diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a327c5c809fb9bbb9b4465f7e3f95e1845aab600 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +# AIOS web — the React shell + FastAPI over the royalimports_os data layer. THE CANONICAL IMAGE: +# this is the one that actually runs, and the one a host migration re-points (C1e). +# +# ⛔ WHY A SECOND DOCKERFILE EXISTS (ops/Dockerfile.api), stated here so nobody "tidies" the wrong +# one away. It is NOT a duplicated service — it is the SAME service built from a DIFFERENT BUILD +# CONTEXT: +# * HF Spaces: the context is the Space repo root, which `deploy_web.py` uploads FLATTENED — +# `api/`, `web/`, `royalimports_os/`, `requirements.txt` all at top level. Hence the paths here. +# * ops/compose.yml: the context is the local repo root, where the same files live at +# `aios-web/api/`, `aios-web/web/`, `royalimports_os/`. Different prefixes, same content. +# ⇒ THE REAL FIX, and it is a follow-up rather than something to do the day of a first deploy: +# make `deploy_web.py` upload with the REPO layout so both contexts are identical, then this file +# serves both and `ops/Dockerfile.api` is deleted. Until then `ops/verify_portability.py` gates +# the two against each other on every property that matters (same app object, same launcher +# contract, non-root, healthchecked, no baked secrets), so they cannot drift silently. +# +# Two stages: node builds the React bundle, python serves it + the JSON API. Login is the app's own +# branded page over a signed session cookie (wave 1 removed the HTTP Basic gate). + +# ---- stage 1: build the React bundle ---- +FROM node:22-slim AS web +WORKDIR /web +# Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash +# 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts). +COPY web/package.json ./ +RUN npm install --no-audit --no-fund --no-package-lock +COPY web/ ./ +RUN rm -f package-lock.json && npm run build # -> /web/dist + +# ---- stage 2: python API serving the bundle ---- +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY api/ ./api/ +COPY royalimports_os/ ./royalimports_os/ # the reused data layer (modules/core/harness) +COPY --from=web /web/dist ./web/dist + +# Run unprivileged — the C2 security baseline, and matched to ops/Dockerfile.api so the two images +# do not differ on a property that would only surface after a migration. uid 1000 is what HF +# Spaces expects. +RUN useradd --create-home --uid 1000 user && chown -R user:user /app +USER user + +ENV RI_DIR=/app/royalimports_os \ + PORT=7860 \ + PYTHONUNBUFFERED=1 +# ⛔ AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound +# call in the process and exists for one developer laptop. ops/verify_portability.py fails if any +# deploy config sets it. +EXPOSE 7860 + +# Liveness via stdlib rather than curl — it keeps the image free of an apt layer that exists only +# for the healthcheck. +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"7860\")}/api/health',timeout=4).status==200 else 1)" + +# ⚠ THE PORT COMES FROM THE ENVIRONMENT, ALWAYS. The application itself reads PORT nowhere — it is +# a launcher argument — which is what makes moving hosts a config change instead of a code change. +# Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time +# and are never baked; the repo-root .dockerignore drops **/.env from the build context. +CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"] diff --git a/README.md b/README.md index 80d116efdce601c28a6d3f43d1afc5475398657e..55d53eccd3c4f8bdc25e8c3e4173e9a1a38df57b 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ ---- -title: Loopable -emoji: 🚀 -colorFrom: red -colorTo: gray -sdk: docker -pinned: false ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +--- +title: Loopable +emoji: 🗂 +colorFrom: indigo +colorTo: gray +sdk: docker +app_port: 7860 +pinned: false +--- + +Loopable — the AIOS platform shell: a React front end (Airtable-grade grid + dashboard pages) +over a FastAPI service reusing the royalimports_os data layer. Login-gated: a branded login page over an HMAC-signed session cookie, with per-user +business-unit scope and module grants. Separate from the Streamlit app. diff --git a/api/aios_session.py b/api/aios_session.py new file mode 100644 index 0000000000000000000000000000000000000000..25739fae016ad54499d8fcbe6897c170bf9ad716 --- /dev/null +++ b/api/aios_session.py @@ -0,0 +1,124 @@ +"""aios_session.py — X3: the signed, STATELESS session cookie (EXIT-3a, 2026-07-30). + + aios_session = base64url(json {t,u,e,i,x}) . base64url(HMAC-SHA256) + +`t` tenant slug · `u` username · `e` the user's revocation epoch · `i` idle deadline · +`x` absolute expiry. Signed with `AIOS_SESSION_SECRET` using stdlib `hmac` — no new dependency, +and nothing to store server-side, which is the whole point: EXIT-4 requires a process that holds +no per-tenant state, and a session TABLE is per-tenant state. D2's Postgres sessions-MIRROR (an +audit view, not the source of truth) arrives with C-2 and is deliberately deferred. + +WHY STATELESS STILL REVOKES. A cookie nobody can delete sounds like a cookie nobody can revoke. +The answer is the `epoch` integer on the user record (`core.users.epoch` / `bump_epoch`): the +cookie carries the epoch it was minted under, every verification re-reads the account's current +epoch, and a password change or a deactivation bumps it — so every outstanding cookie for that +user dies at once, without a session table. + +⚠ NO FALLBACK SECRET, EVER. With `AIOS_SESSION_SECRET` unset this module mints a RANDOM +per-process key: sessions then die on restart, which is an inconvenience. A hardcoded default +would instead let anyone who has read this repo forge a cookie for any user of any tenant — an +inconvenience is the correct trade against a forgery key. `EPHEMERAL_SECRET` records which +happened so the app can say so at startup and a gate can assert it. +""" +import base64 +import hmac +import json +import os +import secrets +import time +from hashlib import sha256 + +COOKIE_NAME = "aios_session" +IDLE_SECONDS = 8 * 60 * 60 # D2: reissued on every authenticated request +ABSOLUTE_SECONDS = 30 * 24 * 60 * 60 # D2: a hard ceiling no amount of activity extends + +_env_secret = os.environ.get("AIOS_SESSION_SECRET") or "" +EPHEMERAL_SECRET = not _env_secret +_SECRET = _env_secret.encode("utf-8") if _env_secret else secrets.token_bytes(32) + + +def _b64(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _unb64(txt: str) -> bytes: + return base64.urlsafe_b64decode(txt + "=" * (-len(txt) % 4)) + + +def _sign(payload: bytes) -> str: + return _b64(hmac.new(_SECRET, payload, sha256).digest()) + + +def mint(tenant, username, epoch, now=None, absolute_expiry=None): + """A fresh cookie value. `absolute_expiry` carries over on reissue so refreshing the idle + window can never extend the 30-day ceiling — that would make the ceiling unreachable.""" + now = int(now if now is not None else time.time()) + claims = {"t": str(tenant), "u": str(username), "e": int(epoch), + "i": now + IDLE_SECONDS, + "x": int(absolute_expiry if absolute_expiry is not None + else now + ABSOLUTE_SECONDS)} + payload = json.dumps(claims, separators=(",", ":"), sort_keys=True).encode("utf-8") + return f"{_b64(payload)}.{_sign(payload)}", claims + + +def read(value, now=None): + """Parse + verify a cookie value → claims dict, or None. + + None for EVERY failure mode — bad shape, bad signature, expired, unparseable. The caller + answers 401 and must not learn which: an error that distinguishes "signature wrong" from + "expired" tells a forger whether their key is right. + """ + if not value or not isinstance(value, str) or value.count(".") != 1: + return None + body, sig = value.split(".", 1) + try: + payload = _unb64(body) + except Exception: + return None + # compare_digest on the SIGNATURE, always — a byte-wise early exit is a timing oracle. + if not hmac.compare_digest(sig, _sign(payload)): + return None + try: + claims = json.loads(payload.decode("utf-8")) + except Exception: + return None + if not isinstance(claims, dict): + return None + if not isinstance(claims.get("u"), str) or not isinstance(claims.get("t"), str): + return None + if not claims["u"] or not claims["t"]: + return None + try: + idle, absolute, epoch = int(claims["i"]), int(claims["x"]), int(claims["e"]) + except (KeyError, TypeError, ValueError): + return None + now = int(now if now is not None else time.time()) + if now >= idle or now >= absolute: + return None + claims["e"], claims["i"], claims["x"] = epoch, idle, absolute + return claims + + +def is_secure_request(request): + """Secure flag ON unless this is plain-HTTP localhost. Set unconditionally in production; + unset on a local http:// dev origin, where a Secure cookie is simply never sent back and + login would appear to succeed and then not work.""" + host = (request.headers.get("host") or "").split(":")[0].lower() + proto = (request.headers.get("x-forwarded-proto") + or request.url.scheme or "http").split(",")[0].strip().lower() + if proto == "https": + return True + return host not in ("localhost", "127.0.0.1", "[::1]", "::1") + + +def set_cookie(response, request, value): + response.set_cookie( + COOKIE_NAME, value, httponly=True, samesite="lax", + secure=is_secure_request(request), max_age=ABSOLUTE_SECONDS, path="/") + + +def clear_cookie(response, request): + # Same attributes as when it was set: a browser matches on path/secure/samesite when + # deleting, and a mismatched delete silently leaves the cookie in place. + response.delete_cookie(COOKIE_NAME, path="/", httponly=True, samesite="lax", + secure=is_secure_request(request)) diff --git a/api/deps.py b/api/deps.py new file mode 100644 index 0000000000000000000000000000000000000000..62b057b6bf18fd663329f03da277ffac5f86e644 --- /dev/null +++ b/api/deps.py @@ -0,0 +1,129 @@ +"""deps.py — the request-scoped session + tenant resolution every v1 route depends on. + +ONE place answers "who is asking, for which tenant, and what may they see?", because a +permission check that exists in four routes has four chances to be forgotten. The routes take +`session: Session = Depends(require_session)` and receive an object that has ALREADY failed +closed if anything was wrong. + +FAIL-CLOSED, spelled out (X2): no cookie → **401**. Cookie present but unverifiable, expired, +epoch-revoked, or naming an unknown tenant → **401**. Authenticated but not granted the surface → +**403**. Never an empty 200: an empty list is a legitimate answer meaning "no rows", and using it +to mean "you are not allowed" is how a permission bug becomes invisible. +""" +import os +import sys +from dataclasses import dataclass +from pathlib import Path + +from fastapi import Depends, HTTPException, Request, Response + +import aios_session + +_RI = Path(os.environ.get("RI_DIR") or Path(__file__).resolve().parents[2] / "royalimports_os") +if str(_RI) not in sys.path: + sys.path.insert(0, str(_RI)) + +import core.perms as perms # noqa: E402 +import core.users as users # noqa: E402 +from harness import runtime # noqa: E402 + + +def err(status, code, message): + """The one error shape (X2): non-2xx JSON `{"error": {"code", "message"}}`.""" + return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}}) + + +@dataclass +class Session: + """A verified request identity. `user` is the PUBLIC record (never a hash or a salt).""" + tenant: str + user: dict + claims: dict + runtime: object + + @property + def uname(self): + return self.user.get("username", "") + + @property + def admin(self): + return perms.is_admin(self.user) + + def require(self, module_key): + """403 unless this session may open `module_key`. Returns nothing — it is a gate.""" + if not perms.may_open(self.user, module_key): + raise err(403, "forbidden", f"your account does not have access to {module_key}") + + +def _user_for(claims): + """The account named by a verified cookie, or None — with the epoch check that makes a + stateless cookie revocable. + + ⛔ THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities `users.verify` + issues.** Any divergence produces the worst failure shape there is — a login that returns 200 + with a cookie and then 401s every request after it. This function got that wrong once (the + emergency-master branch below ran only when NO record existed, while `verify` grants the master + identity even when one does), so it is now written as a deliberate mirror of `verify`'s + structure: try the record, else the master. + + TWO CASES, in `verify`'s own order: + * THE RECORD, if it is readable and satisfies everything a session adds on top of a login: + the account is active, and its CURRENT epoch equals the cookie's — otherwise the session + is revoked (a password change or a deactivation bumped it). + * THE EMERGENCY MASTER — username 'admin' with APP_PASSWORD configured — exactly as + `verify`'s last branch grants it, whether or not a record exists. This is what stops a + store outage, a self-deactivation or a bumped epoch locking the owner out of the product. + ⚠ AND IT MEANS AN `admin` SESSION IS NOT EPOCH-REVOCABLE while APP_PASSWORD is set. That + is not a weakening: whoever holds APP_PASSWORD can simply log in again, so bumping admin's + epoch never revoked them in the first place. Rotating APP_PASSWORD is how you revoke it. + Every OTHER account is fully epoch-revocable, which is asserted in `verify_api.py`. + * anything else → None. An unknown username is NOT admitted just because its signature was + valid: a signature proves the cookie is ours, not that the account still exists. + """ + uname = (claims.get("u") or "").strip().lower() + if not uname: + return None + try: + reg = users.registry() or {} + except Exception: + reg = {} + rec = reg.get(uname) + if (rec and rec.get("active", True) + and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)): + return users._public(uname, rec) + if uname == "admin" and os.environ.get("APP_PASSWORD", ""): + return {"username": "admin", "name": "Administrator", "role": "admin", + "bus": "all", "modules": "all", "epoch": int(claims.get("e") or 0)} + return None + + +def require_session(request: Request, response: Response) -> Session: + """Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is + refreshed on use; the 30-day absolute expiry is carried over, never extended).""" + raw = request.cookies.get(aios_session.COOKIE_NAME) + if not raw: + raise err(401, "no_session", "sign in to continue") + claims = aios_session.read(raw) + if not claims: + raise err(401, "invalid_session", "your session has expired — sign in again") + user = _user_for(claims) + if not user: + raise err(401, "invalid_session", "your session has expired — sign in again") + try: + rt = runtime.get_runtime(claims["t"]) + except KeyError: + # An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this + # deployment, and confirming which slugs exist would answer a question we were not asked. + raise err(401, "invalid_session", "your session has expired — sign in again") + fresh, _ = aios_session.mint(claims["t"], user["username"], int(user.get("epoch") or 0), + absolute_expiry=claims["x"]) + aios_session.set_cookie(response, request, fresh) + return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt) + + +def module_gate(module_key): + """A dependency that 401s without a session and 403s without the grant for `module_key`.""" + def _dep(session: Session = Depends(require_session)) -> Session: + session.require(module_key) + return session + return _dep diff --git a/api/main.py b/api/main.py new file mode 100644 index 0000000000000000000000000000000000000000..e236ca2c65f1240af8fc89aedafb5eeba155c137 --- /dev/null +++ b/api/main.py @@ -0,0 +1,240 @@ +"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. + +It REUSES the `royalimports_os/` data layer verbatim (nothing re-implemented): the canonical, +reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the +JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. + +WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: + + * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared + `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same + anonymous principal and no route could scope anything. Now a signed stateless cookie carries + a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the + user RECORD. `/api/health` stays unauthenticated (liveness only). + * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home + for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). + * **The SSL shim is env-gated.** It used to run at import, unconditionally. + +⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the +session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This +is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB +commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. +""" +import os +import sys +from pathlib import Path + +# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- +# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an +# unverified default context; the HF Space and the container are unaffected. This used to run +# unconditionally at import — i.e. the shipped container disabled TLS verification for every +# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a +# man-in-the-middle away from being someone else's data. It is now opt-in, must be set +# deliberately, and is never on by default. +# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. +if os.environ.get("AIOS_INSECURE_SSL") == "1": + import ssl + ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 + +# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container +# (where the layout differs from the local sibling-dir default). --- +_HERE = Path(__file__).resolve() +_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "royalimports_os")) +for _p in (str(_RI), str(_HERE.parent)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from dotenv import load_dotenv # noqa: E402 +load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed + +from fastapi import Body, Depends, FastAPI, Request # noqa: E402 +from fastapi.responses import JSONResponse # noqa: E402 +from fastapi.staticfiles import StaticFiles # noqa: E402 +from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 + +import aios_session # noqa: E402 +import routes_admin # noqa: E402 +import routes_auth # noqa: E402 +import routes_customers # noqa: E402 +import routes_grid # noqa: E402 +import routes_nav # noqa: E402 +import routes_pages # noqa: E402 +from core import grid_events # noqa: E402 +from deps import Session, module_gate # noqa: E402 + +_WEB_DIST = _HERE.parents[1] / "web" / "dist" + +# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- +# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never +# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the +# ONE canonical file shared with the embedded host (`royalimports_os/aios_grid_fields.json`), so +# embed == standalone by construction. +# +# TWO REASONS THIS IS HERE and not folded into the routes: +# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR +# layout is broken, and discovering that from a 500 on the first customer request instead of +# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a +# rewrite dropped it and this restores it.) +# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and +# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this +# API have not drifted. The rewrite removed them and the referee went red; retargeting the +# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do +# not delete — but only when the subject genuinely moved. Here it should not have moved). +# +# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from +# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and +# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone +# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the +# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. +import json as _json_contract # noqa: E402 +_FIELDS_PATH = _RI / "aios_grid_fields.json" +if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " + "royalimports_os root (it also carries the data layer this API imports)." + ) +_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) +FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract +# text/status/date pass through untouched; every OTHER odoo field is numeric -> rounded. Derived +# from field TYPE (not a hand-kept key list) so a new text/date field can never be wrongly rounded. +_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS + if f["source"] == "odoo" and f["type"] in ("text", "status", "date")} + +app = FastAPI(title="AIOS web API") + + +@app.exception_handler(StarletteHTTPException) +def _error_shape(request: Request, exc: StarletteHTTPException): + """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. + + `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a + 422 from a malformed path param) is wrapped here so a client never has to branch on two + different error bodies. + """ + detail = exc.detail + if isinstance(detail, dict) and "error" in detail: + body = detail + else: + body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} + return JSONResponse(body, status_code=exc.status_code, + headers=getattr(exc, "headers", None)) + + +@app.exception_handler(grid_events.StoreUnavailable) +def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): + """THE SAFETY NET for "the store is down" → 503, from anywhere. + + ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a + `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own + try/except — surfaced as a 500. A store outage is a normal operational state and every route + here touches the store at least twice (the workspace read, then the write), so "remember to + wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler + means a store outage can only ever be a 503, whichever call raised it. + + The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this + adapter, which has no durable session dict to degrade into. A 200 over a write that + evaporated is the failure the whole rule exists to prevent. + """ + return JSONResponse( + {"error": {"code": "store_unavailable", + "message": "the tenant store is unavailable — no change was saved"}}, + status_code=503) + + +@app.get("/api/health") +def health(): + """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not + reveal anything about the deployment beyond "the process is up". No version, no tenant list, + no config: a health endpoint is the one URL every scanner finds first.""" + return {"ok": True} + + +app.include_router(routes_auth.router) +app.include_router(routes_nav.router) +app.include_router(routes_customers.router) +app.include_router(routes_grid.router) +# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user +# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so +# a new page is a builder module plus one registry line, and nothing here changes. +app.include_router(routes_pages.router) +app.include_router(routes_admin.router) + + +# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) +# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic +# behavior. An alias that kept the old auth would be a bypass of everything above it. +_GATE = module_gate(routes_customers.MODULE) + + +@app.get("/api/customers", deprecated=True) +def _customers_alias(session: Session = Depends(_GATE)): + return routes_customers._payload(session) + + +@app.patch("/api/customers/{pid}", deprecated=True) +def _patch_alias(pid: int, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + return routes_customers.patch_customer(pid, body, session) + + +def _assert_contract_floor(): + """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. + + This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the + canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and + the embedded host are serving two different schemas and the grid's own contract has forked — + the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup + on whatever machine is actually running. + """ + import aios_grid + base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] + if [f["key"] for f in base] != [f["key"] for f in FIELDS]: + raise RuntimeError( + "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " + "disagree on the base field set — embed and standalone would serve different schemas. " + "Run aios-web/verify_fields_contract.py.") + + +def _startup_notes(): + """Say the two things an operator must know, once, at import — never in a response body.""" + if aios_session.EPHEMERAL_SECRET: + print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process " + "key. Sessions will not survive a restart and will not work across workers. " + "Set it in production.") + if os.environ.get("AIOS_INSECURE_SSL") == "1": + print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound " + "requests. Local development only; never in a deployed environment.") + + +_assert_contract_floor() +_startup_notes() + + +# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. +# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login +# page cannot live behind a password prompt, so the shell's own assets must be public — which they +# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. +# +# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also +# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are +# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to +# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` +# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), +# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same +# answer as any other path that is not part of the app. +# +# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); +# this guard is what makes the API safe regardless of what the bundle happens to contain. +_DEV_FIXTURES = {"sample_customers.json"} + + +class _AppStatic(StaticFiles): + async def get_response(self, path, scope): + if Path(path).name in _DEV_FIXTURES: + raise StarletteHTTPException(status_code=404, detail="Not Found") + return await super().get_response(path, scope) + + +# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets +if _WEB_DIST.is_dir(): + app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") diff --git a/api/pages.py b/api/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..95996ae623eee1e0778c473c23e11d8d09bc8caf --- /dev/null +++ b/api/pages.py @@ -0,0 +1,362 @@ +"""pages.py — Y1: the page-data envelope, and the ONE place its rules live (W2-2). + +THE POINT OF THIS FILE IS WAVE 3. Collections and Procurement must land by adding a BUILDER, not +by inventing a shape — so every rule the envelope has (the closed `fmt` enum, the drill grammar, +the no-silent-caps contract, how BU scope is resolved) lives here as a constructor a builder calls, +never as a convention a builder is expected to remember. A rule that exists only in a docstring is +a rule the second page gets wrong. + +WHAT A BUILDER OWES, and nothing more: + metrics(team_id, granularity, today) -> a dict of plain data, SCOPE-SHAPED ONLY + blocks(metrics, team_id, granularity) -> [block, ...] built with the constructors below + TITLE / SUBTITLE / MODULE / CONTROLS +`build_envelope` does the rest: it resolves the BU, caches the metrics on the SCOPE key, assembles +the per-request half, and returns the envelope. + +⛔ THE CACHE LINE, WHICH IS WHERE WAVE 1 SHIPPED A CROSS-USER LEAK. What may be cached on a +`(page, team_id, doc_mode, day)` key is the BUILDER'S METRICS — the answer to a question two users +with the same scope are asking identically. What may NEVER be cached there is anything with a +per-USER stratum, and on this envelope that is `controls` (its `bu.options` come from +`perms.allowed_bu_labels(user)`, i.e. from the user RECORD). Wave 1's defect was exactly this shape +one layer down: a whole payload cached on a scope key while carrying per-user fields and overlay +cells. It survived 129 green checks because every fixture user had a distinct scope pair. So the +split is structural here, not remembered: `metrics()` cannot see the session at all — it takes +`team_id`, and `verify_api.py` asserts the cached object holds nothing user-shaped. + +⚠ THE CLIENT FORMATS. The server never sends a pre-formatted number, so one formatter change fixes +every page — and `fmt` is a CLOSED enum for the same reason. Two vocabularies exist and they are +not interchangeable: `FMTS` for kpi/table/validation, and viz `FieldType` for a chart block's +`fields[]` (those objects are fed straight into `chartData`). `money` vs `currency` is the trap. +""" +import time + +# `deps` FIRST, deliberately: it is what puts `royalimports_os/` on `sys.path`, so importing it +# ahead of any `core.*` makes this module importable in whatever order a caller reaches it. +from deps import Session, err, perms # noqa: I001 + +import core.context as ctxlib # noqa: E402 + +# ── the two vocabularies ───────────────────────────────────────────────────────────────────────── +#: The Y1 client-format enum for kpi / table / validation values. CLOSED on purpose: a page that +#: needs a new one adds it HERE and to the client's `fmt` at the same time, which is the property +#: that keeps one formatter authoritative. A table column with NO `fmt` renders verbatim — that is +#: how a text column is expressed without opening the enum. +FMTS = ("money", "num", "pct", "int", "date") + +#: The viz `FieldType` values a chart block's `fields[]` may use — a DIFFERENT vocabulary, because +#: `chartData(spec, rows, fieldByKey)` consumes those objects directly. `currency`, not `money`. +VIZ_TYPES = ("text", "status", "currency", "int", "pct", "date") + +#: `chartData`'s own display cap (`MAX_BUCKETS`). A payload must fit it at the SOURCE: the engine +#: truncates past 12 and, on a date axis, sorts ASCENDING first — so a 13-month series loses the +#: CURRENT month, the one anybody actually looks at. Mirrored here so a builder can size its own +#: series without importing anything from the client tree. +MAX_BUCKETS = 12 + +#: Drill kinds the client is expected to understand. `decomp` names a WINDOW + a scope; the others +#: name an ENTITY. ⚠ THIS WAVE THEY ARE DECLARATIONS — there is no resolver route (§5 Y1 amendment +#: 4). Each descriptor is COMPLETE, so wave 3 adds the resolver with no shape change, and Y6's +#: panel must say it is thin rather than look finished. +DRILL_KINDS = ("decomp", "customer", "sku", "rep", "bu") + + +def _fmt_ok(fmt, allowed, where): + if fmt is not None and fmt not in allowed: + # A wrong `fmt` renders — wrongly — and nothing complains. Refusing it at construction is + # the only moment anybody finds out, so this raises rather than coercing. + raise ValueError(f"{where}: fmt {fmt!r} is not one of {allowed}") + return fmt + + +# ── block constructors ─────────────────────────────────────────────────────────────────────────── +def kpi(key, label, value, fmt="money", delta=None, delta_fmt=None, delta_label=None, + delta_dir=None, note=None, drill=None): + """One KPI card. `delta` is a NUMBER or None — never a string. + + THE ZERO-REVENUE RULE, ported from `app.py:2846`: a period with no orders yet must not show an + alarming −100%. It sends NO number (`delta=None`) plus a `delta_label` and `delta_dir='off'`, + so no formatter is involved and the card states the situation instead of computing a ratio + against nothing. + + THE BIG-RATIO RULE IS THE CLIENT'S: a tiny LY base makes `+14,975%` read as noise, and + Streamlit rewrites it per-card as `+150× LY`. That belongs in ONE formatter (`fmt.pct` renders + |v| ≥ 999 as a multiple), so the server just sends the number. + """ + out = {"key": key, "label": label, "value": value, + "fmt": _fmt_ok(fmt, FMTS, f"kpi {key}")} + if delta is not None: + out["delta"] = delta + out["delta_fmt"] = _fmt_ok(delta_fmt or "pct", FMTS, f"kpi {key} delta") + if delta_label: + out["delta_label"] = delta_label + if delta_dir: + out["delta_dir"] = delta_dir # up | down | flat | off — a TONE hint, not the sign + if note: + out["note"] = note + if drill: + out["drill"] = drill + return out + + +def kpis(key, items): + return {"type": "kpis", "key": key, "items": list(items)} + + +def section(key, label, note=None): + out = {"type": "section", "key": key, "label": label} + if note: + out["note"] = note + return out + + +def column(key, label, fmt=None, align=None): + """A table column. `fmt` omitted ⇒ render verbatim (that is how text columns work).""" + out = {"key": key, "label": label} + if fmt: + out["fmt"] = _fmt_ok(fmt, FMTS, f"column {key}") + if align: + out["align"] = align + elif fmt in ("money", "num", "pct", "int"): + out["align"] = "right" # numbers align right by default, everywhere + return out + + +def table(key, columns, rows, total=None, drill=None, download=None, empty=None): + """A table block. `shown`/`total` ALWAYS ship — `shown == total` still sends both. + + Owner rule [[no-unverifiable-aggregates]]: a silent `[:N]` cap is a defect. `total` defaults to + `len(rows)` so a builder that forgot to truncate cannot accidentally claim it did not. + """ + rows = list(rows) + out = {"type": "table", "key": key, "columns": list(columns), "rows": rows, + "shown": len(rows), "total": int(total if total is not None else len(rows))} + if drill: + out["drill"] = drill + if download: + out["download"] = download + if empty: + out["empty"] = empty + return out + + +def chart(key, spec, series, fields, rows, x_order=None, drill=None, total=None): + """A chart block: ONE x, N SERIES. + + ⛔ WHY `series[]` AND NOT `ChartSpec.splitBy`. The viz engine has no series dimension: + `chartData()` buckets on `x` only and `DashboardView` draws one flat `buckets[]`. `splitBy` is + persisted, validated and offered in the chart editor but never reaches the arithmetic or the + renderer, so a `splitBy` chart draws identically to one without it. A YoY comparison is + therefore N `chartData` calls over the SAME wide rows, one per series `y` — which also means + nothing about `chartData` changes and `verify_charts.py`'s verdict stays put, which W2-5's + pure-move proof requires. + + ⛔ WHY `x_order`. The engine sorts buckets by VALUE unless the x field is date-family — a trend + would render as a revenue-ordered sawtooth. And date-family is not an escape: it buckets to + `YYYY-MM`, so a weekly axis would collapse into months. So the SERVER declares the chronology + and the client re-orders the buckets it got back. + """ + for f in fields: + if f.get("type") not in VIZ_TYPES: + raise ValueError(f"chart {key}: field {f.get('key')!r} type {f.get('type')!r} is not a " + f"viz FieldType {VIZ_TYPES} — `money` is the kpi/table vocabulary") + rows = list(rows) + out = {"type": "chart", "key": key, "spec": dict(spec), "series": list(series), + "fields": list(fields), "rows": rows, + "shown": len(rows), "total": int(total if total is not None else len(rows))} + if x_order is not None: + out["x_order"] = list(x_order) + if drill: + out["drill"] = drill + return out + + +def validation(checks, note=None): + """The reconciliation panel, from a module's `validate()` output. + + `actual` is the module's `a` — the number the page shows. `expected` is its `b` — the + INDEPENDENT aggregate it must tie to. The signed gap rides in `note` rather than being + recomputed client-side, because the module's own tolerance decided `ok` and a second + subtraction in the client could disagree with it. + """ + out = {"type": "validation", "key": "validation", + "checks": [{"name": c.get("check") or c.get("name") or "", + "ok": bool(c.get("ok")), + "actual": c.get("a"), "expected": c.get("b"), + "fmt": "money" if isinstance(c.get("a"), float) else "num", + "note": f"gap {c.get('gap')}"} for c in (checks or [])]} + if note: + out["note"] = note + return out + + +# ── drill descriptors: ONE grammar, three placements ───────────────────────────────────────────── +def decomp_drill(window_row, scope_type="period", scope_value=None, label=None): + """A WINDOW + scope descriptor. Mirrors `ui/drawers._decomp_desc` key-for-key, so the eventual + resolver can serve both front-ends from one function — and accepts both row shapes the sales + module produces (`date_from`/`date_to` on the scorecard, `start`/`end` on a trend row).""" + return {"kind": "decomp", "label": label, + "scope_type": scope_type, "scope_value": scope_value, + "date_from": window_row.get("date_from") or window_row.get("start"), + "date_to": window_row.get("date_to") or window_row.get("end"), + "cmp_from": window_row.get("cmp_from"), "cmp_to": window_row.get("cmp_to")} + + +def entity_drill(kind, id_key, label_key=None): + """Block-level: every row of this block opens the same kind of entity panel.""" + if kind not in DRILL_KINDS: + raise ValueError(f"drill kind {kind!r} is not one of {DRILL_KINDS}") + out = {"kind": kind, "id_key": id_key} + if label_key: + out["label_key"] = label_key + return out + + +def row_drill(row_key="_drill"): + """Block-level: each row carries its OWN descriptor under `row_key`.""" + return {"kind": "decomp", "row_key": row_key} + + +def truncate(rows, limit): + """`(rows[:limit], total)` — the ONLY sanctioned way to cap a list. Returning the total beside + the slice is what makes "showing N of M" possible; a bare `[:N]` is the defect.""" + rows = list(rows) + return rows[:limit], len(rows) + + +# ── BU scope: the one line that could fail open ────────────────────────────────────────────────── +def resolve_bu(session: Session, bu): + """The effective `team_id` for this request. `bu` NARROWS; it never authorises. + + ⛔ THE RULE, AND WHY IT IS SHAPED THIS WAY. `perms.scope_team_id` derives scope from the user + RECORD precisely because a UI selector defaults to 'All' → consolidated, and mirroring that + default on an API with no selector would BE the leak. So a query parameter cannot widen: + + * absent / 'all' → `scope_team_id(user)`, which is None ONLY when the user may see every BU + and is the PINNED team_id otherwise. A Royal-only user asking for 'all' gets 6, and 'all' + for them honestly means "everything you may see". + * an explicit id → must be in `allowed_team_ids(user)`, else **403**. Never silently coerced + to their own BU: a coerced parameter hides a client bug behind data that looks right. + * anything else → 400. + """ + raw = "" if bu is None else str(bu).strip().lower() + if raw in ("", "all"): + return perms.scope_team_id(session.user) + try: + want = int(raw) + except (TypeError, ValueError): + raise err(400, "bad_bu", "bu must be 'all' or a business-unit id") + if want not in perms.allowed_team_ids(session.user): + raise err(403, "bu_forbidden", + "your account does not have access to that business unit") + return want + + +def bu_control(session: Session, team_id): + """The BU picker, built from the user's OWN permitted labels — which is exactly why `controls` + is assembled per request and never cached on a scope key.""" + labels = perms.allowed_bu_labels(session.user) + options = [] + for lab in labels: + if lab == "All": + options.append({"value": "all", "label": "All"}) + continue + tid = ctxlib.BRANDS.get(lab) + if tid is not None: + options.append({"value": tid, "label": lab}) + return {"key": "bu", "kind": "bu", "label": "Business Unit", + "value": "all" if team_id is None else team_id, "options": options} + + +def choice_control(key, label, value, options): + return {"key": key, "kind": "choice", "label": label, "value": value, + "options": [{"value": v, "label": l} for v, l in options]} + + +def bu_label(team_id): + """'All Business Units' / 'Fisch' / 'Royal' — for prose, never for a permission decision.""" + if team_id is None: + return "All Business Units" + for lab, tid in ctxlib.BRANDS.items(): + if tid == team_id: + return lab + return str(team_id) + + +# ── the registry + the assembler ───────────────────────────────────────────────────────────────── +#: page key -> builder module. Wave 3 adds `'ar'` and `'procurement'` HERE and nowhere else — the +#: route, the grant wall, the BU resolution, the caching and the envelope are already written. +#: ⚠ The key must be the REGISTRY key, because it is what `session.require()` gates on. +_BUILDERS = {} + + +def register(key, builder): + _BUILDERS[str(key)] = builder + + +def builder_for(key): + """The builder for `key`, or None. A missing key is a 404 at the route — NOT a 403: an + unported page is a fact about our roadmap, not about this user's grants.""" + if not _BUILDERS: + import pages_sales # noqa: F401 (self-registers on import) + return _BUILDERS.get(str(key or "").strip().lower()) + + +def known_pages(): + builder_for("") # force the lazy import + return sorted(_BUILDERS) + + +#: How long a scope's metrics stay warm. Matches `routes_customers._CACHE_TTL` — the underlying +#: pull is the same order of cost and a second TTL would be a second thing to reason about. +_CACHE_TTL = 900 +#: The metrics cache is bounded for the reason every cache in this process is: an entry per +#: (page × scope × day) that is never evicted is a memory leak with a tenant-shaped growth curve. +_CACHE_MAX = 24 + + +def _cached_metrics(session: Session, page_key, team_id, granularity, builder): + """The SCOPE-shaped half, cached on the runtime (per tenant, LRU-bounded). + + The key carries the DAY: every metric here is relative to "today" (YTD, WTD, the trailing 12 + months), so an entry that outlives midnight serves yesterday's question with today's label. + """ + rt = session.runtime + today = time.strftime("%Y-%m-%d") + key = ("page", page_key, team_id, granularity, today) + hit = rt.pool_cache.get(key) + now = time.time() + if hit and now - hit[0] < _CACHE_TTL: + return hit[1] + metrics = builder.metrics(team_id=team_id, granularity=granularity) + rt.pool_cache[key] = (now, metrics) + # A `while`, not a one-shot slice: evicting a fixed batch leaves the map above its own bound + # whenever it is more than that batch over, which is a bound that does not bind. + # ⚠ This dict is SHARED with `routes_customers._pool_rows`'s pool entries (they key on + # `("pool", …)`, these on `("page", …)`), so either side's eviction can drop the other's entry. + # Harmless — a miss is a rebuild — and deliberately one bounded map per tenant rather than two. + while len(rt.pool_cache) > _CACHE_MAX: + rt.pool_cache.pop(min(rt.pool_cache, key=lambda k: rt.pool_cache[k][0]), None) + return metrics + + +def build_envelope(session: Session, page_key, bu=None, period=None): + """The Y1 envelope for `page_key`, scoped to this session. Raises the contracted errors.""" + builder = builder_for(page_key) + if builder is None: + raise err(404, "unknown_page", f"no page data is published for {page_key!r}") + # The GRANT WALL, on the builder's registry key. 403, never an empty 200 — an empty page reads + # as "you have no data" and is how a permission bug hides in plain sight. + session.require(builder.MODULE) + team_id = resolve_bu(session, bu) + granularity = builder.resolve_granularity(period) + metrics = _cached_metrics(session, page_key, team_id, granularity, builder) + return { + "key": page_key, + "title": builder.TITLE, + "subtitle": builder.subtitle(team_id), + # UTC with an explicit Z. A naive local timestamp from a container whose TZ nobody set is + # indistinguishable from a wrong one. + "as_of": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(metrics["pulled_at"])), + "controls": [bu_control(session, team_id)] + builder.controls(granularity), + "blocks": builder.blocks(metrics, team_id, granularity), + } diff --git a/api/pages_sales.py b/api/pages_sales.py new file mode 100644 index 0000000000000000000000000000000000000000..9b5c4daca4810110e79540aeaf1a9d45d491f98b --- /dev/null +++ b/api/pages_sales.py @@ -0,0 +1,270 @@ +"""pages_sales.py — the Sales page builder (W2-2). The FIRST Y1 builder, and the template. + +It reuses `royalimports_os/modules/sales.py` VERBATIM — the reconciled, validated data layer the +Streamlit page already reads — so there is exactly one definition of every number and the API is a +projection, not a second implementation. Nothing here computes revenue. + +WHAT A BUILDER IS, in three functions: + metrics(team_id, granularity) the SCOPE-shaped data. Cached by `pages._cached_metrics`. + ⛔ It cannot see the session, by design — that is what makes + "nothing user-shaped is cached" a structural property rather + than a rule somebody has to remember. + blocks(metrics, team_id, …) the ordered Y1 blocks, built with `pages`' constructors. + controls(granularity) the page's own toolbar (the BU picker is added by `pages`). + +⛔ THE ONE LEAK THIS FILE HAD TO CLOSE. `sales.headline(team_id=…)` returns a `by_team` map that +**always covers BOTH business units** — its own docstring says "by_team always shows both" — so a +Royal-only request's headline carries Fisch revenue. `metrics()` therefore STRIPS `by_team` when a +BU is pinned, at the source, before it is cached: the other BU's numbers never enter the cache entry +for a scoped user, so a later edit that emits the block unconditionally renders nothing instead of +leaking. Fail-closed by construction, not by review. Asserted in `verify_api.py`. + +⚠ `by_rep` IS NOT THE STREAMLIT PAGE'S "By agent". `sales.by_rep` groups by Odoo `user_id` (the +salesperson on the order); the Streamlit page shows `customers.by_dimension('agent')` (the +CUSTOMER's assigned agent — a different number, see [[invoice-line-agent-commission]]). The split +doc's brief names `by_rep`, so `by_rep` is what ships; the divergence is flagged in the S1 mailbox +for the owner rather than resolved in code. +""" +import sys as _sys +import time + +import core.odoo as O +import core.periods as P +import modules.sales as sales + +import pages + +MODULE = "sales" # the REGISTRY key — what the grant wall gates on +TITLE = "Sales" + +_GRANULARITIES = (("monthly", "Monthly"), ("weekly", "Weekly")) + + +def resolve_granularity(period): + """`?period=` → the trend granularity. + + An unrecognised value is COERCED to the default rather than refused, and that asymmetry with + `bu` is deliberate: `bu` is a SCOPE question, where a coerced parameter would silently show the + wrong book, so it 403s. `period` is a DISPLAY question, and the envelope echoes the resolved + value back in `controls[].value`, so a coerced parameter cannot look accepted. + """ + return "weekly" if str(period or "").strip().lower() in ("weekly", "week") else "monthly" + + +def controls(granularity): + return [pages.choice_control("period", "Granularity", granularity, _GRANULARITIES)] + + +def subtitle(team_id): + return ("Confirmed orders only (Odoo state sale/done), house accounts excluded. Floral " + "wholesale is seasonal, so every comparison is against the SAME period last year.") + + +# ── the scope-shaped half ──────────────────────────────────────────────────────────────────────── +def metrics(team_id=None, granularity="monthly"): + """Everything the blocks need, for ONE scope. Pure data — no session, no user, no formatting.""" + # Pin the recognition basis explicitly. It is thread-local and defaults to 'order', but a + # worker thread reused from a container that once served 'invoice' would otherwise inherit a + # different definition of revenue mid-page, and nothing in the payload would say so. + O.set_doc_mode("order") + t = P.today() + + head = sales.headline(t=t, team_id=team_id) + if team_id is not None: + # ⛔ See the module docstring: `by_team` always covers both BUs. Strip it BEFORE caching. + head = dict(head, by_team={}) + + if granularity == "weekly": + trend = sales.weekly_trend(n_weeks=pages.MAX_BUCKETS, t=t, team_id=team_id) + else: + trend = sales.monthly_trend(n=pages.MAX_BUCKETS, t=t, team_id=team_id) + + return { + "pulled_at": time.time(), + "today": P._d(t), + "scorecard": sales.period_scorecard(t=t, team_id=team_id), + "headline": head, + "trend": trend, + # limit=1e9 so the TOTAL is knowable: the module slices after aggregating, so asking for + # everything costs the same read and lets the block report "showing 25 of 812" instead of + # capping silently. A bare `[:N]` with no total is the defect [[no-unverifiable-aggregates]] + # exists to name. + "by_rep": sales.by_rep(t=t, limit=10 ** 9, team_id=team_id), + "top_customers": sales.top_customers(t=t, limit=10 ** 9, team_id=team_id), + "validation": sales.validate(t=t, team_id=team_id), + } + + +# ── the blocks ─────────────────────────────────────────────────────────────────────────────────── +_LIST_LIMIT = 25 + + +def _delta_dir(v): + if v is None: + return None + return "up" if v > 0 else "down" if v < 0 else "flat" + + +def _scorecard_kpis(scorecard): + """One card per period, every one of them drillable to its exact window.""" + items = [] + for r in scorecard: + drill = pages.decomp_drill(r, "period", None, f"Sales · {r['label']}") + if not r["revenue"]: + # No orders yet in this window. Sending a −100% here is the alarming lie + # `app.py:2846` guards against; send no number at all. + items.append(pages.kpi(r["key"], r["label"], 0.0, "money", + delta_label="no orders yet", delta_dir="off", drill=drill)) + continue + items.append(pages.kpi(r["key"], r["label"], r["revenue"], "money", + delta=r["yoy_pct"], delta_fmt="pct", + delta_label="vs LY" if r["yoy_pct"] is not None else None, + delta_dir=_delta_dir(r["yoy_pct"]), drill=drill)) + return pages.kpis("scorecard", items) + + +def _ytd_context_kpis(scorecard, head): + """The three YTD numbers the Streamlit page puts in a caption (`app.py:2858`). As KPI cards + rather than prose, so each one carries the drill the caption could not.""" + ytd = next((r for r in scorecard if r["key"] == "ytd"), scorecard[-1] if scorecard else {}) + drill = pages.decomp_drill(ytd, "period", None, "Sales · Year to date") + return pages.kpis("ytd_context", [ + pages.kpi("ytd_orders", "Orders (YTD)", head.get("ytd_orders") or 0, "int", drill=drill), + pages.kpi("ytd_aov", "Average order value", head.get("aov") or 0.0, "money", drill=drill), + pages.kpi("ytd_customers", "Active customers (YTD)", head.get("ytd_customers") or 0, + "int", drill=drill), + ]) + + +def _trend_block(trend, granularity, today): + """The YoY trend: ONE x, TWO series, chronology declared by the server. + + See `pages.chart` for why this is `series[]` + `x_order[]` rather than `ChartSpec.splitBy` and + a date-typed axis. `spec.y` is set to the FIRST series so a naive single-series + `chartData(spec, …)` call still draws the primary bars instead of a "choose a number field" + problem card. + """ + label_key = "week" if granularity == "weekly" else "month" + rows, x_order = [], [] + for r in trend: + label = str(r.get(label_key) or "") + rows.append({ + "period": label, + "revenue": r.get("revenue") or 0.0, + "revenue_ly": r.get("revenue_ly") or 0.0, + # The in-progress period compares a PARTIAL window against a full one, so the client + # fades it and drops its YoY (`app.py:2867-2875`). One rule for both granularities: + # the window that CONTAINS today. ISO strings, so string order is date order. + "partial": bool(r.get("start") and r.get("end") + and str(r["start"]) <= today <= str(r["end"])), + "_drill": pages.decomp_drill(r, "period", None, f"Sales · {label}"), + }) + x_order.append(label) + unit = "Week of" if granularity == "weekly" else "Month" + spec = {"id": "sales-trend", "kind": "bar", "x": "period", "y": "revenue", "agg": "sum", + "title": "Revenue trend", "palette": "brand", + "axis": {"x": {"label": unit}, "y": {"format": "currency"}}} + fields = [{"key": "period", "label": unit, "type": "text", "source": "odoo"}, + {"key": "revenue", "label": "Revenue", "type": "currency", "source": "odoo"}, + {"key": "revenue_ly", "label": "Revenue LY", "type": "currency", "source": "odoo"}] + series = [{"y": "revenue", "label": "This year"}, + {"y": "revenue_ly", "label": "Last year"}] + return pages.chart("trend", spec, series, fields, rows, x_order=x_order, + drill=pages.row_drill()) + + +def _by_bu_block(head, scorecard): + """YTD per Business Unit — CONSOLIDATED VIEWS ONLY. + + Omitted entirely when a BU is pinned, matching `app.py:2898`'s `if tid is None`. `metrics()` has + already emptied `by_team` in that case, so this cannot accidentally render the other BU even if + the guard here were removed — belt and braces on the one number that must never cross. + """ + by_team = head.get("by_team") or {} + if not by_team: + return None + ytd = next((r for r in scorecard if r["key"] == "ytd"), scorecard[-1] if scorecard else {}) + rows = [] + for tid in O.TEAM_IDS: + name = O.TEAM_NAMES.get(tid) + e = by_team.get(name) + if not e: + continue + rows.append({ + "bu": name, "team_id": tid, + "revenue": e.get("ytd") or 0.0, "revenue_ly": e.get("ytd_ly") or 0.0, + "yoy_pct": P.yoy_pct(e.get("ytd") or 0.0, e.get("ytd_ly") or 0.0), + "_drill": pages.decomp_drill(ytd, "brand", tid, f"Sales · {name} (YTD)"), + }) + return pages.table( + "by_bu", + [pages.column("bu", "Business Unit"), + pages.column("revenue", "YTD $", "money"), + pages.column("revenue_ly", "LY $", "money"), + pages.column("yoy_pct", "YoY", "pct")], + rows, drill=pages.row_drill(), download={"filename": "sales_by_business_unit"}, + empty="No confirmed orders in either Business Unit this year.") + + +def _by_rep_block(by_rep): + rows, total = pages.truncate(by_rep, _LIST_LIMIT) + return pages.table( + "by_rep", + [pages.column("rep", "Salesperson"), + pages.column("revenue", "YTD $", "money"), + pages.column("orders", "Orders", "int")], + rows, total=total, drill=pages.entity_drill("rep", "uid", "rep"), + download={"filename": "sales_by_rep"}, + empty="No confirmed orders are attributed to a salesperson in this scope.") + + +def _top_customers_block(top_customers): + rows, total = pages.truncate(top_customers, _LIST_LIMIT) + return pages.table( + "top_customers", + [pages.column("customer", "Customer"), + pages.column("revenue", "YTD $", "money"), + pages.column("orders", "Orders", "int")], + rows, total=total, drill=pages.entity_drill("customer", "pid", "customer"), + download={"filename": "top_customers"}, + empty="No confirmed orders in this scope yet.") + + +def blocks(metrics, team_id=None, granularity="monthly"): + """The ORDERED blocks, mirroring `app.py:2833-2961`'s reading order. + + ⚠ The BU table is present only on a consolidated view, so the client must not assume a fixed + block count or index — it renders `blocks` in order and skips a type it does not know. + """ + sc = metrics["scorecard"] + head = metrics["headline"] + out = [ + _scorecard_kpis(sc), + _ytd_context_kpis(sc, head), + pages.section("trend_h", "Revenue trend", + "This period versus the same period last year. Click any bar to decompose " + "that period by customer and SKU. The in-progress period is marked partial " + "and carries no YoY."), + _trend_block(metrics["trend"], granularity, metrics["today"]), + ] + bu = _by_bu_block(head, sc) + if bu is not None: + out.append(pages.section("by_bu_h", "By Business Unit", + "YTD revenue per Business Unit versus last year.")) + out.append(bu) + out.append(pages.section("by_rep_h", "By salesperson", + "YTD revenue per Odoo salesperson on the order.")) + out.append(_by_rep_block(metrics["by_rep"])) + out.append(pages.section("top_customers_h", "Top customers", + "Largest customers by YTD revenue.")) + out.append(_top_customers_block(metrics["top_customers"])) + out.append(pages.validation( + metrics["validation"], + "Every headline number reconciles to an independent Odoo aggregate, and every " + "decomposition sums back to it.")) + return out + + +#: Self-registers on import, so adding a page is one file plus one import — `pages.builder_for` +#: does the import and never needs a per-page branch. +pages.register("sales", _sys.modules[__name__]) diff --git a/api/routes_admin.py b/api/routes_admin.py new file mode 100644 index 0000000000000000000000000000000000000000..05b37f7580f45faddf8f27a085c65e8dc8b138e3 --- /dev/null +++ b/api/routes_admin.py @@ -0,0 +1,325 @@ +"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). + +These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` +modals, over the SAME `core/users` account store — so both front-ends administer one set of +accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. + +ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit +dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record +is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. + +⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` +has no caller anywhere in the shipped app — module grants are only settable by editing the store +JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both +asserted in `verify_api.py` section A: + + * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin + clearing every checkbox to lock an account down would grant it everything. + * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to + `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose + whole point is strict isolation. + +Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in +`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the +WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both +read semantics are untouched — the seam is write-strict / read-unchanged. + +⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` +one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no +accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". +""" +import re + +from fastapi import APIRouter, Body, Depends, Response + +import core.registry as registry +import core.store as store + +from deps import Session, err, perms, require_session, users +from routes_auth import _public_user + +router = APIRouter(prefix="/api/v1") + +#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under +#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no +#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. +_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") +#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog +#: enforces nothing; this does, and the two are allowed to differ because only one of them is +#: reachable from the internet. +_MIN_PW = 8 +_ROLES = ("user", "admin") + + +def admin_gate(session: Session = Depends(require_session)) -> Session: + """401 without a session, 403 without the admin role. Role, not a module grant — mirroring + `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + return session + + +def _require_store(): + """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" + try: + ok = store.available() + except Exception: + ok = False + if not ok: + raise err(503, "store_unavailable", + "the tenant store is unavailable — accounts cannot be read or changed") + + +def _registry(): + _require_store() + try: + reg = users.registry() or {} + except Exception: + raise err(503, "store_unavailable", "the tenant store is unavailable") + return reg + + +def _view(uname, rec): + """What an administrator may see about an account. NEVER `salt` or `hash` — this function is + the only projection these routes use, so there is one place that can leak and it does not.""" + return {"username": uname, + "name": rec.get("name") or uname, + "role": rec.get("role", "user"), + "bus": rec.get("bus", "all"), + "bu_labels": perms.allowed_bu_labels(rec), + "modules": rec.get("modules", "all"), + "agent": rec.get("agent") or None, + "email": rec.get("email") or None, + "active": bool(rec.get("active", True)), + # The session-revocation handle. An admin needs it: it is the only visible evidence + # that "sign them out everywhere" actually happened. + "epoch": int(rec.get("epoch") or 0)} + + +# ── request validation: every rule below is fail-closed ────────────────────────────────────────── +def _clean_username(v): + uname = str(v or "").strip().lower() + if not _UNAME_RE.match(uname): + raise err(400, "bad_username", + "a username is 2-32 characters: lowercase letters, digits, dot, dash or " + "underscore, starting with a letter or digit") + return uname + + +def _clean_password(v): + pw = str(v or "") + if len(pw) < _MIN_PW: + raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") + return pw + + +def _clean_role(v): + role = str(v or "").strip().lower() + if role not in _ROLES: + raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") + return role + + +def _clean_bus(v): + """'all', or a non-empty list of KNOWN business-unit ids. + + Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no + recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_bus", + "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " + "would read as unrestricted)") + out = [] + for b in v: + try: + b = int(b) + except (TypeError, ValueError): + raise err(400, "bad_bus", "a business-unit id must be a number") + if b not in users.BU_LABELS: + raise err(400, "bad_bus", + f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " + "unrecognised id would widen access to every BU") + out.append(b) + return sorted(set(out)) + + +def _clean_modules(v): + """'all', or a non-empty list of KNOWN module keys. + + `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is + refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would + silently lock the account out of the page they meant to grant. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_modules", + "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " + "reads as UNRESTRICTED, which is the opposite of locking an account down") + known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) + out, bad = [], [] + for k in v: + k = str(k or "").strip() + (out if k in known else bad).append(k) + if bad: + raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") + return sorted(set(out)) + + +# ── the routes ─────────────────────────────────────────────────────────────────────────────────── +@router.get("/admin/users") +def list_users(session: Session = Depends(admin_gate)): + reg = _registry() + return {"users": [_view(u, reg[u]) for u in sorted(reg)]} + + +@router.post("/admin/users", status_code=201) +def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): + """Create an account. **409 if the username already exists — `PATCH` is the update path.** + + ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh + record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and + RESURRECT every cookie minted before its last password change. `core/users.py` now + preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / + update a user" calls the same function), but this route still refuses: an upsert that silently + replaces an account's password, role and BU access because someone reused a username is not an + update anybody asked for. + """ + body = body or {} + _require_store() + uname = _clean_username(body.get("username")) + pw = _clean_password(body.get("password")) + role = _clean_role(body.get("role") or "user") + bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") + modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") + name = str(body.get("name") or "").strip() or uname + agent = str(body.get("agent") or "").strip() or None + email = str(body.get("email") or "").strip() or None + + if uname in _registry(): + raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") + users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, + agent=agent, email=email) + reg = _registry() + if uname not in reg: + # The store accepted the write and does not have it. Reporting 201 here would be the + # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. + raise err(503, "store_unavailable", "the account was not saved — try again") + return {"user": _view(uname, reg[uname])} + + +@router.patch("/admin/users/{username}") +def update_user(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Change access on an existing account. Only the keys PRESENT in the body change. + + ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every + request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very + next call — bumping the epoch would only add a forced re-login on top. `active: false` is the + exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its + session rather than keep working until the cookie expires. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg: + raise err(404, "no_such_user", f"no account named {uname!r}") + if not body: + raise err(400, "empty_patch", "no fields to update") + + unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) + if unknown: + # A silently-ignored field is how a UI ends up believing it saved something it did not. + raise err(400, "unknown_fields", f"cannot update: {unknown}") + + is_self = uname == session.uname + role = _clean_role(body["role"]) if "role" in body else None + active = bool(body["active"]) if "active" in body else None + # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is + # not the thing that keeps the product reachable — it just stops an administrator removing + # their own access with a toggle and having to go find the master password. + if is_self and role is not None and role != "admin": + raise err(400, "self_demote", + "you cannot remove your own administrator role — ask another admin") + if is_self and active is False: + raise err(400, "self_deactivate", "you cannot deactivate your own account") + + kw = {} + if role is not None: + kw["role"] = role + if "name" in body: + kw["name"] = str(body["name"] or "").strip() or uname + if "bus" in body: + kw["bus"] = _clean_bus(body["bus"]) + if "modules" in body: + kw["modules"] = _clean_modules(body["modules"]) + if "agent" in body: + kw["agent"] = str(body["agent"] or "").strip() # '' clears the link + if "email" in body: + kw["email"] = str(body["email"] or "").strip() + if kw: + users.set_access(uname, **kw) + if active is not None and active != bool(reg[uname].get("active", True)): + users.set_active(uname, active) # bumps the epoch: kills live sessions + + fresh = _registry() + return {"user": _view(uname, fresh[uname])} + + +@router.post("/admin/users/{username}/password", status_code=204) +def set_password(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump + happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two + can never disagree. + + 204 with no body: there is nothing to say that the caller did not already know, and echoing the + account back would invite a client to diff a response for a password change. + """ + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg: + raise err(404, "no_such_user", f"no account named {uname!r}") + pw = _clean_password((body or {}).get("password")) + before = int(reg[uname].get("epoch") or 0) + users.set_password(uname, pw) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # The whole value of this route is the revocation. If the epoch did not move, the sessions + # were not revoked, and answering 204 would say they were. + raise err(503, "store_unavailable", + "the password change did not persist — outstanding sessions were NOT revoked") + return Response(status_code=204) + + +@router.get("/settings") +def settings(session: Session = Depends(require_session)): + """The session's OWN settings — any authenticated user, not admin-only. + + `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a + client may know about itself (never a hash, never the epoch). `scope` restates it as the two + things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others + would answer a question about our customer list. + """ + modules = perms.allowed_modules(session.user) + return { + "user": _public_user(session.user), + "scope": { + "bus": perms.allowed_bu_labels(session.user), + "team_id": perms.scope_team_id(session.user), + "agent": perms.scope_agent(session.user), + "modules": sorted(modules) if modules is not None else "all", + }, + "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", + session.tenant)}, + # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client + # does not paint a "Manage users" button that 403s. + "admin": perms.is_admin(session.user), + } diff --git a/api/routes_auth.py b/api/routes_auth.py new file mode 100644 index 0000000000000000000000000000000000000000..c054ba7b60d8a686b6849b6463f77e864ffa8a0e --- /dev/null +++ b/api/routes_auth.py @@ -0,0 +1,143 @@ +"""routes_auth.py — X2's auth leg: login / logout / me (EXIT-3a). + +Password auth against the SAME `core/users` accounts the Streamlit `gate()` uses, so both +front-ends share one identity today. The amended D-3 (our own OIDC client against Authentik, with +a branded login page) lands when D-1/D-2 unblock (owner blocker B-4); this is the interim leg it +replaces, and it is deliberately the same account store so the migration is a swap of the +CREDENTIAL check, not of the user model. +""" +import hmac +import os +import time +from collections import OrderedDict + +from fastapi import APIRouter, Body, Depends, Request, Response + +import aios_session +from deps import Session, err, require_session, users, perms + +router = APIRouter(prefix="/api/v1/auth") + +#: THE SERVER OWNS BACKOFF (X5): the login page shows an inline error and never paces itself, so +#: the pacing has to be here or it does not exist. Bounded so it cannot grow into a memory leak. +#: ⚠ HONEST LIMIT: per-PROCESS. Behind several workers an attacker gets this budget per worker, +#: and it resets on deploy. It raises the cost of online guessing against PBKDF2-200k; it is not +#: a substitute for the real thing, which arrives with Authentik (D-1) owning rate limits centrally. +#: ⚠ KEYED BY (TENANT, USERNAME), not by username. Two tenants can each have an 'admin' or a +#: 'boss', and a shared bucket would mean failed logins against tenant B lock the same name out of +#: tenant A — a cross-tenant coupling in the one wave whose point is that tenants cannot reach +#: each other. It is also a (weak) existence oracle: a 429 for a name you never tried here would +#: tell you somebody else did. +_FAILS = OrderedDict() +_MAX_TRACKED = 2048 +_LOCKOUT_AFTER = 8 +_LOCKOUT_SECONDS = 60 + + +def _throttled(key, now=None): + now = now or time.time() + hit = _FAILS.get(key) + if not hit: + return 0 + count, last = hit + if count < _LOCKOUT_AFTER: + return 0 + remaining = int(_LOCKOUT_SECONDS - (now - last)) + return remaining if remaining > 0 else 0 + + +def _note_failure(key, now=None): + now = now or time.time() + count, last = _FAILS.get(key, (0, 0.0)) + # A window that has fully elapsed starts the count again — a user who mistypes twice today + # and twice next week is not an attacker. + if now - last > _LOCKOUT_SECONDS: + count = 0 + _FAILS[key] = (count + 1, now) + _FAILS.move_to_end(key) + while len(_FAILS) > _MAX_TRACKED: + _FAILS.popitem(last=False) + + +def _clear_failures(key): + _FAILS.pop(key, None) + + +def _public_user(user): + """What the client is allowed to know about itself. Never a hash, a salt, or the epoch — + the epoch is a server-side revocation handle and a client has no use for it.""" + return {"username": user.get("username", ""), "name": user.get("name", ""), + "role": user.get("role", "user"), + "bus": perms.allowed_bu_labels(user), + "team_id": perms.scope_team_id(user), + "agent": perms.scope_agent(user), + "modules": sorted(perms.allowed_modules(user) or []) or "all", + "landing": perms.landing_page(user)} + + +@router.post("/login") +def login(request: Request, response: Response, body: dict = Body(default=None)): + body = body or {} + uname = str(body.get("username") or "").strip().lower() + pw = str(body.get("password") or "") + tenant = str(body.get("tenant") or "royal-imports").strip().lower() + + # The tenant is validated BEFORE the credential check so an unknown slug can never mint a + # cookie — but it answers with the SAME 401 as a bad password, because "that tenant does not + # exist" is a fact about our customer list. + from harness import runtime + try: + runtime.get_runtime(tenant) + except KeyError: + raise err(401, "invalid_credentials", "that username and password do not match") + + throttle_key = (tenant, uname) + wait = _throttled(throttle_key) + if wait: + raise err(429, "too_many_attempts", + f"too many failed attempts — try again in {wait} seconds") + + user = users.verify(uname, pw) if (uname and pw) else None + if not user: + # CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no + # record, no PBKDF2) and burns ~200k iterations for a known one, so the response time + # answers "does this account exist?" to anyone with a stopwatch. Burning one equivalent + # hash on the failure path removes that oracle. It costs nothing on the happy path. + try: + users._hash(pw or "x", "00" * 16) + except Exception: + pass + _note_failure(throttle_key) + raise err(401, "invalid_credentials", "that username and password do not match") + + _clear_failures(throttle_key) + value, _claims = aios_session.mint(tenant, user["username"], int(user.get("epoch") or 0)) + aios_session.set_cookie(response, request, value) + return {"user": _public_user(user)} + + +@router.post("/logout", status_code=204) +def logout(request: Request): + """204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from + an already-invalid session, or a user holding a broken cookie has no way to get rid of it. + + ⚠ THE COOKIE IS CLEARED ON THE RETURNED RESPONSE, not on an injected `response` param. When a + handler RETURNS a Response object, FastAPI ships that object — anything written to the + injected `response` is silently dropped. The first version of this route took `response: + Response`, called `clear_cookie` on it, then returned a fresh `Response(204)`: the 204 was + correct, no `Set-Cookie` was ever sent, and the session stayed alive after a "successful" + logout. Caught by asserting `/auth/me` is 401 AFTER the logout rather than trusting the 204. + + ⚠ This clears the BROWSER's copy only — the signed value stays cryptographically valid until + it expires, which is the honest cost of a stateless session. "Sign me out everywhere" is + `core.users.bump_epoch` (a password change already does it); D2's Postgres session mirror + makes per-device revocation possible and arrives with C-2. + """ + out = Response(status_code=204) + aios_session.clear_cookie(out, request) + return out + + +@router.get("/me") +def me(session: Session = Depends(require_session)): + return {"user": _public_user(session.user), "tenant": session.tenant} diff --git a/api/routes_customers.py b/api/routes_customers.py new file mode 100644 index 0000000000000000000000000000000000000000..f3f4cbeeecc2b2dad33626a8d39e183b275703b8 --- /dev/null +++ b/api/routes_customers.py @@ -0,0 +1,208 @@ +"""routes_customers.py — X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a). + +Two things happen here that did not happen in the pre-wave `main.py`: + + 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` — the whole + book, to anyone holding the shared APP_PASSWORD. Now the pool is built with + `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a + Fisch row and an agent-linked login never receives another rep's book. The scope is applied + at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist + in this response. + + 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home + for the same user-owned fields the Streamlit app keeps in the tenant store — two truths, and + whichever process you asked last was right. Reads and writes now both go through + `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE §1a rule 3). + +⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is +cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API +container until its cache is refreshed, and vice versa. Deleting the fork removes the second +SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4, +owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient +proof is single-process and would report green on exactly the thing that is still broken. +""" +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, module_gate, perms + +router = APIRouter(prefix="/api/v1") + +#: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a +#: 403 rather than an empty table that looks like "you have no customers". +MODULE = "customer_data" + +_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before + + +def _pool_rows(session: Session): + """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that + may be shared between users. + + ⛔ WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is + `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is + genuinely scope-shaped — two users with the same BU and the same book are asking the same + question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation). + + The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped + and then removed. `fields` comes from `fields_from_workspace(ws)` — that user's own `custom_` + and `measure_` columns — and every overlay cell comes from `ws['overlays']`; the table + workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e. + PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one + would have been served the FIRST one's private notes and private columns. The scope key was + right for the pool and wrong for everything wrapped around it. + + It survived a green 129-check battery because every fixture user had a DISTINCT + `(team_id, agent)` pair, so no two of them ever collided on the key — the test set could not + express the bug. `verify_api.py` now carries a same-scope second user for exactly this. + """ + import modules.customer_data as cl + + rt = session.runtime + team_id = perms.scope_team_id(session.user) + agent = perms.scope_agent(session.user) + key = ("pool", team_id, agent) + hit = rt.pool_cache.get(key) + now = time.time() + if hit and now - hit[0] < _CACHE_TTL: + return hit[1] + # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled + # builder the Streamlit page uses — passing the scope here is what makes the isolation a + # property of the QUERY instead of a filter someone can forget to apply downstream. + rows_src = cl.pool(agent, team_id) + rt.pool_cache[key] = (now, rows_src) + # Bounded: a scope cache that only ever grows is a memory leak in a shared process. + if len(rt.pool_cache) > 16: + for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]: + rt.pool_cache.pop(stale, None) + return rows_src + + +def _payload(session: Session): + """`{fields, rows, today, pulled_at}` — X2 fixes this at the EXACT shape the pre-wave + `/api/customers` served, which `verify_fields_contract.py` referees. + + The per-USER half (this user's field list, this user's overlay cells) is assembled fresh on + every request from one workspace read. That read is a single store `get`, which is cheap next + to the pool build — and it is the only arrangement in which nothing user-shaped can outlive + the request that asked for it. See `_pool_rows` for what the cache holds and why. + """ + import aios_grid + from core import grid_events + + rows_src = _pool_rows(session) + + # ONE workspace read serves both the field list and the overlay values (doing it per stratum + # would triple the store reads per request). + ws = grid_events.table_workspace(_ctx_for(session, []), allowed_pids=None) + fields = aios_grid.fields_from_workspace(ws) + odoo_keys = [f["key"] for f in fields if f.get("source") == "odoo"] + overlay_keys = [f["key"] for f in fields if f.get("source") == "overlay"] + passthrough = {f["key"] for f in fields + if f.get("source") == "odoo" and f.get("type") in ("text", "status", "date")} + overlays = ws.get("overlays") or {} + + # ⚠ `rows_src` IS THE SHARED CACHED LIST — read it, never mutate it. Every row below is a NEW + # dict; writing an overlay value onto `r` instead of `row` would put one user's private cell + # into the copy the next same-scope user is served, which is the leak this split exists to + # prevent, re-introduced one line lower down. + rows = [] + for r in rows_src: + pid = r.get("pid") + row = {"pid": pid, "_created": r.get("_created") or ""} + for k in odoo_keys: + v = r.get(k) + row[k] = v if k in passthrough else _round(v) + stored = overlays.get(str(pid)) or {} + for k in overlay_keys: + row[k] = stored.get(k, "") + rows.append(row) + + return {"fields": fields, "rows": rows, + # `today` rides the payload because every relative date condition must resolve against + # the TENANT's day, never the browser's — a client that falls back to its own clock + # disagrees with the server for everyone west of it. + "today": time.strftime("%Y-%m-%d"), + "pulled_at": time.strftime("%Y-%m-%d %H:%M")} + + +def _round(v): + # bool is a subclass of int — guard it so True/False never becomes 1/0. Mirrors + # royalimports_os/aios_grid.py._round exactly (embed == standalone on boolean-valued fields). + return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + +def _ctx_for(session: Session, pids): + """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather + than a phantom in-memory workspace an API request cannot persist.""" + from core import grid_events + return grid_events.EventCtx( + uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[], + admin=session.admin, fallback_ws=None, seen_ids={}) + + +def allowed_pids(session: Session): + """The pids this session may touch — the POOL's own ids, so the write wall and the read scope + can never disagree. + + Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through + the full payload would pay for a workspace read and a row assembly on every write. + """ + return frozenset(r["pid"] for r in _pool_rows(session) if r.get("pid") is not None) + + +@router.get("/customers") +def customers(session: Session = Depends(module_gate(MODULE))): + return _payload(session) + + +@router.patch("/customers/{pid}") +def patch_customer(pid: int, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever. + + Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing + the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall + and the truncation rules live, and a second implementation of those would be a second set of + them to keep in step. The response reports what was ACCEPTED, which is not always what was + asked for. + """ + from core import grid_events + + updates = dict(body or {}) + if not updates: + raise err(400, "empty_patch", "no fields to update") + pool = allowed_pids(session) + if pid not in pool: + # 403, not 404: the pid may well exist — it is simply not in this session's book, and + # saying "no such customer" would confirm the opposite to anyone who guessed right. + raise err(403, "out_of_scope", "that customer is not in your book") + payload = _payload(session) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pool, fields=payload["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}) + try: + grid_events.handle_one( + {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch", + "pid": pid, "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable — your change was not saved") + + # What actually landed, read back from the store rather than echoed from the request: a + # refused key or a truncated value must not be reported as accepted. + stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None) + .get("overlays") or {}).get(str(pid)) or {} + accepted = {k: stored.get(k) for k in updates if k in stored} + refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k])) + # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so + # read-your-writes within this runtime is a property of the design rather than of a + # write-through step somebody has to remember. (It was a write-through step while the whole + # payload was cached on a scope key — the arrangement that leaked one user's notes to + # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module + # docstring says why, and Postgres is the fix. + out = {"ok": True, "pid": pid, "updates": accepted} + if refused: + out["refused"] = refused + return out diff --git a/api/routes_grid.py b/api/routes_grid.py new file mode 100644 index 0000000000000000000000000000000000000000..de3372a3641bc619a77053ed4831c48c9464da6e --- /dev/null +++ b/api/routes_grid.py @@ -0,0 +1,155 @@ +"""routes_grid.py — X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c). + +`POST /api/v1/grid/events` takes the component's event objects VERBATIM — the same objects the +Streamlit host receives through the component value slot, unchanged — and runs them through the +same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of +EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated +anything here would be a second wall to keep in step, and the two would drift on the first change. + +DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's +`seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends +its recent 24-event window on every emit inside one page session. A stateless API has no such +dict, and inventing a per-user server-side one would be exactly the resident per-tenant state +EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole +purpose — a batch that repeats an id processes it once), and a genuinely replayed request is +handled by the operations being idempotent. The one event where a replay is observable is +`add_to_list` (it would add the same pids to the same cohort twice — a set union, so the +membership is unchanged, but the toast count repeats). Noted rather than papered over; a +server-side idempotency key belongs with D2's session mirror (C-2). + +STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises +`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read +back. A 200 over a write that evaporated is the failure this rule exists to prevent. +""" +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, module_gate + +router = APIRouter(prefix="/api/v1") + +MODULE = "customer_data" + +#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a +#: legitimate client — refuse it rather than doing 500 store writes on one request. +_MAX_EVENTS = 24 + + +def _ctx(session: Session, fields, pids, **kw): + from core import grid_events + return grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin, + fallback_ws=None, seen_ids={}, **kw) + + +#: The two surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather +#: than over the whole scoped pool — `app.py:page_cohort` calls the same `_table_grid` with +#: `cohort_mode=True, scope_key='cohort'`. +_SCOPES = ("customer", "cohort") + + +def _scope_or_400(raw): + """⛔ REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would + hand a user the whole book on a page they opened to see one cohort — the widening direction, + which is the one that must fail closed.""" + scope = (raw or "customer").strip().lower() + if scope not in _SCOPES: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(_SCOPES)} — refusing to guess") + return scope + + +@router.get("/workspace") +def workspace(scope: str = "customer", + session: Session = Depends(module_gate(MODULE))): + """The durable table workspace for this session: views, fields, overlays, folders. + + ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 — an X2 AMENDMENT, see the split doc). X2 fixes + `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot + ride along in it without forking the thing that gate keeps single-sourced. Without a + workspace route the standalone shell's write path would be WRITE-ONLY: events persist + server-side and nothing reads them back on reload, so a saved view looks lost to the user + even though the store has it. This route closes that read-back gap at its own URL. + + `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader — the + wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a + full-access user. + """ + from core import grid_events + from routes_customers import allowed_pids + + scope = _scope_or_400(scope) + try: + pids = allowed_pids(session) + ws = grid_events.table_workspace(_ctx(session, [], pids), allowed_pids=pids) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the tenant store is unavailable") + + # ⛔ THE SURFACE STAMP — a MIRROR of `app.py:5830-5836`, and it must stay one. These four keys + # are the ONLY thing that tells the client which surface it is drawing: `CustomerGrid.tsx:685` + # reads `payload.workspace.cohortMode` and nothing else. Until this route could express them, + # the standalone shell had no way to render Cohort at all — the write path already accepted + # `scopeKey` (below), so the read path was the whole gap. + # + # ⚠ Deliberately NOT `hideViews`. The Streamlit Cohort page does not set it either: `cohortMode` + # is what swaps the Views panel for the cohort list, so setting both would be two switches for + # one behaviour and they would drift. + ws = dict(ws) # never stamp the object the store handed back + ws["scopeKey"] = scope + if scope == "cohort": + ws["cohortMode"] = True + # The create pane offers "this cohort table only" (default) vs "all customer tables". + ws["scopeChoice"] = True + return {"workspace": ws} + + +@router.post("/grid/events") +def grid_events_route(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """`{events: []}` → `{results, doc?, toast?}`.""" + from core import grid_events + from routes_customers import _payload, allowed_pids + + events = (body or {}).get("events") + if events is None and isinstance(body, dict) and body.get("type"): + events = [body] # a single event object, the legacy shape + if not isinstance(events, list): + raise err(400, "bad_events", "expected {events: [...]}") + if len(events) > _MAX_EVENTS: + raise err(400, "too_many_events", + f"at most {_MAX_EVENTS} events per request (the client's resend window)") + + payload = _payload(session) + pids = frozenset(r["pid"] for r in payload["rows"] if r.get("pid") is not None) + # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be + # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' — so a + # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on + # what a scope is, or the surface you read is not the surface you wrote. + ctx = _ctx(session, payload["fields"], pids, + scope_key=_scope_or_400((body or {}).get("scopeKey"))) + + # Per-event results so the client can tell which of a batch landed — the component's own + # bridge has no response channel at all, so this is strictly more than the embed gets. + results = [] + try: + for one in events: + eid = str(one.get("id") or "") if isinstance(one, dict) else "" + rerender = grid_events.handle_one(one, ctx) + results.append({"id": eid, "rerender": bool(rerender)}) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable — none of your changes were saved") + + out = {"results": results, "rerender": any(r["rerender"] for r in results)} + if ctx.out.doc is not None: + out["doc"] = ctx.out.doc + if ctx.out.toast is not None: + out["toast"] = ctx.out.toast + # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped + # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an + # Odoo column — Odoo is read-only. Everything an event DOES change (overlays, fields, views, + # folders, cohorts) is re-read from the store on the next request. An earlier version cleared + # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh + # data that was never in it. + return out diff --git a/api/routes_nav.py b/api/routes_nav.py new file mode 100644 index 0000000000000000000000000000000000000000..408fb2b26ad26112d935bbb1d8a4651bc1f37697 --- /dev/null +++ b/api/routes_nav.py @@ -0,0 +1,34 @@ +"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. + +SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who +may see what — a nav that hides a link the API would still serve is a UI courtesy, not a +permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), +so the nav and the 403 can never disagree about a grant. + +The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no +`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the +`prefs.json` Library preference is deliberately NOT applied — is documented on +`core.perms.nav_pages`, which is where it belongs: one place, both callers. +""" +from fastapi import APIRouter, Depends + +from deps import Session, perms, require_session + +router = APIRouter(prefix="/api/v1") + + +@router.get("/nav") +def nav(session: Session = Depends(require_session)): + """`{pages: [{key,label,source?,chrome}], landing}`. + + Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you + are not allowed". A session that may open nothing at all is a misconfigured account, and it + gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is + how a permission bug hides in plain sight (X2's never-an-empty-200 rule). + """ + pages = perms.nav_pages(session.user) + if not pages: + from deps import err + raise err(403, "no_surfaces", + "your account has no dashboards assigned — ask an administrator") + return {"pages": pages, "landing": perms.landing_page(session.user)} diff --git a/api/routes_pages.py b/api/routes_pages.py new file mode 100644 index 0000000000000000000000000000000000000000..d178f17c5c2bb8582b12430b025bdcc70caf306b --- /dev/null +++ b/api/routes_pages.py @@ -0,0 +1,65 @@ +"""routes_pages.py — X2/Y1's read of a whole dashboard page: `GET /api/v1/pages/{key}`. + +ONE route serves every ported page. That is the wave-3 promise made mechanical: Collections and +Procurement arrive as a builder registered in `pages.py`, and they inherit this route's session +wall, grant wall, BU resolution, caching and error shape without a line being written here. + +WHY THE GATE IS INSIDE THE HANDLER and not `Depends(module_gate(...))`. `module_gate` closes over a +FIXED module key at import; the key here is a PATH PARAMETER, so the check has to happen once the +key is known. It is the same predicate (`Session.require` → `perms.may_open`), reached from +`pages.build_envelope` so a future builder cannot be added without it. + +FAIL-CLOSED, and the order matters: + no cookie -> 401 (require_session) + unknown / unported page key -> 404 (a fact about our roadmap, not about this user) + no grant for the page -> 403 (never an empty 200 — an empty page reads as "no data") + a BU the account may not see -> 403 (never coerced to their own BU) + a malformed bu -> 400 +""" +from fastapi import APIRouter, Depends, Query + +import pages +# ⛔ IMPORTED HERE AT MODULE LEVEL ON PURPOSE, even though `pages.builder_for` also imports it +# lazily. `deploy_web.py` ships a hand-maintained file set, and W2-11 adds a `check_include()`-style +# IMPORT WALK to catch what that set misses — a walk that only follows module-level imports would +# never reach a builder registered from inside a function body, and a missing module CRASHED the +# live Space on 2026-07-04. One visible import makes the builder reachable by static analysis; the +# lazy import in `pages.py` stays as the fallback that keeps the registry self-assembling. +# (No circularity: `pages` is fully loaded by the line above before `pages_sales` imports it back.) +import pages_sales # noqa: F401 +from deps import Session, require_session + +router = APIRouter(prefix="/api/v1") + + +@router.get("/pages/{key}") +def page_data(key: str, + bu: str = Query(default=None, + description="'all' or a business-unit id. NARROWS the session's own " + "scope; it can never widen it."), + period: str = Query(default=None, + description="Page-specific display parameter. The resolved value " + "is echoed in controls[].value."), + session: Session = Depends(require_session)): + """The Y1 envelope for one page, scoped to this session's BU + own-book grant.""" + return pages.build_envelope(session, key, bu=bu, period=period) + + +@router.get("/pages") +def page_index(session: Session = Depends(require_session)): + """Which pages have published data, filtered to what this session may open. + + The shell needs this to know whether a nav entry leads to a ported page or to the Streamlit + host during the strangler period — and it is `may_open`-filtered for the same reason + `/api/v1/nav` is: the client renders what it is given and never decides who may see what. + + ⚠ AN EMPTY LIST IS A LEGITIMATE 200 HERE, and that is not a hole in the never-an-empty-200 rule. + `/api/v1/nav` 403s on empty because a session that may open NOTHING is a misconfigured account. + This route answers "which of your pages have been PORTED yet", and during the strangler period + "none of them" is the ordinary, truthful answer for a user granted only surfaces that still + live in the Streamlit host. Returning 403 would say they lack access to pages they can see. + """ + from deps import perms + keys = [k for k in pages.known_pages() + if perms.may_open(session.user, (pages.builder_for(k).MODULE))] + return {"pages": keys} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5476ec3cb2f98d94f6ad6a25518072c8827da90b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +# AIOS web API — the FastAPI backend + the royalimports_os data-layer deps it reuses. +# (streamlit is included only because the shared modules/core may import it at load; it is never +# run here. No altair/plotly/openpyxl/reportlab — those are Streamlit-UI/export only.) +fastapi>=0.139 +uvicorn[standard]>=0.30 +python-dotenv>=1.0 +pandas>=2.0 +requests>=2.28 +huggingface_hub>=0.20 +duckdb>=1.0 +pyyaml>=6.0 +streamlit==1.58.0 diff --git a/royalimports_os/aios_grid.py b/royalimports_os/aios_grid.py new file mode 100644 index 0000000000000000000000000000000000000000..3a55bdecf6295fa98387b6cc99e4bac631647164 --- /dev/null +++ b/royalimports_os/aios_grid.py @@ -0,0 +1,1221 @@ +"""aios_grid — embed the AIOS React/glide Airtable-style grid inside Streamlit. + +This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the +standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit +component. The React bundle reads its data from `window.__AIOS_GRID__` (an object +`{fields, rows}`) that we inject into the page before the app's module script runs +— so there is NO backend and NO /api call in the Streamlit container; the browser only ever +sees the derived JSON we hand it. + +Usage (from any page): + import aios_grid + aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS) + +Design notes: + * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and + `rows_from_pool` takes already-built rows — so this helper is tenant/module-agnostic and + sidesteps deploy_hf.py's import guard entirely. + * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to + deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy + dist-embed/index.html -> royalimports_os/aios_grid_embed.html (see build_embed.py). + For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly. + * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args + into the React app and returns guarded events for persistence in the tenant store. The old + injected-HTML path remains a read/local-write fallback when component assets are absent. +""" +import json +import re +from pathlib import Path + +_HERE = Path(__file__).resolve().parent + +# Where the inlined single-file build lives. FIRST match wins: +# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space) +# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy) +_EMBED_CANDIDATES = [ + _HERE / "aios_grid_embed.html", + _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html", +] +_COMPONENT_CANDIDATES = [ + _HERE / "aios_grid_component", + _HERE.parent / "aios-web" / "web" / "dist-embed", +] +_DECLARED_COMPONENTS = {} + +# --- the FIELD CONTRACT — loaded from the CANONICAL source `aios_grid_fields.json` in THIS +# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is +# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo. +# +# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so +# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the +# module preserves that exactly — no import, no guard interaction — while still single-sourcing +# the values (a build-time copy from another tree would reintroduce the drift we're removing). +# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run +# aios-web/verify_fields_contract.py. --- +_FIELDS_PATH = _HERE / "aios_grid_fields.json" + + +def _load_fields(): + if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single " + "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)." + ) + doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) + fields = doc.get("fields") if isinstance(doc, dict) else doc + if not isinstance(fields, list) or not fields: + raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.") + return fields + + +FIELDS = _load_fields() + +# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded — +# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte. + + +def _round(v): + return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + +# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors +# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. +# select — a single-select with its own `options` (the "Status" a user wants to add; distinct +# from the Odoo-sourced `status` lifecycle, which is not user-defined) +# user — an assignee, whose choices come from the HOST's real user list, never from here +#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select — declared +#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every +#: member of it (the `multi` grouping/cell contract the Cohorts column established). +#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') · +#: `phone` / `email` / `url` (text-family with per-type rendering) · `rating` (top-level +#: `max`, SVG stars client-side — never emoji) · `created_time` (READ-ONLY, renders the row's +#: `_created`) · `formula` (READ-ONLY, client-computed from the row's other cells). +CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", + "checkbox", "phone", "email", "url", "rating", "created_time", "formula"} +#: User-created types whose CELLS are read-only: their values are computed (formula — client +#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`). +#: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so +#: the client never offers an editor and the host never accepts a cell write for them. +READONLY_CUSTOM_TYPES = {"created_time", "formula"} +MAX_FIELD_OPTIONS = 50 +MAX_FORMULA_LEN = 500 +#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. +RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 + +#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that +#: IS a measure over a window — `Sales · the last 90 days` as a column. Mirrors the client's +#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata +#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a +#: measure field is READ-ONLY and its values are computed by the host per render. +MEASURE_FIELD_PREFIX = "measure_" +#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range). +MEASURE_FIELD_TYPES = {"currency", "int", "pct"} + + +def _clean_options(raw): + """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. + Mirrors types.ts parseOptions — a choice list that means one thing in the picker and + another in the filter dropdown is a column with two vocabularies.""" + out, seen = [], set() + for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: + if not isinstance(v, (str, int, float)) or isinstance(v, bool): + continue + s = str(v).strip()[:120] + if not s or s.lower() in seen: + continue + seen.add(s.lower()) + out.append(s) + return out[:MAX_FIELD_OPTIONS] + + +def _clean_rating_max(raw): + """A rating's star count, bounded. Anything unusable is the default, not a refusal — the + field still holds its 1..max integers either way.""" + try: + return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX)) + except (TypeError, ValueError): + return RATING_MAX_DEFAULT + + +#: The field types a number-style display format may apply to. `formula` is here because its +#: RESULT is a number the client renders; `pct` already renders in points and takes decimals. +_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"} + + +def _clean_format(raw, ftype): + """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong + types return None (the property is simply absent). Rendering-only — a format can change how + a value reads, never what it is, which is why this needs no parity gate of its own.""" + if not isinstance(raw, dict): + return None + out = {} + if ftype in _NUMBER_FORMAT_TYPES: + if isinstance(raw.get("thousands"), bool): + out["thousands"] = raw["thousands"] + if raw.get("decimals") is not None: + try: + d = int(raw["decimals"]) + except (TypeError, ValueError): + d = None + if d is not None and 0 <= d <= 4: + out["decimals"] = d + if isinstance(raw.get("abbrev"), bool): + out["abbrev"] = raw["abbrev"] + elif ftype in ("date", "created_time"): + if isinstance(raw.get("time"), bool): + out["time"] = raw["time"] + if raw.get("tz") in ("local", "utc"): + out["tz"] = raw["tz"] + return out or None + + +def _clean_permissions(raw): + """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is + the host handler's business (creator/admin, enforced fail-closed there); this validates + only the shape, like every other property here.""" + if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"): + return {"edit": raw["edit"]} + return None + + +_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!]*$") +_FORMULA_REF = re.compile(r"\{([^{}]*)\}") + + +def _clean_formula(raw, valid_keys=None): + """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9). + + Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}` + refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell — never a + wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a + Python mirror of the grammar: a second engine is the filter_sql-class drift risk. + Structure IS checked — charset, length, balanced parens, well-formed non-empty `{refs}` — + and at WRITE time (`valid_keys` given) every ref must name a field this table has, fail + closed. At READ time refs are left alone: a referenced field deleted later must blank the + CELLS, not vaporise the column. + """ + if not isinstance(raw, str): + return None + s = raw.strip() + if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s): + return None + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return None + if depth: + return None + refs = _FORMULA_REF.findall(s) + leftover = _FORMULA_REF.sub("", s) + if "{" in leftover or "}" in leftover: # unbalanced / nested braces + return None + if any(not r.strip() for r in refs): # a `{}` ref names nothing + return None + if valid_keys is not None and any(r not in valid_keys for r in refs): + return None + return s + + +def _field_extras(saved, ftype): + """createdBy / permissions / format / scope — the validated passthrough the created strata + share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it + is carried so the client can gate its menus and the handler can enforce against it. + `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific — the handler + stamps it at create (cohort page only) and preserves it like createdBy; carried here so the + client can label the field, filtered OUT of other pages by fields_from_workspace.""" + out = {} + who = saved.get("createdBy") + if isinstance(who, str) and who.strip(): + out["createdBy"] = who.strip()[:80] + perms = _clean_permissions(saved.get("permissions")) + if perms: + out["permissions"] = perms + fmt = _clean_format(saved.get("format"), ftype) + if fmt: + out["format"] = fmt + if saved.get("scope") == "cohort": + out["scope"] = "cohort" + return out + + +#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27). +#: +#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the +#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the +#: caller has cohorts — the same condition under which the `__cohort__` FILTER field is offered. +#: Putting it in the canonical contract would mean an always-present column that is empty for +#: everyone else, plus three consumers to keep in step for a value none of them can produce. +COHORT_COLUMN = 'cohorts' + + +def cohort_field(label='Cohorts'): + """The derived cohort column's descriptor. + + `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability + off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay + keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what + stops the column menu calling it a "source field" on that basis. + + ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another + column. Text ops over a joined string would ALMOST work and disagree at the edges — `contains + "VIP"` also matches a cohort called "VIP club" — and a filter that is nearly right is worse + than one that is absent. + """ + return { + 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', + 'default': False, 'filterable': False, 'derived': True, 'multi': True, + 'note': 'The cohorts this customer belongs to, newest membership last. A cohort is a ' + 'LIST, so grouping by this column puts a customer under EVERY cohort they are ' + 'in — the group counts therefore add up to more than the record count, which ' + 'stays the number of distinct customers. Read-only: a cohort changes only on ' + 'the Cohort page. To FILTER by one, use the Cohort condition in Filter.', + } + + +def cohort_cells(cohorts, allowed_pids=None): + """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`. + + Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`, + never merged into the cached pool rows — those are shared across users, and stamping one + user's cohorts onto them would leak the membership to everybody else on the next render. + """ + cells = {} + for c in cohorts or []: + # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY + # cohort they are in, so it cannot also occur inside a name. Cohort names are free text + # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather + # than left to break the split silently — one group called "Q3 calls" and another called + # "west" would be two lists that do not exist. The cost is cosmetic and confined to the + # cell; the Cohort page still shows the name the user typed. + name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip() + for pid in c.get('pids') or (): + if allowed_pids is not None and pid not in allowed_pids: + continue + cells.setdefault(pid, []).append(name) + return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()} + + +def clean_measure_field(raw, offered): + """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER. + + `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` — + the same admission the condition builder uses, so a field can only name a measure this + caller could also filter by. Returns the canonical stored shape, or None (fail closed). + + `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused: + the client keys editability off `source == 'overlay'` and the host accepts cell writes only + for overlay keys, so a measure column cannot be typed into at either end. `filterable:False` + because the REPLACEMENT is the measure CONDITION with the same measure and window — the + governed, gate-proved path (CG-8/CG-12) — not text ops over a derived cell. + """ + if not isinstance(raw, dict): + return None + key = str(raw.get("key") or "") + if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80: + return None + spec = raw.get("measure") + if not isinstance(spec, dict): + return None + m = (offered or {}).get(spec.get("key")) + if not m: + return None # not admitted for this caller -> fail closed + window = _clean_window(spec.get("window")) + if window is None: + return None # a measure column with no period is not a column + mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency" + return { + "key": key, + "label": str(raw.get("label") or m.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": True, + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(raw.get("note") or "")[:2000], + "measure": {"key": str(spec.get("key"))[:80], "window": window}, + } + + +def measure_fields_of(fields): + """The formula-measure columns among `fields` — the ones whose values the host must compute + per render (see `rows_from_pool`'s `derived`).""" + return [f for f in fields or [] if isinstance(f.get("measure"), dict)] + + +def fields_from_workspace(workspace=None, cohorts=False, scope_key=None): + """Overlay persisted notes/custom fields onto the immutable source-field contract. + + `cohorts=True` appends the derived cohort column — see `cohort_field`. Appended LAST and + `default: False`, so it never displaces a column somebody already reads; the Fields menu is + where you turn it on. + + Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable), + and `measure_` formula-measure fields (read-only, host-computed — owner item 7). A measure + field was validated against the caller's measure OFFER when it was written + (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero + app-internal imports and cannot know the offer. A measure that has since become + unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value + time, never to an error. + + `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A + saved def carrying `scope` is emitted only when it matches — so a cohort-specific field + never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees + only unscoped (global) fields; base contract fields are never scoped. + """ + saved = dict((workspace or {}).get("fields") or {}) + out = [] + base_keys = {field["key"] for field in FIELDS} + for base in FIELDS: + meta = saved.get(base["key"]) or {} + field = dict(base) + if isinstance(meta.get("note"), str): + field["note"] = meta["note"][:2000] + # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) — how a + # number or date READS, per user. Rendering-only, so this is the whole acceptance. + fmt = _clean_format(meta.get("format"), base.get("type")) + if fmt: + field["format"] = fmt + # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved + # window/label override. Wave 6 deleted every preset member (the owner's + # no-buildable-presets rule) so this branch is currently MEMBERLESS — kept as the + # measure_ path's twin for any future preset-carrying contract, and because deleting + # it would silently change what a re-added preset means. + if base.get("preset") and isinstance(base.get("measure"), dict): + saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {} + window = _clean_window(saved_measure.get("window")) + if window is not None: + field["measure"] = {"key": base["measure"]["key"], "window": window} + if isinstance(meta.get("label"), str) and meta["label"].strip(): + field["label"] = meta["label"][:120] + out.append(field) + for key, field in saved.items(): + if key in base_keys or not isinstance(field, dict): + continue + if field.get("scope") and field.get("scope") != scope_key: + continue # a cohort-specific field on another page (wave-6 item 9) + if (str(key).startswith(MEASURE_FIELD_PREFIX) + and isinstance(field.get("measure"), dict)): + window = _clean_window(field["measure"].get("window")) + mkey = str(field["measure"].get("key") or "") + if window is None or not mkey: + continue + mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": bool(field.get("default", True)), + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(field.get("note") or "")[:2000], + "measure": {"key": mkey[:80], "window": window}, + **_field_extras(field, mtype), + }) + continue + if not str(key).startswith("custom_"): + continue + ftype = field.get("type") + if ftype not in CUSTOM_FIELD_TYPES: + continue + if ftype in READONLY_CUSTOM_TYPES: + # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort + # column's mechanism (source 'odoo' + derived) so the client never offers an + # editor and the host's overlay-write guard excludes them by construction. + # FILTERABLE since wave 6 (owner item 6): their values live client-side + # (formula computes over the row, created_time renders `_created`), this + # table's counts are client-mode, and the windowed count path never sees + # these tables — so the client engine answers them soundly. + entry = { + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "odoo", + "derived": True, + "filterable": True, + "default": bool(field.get("default", True)), + "custom": True, + "note": str(field.get("note") or "")[:2000], + **_field_extras(field, ftype), + } + if ftype == "formula": + formula = _clean_formula(field.get("formula")) + if formula is None: + continue # a formula field without a formula is nothing + entry["formula"] = formula + out.append(entry) + continue + if field.get("source") != "overlay": + continue + options = (_clean_options(field.get("options")) + if ftype in ("select", "multiselect") else []) + if ftype in ("select", "multiselect") and not options: + # A select with no surviving choices can never hold a value. Dropping the COLUMN + # would lose the user's data; degrading it to text keeps every stored value + # readable and lets them re-add choices. + ftype = "text" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "overlay", + "default": bool(field.get("default", True)), + "custom": True, + "agg": field.get("agg") if field.get("agg") in {"sum"} else None, + "note": str(field.get("note") or "")[:2000], + **({"options": options} if ftype in ("select", "multiselect") else {}), + # comma-joined SET semantics (the Cohorts column's contract): the row belongs to + # every member, groups count it under each, the toolbar count stays distinct. + **({"multi": True} if ftype == "multiselect" else {}), + **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}), + **_field_extras(field, ftype), + }) + if cohorts: + out.append(cohort_field()) + return out + + +def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): + """Map customer_data.pool() dicts -> the API row shape the grid expects: + pid + each Odoo field (numeric fields rounded, text/date passed through) + the + persisted external overlay. Mirrors the standalone API payload contract. + + `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than + reads off the pool row — today just the cohort column. A separate argument from `overlays` + on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is + never written back would make the dict mean two things. + """ + fields = fields or FIELDS + odoo_fields = [field for field in fields if field["source"] == "odoo"] + overlay_fields = [field for field in fields if field["source"] == "overlay"] + derived_keys = [field["key"] for field in fields if field.get("derived")] + overlays = overlays or {} + derived = derived or {} + out = [] + for r in pool_rows: + pid = r.get("pid") + # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the + # `created_time` field type renders, regardless of that field's own key. Not a Field: + # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride + # the same way: the Map VIEW's data, nullable, deliberately not a column. + row = {"pid": pid, "_created": r.get("_created") or "", + "lat": r.get("lat"), "lon": r.get("lon")} + for field in odoo_fields: + k = field["key"] + if field.get("derived"): + continue # not on the pool row — filled from `derived` below + v = r.get(k) + row[k] = v if field["type"] in {"text", "status", "date"} else _round(v) + saved = overlays.get(str(pid), {}) or {} + for field in overlay_fields: + row[field["key"]] = saved.get(field["key"], "") + got = derived.get(pid) or {} + for k in derived_keys: + # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a + # text column is the question somebody will ask of it. + row[k] = got.get(k, "") + out.append(row) + return out + + +# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) --------------- +# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are +# VALUE-FREE (they legitimately carry no value and must never be dropped for it). +FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', + 'gt', 'gte', 'lt', 'lte', 'between', 'within'} +#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not +#: a Field and never will be — a cohort is a hand-curated SET, so making it a column would mean +#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD. +COHORT_FIELD = '__cohort__' +#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts. +#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS — see types.ts COHORT_OPS: +#: a set op reaching a column leaf would fall through the client engine's switch to "no +#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. +COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} +#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: +#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the +#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. +COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} +#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS. +MAX_COHORT_IDS = 20 + + +def parse_cohort_ids(value): + """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`. + + Comma-separated in one string because `FilterRule.value` is what all four layers persist and + round-trip, and a one-element list is byte-identical to what the single-cohort leaf already + stored — so every shipped view parses with no migration. Safe because a cohort id is built + from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one. + """ + out = [] + for raw in ('' if value is None else str(value)).split(','): + cid = raw.strip()[:120] + if not cid or cid in out: + continue + out.append(cid) + if len(out) >= MAX_COHORT_IDS: + break + return out +# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays +# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this. +MAX_FILTER_DEPTH = 3 +MAX_FILTER_NODES = 100 # total nodes across the whole tree +MAX_FILTER_SIBLINGS = 50 # per level + + +#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by +#: design, so it does NOT know the window VOCABULARY — `harness/windows.py` owns that, mirrored +#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already +#: need a gate to prevent. This validates SHAPE only. +WINDOW_MAX_N = 3650 + + +def _clean_window(raw): + """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window. + + Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by + `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this + way keeps the grid module reusable and keeps one definition of what "last quarter" means. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if not isinstance(kind, str) or not kind or len(kind) > 40: + return None + out = {'kind': kind} + if raw.get('n') is not None: + try: + out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N)) + except (TypeError, ValueError): + return None + for side in ('from', 'to'): + if raw.get(side) not in (None, ''): + out[side] = str(raw[side])[:32] + return out + + +def _clean_rhs(raw, valid_keys): + """CG-9 — validate `{kind, colId, window?}`, the "compare against another attribute" side. + + SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys` + with the measure keys, exactly as it does for the left side), and a measure rhs must carry a + window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in ('field', 'measure', 'stat'): + return None + if kind == 'stat': + # A STATISTIC carries no column: the population is the comparand. Shape only — which + # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one + # is REFUSED there rather than guessed at, exactly like an unrecognised window kind. + stat = raw.get('stat') + if not isinstance(stat, str) or not stat or len(stat) > 24: + return None + return {'kind': 'stat', 'stat': stat} + col = raw.get('colId') + if col not in valid_keys: + return None + out = {'kind': kind, 'colId': col} + if kind == 'measure': + window = _clean_window(raw.get('window')) + if window is None: + return None # a measure comparand with no period is not a question + out['window'] = window + return out + + +#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8 +#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown +#: display means, so it is never stored. +#: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to +#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the +#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing +#: 'dashboard' here would silently downgrade every already-saved chart view to grid — and live +#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The +#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on +#: READ forever, only ever WRITE 'chart'. +DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart'} + +#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown +#: mode is still rejected rather than accidentally aliased. +_LEGACY_MODES = {'dashboard': 'chart'} + +#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's +#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a +#: kind the client cannot draw is worse than one that does not exist yet. +CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi'} +CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} +MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop +MAX_CHART_TITLE = 60 + +#: Wave-9 I11 (contract C2) — chart customisation, host-validated. +#: +#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex: +#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity) +#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored +#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable — +#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying. +CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'} +CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'} +MAX_AXIS_LABEL = 40 +#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px. +CHART_W_RANGE = (1, 12) +CHART_H_RANGE = (120, 800) + + +def _clean_chart(raw, valid_keys): + """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn. + + A chart's `y` is optional (absent = count of rows, which is what "how many customers per + state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and + silently keeping it would put an empty card on the dashboard with no way to tell why. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in CHART_KINDS: + return None + x = raw.get('x') + if x not in valid_keys: + return None # dead category ref -> the chart goes, not the board + cid = raw.get('id') + if not isinstance(cid, str) or not cid.strip(): + return None # the client owns chart ids; an unidentified card + # cannot be edited or removed, so it must not persist + out = {'id': cid.strip()[:64], 'kind': kind, 'x': x, + 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'} + if raw.get('y') in valid_keys: + out['y'] = raw['y'] + else: + # no measurable column -> the only honest aggregation left is "how many rows" + out['agg'] = 'count' + title = raw.get('title') + if isinstance(title, str) and title.strip(): + out['title'] = title.strip()[:MAX_CHART_TITLE] + + # ── wave-9 I11 (contract C2): customisation ──────────────────────────────────────────── + # `splitBy` is the field whose values become the SERIES. It is deliberately not called + # `colorBy`: that name already means two other things here (`config.colorBy` = row + # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable. + if raw.get('splitBy') in valid_keys and raw['splitBy'] != x: + out['splitBy'] = raw['splitBy'] + # Stacking is only a question once there are series to stack, and only for the two kinds + # that can express it. Anywhere else it is dropped rather than stored as a lie the client + # would have to re-decide. + if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True: + out['stacked'] = True + if raw.get('palette') in CHART_PALETTES: + out['palette'] = raw['palette'] + + axis = raw.get('axis') + if isinstance(axis, dict): + # ⛔ ONE y-scale, always. There is no second-axis key here and there must never be: + # two y-scales on one frame can manufacture any correlation you like by rescaling, and + # the honest alternatives are two charts, small multiples, or indexing to a common base. + # Ruled explicitly in contract C2 against the "Tableau versatility" brief. + clean_axis = {} + for side in ('x', 'y'): + spec = axis.get(side) + if not isinstance(spec, dict): + continue + one = {} + lab = spec.get('label') + if isinstance(lab, str) and lab.strip(): + one['label'] = lab.strip()[:MAX_AXIS_LABEL] + if spec.get('format') in CHART_FORMATS: + one['format'] = spec['format'] + if one: + clean_axis[side] = one + if clean_axis: + out['axis'] = clean_axis + + size = raw.get('size') + if isinstance(size, dict): + one = {} + for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)): + try: + one[key] = max(lo, min(hi, int(size[key]))) + except (KeyError, TypeError, ValueError): + pass # a partial size is fine: the client defaults the missing axis + if one: + out['size'] = one + return out + + +def _clean_display(raw, valid_keys): + """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed. + + `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` — mode + must be a known non-grid mode (grid is the absent default, so storing it would be a second + way to say nothing); every field ref must name a field this table has (a ref to a deleted + field is DROPPED and the client falls back to its per-mode default); unknown keys are + dropped. What each mode MEANS — calendar wants a date-family field, kanban a select-family + stack, map a single-select to colour by and a numeric to size by — is the client's + business: it is the only layer that renders them, and a wrong-typed ref degrades to that + surface's default rather than to an error (the `_clean_window` split). + + Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and + dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — + never the whole array, because losing one column should not cost the user a dashboard they + spent time building. + """ + if not isinstance(raw, dict): + return None + mode = raw.get('mode') + if mode not in DISPLAY_MODES or mode == 'grid': + return None + # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown + # mode is still rejected rather than accidentally aliased into a real one. Every already + # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. + mode = _LEGACY_MODES.get(mode, mode) + out = {'mode': mode} + for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'): + if raw.get(ref) in valid_keys: + out[ref] = raw[ref] + charts = raw.get('charts') + if isinstance(charts, list): + clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c] + # de-dupe by id: two cards sharing an id are one card as far as the client's keyed + # render is concerned, and the second would silently shadow the first + seen, uniq = set(), [] + for c in clean: + if c['id'] in seen: + continue + seen.add(c['id']) + uniq.append(c) + if uniq: + out['charts'] = uniq + return out + + +#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4). +#: +#: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each +#: view or cohort — a deliberate amendment to C4's first wording, recorded in the split doc. +#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort +#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated +#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of +#: what a view IS: keeping it out of the view config means duplicating or exporting a view does +#: not drag a folder reference along with it. One map, one home, both surfaces. +FOLDER_SURFACES = {"views", "cohorts"} +MAX_FOLDERS = 60 +MAX_FOLDER_NAME = 80 + +#: Wave-9 I15 (contract C5) — a user-chosen folder icon, as {shape, tone}. +#: +#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already +#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token, +#: not a colour — so a browser cannot post a hex and defeat a tenant restyle, and a shape the +#: client cannot render can never reach the store. +#: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors — +#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional +#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set +#: without the matching client geometry: an unknown shape falls back to the default folder mark, +#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder. +FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"} +#: Tones are the C1 pastels — FILLS ONLY, never text (the standing palette rule). 'grey' is the +#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed. +FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"} +FOLDER_ICON_DEFAULT_TONE = "grey" + + +#: Wave-9 I17 (contract C4) — who may EDIT a saved view. +#: +#: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today +#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one +#: user's views are invisible to every other user and "collaborative" has nothing to act on +#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable +#: and fail-closed now, so that when shared views land the permission does not need a data +#: migration and no saved view is retro-restricted. It does NOT make anything shared, and +#: nothing in the app currently reads it to grant or deny cross-user access. +#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md. +VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'} +MAX_VIEW_USERS = 50 + + +def clean_view_permissions(raw, default, known_users=None): + """{edit, users?} — fail-closed on both halves. + + `default` is supplied by the CALLER because it splits by path, and that split is a + permission rule rather than a formatting one: absent on a view that already exists means a + pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a + silent takeaway), while absent on CREATE must be 'personal' (a new view must never be + anyone-can-edit purely by omission). + + `known_users` (when given) is the real account list: an unknown name is DROPPED, and an + 'users' grant left with nobody in it collapses to 'personal' rather than to everyone. + """ + mode = (raw or {}).get('edit') if isinstance(raw, dict) else None + if mode not in VIEW_EDIT_MODES: + mode = default if default in VIEW_EDIT_MODES else 'personal' + if mode != 'users': + return {'edit': mode} + names, seen = [], set() + for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]: + u = str(u or '').strip() + if not u or u.lower() in seen: + continue + if known_users is not None and u not in known_users: + continue # fail-closed: a name we cannot resolve grants nothing + seen.add(u.lower()) + names.append(u) + if not names: + return {'edit': 'personal'} # an empty grant is NOT "everyone" + return {'edit': 'users', 'users': names} + + +def clean_folder_icon(raw): + """{shape, tone} or None. Fail-closed on both halves, independently. + + A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather + than losing the icon entirely — losing a user's pick because one half was wrong is the kind + of silent data loss the folder events already avoid elsewhere. + """ + if not isinstance(raw, dict): + return None + shape = raw.get("shape") + if shape not in FOLDER_ICON_SHAPES: + return None + tone = raw.get("tone") + return {"shape": shape, + "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE} + + +def clean_folders(raw): + """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}.""" + out = {} + for surface in FOLDER_SURFACES: + items, seen = [], set() + for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:80] + name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME] + if not fid or not name or fid in seen: + continue # an unidentified or unnamed folder cannot be shown or edited + seen.add(fid) + try: + order = int(f.get("order", len(items))) + except (TypeError, ValueError): + order = len(items) + row = {"id": fid, "name": name, "order": order} + icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark + if icon: + row["icon"] = icon + items.append(row) + items.sort(key=lambda x: x["order"]) + for i, f in enumerate(items): + f["order"] = i # re-index so `order` is always dense and total + if items: + out[surface] = items + return out + + +def clean_item_folders(raw, folders, valid_ids): + """{surface: {itemId: folderId}} — dropping any placement whose ITEM or FOLDER is gone. + + This is what makes a deleted folder's contents fall back to the root rather than vanish: + nothing stores "this item is in no folder", so an unresolvable placement simply disappears + and the item renders at the top level. Same for an item that was deleted elsewhere — its + stale placement can never resurrect it, because the sidebars render ITEMS and consult this + map, never the other way round. + """ + out = {} + for surface in FOLDER_SURFACES: + fids = {f["id"] for f in (folders or {}).get(surface, [])} + ok = {} + for item_id, fid in ((raw or {}).get(surface) or {}).items(): + if not isinstance(item_id, str) or not isinstance(fid, str): + continue + if item_id in (valid_ids or {}).get(surface, ()) and fid in fids: + ok[item_id[:120]] = fid + if ok: + out[surface] = ok + return out + + +def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): + """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). + + Returns a clean tree of leaf conditions ({colId, op, value, value2}) and + groups ({conj, children}). Module-agnostic on purpose: any module embedding + the grid validates its own view state through this one function. + + Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised — + the same contract the rest of the view sanitiser follows, so one bad rule can + never cost a user their whole saved view. Depth, per-level width and total + node count are all capped: the tree is re-evaluated for every row on every + render, so an unbounded structure would be a persistent client-side DoS. + Empty groups are dropped (they carry no meaning once persisted). + + `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is + DROPPED here rather than left for the engine — a deleted cohort would otherwise leave a + condition that can only match nothing, so `List is not [deleted]` would show an empty table + forever with no way to tell why. `None` means this host has no cohorts, and then every + cohort leaf is dropped: fail-closed, like every other unknown key. + """ + if budget is None: + budget = [MAX_FILTER_NODES] + out = [] + for node in list(raw or [])[:MAX_FILTER_SIBLINGS]: + if budget[0] <= 0: + break + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): # a condition GROUP + if depth >= MAX_FILTER_DEPTH: + continue # too deep -> drop + budget[0] -= 1 + children = clean_filter_tree(node['children'], valid_keys, + depth + 1, budget, cohort_ids) + if children: + out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', + 'children': children}) + continue + if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf + op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op')) + named = parse_cohort_ids(node.get('value')) + # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT + # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to + # `is none of [A]` would show every row in B under a count nobody would doubt. This + # is the same all-or-nothing the single-cohort leaf already had, extended to a set. + if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): + budget[0] -= 1 + out.append({'colId': COHORT_FIELD, 'op': op, + 'value': ','.join(named), 'value2': ''}) + continue + if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: + budget[0] -= 1 + # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the + # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client + # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently + # stop filtering and show every row instead of the zero-revenue ones. + val, val2 = node.get('value'), node.get('value2') + leaf = {'colId': node['colId'], 'op': node['op'], + 'value': ('' if val is None else str(val))[:500], + 'value2': ('' if val2 is None else str(val2))[:500]} + # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two + # extra members: a stable client-generated `id`, which is how the server's answer + # finds its way back to the condition that asked (positional matching silently + # re-associates every answer the moment a user deletes a condition), and the + # `window`. Emitted ONLY when the input has them — a column condition's cleaned + # shape is unchanged, so every persisted view deserialises byte-identically and + # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by + # exact structural comparison). + rid = node.get('id') + if rid not in (None, ''): + leaf['id'] = str(rid)[:64] + window = _clean_window(node.get('window')) + if window is not None: + leaf['window'] = window + # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted + # only when the input has them, so a plain column condition's cleaned shape is + # byte-identical to what it always was and `clean_filter_tree` stays idempotent + # (verify_filter_engine.py asserts that by exact structural comparison). Drop the + # carry-through and the next autosave silently strips a date condition back to a + # bare comparison against an empty value — i.e. back to INACTIVE. + date_window = _clean_window(node.get('dateWindow')) + if date_window is not None: + leaf['dateWindow'] = date_window + mode = node.get('dateMode') + # SHAPE only. An unrecognised mode survives here and is refused by + # `windows.resolve_anchor`, which returns None and makes the condition match + # NOTHING — the same split as `_clean_window`, and the reason this module can stay + # free of the date vocabulary it would otherwise have to keep in step. + if isinstance(mode, str) and 0 < len(mode) <= 40: + leaf['dateMode'] = mode + rhs = _clean_rhs(node.get('rhs'), valid_keys) + if rhs is not None: + leaf['rhs'] = rhs + out.append(leaf) + return out + + +def _default_view_config(fields): + shown = [field["key"] for field in fields + if field.get("default") is not False or field["source"] == "overlay"] + hidden = [field["key"] for field in fields if field["key"] not in shown] + return { + # `filters` is the ROOT of the filter tree: leaf conditions and/or nested + # condition groups ({conj, children}); `filterConj` joins the root level. + "filters": [], "filterConj": "and", + "sorts": [], "groupBy": None, "colorBy": None, + "rowHeightMode": "short", "order": shown + hidden, "visible": shown, + "widths": {}, "memberPids": [], + } + + +def views_from_defs(defs, saved_views, fields): + """Convert legacy list formulas into the shared serializable SavedView contract.""" + base = _default_view_config(fields) + views = [{ + "id": "all-customers", "name": "All customers", "kind": "system", + "locked": True, "config": dict(base), + }] + op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq", + "contains": "contains"} + for name, definition in (defs or {}).items(): + filters = [] + for rule in definition.get("rules") or []: + if rule.get("field") not in {field["key"] for field in fields}: + continue + filters.append({ + "colId": rule["field"], + "op": op_map.get(rule.get("op"), "eq"), + "value": str(rule.get("value") if rule.get("value") is not None else ""), + }) + sort = str(definition.get("sort") or "") + sorts = ([{"colId": sort.lstrip("-"), + "dir": "desc" if sort.startswith("-") else "asc"}] + if sort.lstrip("-") in {field["key"] for field in fields} else []) + views.append({ + "id": "list:" + str(name), + "name": str(name), + "kind": "list", + "note": str(definition.get("note") or ""), + "config": { + **base, "filters": filters, "sorts": sorts, + "memberPids": [int(pid) for pid in definition.get("members") or [] + if isinstance(pid, int) or str(pid).isdigit()], + }, + }) + index = {view["id"]: i for i, view in enumerate(views)} + for view_id, saved in (saved_views or {}).items(): + if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict): + continue + clean = dict(saved) + clean["id"] = str(view_id) + if view_id in index: + views[index[view_id]] = clean + else: + views.append(clean) + return views + + +def embed_html_path(): + """The first existing candidate path for the inlined single-file build, or None.""" + for p in _EMBED_CANDIDATES: + if p.is_file(): + return p + return None + + +def scope_counts(shown, matched, total): + """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2). + + `matched` and `total` MUST come from their own queries over the whole scope. Never pass + `len(rows)` as `matched` — that is the silent [:N] this exists to prevent: the page would + report the window size as though it were the result size. + + Refuses the shapes that could only be a mistake, because a wrong count here is invisible on + screen (it looks like a smaller dataset, not like an error). + """ + shown, matched, total = int(shown), int(matched), int(total) + if matched > total: + raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " + f"more rows than the scope holds") + if shown > matched: + raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " + f"more rows than the filter matched") + return {"shown": shown, "matched": matched, "total": total, "windowed": True} + + +def build_html(rows, fields, *, workspace=None, path=None, counts=None): + """PURE: read the inlined single-file build and inject the data handshake. + + Returns a self-contained HTML string with + + inserted immediately after the opening tag — a CLASSIC (non-module) script, so it + executes during parse BEFORE the app's deferred `type="module"` script reads the global. + Raises FileNotFoundError if no build is present (callers should try/except and fall back). + """ + p = Path(path) if path else embed_html_path() + if p is None or not Path(p).is_file(): + raise FileNotFoundError( + "aios_grid: no embed build found. Run `npm run build:embed` in aios-web/web and " + "copy dist-embed/index.html -> royalimports_os/aios_grid_embed.html " + "(python build_embed.py). Looked in: " + + " ; ".join(str(c) for c in _EMBED_CANDIDATES)) + html = Path(p).read_text(encoding="utf-8") + # ensure_ascii=False keeps names readable; default=str is a safety net for stray + # date/Decimal; the " from breaking out. + data = {"fields": fields, "rows": rows, "workspace": workspace or {}} + # Only a WINDOWED table carries counts. Omitting the key leaves the client counting the rows + # it holds, which is correct when it holds all of them. + if counts: + data["counts"] = counts + payload = json.dumps(data, ensure_ascii=False, default=str).replace("window.__AIOS_GRID__ = " + payload + ";" + new_html, n = re.subn(r"(]*>)", lambda m: m.group(1) + tag, html, count=1) + if n == 0: # no (shouldn't happen) — still run it first + new_html = tag + html + return new_html + + +def component_dir(): + for path in _COMPONENT_CANDIDATES: + if (path / "index.html").is_file(): + return path + return None + + +def render(rows, fields, *, workspace=None, height=680, key="aios_grid", + user_options=None, measures=None, measure_sets=None, today=None, viewer=None): + """Render the Airtable-style grid and return its latest guarded host event. + + The preferred custom-component path forwards `key` and supports two-way persistence. + If only the legacy HTML build exists, it renders safely with no return event. + + `user_options` are the assignable people for `user`-typed overlay columns. They come from + the HOST (the tenant's real accounts) rather than from anything the grid could invent, so + an assignee is always someone who can actually log in and see the row. + + `measures` / `measure_sets` are CG-8. A measure is FILTERABLE BUT NOT DISPLAYABLE: "Sales in + the last 90 days" is a question about a window, not a column of this table, and rendering it + as a column would silently pick a window for the user. The HOST resolves each condition to a + set of row ids (`{ruleId: [pid, ...]}`) and the client treats that set as a membership test — + which is why this module can offer measure filtering while keeping ZERO app-internal imports: + it never computes a measure, it only carries one. + + `today` is the TENANT'S current date as an ISO string, and every relative date condition + ("is within the past month", "is before one month ago") resolves against it. It comes from + the host rather than from `new Date()` in the browser for the same three reasons + harness/windows.py takes it as a parameter: a viewer a day ahead of the server would resolve + a different month than the server summed, a window must not shift between the count and the + rows, and the tenant's day boundary is the tenant's. Omit it and those conditions match + NOTHING with a visible marker — never everything. + """ + import streamlit.components.v1 as _c + directory = component_dir() + payload = {"fields": fields, "rows": rows, "workspace": workspace or {}, + "userOptions": [str(u)[:120] for u in (user_options or [])][:500], + "measures": list(measures or []), + "today": str(today) if today else None, + # Wave-5 item 1: who is looking, host-computed — what gates the client's + # "Edit field permissions" menu entry. The HOST-side enforcement is the wall; + # this is the courtesy that keeps the menu honest. + "viewer": ({"name": str(viewer.get("name") or "")[:80], + "isAdmin": bool(viewer.get("isAdmin"))} + if isinstance(viewer, dict) else None), + "measureSets": {str(k): list(v) for k, v in (measure_sets or {}).items()}} + if directory is not None: + cache_key = str(directory) + component = _DECLARED_COMPONENTS.get(cache_key) + if component is None: + component = _c.declare_component("aios_customer_grid", path=cache_key) + _DECLARED_COMPONENTS[cache_key] = component + return component(payload=payload, height=height, key=key, default=None) + html = build_html(rows, fields, workspace=workspace) + _c.html(html, height=height, scrolling=False) + return None diff --git a/royalimports_os/core/auth_google.py b/royalimports_os/core/auth_google.py new file mode 100644 index 0000000000000000000000000000000000000000..43f656d44078f18c71eaf50fb179e6a6a189f3b3 --- /dev/null +++ b/royalimports_os/core/auth_google.py @@ -0,0 +1,91 @@ +"""Sign in with Google — TEMPLATE ONLY (wave-7 W10, 2026-07-28). Nothing here runs in +production yet: every flow function raises until the wiring checklist in +docs/GOOGLE_LOGIN_TODO.md is completed and GOOGLE_LOGIN_ENABLED=1 is set. + +DESIGN (recorded now so the full build has a contract to meet): + - OAuth 2.0 authorization-code flow with PKCE against Google Identity + (https://accounts.google.com/.well-known/openid-configuration). No SDK dependency — + three HTTPS calls (authorize redirect, token exchange, JWKS fetch) keep the surface + auditable and the requirements.txt unchanged until we commit. + - Google is an IDENTITY, not an ACCOUNT SOURCE. A Google sign-in maps to an EXISTING + core/users.py account via its `email` attribute (set_access(..., email=...)); an + unknown email FAILS CLOSED with "no account for this Google identity" — Google login + never creates users, so the per-BU/per-module grant model stays the only door. + - The password gate stays byte-for-byte intact beside it. APP_PASSWORD remains the + bootstrap master; removing password auth is a separate, owner-approved step. + +ENV (placeholders — see .env.example additions in docs/GOOGLE_LOGIN_TODO.md): + GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console + GOOGLE_OAUTH_CLIENT_SECRET its secret (Space secret / .env — never committed) + GOOGLE_OAUTH_REDIRECT_URI e.g. https://royal-imports-cfo-os.hf.space/ (must be + registered VERBATIM in the console; localhost:8501 for dev) + GOOGLE_LOGIN_ENABLED '1' arms the flow; anything else keeps this a template +""" +from __future__ import annotations + +import os + +AUTH_ENDPOINT = "https://accounts.google.com/o/oauth2/v2/auth" +TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token" +JWKS_URI = "https://www.googleapis.com/oauth2/v3/certs" +SCOPES = "openid email profile" + + +def configured() -> bool: + """True when the console credentials exist — the LOGIN BUTTON renders iff this is True. + (Separate from enabled(): a configured-but-disabled deploy shows nothing either, so a + half-finished setup never paints a dead button.)""" + return bool(os.environ.get("GOOGLE_OAUTH_CLIENT_ID")) and enabled() + + +def enabled() -> bool: + return os.environ.get("GOOGLE_LOGIN_ENABLED", "") == "1" + + +def auth_url(state: str, code_challenge: str) -> str: + """The Google consent-screen URL for the authorization-code + PKCE flow. + + `state` must be an unguessable per-session token the callback VERIFIES (CSRF); + `code_challenge` is BASE64URL(SHA256(code_verifier)) with the verifier held in the + session. Template guard: raises until the TODO checklist lands. + """ + raise NotImplementedError( + "Google login is a template — complete docs/GOOGLE_LOGIN_TODO.md, then implement " + "auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, " + "code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)." + ) + + +def exchange_code(code: str, code_verifier: str) -> dict: + """POST the authorization code to TOKEN_ENDPOINT → {'id_token', 'access_token', ...}. + Template guard: raises until implemented (requests.post with client_id/secret, + redirect_uri, grant_type=authorization_code, code, code_verifier).""" + raise NotImplementedError( + "Google login is a template — implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md." + ) + + +def verify_id_token(id_token: str) -> dict: + """Validate the JWT against Google's JWKS (signature, iss, aud=client_id, exp) and + return its claims ({'email', 'email_verified', 'name', ...}). MUST reject + email_verified=False. Template guard: raises until implemented.""" + raise NotImplementedError( + "Google login is a template — implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md." + ) + + +def account_for_claims(claims: dict): + """Map verified Google claims → the core/users.py account, FAIL-CLOSED. + + The one function already real: the mapping rule is the design's heart and testable + today. Returns the users.registry() record whose `email` equals the verified claim + (case-insensitive), only if the account is active; None otherwise — never a new user. + """ + email = str(claims.get("email") or "").strip().lower() + if not email or claims.get("email_verified") is not True: + return None + from core import users + for username, u in (users.registry() or {}).items(): + if str(u.get("email") or "").strip().lower() == email and u.get("active", True): + return users._public(username, u) + return None diff --git a/royalimports_os/core/context.py b/royalimports_os/core/context.py new file mode 100644 index 0000000000000000000000000000000000000000..bc5bc93c11e360b10755f192414d113682339f48 --- /dev/null +++ b/royalimports_os/core/context.py @@ -0,0 +1,59 @@ +"""Request context — the single object that carries the GLOBAL filters through a page render. + +Instead of threading date_from / date_to / team_id / ... through every function, a page builds +one `Ctx` from the sidebar controls and passes it (or its primitives) down. Modules read only +what they need. Add a new global filter once here and every module can opt into it. + +The DBA (doing-business-as) brand filter lives here as `team_id`: + None -> All / consolidated (Fisch + Royal) + 5 -> Fisch + 6 -> Royal +HQ / company-level modules (cash flow, balance sheet, AR/AP, consolidated P&L) ignore brand and +call `ctx.consolidated()`; operational modules (sales, customers, products, margins) honour it. +""" +from dataclasses import dataclass, replace +import core.odoo as O +import core.periods as P + +# label -> team_id used by the sidebar Brand (DBA) selector +BRANDS = {'All': None, 'Fisch': 5, 'Royal': 6} + + +@dataclass(frozen=True) +class Ctx: + date_from: str + date_to: str + period_label: str = 'Year to date' + team_id: int | None = None # None = All DBAs (consolidated); 5 = Fisch; 6 = Royal + doc_mode: str = 'order' # reserved: 'order' vs 'invoice' recognition basis + + @property + def brand(self) -> str: + return O.TEAM_NAMES.get(self.team_id, 'All') if self.team_id else 'All' + + @property + def teams(self) -> list: + """Team ids in scope (single brand, or both when consolidated).""" + return [self.team_id] if self.team_id else list(O.TEAM_IDS) + + @property + def window(self) -> tuple: + return self.date_from, self.date_to + + def consolidated(self) -> 'Ctx': + """All-DBA copy — HQ modules use this so the brand filter never scopes company numbers.""" + return replace(self, team_id=None) + + def with_period(self, date_from, date_to, label=None) -> 'Ctx': + return replace(self, date_from=date_from, date_to=date_to, + period_label=label or self.period_label) + + +def from_ui(period_label, period_window, brand_label='All', doc_mode='order') -> Ctx: + df, dt = period_window + return Ctx(df, dt, period_label, BRANDS.get(brand_label), doc_mode) + + +def default(t=None) -> Ctx: + df, dt = P.ytd(t) + return Ctx(df, dt, 'Year to date', None) diff --git a/royalimports_os/core/grid_events.py b/royalimports_os/core/grid_events.py new file mode 100644 index 0000000000000000000000000000000000000000..b0663a73d003809fd4277a53fd3402b427931044 --- /dev/null +++ b/royalimports_os/core/grid_events.py @@ -0,0 +1,1070 @@ +"""core/grid_events.py — THE WRITE SEAM, lifted out of Streamlit (EXIT-1a, X1, 2026-07-30). + +This is the grid's event handler: the code that takes an UNTRUSTED event object from the +browser component and decides what may be persisted. It was `app.py:_cl_handle_grid_event` / +`_cl_handle_one` and moved here as a PURE MOVE — no behavior change inside a move (MA principle +5). Every body below is the app.py body verbatim; the ONLY mechanical rewrites are the +Streamlit couplings the contract names: + + st.session_state['_cl_grid_seen_ids'] -> ctx.seen_ids (adapter-owned dedup state) + st.session_state['_cl_doc_payload'] -> ctx.out.doc (EventResult.doc) + st.session_state['_cohort_toast'] -> ctx.out.toast (EventResult.toast) + st.session_state['cl_table_workspace'] -> ctx.fallback_ws (store-unavailable fallback) + is_admin() (the bare call inside doc_delete, and _cl_table_workspace's admin=None default) + -> ctx.admin + +There is NO `import streamlit` in this file and there must never be one — `verify_seam.py` +enforces that with a `sys.meta_path` block, which is the only proof that survives refactoring. + +TWO ADAPTERS, ONE HANDLER: + * the Streamlit adapter — app.py keeps `_cl_handle_grid_event(event, uname, allowed_pids, …)` + with its old name and signature, builds an EventCtx whose `seen_ids` and `fallback_ws` are + session-state-backed dicts, and applies the Result to session state. + * the HTTP adapter — `aios-web/api` builds an EventCtx with `fallback_ws=None`. A None + fallback with the store unavailable is a **503**, never a silent in-memory workspace: an + API request has no durable session dict to fall back to, so pretending to persist would + lose the write with a 200 on it. + +⚠ LAYERING, stated honestly. `core/` is not supposed to import `modules/` (ARCHITECTURE §1's +one-directional rule). This handler genuinely straddles: it is the WRITE half of the customer +table and its persistence verbs live in `modules.customer_data` / `modules.cohort`. The +contracted path for the strangler is `core/grid_events.py` (both adapters import it from here), +so the import stays and the violation is RECORDED rather than hidden. It is the one file in +`core/` that points up; when EXIT-5 ports the surfaces, the persistence verbs are what move +down, and this note is the marker for that work. +""" +import datetime as dt +from dataclasses import dataclass, field +from typing import MutableMapping, Optional + +import core.store as store +import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates +import core.users as users +import harness.telemetry as _tel # _tel.error() = THE error sink (all caught errors log) +import modules.cohort as cohort_mod +import modules.customer_data as cl_mod + +#: C5 (owner item 12): customer documents. The cap is enforced HOST-side on the DECODED byte +#: count — the client checks the same number first, but a client check only spares the user a +#: pointless upload; this is what bounds what the tenant's dataset can be made to hold. +#: MOVED from app.py:1410–1412 with the handler (app.py re-imports these during the strangler). +_DOCS_KEY = 'customer_docs' +_DOC_MAX_BYTES = 5 * 1024 * 1024 +_DOC_MAX_PER_ROW = 50 + +#: The props of a field def the CLIENT actually draws — the wave-6 no-blip comparison set. +#: Host-only stamps (createdBy, permissions) are deliberately absent: the client renders +#: nothing from its own copy of those, and a permissions refusal is answered explicitly. +_FIELD_ECHO_PROPS = ('label', 'type', 'note', 'options', 'max', 'formula', 'agg', + 'scope', 'format', 'measure') + + +class StoreUnavailable(RuntimeError): + """The store is down AND the caller offered no durable fallback (HTTP adapter → 503). + + The Streamlit adapter never sees this: it passes a session-state-backed `fallback_ws`, so + its store-down path degrades to the same captioned in-memory workspace it always had. An + API caller has no such dict, and the honest answer to "persist this" with nowhere to + persist it is an error — not a 200 over a write that evaporates. + """ + + +@dataclass +class EventResult: + """What the HANDLER needs to tell its adapter — the three things it cannot do itself. + + `rerender` is the old return value with its old meaning: "this page must re-render from + server state", NOT "handled" (see `handle_one`'s docstring — the distinction is the + difference between a snappy grid and a sluggish one). + """ + rerender: bool = False + doc: Optional[dict] = None # doc_fetch's answer — the bridge has no response channel + toast: Optional[str] = None # add_to_list's confirmation + + +@dataclass +class EventCtx: + """Everything the handler is allowed to know, and nothing about how it is being served. + + ⚠ `allowed_pids` is a SECURITY parameter: it is the pool this caller may touch at all, and + every row-scoped event (overlay_patch, doc_*, cohort membership, add_to_list, memberPids) + is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity — + never anything the browser said about itself. + + `seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every + emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP + passes the caller's per-session mapping. + + `fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username + (exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes + None, and None + store down raises StoreUnavailable (→ 503). + + `out` is the sink for the two parked answers, so `handle_one` can keep returning a bare + bool exactly as it did in app.py while doc/toast still escape. (Additive amendment to X1's + field list, 2026-07-30 — defaulted; recorded in the S1 mailbox.) + """ + uname: str + allowed_pids: frozenset + fields: list + measure_keys: frozenset = frozenset() + resolved_ids: frozenset = frozenset() + cohort_ids: frozenset = frozenset() + measure_offer: tuple = () + admin: bool = False + scope_key: str = 'customer' + seen_ids: Optional[MutableMapping] = None + fallback_ws: Optional[MutableMapping] = None + out: EventResult = field(default_factory=EventResult) + + def __post_init__(self): + # A caller that forgets `seen_ids` gets per-call dedup rather than a crash — the SET is + # what makes re-delivery idempotent, and its absence must not be the thing that decides + # whether an event is processed at all. + if self.seen_ids is None: + self.seen_ids = {} + + +# ------------------------------------------------------------------ the workspace family (X1 amd.) +def table_workspace(ctx, allowed_pids=None): + """Durable Airtable-style view/schema/overlay state, with an honest session fallback. + + Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED + with them. Own views win a same-id collision — a personal view is the user's own object and + must never be shadowed by somebody else's share. + + ⚠ `allowed_pids` is a SECURITY parameter, not a convenience. A shared view carries + `memberPids`, which the writer's scope validated on the way IN — but the READER may have a + narrower scope. Handing a Fisch-scoped user a view whose member list was built by a + full-access user would leak the other BU's customers straight through the strict-isolation + rule. So membership is re-filtered against the READER on the way OUT. Pass it wherever the + result reaches a renderer or an event handler; omitting it drops memberPids entirely rather + than trusting the stored list (fail-closed: a smaller view, never a wider one). + + MOVED from app.py:5323 (`_cl_table_workspace`). `admin` was a parameter defaulting to a live + `is_admin()` call; it now always comes from `ctx.admin`, which is identical at the one call + site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids` + stays an EXPLICIT argument because the folder leg deliberately omits it. + """ + uname = ctx.uname + if store.available(): + ws = cl_mod.table_workspace(uname) + try: + shared = cl_mod.shared_table_views(uname, bool(ctx.admin)) + except Exception as _se: + _tel.error('sharedviews:read', _se) + shared = {} + if shared: + merged = dict(shared) + merged.update(ws.get('views') or {}) # own views win the id collision + ws['views'] = {vid: scope_view(v, uname, allowed_pids) + for vid, v in merged.items()} + return ws + # Store down. Streamlit degrades to its captioned session workspace exactly as before; an + # API caller has no durable dict and must be told, not quietly served memory. + if ctx.fallback_ws is None: + raise StoreUnavailable('table workspace unavailable: no store and no session fallback') + return ctx.fallback_ws.setdefault( + uname, {'views': {}, 'fields': {}, 'overlays': {}}) + + +def scope_view(view, uname, allowed_pids): + """Re-scope a view to the READER. A no-op for a view the reader created. + + Two things travel inside a shared view that are SCOPE, not presentation: + 1. `memberPids` — a customer list. Re-filtered against the reader's permitted pool. + 2. cohort conditions — a cohort id is a permission handle owned by ONE user. For a reader + who does not own it the leaf is UNANSWERABLE, and partial evaluation of an unanswerable + leaf WIDENS `is none of` ([[cg-cohort-set-conditions]]) — i.e. it would show MORE rows, + not fewer. So the whole filter tree is dropped for a foreign reader rather than + evaluated with a hole in it. A visibly unfiltered shared view is recoverable; a + silently widened one is a leak. + """ + if not isinstance(view, dict) or view.get('createdBy') == uname: + return view + out = dict(view) + cfg = dict(out.get('config') or {}) + mp = cfg.get('memberPids') + if isinstance(mp, list): + cfg['memberPids'] = ([p for p in mp if p in (allowed_pids or ())] + if allowed_pids is not None else []) + if tree_has_cohort(cfg.get('filters')): + cfg['filters'] = [] + cfg['_scopedOut'] = 'cohort' # the client says so rather than pretending + out['config'] = cfg + return out + + +def tree_has_cohort(node): + """True when a filter tree references the Cohorts column anywhere (any depth).""" + import aios_grid as _agc + col = _agc.COHORT_COLUMN + if isinstance(node, list): + return any(tree_has_cohort(n) for n in node) + if not isinstance(node, dict): + return False + if node.get('colId') == col or node.get('field') == col: + return True + return tree_has_cohort(node.get('conditions') or node.get('children')) + + +def measure_leaves(nodes): + """Every leaf of a filter tree that carries a date WINDOW — i.e. every measure condition. + + Keyed off the window rather than the column name on purpose: a window is the one thing no + column condition has, so this cannot mistake `revenue_ytd > 5000` for a measure. + """ + out = [] + for node in nodes or []: + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + out.extend(measure_leaves(node['children'])) + elif node.get('window') is not None: + out.append(node) + return out + + +# ------------------------------------------------------------------ private handler helpers +def _field_echo_equal(accepted, raw): + """Does the ACCEPTED def match what the client already renders? (wave-6 item 3) + + The no-blip law: a field_upsert whose acceptance changed NOTHING the client draws needs no + re-render — the client is the ORIGIN of the change, its optimistic copy is already right, + and the rerun it would buy costs a full payload re-serialisation plus an iframe remount + (the "blip" the owner named). STRICT per-prop equality after collapsing the empty forms + ('' / [] / {} / absent all render identically): any normalisation a validator applied is a + real difference the client must be SHOWN. Fail-closed — unsure means not equal means rerun. + """ + empty = ('', [], {}) + for k in _FIELD_ECHO_PROPS: + a, b = accepted.get(k), raw.get(k) if isinstance(raw, dict) else None + if k == 'scope': + # 'global' and absent are the SAME rendering (an unscoped field) — the event may + # say 'global' explicitly while the stored def says it by omission. + a = None if a == 'global' else a + b = None if b == 'global' else b + if (None if a in empty else a) != (None if b in empty else b): + return False + return True + + +def _known_usernames(): + """Real account names, for C4's fail-closed `users` grant. None = "could not resolve". + + None means DO NOT FILTER, deliberately. Returning an empty set on a transient registry + error would silently collapse a user's stored 'specific users' grant to 'personal' on the + very next autosave — destroying a setting because a read blipped. The grant is inert today + anyway (views are per-user; nothing reads it to allow cross-user access), so the honest + trade is: never invent access, never destroy a stored intent either. + """ + try: + reg = set(users.registry() or {}) + except Exception as _ue: + _tel.error('viewperms:users', _ue) + return None + # ⚠ EMPTY IS "UNRESOLVED", NOT "NOBODY EXISTS". registry() returns {} rather than raising + # when the user store cannot be read (a store blip logs `store:get:users` and hands back an + # empty dict) — and a working app ALWAYS has at least the bootstrap admin. Treating {} as a + # real answer would filter every name out of a stored grant and collapse it to 'personal' + # on the next autosave. Caught by testing this in BARE mode with no store token. + return reg or None + + +# ------------------------------------------------------------------ THE SEAM +def handle_events(events, ctx): + """The event LOG entry point → one EventResult. The `[-24:]` window lives here. + + ⚠ THE VALUE IS AN EVENT LOG `{'events': [...]}`, NOT ONE EVENT (2026-07-27). A component + has exactly one value slot, and two quick emits share it: creating a field emits + `field_upsert` and the insertColumn autosave emits `view_upsert` 420ms later — with the + server still inside the first rerun, the second write clobbered the slot before any run + read it, and the field creation VANISHED with no error anywhere (measured live; the same + race sat latent under overlay-field creation, masked by localStorage). The client now + sends its recent window of events; each is processed exactly once, keyed by its id. The + single-event shape is still accepted — it is what an older client in a stale tab sends. + + Accepts everything the component value can be: the log dict, a bare list of events (what + the HTTP body's `events` array unwraps to), or one legacy single-event object. + + `EventResult.rerender` is True when ANY processed event demands a re-render from server + state — the same OR-fold `_cl_handle_grid_event` did. + """ + if isinstance(events, dict) and isinstance(events.get('events'), list): + seq = events['events'][-24:] + elif isinstance(events, list): + seq = events[-24:] + else: + seq = [events] + rerun = False + for one in seq: + rerun = handle_one(one, ctx) or rerun + ctx.out.rerender = rerun + return ctx.out + + +def handle_one(event, ctx): + """Validate ONE untrusted component event. + + Odoo-source fields are never accepted as cell updates. View and schema objects are + user-scoped; row patches additionally require the pid to be in the currently permitted + Customer List pool. + + RETURN VALUE = "this page must re-render from server state", NOT "handled". + The distinction is the difference between a snappy grid and a sluggish one. + Persisting is not by itself a reason to rerun: Streamlit has ALREADY re-run the + script once simply to deliver the component value, and the client is the ORIGIN + of the change, so its own state is correct before the server hears about it. An + extra st.rerun() therefore buys nothing and costs a second full page run. + Measured on the 1,550-row book (2026-07-25): one filter tweak produced TWO + page_customer_data runs ~2.8s apart, each re-serialising the whole ~576 KB row + payload — and the handler on the second run is a guaranteed no-op, because the + event-id dedupe two lines below has already consumed that id. + + THE WAVE-6 NO-BLIP LAW (item 3, 2026-07-27) — True ONLY when the next render + genuinely differs from what the client already draws: + field_upsert — False when the accepted def ECHOES the client's copy + (_field_echo_equal); True on any validator mutation, any + permissions refusal, and ALWAYS when a measure window changed + (values must be re-resolved server-side). + overlay_patch — False when every requested update was accepted verbatim; + True the moment anything was refused or truncated. + field_delete — False: the client removed the column optimistically and the + store scrub is invisible to the next render. + field_duplicate — True only when host-held VALUES must flow back (custom_ cell + copies, measure_ recomputes); formula/created_time clones + compute client-side. + view_delete — False (the deleted view simply stops being served). + view_upsert — unchanged: False, except an unresolved measure condition. + cohort_* / add_to_list — True: the lists payload is server truth, and these are + not the hot path. + + The id dedupe is a SET (not a last-id slot): the event log resends the whole recent + window on every emit, so every id must stay consumed, not just the latest one. + """ + # The ctx fields the moved body reads as plain locals — bound once here so every branch + # below is textually the app.py body (which is what makes this move reviewable). + uname = ctx.uname + allowed_pids = ctx.allowed_pids + fields = ctx.fields + measure_keys = ctx.measure_keys + resolved_ids = ctx.resolved_ids + cohort_ids = ctx.cohort_ids + measure_offer = ctx.measure_offer + admin = ctx.admin + scope_key = ctx.scope_key + + if not isinstance(event, dict): + return False + event_id = str(event.get('id') or '')[:180] + seen = ctx.seen_ids + if not event_id or event_id in seen: + return False + seen[event_id] = True + if len(seen) > 400: # a session-lifetime set, kept bounded + for stale in list(seen)[:200]: + seen.pop(stale, None) + kind = event.get('type') + field_by_key = {f['key']: f for f in fields} + valid_keys = set(field_by_key) + overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'} + ws = table_workspace(ctx, allowed_pids=allowed_pids) + + def _session_ready(): + ws.setdefault('views', {}) + ws.setdefault('fields', {}) + ws.setdefault('overlays', {}) + + if kind == 'view_upsert': + raw = event.get('view') + if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict): + return False + view_id = str(raw.get('id') or '').strip()[:120] + name = str(raw.get('name') or '').strip()[:120] + if not view_id or not name: + return False + cfg = raw['config'] + # The filter TREE (leaf conditions and/or nested condition groups) is + # validated by the module-agnostic grid contract helper — imported lazily + # here for the same reason the page does it: a missing embed build must + # never break page import. + import aios_grid as _ag + # A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered, + # shown, grouped or coloured by, because it is not a column of this table. Hence the + # widened key set here and the plain `valid_keys` everywhere below. + # `cohort_ids` is a PERMISSION list, not a convenience: a cohort rule naming anything + # else is dropped here. Leaving it in would persist a condition the engine can only + # answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table + # forever with nothing on screen explaining why. + filters = _ag.clean_filter_tree(cfg.get('filters'), + valid_keys | set(measure_keys), + cohort_ids=set(cohort_ids)) + sorts = [] + for rule in list(cfg.get('sorts') or [])[:20]: + if (isinstance(rule, dict) and rule.get('colId') in valid_keys + and rule.get('dir') in ('asc', 'desc')): + sorts.append({'colId': rule['colId'], 'dir': rule['dir']}) + order = list(dict.fromkeys( + key for key in list(cfg.get('order') or []) if key in valid_keys)) + order += [field['key'] for field in fields if field['key'] not in order] + visible = list(dict.fromkeys( + key for key in list(cfg.get('visible') or []) if key in valid_keys)) + widths = {} + raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {} + for key, value in raw_widths.items(): + if key in valid_keys: + try: + widths[key] = max(60, min(600, round(float(value)))) + except (TypeError, ValueError): + pass + config = { + 'filters': filters, + # conjunction joining the ROOT-level conditions; anything but an + # explicit 'or' means 'and' (legacy views had an implicit AND). + 'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and', + 'sorts': sorts, + 'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None, + 'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None, + 'rowHeightMode': (cfg.get('rowHeightMode') + if cfg.get('rowHeightMode') in ('short', 'medium', 'tall') + else 'short'), + 'order': order, + 'visible': visible, + 'widths': widths, + 'memberPids': [pid for pid in list(cfg.get('memberPids') or []) + if isinstance(pid, int) and pid in allowed_pids], + } + # Wave-6 item 10: a non-grid display mode rides the view config, validated + # structurally (mode whitelist, refs must be this table's fields). + display = _ag._clean_display(cfg.get('display'), valid_keys) + if display: + config['display'] = display + # Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling + # sideways. Clamped 1..8 — 0 would unfreeze the identity column the whole + # surface pivots on; absent = the legacy single frozen column. + if cfg.get('frozenCount') is not None: + try: + config['frozenCount'] = max(1, min(8, int(cfg['frozenCount']))) + except (TypeError, ValueError): + pass + # ── wave-9 C3 (lock) + C4 (permissions) ──────────────────────────────────────────── + # The PRIOR stored view decides both: whether this is a create or an update (C4's + # default splits on exactly that) and who is allowed to change the lock (C3). + prior = (ws.get('views') or {}).get(view_id) or {} + # ⚠ THE SHARED WALL (wave-9 I17). `ws` is already visibility-filtered, so a view the + # caller may not SEE reads as absent — which would look like a CREATE and let anybody + # overwrite somebody else's shared view by guessing its id. Ask the store for the raw + # record and refuse before deciding anything else. Fail-closed on both branches. + _raw_shared = cl_mod.shared_table_view(view_id) if store.available() else None + if _raw_shared is not None and not prior: + return False # exists, not visible to this caller: refuse + if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin): + return False # visible but not editable by this caller + is_new = not prior + # C4: the creator is HOST-STAMPED and never read from the browser — the same rule as + # the field-level createdBy. An existing view keeps whoever made it. + creator = prior.get('createdBy') or uname + # Only the creator or an admin may change the PERMISSIONS themselves. A collaborator who + # could rewrite them could grant themselves sole ownership of someone else's view, or + # quietly widen a users-scoped view to everyone — privilege escalation by edit. + if not is_new and not _tstore._may_administer(prior, uname, admin): + raw = dict(raw) + raw['permissions'] = prior.get('permissions') + raw['locked'] = prior.get('locked') + # ⚠ A SYSTEM VIEW IS PER-USER STATE AND MUST NEVER BE SHARED. Caught in the wave-9 + # integration pass, before deploy: `all-customers` is every user's DEFAULT view and it + # carries their own column widths, visible fields and row height. C4's read-default + # ('collaborative' when permissions are absent, so nobody's saved view is + # retro-restricted) is right for a view somebody AUTHORED — but applied to the system + # view it routed `all-customers` into the shared bucket, and the first user to resize a + # column would have published their layout to everyone. Verified in the store: + # shared_table_views() was returning ['all-customers']. + # System views are pinned personal, so they stay in each user's own workspace exactly as + # before this wave. `kind` is host-validated two lines below; read it here from `raw` and + # treat the stored kind as authoritative for an existing view. + _kind = (prior.get('kind') if not is_new + else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom') + else 'custom')) + if _kind == 'system' or view_id == 'all-customers': + perms = {'edit': 'personal'} + else: + perms = _ag.clean_view_permissions( + raw.get('permissions'), + default='personal' if is_new else 'collaborative', + known_users=_known_usernames()) + # C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever + # is already stored. Enforced here rather than hidden in the client. This composes with + # C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze + # the display MODE. + # ⚠ Inert in practice TODAY — a user can only ever write their own per-user workspace, + # so `uname == creator` always holds. Correct plumbing for shared views, not a live gate. + may_lock = admin or uname == creator + locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked')) + # C3: the LOCK's actual job — a locked view's display mode is frozen, so a locked Kanban + # stays Kanban. This half IS live per-user and is the part the owner asked for. Filters, + # sorts and columns stay editable by design. + if locked and not is_new: + was = (prior.get('config') or {}).get('display') + now = config.get('display') + if (was or {}).get('mode') != (now or {}).get('mode'): + if was: + config['display'] = was + else: + config.pop('display', None) + view = {'id': view_id, 'name': name, + 'kind': raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom') + else 'custom', + 'locked': locked, + 'createdBy': creator, + 'permissions': perms, + 'note': str(raw.get('note') or '')[:2000], + 'config': config} + if store.available(): + cl_mod.save_table_view(uname, view) + else: + _session_ready() + ws['views'][view_id] = view + # PERSISTED, but deliberately NO rerun — see the return-value contract in + # this function's docstring. This is the hot path: every filter, sort, + # group, colour, column-resize and column-toggle autosaves through here. + # + # CG-8 is the ONE exception, and it is self-limiting. A measure condition is answered + # server-side, so if this view carries one whose answer was NOT in the payload we just + # shipped, the client is showing a pending marker with nothing coming. Ask for exactly + # one more run — and only then, so the common case (a column filter) still costs zero. + # If the top-of-run read of the component's own value already covered it, this is False + # and nothing extra happens; if that read is ever stale, this is what stops the + # condition hanging unresolved. + # + # ⚠ COMPLETE rules only (wave-6 item 3, found by the live roundtrip on this rev): + # picking a measure FIELD mints a valueless rule whose autosave used to land here as + # "unresolved" and trigger a rerun that could resolve NOTHING — `rule_complete` refuses + # half-asked questions — so the only effect was a remount that closed the builder + # popover mid-build (the wave-5 "gate flake" runs, now explained). A rule still being + # typed stays client-side PENDING by design; the run that matters fires when its VALUE + # arrives and the question becomes answerable. + from harness import measure_filter as _mfc + unresolved = {str(r.get('id') or '') for r in measure_leaves(filters) + if _mfc.rule_complete(r)} - set(resolved_ids) + return bool(unresolved - {''}) + + if kind == 'view_delete': + view_id = str(event.get('viewId') or '')[:120] + if not view_id or view_id == 'all-customers': + return False + # Wave-9 I17: deleting a SHARED view is narrower than editing it — creator or admin + # only. A collaborator can change what a shared view shows; they cannot destroy + # everybody else's copy of it, because there is only one copy. + if store.available(): + _raw_shared = cl_mod.shared_table_view(view_id) + if _raw_shared is not None and not _tstore._may_administer( + _raw_shared, uname, admin): + return False + cl_mod.delete_table_view(uname, view_id) + else: + _session_ready() + ws['views'].pop(view_id, None) + # Wave-6 item 3: the client removed the view optimistically; a deleted view simply + # stops being served, so the next render differs in nothing the client still shows. + return False + + if kind == 'field_upsert': + raw = event.get('field') + if not isinstance(raw, dict): + return False + key = str(raw.get('key') or '')[:80] + import aios_grid as _ag2 + if key.startswith(_ag2.MEASURE_FIELD_PREFIX): + # A FORMULA-MEASURE column (owner item 7). Validated against THIS caller's measure + # offer — the same admission the condition builder uses — so a field can only name a + # measure this user could also filter by. Fail closed on anything else. + field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer}) + if field is None: + return False + fmt = _ag2._clean_format(raw.get('format'), field.get('type')) + if fmt: + field['format'] = fmt + elif key in field_by_key and not key.startswith('custom_'): + base = field_by_key[key] + field = {**base, 'note': str(raw.get('note') or '')[:2000]} + # Wave-5 item 10: a DISPLAY format on any base field (presets included) — how a + # number/date reads, per user. Rendering-only; validated fail-closed per type. + fmt = _ag2._clean_format(raw.get('format'), base.get('type')) + if fmt: + field['format'] = fmt + else: + field.pop('format', None) # cleared -> back to the type's default render + # PRESET measure fields (wave-2 item 8): the browser may change the PERIOD (and the + # auto-renamed label) of revenue_ytd / revenue_ly / orders_24m — nothing else. The + # whitelist IS the base contract: only a field that shipped with preset+measure can + # take a window here, and the measure KEY is pinned to the base's (fail closed). + if (base.get('preset') and isinstance(base.get('measure'), dict) + and isinstance(raw.get('measure'), dict)): + window = _ag2._clean_window(raw['measure'].get('window')) + if window is not None: + field['measure'] = {'key': base['measure']['key'], 'window': window} + if isinstance(raw.get('label'), str) and raw['label'].strip(): + field['label'] = str(raw['label'])[:120] + elif key.startswith('custom_') and raw.get('type') in _ag2.CUSTOM_FIELD_TYPES: + ftype = raw['type'] + readonly = ftype in _ag2.READONLY_CUSTOM_TYPES + # The editable family must SAY it is overlay (the client's own contract); the + # read-only pair (formula / created_time) is stored under the cohort column's + # mechanism instead, so no cell write can ever be accepted for it. + if not readonly and raw.get('source') != 'overlay': + return False + field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120], + 'type': ftype, 'source': 'odoo' if readonly else 'overlay', + 'default': True, 'custom': True, + 'note': str(raw.get('note') or '')[:2000]} + if readonly: + field['derived'] = True + # FILTERABLE since wave 6 (owner item 6): formula/created_time values live + # client-side, and this table's counts are client-mode — the client engine + # answers conditions on them soundly. Mirrors fields_from_workspace. + field['filterable'] = True + if raw.get('agg') == 'sum' and not readonly: + field['agg'] = 'sum' + # A `select`/`multiselect` carries its own vocabulary; a `user` does NOT — its + # choices are the tenant's accounts, resolved fresh on every render, so persisting + # a snapshot here would go stale the moment somebody joins or leaves. + if ftype in ('select', 'multiselect'): + field['options'] = _ag2._clean_options(raw.get('options')) + if not field['options']: + return False # a select with no choices can never hold a value + if ftype == 'rating': + field['max'] = _ag2._clean_rating_max(raw.get('max')) + if ftype == 'formula': + # WRITE-time is where refs must exist (fail closed); read-time leaves them + # alone so a later-deleted ref blanks the CELLS, not the column. + formula = _ag2._clean_formula(raw.get('formula'), valid_keys) + if formula is None: + return False + field['formula'] = formula + fmt = _ag2._clean_format(raw.get('format'), ftype) + if fmt: + field['format'] = fmt + else: + return False + # Wave-5 item 1 — createdBy + permissions on the CREATED strata (custom_/measure_). + # `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the + # client never supplies it, so a browser cannot claim someone else's field. A + # permissions CHANGE is accepted only from the creator or an admin; a legacy field + # with no createdBy is admin-only (fail closed, both directions). The table workspace + # is per-user today, so this wall becomes load-bearing when sharing arrives — but the + # admin path and the legacy path are enforceable (and NC'd) right now. + needs_values = False + refused_perms = False + if key.startswith('custom_') or key.startswith(_ag2.MEASURE_FIELD_PREFIX): + prior = ws.get('fields', {}).get(key) + prior = prior if isinstance(prior, dict) else None + if prior is None: + field['createdBy'] = uname + # Wave-6 item 9: scope is chosen at CREATE, on the cohort page only — + # 'cohort' marks the field cohort-specific; 'global' or absent stays + # unscoped, so an old bundle that sends no scope keeps today's behavior. + if scope_key == 'cohort' and raw.get('scope') == 'cohort': + field['scope'] = 'cohort' + else: + if isinstance(prior.get('createdBy'), str) and prior['createdBy'].strip(): + field['createdBy'] = prior['createdBy'] + # Scope is PRESERVED like createdBy: the client copy never moves an + # existing field between scopes. + if prior.get('scope') == 'cohort': + field['scope'] = 'cohort' + want = _ag2._clean_permissions(raw.get('permissions')) + have = _ag2._clean_permissions((prior or {}).get('permissions')) + owner = field.get('createdBy') + if want is not None and (admin or (owner and owner == uname)): + field['permissions'] = want + else: + if have is not None: + field['permissions'] = have + # Asked for a permissions change and did not get it: the client's + # optimistic copy is a lie that must be repainted (wave-6 item 3). + refused_perms = want is not None and want != have + # A NEW measure column, or an existing one whose window moved: the VALUES + # are host-computed, so the next render genuinely differs. + if key.startswith(_ag2.MEASURE_FIELD_PREFIX): + needs_values = prior is None or prior.get('measure') != field.get('measure') + if store.available(): + cl_mod.save_table_field(uname, field) + else: + _session_ready() + if not field.get('custom') and not str(field.get('note') or '').strip(): + ws['fields'].pop(key, None) + else: + ws['fields'][key] = field + if needs_values or refused_perms: + return True + # THE NO-BLIP RETURN (wave-6 item 3): an accepted def that echoes what the client + # already renders needs no rerun — see _field_echo_equal and the docstring's law. + return not _field_echo_equal(field, raw) + + if kind in ('cohort_add', 'cohort_remove', 'cohort_rename', 'cohort_delete'): + # Cohort membership/lifecycle edits from the grid's cohort panel (wave-2 item 2 — + # replaces the retired Manage popover, so rename/delete keep a surface). Fail-closed: + # only the caller's OWN cohorts (all_for), and member pids are intersected with the + # caller's permitted pool — the same guard add_to_list rides. + if not store.available(): + return False + cohort_id = str(event.get('cohortId') or '').strip()[:120] + if not cohort_id or cohort_id not in cohort_mod.all_for(uname): + return False + try: + if kind == 'cohort_rename': + name = str(event.get('name') or '').strip()[:120] + if not name: + return False + cohort_mod.rename(uname, cohort_id, name) + elif kind == 'cohort_delete': + cohort_mod.delete(uname, cohort_id) + else: + raw_pids = event.get('pids') + if not isinstance(raw_pids, list): + return False + pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids] + if not pids: + return False + if kind == 'cohort_add': + cohort_mod.add_members(uname, cohort_id, pids) + else: + cohort_mod.remove_members(uname, cohort_id, pids) + except ValueError as e: + _tel.error('cohort:panel-edit', e) + return False + # True: the lists payload (names, membership, counts) must reflect this on the next + # render — the panel is showing server truth, not local optimism. + return True + + if kind in ('doc_add', 'doc_delete', 'doc_fetch'): + # DOCUMENTS on a customer row (owner item 12, contract C5). Metadata in the store key + # `customer_docs`; the BYTES live at their own path in the tenant dataset, never in a + # JSON blob — see core.store.upload_bytes for why that distinction is load-bearing. + import base64 as _b64c + import re as _re_mod + if not store.available(): + return False + pid = event.get('pid') + doc_id = str(event.get('docId') or '').strip()[:80] + # The pid wall is the SAME one every other row-scoped event rides: a caller may only + # touch customers inside their own permitted pool. Without it, documents would be the + # one surface where a scoped agent could read another book's rows. + if not isinstance(pid, int) or pid not in allowed_pids or not doc_id: + return False + meta_all = store.get(_DOCS_KEY) or {} + row = list((meta_all.get(str(pid)) or [])) + + if kind == 'doc_add': + raw_b64 = event.get('data_b64') + name = str(event.get('name') or 'document')[:200] + if not isinstance(raw_b64, str) or not raw_b64: + return False + try: + blob = _b64c.b64decode(raw_b64, validate=True) + except Exception: + return False # not base64 -> not a file we will store + # Re-check the DECODED size host-side. The client checks it too, but a client check + # is a courtesy: this is the number that bounds what the tenant's repo can be made + # to hold by anyone who can reach the endpoint. + if not blob or len(blob) > _DOC_MAX_BYTES: + return False + if len(row) >= _DOC_MAX_PER_ROW or any(d.get('id') == doc_id for d in row): + return False + safe = _re_mod.sub(r'[^A-Za-z0-9._-]', '_', name)[:80] or 'file' + path = f'docs/{pid}/{doc_id}_{safe}' + try: + store.upload_bytes(path, blob, message=f'doc {doc_id} for customer {pid}') + except Exception as _ue: + _tel.error('docs:upload', _ue) + return False + row.append({'id': doc_id, 'name': name, + 'mime': str(event.get('mime') or 'application/octet-stream')[:120], + 'size': len(blob), 'by': uname, + 'ts': dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), + 'path': path}) + elif kind == 'doc_delete': + hit = next((d for d in row if d.get('id') == doc_id), None) + if not hit: + return False + # THE WALL: the uploader or an admin. Enforced here regardless of what the client + # chose to render — `canDelete` in the payload only hides a button. + if hit.get('by') != uname and not admin: + return False + store.delete_path(hit.get('path') or '') + row = [d for d in row if d.get('id') != doc_id] + else: # doc_fetch + hit = next((d for d in row if d.get('id') == doc_id), None) + if not hit: + return False + blob = store.download_bytes(hit.get('path') or '') + if blob is None: + return False + # The bridge is a fire-and-forget event LOG with no response channel, so the answer + # is PARKED on the Result and rides the next render's payload (C5). + ctx.out.doc = { + 'pid': pid, 'docId': doc_id, 'name': hit.get('name') or 'document', + 'mime': hit.get('mime') or 'application/octet-stream', + 'data_b64': _b64c.b64encode(blob).decode('ascii')} + return True # re-render to deliver it + + def _write(cur): + cur[str(pid)] = row + return cur + store.update(_DOCS_KEY, _write, flush='async') + return True + + if kind in ('folder_create', 'folder_rename', 'folder_delete', 'folder_duplicate', + 'item_move'): + # FOLDERS over the Views / Cohorts sidebars (owner item 11, contract C4). All five + # events touch ONE home — the table workspace's `folders` + `itemFolders` — so a cohort + # can be filed without the cohort STORE learning about folders at all. + import aios_grid as _agf + if not store.available(): + return False + surface = str(event.get('surface') or '') + if surface not in _agf.FOLDER_SURFACES: + return False + # The ids this user may actually file: their own views, their own cohorts. Fail-closed — + # an item_move naming somebody else's cohort is dropped by clean_item_folders anyway, + # but refusing here means we never write it in the first place. + def _own_ids(sfc, cur_ws): + if sfc == 'views': + return set((cur_ws.get('views') or {})) + return set(cohort_mod.all_for(uname)) + + def _mutate(cur_ws): + folders = _agf.clean_folders(cur_ws.get('folders')) + placed = dict(cur_ws.get('itemFolders') or {}) + lst = list(folders.get(surface) or []) + # ⚠ the wire field is `folderId` on EVERY folder event (CLIENT's types.ts is the + # built contract; the split doc's first draft said `id`). Converged here rather + # than asking the client to re-emit — see the C4 reconciliation note in the doc. + fid = str(event.get('folderId') or '').strip()[:80] + if kind == 'folder_create': + name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] + if not fid or not name or len(lst) >= _agf.MAX_FOLDERS: + return None + if any(f['id'] == fid for f in lst): + return None + row = {'id': fid, 'name': name, 'order': len(lst)} + icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5) + if icon: + row['icon'] = icon + lst.append(row) + elif kind == 'folder_rename': + name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] + hit = next((f for f in lst if f['id'] == fid), None) + if not hit or not name: + return None + hit['name'] = name + # wave-9 I15 (C5): the rename pane is also where the icon is changed. An event + # that carries NO icon key leaves the existing one alone (a rename must not + # silently strip a chosen icon); an explicit null clears it back to default. + if 'icon' in event: + icon = _agf.clean_folder_icon(event.get('icon')) + if icon: + hit['icon'] = icon + else: + hit.pop('icon', None) + elif kind == 'folder_delete': + if not any(f['id'] == fid for f in lst): + return None + lst = [f for f in lst if f['id'] != fid] + # CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that: + # an item with no placement renders at the top level. + placed[surface] = {k: v for k, v in (placed.get(surface) or {}).items() + if v != fid} + elif kind == 'folder_duplicate': + new_id = str(event.get('newId') or '').strip()[:80] + src = next((f for f in lst if f['id'] == fid), None) + if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS: + return None + if any(f['id'] == new_id for f in lst): + return None + lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME], + 'order': len(lst)}) + # ⚠ The folder's CONTENTS are duplicated by the client emitting the ordinary + # per-item duplicate events it already owns (view_upsert with a fresh id, the + # cohort copy path) and then item_move-ing them here. Copying views/cohorts + # inside this handler would be a SECOND implementation of "duplicate an item", + # and the two would drift the first time either one changed. + else: # item_move + item_id = str(event.get('itemId') or '').strip()[:120] + target = event.get('folderId') + if not item_id or item_id not in _own_ids(surface, cur_ws): + return None + cur = dict(placed.get(surface) or {}) + if target is None: + cur.pop(item_id, None) # back to the root + else: + tid = str(target)[:80] + if not any(f['id'] == tid for f in lst): + return None + cur[item_id] = tid + placed[surface] = cur + folders[surface] = lst + clean = _agf.clean_folders(folders) + return clean, _agf.clean_item_folders( + placed, clean, + {sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES}) + + # ⚠ NO allowed_pids ON PURPOSE — this is app.py's bare `_cl_table_workspace(uname)` + # call at the old :6268, preserved exactly. Omitting the pool drops memberPids from + # any foreign shared view (fail-closed), which is harmless here because the folder + # legs read only view IDS and the folder maps. Passing the pool would be a behavior + # change inside a pure move. + _res = _mutate(table_workspace(ctx)) + if _res is None: + return False # refused: nothing is written + cl_mod.save_table_folders(uname, _res[0], _res[1]) + # True: the sidebars render from server state (which folder holds what), not from local + # optimism — the same contract the cohort panel's membership edits ride. + return True + + if kind == 'field_delete': + # Delete a USER-CREATED column outright (owner gap closed 2026-07-27). The prefix IS + # the permission: only the created strata (`custom_` overlay fields, `measure_` formula + # columns) are deletable — a base field's key fails the prefix test no matter what the + # browser claims, so the table's contract cannot be deleted from the client. The store + # scrubs the field's stored cell values with it (delete_table_field); views naming the + # key self-heal on their next autosave, the rule every stale colId already rides. + key = str(event.get('key') or '')[:80] + import aios_grid as _ag3 + if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)): + return False + if store.available(): + cl_mod.delete_table_field(uname, key) + else: + _session_ready() + ws['fields'].pop(key, None) + for _row in ws['overlays'].values(): + if isinstance(_row, dict): + _row.pop(key, None) + # Wave-6 item 3: the client dropped the column optimistically; the store scrub is + # invisible to the next render (the values leave WITH the column). + return False + + if kind == 'field_duplicate': + # Wave-5 item 1. The DESTINATION key is client-generated (the established pattern — + # the client must know it synchronously to place the clone right of the source), and + # both keys must sit in the SAME created stratum: base odoo fields offer no Duplicate, + # because duplicating the source-of-truth into an editable copy would need a + # value-snapshot mechanism nobody asked for. Overlay VALUES are copied only for + # `custom_` sources (a measure clone recomputes; a formula clone re-evaluates). + # The duplicator becomes the clone's creator — it is a NEW field they made. + src_key = str(event.get('sourceKey') or '')[:80] + new_key = str(event.get('key') or '')[:80] + import aios_grid as _ag5 + prefix = ('custom_' if src_key.startswith('custom_') + else _ag5.MEASURE_FIELD_PREFIX + if src_key.startswith(_ag5.MEASURE_FIELD_PREFIX) else None) + if (prefix is None or not new_key.startswith(prefix) or new_key == src_key + or new_key in field_by_key): + return False + src = ws.get('fields', {}).get(src_key) + if not isinstance(src, dict): + return False + clone = dict(src) + clone['key'] = new_key + label = str(event.get('label') or '').strip()[:120] + clone['label'] = label or (str(src.get('label') or 'Untitled')[:110] + ' copy') + clone['createdBy'] = uname + # Wave-6 item 9: the clone inherits the source's scope via dict(src); on the cohort + # page the duplicate surface may explicitly place it instead (same create-time rule + # as field_upsert — a page that is not the cohort page cannot mint a cohort field). + if scope_key == 'cohort' and event.get('scope') in ('cohort', 'global'): + if event['scope'] == 'cohort': + clone['scope'] = 'cohort' + else: + clone.pop('scope', None) + if store.available(): + cl_mod.duplicate_table_field(uname, src_key, new_key, clone) + else: + _session_ready() + ws['fields'][new_key] = clone + if src_key.startswith('custom_'): + for _row in ws['overlays'].values(): + if isinstance(_row, dict) and src_key in _row: + _row[new_key] = _row[src_key] + # Wave-6 item 3: True only when host-held VALUES must flow back — copied overlay + # cells (custom_ sources) or host-computed measure values. formula/created_time + # clones compute client-side, so the optimistic insert is already complete. + return src.get('type') not in ('formula', 'created_time') + + if kind == 'add_to_list': + # "Add to list" (owner item 6): a view's current matches become fixed cohort members. + # + # ⚠ `pids` arrives from the BROWSER. It is intersected with `allowed_pids` — the pool this + # user may actually see — before anything is stored. The client computing the set is a + # correctness decision (it is the only engine that already knows the answer, and asking a + # second engine the same question is the drift verify_filter_engine.py exists to prevent); + # it is NOT a trust decision. + raw_pids = event.get('pids') + if not isinstance(raw_pids, list): + return False + pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids] + if not pids: + return False + cohort_id = str(event.get('cohortId') or '').strip()[:120] + name = str(event.get('name') or '').strip()[:120] + if not store.available(): + return False + try: + if not cohort_id: + if not name: + return False + cohort_id = cohort_mod.create(uname, name, pids) + else: + if cohort_id not in cohort_mod.all_for(uname): + return False # not this user's cohort — fail closed, do not create + cohort_mod.add_members(uname, cohort_id, pids) + except ValueError as e: + _tel.error('cohort:add_to_list', e) + return False + # True: the Cohort page's list and counts must reflect this, and the toast below only + # renders on a fresh run. + ctx.out.toast = ( + f"Added {len(pids):,} customer{'' if len(pids) == 1 else 's'} to " + f"{name or 'the list'}.") + return True + + if kind == 'overlay_patch': + pid = event.get('pid') + if not isinstance(pid, int) or pid not in allowed_pids: + return False + updates = {} + refused = False + raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {} + for key, value in raw_updates.items(): + if key not in overlay_keys or not isinstance(value, (str, int, float)): + refused = True + continue + # Wave-5 item 1: `permissions.edit` is enforced HERE, per key — the client hiding + # its editor is courtesy, this is the wall. Fail-closed: a restricted field with + # no readable createdBy admits only admins. + perms = (field_by_key.get(key) or {}).get('permissions') or {} + edit = perms.get('edit') + if edit == 'admins' and not admin: + refused = True + continue + if edit == 'creator' and not ( + admin or (field_by_key.get(key) or {}).get('createdBy') == uname): + refused = True + continue + clean = str(value)[:10000] + if isinstance(value, str) and clean != value: + # Truncated: the stored truth is shorter than the client's copy. (An + # int/float arriving as its string form renders identically — not a refusal.) + refused = True + updates[key] = clean + if updates: + if store.available(): + cl_mod.patch_table_overlay(uname, pid, updates) + else: + _session_ready() + ws['overlays'].setdefault(str(pid), {}).update(updates) + # Wave-6 item 3: the client's cell already shows the typed value — rerun ONLY when + # some of what it shows was refused or clamped and must be repainted to the truth. + return refused + + return False diff --git a/royalimports_os/core/links.py b/royalimports_os/core/links.py new file mode 100644 index 0000000000000000000000000000000000000000..46ea71d46633a9bfe59545777ed5d0acc3cb3d25 --- /dev/null +++ b/royalimports_os/core/links.py @@ -0,0 +1,59 @@ +"""Deep links into the app — ?page=&entity=:&src=. + +The app (app.py) consumes these params once per session right after login and routes to the page, +opening the entity drawer when an entity is present. Digest/alert/mention emails build their links +here so every emailed number lands the reader on the exact row it came from. `src` tags where the +click came from (digest / alert / mention) — the app logs it as the click-through read receipt. +""" +import os +import sys +from urllib.parse import quote + +#: ⛔ THE ONE HOST LITERAL LEFT IN THE RUNTIME PATH, and it is a MIGRATION HAZARD — named here +#: rather than buried in a default argument, because the day we change hosts this is what silently +#: keeps mailing customers a link to the DEAD one. Found by `ops/verify_portability.py` on its +#: first run (2026-07-30). +#: +#: Why it is still here instead of raising: `deeplink()` has exactly one family of callers +#: (`modules/digest.py`, 7 sites, all building email hrefs), `DIGEST_ENABLED` defaults to ON, and +#: no deploy sets `APP_BASE_URL` today — so refusing to guess would break a LIVE feature to fix a +#: latent one. Instead: both deploy scripts now push `APP_BASE_URL`, which makes the fallback +#: unreachable in a deployed environment, and reaching it prints an operator warning. +#: +#: ⇒ EXIT CONDITION for this constant: once a deploy has set `APP_BASE_URL` and it is confirmed +#: present in the running environment, delete the fallback and let `deeplink()` raise instead. It +#: is declared in the portability gate's RESIDUALS so the gate reports it honestly rather than +#: reporting green over it. +_LEGACY_HF_FALLBACK = 'https://royal-imports-cfo-os.hf.space' + +BASE_URL = (os.environ.get('APP_BASE_URL') or _LEGACY_HF_FALLBACK).rstrip('/') + +#: Warned LAZILY and ONCE — at the first link actually built, not at import. An import-time print +#: fires in every gate, every subprocess and every unit test that so much as touches `core`, which +#: is how a real warning becomes noise nobody reads. +_WARNED = [False] + + +def _warn_once_if_defaulted(): + if _WARNED[0] or os.environ.get('APP_BASE_URL'): + return + _WARNED[0] = True + print(f"[links] APP_BASE_URL is not set - emailed deep links will point at " + f"{_LEGACY_HF_FALLBACK}, which is correct ONLY while that is where the app lives. " + f"Set APP_BASE_URL in the deployed environment.", file=sys.stderr) + +# entity kinds the URL scheme supports (must stay in sync with app.py's _desc_from_param) +LINK_KINDS = ('customer', 'sku', 'agent', 'account') + + +def deeplink(page=None, kind=None, ident=None, src=None): + """An absolute app URL. deeplink('customer_data', 'customer', 1234, src='digest').""" + _warn_once_if_defaulted() + parts = [] + if page: + parts.append(f"page={quote(str(page), safe='')}") + if kind in LINK_KINDS and ident not in (None, ''): + parts.append(f"entity={quote(f'{kind}:{ident}', safe=':')}") + if src: + parts.append(f"src={quote(str(src), safe='')}") + return BASE_URL + ('/?' + '&'.join(parts) if parts else '/') diff --git a/royalimports_os/core/management_pl.py b/royalimports_os/core/management_pl.py new file mode 100644 index 0000000000000000000000000000000000000000..1546b097d465713ad2b19f039b91a1ee57d5f1d8 --- /dev/null +++ b/royalimports_os/core/management_pl.py @@ -0,0 +1,161 @@ +"""core/management_pl.py — reproduce the operating model's MANAGEMENT P&Ls +(FFS-M / RI-M / GD-M / BU-M) monthly from RAW ODOO, Sep-2024 cutover onward. READ-ONLY. + +Decoded from v42 (see 17. Application development/financial_analysis/MODEL_LOGIC_MAP.md §10): + + -M[entity][account] = clean_entity[account] # live from Odoo (PROVEN to the cent) + + hq_overlay[entity][account] # HQ cost allocated to the entity + + adj_overrides[entity][account] # tiny, transition-month only + + clean_entity = account.analytic.line grouped by general_account_id (GL row) x account_id + (analytic = ENTITY column). Analytic ids: Fisch=1 Royal=2 GiftwareDeals=3 + HQ=4 Internal=5. Sections by account_type. This is build_income_statements.py + (verified exact). Proven live: Oct-2025 acct 51000 -> Fisch 327,531.85 etc. + + hq_overlay / adj_overrides come from data/mgmt_overlay.json (extracted read-only from v42 by + build_mgmt_overlay.py). The HQ allocation is HARDCODED in the model (PNL (HQ-*) tabs are literal + values, not formulas), so storing it is faithful to how the model is actually maintained. + +Consolidation: BU-M = FFS-M + RI-M + GD-M (the Internal analytic is eliminated; HQ is allocated out). + +Pure: data in -> plain dicts out. No Streamlit, no writes. +""" +from __future__ import annotations +import json +import os +from functools import lru_cache + +from core import odoo as O + +# ---- entity <-> Odoo analytic account ids (confirmed live) -------------------------------------- +ENTITY_ANALYTIC = {'FFS': 1, 'RI': 2, 'GD': 3, 'HQ': 4, 'Internal': 5} +ANALYTIC_ENTITY = {v: k for k, v in ENTITY_ANALYTIC.items()} +BUS = ['FFS', 'RI', 'GD'] # the business units that get a -M statement +ENTITY_LABEL = {'FFS': 'Fisch (FFS)', 'RI': 'Royal (RI)', 'GD': 'Giftware Deals (GD)', 'BU': 'Consolidated (BU)'} + +# P&L sections by Odoo account_type (mirrors build_income_statements.py) +INCOME_TYPES = {'income'} +OTHER_INCOME_TYPES = {'income_other'} +COGS_TYPES = {'expense_direct_cost'} +EXPENSE_TYPES = {'expense'} +DEPREC_TYPES = {'expense_depreciation'} +PL_TYPES = INCOME_TYPES | OTHER_INCOME_TYPES | COGS_TYPES | EXPENSE_TYPES | DEPREC_TYPES + +_OVERLAY_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'data', 'mgmt_overlay.json') + + +def month_bounds(year: int, month: int) -> tuple[str, str]: + import calendar + last = calendar.monthrange(year, month)[1] + return f'{year}-{month:02d}-01', f'{year}-{month:02d}-{last:02d}' + + +@lru_cache(maxsize=1) +def account_master() -> dict: + """{gl_account_id: {'code','name','type'}} for every account.""" + rows = O.search_read('account.account', [], ['id', 'code', 'name', 'account_type']) + return {r['id']: {'code': r.get('code') or '', 'name': r.get('name') or '', + 'type': r.get('account_type')} for r in rows} + + +def _display(acc_type: str, raw: float) -> float: + """Income shown raw; costs/expenses/depreciation flipped positive (model convention).""" + if acc_type in INCOME_TYPES or acc_type in OTHER_INCOME_TYPES: + return raw + return -raw + + +@lru_cache(maxsize=64) +def clean_entities(year: int, month: int) -> dict: + """The clean per-entity P&L straight from Odoo analytic lines, for one month. + + Returns {entity_key: {account_code: display_amount}} for FFS/RI/GD/HQ/Internal, restricted to + P&L account types. This is the LIVE, proven-exact foundation (= build_income_statements.py). + """ + start, end = month_bounds(year, month) + rows = O.read_group('account.analytic.line', + domain=[('date', '>=', start), ('date', '<=', end)], + fields=['amount:sum'], groupby=['general_account_id', 'account_id'], lazy=False) + am = account_master() + out = {k: {} for k in ENTITY_ANALYTIC} + for r in rows: + gen = r.get('general_account_id'); ana = r.get('account_id') + if not gen or not ana: + continue + gid, aid = gen[0], ana[0] + ent = ANALYTIC_ENTITY.get(aid) + acc = am.get(gid) + if ent is None or not acc or acc['type'] not in PL_TYPES: + continue + amt = _display(acc['type'], r.get('amount') or 0.0) + if abs(amt) < 0.005: + continue + out[ent][acc['code']] = out[ent].get(acc['code'], 0.0) + amt + return out + + +@lru_cache(maxsize=1) +def _overlay() -> dict: + """The stored HQ-allocation + transition-reconciliation overlay (build_mgmt_overlay.py). + Shape: {'YYYY-MM': {'FFS': {acct: amt}, ...}}. These are financial figures, so they live in the + PRIVATE HF Dataset store on the Space (never in the public Space repo); locally they load from the + committed JSON. File first, so local runs need no token. Empty -> -M = clean entity only.""" + try: + with open(_OVERLAY_PATH, encoding='utf-8') as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + pass + try: + from core import store + return store.get('mgmt_overlay') or {} + except Exception: + return {} + + +def management_pl(year: int, month: int, entity: str) -> dict: + """The management P&L for one entity-month: {account_code: amount} = clean + overlay. + + entity in {'FFS','RI','GD'}; 'BU' returns the consolidation (sum of the three). + """ + if entity == 'BU': + agg: dict = {} + for bu in BUS: + for code, amt in management_pl(year, month, bu).items(): + agg[code] = agg.get(code, 0.0) + amt + return agg + clean = dict(clean_entities(year, month).get(entity, {})) + ov = _overlay().get(f'{year}-{month:02d}', {}).get(entity, {}) + for code, amt in ov.items(): + clean[code] = clean.get(code, 0.0) + amt + return clean + + +# ---- section roll-ups (for the statement view) -------------------------------------------------- +def _bucket(code: str) -> str | None: + acc = next((a for a in account_master().values() if a['code'] == code), None) + if not acc: + return None + t = acc['type'] + if t in INCOME_TYPES: return 'income' + if t in OTHER_INCOME_TYPES: return 'other_income' + if t in COGS_TYPES: return 'cogs' + if t in EXPENSE_TYPES: return 'expense' + if t in DEPREC_TYPES: return 'deprec' + return None + + +def statement(year: int, month: int, entity: str) -> dict: + """Section subtotals + net for one entity-month (income / cogs / gross / opex / deprec / net).""" + lines = management_pl(year, month, entity) + sec = {'income': 0.0, 'other_income': 0.0, 'cogs': 0.0, 'expense': 0.0, 'deprec': 0.0} + for code, amt in lines.items(): + b = _bucket(code) + if b: + sec[b] += amt + income = sec['income'] + sec['other_income'] + gross = income - sec['cogs'] + net = gross - sec['expense'] - sec['deprec'] + return {'income': income, 'cogs': sec['cogs'], 'gross_profit': gross, + 'operating_expense': sec['expense'], 'depreciation': sec['deprec'], 'net_profit': net, + 'lines': lines} diff --git a/royalimports_os/core/odoo.py b/royalimports_os/core/odoo.py new file mode 100644 index 0000000000000000000000000000000000000000..4cd1bd2b4aef99429b92a1c9ccbfd391dee65338 --- /dev/null +++ b/royalimports_os/core/odoo.py @@ -0,0 +1,201 @@ +"""CFO-OS shared Odoo data layer (READ-ONLY). + +Reuses the existing write-guarded client (ffs_dashboard/odoo_client.py) so there is +a single source of truth for the connection. Adds: + - cached singleton client + - read_group / search_read passthroughs with light cleanup + - business constants (teams, excluded partners) + - the "sales universe" base domain (confirmed orders, RI+FFS only) + +NOTHING here writes to Odoo — the underlying client hard-blocks writes. +""" +import os +import sys +import queue as _queue +import threading +from concurrent.futures import ThreadPoolExecutor +from functools import lru_cache +from pathlib import Path + +# Self-contained read-only client lives at the app root (royalimports_os/odoo_client.py) with its +# own .env — so this whole folder can be moved/hosted anywhere without external paths. +_ROOT = Path(__file__).resolve().parents[1] # royalimports_os/ +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) +from odoo_client import OdooClient # noqa: E402 (read-only, write-guarded) + +# ---- business constants ---- +TEAM_IDS = [5, 6] +TEAM_NAMES = {5: 'Fisch', 6: 'Royal'} +# Partners to exclude from analytics (not part of RI/FFS). Sourced from env so no +# customer name is ever hardcoded in the (potentially public) repo; set as a Space secret. +EXCLUDE_PARTNER_NAMES = {n.strip() for n in os.environ.get('EXCLUDE_PARTNER_NAMES', '').split(',') if n.strip()} + +# Confirmed sale.order.line scope for RI+FFS (the analytics "sales universe"). +SALE_LINE_BASE = [ + ('state', 'in', ['sale', 'done']), + ('order_id.team_id', 'in', TEAM_IDS), + ('product_id', '!=', False), +] +# Posted customer invoices/credit notes (AR + revenue recognised). +INVOICE_BASE = [ + ('move_type', 'in', ['out_invoice', 'out_refund']), + ('state', '=', 'posted'), +] + + +@lru_cache(maxsize=1) +def _global_client(): + """The single read-only client used for normal (main-thread) calls.""" + return OdooClient() + + +# ---- concurrent read pool ------------------------------------------------------------------- +# Each Odoo XML-RPC call is a round-trip; a heavy drawer fires a dozen of them and they add up to +# many seconds when run one-by-one. `parallel()` runs independent READ tasks across a small pool of +# extra connections, with each task transparently using its own connection via a thread-local — so +# the existing read_group/search_read/sum_field helpers need no change. READ-ONLY only. +_POOL_MAX = 8 +_tlocal = threading.local() +_pool_q = _queue.Queue() +_pool_n = 0 +_pool_lock = threading.Lock() + + +def _checkout(): + global _pool_n + try: + return _pool_q.get_nowait() + except _queue.Empty: + with _pool_lock: + grow = _pool_n < _POOL_MAX + if grow: + _pool_n += 1 + return OdooClient() if grow else _pool_q.get() # else block until one frees + + +def _checkin(c): + _pool_q.put(c) + + +def get_odoo(): + """The active read-only client. Inside a parallel() task this is that task's pooled connection; + otherwise it's the shared singleton. Transparent to every read_group/search_read/... caller.""" + c = getattr(_tlocal, 'client', None) + return c if c is not None else _global_client() + + +def set_doc_mode(mode): + """Recognition basis for THIS thread's domain builds: 'order' (all confirmed orders) or 'invoice' + (fully-invoiced orders only). Cached bundles call this before pulling; parallel() propagates it to + each worker. Thread-local, so concurrent user sessions never race on each other's basis.""" + _tlocal.doc_mode = 'invoice' if mode == 'invoice' else 'order' + + +def doc_mode(): + return getattr(_tlocal, 'doc_mode', 'order') + + +def parallel(tasks): + """Run independent zero-arg READ callables concurrently across the connection pool; returns + results in order. Inside each task, get_odoo() (hence read_group/search_read/sum_field/ + search_count) uses that task's own connection. Never use for writes.""" + tasks = list(tasks) + if not tasks: + return [] + if len(tasks) == 1: + return [tasks[0]()] + + caller_mode = doc_mode() # propagate the submitter's basis to each worker + def _run(fn): + c = _checkout() + _tlocal.client = c + _tlocal.doc_mode = caller_mode + try: + return fn() + finally: + _tlocal.client = None + _tlocal.doc_mode = 'order' + _checkin(c) + with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex: + return list(ex.map(_run, tasks)) + + +def parallel_map(task_dict): + """parallel() over a {name: callable} mapping -> {name: result}.""" + keys = list(task_dict) + return dict(zip(keys, parallel([task_dict[k] for k in keys]))) + + +def m2o_id(v): + return v[0] if isinstance(v, list) and v else None + + +def m2o_name(v): + return v[1] if isinstance(v, list) and len(v) > 1 else '' + + +@lru_cache(maxsize=1) +def excluded_partner_ids(): + """Resolve the excluded partner names (from EXCLUDE_PARTNER_NAMES) to ids, for fast + domain filtering. Cached for the process.""" + o = get_odoo() + if not EXCLUDE_PARTNER_NAMES: + return tuple() + rows = o.search_read('res.partner', + [('name', 'in', list(EXCLUDE_PARTNER_NAMES))], ['id']) + return tuple(r['id'] for r in rows) + + +def sale_line_domain(date_from=None, date_to=None, team_id=None, extra=None, partner_ids=None): + """Build a sale.order.line domain in the RI+FFS confirmed-order scope, excluding + the configured house accounts, with optional date window / single team / extra clauses. + partner_ids (a collection, possibly empty) restricts to those customers — the carrier for + the Customer module's Agent filter; None = no partner restriction (an empty set matches none).""" + dom = list(SALE_LINE_BASE) + if team_id is not None: + dom = [d for d in dom if not (isinstance(d, tuple) and d[0] == 'order_id.team_id')] + dom.append(('order_id.team_id', '=', team_id)) + if date_from: + dom.append(('order_id.date_order', '>=', f'{date_from} 00:00:00')) + if date_to: + dom.append(('order_id.date_order', '<=', f'{date_to} 23:59:59')) + ex = excluded_partner_ids() + if ex: + dom.append(('order_partner_id', 'not in', list(ex))) + if partner_ids is not None: + dom.append(('order_partner_id', 'in', list(partner_ids))) + if doc_mode() == 'invoice': + dom.append(('order_id.invoice_status', '=', 'invoiced')) + if extra: + dom.extend(extra) + return dom + + +def read_group(model, domain, fields, groupby, **kw): + return get_odoo().read_group(model, domain=domain, fields=fields, groupby=groupby, **kw) + + +def search_read(model, domain=None, fields=None, **kw): + return get_odoo().search_read(model, domain=domain or [], fields=fields or [], **kw) + + +def sum_field(model, domain, field): + """One-line aggregate sum of `field` over a domain (uses read_group, no row fetch). + + Robust to the empty match set: a groupby=[] read_group over zero rows makes Odoo return a + None aggregate, which the server's XML-RPC layer cannot marshal ("cannot marshal None"). + An empty set simply means the sum is 0.0, so we treat that specific Fault as zero.""" + try: + g = get_odoo().read_group(model, domain=domain, fields=[f'{field}:sum'], groupby=[], lazy=False) + except Exception as e: # noqa: BLE001 — narrow check below + if 'cannot marshal None' in str(e): + return 0.0 + raise + return (g[0].get(field) or 0.0) if g else 0.0 + + +def distinct_count(model, domain, field): + """Distinct count of a (m2o) field over a domain via grouped read_group.""" + g = get_odoo().read_group(model, domain=domain, fields=[field], groupby=[field], lazy=False) + return sum(1 for r in g if r.get(field)) diff --git a/royalimports_os/core/periods.py b/royalimports_os/core/periods.py new file mode 100644 index 0000000000000000000000000000000000000000..fe92ec7228c03056d11b149cc95eafb76c077ace --- /dev/null +++ b/royalimports_os/core/periods.py @@ -0,0 +1,155 @@ +"""Period helpers — floral wholesale is highly seasonal, so YoY / same-period +comparisons are first-class. Everything returns ISO date strings (yyyy-mm-dd). + +`today` is injectable for testing/reproducibility. +""" +import datetime as dt + + +def _d(x): + return x.isoformat() + + +def today(): + return dt.date.today() + + +def ytd(t=None): + """(start, end) for year-to-date through `t`.""" + t = t or today() + return _d(dt.date(t.year, 1, 1)), _d(t) + + +def ytd_last_year(t=None): + """Same calendar window one year earlier (Jan 1 LY → same month/day LY). + Clamps Feb 29 → Feb 28 in non-leap years.""" + t = t or today() + try: + end = dt.date(t.year - 1, t.month, t.day) + except ValueError: + end = dt.date(t.year - 1, t.month, t.day - 1) + return _d(dt.date(t.year - 1, 1, 1)), end if isinstance(end, str) else _d(end) + + +def ltm(t=None): + """Trailing 12 months ending `t`.""" + t = t or today() + return _d(t - dt.timedelta(days=365)), _d(t) + + +def prior_ltm(t=None): + t = t or today() + return _d(t - dt.timedelta(days=730)), _d(t - dt.timedelta(days=365)) + + +def mtd(t=None): + t = t or today() + return _d(dt.date(t.year, t.month, 1)), _d(t) + + +def wtd(t=None): + """(Monday of this ISO week, t) — week-to-date.""" + t = t or today() + return _d(t - dt.timedelta(days=t.weekday())), _d(t) + + +def qtd(t=None): + """(first day of this calendar quarter, t) — quarter-to-date.""" + t = t or today() + q_start_month = ((t.month - 1) // 3) * 3 + 1 + return _d(dt.date(t.year, q_start_month, 1)), _d(t) + + +def quarter_range(t=None): + """(first day, last day) of the calendar quarter containing t.""" + t = t or today() + qs = ((t.month - 1) // 3) * 3 + 1 + first = dt.date(t.year, qs, 1) + nxt = dt.date(t.year + (1 if qs + 3 > 12 else 0), ((qs + 2) % 12) + 1, 1) + return _d(first), _d(nxt - dt.timedelta(days=1)) + + +def shift_year(date_from, date_to, weeks=False): + """Same window one year earlier. For week-aligned comparisons (weeks=True) shift by exactly 52 + weeks (364 days) so the weekday lines up; otherwise shift by calendar year (clamping Feb 29).""" + f = dt.date.fromisoformat(date_from) + t = dt.date.fromisoformat(date_to) + if weeks: + return _d(f - dt.timedelta(days=364)), _d(t - dt.timedelta(days=364)) + + def back(d): + try: + return dt.date(d.year - 1, d.month, d.day) + except ValueError: + return dt.date(d.year - 1, d.month, d.day - 1) + return _d(back(f)), _d(back(t)) + + +def last_n_days(n, t=None): + t = t or today() + return _d(t - dt.timedelta(days=n)), _d(t) + + +def year_range(year): + return f'{year}-01-01', f'{year}-12-31' + + +def month_range(year, month): + """(first day, last day) of a given calendar month, ISO strings.""" + first = dt.date(year, month, 1) + nxt = dt.date(year + (1 if month == 12 else 0), 1 if month == 12 else month + 1, 1) + return _d(first), _d(nxt - dt.timedelta(days=1)) + + +def period_options(t=None, n_months=15): + """Ordered {label: (date_from, date_to)} for the close/reconciliation period picker.""" + t = t or today() + opts = {} + opts['This month (to date)'] = (_d(dt.date(t.year, t.month, 1)), _d(t)) + lm_year, lm_month = (t.year, t.month - 1) if t.month > 1 else (t.year - 1, 12) + lf, lt = month_range(lm_year, lm_month) + opts[f'Last month ({lf[:7]})'] = (lf, lt) + opts['Quarter to date'] = (_d(dt.date(t.year, ((t.month - 1) // 3) * 3 + 1, 1)), _d(t)) + opts['Year to date'] = (_d(dt.date(t.year, 1, 1)), _d(t)) + for label, mf, mt in reversed(month_starts(n_months, t)): + if label == f'{t.year}-{t.month:02d}' or label == lf[:7]: + continue + opts[label] = (mf, mt) + return opts + + +def month_starts(n_back=13, t=None): + """List of (yyyy-mm, start, end) for the last n_back months including current.""" + t = t or today() + out = [] + y, m = t.year, t.month + for _ in range(n_back): + start = dt.date(y, m, 1) + end = (dt.date(y + (m // 12), (m % 12) + 1, 1) - dt.timedelta(days=1)) + out.append((f'{y}-{m:02d}', _d(start), _d(end))) + m -= 1 + if m == 0: + m = 12 + y -= 1 + return list(reversed(out)) + + +def week_starts(n_back=12, t=None): + """List of (label, start, end) for the last n_back ISO weeks (Mon–Sun) including the current + (partial) week. label = the Monday as 'MM-DD' (compact, chart-friendly); end is clamped to t + for the current week so weekly totals never include future days.""" + t = t or today() + this_mon = t - dt.timedelta(days=t.weekday()) + out = [] + for i in range(n_back): + mon = this_mon - dt.timedelta(weeks=i) + sun = mon + dt.timedelta(days=6) + end = min(sun, t) + out.append((f'{mon.month:02d}-{mon.day:02d}', _d(mon), _d(end))) + return list(reversed(out)) + + +def yoy_pct(this_v, last_v): + if last_v: + return (this_v - last_v) / last_v * 100.0 + return None diff --git a/royalimports_os/core/perms.py b/royalimports_os/core/perms.py new file mode 100644 index 0000000000000000000000000000000000000000..47a19fffa6b94195804d84c28f6e3bf53234a339 --- /dev/null +++ b/royalimports_os/core/perms.py @@ -0,0 +1,164 @@ +"""core/perms.py — the permission + scope predicates as PURE functions of a user record. + +EXIT-3b (2026-07-30). `ui/session.py` holds the same predicates keyed off `st.session_state`; +these take the user record explicitly, so the API can answer "what may this session see?" without +a Streamlit session. ONE implementation of each rule, two callers — and `verify_api.py` asserts +lock-step against `ui/session.py`'s source (the [[date-window-vocabulary]] discipline: a mirrored +vocabulary that is not gated has already drifted). + +⚠ THE TRAP THIS FILE EXISTS TO AVOID. `ui/session.brand_team()` reads the sidebar's BU selector +and defaults to `'All'` → `team_id=None` → CONSOLIDATED. Mirroring it literally on the API would +be a fail-OPEN: an API session has no selector, so a Royal-only user would be handed Fisch rows — +the exact leak strict isolation exists to prevent. The scope must be derived from the USER RECORD +(`bus`), never from a UI default. `scope_team_id()` below is that derivation. + +Fail-closed everywhere: an unknown module key is denied, an unreadable `bus` narrows rather than +widens, and a user with no grant sees nothing rather than everything. +""" +import core.context as ctxlib +import core.registry as registry +import core.users as users + +# Old module keys migrate ON READ — grants, Library prefs and ?page= deep links keep working +# without ever editing a stored user record. MUST stay identical to ui/session._LEGACY_KEYS +# (gated in verify_api.py): 'myday' shipped before the Customer List rename, 'customer_list' +# before the 2026-07-26 rename to 'customer_data', 'map' joined in wave 8 when the Map PAGE +# became a VIEW of the customer table. A chain is fine only if every hop resolves to the CURRENT +# key, so 'myday' points at the END of the chain, not at the middle. +# ⚠ 'customers' is deliberately NOT here. It is a real registry key — the parent of +# customer_data + cohort — and may_open() gives a parent grant to its children. Mapping it away +# would silently narrow every "customers" grant to one child. +_LEGACY_KEYS = {'myday': 'customer_data', 'customer_list': 'customer_data', + 'map': 'customer_data', 'library': 'settings'} + + +def is_admin(user): + return (user or {}).get('role') == 'admin' + + +def allowed_modules(user): + """Registry keys this user may open — None means ALL (admins / unrestricted users).""" + m = (user or {}).get('modules', 'all') + if not m or m == 'all': + return None + return {_LEGACY_KEYS.get(k, k) for k in m} + + +def may_open(user, key): + """True iff `user` may OPEN module `key`. A PARENT grant covers its sub-modules (granting + 'customers' grants 'customer_data'). Fail-closed — an unknown key stays denied.""" + am = allowed_modules(user) + if am is None or key in am: + return True + parent = registry.BY_KEY.get(key, {}).get('parent') + return parent in am if parent else False + + +# ------------------------------------------------------------------ BU (business unit) scope +def allowed_bu_labels(user): + """The BU labels this user may see — `core.users.allowed_bus_labels`, which is already a + pure function of the record. Named here so callers have one import for the whole rule set.""" + return users.allowed_bus_labels(user) + + +def allowed_team_ids(user): + """The concrete Odoo team_ids this user may see, as a frozenset. `All` expands to every BU.""" + labels = allowed_bu_labels(user) + if 'All' in labels: + return frozenset(t for t in ctxlib.BRANDS.values() if t is not None) + return frozenset(ctxlib.BRANDS[l] for l in labels + if l in ctxlib.BRANDS and ctxlib.BRANDS[l] is not None) + + +def scope_team_id(user): + """The `team_id` an API request must query with — the ONE number that enforces BU isolation. + + None means CONSOLIDATED, and it is returned ONLY when the user may genuinely see every BU + (`bus='all'`, or a multi-BU grant, both of which put 'All' in the label list). A user + permitted exactly one BU gets that BU's id PINNED, never None. + + ⚠ This is the opposite default from `ui/session.brand_team()`, which falls back to 'All' + because a UI selector has to start somewhere. Here there is no selector to read, so the + fallback would BE the leak. A single-BU user's rows are pinned at the query, not filtered + afterwards, so there is no window in which the other BU's rows exist in the response. + """ + labels = allowed_bu_labels(user) + if 'All' in labels: + return None + ids = sorted(allowed_team_ids(user)) + return ids[0] if len(ids) == 1 else None + + +def scope_agent(user): + """The sales agent whose OWN BOOK this session is confined to, or None for the whole book. + + ⚠ Mirrors `page_customer_data` (app.py), which reads `u.get('agent')` with NO admin + exemption — deliberately different from `page_agent`'s `None if is_admin() else …`. The + customer TABLE is the surface `/api/v1/customers` serves, so its rule is the one that + travels: an account linked to an agent sees that agent's book, admin or not. Getting this + wrong in the permissive direction would hand a restricted sales-agent login the whole book. + """ + return (user or {}).get('agent') or None + + +# ------------------------------------------------------------------ nav projection +def nav_pages(user): + """The nav this user may see: registry order, `may_open`-filtered, archived EXCLUDED. + + Shape (X2's `{key,label,source?,parent?}` plus ONE additive field, `chrome`): + [{key, label, source?, chrome: 'main'|'utility'}, …] + + Four rules, each of which has bitten before: + + * ARCHIVED IS INVISIBLE TO EVERYONE ([[module-archive-semantics]]) — not "hidden unless + admin". 18 of the 28 registry rows are archived and none may appear here. + + * `group_only` ROWS ARE EXCLUDED ENTIRELY, and therefore **no `parent` is ever emitted**. + Wave-9 I8 flipped the host nav to a FLAT list: children became top-level and the folder + key was dropped, because a `group_only` key names a FAMILY and has no `PAGE_FUNCS` entry + — rendering a row for it is a white screen one click away (app.py:8182-8187 says exactly + this). `customers` is the only such row. Emitting `parent: 'customers'` on its children + while the parent itself is filtered out would hand the client a DANGLING reference to a + row that is not in the payload, which is worse than a flat list — so the flat list is + what ships, matching the surface a user actually sees today. + + * `nav: False` IS NOT `archived`. `settings`, `analyst` and `dictionary` are LIVE surfaces + the host renders outside the module list (Analyst is the unrestricted landing page, + Settings sits in the chrome). Dropping them would leave the shell unable to reach two of + the nine live surfaces, so they ship flagged `chrome: 'utility'` and the client places + them where the host does. `chrome: 'main'` is the module list. + + * NOT filtered by `ui/session.user_enabled_modules()`. That is a PREFERENCE read from + `royalimports_os/data/store/prefs.json` — a local file on one box. Consuming it here + would put per-user FILESYSTEM state inside a process EXIT-4 requires to be stateless, + and the same user's nav would differ per container. It is a Library setting, not a + permission; the permission gate is `may_open`, and that is what this applies. + """ + out = [] + for m in registry.REGISTRY: + if m.get('archived') or m.get('group_only'): + continue + if not may_open(user, m['key']): + continue + row = {'key': m['key'], 'label': m['label'], + 'chrome': 'main' if m.get('nav', True) else 'utility'} + if m.get('source'): + row['source'] = m['source'] + out.append(row) + return out + + +def landing_page(user): + """Where a restricted user lands: the first MAIN-chrome page in their grant. None for + unrestricted users (the client lands on the Analyst, as the host does). + + Mirrors `ui/session.first_allowed_module` minus its archived last-resort leg — that leg + exists so a misconfigured Streamlit login is not bricked; an API caller gets an explicit + answer instead, and serving an archived key would contradict the rule above. + """ + if allowed_modules(user) is None: + return None + for row in nav_pages(user): + if row['chrome'] == 'main': + return row['key'] + return None diff --git a/royalimports_os/core/registry.py b/royalimports_os/core/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..7ef3fb46d6f242dae6ea2293573879f4c22c350b --- /dev/null +++ b/royalimports_os/core/registry.py @@ -0,0 +1,137 @@ +"""Module registry — the single source of truth for the dashboards. + +`app.py` builds the sidebar nav from this, and decides whether the global Brand (DBA) filter +applies to a page. `validate.py` runs each module's validate() from this. Adding a module = +add one row here + a page function in app.py + a module in modules/. + +Fields: + key stable id (matches the modules/.py file and the app page_ function) + label sidebar/title text (no emojis) + brand True -> DBA-filterable (honours the Fisch/Royal selector) + False -> consolidated/company-level (ignores the brand selector) + hq True -> HQ / company-level (cash flow, balance sheet, AR/AP, consolidated) + validate True -> module exposes validate() and is included in validate.py + parent set -> this row is a SUB-MODULE of : it leaves the Workflows dropdown and + renders as a sub-nav button when its parent (or a sibling) is the active module; a + parent grant covers its sub-modules (owner nav redesign 2026-07-23) + archived True -> RETIRED from the UI for EVERYONE (owner 2026-07-27, supersedes the + 2026-07-23 per-user-restore semantics): no nav entry, no Settings>Modules checkbox, + prewarm and validate.py skip it. Code and data layer stay in the repo but are NOT + maintained — spend no effort on archived modules until the owner un-archives. + 2026-07-23 pass: backorders, pricecomp, o2c, bookings, spend, expenses. + 2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement + + Collections + Analyst/plumbing): products, assortment, financial, management, + pricing, inventory, vendor, warehouse, returns, outreach, complexity, health. + note one-line scope note shown in the UI / docs +""" +# WAVE-9 I8 — `'source'`: which connected system a module's data comes from. +# +# The owner's model: every nav entry is "a piece of database from a source", so the nav shows the +# source beside the name ("Sales · Odoo"). It is a REGISTRY FACT rather than a string in the nav +# code precisely so the nav needs no edit when a second connector lands — a user-created blank +# table simply carries a different source, and a module with no `source` (Metric Dictionary, +# Settings, the Analyst) renders no badge because it is not a database at all. +# See [[connector-onboarding]]: the plug-in seam is datastore.py, and Odoo has no delegated +# OAuth, so the "+ New" connect flow is the credential-form shape, not an OAuth redirect. +REGISTRY = [ + {'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', + 'note': 'Revenue, YoY, seasonality, reps, customers, SKUs — by BU.'}, + # ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer + # dashboard completely, i want to redesign it, exactly with the backend we have". So the + # PAGE goes and the DATA LAYER stays — modules/customers.py still powers the Customer table's + # metrics and still validates against Odoo, which is what the redesign will be built on. + # + # ⚠ NOT 'archived': True. Archived means invisible to everyone, and this row is the parent of + # customer_data + cohort — archiving it would take the whole family out of the nav and strand + # its children. 'group_only' says exactly what is true: this key names a FOLDER, never a + # destination. It has no PAGE_FUNCS entry, the nav never renders it as a leaf, and a stale + # deep link to it redirects to the family's first visible member. + {'key': 'customers', 'label': 'Customers', 'brand': True, 'hq': False, 'validate': True, + 'group_only': True, + 'note': 'Folder for the customer surfaces (Customer table, Cohort). The old Customers ' + 'dashboard was retired for redesign (owner item 14, 2026-07-28); its metric layer ' + '(modules/customers.py) still backs the table and still validates against Odoo.'}, + # LABEL renamed 'Customer List' -> 'Data' (owner 2026-07-25) -> 'Customer' (owner item 5, + # 2026-07-27); KEY renamed 'customer_list' -> 'customer_data' (owner 2026-07-26). The key + # rename is safe ONLY because _LEGACY_KEYS (ui/session.py) migrates grants, Library prefs + # and ?page= deep links ON READ — stored user records are never edited, exactly as + # myday->customer_list was done. The STORE keys are deliberately NOT renamed: + # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them + # would orphan every saved view. Label-only changes need none of that machinery. + {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', + 'note': 'Build, filter and save customer lists: the whole scoped book with per-customer ' + 'metrics; saved lists are visible, editable formulas (Call list and Win-back ship ' + 'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'}, + # Cohort (owner item 6, 2026-07-26): the SAME table as Data over a FIXED set. The + # difference is membership semantics, not layout — a saved view re-populates as the + # data moves, a cohort only changes when a person edits it. validate:False because + # there is no Odoo aggregate to reconcile a hand-picked list against; its metrics are + # the customers module's, already validated there. + {'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', + 'note': 'Hand-curated, unchanging customer lists. Same grid as Data (filter, sort, ' + 'group, custom columns) but the SET is fixed: it changes only when someone ' + 'adds or removes a member. Populated from a view via ... > Add to list.'}, + {'key': 'agent', 'label': 'Agents', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', + 'note': 'Per-agent book: sales (custom dates), returns, top SKUs, full customer list incl. inactive. Returns consolidated.'}, + {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'SKU movers, zombies, velocity, coverage, drawers — by BU (sales-derived).'}, + {'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness — by BU.'}, + {'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'}, + {'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Per-SKU margin/markup/LTM sales + allocated net P&L (channel-rate cost-to-SKU) for pricing decisions.'}, + {'key': 'management', 'label': 'Management P&L', 'brand': False, 'hq': True, 'validate': True, 'archived': True, + 'nav': False, # rendered INSIDE the Financial module (view toggle) — archived WITH financial + 'note': 'Operating-model management income statements (FFS/RI/GD/BU-M) reproduced from raw Odoo, Sep-2024+.'}, + {'key': 'inventory', 'label': 'Inventory', 'brand': False, 'hq': False, 'validate': True, 'archived': True, + 'note': 'On-hand stock is physically consolidated (shared warehouse); not DBA-split.'}, + {'key': 'procurement', 'label': 'Procurement', 'brand': False, 'hq': False, 'validate': True, 'source': 'Odoo', + 'note': 'Biweekly buy list: lead-time-cover trigger, 8-month demand baseline + YTD trend, by supplier.'}, + {'key': 'vendor', 'label': 'Vendors', 'brand': False, 'hq': False, 'validate': False, 'archived': True, + 'note': 'Vendor master data + the SKUs each supplies; multiple-vendor / cheapest-price view.'}, + {'key': 'warehouse', 'label': 'Warehouse', 'brand': False, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Movement & efficiency: on-time ship, Pick-Pack-Ship cycle times, throughput, backlog age, shrinkage. Physical ops (consolidated).'}, + {'key': 'ar', 'label': 'Collections', 'brand': False, 'hq': True, 'validate': True, + 'source': 'Odoo', + 'note': 'Receivables are company-level (consolidated).'}, + {'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True, + 'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'}, + {'key': 'outreach', 'label': 'Outreach', 'brand': True, 'hq': False, 'validate': False, 'archived': True, + 'note': 'Campaign emails to customer segments: templates, suppression, send log, revenue attribution. Sending is admin-gated behind SAFE_MODE.'}, + {'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'}, + {'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE — fix the rule, not the rep.'}, + {'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'}, + {'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True, + 'note': 'Pre-season booking coverage: cumulative booked $ for Aug-Dec delivery vs last year same-week; category TY-vs-LY-as-of-date. The container-program decision, months ahead.'}, + {'key': 'spend', 'label': 'Spend & Payables', 'brand': False, 'hq': True, 'validate': True, 'archived': True, + 'note': 'Vendor-bill side: payment-terms capture (early-pay cash surrendered), duplicate-bill review, the opex spend cube (fragmentation, consolidate+rebid) and freight recovery. Company-level.'}, + {'key': 'complexity', 'label': 'SKU Complexity', 'brand': False, 'hq': False, 'validate': True, 'archived': True, + 'note': 'BCG tail rationalization: every SKU re-costed by ACTIVITY (lines, picks, returns) + 25%/yr carrying; KILL candidates rest on hard math (GM minus carrying), REVIEW on the pooled activity estimate. All-channel.'}, + {'key': 'expenses', 'label': 'Expenses', 'brand': False, 'hq': True, 'validate': True, 'archived': True, + 'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'}, + {'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True, + 'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE ' + '(owner 2026-07-23): every discrepancy / potential-error marker lives here — ' + 'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-' + 'order — so operational workflows stay clean for doing the work.'}, + {'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True, + 'nav': False, # sidebar sentinel button (like Feedback) — not a Dashboards/Workflows entry + 'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) — row kept ONLY so validate.py keeps running the semantic-layer contracts (the Analyst grounding proof). No nav, no page.'}, + {'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False, + 'nav': False, # sidebar sentinel (account group) — owner IA 2026-07-12 + 'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis ' + 'toggle (Orders vs Invoiced) moved here from the sidebar.'}, + {'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False, + 'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py — live LLM cost) + 'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer — answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'}, +] + +BY_KEY = {m['key']: m for m in REGISTRY} + + +def brand_filterable(key) -> bool: + return bool(BY_KEY.get(key, {}).get('brand')) diff --git a/royalimports_os/core/store.py b/royalimports_os/core/store.py new file mode 100644 index 0000000000000000000000000000000000000000..95a564b571f507d6655487ebecc1f4bf90421426 --- /dev/null +++ b/royalimports_os/core/store.py @@ -0,0 +1,307 @@ +"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo — the platform's ONLY writable store +(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags. + +Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The +repo is created private on first write. Reads use a short in-process cache that writes refresh, so +within a running instance reads are consistent with writes; a fresh instance re-fetches the latest. +""" +import io +import os +import json +import threading +import time as _time +from pathlib import Path + +try: + from dotenv import load_dotenv + load_dotenv(Path(__file__).resolve().parents[1] / '.env') +except Exception: + pass + +from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError + +REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data') +_LOCK = threading.RLock() +_CACHE = {} +# --- async flush state (wave-7 W3, 2026-07-28) -------------------------------------------- +# One typed character in a Notes cell used to cost THREE synchronous network calls inside the +# component round-trip (strict re-download + create_repo + full re-upload = one hub commit per +# edit — measured seconds each, and a commit-budget hazard). update(..., flush='async') now +# applies the mutation to the in-process cache synchronously (read-your-writes: every reader +# already consults _CACHE first) and a background worker coalesces uploads. The restart-wipe +# protection is UNCHANGED where it matters: the FIRST update of a key in a fresh process still +# strict-reads (a transient read error aborts the write, never merges into {}); after that the +# cache we last wrote IS the latest known state, so re-downloading it per keystroke bought +# nothing but latency. Known, accepted: with a second concurrent writer (local QA against the +# shared store) the stale window widens from ~one round-trip to ~the flush gap. +_OWNED = {} # name -> True once _CACHE[name] reflects our own last accepted write +_DIRTY = {} # name -> cache is ahead of the hub +_FLUSHERS = {} # name -> live worker thread +_REPO_OK = [False] # create_repo(exist_ok=True) confirmed once per process +_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit +_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget) +_LAST_FLUSH = {} # name -> monotonic time of the last completed upload + + +def _token(): + return os.environ.get('HF_TOKEN') or None + + +def available(): + """True when a token is configured — callers degrade gracefully (read-only UI) without one.""" + return bool(_token()) + + +def _download(name): + """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent. Any + other failure (network, auth, rate-limit, corrupt/partial JSON) RAISES — a transient read + error must never be mistaken for an empty store, or a caller could overwrite real data with + an empty dict (this is what wiped the user registry on restart).""" + try: + path = hf_hub_download(REPO, f'{name}.json', repo_type='dataset', token=_token(), + force_download=True) + with open(path, encoding='utf-8') as f: + return json.load(f) + except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError): + return {} + + +def get(name, fresh=False): + """Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error, hand + back the last known-good cached value if we have one, else {}. Callers that MUTATE the store + must go through update()/_read_strict, which RAISE on such errors so they never clobber real + data with a phantom-empty read.""" + with _LOCK: + if not fresh and name in _CACHE: + return json.loads(json.dumps(_CACHE[name])) # hand back a copy + try: + data = _download(name) + except Exception as e: + try: # lazy import — core stays layer-clean + import harness.telemetry as _tel # noqa: PLC0415 + _tel.error(f'store:get:{name}', e, + fallback='cached' if name in _CACHE else 'empty') + except Exception: + pass + if name in _CACHE: + return json.loads(json.dumps(_CACHE[name])) + return {} + _CACHE[name] = data + return json.loads(json.dumps(data)) + + +def _read_strict(name): + """Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely + absent file). Read-modify-write callers use this so a failed read ABORTS the write instead + of merging changes into an empty dict and overwriting the store.""" + with _LOCK: + data = _download(name) + _CACHE[name] = data + return json.loads(json.dumps(data)) + + +def exists(name): + """True if {name}.json is confirmed present, False if confirmed absent, and True on ANY + uncertainty — so a caller that would seed or overwrite the store never clobbers a registry + we merely failed to reach.""" + try: + files = HfApi(token=_token()).list_repo_files(REPO, repo_type='dataset') + return f'{name}.json' in files + except RepositoryNotFoundError: + return False + except Exception: + return True + + +def _upload(name, data): + """The raw hub write. create_repo is confirmed ONCE per process — it was a full extra API + round-trip on every put(), pure latency after the first call ever succeeded.""" + api = HfApi(token=_token()) + if not _REPO_OK[0]: + api.create_repo(REPO, repo_type='dataset', private=True, exist_ok=True) + _REPO_OK[0] = True + buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8')) + api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=REPO, + repo_type='dataset', commit_message=f'update {name}') + + +def upload_bytes(path_in_repo, data, message=None): + """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents). + + The rest of this module is JSON-per-key; a customer's attachment is neither JSON nor + per-key, so it gets its own path rather than being base64'd into somebody's blob. That + matters for a reason beyond tidiness: the workspace blob is READ ON EVERY PAGE RENDER, and + a few megabytes of base64 inside it would be paid on every keystroke by every user of that + table, not just by whoever opened the document. + + Bytes are never cached in-process: they are large, cold, and fetched one at a time on an + explicit user action. + """ + if not available(): + raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + with _LOCK: + api = HfApi(token=_token()) + if not _REPO_OK[0]: + api.create_repo(REPO, repo_type='dataset', private=True, exist_ok=True) + _REPO_OK[0] = True + api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo, + repo_id=REPO, repo_type='dataset', + commit_message=message or f'add {path_in_repo}') + + +def download_bytes(path_in_repo): + """Raw bytes back, or None when absent/unreachable. + + None rather than an exception: a missing attachment must degrade to "that file is gone", + never to a broken page — the same contract every other read here follows. + """ + if not available(): + return None + try: + local = hf_hub_download(REPO, path_in_repo, repo_type='dataset', token=_token()) + return Path(local).read_bytes() + except (EntryNotFoundError, RepositoryNotFoundError): + return None + except Exception: + return None + + +def delete_path(path_in_repo): + """Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal.""" + if not available(): + return False + try: + HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=REPO, + repo_type='dataset', + commit_message=f'delete {path_in_repo}') + return True + except (EntryNotFoundError, RepositoryNotFoundError): + return True + except Exception: + return False + + +def put(name, data): + """Persist {name}.json (creates the private repo on first write). Updates the in-process cache.""" + if not available(): + raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + with _LOCK: + _upload(name, data) + _CACHE[name] = json.loads(json.dumps(data)) + _OWNED[name] = True + _LAST_FLUSH[name] = _time.monotonic() + return _CACHE[name] + + +def _flush_worker(name): + """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure — + the cache stays the truth and the dirty mark survives, so nothing is ever lost silently.""" + backoff = 2.0 + while True: + _time.sleep(_FLUSH_DELAY) + with _LOCK: + gap = _time.monotonic() - _LAST_FLUSH.get(name, 0.0) + if gap < _FLUSH_MIN_GAP: + wait = _FLUSH_MIN_GAP - gap + elif not _DIRTY.get(name): + _FLUSHERS.pop(name, None) + return + else: + _DIRTY[name] = False + snapshot = json.loads(json.dumps(_CACHE.get(name, {}))) + wait = None + if wait is not None: + _time.sleep(wait) + continue + try: + _upload(name, snapshot) + with _LOCK: + _LAST_FLUSH[name] = _time.monotonic() + backoff = 2.0 + except Exception as e: + with _LOCK: + _DIRTY[name] = True # keep it queued; cache is still the truth + try: + import harness.telemetry as _tel # noqa: PLC0415 + _tel.error(f'store:flush:{name}', e) + except Exception: + pass + _time.sleep(backoff) + backoff = min(backoff * 2, 60.0) + + +def _schedule_flush(name): + """Mark dirty + ensure exactly one live worker for the key. Caller holds _LOCK.""" + _DIRTY[name] = True + t = _FLUSHERS.get(name) + if t is not None and t.is_alive(): + return + t = threading.Thread(target=_flush_worker, args=(name,), daemon=True, + name=f'store-flush-{name}') + _FLUSHERS[name] = t + t.start() + + +def flush(name=None, timeout=30.0): + """Block until pending async writes for `name` (or ALL keys) reach the hub — QA gates and + shutdown hooks use this; the app itself never needs to.""" + deadline = _time.monotonic() + timeout + names = [name] if name else None + while _time.monotonic() < deadline: + with _LOCK: + pending = {k for k, v in _DIRTY.items() if v} | { + k for k, t in _FLUSHERS.items() if t.is_alive()} + if names is not None: + pending &= set(names) + if not pending: + return True + _time.sleep(0.2) + return False + + +def _flush_now_at_exit(): + """Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing + gap — a Space restart must not eat the last seconds of the async window.""" + with _LOCK: + dirty = [k for k, v in _DIRTY.items() if v] + snaps = {k: json.loads(json.dumps(_CACHE.get(k, {}))) for k in dirty} + for k in dirty: + _DIRTY[k] = False + for k, d in snaps.items(): + try: + _upload(k, d) + except Exception: + pass # exiting anyway — nothing left to reschedule + + +import atexit # noqa: E402 (registered after the helpers it needs) +atexit.register(_flush_now_at_exit) + + +def update(name, fn, flush='sync'): + """Read-modify-write. STRICT first read (raises on a transient error rather than starting + from an empty dict — a network blip during 'add user' can never wipe the accounts), apply + fn(data)->data, persist. + + flush='sync' (default) = exactly the historical behavior: fresh strict read + blocking + upload. flush='async' (the table-workspace hot path, wave-7 W3) = the mutation applies to + the in-process cache and returns immediately; a coalescing worker commits to the hub. The + strict read still guards the FIRST async update of a key per process; after we own the + cache, it IS the latest known state and re-downloading it per edit was pure latency.""" + with _LOCK: + if flush == 'async' and _OWNED.get(name) and name in _CACHE: + data = json.loads(json.dumps(_CACHE[name])) + else: + data = _read_strict(name) + result = fn(data) + data = result if result is not None else data + if flush == 'async': + if not available(): + raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + _CACHE[name] = json.loads(json.dumps(data)) + _OWNED[name] = True + _schedule_flush(name) + return data + put(name, data) + return data diff --git a/royalimports_os/core/store_backend.py b/royalimports_os/core/store_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..0de29b97c099dc3dd3fe0bf66727937681045ea9 --- /dev/null +++ b/royalimports_os/core/store_backend.py @@ -0,0 +1,85 @@ +"""core/store_backend.py — which store is active: `STORE_BACKEND=hf|pg` (X4 / EXIT-2b). + + import core.store_backend as store # new code + store.get('users') # goes to whichever backend is selected + +⛔ WHAT THIS DOES **NOT** DO, stated first because assuming otherwise would be a silent data +split. Flipping `STORE_BACKEND=pg` does NOT redirect the ~40 existing callers that say +`import core.store as store` — they are bound to the HF module directly. Rewiring them is task +**C-4** (the HF→Postgres cutover with its dual-read window), and C-4 cannot start until the +database exists (owner blocker **B-3**). Until then this selector is the SEAM, exercised by new +code and by the gate, and the default is `hf`, which is exactly today's behaviour. + +Doing it any other way would be worse. The tempting shortcut — have `core/store.py` itself +delegate — would mean every existing caller silently changes backend the moment an env var is set +in some shell, with no dual-read window and no way to compare the two stores' contents first. +"One store" is the goal; getting there through a flag nobody audited would produce two. + +THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise. An unrecognised value is a +configuration error and RAISES rather than falling back — a typo'd backend name that quietly +served the old store would be discovered by data going missing. +""" +import os + +import core.store as _hf + +_NAMES = ('available', 'get', 'exists', 'put', 'update', 'upload_bytes', 'download_bytes', + 'delete_path', 'flush') + + +def name(): + """The selected backend name, validated. `hf` unless explicitly told otherwise.""" + raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower() + if raw not in ('hf', 'pg'): + raise RuntimeError( + f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store, the " + f"default) or 'pg' (Postgres, needs DATABASE_URL). Refusing to guess: a typo that " + f"silently served the other store is how data ends up in two places.") + return raw + + +def active(): + """The backend MODULE. Resolved per call, so a test can flip the env var without a reimport.""" + if name() == 'pg': + import core.store_pg as _pg + return _pg + return _hf + + +# --- the nine delegates. Written out rather than generated with setattr so that reading this file +# tells you the whole interface, and so a new store operation cannot be added to core/store.py +# without something here failing to match (verify_store_pg.py compares the signatures). +def available(): + return active().available() + + +def get(name_, fresh=False): + return active().get(name_, fresh=fresh) + + +def exists(name_): + return active().exists(name_) + + +def put(name_, data): + return active().put(name_, data) + + +def update(name_, fn, flush='sync'): + return active().update(name_, fn, flush=flush) + + +def upload_bytes(path_in_repo, data, message=None): + return active().upload_bytes(path_in_repo, data, message=message) + + +def download_bytes(path_in_repo): + return active().download_bytes(path_in_repo) + + +def delete_path(path_in_repo): + return active().delete_path(path_in_repo) + + +def flush(name_=None, timeout=30.0): + return active().flush(name_, timeout=timeout) diff --git a/royalimports_os/core/store_pg.py b/royalimports_os/core/store_pg.py new file mode 100644 index 0000000000000000000000000000000000000000..0710853afdc7979caa7979d61eeec0e0c065af2c --- /dev/null +++ b/royalimports_os/core/store_pg.py @@ -0,0 +1,249 @@ +"""core/store_pg.py — `core/store.py`'s interface, backed by Postgres (X4 / EXIT-2b, 2026-07-30). + +THE INTERFACE IS THE CONTRACT, and it is `core/store.py`'s nine functions: + + available() · get(name, fresh=False) · exists(name) · put(name, data) + update(name, fn, flush='sync') · upload_bytes(path, data, message=None) + download_bytes(path) · delete_path(path) · flush(name=None, timeout=30.0) + +Same names, same signatures, same return types, same failure semantics — so switching backends is +an env var (`STORE_BACKEND=hf|pg`) and not a rewrite of every caller. `core/store_backend.py` +does the selection; nothing above it needs to know which store it is talking to. + +⛔ NOT VERIFIED AGAINST A REAL SERVER — owner blocker **B-3** (choose + provision managed +Postgres, US-region, co-located with Ashburn). `verify_store_pg.py` is written and it SKIPS LOUDLY +without `DATABASE_URL`; it is not reported as passing, and this module is not the default backend. +The schema is `harness/pg/schema.sql`. Sequenced around the blocker, never stubbed past it. + +WHAT POSTGRES FIXES, precisely (C1c — these are the reasons, not a preference): + * **`get()` is cache-first on the HF store**, so a write from another process is invisible to a + running app until restart. Observed live in wave 11. Here a read is a SELECT: cross-process + coherence is the default rather than a thing to remember. + * **`update()` was read-modify-write with no concurrency control** — two writers raced and the + loser vanished silently. Here it runs inside ONE transaction with `SELECT … FOR UPDATE`, so + the second writer waits and then sees the first writer's value. + * **A write was an HTTP commit** against a 256-commits/hour repo budget, which is why the hot + path needed a coalescing async flusher at all. A Postgres write is a write; `flush='async'` + stays in the signature and becomes a no-op, because there is nothing to coalesce. + +⚠ ONE DELIBERATE DIFFERENCE FROM THE HF BACKEND, and it is a fix. `store.get()` swallows transient +read failures and hands back a cached-or-empty dict, because a display read must not break a page. +That leniency is exactly what wiped the user registry once (an empty read merged into a write), and +`_read_strict` exists to opt out of it. Postgres has no equivalent "the network blipped" state +worth hiding: a connection failure RAISES from `get()` here. Callers that must not break on a read +should catch it — silently returning `{}` from a database that is simply unreachable is how an +empty dict gets written back over real data. +""" +import json +import os +import threading + +_URL_ENV = 'DATABASE_URL' +_LOCK = threading.RLock() +_POOL = {'pool': None, 'url': None, 'schema': None} + + +def _url(): + return os.environ.get(_URL_ENV) or None + + +def _schema_for(tenant_slug=None): + """`t_` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier). + + Mirrors `control.provision_tenant_schema` exactly — the two MUST agree or a write lands in a + schema the DDL never created. `AIOS_TENANT` names the tenant this process serves; it defaults + to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for` + follows for the DuckDB filename. + """ + slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower() + return 't_' + slug.replace('-', '_') + + +def _pool(): + """The psycopg connection pool, opened once per process. + + psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency + to be installed at all, which is what keeps this file safe to commit before B-3 unblocks. + """ + with _LOCK: + url = _url() + if not url: + raise RuntimeError( + f'{_URL_ENV} is not set — the Postgres store backend has nothing to connect to.') + if _POOL['pool'] is not None and _POOL['url'] == url: + return _POOL['pool'] + try: + from psycopg_pool import ConnectionPool + except ImportError as e: # pragma: no cover - depends on B-3 + raise RuntimeError( + 'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in ' + 'requirements.txt until the database exists (owner blocker B-3), so that the ' + 'default hf backend installs nothing it does not use.') from e + if _POOL['pool'] is not None: + try: + _POOL['pool'].close() + except Exception: + pass + # min_size=0 so a process that never touches the store opens no connection at all — the + # shared API serves plenty of requests (health, static, a cached payload) that never read. + _POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0) + _POOL['url'] = url + return _POOL['pool'] + + +def available(): + """True when a connection can actually be MADE, not merely when a URL is configured. + + `core.store.available()` only checks for a token, which is all it can cheaply do. Here the + honest answer needs a round-trip, so the result is memoised per process: callers use this to + decide whether to degrade the UI, and a `SELECT 1` per render would be absurd. + """ + if not _url(): + return False + if _POOL.get('ok'): + return True + try: + with _pool().connection() as con: + con.execute('SELECT 1') + _POOL['ok'] = True + return True + except Exception: + return False + + +def _table(kind, tenant_slug=None): + """`"t_slug"."store_kv"` — the qualified table, safely quoted.""" + from psycopg import sql + return sql.SQL('{}.{}').format(sql.Identifier(_schema_for(tenant_slug)), + sql.Identifier(kind)) + + +def _q(template, kind, tenant_slug=None): + """Compose `template` with `{tbl}` = the QUALIFIED table and `{bare}` = the table name alone. + + ⚠ THE TWO ARE NOT INTERCHANGEABLE, and getting it wrong is a runtime syntax error nothing + local would catch (no database here — owner blocker B-3). Inside `ON CONFLICT … DO UPDATE`, + Postgres refers to the conflicting row by the table's ALIAS, which defaults to the bare + table name: `store_kv.rev` is valid, `t_royal_imports.store_kv.rev` is not. So the FROM + position takes `{tbl}` and the DO-UPDATE position takes `{bare}`. + + Identifiers go through `psycopg.sql.Identifier`, so the schema name — derived from a tenant + slug — cannot be an injection vector even though it is interpolated. The slug is ALSO + regex-constrained at its source (`control.tenants`), which is the belt to this brace. + """ + from psycopg import sql + return sql.SQL(template).format(tbl=_table(kind, tenant_slug), + bare=sql.Identifier(kind)) + + +def get(name, fresh=False, tenant_slug=None): + """The value stored under `name`, or `{}` when the key is genuinely absent. + + `fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and + there is no cache here — every read is a SELECT. Keeping the parameter means callers do not + branch on the backend. + """ + with _pool().connection() as con: + row = con.execute( + _q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), + (str(name),)).fetchone() + return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {}) + + +def exists(name, tenant_slug=None): + """True/False — and unlike the HF backend this is NEVER "True on uncertainty". + + `core.store.exists` returns True on any error so a caller that would seed the store cannot + clobber a registry it merely failed to reach. That guard exists because a FILE HOST cannot + distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an + unreachable server raises instead of answering. + """ + with _pool().connection() as con: + row = con.execute( + _q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), + (str(name),)).fetchone() + return row is not None + + +def put(name, data, tenant_slug=None): + with _pool().connection() as con: + con.execute( + _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' + 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' + 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug), + (str(name), json.dumps(data))) + return data + + +def update(name, fn, flush='sync', tenant_slug=None): + """Read-modify-write inside ONE transaction, with the row LOCKED for the duration. + + This is the operation the HF backend could not make safe. There, `update` read over HTTP, + applied `fn`, and uploaded — so two concurrent writers both read the old value and the second + upload silently discarded the first writer's change. `SELECT … FOR UPDATE` makes the second + writer WAIT and then apply `fn` to the first writer's result, which is what "read-modify-write" + was always supposed to mean. + + `flush` is accepted and ignored (see the module docstring): there is no commit budget to + coalesce against, so 'async' has nothing to defer. + """ + with _pool().connection() as con: + with con.transaction(): + row = con.execute( + _q('SELECT value FROM {tbl} WHERE key = %s FOR UPDATE', + 'store_kv', tenant_slug), + (str(name),)).fetchone() + data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {}) + result = fn(data) + data = result if result is not None else data + con.execute( + _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' + 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' + 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug), + (str(name), json.dumps(data))) + return data + + +def upload_bytes(path_in_repo, data, message=None, tenant_slug=None): + with _pool().connection() as con: + con.execute( + _q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) ' + 'ON CONFLICT (path) DO UPDATE SET bytes = EXCLUDED.bytes, ' + 'message = EXCLUDED.message, updated_at = now()', 'store_blobs', tenant_slug), + (str(path_in_repo), bytes(data), message)) + + +def download_bytes(path_in_repo, tenant_slug=None): + """Raw bytes, or None when absent — the same contract as the HF backend: a missing attachment + degrades to "that file is gone", never to a broken page.""" + with _pool().connection() as con: + row = con.execute( + _q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug), + (str(path_in_repo),)).fetchone() + return bytes(row[0]) if row else None + + +def delete_path(path_in_repo, tenant_slug=None): + """Absent is SUCCESS — deleting what is already gone is the goal (HF backend's contract).""" + with _pool().connection() as con: + con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug), + (str(path_in_repo),)) + return True + + +def flush(name=None, timeout=30.0): + """A no-op that returns True: every write above is already committed when it returns. Kept in + the interface because QA gates and shutdown hooks call it and must not branch on the backend.""" + return True + + +def close(): + """Release the pool (tests, and a process that is shutting down cleanly).""" + with _LOCK: + if _POOL['pool'] is not None: + try: + _POOL['pool'].close() + finally: + _POOL['pool'] = None + _POOL['url'] = None + _POOL.pop('ok', None) diff --git a/royalimports_os/core/table_store.py b/royalimports_os/core/table_store.py new file mode 100644 index 0000000000000000000000000000000000000000..5146e0d63890b0b4eca27216667d6aa35b63e35e --- /dev/null +++ b/royalimports_os/core/table_store.py @@ -0,0 +1,281 @@ +"""The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory. + +One durable store key holds one table OBJECT's per-user Airtable-style state: + + {username: {'views': {view_id: SavedView}, + 'fields': {field_key: Field}, # notes + custom_ + measure_ strata + 'overlays': {str(pid): {field_key: value}}}} + +`make(table_key)` returns the six operations a table page's host loop needs, closed over that +key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are +these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table +pattern to a new object is a registry row + a config, not a copy of the store plumbing +(owner directive: the table-page factory). + +A LIST (membership/formula semantics) is deliberately a different store from a VIEW +(presentation/query state) — see modules/customer_data.py's customer_lists key. +""" +import core.store as store + +#: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but +#: their creator live here instead of in a personal workspace, under a key that cannot collide +#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared` +#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from +#: here, and a view shared is removed from its creator's workspace. Two homes would mean two +#: divergent copies the moment either was edited. +SHARED_KEY = '__shared__' + + +def _may_see(view, viewer, is_admin=False): + """Visibility for ONE shared view, fail-closed. + + 'collaborative' = everyone who can already open the module (the caller has gated that). + 'users' = the named users, plus the creator, plus admins — an admin who could not + see a view could not administer it either. + Anything unrecognised returns False rather than defaulting open: an unreadable permission + must never widen access ([[aios-permissioning]] — no fail-open defaults). + """ + if not isinstance(view, dict): + return False + if view.get('createdBy') == viewer or is_admin: + return True + perms = view.get('permissions') or {} + edit = perms.get('edit') + if edit == 'collaborative': + return True + if edit == 'users': + return viewer in set(perms.get('users') or ()) + return False # 'personal', absent, or junk + + +def _may_edit(view, viewer, is_admin=False): + """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who + can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a + separate function so they can diverge (a future read-only share) without hunting callers.""" + return _may_see(view, viewer, is_admin) + + +def _may_administer(view, viewer, is_admin=False): + """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY. + + Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they + could grant themselves sole ownership of somebody else's view, or quietly widen a + users-scoped view to everyone — the classic privilege-escalation-by-edit hole. + """ + if not isinstance(view, dict): + return False + return bool(is_admin) or view.get('createdBy') == viewer + + +def is_shared(view): + """A view belongs in the shared bucket when its permissions reach beyond its creator.""" + return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users') + + +class TableStore: + """The six store operations for one table object's workspace, closed over its store key.""" + + def __init__(self, table_key): + self.table_key = table_key + + # ---------------------------------------------------------------- read + def workspace(self, username): + """One user's durable workspace: always the full three-strata shape.""" + try: + ws = (store.get(self.table_key) or {}).get(username, {}) or {} + except Exception: + ws = {} + return { + 'views': dict(ws.get('views') or {}), + 'fields': dict(ws.get('fields') or {}), + 'overlays': dict(ws.get('overlays') or {}), + # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH + # stratum rather than a key on each item — see aios_grid.clean_folders for why + # (a cohort lives in another store, and filing is an organising act, not part of + # what a view is). Absent for every workspace saved before this wave, which is + # exactly "no folders yet". + 'folders': dict(ws.get('folders') or {}), + 'itemFolders': dict(ws.get('itemFolders') or {}), + } + + # ---------------------------------------------------------------- write + def _update(self, username, change): + def _up(data): + ws = data.setdefault(username, {}) + ws.setdefault('views', {}) + ws.setdefault('fields', {}) + ws.setdefault('overlays', {}) + ws.setdefault('folders', {}) + ws.setdefault('itemFolders', {}) + change(ws) + return data + # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak, + # column note and typed overlay cell lands here inside the component round-trip, and + # the historical synchronous hub commit cost seconds per edit. The mutation applies to + # the in-process cache (read-your-writes for every subsequent render); the hub write + # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'. + store.update(self.table_key, _up, flush='async') + + def save_folders(self, username, folders, item_folders): + """Replace the folder stratum wholesale (wave-8 I11). + + Wholesale rather than per-folder because the caller has ALREADY validated the complete + picture through aios_grid.clean_folders / clean_item_folders, and those two are + interdependent: a placement is only legal while its folder exists, so committing them + separately would leave a window where a reader sees an item filed into a folder that is + not there yet. One write, one consistent state. + """ + def _set(ws): + ws['folders'] = dict(folders or {}) + ws['itemFolders'] = dict(item_folders or {}) + self._update(username, _set) + + def shared_views(self, viewer, is_admin=False): + """Every SHARED view this viewer may see, by id (wave-9 I17). + + Read-only and independent of the viewer's own workspace: the caller merges. Returns + only what `_may_see` allows, so a caller cannot accidentally render somebody else's + personal view by forgetting to filter. + """ + try: + bucket = ((store.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {} + except Exception: + return {} + return {vid: dict(v) for vid, v in bucket.items() + if _may_see(v, viewer, is_admin)} + + def shared_view(self, view_id): + """One shared view RAW — no visibility filter. For authorisation decisions only: a + caller must know a view exists and who owns it before it can decide whether the actor + may touch it. Never hand the result to a renderer without checking `_may_see`.""" + try: + return ((store.get(self.table_key) or {}).get(SHARED_KEY) or {} + ).get('views', {}).get(str(view_id)) + except Exception: + return None + + def save_view(self, username, view, shared=None): + """Upsert a SavedView into its ONE home — personal workspace or the shared bucket. + + `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home + it lands in, the view is REMOVED from the other, so a view can never exist as two + copies that diverge on the next edit. + + ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer + moves data and does not know who is asking. `_cl_handle_one` is the wall. + """ + view_id = str((view or {}).get('id') or '').strip() + if not view_id: + raise ValueError('view id is required') + if username == SHARED_KEY: + raise ValueError('reserved username') + to_shared = is_shared(view) if shared is None else bool(shared) + payload = dict(view) + + def _up(data): + if to_shared: + bucket = data.setdefault(SHARED_KEY, {}) + bucket.setdefault('views', {})[view_id] = payload + # it may have lived in the creator's workspace before being shared + owner = data.get(payload.get('createdBy') or username) or {} + (owner.get('views') or {}).pop(view_id, None) + else: + ws = data.setdefault(username, {}) + ws.setdefault('views', {})[view_id] = payload + (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None) + return data + + store.update(self.table_key, _up, flush='async') + + def delete_view(self, username, view_id): + """Delete a custom/list view override. The system all-rows view is guarded by caller. + + Removes from BOTH homes: the caller has already authorised the delete, and leaving a + stale copy in the other bucket would resurrect the view on the next read. + """ + vid = str(view_id) + + def _up(data): + (data.get(username, {}).get('views') or {}).pop(vid, None) + (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None) + return data + + store.update(self.table_key, _up, flush='async') + + def save_field(self, username, field): + """Persist a column note or a user-created (custom_/measure_) field definition.""" + key = str((field or {}).get('key') or '').strip() + if not key: + raise ValueError('field key is required') + + def _save(ws): + # A cleared note on an immutable source field returns to the canonical schema + # instead of leaving a meaningless override row. Custom fields remain even with + # an empty note — and so does a PRESET field carrying a measure-window override + # (wave-2 item 8) or a DISPLAY-format override (wave-5 item 10): the note may be + # empty but the period/format is real user state. + if (not field.get('custom') and not str(field.get('note') or '').strip() + and not isinstance(field.get('measure'), dict) + and not isinstance(field.get('format'), dict)): + ws['fields'].pop(key, None) + else: + ws['fields'][key] = dict(field) + + self._update(username, _save) + + def duplicate_field(self, username, source_key, new_key, field): + """Clone a user-created field in ONE store transaction (wave-5 item 1): the new + definition plus — for `custom_` overlay sources only — every stored cell value under + the source key. One transaction, because a def without its values (or values without a + def) is exactly the orphan state delete_field exists to prevent, in reverse. + The caller validated both keys (same created stratum) and stamped the clone's + createdBy; this layer only moves data.""" + source_key = str(source_key or '').strip() + new_key = str(new_key or '').strip() + if not source_key or not new_key or source_key == new_key: + raise ValueError('duplicate_field needs two distinct keys') + + def _dup(ws): + ws['fields'][new_key] = dict(field) + if source_key.startswith('custom_'): + for row in ws['overlays'].values(): + if isinstance(row, dict) and source_key in row: + row[new_key] = row[source_key] + + self._update(username, _dup) + + def delete_field(self, username, key): + """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27). + + Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula + columns — the caller enforces the prefix). The stored overlay VALUES for the key are + scrubbed with it: a deleted column's cells must not linger as orphan data that would + silently resurface if the key were ever reused. Views referencing the key self-heal on + their next autosave (an unknown colId is dropped) — the rule every stale key rides. + """ + key = str(key or '').strip() + if not key: + return + + def _drop(ws): + ws['fields'].pop(key, None) + for row in ws['overlays'].values(): + if isinstance(row, dict): + row.pop(key, None) + + self._update(username, _drop) + + def patch_overlay(self, username, pid, updates): + """Patch only the external editable stratum; never writes to the source system.""" + clean = dict(updates or {}) + if not clean: + return + + def _patch(ws): + ws['overlays'].setdefault(str(int(pid)), {}).update(clean) + + self._update(username, _patch) + + +def make(table_key): + return TableStore(table_key) diff --git a/royalimports_os/core/user_tables.py b/royalimports_os/core/user_tables.py new file mode 100644 index 0000000000000000000000000000000000000000..499c6b48863be013518092d5d93827325ce80331 --- /dev/null +++ b/royalimports_os/core/user_tables.py @@ -0,0 +1,179 @@ +"""USER-CREATED TABLES (wave-9 I8 / contract C6) — databases that are not a connector. + +The owner's "+ New" flow has two branches: connect a SOURCE (Odoo today) or *"create a completely +new database blank, where this time the user can actually insert row that they want."* The first +branch is the connector seam and lives in `harness/datastore.py`; THIS module is the second. + +WHY A SEPARATE STORE AND NOT A REGISTRY ROW. `core/registry.py` is a static Python list read at +import time — it is the catalogue of modules the PRODUCT ships, and a tenant cannot append to it +at runtime without editing code. A user-created table is tenant data, so it lives in the tenant +store and is MERGED into the nav beside the registry rows. That also keeps the archived/allowed +machinery honest: a user table is never "archived", it is deleted, and it is never in +`allowed_modules` because it is not a module. + +SHAPE (one store key, `user_tables`): + + {table_key: {'key', 'label', 'source', 'createdBy', 'created', 'fields': [...], 'rows': {...}}} + +`source` is deliberately NOT 'Odoo' — the nav badge is generated from it, so a blank table reads +"Blank" and can never be mistaken for connected data. `rows` is `{row_id: {field_key: value}}`, +which is the same overlay shape `TableStore` already uses, so the grid's write path needs no new +storage concept. + +⚠ ROW WRITES ARE ONLY EVER LEGAL HERE. An Odoo-sourced table is READ-ONLY at the source and +`_cl_handle_one`'s row_add / row_delete refuse unless `is_user_table(key)` — a user must not be +able to invent a customer in Odoo by typing into a grid. +""" +import datetime as _dt +import re + +import core.store as store + +STORE_KEY = 'user_tables' +MAX_TABLES = 40 +MAX_LABEL = 60 +MAX_ROWS = 5000 +#: the badge the nav paints for a table with no connector behind it +BLANK_SOURCE = 'Blank' +#: every user table's key is prefixed, so it can never collide with a registry module key +KEY_PREFIX = 'ut_' + + +def _slug(label): + s = re.sub(r'[^a-z0-9]+', '_', str(label or '').strip().lower()).strip('_') + return (s or 'table')[:40] + + +def all_tables(): + """{key: definition} — every user-created table in this tenant.""" + try: + return dict(store.get(STORE_KEY) or {}) + except Exception: + return {} + + +def get(table_key): + return all_tables().get(str(table_key)) + + +def is_user_table(table_key): + """THE permission predicate for row writes. Cheap and total: a key that is not in this store + is not a user table, so it cannot accept invented rows.""" + return str(table_key or '').startswith(KEY_PREFIX) and get(table_key) is not None + + +def create(label, username, fields=None): + """Create a blank table. Returns its key, or None if refused. + + Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) — + the caller is a UI form and a refusal is a message, not an exception. + """ + label = str(label or '').strip()[:MAX_LABEL] + if not label: + return None + existing = all_tables() + if len(existing) >= MAX_TABLES: + return None + key = KEY_PREFIX + _slug(label) + if key in existing: # a second table of the same name gets a suffix + for n in range(2, 50): + if f'{key}_{n}' not in existing: + key = f'{key}_{n}' + break + else: + return None + # A blank table still needs ONE column to be a table at all: a row with no fields cannot be + # displayed, edited or identified. 'name' is the identity column, the same role `customer` + # plays in the Customer table. + defn = { + 'key': key, + 'label': label, + 'source': BLANK_SOURCE, + 'createdBy': username, + 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), + 'fields': list(fields or [{'key': 'name', 'label': 'Name', 'type': 'text', + 'source': 'overlay', 'default': True}]), + 'rows': {}, + } + + def _add(cur): + cur[key] = defn + return cur + + store.update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it + return key + + +def delete(table_key): + def _drop(cur): + cur.pop(str(table_key), None) + return cur + store.update(STORE_KEY, _drop, flush='sync') + + +def add_row(table_key, values=None, username=None): + """Append one row. Returns the new row id, or None if refused.""" + if not is_user_table(table_key): + return None # ⚠ never on a connector-backed table + defn = get(table_key) or {} + rows = defn.get('rows') or {} + if len(rows) >= MAX_ROWS: + return None + rid = str(max((int(r) for r in rows if str(r).isdigit()), default=0) + 1) + valid = {f['key'] for f in (defn.get('fields') or [])} + clean = {k: v for k, v in (values or {}).items() if k in valid} + + def _add(cur): + t = cur.get(str(table_key)) + if t is not None: + t.setdefault('rows', {})[rid] = clean + return cur + + store.update(STORE_KEY, _add, flush='async') + return rid + + +def delete_row(table_key, row_id): + if not is_user_table(table_key): + return False + + def _drop(cur): + t = cur.get(str(table_key)) + if t is not None: + (t.get('rows') or {}).pop(str(row_id), None) + return cur + + store.update(STORE_KEY, _drop, flush='async') + return True + + +def may_open(table_key, viewer, is_admin=False): + """FAIL-CLOSED visibility: the creator, or an admin. Nothing else. + + A user table cannot be gated by `allowed_modules` — a module grant written last month cannot + describe a table created this morning — so it needs its own rule, and the safe rule is the + narrow one. Tables are stored tenant-wide under one key, so without this ANY user (including + a BU-scoped sales agent) could open a table somebody else created just by knowing its key. + Sharing a user table with named colleagues is a follow-on, and it should reuse the shared- + VIEW vocabulary rather than inventing a second one. + """ + t = get(table_key) + if not t: + return False + return bool(is_admin) or t.get('createdBy') == viewer + + +def nav_entries(viewer=None, is_admin=False): + """Registry-SHAPED dicts so the nav can render user tables beside real modules with no + special-casing: the same keys the flat nav reads (`key`, `label`, `source`). + + Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse. + """ + out = [] + for key, t in sorted(all_tables().items(), key=lambda kv: (kv[1].get('label') or '').lower()): + if viewer is not None and not may_open(key, viewer, is_admin): + continue + out.append({'key': key, 'label': t.get('label') or key, + 'source': t.get('source') or BLANK_SOURCE, + 'user_table': True, 'nav': True, 'validate': False}) + return out diff --git a/royalimports_os/core/users.py b/royalimports_os/core/users.py new file mode 100644 index 0000000000000000000000000000000000000000..c2a380a8888cfe85cd1ea47c3771cd72779d9aad --- /dev/null +++ b/royalimports_os/core/users.py @@ -0,0 +1,238 @@ +"""Per-user accounts for the platform, persisted in the HF Dataset store (users.json). + +Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) — never stored or logged in plaintext. +A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create +users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable. + +Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives +allowed_bus() — the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch). +""" +import os +import hmac +import hashlib +import secrets + +import core.store as store + +BU_LABELS = {5: 'Fisch', 6: 'Royal'} +_ITER = 200_000 + + +def _hash(pw, salt): + return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex() + + +def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None): + salt = secrets.token_hex(16) + return {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role, + 'bus': bus, 'active': active, 'modules': modules, + 'agent': agent or None, 'email': email or None} + + +def registry(): + return store.get('users') + + +def ensure_bootstrap(): + """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet). + Idempotent; no-op if the store is unavailable (the app then falls back to the master-password + path in verify()). + + Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users') + is definitively False. A transient read failure at startup used to return {} and make this + re-seed just {admin} over the real accounts — that is the bug that wiped users on restart.""" + if not store.available(): + return + if store.exists('users'): # present, or uncertain -> never clobber + return + try: + reg = store.get('users', fresh=True) + except Exception: + return + if reg: + return + master = os.environ.get('APP_PASSWORD', '') + if not master: + return + try: + store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')}) + except Exception: + pass + + +def _public(username, u): + return {'username': username, 'name': u.get('name', username), + 'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'), + 'modules': u.get('modules', 'all'), + 'agent': u.get('agent'), 'email': u.get('email'), + 'epoch': int(u.get('epoch') or 0)} + + +# ------------------------------------------------------------------ session revocation (X3) +# The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to +# delete from, so "log this user out everywhere" needs a number that lives with the account. The +# cookie carries the epoch it was minted under; bumping the account's epoch makes every +# outstanding cookie for that user fail verification on its next use. Absent == 0, so every +# record written before this wave is valid without a migration. +def epoch(username): + """The current session epoch for `username`. None when there is no such account. + + None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" — + which the session verifier must treat as a reason to refuse, not as a default to compare + against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles + that case explicitly rather than inventing an epoch for it here.) + """ + username = (username or '').strip().lower() + try: + u = (store.get('users') or {}).get(username) + except Exception: + return None + return int((u or {}).get('epoch') or 0) if u else None + + +def bump_epoch(username): + """Revoke every outstanding API session for this account.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def verify(username, pw): + """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the + 'admin' login even if the store is unreachable, so the owner is never locked out.""" + username = (username or '').strip().lower() + if not username or not pw: + return None + master = os.environ.get('APP_PASSWORD', '') + try: + # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once + reg = store.get('users', fresh=True) + except Exception: + reg = {} + u = reg.get(username) + if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']): + return _public(username, u) + # emergency master: admin + APP_PASSWORD always works (covers first run / store outage) + if username == 'admin' and master and hmac.compare_digest(str(pw), master): + # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie + # the API mints from this dict AGREES with the stored account. + # + # This is not what stops the emergency lockout — `deps._user_for`'s master fallback does + # that, and a negative control confirmed the lockout is gone with or without this line. + # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the + # record falls through to that master fallback, which hands back a SYNTHETIC identity + # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would + # therefore be silently WIDENED to consolidated, all-module access for the life of that + # session. Matching the epoch means the record branch wins and the account's real scope + # applies, leaving the master fallback as the true last resort it is meant to be. + # 0 when the store is unreachable, which is the case this branch was written for. + return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all', + 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)} + return None + + +def create_user(username, pw, name, role='user', bus='all', modules='all', + agent=None, email=None): + """Create — or, from app.py's dialog, OVERWRITE — an account. + + ⛔ X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh + record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset + the epoch to 0, and every cookie minted before that account's last password rotation started + verifying again — a silent un-revocation. `app.py`'s "Add / update a user" calls this function + for BOTH add and update, so the hole was reachable from the shipped UI. + + Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so + the carry-and-bump lives here rather than in each caller: an overwrite is at least as + session-invalidating as a password change, and it usually IS one. + """ + username = (username or '').strip().lower() + if not username or not pw: + raise ValueError('username and password are required') + + def _add(reg): + prior = reg.get(username) or {} + rec = _record(pw, name or username, role, bus, modules=modules, + agent=agent, email=email) + if prior: + rec['epoch'] = int(prior.get('epoch') or 0) + 1 + reg[username] = rec + return reg + store.update('users', _add) + + +def set_password(username, pw): + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + u['salt'] = secrets.token_hex(16) + u['hash'] = _hash(pw, u['salt']) + # X3: a password change revokes every outstanding API session for the account. Bumped + # INSIDE the same read-modify-write as the hash so the two can never disagree — a + # separate update() could rotate the password and leave old cookies live if the second + # write failed. + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def set_active(username, active): + username = (username or '').strip().lower() + + def _set(reg): + if username in reg: + reg[username]['active'] = bool(active) + # X3: DEACTIVATION must kill live sessions, not just future logins — otherwise a + # disabled account keeps working until its cookie expires. Bumped on reactivation too: + # cheap, and it means a re-enabled account never resurrects a stale cookie. + reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def set_access(username, role=None, bus=None, modules=None, agent=None, email=None, + name=None): + """Update access fields. agent/email: pass '' to clear, None to leave unchanged — + the user↔agent link scopes the Customer List page / digests to that agent's book. + + `name` follows the same None-means-unchanged idiom. It is here because it had no setter at all: + a display name could previously only be changed by re-creating the record through + `create_user`, i.e. by also resetting the password (and, before the fix above, the session + epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + if name is not None: + u['name'] = name + if role is not None: + u['role'] = role + if bus is not None: + u['bus'] = bus + if modules is not None: + u['modules'] = modules + if agent is not None: + u['agent'] = agent or None + if email is not None: + u['email'] = email or None + return reg + store.update('users', _set) + + +def allowed_bus_labels(user): + """BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no + 'All', so the other BU is never reachable); multiple -> All + each.""" + bus = (user or {}).get('bus', 'all') + if bus == 'all': + return ['All', 'Fisch', 'Royal'] + labels = [BU_LABELS[b] for b in bus if b in BU_LABELS] + if not labels: + return ['All', 'Fisch', 'Royal'] + return (['All'] + labels) if len(labels) > 1 else labels diff --git a/royalimports_os/evals/analyst_golden.yml b/royalimports_os/evals/analyst_golden.yml new file mode 100644 index 0000000000000000000000000000000000000000..a07bd1bb21f634affc9e55278365515601384b3d --- /dev/null +++ b/royalimports_os/evals/analyst_golden.yml @@ -0,0 +1,104 @@ +# Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the +# Jul-15 Odoo mass-recompute + hard-delete reconciliation — closed windows DO drift when docs +# are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances). +# Original header: truths computed from the PARITY-PROVEN +# semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot +# break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use +# assertions) — the validate() culture applied to AI answers. Run: harness/evals.py. +# expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive). +# expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools). +# expect.not_tools: tool names that must NOT appear (the over-refusal guard — an answerable ask +# must never end in report_gap; a gap ask must never end in a forced wrong query). +items: + - id: rev_h1 + ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?" + expect: {value: 3501508.33, rel: 0.001, tools: [run_semantic_query]} + - id: rev_h1_fisch + ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?" + expect: {value: 2494244.70, rel: 0.001, tools: [run_semantic_query]} + - id: rev_h1_royal + ask: "And Royal's revenue for the same H1 2026 window?" + expect: {value: 1007263.63, rel: 0.001, tools: [run_semantic_query]} + - id: rev_june + ask: "Revenue for June 2026?" + expect: {value: 535046.06, rel: 0.001, tools: [run_semantic_query]} + - id: margin_h1 + ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?" + expect: {value: 2067509.45, rel: 0.001, tools: [run_semantic_query]} + - id: margin_pct_h1 + ask: "What gross margin percentage did we run in H1 2026?" + expect: {value: 0.59, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]} + - id: orders_h1 + ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?" + expect: {value: 4989, abs: 1, tools: [run_semantic_query]} + - id: customers_h1 + ask: "How many distinct customers bought from us in H1 2026?" + expect: {value: 1037, abs: 1, tools: [run_semantic_query]} + - id: top_customer + ask: "Who was our biggest customer by revenue in H1 2026?" + expect: {contains: ["Poppy Flowers"], tools: [run_semantic_query], not_tools: [report_gap]} + - id: opex_h1 + ask: "Total operating expense on the GL for H1 2026 (Jan 1 - Jun 30)?" + expect: {value: 4683428.81, rel: 0.001, tools: [run_semantic_query]} + - id: top_expense + ask: "What is our single largest expense account in H1 2026?" + expect: {contains: ["FBA"], tools: [run_semantic_query]} + - id: chart_request + ask: "Make me a line chart of monthly wholesale revenue for H1 2026." + expect: {artifact: chart, tools: [run_semantic_query, make_chart]} + - id: bu_guard + ask: "Show one single revenue line combining Fisch and Royal together per month for H1 2026." + expect: {tools: [run_semantic_query]} # combined-total ok; the guard is it must NOT error + - id: name_resolution + ask: "How much revenue did the customer 'flower' — I don't remember the exact name, something with FLOWERS in it — bring in H1 2026? Pick the top match." + expect: {tools: [get_field_values, run_semantic_query]} + - id: map_request + ask: "Put my top 20 customers by H1 2026 revenue on a map." + expect: {artifact: chart, tools: [run_semantic_query, make_chart]} + # --- the gap loop + location/YoY competence (added 2026-07-16 with the debug fixes) --- + - id: city_filter + ask: "Which customer in Brooklyn brought us the most revenue in H1 2026 (Jan 1 - Jun 30)?" + expect: {value: 48785.00, rel: 0.001, contains: ["J KAY"], + tools: [get_field_values, run_semantic_query], not_tools: [report_gap]} + - id: yoy_decliners + ask: "Which Fisch customer declined the most in revenue in H1 2026 (Jan 1 - Jun 30) versus the same window of 2025? Give the dollar decline." + expect: {value: 16451.25, rel: 0.005, contains: ["JUNE"], + tools: [run_semantic_query, transform_result], not_tools: [report_gap]} + # --- the agent dimension (added 2026-07-17: res.partner.agent_ids m2m sync — the #1 logged + # gap, closed). agent_book is VERBATIM the old gap_refusal ask: yesterday's honest refusal + # must become today's answer. + - id: agent_book + ask: "Who are the top customers handled by our salesperson Naomi in H1 2026?" + expect: {contains: ["Poppy Flowers"], tools: [get_field_values, run_semantic_query], + not_tools: [report_gap]} + - id: agent_revenue + ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?" + expect: {value: 872869.98, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]} + # --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a + # per-invoice-line commission agent; they name different people, and the commission table also + # carries internal SALESPEOPLE who are not agents. These three lock the distinction: the + # unallocated bucket must be answerable at all, and Anna/Shantal must never be called agents. + - id: agent_unallocated + ask: "Using the invoice-line basis, how much of Fisch's 2025 sales was NOT allocated to an agent?" + expect: {value: 1447579.59, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]} + - id: agent_not_salesperson + ask: "Who are our agents on the Fisch invoice lines in 2025, and what did each generate?" + expect: {contains: ["Pasternak"], not_contains: ["Anna", "Shantal"], + tools: [run_semantic_query], not_tools: [report_gap]} + - id: agent_allocated_2025 + ask: "On the invoice-line basis, how much of Fisch's 2025 sales was allocated to a real agent?" + expect: {value: 2995664.75, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]} + - id: gap_refusal + ask: "How many units of our best-selling SKU do we physically have in stock right now?" + expect: {tools: [report_gap]} # stock-on-hand has NO topic in the store — must self-report, never guess + # --- the analytics transform library (added 2026-07-18: 60-op Tableau-class set). Truths from + # the parity-proven semantic layer over the CLOSED H1 2026 window. + - id: abc_a_tier + ask: "Using an ABC / Pareto analysis of H1 2026 revenue (Jan 1 - Jun 30), how many customers fall in the A tier — the vital few that make up the first 80% of revenue?" + expect: {value: 311, abs: 6, tools: [run_semantic_query, transform_result], not_tools: [report_gap]} + - id: ytd_customers + ask: "How many distinct customers had we sold to cumulatively from Jan 1 through the end of March 2026?" + expect: {value: 790, abs: 1, tools: [run_semantic_query], not_tools: [report_gap]} + - id: concentration_top5 + ask: "What percent of our H1 2026 revenue (Jan 1 - Jun 30) is accounted for by our top 5 customers?" + expect: {value: 7.84, abs: 0.5, tools: [run_semantic_query], not_tools: [report_gap]} diff --git a/royalimports_os/harness/__init__.py b/royalimports_os/harness/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4855a958c985c127800ef1d441e4167a4b4770f3 --- /dev/null +++ b/royalimports_os/harness/__init__.py @@ -0,0 +1,3 @@ +"""Harness — the multi-source financial-dashboarding platform (connectors + canonical model + skills).""" +from .canonical import Capability, Entity, Query, Filter +from .base import Connector, Tenant, Skill, register_skill, skills_for diff --git a/royalimports_os/harness/analyst.py b/royalimports_os/harness/analyst.py new file mode 100644 index 0000000000000000000000000000000000000000..8cb6c2a0453e34a298f504a72826ba1f10596a06 --- /dev/null +++ b/royalimports_os/harness/analyst.py @@ -0,0 +1,265 @@ +"""harness/analyst.py — the AIOS Analyst loop (OM-4, 2026-07-11). + +A CHEAP small model (via OpenRouter, OpenAI-compatible) drives the compounding tool registry +(harness/tools.py) to answer questions, build charts, and spawn dashboards over the semantic +layer. The model NEVER writes SQL — it picks registry keys, guided by the in-product SKILL +recipes (model/skills/*.skill.yml) injected into its system prompt. Plan: omni-adoption addendum. + +Architecture (mirrors the Omni coordinator pattern): + ask(question) -> bounded tool loop: chat -> tool_calls -> tools.dispatch() -> results -> chat … + until a plain answer or MAX_ITERS. chat_fn is INJECTABLE: live = OpenRouter POST; tests = a + scripted double — the loop mechanics are provable offline (no key, no tokens). + +Correctness posture: bounded iterations (never endless); tool errors go BACK to the model as +readable envelopes (it self-corrects); the final answer carries the tool trace (auditability + +the drill guarantee); exhausted-budget is reported AS exhausted, never as success (loop rules). +Cost posture: DEFAULT_MODEL is a cheap tool-calling model; override per-tenant/env; prompt = the +skills + topic summary (small, cacheable), NOT raw data. +""" +import json +import os +from pathlib import Path + +import harness.tools as T + +# Provider LADDER (owner 2026-07-12: "use Groq and Cerebras"): all OpenAI-compatible chat +# endpoints; tried in order, failing over on payment/rate/server errors — an exhausted or +# rate-limited provider degrades to the next instead of taking the Analyst down (the +# OpenRouter-402 incident). Pin one with ANALYST_PROVIDER=groq|cerebras|openrouter; override +# its model with ANALYST_MODEL. Every model here must re-pass the eval gate (§4b). +PROVIDERS = [ + {"name": "groq", "env": "GROQ_API_KEY", + "url": "https://api.groq.com/openai/v1/chat/completions", + "model": "llama-3.3-70b-versatile"}, + {"name": "cerebras", "env": "CEREBRAS_API_KEY", + "url": "https://api.cerebras.ai/v1/chat/completions", + "model": "gpt-oss-120b"}, # this account's tool-capable model (from /v1/models) + {"name": "openrouter", "env": "OPENROUTER_API_KEY", + "url": "https://openrouter.ai/api/v1/chat/completions", + "model": "openai/gpt-4o-mini"}, +] +DEFAULT_MODEL = None # None = the selected provider's default model +MAX_ITERS = 8 + + +def available_providers(): + pin = os.environ.get("ANALYST_PROVIDER") + ps = [p for p in PROVIDERS if os.environ.get(p["env"])] + if pin: + ps = [p for p in ps if p["name"] == pin] or ps + return ps + + +def llm_configured(): + return bool(available_providers()) +SKILLS_DIR = Path(__file__).resolve().parents[1] / "model" / "skills" + + +def _skills_text(): + parts = [] + for f in sorted(SKILLS_DIR.glob("*.skill.yml")): + parts.append(f.read_text(encoding="utf-8")) + return "\n\n".join(parts) or "(no skill files installed)" + + +def system_prompt(): + """The small model's standing instructions: the platform rules + the skill recipes. + Stable content (cache-friendly); per-question content goes in the user turn.""" + topics = T.list_topics() + topic_summary = "\n".join( + f"- {t['topic']}: {t['label']} | metrics: {', '.join(t['metrics'])} | dims: {', '.join(t['dims'])}" + for t in topics) + return f"""You are the AIOS Analyst for a wholesale business. You answer data questions by +calling TOOLS over a governed semantic layer. THE RULES (non-negotiable): +1. You NEVER write SQL or invent field names — only registry keys from the tools' schemas. +2. Start unknown asks with list_topics/describe_topic. Resolve typed names (customers, products) + with get_field_values BEFORE filtering. Business units: Fisch=5, Royal=6 — never mix them in + one series unless explicitly comparing. +3. Every number you present must come from a tool result and keep its result_id (drillable). + If a tool errors, read the error and correct your call — do not make numbers up, ever. +4. Artifact actions (save_view, compose_dashboard, schedule_report, create_alert) need the + user's explicit confirmation first. +5. Answer plainly and lead with the finding. State the window and scope you used — READ the + window from the tool result's 'window' field, never guess it: a query without dates covers + ALL recorded history (there is no hidden default window). Time-bound asks ("this year", + "past 4 weeks") must pass explicit date_from/date_to. +6. A single-number question gets THE SINGLE NUMBER first (the total for the asked window/scope), + THEN any breakdown. Never answer a "what was X" question with only components: no monthly + lists without the total, no per-BU splits when the consolidated figure was asked. If the ask + is consolidated, query WITHOUT group_by (or sum before answering). +7. NEVER state an aggregate you did not obtain from a tool result. If you want to mention a + total across the rows you fetched, either run the scalar query for it or ADD UP those exact + rows — a round "approximately" figure is a fabrication and forbidden. When in doubt, present + the rows without a total. Joining or diffing two result sets BY HAND (e.g. this year vs last + year per customer) is equally forbidden — that is transform_result's job ({{"op":"yoy"}} adds + _ly, _delta, _yoy_pct per row); if no transform can compute it, say you cannot. +8. THE GAP PROTOCOL. If — and only if — after checking the schema (list_topics/describe_topic) + no registered dimension, metric, transform or chart kind can answer, do NOT guess and do NOT + force a wrong query: call report_gap(kind, missing, question, workaround) ONCE, then answer + in one honest sentence what is missing and offer the nearest ask that IS answerable. Before + concluding a gap, check the TRANSFORMS: year-over-year / decline / growth comparisons (yoy), + rankings (top_n/sort/head), shares, running totals and distributions are ALL answerable via + transform_result — a derived-analytics question is never a gap. Both directions are + failures: refusing without report_gap, and calling report_gap for something the tools + support. +9. NEVER emit markdown images, data: URIs, or links to charts — you cannot draw. Charts exist + ONLY through make_chart / make_table; the platform renders them below your answer. Refer to + a chart as "the chart below". Derived analytics (top-N, YoY, shares, running totals, moving + averages, ranks, bins) exist ONLY through transform_result — never arithmetic in your head. + +CHART PICKER (the form follows the comparison — pick by what the question compares): +- trend over time -> line (few periods -> bar); seasonal/two-dim intensity -> heatmap +- this year vs last year by period -> transform_result [{{"op":"yoy"}}] then yoy_bars +- ranking / "top N" -> transform_result top_n then ranked_bar (sorted, labeled) +- part-to-whole -> pie|donut ONLY when <=6 slices (top_n first); over time -> stacked_bar; + shares of a shifting total -> stacked_pct; hierarchy of shares -> treemap +- what drove the change / contribution -> waterfall (signed steps; platform adds the Total) +- concentration ("do the top X carry the book?") -> pareto +- correlation of two measures -> scatter; + a third measure -> bubble (size) +- distribution of a measure -> histogram (or transform bin -> bar) +- stage conversion -> funnel; actual vs target -> bullet (y=actual, y2=target) +- level + rate together (e.g. GM$ + GM%) -> combo (y=bars left, y2=line right) +- same small chart per segment -> facet on line|bar|area|scatter +- exact figures, many columns, lists -> make_table (totals row included) +- customer geography -> map. When two forms tie, pick the simpler one. + +DATASETS: +{topic_summary} + +YOUR SKILL RECIPES (follow these plans; adjust the [bracketed] parameters): +{_skills_text()}""" + + +_DEAD_STATUS = {401, 402, 403} # payment/auth: skip the provider immediately, no retry +_RETRY_STATUS = {429, 500, 502, 503} # rate/server: retry with backoff before failing over +_MAX_ANSWER_TOKENS = 1500 # answers are short; also keeps low-credit providers usable +_COOLDOWN = {} # provider name -> unix ts until which we skip it (in-proc) + + +def _live_chat(messages, tools, model, tool_choice="auto"): + """POST down the provider ladder with LATENCY-FIRST failover (a chat user is waiting): + a rate-limited provider is SKIPPED IMMEDIATELY in favor of the next one — we only sleep + when the ENTIRE ladder is limited (then short ladder-level backoffs). A LONG Retry-After + (daily quota, e.g. Groq TPD) puts the provider in an in-process COOLDOWN; 401/402/403 skip + the provider outright.""" + import time as _time + + import requests + provs = available_providers() + if not provs: + raise RuntimeError("no LLM provider configured — set GROQ_API_KEY / CEREBRAS_API_KEY / " + "OPENROUTER_API_KEY in royalimports_os/.env") + last_err = None + # Two patience profiles: CHAT (default) fails over fast — a user is waiting; BATCH + # (ANALYST_PATIENT=1, set by the eval gate / insight runner) waits out per-minute rate + # windows across more ladder rounds instead of exhausting the ladder in seconds. + patient = os.environ.get("ANALYST_PATIENT") == "1" + round_sleeps = (5, 10, 20, 30, 45, 60) if patient else (2, 6, 12) + for rnd in range(len(round_sleeps)): # ladder rounds + saw_retryable = False + for p in provs: + if _COOLDOWN.get(p["name"], 0) > _time.time(): + last_err = RuntimeError(f"{p['name']}: cooling down (quota)") + continue + try: + r = requests.post(p["url"], timeout=120, + headers={"Authorization": f"Bearer {os.environ[p['env']]}", + "HTTP-Referer": "https://aios.local", + "X-Title": "AIOS Analyst"}, + json={"model": (model or os.environ.get("ANALYST_MODEL") + or p["model"]), + "messages": messages, "tools": tools, + "tool_choice": tool_choice, "temperature": 0.1, + "max_tokens": _MAX_ANSWER_TOKENS}) + if r.status_code in _DEAD_STATUS: + last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}") + _COOLDOWN[p["name"]] = _time.time() + 300 # don't re-poke dead auth/payment + continue + if r.status_code in _RETRY_STATUS: + last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}") + try: + wait = float(r.headers.get("retry-after") or 0) + except ValueError: + wait = 0 + if wait > 60: # daily/long quota — cool down + _COOLDOWN[p["name"]] = _time.time() + wait + else: + saw_retryable = True # NO sleep — next provider NOW + continue + r.raise_for_status() + usage = r.json().get("usage", {}) + return r.json()["choices"][0]["message"], usage + except requests.RequestException as e: + last_err = e + continue # network blip — next provider + if not saw_retryable: + break # nothing retryable left anywhere + _time.sleep(round_sleeps[rnd]) # whole ladder limited — pause a round + raise RuntimeError(f"all LLM providers failed (last: {last_err})") + + +def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=MAX_ITERS, + telemetry_kind="analyst"): + """Run one Analyst turn. Returns {answer, artifacts, tool_trace, iterations, exhausted, + usage}. chat_fn(messages, tools, model) -> (assistant_message_dict, usage_dict).""" + chat = chat_fn or _live_chat + tools = T.openai_tools() + messages = [{"role": "system", "content": system_prompt()}] + messages += list(history or []) + # The model has no clock: anchor relative windows ("last week", "this month") to today. + # Lives in the USER turn so the system prompt stays byte-stable (prompt-cache friendly). + import datetime as _dt + messages.append({"role": "user", + "content": f"(today is {_dt.date.today().isoformat()})\n{question}"}) + trace, artifacts, usage_total = [], [], {"prompt_tokens": 0, "completion_tokens": 0} + + for i in range(max_iters): + msg, usage = chat(messages, tools, model) + for k in usage_total: + usage_total[k] += usage.get(k, 0) or 0 + tool_calls = msg.get("tool_calls") or [] + # Free-tier models INTERMITTENTLY answer a data question without calling any tool + # (~1 in 4 observed on gpt-oss-120b — the "no chart" defect). Deterministic fix: if + # the VERY FIRST turn used no tools, redo it once with tool_choice='required'. Meta + # questions just get a harmless schema lookup before answering. + if i == 0 and not tool_calls and chat is _live_chat: + msg, usage = _live_chat(messages, tools, model, tool_choice="required") + for k in usage_total: + usage_total[k] += usage.get(k, 0) or 0 + tool_calls = msg.get("tool_calls") or [] + messages.append({"role": "assistant", "content": msg.get("content") or "", + **({"tool_calls": tool_calls} if tool_calls else {})}) + if not tool_calls: + answer = (msg.get("content") or "").strip() + if not answer: + # A model occasionally returns NO content and NO tool calls (observed live + # 2026-07-16: the reply bubble was simply empty and the next turn hallucinated). + # Never surface silence — report the failure honestly instead. + answer = ("(the model returned an empty answer — nothing was computed; " + "please rephrase or ask again)") + import harness.telemetry as _tel + _tel.log(telemetry_kind, question=question[:200], + model=model or (available_providers() or [{}])[0].get("name", "auto"), + iterations=i + 1, + tools=[t["tool"] for t in trace], + tokens=usage_total["prompt_tokens"] + usage_total["completion_tokens"]) + return {"answer": answer, "artifacts": artifacts, + "tool_trace": trace, "iterations": i + 1, "exhausted": False, + "usage": usage_total} + for tc in tool_calls: + name = tc["function"]["name"] + args = tc["function"].get("arguments") or "{}" + res = T.dispatch(name, args) + trace.append({"tool": name, "args": args if isinstance(args, str) else json.dumps(args), + "ok": res["ok"], "error": res.get("error")}) + if res["ok"] and isinstance(res["data"], dict) and ( + "chart" in res["data"] or "kpi" in res["data"] or "table" in res["data"] + or "dashboard" in res["data"]): + artifacts.append(res["data"]) + messages.append({"role": "tool", "tool_call_id": tc.get("id", name), + "content": json.dumps(res, default=str)[:8000]}) + # budget exhausted — report honestly (never as success) + return {"answer": "(stopped: tool-call budget exhausted before a final answer — " + "narrow the question or raise max_iters)", + "artifacts": artifacts, "tool_trace": trace, "iterations": max_iters, + "exhausted": True, "usage": usage_total} diff --git a/royalimports_os/harness/base.py b/royalimports_os/harness/base.py new file mode 100644 index 0000000000000000000000000000000000000000..b538ac4474ad4e36f66e30b4edf6e5dcd8500822 --- /dev/null +++ b/royalimports_os/harness/base.py @@ -0,0 +1,123 @@ +"""Harness core — the platform contract that turns 'a dashboard' into 'a product'. + +Three concepts: + Connector a source adapter (Odoo / QuickBooks / Amazon / …). Declares its capabilities and + answers canonical Queries. Adding a source = subclass Connector. + Tenant one client (Royal Imports, …). Bundles the connectors they've hooked up + their scope + config (business units, excluded accounts, channel map, branding). Multi-tenant lives here. + Skill a portable dashboard/metric written ONCE against the canonical model; it declares the + capabilities it needs so a tenant only ever sees skills its sources can serve. + +The UI (app.py) becomes a thin renderer over (tenant → available skills). Nothing above the +connector layer knows what Odoo is. +""" +from __future__ import annotations +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Callable +from .canonical import Capability, Query, Entity + + +class Connector(ABC): + """A data source. Implement `capabilities()` + `aggregate()` (and optionally `records()`), map the + canonical model to the source, and the source is usable by every compatible skill.""" + key: str = 'connector' + name: str = 'Connector' + + @abstractmethod + def capabilities(self) -> set[Capability]: + """The canonical capabilities this source can serve for the given account.""" + + @abstractmethod + def aggregate(self, q: Query) -> list[dict]: + """Answer an aggregate Query → list of canonical dicts (keys = measures/group_by, + 'count').""" + + def records(self, q: Query) -> list[dict]: + """Optional row-level fetch (canonical dicts). Default: not supported.""" + raise NotImplementedError(f'{self.key}: row-level records() not implemented') + + def distinct_count(self, q: Query, fieldname: str) -> int: + """Number of distinct values of a canonical field over the query's scope (e.g. active + customers). Default: derive from an aggregate group_by; connectors may override efficiently.""" + from dataclasses import replace + rows = self.aggregate(replace(q, measures=(), aggregate='count', group_by=(fieldname,))) + return sum(1 for r in rows if r.get(fieldname) not in (None, False)) + + def supports(self, cap: Capability) -> bool: + return cap in self.capabilities() + + def entities(self) -> set[Entity]: + """Canonical entities this connector can materialize (advertised by subclasses).""" + return set() + + +@dataclass +class Tenant: + """One client of the platform. `sources` maps a connector key → Connector instance. `config` holds + the client's scope semantics (which is what used to be hard-coded in core/odoo.py): + business_units {label: source_scope_value} e.g. {'Fisch': 5, 'Royal': 6} + confirmed_statuses [...] which order states count as 'real' + excluded_customers [ids] house / inter-co accounts to drop + channels {label: match} e.g. {'Amazon': } (marketplace split) + branding {...} palette, name, logo + A skill reads tenant.config for scope and asks tenant.source(capability) for the right connector.""" + key: str + name: str + sources: dict[str, Connector] = field(default_factory=dict) + config: dict = field(default_factory=dict) + + def add(self, conn: Connector) -> 'Tenant': + self.sources[conn.key] = conn + return self + + def capabilities(self) -> set[Capability]: + caps: set[Capability] = set() + for c in self.sources.values(): + caps |= c.capabilities() + return caps + + def source(self, cap: Capability, prefer: str | None = None) -> Connector: + """The connector that serves `cap` (a tenant may hook several sources; `prefer` picks one, + else first that supports it). Raises if no connected source provides the capability.""" + if prefer and prefer in self.sources and self.sources[prefer].supports(cap): + return self.sources[prefer] + for c in self.sources.values(): + if c.supports(cap): + return c + raise LookupError(f"tenant {self.key!r} has no source for capability {cap.value!r}") + + def has(self, *caps: Capability) -> bool: + mine = self.capabilities() + return all(c in mine for c in caps) + + +# --------------------------------------------------------------------------- skill registry +@dataclass(frozen=True) +class Skill: + """A portable dashboard. `requires` are the capabilities it needs; the harness only offers it to a + tenant whose connected sources cover them. `render` is the UI entrypoint (given tenant + context).""" + key: str + label: str + requires: tuple[Capability, ...] + render: Callable | None = None + hq: bool = False # consolidated / company-level (ignores the business-unit filter) + + def available_for(self, tenant: Tenant) -> bool: + return tenant.has(*self.requires) + + +_SKILLS: dict[str, Skill] = {} + + +def register_skill(skill: Skill) -> Skill: + _SKILLS[skill.key] = skill + return skill + + +def skills_for(tenant: Tenant) -> list[Skill]: + """The skills a tenant can actually run, given its connected sources — this is the dynamic nav.""" + return [s for s in _SKILLS.values() if s.available_for(tenant)] + + +def all_skills() -> list[Skill]: + return list(_SKILLS.values()) diff --git a/royalimports_os/harness/canonical.py b/royalimports_os/harness/canonical.py new file mode 100644 index 0000000000000000000000000000000000000000..8b0f3337e9ad132f6e8b9e3cff40875c1bc4cb78 --- /dev/null +++ b/royalimports_os/harness/canonical.py @@ -0,0 +1,101 @@ +"""Canonical financial data model — the SHARED vocabulary of the harness. + +Skills speak ONLY in these terms; connectors (Odoo / QuickBooks / Amazon / …) translate them to +their own source. This is the seam that decouples the dashboards from any single system: add a new +source by writing a connector that maps its API onto these entities/fields — every skill that needs +only the capabilities that source provides then works, unchanged. + +Design rules: +- Entities and canonical field names are source-agnostic (no `amount_untaxed`, no `sale.order`). +- A skill builds a `Query` (what business question), a connector answers it (how, per source). +- Capabilities let a skill declare what it needs and a tenant show only the dashboards its + connected sources can actually serve. +""" +from __future__ import annotations +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + + +class Capability(str, Enum): + """What a connector can serve. Skills require a set of these; a tenant's nav is the union of + what its connectors provide. (A pure-accounting source like QuickBooks has LEDGER+INVOICING+ + CUSTOMERS but not SKU-level SALES velocity; Amazon has SALES but no LEDGER; Odoo has all.)""" + SALES = 'sales' # orders / order lines (SKU-level revenue, units) + INVOICING = 'invoicing' # customer invoices + credit notes (AR, returns) + LEDGER = 'ledger' # GL accounts + journal lines (P&L, balance sheet) + PAYABLES = 'payables' # vendor bills / AP + INVENTORY = 'inventory' # on-hand stock levels + valuation + PRODUCTS = 'products' # product master (SKU, category, cost) + CUSTOMERS = 'customers' # customer master (name, geo, assigned agent) + + +class Entity(str, Enum): + """Canonical business objects. Each connector declares which it can materialize.""" + SALES_ORDER = 'sales_order' + SALES_ORDER_LINE = 'sales_order_line' + INVOICE = 'invoice' # customer invoice / credit note + JOURNAL_LINE = 'journal_line' # GL posting + GL_ACCOUNT = 'gl_account' + VENDOR_BILL = 'vendor_bill' + STOCK_LEVEL = 'stock_level' + PRODUCT = 'product' + CUSTOMER = 'customer' + + +# Canonical field names per entity — the ONLY field identifiers a skill may use. Connectors own the +# mapping to their native fields. `date` is the entity's primary time axis (used by Query windows). +FIELDS: dict[Entity, set[str]] = { + Entity.SALES_ORDER: {'date', 'amount', 'customer', 'salesperson', 'agent', 'status', 'business_unit', 'channel'}, + Entity.SALES_ORDER_LINE: {'date', 'amount', 'qty', 'margin', 'product', 'category', 'customer', 'status', 'business_unit'}, + Entity.INVOICE: {'date', 'amount', 'customer', 'doc_type', 'status', 'salesperson', 'business_unit'}, + Entity.JOURNAL_LINE: {'date', 'amount', 'account', 'account_type', 'business_unit', 'status'}, + Entity.GL_ACCOUNT: {'code', 'name', 'type'}, + Entity.VENDOR_BILL: {'date', 'amount', 'vendor', 'status'}, + Entity.STOCK_LEVEL: {'product', 'qty', 'value', 'location'}, + Entity.PRODUCT: {'id', 'code', 'name', 'category', 'cost', 'active'}, + Entity.CUSTOMER: {'id', 'name', 'city', 'state', 'agent', 'active'}, +} + +# Allowed filter/aggregate vocabulary (connectors translate these; they never see source operators). +OPS = frozenset({'eq', 'ne', 'in', 'not_in', 'gte', 'lte', 'gt', 'lt', 'like'}) +AGGS = frozenset({'sum', 'count', 'avg', 'min', 'max'}) +GRAINS = frozenset({'day', 'week', 'month', 'quarter', 'year'}) # time-bucket for a trend + + +@dataclass(frozen=True) +class Filter: + """A canonical predicate: (canonical_field, op, value). op ∈ OPS.""" + fieldname: str + op: str + value: Any + + def __post_init__(self): + if self.op not in OPS: + raise ValueError(f'unknown op {self.op!r}; allowed {sorted(OPS)}') + + +@dataclass(frozen=True) +class Query: + """A source-agnostic analytical request. `measures` are canonical fields aggregated by `aggregate`, + optionally grouped by canonical `group_by` fields, filtered by `filters`, over [date_from, date_to] + on the entity's `date` axis. A connector returns a list of plain dicts keyed by the canonical + measure/group names (+ 'count'). Skills post-process those dicts — never source rows.""" + entity: Entity + measures: tuple[str, ...] = () + aggregate: str = 'sum' + group_by: tuple[str, ...] = () + filters: tuple[Filter, ...] = () + date_from: str | None = None + date_to: str | None = None + grain: str | None = None # bucket the date axis into a trend; result rows carry 'period' (e.g. '2026-05') + + def __post_init__(self): + if self.aggregate not in AGGS: + raise ValueError(f'unknown aggregate {self.aggregate!r}') + if self.grain is not None and self.grain not in GRAINS: + raise ValueError(f'unknown grain {self.grain!r}; allowed {sorted(GRAINS)}') + valid = FIELDS.get(self.entity, set()) + for f in list(self.measures) + list(self.group_by) + [x.fieldname for x in self.filters]: + if f not in valid: + raise ValueError(f'{f!r} is not a canonical field of {self.entity.value} (have {sorted(valid)})') diff --git a/royalimports_os/harness/chatlog.py b/royalimports_os/harness/chatlog.py new file mode 100644 index 0000000000000000000000000000000000000000..9f09aad200ae632610c331cd0ebea43c89a87b04 --- /dev/null +++ b/royalimports_os/harness/chatlog.py @@ -0,0 +1,102 @@ +"""harness/chatlog.py — durable Analyst conversations (2026-07-16). + +Every Analyst conversation is a logged SESSION that survives reruns, page navigation and Space +restarts. Persistence rides the platform's only writable store (core/store.py → the private HF +dataset), the SAME mechanism as users/notes — Odoo stays read-only. + +Shape (store key 'analyst_sessions'): { username: [ session, ... ] } (newest first), each + session = {id, title, started_at, updated_at, ui:[[role,content,artifacts],...], + llm:[{role,content},...]}. +ui = the redisplay transcript; llm = the trimmed model-context transcript. + +Write posture (per the design review): the LIVE turn stays in st.session_state for a snappy chat; +we persist in a BACKGROUND daemon thread (core/store.update is a full dataset commit — never block +the UI on it), and bound growth (MAX_SESSIONS per user, MAX_TURNS per session) so the JSON stays +small. A failed/absent store degrades to in-memory only (chat still works this session). +""" +import threading +import time +import uuid + +import core.store as store + +KEY = "analyst_sessions" +MAX_SESSIONS = 40 # keep the last N conversations per user (bounds the JSON) +MAX_TURNS = 60 # ui/llm entries kept per conversation +TITLE_LEN = 60 + + +def new_id(): + return uuid.uuid4().hex[:12] + + +def new_session(sid=None): + now = time.strftime("%Y-%m-%d %H:%M:%S") + return {"id": sid or new_id(), "title": "", "started_at": now, "updated_at": now, + "ui": [], "llm": []} + + +def _safe(obj): + """JSON round-trip a value (chart/kpi artifacts) so only serializable content is stored; + anything that won't serialize is dropped rather than breaking the whole write.""" + import json + try: + return json.loads(json.dumps(obj, default=str)) + except Exception: + return None + + +def load_sessions(username): + """The user's conversations, newest first (lenient read — never raises on a store blip).""" + if not store.available(): + return [] + try: + all_s = store.get(KEY) or {} + except Exception: + return [] + return list(all_s.get(username, [])) + + +def touch(session, question): + """Stamp updated_at and derive a title from the first user question.""" + session["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S") + if not session.get("title") and question: + t = " ".join(str(question).split()) + session["title"] = t[:TITLE_LEN] + ("…" if len(t) > TITLE_LEN else "") + # bound in-memory transcript length (keep the most recent turns) + if len(session.get("ui", [])) > MAX_TURNS: + session["ui"] = session["ui"][-MAX_TURNS:] + if len(session.get("llm", [])) > MAX_TURNS: + session["llm"] = session["llm"][-MAX_TURNS:] + return session + + +def upsert(sessions, session): + """Return the sessions list with `session` moved to the front (deduped by id), capped.""" + out = [s for s in (sessions or []) if s.get("id") != session.get("id")] + out.insert(0, session) + return out[:MAX_SESSIONS] + + +def _persist(username, session): + # sanitize artifacts for storage (charts are plain dicts; drop any non-serializable stragglers) + stored = dict(session) + stored["ui"] = [[role, content, [a for a in (_safe(x) for x in (arts or [])) if a is not None]] + for (role, content, arts) in session.get("ui", [])] + + def _fn(data): + data = data or {} + data[username] = upsert(data.get(username, []), stored) + return data + try: + store.update(KEY, _fn) + except Exception: + pass # store blip / no token → in-memory only this run + + +def save_async(username, session): + """Persist in a daemon thread so the chat never waits on a dataset commit.""" + if not store.available(): + return + threading.Thread(target=_persist, args=(username, dict(session)), + daemon=True, name="analyst-chatlog").start() diff --git a/royalimports_os/harness/connectors/__init__.py b/royalimports_os/harness/connectors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b1197d1f9dcafee9fa927dcf2738b0dca5e9358 --- /dev/null +++ b/royalimports_os/harness/connectors/__init__.py @@ -0,0 +1 @@ +"""Source connectors — one module per system (odoo, quickbooks, amazon, ...).""" diff --git a/royalimports_os/harness/connectors/odoo.py b/royalimports_os/harness/connectors/odoo.py new file mode 100644 index 0000000000000000000000000000000000000000..20bb067b0d8b696babd8fd3131c402775e49cb7b --- /dev/null +++ b/royalimports_os/harness/connectors/odoo.py @@ -0,0 +1,147 @@ +"""Odoo connector — the reference implementation. + +ALL Odoo specifics (model names, field names, XML-RPC quirks, the read_group dot-path gotchas) live +here and nowhere else. It maps the canonical model onto the existing, battle-tested `core.odoo` data +layer (read-only, write-guarded) so nothing is re-plumbed — the harness just gives it a source-agnostic +face. Every other connector (QuickBooks, Amazon, …) is a sibling of this file. +""" +from __future__ import annotations +import sys +from dataclasses import replace +from functools import lru_cache +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # royalimports_os/ +import core.odoo as O +from harness.canonical import Capability, Entity, Query, Filter + + +@lru_cache(maxsize=1) +def _product_category_map(): + """product_id -> main category name (Odoo-side enrichment; reuses the existing resolver).""" + import modules.sales as _sales + return _sales._product_cat() + + +def _replace_group(q: Query, new_group_by): + return replace(q, group_by=new_group_by, grain=None) + +_OPS = {'eq': '=', 'ne': '!=', 'in': 'in', 'not_in': 'not in', + 'gte': '>=', 'lte': '<=', 'gt': '>', 'lt': '<', 'like': 'ilike'} + +# Canonical entity -> Odoo model + (canonical field -> Odoo field) + its time axis. +_MAP = { + Entity.SALES_ORDER: { + 'model': 'sale.order', 'date': 'date_order', 'dt': True, + 'f': {'amount': 'amount_untaxed', 'customer': 'partner_id', 'salesperson': 'user_id', + 'status': 'state', 'business_unit': 'team_id'}}, + Entity.SALES_ORDER_LINE: { + 'model': 'sale.order.line', 'date': 'order_id.date_order', 'dt': True, + 'f': {'amount': 'price_subtotal', 'qty': 'product_uom_qty', 'margin': 'margin', + 'product': 'product_id', 'customer': 'order_partner_id', + 'status': 'order_id.state', 'business_unit': 'order_id.team_id'}}, + Entity.INVOICE: { + 'model': 'account.move', 'date': 'invoice_date', 'dt': False, + 'f': {'amount': 'amount_untaxed_signed', 'customer': 'partner_id', 'doc_type': 'move_type', + 'status': 'state', 'salesperson': 'invoice_user_id', 'business_unit': 'team_id'}}, + Entity.STOCK_LEVEL: { + 'model': 'stock.quant', 'date': None, 'dt': False, + 'f': {'product': 'product_id', 'qty': 'quantity', 'location': 'location_id'}}, + Entity.PRODUCT: { + 'model': 'product.product', 'date': None, 'dt': False, + 'f': {'id': 'id', 'code': 'default_code', 'name': 'name', 'category': 'categ_id', + 'cost': 'standard_price', 'active': 'active'}}, + Entity.CUSTOMER: { + 'model': 'res.partner', 'date': None, 'dt': False, + 'f': {'id': 'id', 'name': 'name', 'city': 'city', 'state': 'state_id', + 'agent': 'agent_ids', 'active': 'active'}}, +} + + +class OdooConnector: + key = 'odoo' + name = 'Odoo ERP' + + def __init__(self, doc_mode: str = 'order'): + self._doc_mode = doc_mode + + def capabilities(self) -> set[Capability]: + return {Capability.SALES, Capability.INVOICING, Capability.LEDGER, Capability.PAYABLES, + Capability.INVENTORY, Capability.PRODUCTS, Capability.CUSTOMERS} + + def entities(self) -> set[Entity]: + return set(_MAP) + + def supports(self, cap: Capability) -> bool: + return cap in self.capabilities() + + # ------------------------------------------------------------------ query translation + def _domain(self, q: Query, meta): + dom = [] + if q.date_from and meta['date']: + dom.append((meta['date'], '>=', q.date_from + (' 00:00:00' if meta['dt'] else ''))) + if q.date_to and meta['date']: + dom.append((meta['date'], '<=', q.date_to + (' 23:59:59' if meta['dt'] else ''))) + for flt in q.filters: + dom.append((meta['f'].get(flt.fieldname, flt.fieldname), _OPS[flt.op], flt.value)) + return dom + + def aggregate(self, q: Query) -> list[dict]: + meta = _MAP[q.entity] + O.set_doc_mode(self._doc_mode) + # 'category' isn't a native line field (dot-path groupby fails) — enrich from the product master. + if 'category' in q.group_by: + return self._by_category(q, meta) + dom = self._domain(q, meta) + measures = [f"{meta['f'].get(m, m)}:{q.aggregate}" for m in q.measures] + groupby = [meta['f'].get(g, g) for g in q.group_by] + if q.grain and meta['date']: # time-bucket -> Odoo's date:grain groupby + groupby.append(f"{meta['date']}:{q.grain}") + rows = O.read_group(meta['model'], dom, measures, groupby, lazy=False) + return [self._canon_row(r, q, meta) for r in rows] + + def _canon_row(self, r, q, meta): + rec = {} + for m in q.measures: + rec[m] = r.get(meta['f'].get(m, m)) or 0.0 + for g in q.group_by: + v = r.get(meta['f'].get(g, g)) + rec[g] = ({'id': O.m2o_id(v), 'name': O.m2o_name(v)} if isinstance(v, (list, tuple)) else v) + if q.grain and meta['date']: + rng = (r.get('__range') or {}).get(f"{meta['date']}:{q.grain}") or {} + rec['period'] = (rng.get('from') or '')[:7] + rec['count'] = r.get('__count') or 0 + return rec + + def _by_category(self, q, meta): + """Group SALES_ORDER_LINE by main product category by grouping on product then folding + product->category in-connector (source-specific enrichment lives here, not in the skill).""" + catmap = _product_category_map() + sub = _replace_group(q, tuple(g for g in q.group_by if g != 'category') + ('product',)) + agg = {} + for r in self.aggregate(sub): + prod = r.get('product') or {} + cat = catmap.get(prod.get('id') if isinstance(prod, dict) else prod, '(uncategorized)') + e = agg.setdefault(cat, {'category': cat, 'count': 0}) + for m in q.measures: + e[m] = e.get(m, 0.0) + (r.get(m) or 0.0) + e['count'] += r.get('count') or 0 + return sorted(agg.values(), key=lambda x: -(x.get(q.measures[0]) if q.measures else 0)) + + def distinct_count(self, q: Query, fieldname: str) -> int: + meta = _MAP[q.entity] + O.set_doc_mode(self._doc_mode) + return O.distinct_count(meta['model'], self._domain(q, meta), meta['f'].get(fieldname, fieldname)) + + def records(self, q: Query) -> list[dict]: + meta = _MAP[q.entity] + O.set_doc_mode(self._doc_mode) + fields = [meta['f'].get(m, m) for m in (q.measures or tuple(meta['f']))] + rows = O.search_read(meta['model'], self._domain(q, meta), fields) + inv = {v: k for k, v in meta['f'].items()} + return [{inv.get(k, k): (O.m2o_name(v) if isinstance(v, (list, tuple)) else v) + for k, v in r.items()} for r in rows] + + # convenience for tenant setup: resolve names → ids for scope config + @staticmethod + def excluded_customer_ids() -> list[int]: + return list(O.excluded_partner_ids()) diff --git a/royalimports_os/harness/connectors/quickbooks.py b/royalimports_os/harness/connectors/quickbooks.py new file mode 100644 index 0000000000000000000000000000000000000000..c6f771fd27321b4e8196395d3c7015dbf0d45617 --- /dev/null +++ b/royalimports_os/harness/connectors/quickbooks.py @@ -0,0 +1,37 @@ +"""QuickBooks Online connector — STUB (scaffolding for the next real integration). + +It exists to prove the harness is source-agnostic: it declares the capabilities QBO actually has +(accounting + AR + customers, but NOT SKU-level sales velocity or warehouse inventory), so the +platform will automatically OFFER ledger/AR/customer skills on a QuickBooks-only tenant and HIDE the +SKU/inventory skills — no per-skill if-statements anywhere. Fill in aggregate() against the QBO +Reports/Query API (OAuth2) to make it live. +""" +from __future__ import annotations +from harness.canonical import Capability, Entity, Query + + +class QuickBooksConnector: + key = 'quickbooks' + name = 'QuickBooks Online' + + def __init__(self, realm_id: str | None = None, token: str | None = None): + self.realm_id = realm_id + self._token = token # OAuth2 access token (never logged) + + def capabilities(self) -> set[Capability]: + # QBO gives you the books + receivables + the customer list; it does not model SKU-level + # order lines or physical stock the way an ERP/marketplace does. + return {Capability.LEDGER, Capability.INVOICING, Capability.PAYABLES, Capability.CUSTOMERS} + + def entities(self) -> set[Entity]: + return {Entity.INVOICE, Entity.JOURNAL_LINE, Entity.GL_ACCOUNT, Entity.VENDOR_BILL, Entity.CUSTOMER} + + def supports(self, cap: Capability) -> bool: + return cap in self.capabilities() + + def aggregate(self, q: Query) -> list[dict]: + raise NotImplementedError( + "QuickBooksConnector.aggregate: TODO — translate the canonical Query to the QBO " + "Query/Reports API (GET /v3/company/{realm}/query) and map rows back to canonical dicts. " + "Entity map: INVOICE->Invoice/CreditMemo, JOURNAL_LINE->JournalEntry lines, " + "GL_ACCOUNT->Account, VENDOR_BILL->Bill, CUSTOMER->Customer.") diff --git a/royalimports_os/harness/datastore.py b/royalimports_os/harness/datastore.py new file mode 100644 index 0000000000000000000000000000000000000000..4cb3126cdd0a7bbfb649d473269839cbe79f92ff --- /dev/null +++ b/royalimports_os/harness/datastore.py @@ -0,0 +1,750 @@ +"""harness/store.py — the tenant data store (OM-1, 2026-07-11). + +Incremental Odoo → DuckDB mirror: the local analytical store that saved views/dashboards (OM-3) +and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC +per question. +Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1. + +Design rules: + - The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion, + confirmed-only) is applied by the SEMANTIC layer at query time (OM-2) — one source of truth. + - Sync is CHECKPOINTED + BOUNDED + RESUMABLE (loop-library discipline): backfill paginates by + id; live mode advances a write_date cursor; every run is bounded by max_batches and safe to + kill/re-run (upserts are idempotent). + - validate() compares the store against LIVE Odoo (raw fidelity: counts + monthly sums to the + cent) — the store never validates itself. Odoo hard-deletes (unlink) don't move write_date; + the count checks are the drift detector for those. + - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo. +""" +import datetime as dt +import json +import os +import sys +import time +from pathlib import Path + +import duckdb + +import core.odoo as O + +_STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store" + +# ⚠ MUTABLE ON PURPOSE, and read LATE by every function below (Python resolves a module global at +# CALL time), which is how `provision_tenant.py` already points a fresh instance at its own file. +# EXIT-4a (X7) formalises that: `AIOS_DUCKDB_PATH` lets a container name the file without an edit, +# `path_for()` owns the per-tenant naming convention, and `use_path()` is the SAFE way to switch — +# see its docstring for why a bare assignment is not. +# The DEFAULT is unchanged: /data/store/royal.duckdb. +DB_PATH = Path(os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb")) + + +def path_for(tenant_key): + """The canonical DuckDB file for a tenant slug. `royal-imports` keeps the historical + `royal.duckdb` name so tenant #0's existing 175 MB file is not orphaned by a rename. + + File-per-tenant IS the isolation model (C1c): DuckDB has no row-level security, so the only + boundary that holds is the operating system's — one file, one tenant. + """ + key = str(tenant_key or "").strip().lower() + if key in ("", "royal-imports"): + return DB_PATH if key == "" else Path( + os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb")) + safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60] + return _STORE_DIR / f"{safe}.duckdb" + + +def use_path(path): + """Repoint the store at `path`, CLOSING the process singleton first. + + ⚠ A bare `datastore.DB_PATH = …` is not enough and is the more dangerous half of this + operation. `_instance()` caches ONE open connection for the life of the process, so after a + plain reassignment every read keeps being served from the PREVIOUSLY opened file — a silent + cross-tenant read, returning real rows that belong to somebody else. Closing the connection + and clearing the readiness memo (which is also per-file) is what makes the switch honest. + + ⚠ AND CLOSING THE CONNECTION IS STILL NOT ENOUGH ON ITS OWN, because `ro_con()` caches a + cursor in a `threading.local`: this function can only clear the CALLING thread's, and + `ro_con`'s liveness probe (`SELECT 1`) SUCCEEDS on a cursor whose underlying connection was + closed out from under it in some cases — so another thread would keep answering from the old + file with no error to notice. The generation counter below is what closes that: `ro_con` + compares the generation its cursor was opened under and reopens when it has moved, which is + a check no individual thread can forget to do. + + Returns the new path. Callers: `provision_tenant.py`, and any single-tenant worker process. + NOT a per-request operation — one process serves one analytical store at a time, which is why + `harness.runtime` hands out the PATH rather than switching the global underneath a request. + """ + global DB_PATH + with _INSTANCE_LOCK: + con = _INSTANCE.get("con") + if con is not None: + try: + con.close() + except Exception: + pass + _INSTANCE["con"] = None + _RO_TLS.__dict__.pop("cur", None) + _READY["ok"] = False + _GENERATION[0] += 1 # every other thread's cached cursor is now stale + DB_PATH = Path(path) + return DB_PATH + +# Entity specs: Odoo model → store table. m2o fields land as _id + _name. +# 'archivable' entities are pulled with active in [True, False] (the re-SKU/merge rule). +ENTITIES = { + "sale_order": { + "model": "sale.order", "archivable": False, + "fields": ["name", "date_order", "partner_id", "team_id", "user_id", "state", + "amount_untaxed", "invoice_status", "write_date"], + }, + "sale_order_line": { + "model": "sale.order.line", "archivable": False, + "fields": ["order_id", "order_partner_id", "product_id", "product_uom_qty", + "price_subtotal", "margin", "purchase_price", "write_date"], + }, + "res_partner": { + "model": "res.partner", "archivable": True, + # `agent` (bool, labelled "Creditor/Agent") is THE discriminator between a real sales + # AGENT and an internal SALESPERSON who merely carries commission lines — the commission + # table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent") + # marks agents who came from staff; the owner ruling on whether those count as agents is + # OUTSTANDING, so both flags are synced and neither reading is baked in. + "fields": ["name", "team_id", "city", "state_id", "country_id", "active", + "agent", "salesman_as_agent", "write_date"], + "backfill_fields": ["agent", "salesman_as_agent"], # added 2026-07-28 + # m2m fields land as _id (the FIRST linked id — the Customers-module convention: + # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a + # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be + # indistinguishable from an m2o (id, name) pair in _flatten. + "m2m": ["agent_ids"], + }, + "product_product": { + "model": "product.product", "archivable": True, + "fields": ["default_code", "name", "categ_id", "type", "standard_price", "active", + "write_date"], + }, + "account_move": { + "model": "account.move", "archivable": False, + "fields": ["name", "move_type", "state", "invoice_date", "invoice_date_due", + "partner_id", "amount_untaxed_signed", "amount_residual_signed", + "payment_state", "write_date"], + }, + "account_move_line": { + "model": "account.move.line", "archivable": False, + # display_type/move_type/parent_state/price_subtotal/product_id added 2026-07-28 so the + # invoice-LINE grain is queryable (topic `invoice_lines`). + # ⚠ display_type values here are 'product', 'cogs', 'payment_term', 'line_note', + # 'line_section'. Commission rows attach to 'cogs' lines as well as 'product' ones + # (96,936 of 147,160 in 2025). Filtering to 'product' is a GRAIN guard: measured + # 2026-07-29, omitting it does not move revenue (non-product lines carry + # price_subtotal = 0) but it inflates commission ROW COUNTS ~80% and would corrupt any + # count-of-lines measure. + "fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit", + "balance", "display_type", "move_type", "parent_state", "price_subtotal", + "product_id", "write_date"], + "backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal", + "product_id"], # added 2026-07-28 + # the BU bridge: account.move carries NO business unit (every invoice sits on team 1 — + # see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable + # through the originating sale order. sale_line_id (the FIRST linked sale line) is + # lossless here: ZERO 2025 invoice lines span more than one team (measured). + "m2m": ["sale_line_ids"], + }, + "account_invoice_line_agent": { + # The OCA sale-commission module: one row per (invoice line × agent). This is the SECOND + # agent source — per-line commission attribution — and it names a different population + # than res_partner.agent_ids (the customer-master book). See topic `invoice_lines`. + # ⚠ TWO similarly-named FKs, do not confuse them: + # object_id -> account_move_line (THE grain; join on this) + # invoice_id -> account_move (the parent document) + "model": "account.invoice.line.agent", "archivable": False, + "fields": ["agent_id", "commission_id", "amount", "invoice_id", "object_id", + "invoice_date", "settled", "write_date"], + }, + "account_account": { + # NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 — + # a ('active','in',...) domain 500s); treat as non-archivable. + "model": "account.account", "archivable": False, + "fields": ["code", "name", "account_type", "write_date"], + }, +} + +BATCH = 2000 + +# Fields that are genuinely BOOLEAN. This list is load-bearing in TWO places: _cols types the +# column, and _flatten must NOT collapse a real False into NULL. Odoo returns False both for +# "empty" and for "boolean false", so a bool field missing from here silently becomes NULL — and +# for res_partner.agent that would erase the ONE flag separating an AGENT from a SALESPERSON +# (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably +# from "unknown". +BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free") + + +import threading as _thr + +_RO_TLS = _thr.local() +_READY = {"ok": False} +_INSTANCE = {"con": None} +_INSTANCE_LOCK = _thr.Lock() +#: Bumped by `use_path()` whenever the store FILE changes. Every per-thread cursor is stamped with +#: the generation it was opened under, so a thread that never called `use_path` still cannot keep +#: reading the previous tenant's file — see `ro_con`. A plain "is the cursor alive?" probe cannot +#: answer that question, which is why this counter exists rather than more probing. +_GENERATION = [0] + + +def mark_ready(): + """In-process signal that the store is fully synced (every entity live). The sync writer calls + this after an all-live pass so a reader never has to (re)discover usability by touching the + file. Belt-and-suspenders alongside the shared-instance model below.""" + _READY["ok"] = True + + +def _instance(): + """THE one process-wide read-write DuckDB connection. Readers take .cursor() off it and the + sync writer uses it too — DuckDB serves concurrent reads + a single writer from ONE instance + via MVCC, so a read NEVER contends with a sync pass for the file lock. + + This is the fix for the 'connection error unless I refresh' bug (2026-07-16): an independent + read-only `duckdb.connect()` FAILS while the writer holds the lock ('Can't open a connection + to same database file with a different configuration than existing connections'); a cursor on + the shared instance does not. Openers must ensure the file is fully present first — ensure_seed + writes atomically (os.replace) and every read path guards on DB_PATH.exists() — so the instance + never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole + life, so a SEPARATE process — e.g. mcp_server.py — cannot open the same store file concurrently; + in production only the app runs, and that was already true during any sync pass.)""" + con = _INSTANCE["con"] + if con is not None: + return con + with _INSTANCE_LOCK: + if _INSTANCE["con"] is None: + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + _INSTANCE["con"] = duckdb.connect(str(DB_PATH)) + return _INSTANCE["con"] + + +def ro_cursor(): + """A read cursor on the shared instance — the contention-free way to read the store.""" + return _instance().cursor() + + +def ready(): + """True once EVERY entity has completed its backfill (phase 'live'). Gate all store READS on + this: a mid-backfill store would return PARTIAL totals silently — worse than an error. Positive + results cache for the process (backfill never regresses). Reads via a shared-instance cursor, + so it is never mis-reported as 'not ready' just because a background sync pass is mid-flight.""" + if _READY["ok"]: + return True + if not DB_PATH.exists(): + return False + try: + cur = ro_cursor() + try: + phases = {r[0]: r[1] for r in + cur.execute("SELECT entity, phase FROM _sync_state").fetchall()} + finally: + cur.close() + except Exception: + return False + if all(phases.get(k) == "live" for k in ENTITIES): + _READY["ok"] = True + return True + return False + + +def ro_con(): + """Per-thread cached read CURSOR on the shared instance for page/module store reads (the + open cost dominates repeated small queries — the Expenses retrofit measured 7.7s→1.0s). No + lock contention with the sync writer; a cursor that has gone bad is reopened once. REFUSES + until ready() — callers either fall back to live reads (modules) or relay a readable message + (Analyst). + + ⚠ THE GENERATION CHECK IS A CORRECTNESS GUARD, NOT AN OPTIMISATION (EXIT-4a). The liveness + probe below asks "does this cursor still work?", which is a different question from "is this + cursor still pointing at the file we are supposed to be reading?". After `use_path()` switches + tenants, a cursor cached in ANOTHER thread's `threading.local` can still answer `SELECT 1` — + and would then serve that thread real, plausible rows from the PREVIOUS tenant's file, with no + exception anywhere. `use_path` bumps the generation; a cursor opened under an older one is + discarded here, which is the only place every thread is guaranteed to pass through.""" + cur = getattr(_RO_TLS, "cur", None) + if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]: + try: + cur.close() + except Exception: + pass + cur, _RO_TLS.cur = None, None # the store moved underneath this thread + if cur is not None: + try: + cur.execute("SELECT 1") # cheap liveness probe + return cur + except Exception: + _RO_TLS.cur = None # stale → reopen below + if not ready(): + raise RuntimeError("the tenant data store is completing its FIRST sync on this " + "deployment — dashboards work meanwhile (live reads); retry " + "store queries in a few minutes") + cur = ro_cursor() + _RO_TLS.cur = cur + _RO_TLS.gen = _GENERATION[0] # stamp WHICH file this cursor belongs to + return cur + + +SEED_DATASET = os.environ.get("STORE_SEED_DATASET", "royal-imports/cfo-os-data") + + +def ensure_seed(): + """Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store + from the PRIVATE dataset's seed snapshot — Analyst-ready in ~a minute instead of a 20-40 min + XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores + the pilot views/routines when absent. Fail-quiet: no seed/token → normal backfill.""" + import shutil + if DB_PATH.exists(): + return False + tok = os.environ.get("HF_TOKEN") + if not tok: + return False + try: + from huggingface_hub import hf_hub_download + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + p = hf_hub_download(SEED_DATASET, "store_seed/royal.duckdb", + repo_type="dataset", token=tok) + # Atomic install: copy to a temp path then os.replace — a reader that guards on + # DB_PATH.exists() must never bind a half-written file into the shared instance. + tmp = DB_PATH.with_name(DB_PATH.name + ".tmp") + shutil.copyfile(p, tmp) + os.replace(tmp, DB_PATH) + for fn in ("views.json", "routines.json"): + tgt = DB_PATH.parent / fn + if not tgt.exists(): + try: + q = hf_hub_download(SEED_DATASET, f"store_seed/{fn}", + repo_type="dataset", token=tok) + shutil.copyfile(q, tgt) + except Exception: + pass + return True + except Exception: + return False + + +def connect(): + """A cursor on the shared instance for the SYNC WRITER (and status/validate). Returned as a + cursor so callers' con.close() frees the cursor without closing the process-wide instance; + one writer cursor is active at a time (the sync thread is sequential), readers use others.""" + cur = _instance().cursor() + cur.execute("""CREATE TABLE IF NOT EXISTS _sync_state ( + entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, + cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") + return cur + + +def _flatten(rec, fields, m2m=()): + out = {"id": rec["id"]} + for f in m2m: + v = rec.get(f) or [] + base = f.removesuffix("_ids") + out[f"{base}_id"] = v[0] if v else None + out[f] = json.dumps(list(v)) if v else None + for f in fields: + v = rec.get(f) + if isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int): + out[f"{f.removesuffix('_id')}_id"] = v[0] + out[f"{f.removesuffix('_id')}_name"] = str(v[1]) + elif isinstance(v, (list, tuple)): + out[f] = json.dumps(v) + elif v is False and f not in BOOL_FIELDS: # Odoo False = NULL for non-bool fields + out[f] = None + else: + out[f] = v + return out + + +def _cols(spec): + cols = ["id BIGINT PRIMARY KEY"] + for f in spec["fields"]: + base = f.removesuffix("_id") + if f.endswith("_id"): + cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"] + elif f in BOOL_FIELDS: + cols.append(f"{f} BOOLEAN") + elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed", + "price_subtotal", "margin", "purchase_price", "product_uom_qty", + "standard_price", "debit", "credit", "balance", "amount"): + cols.append(f"{f} DOUBLE") + else: + cols.append(f"{f} VARCHAR") + for f in spec.get("m2m") or []: + base = f.removesuffix("_ids") + cols += [f"{base}_id BIGINT", f"{f} VARCHAR"] + return cols + + +def _ensure_table(con, key, spec): + con.execute(f"CREATE TABLE IF NOT EXISTS {key} ({', '.join(_cols(spec))})") + + +def _ensure_columns(con, key, spec): + """Schema evolution: existing stores/seeds predate columns a newer spec introduces — + CREATE TABLE IF NOT EXISTS won't add them. Returns the newly added column names.""" + have = {r[1] for r in con.execute(f"PRAGMA table_info('{key}')").fetchall()} + added = [] + for c in _cols(spec): + name = c.split()[0] + if name not in have: + con.execute(f"ALTER TABLE {key} ADD COLUMN {c}") + added.append(name) + return added + + +def _field_cols(f): + """The store column(s) one spec field materialises into (m2o fields become _id + _name).""" + if f.endswith("_id"): + base = f.removesuffix("_id") + return [f"{base}_id", f"{base}_name"] + return [f] + + +def _backfill_columns(con, key, spec, log=print): + """Scalar/m2o schema evolution — the twin of _backfill_m2m, and for the same reason. + + _ensure_columns ALTERs a new column in, but existing rows NEVER re-sync (WE moved the schema; + their write_date didn't move), so a newly added field stays NULL forever — silently, and NULL + is indistinguishable from a legitimately empty value. That is how a store starts answering + 'no rows match' instead of erroring. Re-pull just the affected fields for every row, + paginated by id, marked durably so a crash retries rather than declaring completion. + + The fields are DECLARED in the spec (`backfill_fields`), never inferred from "which columns + did _ensure_columns just add" — that inference is wrong on the second run, when the ALTER has + already happened and the list comes back empty while the data is still missing. Declaring it + makes the migration re-entrant and reviewable. Leave the entry in place after it completes; + the durable marker, not the absence of the declaration, is what stops it re-running. + """ + fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"] + if not fields: + return + marker = f"{key}.cols.{','.join(sorted(fields))}" + st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?", + [marker]).fetchone() + if st and st[0] == "done": + return + # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses + # an hour of work to one dropped connection gets skipped by the next person under time + # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an + # interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring + # completion it never reached. + last, n = (st[1] or 0, st[2] or 0) if st else (0, 0) + if last: + log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)") + cols = [c for f in fields for c in _field_cols(f)] + setter = ", ".join(f"{c}=?" for c in cols) + while True: + recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], + fields, limit=5000, order="id asc") + if not recs: + break + rows = [_flatten(r, fields) for r in recs] + con.executemany(f"UPDATE {key} SET {setter} WHERE id=?", + [[r.get(c) for c in cols] + [r["id"]] for r in rows]) + last, n = recs[-1]["id"], n + len(recs) + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) + log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)") + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) + log(f" {key}: column backfill complete -> {n:,} rows") + + +def _backfill_m2m(con, key, spec, log=print): + """One-time targeted backfill after a schema migration: existing rows never re-sync (their + write_date didn't move when WE added a column), so pull every record where the m2m field is + SET and update in place. Rows with the field empty keep NULL — correct and free. Completion + is a durable _sync_state marker ('.m2m.') — NOT 'column just added': a crash + between the ALTER and this backfill (seen live: the SSL-failed first run) must retry.""" + for f in spec.get("m2m") or []: + marker = f"{key}.m2m.{f}" + if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone(): + continue + base = f.removesuffix("_ids") + # PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line + # has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone + # missing with no error — a partial backfill that reads as a complete one (the + # [[no-unverifiable-aggregates]] no-silent-caps rule). + rows, last, page = [], 0, 20000 + while True: + recs = O.search_read(spec["model"], + _base_domain(spec) + [(f, "!=", False), ("id", ">", last)], + [f], limit=page, order="id asc") + if not recs: + break + rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)] + last = recs[-1]["id"] + log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)") + if rows: + con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows) + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")]) + log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated") + + +def _upsert(con, key, spec, recs): + if not recs: + return + rows = [_flatten(r, spec["fields"], spec.get("m2m") or ()) for r in recs] + colnames = [c.split()[0] for c in _cols(spec)] + ids = [r["id"] for r in rows] + con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})") + con.executemany( + f"INSERT INTO {key} ({', '.join(colnames)}) VALUES ({', '.join('?' for _ in colnames)})", + [[r.get(c) for c in colnames] for r in rows]) + + +def _state(con, key): + row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?", + [key]).fetchone() + return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None + + +def _save_state(con, key, phase, last_id, cursor_wd): + n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""", + [key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")]) + + +def _base_domain(spec): + return [("active", "in", [True, False])] if spec["archivable"] else [] + + +def sync_entity(key, max_batches=200, batch=BATCH, log=print): + """One bounded, resumable sync pass for an entity. Backfill (by id) → live (by write_date). + Returns (phase, pulled_rows).""" + spec = ENTITIES[key] + con = connect() + _ensure_table(con, key, spec) + _ensure_columns(con, key, spec) + _backfill_columns(con, key, spec, log=log) + _backfill_m2m(con, key, spec, log=log) + st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0} + fields = spec["fields"] + list(spec.get("m2m") or []) + pulled = 0 + try: + if st["phase"] == "backfill": + last_id = st["last_id"] or 0 + for i in range(max_batches): + dom = _base_domain(spec) + [("id", ">", last_id)] + recs = O.search_read(spec["model"], dom, ["id"] + fields, + limit=batch, order="id asc") + if not recs: + # backfill complete → switch to live mode anchored at max write_date seen + wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0] + _save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00") + log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode") + return "live", pulled + _upsert(con, key, spec, recs) + last_id = recs[-1]["id"] + pulled += len(recs) + _save_state(con, key, "backfill", last_id, None) + if (i + 1) % 10 == 0: + log(f" {key}: backfill …{pulled:,} rows (id≤{last_id})") + log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)") + return "backfill", pulled + # live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP + # FALLBACK (found live 2026-07-17): a batch job can stamp >batch rows with ONE + # write_date second (an Odoo recompute stamped ~88k account_move rows '2026-07-15 + # 16:35:03'; stored values carry microseconds, XML-RPC strings don't) — then the + # write_date cursor can NEVER advance and every pass re-pulls the same first page + # forever. When a full batch leaves the cursor unchanged we switch to an ID-WALK over + # write_date >= cursor (order id asc, id > tie), checkpointed as 'cursor|' in + # cursor_wd so a killed walk resumes; on completion the cursor jumps one second past + # the stamp (safe: the walk covered every row in that second, and the stamp is in the + # past — guarded by a 2-minute recency check). + raw = st["cursor_wd"] or "1970-01-01 00:00:00" + cursor, _pipe, _tie = raw.partition("|") + walking, tie_id = bool(_pipe), int(_tie or 0) + walk_max = cursor + for _ in range(max_batches): + if walking: + dom = _base_domain(spec) + [("write_date", ">=", cursor), ("id", ">", tie_id)] + recs = O.search_read(spec["model"], dom, ["id"] + fields, + limit=batch, order="id asc") + if not recs: # walk complete → advance past the stamp + try: + wm = dt.datetime.strptime(walk_max, "%Y-%m-%d %H:%M:%S") + if wm < dt.datetime.utcnow() - dt.timedelta(minutes=2): + cursor = (wm + dt.timedelta(seconds=1)).strftime("%Y-%m-%d %H:%M:%S") + else: + cursor = walk_max + except ValueError: + cursor = walk_max + walking = False + log(f" {key}: id-walk complete -> cursor {cursor}") + break + _upsert(con, key, spec, recs) + pulled += len(recs) + tie_id = recs[-1]["id"] + walk_max = max(walk_max, max(str(r["write_date"]) for r in recs)) + _save_state(con, key, "live", st["last_id"], f"{cursor}|{tie_id}") + continue + dom = _base_domain(spec) + [("write_date", ">=", cursor)] + recs = O.search_read(spec["model"], dom, ["id"] + fields, + limit=batch, order="write_date asc, id asc") + if not recs or (len(recs) == 1 and str(recs[0].get("write_date")) == cursor + and pulled == 0 and _already(con, key, recs[0])): + break + new_cursor = str(recs[-1]["write_date"]) + if new_cursor == cursor and len(recs) == batch: + # full batch stuck on one second: enter the id-walk from id 0 (the write_date + # ordering shuffles ids within the stamp second, so the batch just pulled is + # NOT an id prefix — restart coverage by id; upserts make the overlap free) + walking, tie_id, walk_max = True, 0, cursor + _save_state(con, key, "live", st["last_id"], f"{cursor}|0") + log(f" {key}: mass-stamp second at {cursor} -> switching to id-walk") + continue + _upsert(con, key, spec, recs) + pulled += len(recs) + if new_cursor == cursor and len(recs) < batch: + cursor = new_cursor + break + cursor = new_cursor + # Checkpoint the cursor EVERY batch: a big delta (88k account_move rows after the + # 2026-07-15 recompute) means a killed pass otherwise re-pulls everything — + # upserts are idempotent, but the wasted RPC is real (learned 2026-07-17). + _save_state(con, key, "live", st["last_id"], cursor) + _save_state(con, key, "live", st["last_id"], + f"{cursor}|{tie_id}" if walking else cursor) + if pulled: + log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}" + + (f" walking id>{tie_id}" if walking else "") + ")") + return "live", pulled + finally: + con.close() + + +def _already(con, key, rec): + return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone()) + + +def sync_all(entities=None, max_batches=200, log=print): + out = {} + for key in (entities or list(ENTITIES)): + t0 = time.time() + phase, pulled = sync_entity(key, max_batches=max_batches, log=log) + out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)} + # A full pass that lands every entity in live mode latches process readiness in-process — so + # readers never race the writer's lock to (re)discover the store is usable. + if entities is None and out and all(v["phase"] == "live" for v in out.values()): + mark_ready() + return out + + +def reconcile_deletes(entities=None, log=print): + """Remove store rows hard-deleted in Odoo. unlink() doesn't move write_date, so the cursor + sync can never see a deletion — the count parity DETECTS the drift (by design); this repairs + it. Pulls the full live id set per entity in bounded id-ascending pages (id-only reads are + cheap) and deletes store ids not present live. Run from sync_runner / maintenance, not the + app's frequent passes.""" + con = connect() + try: + out = {} + for key in (entities or list(ENTITIES)): + spec = ENTITIES[key] + live_ids, last = set(), 0 + while True: + recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], + ["id"], limit=50000, order="id asc") + if not recs: + break + live_ids.update(r["id"] for r in recs) + last = recs[-1]["id"] + store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()} + dead = store_ids - live_ids + if dead: + con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)") + con.execute("DELETE FROM _dead") + con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead]) + con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)") + con.execute("DELETE FROM _dead") + n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key]) + out[key] = len(dead) + log(f" {key}: reconciled {len(dead):,} hard-deleted rows") + return out + finally: + con.close() + + +def status(): + con = connect() + try: + return {r[0]: {"phase": r[1], "cursor": r[3], "rows": r[4], "updated": r[5]} + for r in con.execute("SELECT * FROM _sync_state").fetchall()} + finally: + con.close() + + +# ------------------------------------------------------------------ parity (store vs LIVE Odoo) + +def _check(name, ours, live, tol=0.01): + gap = abs((ours or 0) - (live or 0)) + return {"check": name, "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "live": live} + + +def validate(pre=None, months=("2026-01-01", "2026-07-01")): + """Raw-fidelity parity: the store must mirror live Odoo — row counts per synced entity + + unscoped monetary sums over a window, to the cent. Only checks entities in LIVE phase.""" + con = connect() + out = [] + try: + live_state = status() + d0, d1 = months + for key, spec in ENTITIES.items(): + if live_state.get(key, {}).get("phase") != "live": + continue + ours = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + live = O.get_odoo().search_count(spec["model"], _base_domain(spec)) + out.append(_check(f"store.{key} row count == live search_count", ours, live, tol=0)) + if live_state.get("sale_order_line", {}).get("phase") == "live": + ours = con.execute( + "SELECT coalesce(sum(l.price_subtotal),0) FROM sale_order_line l " + "JOIN sale_order o ON o.id = l.order_id " + "WHERE o.date_order >= ? AND o.date_order < ?", [d0, d1]).fetchone()[0] + live = O.sum_field("sale.order.line", + [("order_id.date_order", ">=", f"{d0} 00:00:00"), + ("order_id.date_order", "<", f"{d1} 00:00:00")], + "price_subtotal") + out.append(_check(f"store Σ line revenue [{d0}..{d1}) == live (unscoped)", ours, live)) + if live_state.get("sale_order", {}).get("phase") == "live": + ours = con.execute( + "SELECT coalesce(sum(amount_untaxed),0) FROM sale_order " + "WHERE date_order >= ? AND date_order < ?", [d0, d1]).fetchone()[0] + live = O.sum_field("sale.order", + [("date_order", ">=", f"{d0} 00:00:00"), + ("date_order", "<", f"{d1} 00:00:00")], "amount_untaxed") + out.append(_check(f"store Σ order amount_untaxed [{d0}..{d1}) == live (unscoped)", + ours, live)) + finally: + con.close() + return out + + +if __name__ == "__main__": + args = sys.argv[1:] + if args and args[0] == "status": + for k, v in status().items(): + print(f" {k:18s} {v['phase']:9s} rows={v['rows']:>9,} cursor={v['cursor']} ({v['updated']})") + elif args and args[0] == "reconcile": + ents = args[1].split(",") if len(args) > 1 else None + for k, n in reconcile_deletes(ents).items(): + print(f" {k:18s} -{n:,} hard-deleted rows") + elif args and args[0] == "validate": + ok = True + for c in validate(): + mark = "OK " if c["ok"] else "XX " + ok &= c["ok"] + print(f" {mark}{c['check'][:80]} gap={c['gap']}") + print("PARITY:", "GREEN" if ok else "RED") + sys.exit(0 if ok else 1) + else: + ents = args[0].split(",") if args else None + mb = int(args[1]) if len(args) > 1 else 200 + for k, v in sync_all(ents, max_batches=mb).items(): + print(f" {k:18s} {v['phase']:9s} +{v['pulled']:,} rows in {v['secs']}s") diff --git a/royalimports_os/harness/evals.py b/royalimports_os/harness/evals.py new file mode 100644 index 0000000000000000000000000000000000000000..d64172f90f9c2c9dc104254e328290b92d778b2f --- /dev/null +++ b/royalimports_os/harness/evals.py @@ -0,0 +1,134 @@ +"""harness/evals.py — the Analyst eval gate (OM-4, 2026-07-11). + +Runs the golden set (evals/analyst_golden.yml — truths from the parity-proven semantic layer over +CLOSED windows) against the Analyst and scores DETERMINISTICALLY: expected number found in the +answer (tolerance-aware, handles $/commas/% forms), required substrings, required tools in the +trace, required artifacts. No LLM judge in v1 — the validate() culture applied to AI answers. + +Gate rule (from the plan): run before any deploy that touches model/ or the Analyst; ship only on +pass-rate >= the bar (start 80%, tighten as the model/skills improve). + +Usage: + python -X utf8 -c "import ssl; ssl._create_default_https_context=ssl._create_unverified_context; + import harness.evals as E; E.run()" # live (needs OPENROUTER_API_KEY) + E.run(chat_fn=scripted) # offline harness test +""" +import re +from pathlib import Path + +import yaml + +GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml" + + +def _numbers(text): + """Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too.""" + out = [] + for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text): + try: + v = float(m.group(1).replace(",", "")) + except ValueError: + continue + out.append(v) + if m.group(2): # percent form: also offer the fraction + out.append(v / 100.0) + return out + + +def _norm_ws(s): + """Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break + space) and friends inside names — 'Poppy\\u202fFlowers' failed a plain substring check on + two different models (the 'top_customer flake', diagnosed 2026-07-12).""" + return re.sub(r"\s+", " ", s) + + +def _check(item, result): + exp = item.get("expect") or {} + answer = _norm_ws(result.get("answer") or "") + reasons = [] + if result.get("exhausted"): + reasons.append("tool budget exhausted") + if "value" in exp: + want = float(exp["value"]) + tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01) + nums = _numbers(answer) + if exp.get("allow_pct_form"): + nums += [n for n in list(nums)] + if not any(abs(n - want) <= tol for n in nums): + reasons.append(f"expected {want:,.2f} (±{tol:,.2f}) not in answer") + for s in exp.get("contains", []): + if _norm_ws(s).lower() not in answer.lower(): + reasons.append(f"missing substring {s!r}") + # WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing; + # some failures are about saying a WRONG thing that a right answer never says — e.g. naming an + # internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative + # assertion or the eval passes on an answer that is confidently incorrect. + for s in exp.get("not_contains", []): + if _norm_ws(s).lower() in answer.lower(): + reasons.append(f"forbidden substring {s!r} present") + used = [t["tool"] for t in result.get("tool_trace", [])] + for t in exp.get("tools", []): + if t not in used: + reasons.append(f"tool {t!r} not used (trace: {used})") + for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear + if t in used: + reasons.append(f"tool {t!r} used but forbidden here (trace: {used})") + if exp.get("artifact") == "chart": + if not any("chart" in a for a in result.get("artifacts", [])): + reasons.append("no chart artifact produced") + return (not reasons), reasons + + +def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True): + """Run the golden set. Returns {passed, total, pass_rate, gate_ok, results}.""" + import harness.analyst as A + items = yaml.safe_load(GOLDEN.read_text(encoding="utf-8"))["items"] + if only: + items = [i for i in items if i["id"] in only] + results, passed = [], 0 + import os as _os + import time as _time + _os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast + infra_fails = 0 + aborted = False + for item in items: + _time.sleep(6) # pace the free-tier per-minute windows (Groq/Cerebras) + kw = {"chat_fn": chat_fn} if chat_fn else {} + if model: + kw["model"] = model + try: + res = A.ask(item["ask"], telemetry_kind="analyst_eval", **kw) + ok, reasons = _check(item, res) + infra_fails = 0 + except Exception as e: + res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"] + # Provider exhaustion is NOT a model failure: two in a row means the ladder is out + # of quota — abort and report INCONCLUSIVE instead of burning the remaining items + # into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%"). + if "all LLM providers failed" in str(e): + infra_fails += 1 + if infra_fails >= 2: + aborted = True + passed += ok + results.append({"id": item["id"], "ok": ok, "reasons": reasons, + "usage": res.get("usage"), "iterations": res.get("iterations")}) + if verbose: + mark = "OK " if ok else "XX " + print(f" {mark}{item['id']:18s}" + ("" if ok else f" — {'; '.join(reasons)[:90]}")) + if aborted: + break + if aborted: + ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")] + if verbose: + print(f"\nEVAL ABORTED — LLM providers exhausted after {len(results)} item(s); " + f"{passed}/{len(ran) or 1} of the items that actually ran passed. " + "Gate is INCONCLUSIVE (not a model failure) — re-run when quota recovers.") + return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False, + "inconclusive": True, "results": results} + rate = passed / len(items) if items else 0.0 + gate = rate >= bar + if verbose: + print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): " + f"{'PASS' if gate else 'FAIL'}") + return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate, + "results": results} diff --git a/royalimports_os/harness/filter_sql.py b/royalimports_os/harness/filter_sql.py new file mode 100644 index 0000000000000000000000000000000000000000..b49376a9ad40ae7d9262fed449ae691e0b0d84f5 --- /dev/null +++ b/royalimports_os/harness/filter_sql.py @@ -0,0 +1,654 @@ +"""harness/filter_sql.py — the grid's filter/sort engine, compiled to SQL (CG-1). + +THE PORT. `customer-grid/useVisibleRows.ts` evaluates the Airtable-parity filter TREE and the +multi-level sort in TypeScript, per row, in the browser. That works at customer grain (1,550 rows, +576 KB) and does NOT work at line grain, so a line-grain table has to filter server-side. This +module is the second implementation of that contract, emitting parameterized SQL instead of +booleans. `aios_grid.clean_filter_tree` (the validator) is the third. All three are held in +lock-step by `aios-web/verify_filter_engine.py`, which runs the SAME cases through the TS engine, +this compiler (executed on DuckDB), and the validator, and compares row sets. + +Design: this file is PURE — no model, no store, no app imports. It takes a filter tree plus a +`columns` spec and returns SQL + params. `harness/semantic.store_columns` builds the spec from a +topic; nothing else here knows what a topic is. + + columns = {colId: {"sql": , "type": , "aggregate": bool}} + +`sql` is a trusted IDENTIFIER/expression from the semantic model (never user input); every VALUE +is a bound parameter. That split is the whole injection story. + +-------------------------------------------------------------------------------------------- +TRI-STATE, AND WHY IT BECOMES COMPILE-TIME PRUNING +-------------------------------------------------------------------------------------------- +`evalNode` in TS returns true | false | null, where null means "this node is INACTIVE and its +parent must IGNORE it". Get that wrong and a half-typed condition returns true, makes its `or` +group true, and shows every record. + +Activeness reads ONLY the rule (op, value, value2, is-the-column-known) — never the row. So in +SQL it is not a third truth value at all: it is decided while COMPILING. An inactive node emits +no SQL, a group with no active children emits no SQL, and a root with nothing active emits NO +PREDICATE AT ALL (`compile_filter_tree` returns None) — which is exactly `matchFilterTree` +returning true for every row. + +-------------------------------------------------------------------------------------------- +THE FOUR PLACES A NAIVE PORT SILENTLY DIVERGES +-------------------------------------------------------------------------------------------- +JavaScript coerces null to 0 / "" before comparing; SQL propagates NULL and drops the row. + + 1. `toNum(null)` is 0, so a NULL revenue PASSES `>= 0`. SQL: NULL >= 0 -> NULL -> dropped. + 2. `String(null)` is "", so a NULL state PASSES `is not FL`. SQL: NULL <> 'fl' -> dropped. + 3. `"" <= "2026-01-01"` is true, so a NULL date PASSES `is before`. SQL: dropped. + 4. `contains` is a LITERAL substring test. `LIKE '%v%'` treats % and _ as wildcards, so a + value containing either matches the wrong rows. Hence `strpos`, not LIKE. + +Every column is therefore read through `_value_sql`, which COALESCEs to the same zero-value +JavaScript would have coerced to. The one thing that must NOT be coalesced away is the +blank test itself — `isBlank` looks at the RAW value, and numeric 0 is NOT blank. + +-------------------------------------------------------------------------------------------- +AND THE FIFTH: `default: return true` IS ACTIVE, NOT INACTIVE +-------------------------------------------------------------------------------------------- +`matchFilter`'s per-type switch ends in `default: return true`, reachable whenever an operator +does not apply to the field type — `contains` on a currency/date field, or `gt`/`between` on a +text field. A persisted tree can hold such a pair. That rule is ACTIVE and evaluates TRUE for +every row, which is NOT the same as inactive: under `or` it makes the whole group true and shows +everything. We emit `1=1` for those, and the gate asserts the difference is observable. + +-------------------------------------------------------------------------------------------- +INPUT CONTRACT +-------------------------------------------------------------------------------------------- +Feed this CLEANED trees (`aios_grid.clean_filter_tree`). It is deliberately faithful to the TS +engine rather than fail-closed on garbage — an unknown OP compiles to `1=1` exactly as TS's +`default:` branch returns true — because the validator's job is to reject garbage and this +module's job is to mean the same thing as the client. The gate proves the composition. + +Ported ahead of their consumer (CG-2) and gated at the same fidelity as the tree: +`compile_search` (`matchSearch`) and `member_ids` (the hand-picked-members union in +`useVisibleRows`). +""" +import math +import re +from collections import namedtuple + +# Mirrors customer-grid/types.ts FilterOp. verify_filter_engine.py asserts this set is +# byte-identical to aios_grid.FILTER_OPS — three copies of a vocabulary need a drift gate. +FILTER_OPS = frozenset({ + 'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', + 'gt', 'gte', 'lt', 'lte', 'between', 'within', +}) +#: The RESERVED pseudo-column of a cohort-membership leaf. Mirrors types.ts COHORT_FIELD. +COHORT_FIELD = '__cohort__' +#: Set operators over a SET of cohorts. Mirrors types.ts COHORT_OPS; DISJOINT from FILTER_OPS. +COHORT_OPS = frozenset({'anyOf', 'allOf', 'noneOf'}) +#: The single-cohort ops the leaf shipped with, kept as permanent aliases. types.ts mirrors this. +COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} +#: Mirrors types.ts MAX_COHORT_IDS. +MAX_COHORT_IDS = 20 + + +def parse_cohort_ids(value): + """The cohorts a leaf names. Mirrors types.ts `cohortIds()` and aios_grid.parse_cohort_ids.""" + out = [] + for raw in ('' if value is None else str(value)).split(','): + cid = raw.strip()[:120] + if not cid or cid in out: + continue + out.append(cid) + if len(out) >= MAX_COHORT_IDS: + break + return out +# types.ts VALUE_FREE_OPS: no value control is rendered, and they must never be judged +# "inactive because the value is blank" — that would turn them into no-ops. +VALUE_FREE_OPS = frozenset({'isEmpty', 'isNotEmpty'}) +# types.ts isNumericType() +NUMERIC_TYPES = frozenset({'currency', 'int', 'pct'}) +TEXTUAL_TYPES = frozenset({'text', 'status'}) + +SQL_TRUE = '1=1' + +_NUM_CMP = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'} +_DATE_CMP = dict(_NUM_CMP) + +#: sql = the predicate (already parenthesised); params = bound values, positional; +#: uses_aggregate = the predicate references an aggregate column, so it belongs in HAVING +#: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an +#: always-true `1=1` branch references nothing, and must not force HAVING routing). +Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used') + + +# --------------------------------------------------------------------------- value coercion + +# JS `Number(string)` trims exactly this set (WhiteSpace + LineTerminator). Python's float() and +# str.strip() use a DIFFERENT set -- notably U+0085 NEL, which Python treats as whitespace and JS +# does not -- so the set is spelled out rather than left to strip(). +_JS_WS = "".join(chr(c) for c in (9, 10, 11, 12, 13, 32, 160, 5760, 8232, 8233, + 8239, 8287, 12288, 65279)) + \ + "".join(chr(c) for c in range(8192, 8203)) +# ASCII-ONLY on purpose: `\d` in a Python str pattern matches every Unicode decimal digit, so +# `\d` here would parse the fullwidth "250" that JS rejects -- reintroducing the exact bug +# this function exists to avoid. +_JS_DEC = re.compile(r"^[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$") +_JS_INF = re.compile(r"^[+-]?Infinity$") +_JS_RADIX = ((re.compile(r"^0[xX][0-9a-fA-F]+$"), 16), + (re.compile(r"^0[oO][0-7]+$"), 8), + (re.compile(r"^0[bB][01]+$"), 2)) + + +def to_num(v): + """Port of `toNum`: JS `Number(v)`, then `Number.isFinite` or 0. + + NOT `float(v)`. Python's float() is MORE permissive than JS's Number() in ways that make SQL + narrow where the client shows everything -- the worse direction, because at line grain that + is a silent under-fetch sitting beside an honest-looking row_count: + + "1_000" float() -> 1000.0 (PEP-515 underscores) Number() -> NaN -> 0 + "250" float() -> 250.0 (any Unicode Nd) Number() -> NaN -> 0 + "\\u0085" + "12" float() -> 12.0 (NEL is Python whitespace) Number() -> NaN -> 0 + + and LESS permissive in one family, which errs the other way: + + "0x10" float() -> raises -> 0 Number() -> 16 (also 0b, 0o) + + A tree carrying any of these is reachable: the React builder renders , + but persisted / seeded / imported view JSON goes through `clean_filter_tree`, which passes + the value string through untouched by design (it validates structure, not numeric syntax). + + Agreed by construction on the rest: ""/None -> 0, "abc"/"12,000" -> 0, " 250 " -> 250, + ".5"/"5."/"+5", "1e400" -> Infinity -> 0, "-0" -> 0, and float64 truncation of + "9007199254740993" -> 9007199254740992. + """ + if isinstance(v, bool): + return 1.0 if v else 0.0 # JS Number(true) === 1 + if isinstance(v, (int, float)): + n = float(v) + return n if math.isfinite(n) else 0.0 + if v is None: + return 0.0 # JS Number(null) === 0 + s = str(v).strip(_JS_WS) + if s == "": + return 0.0 # JS Number("") === 0 + if _JS_INF.match(s): + return 0.0 # Infinity is not finite -> 0 + for pattern, base in _JS_RADIX: + if pattern.match(s): + return float(int(s[2:], base)) + if not _JS_DEC.match(s): + return 0.0 # JS NaN -> 0 + n = float(s) + return n if math.isfinite(n) else 0.0 + + +def _value_sql(spec): + """The ONE expression for a column, used by BOTH the filter and the sort. + + Two expressions would let two rows tie in the grid and not tie in SQL (dates truncated for + comparison but ordered by the raw timestamp is the obvious way to get that wrong), so the + next sort key would break the tie in one engine and not the other. + + numeric: `round_even(x, 0)` reproduces `aios_grid._round`, which is Python's round() = + BANKER'S rounding. DuckDB's plain ROUND() rounds half AWAY from zero, so ROUND(1234.5)=1235 + while the grid shows 1234 — `eq 1234` would then match on screen and miss in SQL. The grid + displays rounded values; filters must agree with what is on screen. + date: truncate to the 10-char ISO shape the payload carries, so lexical compare is + chronological and a timestamp column cannot smuggle ' 00:00:00' into the comparison. + text: lowercased, matching `String(raw ?? "").toLowerCase()`. + """ + e = spec['sql'] + t = spec['type'] + if t in NUMERIC_TYPES: + return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), 0), 0)" + if t == 'date': + return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')" + return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))" + + +def _blank_sql(spec): + """Port of `isBlank`: null/undefined/"" are blank — numeric 0 is NOT. + + Reads the RAW column, never `_value_sql` (which has already coalesced blanks into 0/"", + the very distinction this test exists to make). + """ + e = spec['sql'] + if spec['type'] in NUMERIC_TYPES: + # 0 stays non-blank: CAST(0 AS VARCHAR) is '0'. The '' arm covers a text-typed column + # holding "", which `_round` passes through unchanged into the payload. + return f"(({e}) IS NULL OR CAST({e} AS VARCHAR) = '')" + if spec['type'] == 'date': + return f"(COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '') = '')" + return f"(COALESCE(CAST({e} AS VARCHAR), '') = '')" + + +# --------------------------------------------------------------------------- activeness + +def is_rule_active(rule, columns): + """Port of `isRuleActive` + `evalNode`'s two leaf guards. Row-independent by construction. + + Unknown column -> inactive (evalNode returns null before matchFilter is ever called). + Value-free ops are always active. `between` needs BOTH bounds. + + ⚠ The four VALUE-FREE ANCHOR MODES (`is before [today]`) carry no value, and the historical + "blank value = inactive" rule would IGNORE them — widening the result under a count nobody + would doubt. Same shape as the isEmpty/isNotEmpty no-op bug, arriving by a different door. + """ + if not isinstance(rule, dict): + return False + col = rule.get('colId') + if col == COHORT_FIELD: # a cohort leaf has no column to look up + # Driven by the SET it names, so a value of "," is INACTIVE rather than active-and- + # unanswerable. Mirrors the cohort branch at the top of types.ts isRuleActive. + return bool(parse_cohort_ids(rule.get('value'))) + if col not in columns: + return False + op = rule.get('op') + if op in VALUE_FREE_OPS: + return True + if op == 'within': + return _window_bounds(rule.get('dateWindow'), None, probe=True) + if rule.get('rhs') is not None: + rhs = rule['rhs'] + # An rhs naming a column this table does not have is IGNORED, by the same rule as an + # unknown left column two lines above — for EITHER kind. Ignoring one side and hiding + # every row for the other would make the comparison asymmetric in a way nobody could + # predict. Doing it here rather than in _compile_leaf keeps activeness the single place + # that decides, which is what makes the compiled branch below provably unreachable. + if not isinstance(rhs, dict) or rhs.get('colId') not in columns: + return False + return _rhs_complete(rhs) + if rule.get('dateMode') in _ANCHOR_VALUE_FREE: + return True + value = _as_str(rule.get('value')) + if op == 'between': + v2 = rule.get('value2') + return value != '' and v2 is not None and _as_str(v2) != '' + return value != '' + + +def _rhs_complete(rhs): + """Port of types.ts isRhsComplete. An incomplete rhs makes the rule INACTIVE, exactly as a + half-typed value does.""" + if not isinstance(rhs, dict) or not rhs.get('colId'): + return False + if rhs.get('kind') == 'measure': + return _windows().normalize(rhs.get('window')) is not None + return rhs.get('kind') == 'field' + + +def _windows(): + """`harness.windows` imported lazily, so this module stays importable on its own. + + filter_sql is otherwise PURE (no model/store/app imports) and that property is what lets + aios_grid embed the grid anywhere. windows.py is the one exception and it is a deliberate + one: the date vocabulary is a two-ENGINE contract, and a third copy of "what the past month + means" is precisely the drift verify_windows.py exists to prevent. It imports nothing but + datetime, so purity is preserved in substance. + """ + from harness import windows as _w + return _w + + +#: Mirrors windows.ANCHOR_VALUE_FREE, resolved lazily on first use (see _windows). +class _AnchorValueFree: + def __contains__(self, mode): + return mode in _windows().ANCHOR_VALUE_FREE + + +_ANCHOR_VALUE_FREE = _AnchorValueFree() + + +def _window_bounds(spec, today, probe=False): + """Resolve a date-condition RANGE, or refuse. + + `probe=True` answers only "is this a resolvable window?" without needing `today` — which is + what activeness has to know while compiling, before any date is in hand. + + ⚠ A missing `today` on the real path RAISES rather than widening. The client's answer to the + same gap is to match nothing; the server's must be to refuse, because a server that quietly + drops a condition reports an authoritative row_count for a query nobody asked for. Same rule + as the sibling/depth caps: truncate on the client, refuse on the server. + """ + w = _windows() + if probe: + return w.normalize(spec) is not None + if today is None: + raise ValueError( + 'a relative-date condition needs `today` — compile_filter_tree(today=...). ' + 'Refusing rather than dropping the condition, which would widen the result while ' + 'the count beside it still looked authoritative.') + return w.resolve(spec, today) + + +def _as_str(v): + """The builder stores values as raw input strings; be tolerant of a number that survived + an older payload. `str(500) != ''` is active and `to_num` reads it back, matching JS where + `500 === ""` is false.""" + return '' if v is None else str(v) + + +# --------------------------------------------------------------------------- leaf + +def _compile_leaf(rule, columns, state): + """Port of `matchFilter`. Returns (sql, params) or None when the rule is INACTIVE.""" + if not is_rule_active(rule, columns): + return None + col_id = rule['colId'] + op = rule.get('op') + value = _as_str(rule.get('value')) + + # --- owner item 5: a COHORT leaf, before any column lookup ------------------------------- + # `__cohort__` is not a column. Handled first for the same reason evalNode handles it first: + # falling through to the column path would make it INACTIVE, and inactive means every row. + if col_id == COHORT_FIELD: + return _compile_cohort(rule, state) + + spec = columns[col_id] + + def used(sql, params=()): + state['used'].add(col_id) + if spec.get('aggregate'): + state['aggregate'] = True + return (sql, list(params)) + + # --- value-free ops, BEFORE any value-based short-circuit (the no-op bug) --------------- + if op == 'isEmpty': + return used(_blank_sql(spec)) + if op == 'isNotEmpty': + return used(f"NOT {_blank_sql(spec)}") + + v = _value_sql(spec) + t = spec['type'] + + # --- CG-9: the comparand is another COLUMN, coerced through the LEFT field's type -------- + # `_value_sql` on a spec whose `sql` is the RIGHT column and whose `type` is the LEFT one — + # exactly what TS does (`toNum(other)` for a numeric LHS, `String(other ?? "")` for a date, + # `.toLowerCase()` for text). Reading the right column through its OWN type would compare a + # rounded number against an unrounded one and disagree on the ties. + rhs_v = None + if rule.get('rhs') is not None: + # No `rspec is None` branch: `is_rule_active` already rejected an rhs naming a column + # this table does not have, so it cannot be missing here. A defensive fallback would be + # unreachable, and the negative control showed exactly that — deleting it changed + # nothing, which is how dead code hides behind a careful-looking line. + rspec = columns[rule['rhs']['colId']] + rhs_v = _value_sql({'sql': rspec['sql'], 'type': t}) + state['used'].add(rule['rhs']['colId']) + if rspec.get('aggregate'): + state['aggregate'] = True + + if t in NUMERIC_TYPES: + if rhs_v is not None: + return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, []) + if op == 'between': + a, b = to_num(value), to_num(rule.get('value2')) + return used(f"({v} >= ? AND {v} <= ?)", [min(a, b), max(a, b)]) + if op in _NUM_CMP: + return used(f"({v} {_NUM_CMP[op]} ?)", [to_num(value)]) + return (SQL_TRUE, []) # contains/doesNotContain on a number -> default: true + + if t == 'date': + if op == 'within': + bounds = _window_bounds(rule.get('dateWindow'), state.get('today')) + if bounds is None: + return ('1=0', []) # unreachable: activeness probed it. Never widen. + lo, hi = bounds + # `{v} <> ''` is NOT redundant. `_value_sql` COALESCEs a NULL date to '', and + # '' <= any upper bound is true — so an open-ended window would sweep in every + # never-ordered customer. TS guards it with `if (a === "") return false`. + parts, params = [f"{v} <> ''"], [] + if lo is not None: + parts.append(f"{v} >= ?") + params.append(lo) + if hi is not None: + parts.append(f"{v} <= ?") + params.append(hi) + return used('(' + ' AND '.join(parts) + ')', params) + if rhs_v is not None: + return used(f"({v} {_DATE_CMP[op]} {rhs_v})") if op in _DATE_CMP else (SQL_TRUE, []) + if op == 'between': + v2 = _as_str(rule.get('value2')) + lo, hi = (value, v2) if value < v2 else (v2, value) + return used(f"({v} >= ? AND {v} <= ?)", [lo, hi]) + if op in _DATE_CMP: + anchor = _resolve_anchor(rule.get('dateMode'), value, state.get('today')) + if anchor is None: + return ('1=0', []) # unresolvable anchor -> matches nothing (TS: false) + return used(f"({v} {_DATE_CMP[op]} ?)", [anchor]) + return (SQL_TRUE, []) # contains/doesNotContain on a date -> default: true + + # text / status — case-insensitive. strpos, not LIKE: `includes` is a LITERAL substring + # test, so a value holding % or _ must not act as a wildcard. + # + # BOTH SIDES fold in SQL (`LOWER(?)`, never Python's str.lower()). Python and JS agree on + # case folding; DuckDB does NOT — LOWER('ISTANBUL' with a dotted capital I) drops the dot + # while both languages keep it as i + combining dot. Folding the needle in Python and the + # haystack in DuckDB therefore produced two different strings, and a value could fail to + # match ITSELF. One folder for both operands is the only version that cannot do that. + if rhs_v is not None: + # Both operands are already folded by `_value_sql` (text branch), so no LOWER(?) here. + if op == 'contains': + return used(f"(strpos({v}, {rhs_v}) > 0)") + if op == 'doesNotContain': + return used(f"(strpos({v}, {rhs_v}) = 0)") + if op == 'eq': + return used(f"({v} = {rhs_v})") + if op == 'neq': + return used(f"({v} <> {rhs_v})") + return (SQL_TRUE, []) + if op == 'contains': + return used(f"(strpos({v}, LOWER(?)) > 0)", [value]) + if op == 'doesNotContain': + return used(f"(strpos({v}, LOWER(?)) = 0)", [value]) + if op == 'eq': + return used(f"({v} = LOWER(?))", [value]) + if op == 'neq': + return used(f"({v} <> LOWER(?))", [value]) + return (SQL_TRUE, []) # gt/gte/lt/lte/between/within on text -> default: true + + +def _resolve_anchor(mode, value, today): + """A date condition's anchor -> one ISO date, or None. `today` may be absent ONLY for the + `exact` mode, which does not use it. + + The sentinel is deliberate and load-bearing for BACK-COMPAT: every date filter that existed + before anchors is `exact`, and every caller that compiles one (store_query, store_rows) + passes no `today`. Requiring it unconditionally would turn a working filter into an + exception. A RELATIVE anchor still refuses — the server never widens. + """ + mode = mode or 'exact' + if today is None: + if mode != 'exact': + raise ValueError( + f"the date anchor {mode!r} is relative and needs `today` — " + f"compile_filter_tree(today=...). Refusing rather than dropping the condition, " + f"which would widen the result under an authoritative-looking count.") + # `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and + # gets the same answer every time), so any valid date serves as the unused argument. + today = '1970-01-01' + return _windows().resolve_anchor(mode, value, today) + + +def _compile_cohort(rule, state): + """Port of evalNode's cohort branch: membership in a SET of hand-curated sets of row ids. + + Owner 2026-07-27: the leaf names one OR MORE cohorts and asks `anyOf` / `allOf` / `noneOf`. + A single id is a one-element set, so the shipped `is part of` compiles down this same path. + + States, mirroring TS exactly: + every set present -> the union (anyOf / noneOf) or the intersection (allOf) + a set present but EMPTY -> that member contributes 1=0, because nobody is in an empty + cohort. So `allOf` collapses to 1=0 and `noneOf` to 1=1 — + which is what `set.has(pid)` on an empty Set does in TS. + ANY set ABSENT -> 1=0 for ALL THREE ops. The condition is unanswerable, and + unanswerable matches nothing: answering "everything" to + `is none of [a cohort we cannot see]` is the widening sin + wearing a plausible face, and answering it for the members we + DO have would silently ask a different question. + + COALESCE(... , FALSE) because a NULL row id makes `id IN (...)` NULL, and `NOT NULL` is NULL + — SQL would drop the row where TS keeps it. + """ + sets = state.get('cohort_sets') + if sets is None: + raise ValueError( + 'a cohort condition needs cohort_sets — compile_filter_tree(cohort_sets=...). ' + 'Refusing rather than dropping it: a dropped condition widens the result while the ' + 'count beside it still looks authoritative.') + id_sql = state.get('id_sql') + if not id_sql: + raise ValueError('a cohort condition needs id_sql (the row-identity column)') + op = COHORT_OP_ALIASES.get(rule.get('op'), rule.get('op')) + named = parse_cohort_ids(rule.get('value')) + members = [sets.get(cid) for cid in named] + if not named or any(m is None for m in members): + return ('1=0', []) + + parts, params = [], [] + for ids in members: + ids = [int(x) for x in ids] + if not ids: + parts.append('1=0') + continue + parts.append(f"COALESCE({id_sql} IN ({','.join('?' for _ in ids)}), FALSE)") + params.extend(ids) + if op == 'allOf': + return ('(' + ' AND '.join(parts) + ')', params) + union = '(' + ' OR '.join(parts) + ')' + return ((f"(NOT {union})" if op == 'noneOf' else union), params) + + +# --------------------------------------------------------------------------- tree + +def _compile_node(node, columns, state): + """Port of `evalNode`. Returns (sql, params), or None when the node is INACTIVE. + + A group combines only its ACTIVE children with its OWN conjunction; a group with no active + children is itself inactive and disappears. That is what stops a half-built group from + either zeroing the grid (`and`) or disabling it (`or`). + """ + if not isinstance(node, dict): + return None + if isinstance(node.get('children'), list): + parts, params = [], [] + for child in node['children']: + got = _compile_node(child, columns, state) + if got is None: + continue # inactive -> ignored entirely + parts.append(got[0]) + params.extend(got[1]) + if not parts: + return None + joiner = ' OR ' if node.get('conj') == 'or' else ' AND ' + return ('(' + joiner.join(parts) + ')', params) + return _compile_leaf(node, columns, state) + + +def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql=None, + today=None, cohort_sets=None): + """Port of `matchFilterTree` (+ the `memberPids` union in `useVisibleRows`). + + Returns a `Compiled`, or **None when nothing is active** — meaning no predicate at all, not + a false one. `matchFilterTree` returns true when the tree is inactive, so emitting anything + here would narrow rows the client would have shown. + + `member_ids` reproduces `members.has(r.pid) || matchFilterTree(...)`. Note the union lives + INSIDE `if (filters.length)` and is OR'd with a tree that is true for everything when + inactive — so an inactive tree must emit NOTHING, never `pid IN (...)`, which would narrow + to just the hand-picked members. Same family of bug as the tri-state one, inverted. + + `today` is the TENANT'S today, required by any relative date condition (`is within …`, + `is before [one month ago]`) and by nothing else — an `exact` date needs none, which is what + keeps every pre-anchor caller working unchanged. `cohort_sets` is `{cohortId: [pid, ...]}` + for cohort-membership leaves. Both REFUSE when needed and missing; neither defaults. + """ + state = {'aggregate': False, 'used': set(), + 'today': today, 'cohort_sets': cohort_sets, 'id_sql': id_sql} + got = _compile_node({'conj': conj, 'children': list(nodes or [])}, columns or {}, state) + if got is None: + return None + sql, params = got + if member_ids: + if not id_sql: + raise ValueError('member_ids requires id_sql (the row-identity column)') + ids = [int(x) for x in member_ids] + sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))" + params = list(params) + ids + return Compiled(sql, list(params), state['aggregate'], frozenset(state['used'])) + + +def compile_search(q, columns): + """Port of `matchSearch`: the narrowing search box. + + True if ANY human-readable (text/status) column contains `q`, case-insensitively. NUMERIC + columns are skipped on purpose so a query like "10" does not match revenue or order counts. + Dates are skipped too — `matchSearch` tests only text and status. + + The caller ANDs this with the filter predicate, matching the TS pipeline order + (filter -> search). It is deliberately OUTSIDE the `member_ids` union: in `useVisibleRows` + the union applies to the filter step only, so a hand-picked member is still searchable away. + """ + # `.strip(_JS_WS)`, not `.strip()`: the client trims with JS String.trim(), whose whitespace + # set differs from Python's in BOTH directions (Python strips U+0085 NEL and U+001C..1F, + # JS does not; JS strips U+FEFF, Python does not). A query of just those characters is + # blank to one engine and a real search to the other. Folding is left to SQL — see the + # LOWER(?) note in _compile_leaf. + q = (q or '').strip(_JS_WS) + if not q: + return None + parts, params, used = [], [], set() + aggregate = False + for col_id, spec in (columns or {}).items(): + if spec['type'] not in TEXTUAL_TYPES: + continue + parts.append(f"(strpos({_value_sql(spec)}, LOWER(?)) > 0)") + params.append(q) + used.add(col_id) + aggregate = aggregate or bool(spec.get('aggregate')) + if not parts: + # No searchable column: `matchSearch` returns false for every row, so the search + # narrows to nothing. That is a real predicate, not an absent one. + return Compiled('1=0', [], False, frozenset()) + return Compiled('(' + ' OR '.join(parts) + ')', params, aggregate, frozenset(used)) + + +def compile_order_by(sorts, columns, tiebreak_sql=None): + """Port of `makeComparator`. Returns an ORDER BY body, or None when nothing sorts. + + `tiebreak_sql` (the row-identity column) is appended as a final ASC key and is REQUIRED for + any windowed fetch. `Array.prototype.sort` is stable, so TS leaves tied rows in their + incoming order; SQL guarantees NOTHING for ties, so LIMIT/OFFSET paging over a non-total + order can repeat a row on one page and drop it from another. Appending row identity makes + the order total. It is a STRENGTHENING of the TS guarantee, not a divergence: the payload a + windowed fetch produces is itself in this order, so the client never re-sorts it differently. + + BLANKS SORT LAST IN BOTH DIRECTIONS (Airtable's behavior, and the reason descending + "Last order" leads with the most recent rather than with never-ordered customers). Each key + contributes TWO terms: an always-ASC blank flag, then the value in the requested direction. + Two blanks tie on both terms, so the NEXT key decides — matching the comparator's + `if (aBlank && bBlank) continue`. + + An unknown column is skipped: in TS its values are `undefined` on both sides, so both are + blank and the key is a no-op tiebreak. + + Direction: `s.dir === "asc" ? c : -c` — only the exact string "asc" ascends, anything else + descends. Faithfully reproduced; do not "fix" it to default ASC. + + Residual divergence, deliberate: TS uses `localeCompare`, we use LOWER() + DuckDB's binary + collation. They agree on case-insensitive ordering of ASCII and on ISO dates (unaffected by + LOWER), and differ on locale tie-breaks between case-variants of the same string ("a" vs + "A" sort adjacent under localeCompare, tie under LOWER) and on accent ordering. + """ + terms = [] + for s in (sorts or []): + if not isinstance(s, dict): + continue + spec = (columns or {}).get(s.get('colId')) + if spec is None: + continue + terms.append(f"CASE WHEN {_blank_sql(spec)} THEN 1 ELSE 0 END ASC") + terms.append(f"{_value_sql(spec)} {'ASC' if s.get('dir') == 'asc' else 'DESC'}") + # Append the tiebreak even when NO sort key survived (all unknown, or none given). A + # windowed fetch needs a total order regardless of what the user sorted by — an unsorted + # LIMIT/OFFSET is exactly as unstable as a tied one. Callers wanting pure client parity + # pass no tiebreak_sql and still get None. + if tiebreak_sql: + terms.append(f"{tiebreak_sql} ASC") + return ', '.join(terms) or None diff --git a/royalimports_os/harness/measure_filter.py b/royalimports_os/harness/measure_filter.py new file mode 100644 index 0000000000000000000000000000000000000000..2f42139e6505a339bf715bd620a91e4fcae58036 --- /dev/null +++ b/royalimports_os/harness/measure_filter.py @@ -0,0 +1,666 @@ +"""harness/measure_filter.py — answering `Where [Sales] [in the last 90 days] [>] [5,000]` (CG-8). + +GATED 2026-07-26 by `royalimports_os/verify_measure_filter.py` — 119 assertions against the real +store (every resolved id set compared to a per-customer aggregate summed in Python from raw rows) +plus a synthetic store for the cases live data cannot produce. Negative control: 24 faithful +mutations, 24 caught. The gate found two things the reasoning below asserted and the code did not +do, both now fixed and both re-caught by the control: + + - a rule with NO window resolved to ALL TIME. The error message four lines below says it + refuses to do exactly that; the line above it did it anyway. + - `sum()` over an all-NULL group is NULL, and `NULL > x` and `NULL <= x` are BOTH false, so + such a customer fell out of the two halves of the same question. Now COALESCEd to 0. + +THE PROBLEM. `revenue_ytd` is a pre-computed column in `customer_data.pool()`. "Sales in the last +90 days" is a DIFFERENT aggregate over a DIFFERENT window, and no column holds it. So a measure +condition cannot be answered by looking at the row in hand — it has to be a query. + +THE SHAPE. Resolve each measure condition server-side to a **set of customer ids**, and let the +existing client engine treat that set as a membership test. Per condition, one query: + + SELECT rp_id FROM WHERE GROUP BY rp_id + HAVING ? + +Three properties that fall out, none incidental: + + * **No second definition of a number.** The aggregate comes from `semantic._measure_sql` — the + same expression the Analyst and every metric already use. This cannot disagree with "revenue" + elsewhere because it IS that computation. + * **The displayed columns are untouched.** `revenue_ytd` still comes from `pool()`. We ADD a + predicate; we do not re-source the table. So this needs no `pool()`→store migration, and does + not inherit that migration's parity problem. + * **Counts stay honest.** The client still holds every row, so the table stays whole-book and + `countLabel` keeps counting what it actually has (CG-3 `whole-book` mode). + +⚠ WHAT MAY BE OFFERED, AND WHY IT IS AN ALLOWLIST +`store_parity()` validates exactly five metrics against live Odoo. Every OTHER sum-metric on the +topic would get an expression with no independent check behind it — self-consistent with the rest +of the app and still wrong if the expression is wrong. Against "robust for EVERY numerically +driven object", that gap is load-bearing, so admission is gated on HAVING A CONTRACT and spelled +as a list. An allowlist fails CLOSED when a metric is added; iterating the topic's metrics would +fail open, silently offering an unvalidated number in the one place a number looks authoritative. + +⚠ AN UNRESOLVABLE CONDITION RAISES. It never returns "everything" or "nothing": both are answers, +and a wrong answer here is invisible — the grid just shows a different set of customers under a +count that looks authoritative. `ModelError` is the only safe failure. +""" +from harness import semantic as _sem +from harness import windows as _win + +#: metric key -> how it reads in the condition builder. ONLY metrics with an independent +#: validation contract (semantic.store_parity / a metric `validate:` block) appear here. +#: `orders` lives on sales_orders and `customers` is a property of a SET, not of a customer — +#: neither is a per-customer measure, so neither is offered. +#: Each entry carries its TOPIC, because admission is no longer single-topic. `orders` lives on +#: sales_orders (order-header grain) while the other three live on sales_lines — and the topic +#: decides the FROM clause, the scope SQL and which dim identifies a customer. Letting the caller +#: pass the topic (the old shape) meant a caller could pass a wrong one; deriving it from the +#: measure means there is nothing to get wrong. +ADMITTED = { + 'revenue': {'label': 'Sales', 'topic': 'sales_lines'}, + 'margin': {'label': 'Gross Margin $', 'topic': 'sales_lines'}, + 'units': {'label': 'Units', 'topic': 'sales_lines'}, + # `orders` is admitted on the SAME rule as the other three and not by exception: + # `semantic.store_parity()` checks it store-vs-live per BU, and the metric carries an + # `order_count` validate contract. It is the replacement for the retired `Orders 24M` + # column — same question, with the period as a parameter instead of frozen at 24 months. + 'orders': {'label': 'Orders', 'topic': 'sales_orders'}, + # ⚠ The first COMPOSITE measure (2026-07-27, owner's follow-up): a RATIO, not an aggregate. + # `aov` is `revenue` (line grain, sales_lines) over `orders` (header grain, sales_orders), so + # it has no single query and `_measure_sql` REFUSES it — which is the fail-closed shape that + # makes a mis-wired ratio raise instead of quietly computing something else. It is resolved + # by computing each BASE per customer and combining them through `semantic._post_compute`, + # the same function every other surface uses, so AOV here cannot disagree with AOV anywhere. + # `topic` names where the metric is DECLARED; the bases each resolve on their own. + 'aov': {'label': 'Avg order $', 'topic': 'sales_lines'}, + # ⚠ `input_scale` SETTLES the question this measure was withheld over. The metric stays a + # FRACTION (margin/revenue, 0-1) — it is the semantic model's number and every other surface + # reads it — while the USER'S TYPED VALUE is divided by 100 on the way in, so + # `Gross Margin % > 20` compares against 0.20. The factor moves exactly once, at the input + # boundary, and never touches the metric. Scaling a COMPUTED comparand (a statistic, or the + # other side of a measure-vs-measure) would divide by 100 twice, so it is applied to the + # literal alone and the gate proves that by construction. + 'margin_pct': {'label': 'Gross Margin %', 'topic': 'sales_lines', 'input_scale': 100.0}, + # ⚠ COMPANY-LEVEL (2026-07-27, owner's ask). Credit notes carry team_id = 1 for everything, + # so a business-unit total is not a thing that exists — `_query_parts` refuses one and the + # live domain builder refuses one. `measure_fields(team_id)` therefore stops OFFERING these + # to a BU-scoped caller rather than letting them build a condition that can only error: a + # dead entry in the picker is worse than an absent one, and hiding what cannot be answered + # is what `filterable: false` already does for columns. + 'returns': {'label': 'Returns $', 'topic': 'credit_notes'}, + 'returns_pct': {'label': 'Return %', 'topic': 'credit_notes', 'input_scale': 100.0}, +} + +#: The metrics `semantic.store_parity()` actually exercises against live Odoo. ADMITTED must be +#: a SUBSET of this — that is the whole admission rule ("does this measure carry an independent +#: validation contract"), and stating it as data lets the gate assert it instead of trusting the +#: comment above. Adding a measure here without adding it to store_parity's loop fails the gate. +PARITY_CHECKED = frozenset({'revenue', 'margin', 'units', 'orders', 'customers', + 'returns', 'invoiced'}) + +#: numeric comparison ops, mirroring filter_sql._NUM_CMP. `between` is expressed as two +#: conditions by the caller rather than a third operator shape. +OPS = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'} + +#: CG-9 — the ops allowed when the right-hand side is ANOTHER MEASURE. `eq`/`neq` are withheld, +#: and not out of caution: both sides are sums of floats the user never sees, so `=` would match +#: nobody and `≠` would match everybody, and both would LOOK like they worked. CG-8 could offer +#: `=` honestly because the other side was a number a person typed. Mirrors types.ts +#: MEASURE_PAIR_OPS; the gate asserts the two lists are the same. +PAIR_OPS = ('gte', 'gt', 'lte', 'lt') + +#: The statistics a condition may compare against — "who is in the top quartile" (owner, +#: 2026-07-27). Each is computed over the CALLER'S POOL for the same measure and window. +#: The labels are the sentence the user reads; the client mirrors them. +STATS = { + 'avg': 'the average', + 'median': 'the median', + 'p10': 'the bottom 10%', + 'p25': 'the bottom quartile', + 'p75': 'the top quartile', + 'p90': 'the top 10%', +} +# ⚠ The owner's OWN words, and short enough to fit: "the 75th percentile (top quartile)" clipped +# to "the 75th perce" in the value select, and a truncated parenthetical reads as a rendering +# bug. `> the top quartile` is also how a person says it out loud, so the shorter label is the +# better one twice over. Seen in a live screenshot of a run whose every assertion was green. + +#: ⚠ THE POPULATION IS THE CALLER'S POOL, INCLUDING ZEROS. A customer who bought nothing has a +#: measure of 0, not of nothing — that is the convention the zero-group already establishes +#: everywhere else, and "the median customer" means the median of the customers you can see. +#: Two consequences worth knowing before reading a result: +#: · it is a PERMISSION boundary as well as a definition — a scoped user's median is theirs, +#: and a statistic over the whole book would leak the shape of data they cannot open; +#: · on a measure most customers score 0 on, the 75th percentile IS 0, so "top quartile" +#: degenerates to "anyone with any activity at all". True, and surprising. + + +def percentile(values, q): + """The `q` quantile of `values` by LINEAR INTERPOLATION between closest ranks — the R-7 + convention, which is what Excel's PERCENTILE.INC computes. `q` may also be a STATS key. + + Pinned to one convention on purpose: nearest-rank and interpolation disagree on any even + population (the median of [1,2,3,4] is 2.5 here and 2 or 3 under nearest-rank), and the gate + checks it against hand-computed values rather than against `statistics.median` — a reference + borrowed from a library only proves we match that library. + + Returns None for an empty population: there is no median of nobody, and 0 would be a number + the caller could act on. + """ + if isinstance(q, str): + if q == 'avg': + xs = list(values) + return (sum(xs) / len(xs)) if xs else None + if q not in STATS: + raise _sem.ModelError( + f"{q!r} is not a statistic. Offered: {sorted(STATS)}") + q = 0.5 if q == 'median' else int(q[1:]) / 100.0 + xs = sorted(float(v) for v in values) + if not xs: + return None + if len(xs) == 1: + return xs[0] + pos = (len(xs) - 1) * float(q) + lo = int(pos) + hi = min(lo + 1, len(xs) - 1) + return xs[lo] + (xs[hi] - xs[lo]) * (pos - lo) + + +#: The same six comparisons as OPS, as Python callables — used wherever the comparison happens in +#: Python rather than in a HAVING (a composite measure, or a two-measure pair). +_CMP = {'gt': lambda a, b: a > b, 'gte': lambda a, b: a >= b, + 'lt': lambda a, b: a < b, 'lte': lambda a, b: a <= b, + 'eq': lambda a, b: a == b, 'neq': lambda a, b: a != b} + + +def is_company_level(key): + """True when this measure cannot be scoped to a business unit. + + Checked over every BASE the measure decomposes into, not just the declared topic: `Return %` + names credit_notes but also reads customer_invoices, and one un-scopable half is enough to + make the whole ratio un-scopable. + """ + bases, _ = _sem._expand_measures([key]) + tps = _sem.topics() + for b in bases: + store = (tps.get(_sem.metrics()[b].get('topic')) or {}).get('store') or {} + if not store.get('team_col'): + return True + return False + + +def measure_fields(team_id=None): + """The measures the condition builder may offer, in the payload's field shape. + + These are FILTERABLE BUT NOT DISPLAYABLE — a measure is a question about a window, not a + column of the table, and rendering it as a column would silently pick a window for the user. + + ⚠ `team_id` NARROWS THE OFFER. A COMPANY-LEVEL measure (returns, and anything else built on + documents that carry no trustworthy BU tag) cannot be answered for one business unit — + `_query_parts` refuses, and refusing is right. But a picker entry whose every use ends in an + error is worse than no entry, so a BU-scoped caller is not offered them at all. Same + principle as `filterable: false` on a column: do not offer what cannot be answered. + """ + mts = _sem.metrics() + out = [] + for key, spec in ADMITTED.items(): + m = mts.get(key) + if not m: + continue + if team_id and is_company_level(key): + continue + out.append({'key': key, 'label': spec['label'], + 'type': _sem._FORMAT_TYPE.get(m.get('format'), 'currency')}) + return out + + +def topic_for(key): + """The topic a measure is answered on. Raises for anything not admitted.""" + spec = ADMITTED.get(key) + if not spec: + raise _sem.ModelError( + f"{key!r} is not an admitted measure. Offered: {sorted(ADMITTED)} — a measure " + f"without an independent validation contract is not filterable (see this module's " + f"docstring).") + return spec['topic'] + + +def _entity_sql(topic): + """(from-clause, the id column customers are grouped by, the store binding) for a topic.""" + t = _sem.topics().get(topic) + if not t or 'store' not in t: + raise _sem.ModelError(f"topic {topic!r} has no store binding") + s = t['store'] + dims = s.get('dims') or {} + # sales_lines calls it `order_partner` (the line's parent order's partner); sales_orders + # calls the same thing `partner`. One name per topic, both meaning "the customer this row + # belongs to" — which is the only dim this module ever groups by. + dim = dims.get('order_partner') or dims.get('partner') + if not dim: + raise _sem.ModelError(f"topic {topic!r} has no customer dim to group by") + return f"{s['table']} {s['alias']} {s.get('join', '')}", dim['col'], s + + +def is_composite(key): + """True for a measure with no single aggregate expression — a `ratio` or a `derived`. + + These are answered by computing each BASE per customer and combining in Python; they can + never be a HAVING, which is why `_measure_sql` refusing them is a feature. + """ + return _sem.metrics().get(key, {}).get('agg') in ('ratio', 'derived') + + +def _base_topic(key): + """The topic a BASE metric aggregates on — read from the METRIC, never from ADMITTED. + + A composite decomposes into bases that need not themselves be offerable (and whose topics may + differ from each other: `aov`'s numerator is on sales_lines and its denominator on + sales_orders). Resolving the topic from the allowlist would make that undecidable. + """ + m = _sem.metrics().get(key) + if not m: + raise _sem.ModelError(f"unknown metric {key!r}") + topic = m.get('topic') + if not topic: + raise _sem.ModelError(f"metric {key!r} declares no topic to aggregate on") + return topic + + +def _query_parts(key, window, today, team_id): + """The FROM / GROUP-BY id / aggregate / WHERE for one (measure, window), shared by the + one-sided and the two-sided resolver so there is one definition of the scope.""" + topic = _base_topic(key) + # NO DEFAULT WINDOW. A rule that carries no window is not "all time" — it is a rule whose + # window we do not know, and answering it over all time WIDENS the condition while the count + # beside it still looks authoritative. `all_time` is a legitimate ANSWER the user can pick; + # it is not a legitimate GUESS. (This line shipped as `rule.get('window') or {...}` and the + # gate caught it contradicting the error message two lines below.) + bounds = _win.resolve(window, today) + if bounds is None: + raise _sem.ModelError( + f"unresolvable window {window!r} — refusing rather than defaulting to all " + f"time, which would widen the condition while the count still looked authoritative") + date_from, date_to = bounds + + frm, id_col, s = _entity_sql(topic) + m = _sem.metrics()[key] + # COALESCE, because SQL's sum() over an all-NULL group is NULL, and `NULL > x` and + # `NULL <= x` are BOTH false: the customer would vanish from the two halves of the same + # question, so `margin <= 0` and `margin > 0` would not cover the book. The zero-group below + # cannot rescue them either — they DO have rows in the window, just no values. Python's sum + # of the same rows is 0.0, and 0 is what the grid would show, so 0 is the answer. + agg = f"COALESCE({_sem._measure_sql(m, s['alias'])}, 0)" + + where, params = [_sem.render_scope(s['scope_sql'].strip())], [] + if date_from: + where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") + params.append(f"{date_from} 00:00:00") + if date_to: + where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?") + params.append(f"{date_to} 23:59:59") + if team_id: + if not s.get('team_col'): + raise _sem.ModelError(f"topic {topic!r} is company-level — it has no business-unit " + f"filter, so a BU-scoped caller cannot be answered safely") + where.append(f"{s['team_col']} = ?") + params.append(int(team_id)) + return frm, id_col, agg, ' AND '.join(where), params + + +def _scaled(key, value): + """A TYPED value in the units the metric actually holds. + + `Gross Margin % > 20` is typed in percentage points and the metric is a fraction, so the 20 + becomes 0.20. Applied to a LITERAL only — a statistic and the other side of a + measure-vs-measure are already on the metric's scale, and dividing them too would apply the + factor twice for a result that still looks plausible. + """ + scale = ADMITTED.get(key, {}).get('input_scale') + return (value / scale) if scale else value + + +def resolve_stat(rule, today, team_id=None, allowed_pids=None): + """`Where [Sales] [this year] [>] [the 75th percentile]` — the top quartile of the pool. + + One totals query for the measure, the statistic taken over the CALLER'S POOL (see the STATS + note on why the population is the pool and what that costs), then a comparison per customer. + """ + stat = (rule.get('rhs') or {}).get('stat') + if stat not in STATS: + raise _sem.ModelError(f"{stat!r} is not a statistic. Offered: {sorted(STATS)}") + key = rule.get('colId') + topic_for(key) + op = rule.get('op') + if op not in PAIR_OPS: + raise _sem.ModelError( + f"{op!r} cannot compare a measure with a statistic ({sorted(PAIR_OPS)}). Equality " + f"against a computed cut-off matches nobody except by coincidence.") + pool = _pool_of(allowed_pids) + if pool is None: + raise _sem.ModelError( + "a statistic needs a population to be computed over, and none was supplied — " + "refusing rather than taking it over whichever customers happen to have rows, " + "which would be a different question with an authoritative-looking answer") + con = _sem._store_con() + try: + totals = _totals(con, key, rule.get('window'), today, team_id) + finally: + con.close() + # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision, + # 2026-07-27: *"top quartile of customers who actually has returned something"*. + # + # It was the whole pool including zeros, which is the convention everywhere else and was + # wrong HERE. The pool carries lapsed customers, so ~30% of it sits at zero for any 12-month + # window — that dragged the 25th percentile to exactly 0.0, made `< the bottom quartile` + # match NOBODY, and would have been worse for Return %, where most customers never return + # anything and "the top quartile" would have degenerated to "anyone with a single refund". + # + # NON-ZERO, not has-rows: a zero Return % means nothing was returned, which is precisely the + # customer the owner is excluding. It also reads as one sentence — the question is asked + # among customers who have that measure — and it generalises: "the median Sales" means the + # median among customers who sold something. + # + # The ANSWER is restricted to the same population, so `< the bottom quartile` returns a + # quarter of the returners rather than every customer who never returned. + population = {pid: v for pid, v in totals.items() if pid in pool and v} + cut = percentile(list(population.values()), stat) + if cut is None: # nobody has this measure -> nobody qualifies + return set() + return {pid for pid, v in population.items() if _CMP[op](v, cut)} + + +def _pool_of(allowed_pids): + """The caller's visible pool as a set of plain ints, or None. + + ⚠ The pool arrives from `pool()`/a cohort and may hold numpy ints or id strings; the store + returns BIGINTs. A set difference across two int types silently finds nothing in common, + which here would look like "no lapsed customers" rather than an error. + """ + return None if allowed_pids is None else {int(p) for p in allowed_pids if p is not None} + + +def rule_complete(rule): + """Is this rule's right-hand side FULLY present — a question that has actually been asked? + + THE GAP THIS CLOSES (2026-07-27, the cold-container race). The component's view state + passes through the host while the user — or the client's own initialisation — is mid-edit, + and a rule can transiently arrive with its window, statistic or value missing. `to_num(None)` + is 0 by the engines' shared contract, so resolution used to ANSWER that half-asked question + as ` 0`: the id landed in `measureSets`, the client read the rule as answered and + cleared its pending marker, and the grid stood behind a count for a question nobody asked. + Refusing instead leaves the id OUT of the payload, which the client renders as PENDING + ("Calculating…") — the honest state for a question still being typed. Exactly the class the + module docstring already names: an unresolvable condition never becomes an answer. + + INCOMPLETE is distinct from MALFORMED, on purpose. An ABSENT window / stat / value is a + normal transient (the caller skips it silently — mid-edit is not a telemetry event); a + PRESENT-but-bogus one is a real defect and still RAISES downstream (`unresolvable window`, + `not a statistic`, `not admitted`). `value` presence is textual, not numeric: "0" is a + typed answer, while parseability stays `to_num`'s business so the two engines keep one + contract for garbage input. + """ + if not isinstance(rule, dict) or not rule.get('op'): + return False + if rule.get('window') is None: + return False + rhs = rule.get('rhs') + if isinstance(rhs, dict) and rhs.get('kind') == 'measure': + return bool(rhs.get('colId')) and rhs.get('window') is not None + if isinstance(rhs, dict) and rhs.get('kind') == 'stat': + return bool(rhs.get('stat')) + v = rule.get('value') + return v is not None and str(v).strip() != '' + + +def resolve_rule(rule, today, team_id=None, allowed_pids=None): + """One measure condition -> the SET OF CUSTOMER IDS satisfying it. + + `rule` = {colId: , window: , op: , value: }, + or — CG-9 — the same with `rhs: {kind:'measure', colId, window}` instead of a value, which + hands off to `resolve_pair`. + + ⚠ Customers with NO rows in the window are absent from the GROUP BY, so a condition like + `revenue < 100` would miss them even though 0 < 100 is true. That is the single most + dangerous thing about this query shape: the missing rows are exactly the lapsed customers a + "who has gone quiet" filter is FOR. So the caller passes `allowed_pids` (the pool) and the + zero-group is reconstructed explicitly below rather than left to the SQL. + """ + if not rule_complete(rule): + # An absent value would coerce to 0 and resolve as a DIFFERENT question under an + # authoritative-looking count — the same failure class as defaulting a missing window + # to all time, caught by the same principle: refuse, never guess. + raise _sem.ModelError( + 'incomplete measure condition — its window, statistic or value is not present; ' + 'refusing to answer a question that has not been fully asked') + rhs = rule.get('rhs') + if isinstance(rhs, dict) and rhs.get('kind') == 'measure': + return resolve_pair(rule, today, team_id, allowed_pids) + if isinstance(rhs, dict) and rhs.get('kind') == 'stat': + return resolve_stat(rule, today, team_id, allowed_pids) + key = rule.get('colId') + topic_for(key) # raises for anything not admitted + op = rule.get('op') + if op not in OPS: + raise _sem.ModelError(f"{op!r} is not a numeric comparison ({sorted(OPS)})") + + from harness import filter_sql as _fs + # ⚠ THE SCALE IS APPLIED HERE, TO THE TYPED VALUE, AND NOWHERE ELSE. + # + # Its position relative to `_zero_satisfies` turns out NOT to matter, and the negative + # control is what established that rather than reasoning: dividing by a POSITIVE constant + # cannot change how a number compares to zero, so the zero-group is identical either way. + # (An earlier version of this comment claimed the opposite. It was wrong.) What DOES matter + # is that a COMPUTED comparand never passes through here — see `_scaled`. + value = _scaled(key, _fs.to_num(rule.get('value'))) + + pool = _pool_of(allowed_pids) + if is_composite(key): + # A ratio has no HAVING to push the comparison into: compute the value per customer, then + # compare in Python — the same shape `resolve_pair` uses, and for the same reason. + con = _sem._store_con() + try: + totals = _totals(con, key, rule.get('window'), today, team_id) + finally: + con.close() + # No pool means we can only speak about customers a base has seen. WITH a pool, everyone + # else scores 0 through the ratio's guard — which is what makes the zero-group work for a + # ratio exactly as it does for a sum ("AOV <= 250" includes those who bought nothing). + universe = pool if pool is not None else set(totals) + return {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)} + + frm, id_col, agg, where_sql, params = _query_parts( + key, rule.get('window'), today, team_id) + sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} " + f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?") + + con = _sem._store_con() + try: + matched = {int(r[0]) for r in con.execute(sql, params + [value]).fetchall() + if r[0] is not None} + # The zero-group: every customer in scope with NO rows in the window aggregates to 0, and + # SQL cannot return a group that has no rows. Reconstruct it from the pool rather than + # pretending those customers do not exist. + if pool is not None: + present = {int(r[0]) for r in con.execute( + f"SELECT DISTINCT {id_col} FROM {frm} WHERE {where_sql}", + params).fetchall() if r[0] is not None} + absent = pool - present + if absent and _zero_satisfies(op, value): + matched |= absent + finally: + con.close() + if pool is not None: + matched &= pool + return matched + + +def _totals(con, key, window, today, team_id): + """`{pid: value}` for one (measure, window), COMPOSITES included. + + For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived + metric it is one query per BASE — each on that base's own topic — combined per customer by + `semantic._post_compute`, the same function `store_query` uses for the un-grouped number. So + AOV per customer is the app's AOV, computed once and defined once. + + ⚠ The pid keys come from DIFFERENT dims for a cross-topic ratio (`l.order_partner_id` and + `o.partner_id`). The union is taken rather than the numerator's keys, so a customer either + side has seen gets a number; the ratio's own `den == 0 -> 0.0` guard covers the rest. + """ + if is_composite(key): + bases, _ = _sem._expand_measures([key]) + parts = {b: _totals(con, b, window, today, team_id) for b in bases} + pids = set() + for got in parts.values(): + pids |= set(got) + return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key) + for pid in pids} + frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id) + sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}") + # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in + # COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and + # the negative control proved it (deleting it changed nothing). If one ever DID arrive, a + # TypeError is the right outcome: it would mean the SQL-level guard had been removed, which + # is the bug worth hearing about rather than papering over. + return {int(r[0]): float(r[1]) + for r in con.execute(sql, params).fetchall() if r[0] is not None} + + +def resolve_pair(rule, today, team_id=None, allowed_pids=None): + """CG-9 — `Where [Sales] [this year] [>] [Sales] [year to date, last year]`. + + TWO QUERIES, compared in Python. The obvious alternative — one query with two conditional + aggregates and a HAVING comparing them — would have to reach inside `_sem._measure_sql` to + inject a per-side window into an expression that is already a `sum(CASE WHEN … )` for + filtered measures. Two queries keep ONE definition of each number (the metric's own + aggregate, unmodified), let the two sides sit on DIFFERENT TOPICS, and make the zero-group + fall out for free: absent from a side's result means 0 on that side. + + ⚠ The zero-group is LOUDER here than it is for a one-sided rule. Every customer with no rows + in EITHER window compares 0 against 0, so `<=` and `>=` match all of them — correct, and it + will look wrong on a narrow window. That is why `eq`/`neq` are not offered at all + (PAIR_OPS): two float sums are never exactly equal except at zero, where they always are. + """ + rhs = rule.get('rhs') or {} + lkey, rkey = rule.get('colId'), rhs.get('colId') + topic_for(lkey), topic_for(rkey) # raises for anything not admitted + op = rule.get('op') + if op not in PAIR_OPS: + raise _sem.ModelError( + f"{op!r} cannot compare two measures ({sorted(PAIR_OPS)}). Equality between two sums " + f"of floats matches nobody and inequality matches everybody; neither answers the " + f"question, and both look like they work.") + cmp = _CMP[op] + + pool = _pool_of(allowed_pids) + con = _sem._store_con() + try: + left = _totals(con, lkey, rule.get('window'), today, team_id) + right = _totals(con, rkey, rhs.get('window'), today, team_id) + finally: + con.close() + # With no pool we can only speak about customers at least one side has seen — the zero-vs- + # zero group is unknowable without being told who exists. With a pool, it is every member. + universe = pool if pool is not None else (set(left) | set(right)) + return {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))} + + +def _zero_satisfies(op, value): + """Does a customer with NO activity in the window (measure = 0) satisfy this condition?""" + return _CMP[op](0.0, value) + + +def resolve_values(key, window, today, team_id=None, allowed_pids=None): + """`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN + version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD). + + Same `_totals` the filter conditions and statistics use, so the column and the condition + cannot disagree. Three display decisions live here, not in the client: + + * ZERO-FILLED over the pool. A customer with no rows in the window has a measure of 0 — + the zero-group convention everywhere else — and 0 is a real value the cell shows. Blank + is reserved for "could not compute" (the caller omits the column entirely on failure). + * `input_scale` applied OUTWARD. `margin_pct` is a fraction (0-1) internally; the user + types percentage points into the filter (input divides by 100), so the CELL shows points + too (value × 100) — one boundary, both directions, and `yoy_pct` beside it already reads + in points. + * Rounded like every other numeric column: whole numbers for money/counts (the pool's own + `_round` convention), one decimal for a percent. + + Raises for anything not admitted / an unresolvable window — the caller degrades that column + to blank rather than to a number nobody computed. + """ + topic_for(key) # raises for anything not admitted + # The window must RESOLVE before any query runs — same refusal as `_query_parts`, checked + # here too because a composite's `_totals` path resolves per base and would otherwise raise + # deeper with a less useful message. + if _win.resolve(window, today) is None: + raise _sem.ModelError( + f"unresolvable window {window!r} for the {key!r} column — refusing rather than " + f"defaulting to all time") + pool = _pool_of(allowed_pids) + con = _sem._store_con() + try: + totals = _totals(con, key, window, today, team_id) + finally: + con.close() + scale = ADMITTED.get(key, {}).get('input_scale') or 1.0 + mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency') + nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal + universe = pool if pool is not None else set(totals) + # round(x, None) returns an int — exactly the pool's `_round` convention for money/counts. + return {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe} + + +def resolve_tree(rules, today, team_id=None, allowed_pids=None): + """{rule_id: sorted pids} for every measure condition in a tree. + + Keyed by the rule's client-generated `id` so the browser can map an answer back to the + condition that asked it — positional matching would silently re-associate every answer the + moment a user deleted a condition. + """ + out = {} + for r in rules or []: + rid = str(r.get('id') or '') + if not rid: + raise _sem.ModelError('a measure condition needs a stable id to carry its answer back') + out[rid] = sorted(resolve_rule(r, today, team_id, allowed_pids)) + return out + + +def signature(rule): + """The identity of the QUESTION a rule asks, ignoring which condition row asked it. + + Two conditions with the same measure, window, operator and value have the same answer, so + they cost one query between them — and a saved view re-rendered unchanged costs none, since + the caller can memoise on this. `id` is deliberately absent: it names the asker, not the + question. + + ⚠ The RHS is part of the question. Without it, `Sales[ytd] > Sales[last year]` and + `Sales[ytd] > Sales[ytd_last_year]` would share one memo entry — same measure, same op, same + (empty) value — and the second would be served the first one's answer. Silent, and wrong in + exactly the comparison this was built for. + """ + w = _win.normalize(rule.get('window')) + rhs = rule.get('rhs') + rsig = None + if isinstance(rhs, dict): + rw = _win.normalize(rhs.get('window')) + # `stat` rides the same tuple: `> the median` and `> the 75th percentile` are different + # questions with the same measure, window, op and (empty) value. + rsig = (rhs.get('kind'), rhs.get('colId'), rhs.get('stat'), + None if rw is None else tuple(sorted(rw.items()))) + return (rule.get('colId'), rule.get('op'), str(rule.get('value') or ''), + None if w is None else tuple(sorted(w.items())), rsig) + + +def collect(nodes): + """Every MEASURE leaf in a filter tree, flattened (groups contribute their children).""" + found = [] + for n in nodes or []: + if not isinstance(n, dict): + continue + if isinstance(n.get('children'), list): + found.extend(collect(n['children'])) + elif n.get('colId') in ADMITTED: + found.append(n) + return found diff --git a/royalimports_os/harness/routines.py b/royalimports_os/harness/routines.py new file mode 100644 index 0000000000000000000000000000000000000000..f7b0ac1b635dd2036e91077d02ea53059e9c19ce --- /dev/null +++ b/royalimports_os/harness/routines.py @@ -0,0 +1,259 @@ +"""harness/routines.py — OM-5 v0: scheduled reports + KPI alerts over the semantic layer. + +Two of the four productization user jobs (Scheduled Reporting, KPI monitoring). Definitions live +in data/store/routines.json (tenant data, git-ignored); evaluation happens ON READ against the +tenant store through the same governed queries as everything else — an alert is a metric key + +a mode, never a hand-rolled query. + +v0 boundaries (deliberate): +- Channel is 'in-app' only. Email/Slack delivery is OUTBOUND and ships only with owner approval + (the Odoo mail client is whitelisted for Collections statements ONLY — nothing else sends). +- No background scheduler: reports record cadence + show due-state in-app; alerts evaluate live + whenever the Routines page (or a tool) asks. The enterprise scheduler lands with OM-5 delivery. + +Alert modes: +- 'xmr' Wheeler individuals chart on complete monthly values (promoted from + modules/warehouse._xmr, generalized: no 0/100 clamps — metrics aren't percentages). + Three detection rules only (more = alarm fatigue, per the warehouse brief). +- 'threshold' last complete month above/below a fixed value. +""" +import json +import time +from datetime import date, timedelta + +import harness.semantic as SEM +from harness.tools import VIEWS_PATH + +ROUTINES_PATH = VIEWS_PATH.parent / "routines.json" + +CADENCES = ("daily", "weekly", "monthly") +CHANNELS = ("in-app",) # email/slack gate on owner approval (outbound send) +ALERT_MODES = ("xmr", "threshold") +XMR_MONTHS = 18 # complete months fed to the individuals chart + + +def load(): + if ROUTINES_PATH.exists(): + return json.loads(ROUTINES_PATH.read_text(encoding="utf-8")) + return {"reports": {}, "alerts": {}} + + +def _save(d): + ROUTINES_PATH.parent.mkdir(parents=True, exist_ok=True) + ROUTINES_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8") + + +# ------------------------------------------------------------------ scheduled reports + +def schedule_report(view, cadence, channel="in-app"): + import harness.views as V + if view not in V.views() and view not in V.dashboards(): + raise SEM.ModelError(f"unknown view/dashboard {view!r} — save_view/compose_dashboard first") + if cadence not in CADENCES: + raise SEM.ModelError(f"cadence must be one of {CADENCES}") + if channel not in CHANNELS: + raise SEM.ModelError(f"channel {channel!r} not available yet — v0 delivers in-app " + "(email/Slack ship with the delivery gate); use channel='in-app'") + d = load() + d["reports"][view] = {"cadence": cadence, "channel": channel, + "created_at": time.strftime("%Y-%m-%d %H:%M")} + _save(d) + return {"scheduled": view, "cadence": cadence, "channel": channel, + "note": "runs in-app (Workflows > Routines); external delivery ships with OM-5 email approval"} + + +def delete_report(view): + d = load() + d["reports"].pop(view, None) + _save(d) + + +_CADENCE_DAYS = {"daily": 1, "weekly": 7, "monthly": 30} + + +def mark_viewed(view): + """Called when the scheduled view/dashboard is actually opened — resets its due-ness.""" + d = load() + r = d.get("reports", {}).get(view) + if r is not None: + r["last_viewed"] = time.strftime("%Y-%m-%d %H:%M") + _save(d) + + +def due_reports(): + """Scheduled reports whose cadence has elapsed since last viewed (v0 in-app delivery via + the Routines page). Never-viewed ages from created_at.""" + from datetime import datetime + out = [] + for name, r in load().get("reports", {}).items(): + days = _CADENCE_DAYS.get(r.get("cadence"), 7) + anchor = r.get("last_viewed") or r.get("created_at") or "" + try: + age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days + except Exception: + age = days + if age >= days: + out.append({"name": name, "cadence": r.get("cadence"), + "last_viewed": r.get("last_viewed"), "age_days": age}) + return out + + +# ------------------------------------------------------------------ insight routines (OM-5) +# A saved PROMPT on a cadence: the Analyst runs it in the background and the narrative (plus its +# governed artifacts) lands in-app on the Routines page. The narrative is a deliberate +# point-in-time SNAPSHOT (it narrates a moment); its charts still carry re-runnable queries. + +def schedule_insight(name, prompt, cadence): + if cadence not in CADENCES: + raise SEM.ModelError(f"cadence must be one of {CADENCES}") + if not (prompt or "").strip(): + raise SEM.ModelError("an insight needs a prompt — what should the Analyst look at?") + d = load() + d.setdefault("insights", {})[name] = {"prompt": prompt.strip(), "cadence": cadence, + "created_at": time.strftime("%Y-%m-%d %H:%M")} + _save(d) + return {"insight": name, "cadence": cadence, + "note": "runs in the background when due; the narrative lands on " + "Workflows > Routines"} + + +def delete_insight(name): + d = load() + (d.get("insights") or {}).pop(name, None) + _save(d) + + +def due_insights(): + from datetime import datetime + out = [] + for name, i in (load().get("insights") or {}).items(): + days = _CADENCE_DAYS.get(i.get("cadence"), 7) + anchor = i.get("last_run_at") or "" + try: + age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days + except Exception: + age = days # never run -> due now + if age >= days: + out.append(name) + return out + + +def run_insight(name): + """Execute one insight through the Analyst and persist the narrative + artifacts.""" + d = load() + ins = (d.get("insights") or {}).get(name) + if not ins: + raise SEM.ModelError(f"unknown insight {name!r}") + import harness.analyst as A + r = A.ask(ins["prompt"]) + now = time.strftime("%Y-%m-%d %H:%M") + ins["last_run_at"] = now + ins["last"] = {"at": now, "answer": r["answer"], "artifacts": r.get("artifacts") or [], + "tokens": (r.get("usage") or {}).get("prompt_tokens", 0) + + (r.get("usage") or {}).get("completion_tokens", 0)} + _save(d) + return ins["last"] + + +# ------------------------------------------------------------------ KPI alerts + +def create_alert(metric, mode="xmr", team_id=None, threshold=None, direction=None): + m = SEM.metrics().get(metric) + if not m: + raise SEM.ModelError(f"unknown metric {metric!r} (use list_topics)") + if mode not in ALERT_MODES: + raise SEM.ModelError(f"mode must be one of {ALERT_MODES}") + if mode == "threshold": + if threshold is None or direction not in ("above", "below"): + raise SEM.ModelError("threshold mode needs threshold= and " + "direction='above'|'below'") + d = load() + key = metric + (f"_team{team_id}" if team_id else "") + d["alerts"][key] = {"metric": metric, "mode": mode, "team_id": team_id, + "threshold": threshold, "direction": direction, + "created_at": time.strftime("%Y-%m-%d %H:%M")} + _save(d) + return {"alert": key, **evaluate_alert(d["alerts"][key])} + + +def delete_alert(key): + d = load() + d["alerts"].pop(key, None) + _save(d) + + +def xmr(series): + """Wheeler XmR individuals limits + the three detection rules — the model-level primitive + (generalized from modules/warehouse._xmr; no percentage clamps). series=[(label, value)].""" + vals = [v for _, v in series] + if len(vals) < 8: + return {"center": None, "unpl": None, "lnpl": None, + "points": [{"label": l, "value": v, "signal": ""} for l, v in series]} + center = sum(vals) / len(vals) + mrs = [abs(vals[i] - vals[i - 1]) for i in range(1, len(vals))] + mr_bar = sum(mrs) / len(mrs) if mrs else 0.0 + unpl = center + 2.66 * mr_bar + lnpl = center - 2.66 * mr_bar + pts = [] + for i, (l, v) in enumerate(series): + sig = "" + if v > unpl or v < lnpl: + sig = "outside limits" + if not sig and i >= 7 and all((x - center) > 0 for _, x in series[i - 7:i + 1]): + sig = "sustained shift (8 above center)" + if not sig and i >= 7 and all((x - center) < 0 for _, x in series[i - 7:i + 1]): + sig = "sustained shift (8 below center)" + if not sig and i >= 3: + last4 = [x for _, x in series[i - 3:i + 1]] + near = sum(1 for x in last4 + if abs(x - center) > 0.5 * ((unpl - center) if x >= center + else (center - lnpl))) + if near >= 3: + sig = "moderate shift emerging" + pts.append({"label": l, "value": v, "signal": sig}) + return {"center": center, "unpl": unpl, "lnpl": lnpl, "points": pts} + + +def _month_window(n=XMR_MONTHS): + """Last n COMPLETE months (the in-progress month would false-signal every alert).""" + first_cur = date.today().replace(day=1) + y, mo = first_cur.year, first_cur.month + for _ in range(n): + mo -= 1 + if mo == 0: + y, mo = y - 1, 12 + start = date(y, mo, 1) + return start.isoformat(), (first_cur - timedelta(days=1)).isoformat() + + +def evaluate_alert(a): + """Live evaluation: monthly series of the metric (complete months only) -> mode verdict.""" + metric = a["metric"] + topic = SEM.metrics()[metric]["topic"] + date_from, date_to = _month_window() + res = SEM.store_query(topic, [metric], grain="month", date_from=date_from, date_to=date_to, + team_id=a.get("team_id")) + series = [(r["period"][:7], r.get(metric) or 0) for r in res["rows"]] + out = {"metric": metric, "mode": a["mode"], "team_id": a.get("team_id"), + "window": f"{date_from}..{date_to}", "series": series} + if not series: + return {**out, "status": "no data", "fired": False} + last_label, last_val = series[-1] + if a["mode"] == "threshold": + thr, direc = a.get("threshold"), a.get("direction") + fired = (last_val > thr) if direc == "above" else (last_val < thr) + return {**out, "last": {"label": last_label, "value": last_val}, + "threshold": thr, "direction": direc, + "status": (f"FIRED — {last_label} is {direc} {thr:,.0f}" if fired else "OK"), + "fired": bool(fired)} + lim = xmr(series) + sig = lim["points"][-1]["signal"] if lim["points"] else "" + return {**out, "last": {"label": last_label, "value": last_val}, + "center": lim["center"], "unpl": lim["unpl"], "lnpl": lim["lnpl"], + "points": lim["points"], + "status": (f"SIGNAL — {sig} ({last_label})" if sig else "OK"), + "fired": bool(sig)} + + +def evaluate_all(): + return {k: evaluate_alert(a) for k, a in load().get("alerts", {}).items()} diff --git a/royalimports_os/harness/runtime.py b/royalimports_os/harness/runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..c1ba30ca4ea21a9f573fdd96823312ba9d0f1f8b --- /dev/null +++ b/royalimports_os/harness/runtime.py @@ -0,0 +1,186 @@ +"""harness/runtime.py — X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a). + +THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today +each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles +and its own DuckDB — EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly +(tenant B came within 2% of tenant A; nothing is shared). The target is ~30–80 MB. That is only +reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between +requests except a bounded cache it is willing to throw away. + +WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton — one process, one tenant, +caches that are never evicted by design. That is correct for Streamlit and fatal for a shared +process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches, +accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6). + +THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked: + 1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime + reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is + exactly what makes "which tenant is this process serving?" unanswerable. + 2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted. + An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve — the + thing this whole program exists to remove. + 3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace + even if a caller passes the wrong key downstream (EXIT-4b proof #3). + +⚠ THE DUCKDB CAVEAT, STATED NOT HIDDEN. `harness.datastore` holds ONE process-wide connection +(the fix for the 2026-07-16 "connection error unless I refresh" bug), so one PROCESS can serve +one analytical store at a time. This module therefore hands out `datastore_path` — the file a +tenant's analytical reads belong to — and does NOT switch the global underneath a live request. +Until the singleton is per-tenant (EXIT-5), an API process answering measure/Analyst queries must +be pinned to one tenant's file, and `assert_datastore_matches()` below is what turns a +mis-pinned deployment into a loud failure instead of a silent cross-tenant read. +""" +from __future__ import annotations + +import os +import threading +from collections import OrderedDict +from dataclasses import dataclass, field + +import core.store as store +from harness import datastore as _ds +from harness import tenants as _tenants + +#: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a +#: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement. +_MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32) + +#: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring +#: this becomes "a tenants store (DB / HF dataset)" — X4's control DB is where it goes, and +#: `register()` below is the seam that makes the swap a one-liner rather than a refactor. +_BUILDERS = {"royal-imports": _tenants.royal_imports} + +# X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a +# slug an attacker can mint a session against in production; gated, it exists exactly where it is +# needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be +# flipped by a request. +if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1": + _BUILDERS["qa-b"] = _tenants.qa_tenant_b + +_CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict() +_LOCK = threading.RLock() + + +@dataclass +class TenantRuntime: + """Everything a request needs to serve ONE tenant, and nothing about any other. + + `store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the + EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, …) are found + exactly where they already live — a namespacing change that renamed tenant #0's keys would be + a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes + proof #3 hold. + """ + key: str + tenant: object # harness.base.Tenant + store_namespace: str = "" + pool_cache: dict = field(default_factory=dict) + _lock: threading.RLock = field(default_factory=threading.RLock, repr=False) + + @property + def name(self): + return getattr(self.tenant, "name", self.key) + + @property + def datastore_path(self): + """The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the + isolation model — DuckDB has no RLS, so the OS boundary is the only one that holds).""" + return _ds.path_for(self.key) + + # ---------------------------------------------------------------- the store handle + def store_key(self, name): + """Namespace a product-data key for this tenant. The ONE place the prefix is applied.""" + return f"{self.store_namespace}{name}" if self.store_namespace else str(name) + + def get(self, name, fresh=False): + return store.get(self.store_key(name), fresh=fresh) + + def put(self, name, data): + return store.put(self.store_key(name), data) + + def update(self, name, fn, flush="sync"): + return store.update(self.store_key(name), fn, flush=flush) + + def exists(self, name): + return store.exists(self.store_key(name)) + + def available(self): + return store.available() + + def assert_datastore_matches(self): + """RAISE if the process-wide DuckDB file is not this tenant's. + + The alternative is the failure mode that matters: an analytical read served from another + tenant's file returns real, plausible, wrong rows — no exception, no telemetry, and the + number reconciles against the wrong book. A loud failure is strictly better, and this is + the assertion EXIT-5 deletes when the connection becomes per-tenant. + """ + want, have = self.datastore_path.resolve(), _ds.DB_PATH.resolve() + if want != have: + raise RuntimeError( + f"tenant {self.key!r} expects the analytical store at {want}, but this process " + f"has {have} open. Refusing the read — a cross-tenant answer would reconcile " + f"against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or call " + f"harness.datastore.use_path before serving this tenant.)") + + +def register(key, builder): + """Add a tenant to the registry (a stub connector is a legitimate builder — see X8's qa-b). + Invalidates any cached runtime for the slug so a re-registration cannot be shadowed.""" + _BUILDERS[str(key)] = builder + invalidate(key) + + +def known_tenants(): + return sorted(_BUILDERS) + + +def get_runtime(tenant_key): + """The runtime for `tenant_key`, built on first use and LRU-cached. + + Raises KeyError for an unknown slug — the caller turns that into a 401 (fail-closed: an + unknown tenant is not a 404, because confirming which slugs exist answers a question the + request was not entitled to ask). + """ + key = str(tenant_key or "").strip().lower() + if not key: + raise KeyError("no tenant") + with _LOCK: + rt = _CACHE.get(key) + if rt is not None: + _CACHE.move_to_end(key) # LRU: touch on read + return rt + builder = _BUILDERS.get(key) + if builder is None: + raise KeyError(key) + # Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the + # registry lock through it would serialise every other tenant's requests behind it. + tenant = builder() + with _LOCK: + existing = _CACHE.get(key) + if existing is not None: # another thread won the race — keep ONE runtime + _CACHE.move_to_end(key) + return existing + rt = TenantRuntime(key=key, tenant=tenant, + store_namespace="" if key == "royal-imports" else f"t/{key}/") + _CACHE[key] = rt + while len(_CACHE) > _MAX_RUNTIMES: + _CACHE.popitem(last=False) # evict least recently used + return rt + + +def invalidate(tenant_key=None): + """Drop one tenant's runtime, or all of them. Explicit, per X7 — a cache with no documented + way to clear it becomes a restart.""" + with _LOCK: + if tenant_key is None: + _CACHE.clear() + else: + _CACHE.pop(str(tenant_key or "").strip().lower(), None) + + +def resident(): + """The tenant slugs currently holding a runtime — LRU order, oldest first. For the EXIT-4 + measurement and for asserting the bound actually binds.""" + with _LOCK: + return list(_CACHE) diff --git a/royalimports_os/harness/semantic.py b/royalimports_os/harness/semantic.py new file mode 100644 index 0000000000000000000000000000000000000000..36cc20b2bac287f24f05d733b58a96726a280ca8 --- /dev/null +++ b/royalimports_os/harness/semantic.py @@ -0,0 +1,936 @@ +"""harness/semantic.py — the semantic seam (OM-0a, 2026-07-11). + +The ONE governed registry of topics + metrics (royalimports_os/model/*.yml) and the resolver that +turns a metric key into a number through the SAME proven data layer the validated modules use +(core/odoo domain builders + aggregate helpers) — so scope parity is by construction, and every +metric carries its definition, ai_context, and an independent validate contract. + +Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved- +dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4: +list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan: +.claude/wiki/research/omni-adoption.md (Part IV + the productization addendum). + +Design constraints honored (from /odoo-api): + - read_group dot-path GROUPBY fails → scalar aggregates here use sum_field/distinct_count; + grouped queries (OM-3+) must group by direct m2o fields only. + - Scope is never hand-rolled: topics BIND to a named domain builder (sale_line_domain, ...); + the YAML `scope:` block is the declarative statement of what that builder bakes in. + - READ-ONLY on Odoo, always. +""" +import ast +import functools +import operator +from pathlib import Path + +import yaml + +import core.odoo as O + +MODEL_DIR = Path(__file__).resolve().parents[1] / "model" + +# Topic scope binds to a NAMED, reviewed domain builder — never a YAML-assembled domain (one +# source of truth for scope until OM-2 makes topics executable against the store). +def _order_domain(*a, **kw): + # lazy: order scope currently lives in modules/sales.py; OM-2 lifts it into core/the store + import modules.sales as S + return S.order_domain(*a, **kw) + + +def _customer_move_domain(move_type): + """A posted customer document domain, matching `modules/returns.py`'s `_mdom` exactly. + + ⚠ COMPANY-LEVEL, and it REFUSES a team rather than ignoring one. Credit notes all carry + team_id = 1, so a BU filter on the document is meaningless — silently dropping the argument + would hand a business-unit user the company's number under their own name, which is the + widening sin in a different currency. + """ + def build(date_from=None, date_to=None, team_id=None, **kw): + if team_id: + raise ModelError( + f"customer {move_type} documents are COMPANY-LEVEL — credit notes and invoices " + f"carry no trustworthy business-unit tag, so a team-scoped total cannot be " + f"produced. Refusing rather than returning the company number.") + dom = [("move_type", "=", move_type), ("state", "=", "posted")] + if date_from: + dom.append(("invoice_date", ">=", date_from)) + if date_to: + dom.append(("invoice_date", "<=", date_to)) + return dom + return build + + +_BUILDERS = { + "sale_line_domain": O.sale_line_domain, + "sale_order_domain": _order_domain, + "credit_note_domain": _customer_move_domain("out_refund"), + "customer_invoice_domain": _customer_move_domain("out_invoice"), +} + + +class ModelError(Exception): + """A model-file problem (unknown key, bad reference, unparseable expr).""" + + +# ---------------------------------------------------------------- loading + +@functools.lru_cache(maxsize=1) +def _tenant(): + f = MODEL_DIR / "tenant.yml" + return yaml.safe_load(f.read_text(encoding="utf-8")) if f.exists() else {"params": {}} + + +def _sql_value(v): + """Render a tenant param into SQL safely (params are versioned config, not user input — + but escape anyway): ints as-is, lists comma-joined, strings single-quoted + escaped.""" + if isinstance(v, bool): + raise ModelError("boolean tenant params not supported in scope_sql") + if isinstance(v, (int, float)): + return str(v) + if isinstance(v, (list, tuple)): + return ", ".join(_sql_value(x) for x in v) + return "'" + str(v).replace("'", "''") + "'" + + +def render_scope(sql): + """Substitute {param} placeholders in a store scope/filter SQL fragment from tenant.yml.""" + params = _tenant().get("params") or {} + out = sql + for k, v in params.items(): + out = out.replace("{" + k + "}", _sql_value(v)) + if "{" in out and "}" in out: + import re as _re + missing = _re.findall(r"\{([a-z_]+)\}", out) + if missing: + raise ModelError(f"scope_sql references unknown tenant params: {missing}") + return out + + +@functools.lru_cache(maxsize=1) +def _model(): + topics, metrics = {}, {} + for f in sorted((MODEL_DIR / "topics").glob("*.yml")): + d = yaml.safe_load(f.read_text(encoding="utf-8")) + if not d or "key" not in d: + raise ModelError(f"topic file {f.name} lacks a 'key'") + if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS: + raise ModelError(f"topic {d['key']}: unknown domain_builder {d.get('domain_builder')!r}") + # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises + # a clear error if a live resolution is attempted on it + topics[d["key"]] = d + for f in sorted((MODEL_DIR / "metrics").glob("*.yml")): + d = yaml.safe_load(f.read_text(encoding="utf-8")) + topic = d.get("topic") + if topic not in topics: + raise ModelError(f"metrics file {f.name}: unknown topic {topic!r}") + for m in d.get("metrics", []): + if "key" not in m: + raise ModelError(f"metrics file {f.name}: a metric lacks a 'key'") + if m["key"] in metrics: + raise ModelError(f"duplicate metric key {m['key']!r}") + m.setdefault("topic", topic) # a metric may name its own topic (rare) + if m["topic"] not in topics: + raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}") + metrics[m["key"]] = m + return {"topics": topics, "metrics": metrics} + + +def reload_model(): + _model.cache_clear() + _tenant.cache_clear() + return _model() + + +def topics(): + return dict(_model()["topics"]) + + +def metrics(): + return dict(_model()["metrics"]) + + +def describe(key): + """Full metadata for a metric (the dictionary page / AI describe_topic feed).""" + m = _model()["metrics"].get(key) + if not m: + raise ModelError(f"unknown metric {key!r}") + return {**m, "topic_def": _model()["topics"][m["topic"]]} + + +# ---------------------------------------------------------------- resolving + +def _domain(topic_def, date_from, date_to, team_id): + b = topic_def.get("domain_builder") + if not b: + raise ModelError(f"topic {topic_def['key']!r} is store-only — use store_query() " + f"(no live-path domain builder bound yet)") + return _BUILDERS[b](date_from, date_to, team_id=team_id) + + +_OPS = {ast.Add: operator.add, ast.Sub: operator.sub, + ast.Mult: operator.mul, ast.Div: operator.truediv} + + +def _eval_expr(expr, resolve): + """Safely evaluate a derived-metric expression: metric keys, numbers, + - * / and parens.""" + def ev(node): + if isinstance(node, ast.Expression): + return ev(node.body) + if isinstance(node, ast.BinOp) and type(node.op) in _OPS: + return _OPS[type(node.op)](ev(node.left), ev(node.right)) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return -ev(node.operand) + if isinstance(node, ast.Num): # py<3.8 compat name; Constant below + return node.n + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): + return node.value + if isinstance(node, ast.Name): + return resolve(node.id) + raise ModelError(f"disallowed token in derived expr: {ast.dump(node)}") + try: + return ev(ast.parse(expr, mode="eval")) + except SyntaxError as e: + raise ModelError(f"unparseable derived expr {expr!r}: {e}") + + +def metric(key, date_from=None, date_to=None, team_id=None, _seen=None): + """Resolve a registered metric to a number for a window (+optional BU). The only public + compute path — every surface that shows this metric goes through here.""" + _seen = _seen or set() + if key in _seen: + raise ModelError(f"circular metric reference at {key!r}") + _seen = _seen | {key} + m = _model()["metrics"].get(key) + if not m: + raise ModelError(f"unknown metric {key!r}") + agg = m.get("agg") + if agg == "ratio": + num = metric(m["numerator"], date_from, date_to, team_id, _seen) + den = metric(m["denominator"], date_from, date_to, team_id, _seen) + return (num / den) if den else 0.0 + if agg == "derived": + return _eval_expr(m["expr"], lambda k: metric(k, date_from, date_to, team_id, _seen)) + t = _model()["topics"][m["topic"]] + dom = _domain(t, date_from, date_to, team_id) + if agg == "sum": + # ⚠ `negate` applies HERE too, not only in `_measure_sql`. It existed for GL income on a + # STORE-ONLY topic, so the live path never met it — the first metric that is both negated + # and parity-checked (`returns`) would otherwise compare a negative live number against a + # positive store one, and the "parity failure" would look like a data problem. + got = O.sum_field(t["entity"], dom, m["field"]) + return -got if m.get("negate") else got + if agg == "count_distinct": + return O.distinct_count(t["entity"], dom, m["field"]) + if agg == "count": + return O.get_odoo().search_count(t["entity"], dom) + raise ModelError(f"metric {key!r}: unknown agg {agg!r}") + + +# ---------------------------------------------------------------- validate contracts + +def _v_order_level_revenue(date_from, date_to, team_id): + """Independent check: line-level Σ price_subtotal == order-level Σ amount_untaxed.""" + import modules.sales as S + line = metric("revenue", date_from, date_to, team_id) + order = O.sum_field("sale.order", S.order_domain(date_from, date_to, team_id), "amount_untaxed") + return line, order + + +def _v_order_count(date_from, date_to, team_id): + """Exact reconciliation: order-level count == distinct line-parent orders + line-LESS orders. + (Confirmed orders with zero lines exist in the data — 2 found at first proof, 2026-07-11; + counting them explicitly makes the identity exact instead of hiding them in a tolerance, + and surfaces them as a data-health signal.)""" + import modules.sales as S + order_level = metric("orders", date_from, date_to, team_id) + dom = S.order_domain(date_from, date_to, team_id) + from_lines = O.distinct_count("sale.order.line", + O.sale_line_domain(date_from, date_to, team_id), "order_id") + empty = O.get_odoo().search_count("sale.order", dom + [("order_line", "=", False)]) + return order_level, from_lines + empty + + +def _v_gl_opex(date_from, date_to, team_id): + """Store-path opex == the live Odoo aggregate with the Expenses-module domain (posted, + expense-type accounts), to the cent. team_id N/A (company-level).""" + res = store_query("gl_lines", ["opex"], date_from=date_from, date_to=date_to) + ours = res["rows"][0]["opex"] if res["rows"] else 0 + dom = [("move_id.state", "=", "posted"), + ("account_id.account_type", "in", ["expense", "expense_depreciation"])] + if date_from: + dom.append(("date", ">=", date_from)) + if date_to: + dom.append(("date", "<=", date_to)) + live = O.sum_field("account.move.line", dom, "balance") + return ours, live + + +def _v_ar_outstanding(date_from, date_to, team_id): + """Store-path AR outstanding == live Odoo aggregate (posted out_invoice/out_refund residuals). + No date window — outstanding is an as-of-now balance.""" + res = store_query("receivables", ["ar_outstanding"]) + ours = res["rows"][0]["ar_outstanding"] if res["rows"] else 0 + live = O.sum_field("account.move", + [("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"])], + "amount_residual_signed") + return ours, live + + +def _v_returns_sign(date_from, date_to, team_id): + """Independent check on the NEGATION: `returns` must equal the ABSOLUTE untaxed total of the + same posted credit notes, read straight from Odoo without going through the metric. + + A lost negation makes returns negative; a doubled one makes it negative again. Both read as + "no returns" rather than as an error, and neither is visible to any self-consistency check — + the store and the live path would agree with each other perfectly while both being wrong. + """ + ours = metric("returns", date_from, date_to, team_id) + raw = O.sum_field("account.move", + _BUILDERS["credit_note_domain"](date_from, date_to, team_id), + "amount_untaxed_signed") + return ours, abs(raw) + + +def _v_line_vs_move_grain(date_from, date_to, team_id): + """Independent check on the invoice-LINE topic: the store's netted product-line total must + equal the same figure read straight from LIVE Odoo. + + Two things this catches that self-consistency cannot. (1) A LOST CREDIT-NOTE NEGATION — + `price_subtotal` is stored POSITIVE on a refund line, so a dropped subtraction inflates + revenue while every internal check still agrees. (2) A WRONG `display_type` FILTER — the + invoice tables carry 'cogs' and 'payment_term' rows alongside 'product' ones, and including + them changes the total silently rather than erroring. + + Company-level (team_id None) uses the strongest possible reference: the MOVE-grain + `amount_untaxed_signed`, a different table entirely. Per-BU there is no move-grain analogue + (account.move carries no business unit — every invoice sits on team 1), so the reference is + the live line-level aggregate reached through the sale-order dot path: still an independent + engine and query path from the DuckDB store, which is what the contract requires. + """ + ours = (store_query("invoice_lines", ["invoiced_line_sales"], + date_from=date_from, date_to=date_to, team_id=team_id) + .get("rows") or [{}])[0].get("invoiced_line_sales") or 0.0 + if team_id is None: + raw = O.sum_field("account.move", + [("state", "=", "posted"), + ("move_type", "in", ["out_invoice", "out_refund"]), + ("invoice_date", ">=", date_from), ("invoice_date", "<=", date_to)], + "amount_untaxed_signed") + return ours, raw + dom = [("parent_state", "=", "posted"), ("display_type", "=", "product"), + ("date", ">=", date_from), ("date", "<=", date_to), + ("sale_line_ids.order_id.team_id", "=", team_id)] + gross = O.sum_field("account.move.line", dom + [("move_type", "=", "out_invoice")], + "price_subtotal") + refunds = O.sum_field("account.move.line", dom + [("move_type", "=", "out_refund")], + "price_subtotal") + return ours, (gross or 0) - (refunds or 0) + + +_VALIDATORS = { + "returns_sign": _v_returns_sign, + "line_vs_move_grain": _v_line_vs_move_grain, + "order_level_revenue": _v_order_level_revenue, + "order_count": _v_order_count, + "gl_opex": _v_gl_opex, + "ar_outstanding": _v_ar_outstanding, +} + + +def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01): + """Run a metric's declared independent cross-check. Returns the standard check dict + (the validate.py convention: check/ok/gap).""" + m = _model()["metrics"].get(key) + if not m: + raise ModelError(f"unknown metric {key!r}") + contract = m.get("validate") + if not contract: + return {"check": f"semantic.{key}: no independent contract declared", "ok": True, "gap": 0.0, + "note": "declare one in model/metrics when an independent aggregate exists"} + fn = _VALIDATORS.get(contract["method"]) + if not fn: + raise ModelError(f"metric {key!r}: unknown validate method {contract['method']!r}") + ours, independent = fn(date_from, date_to, team_id) + gap = abs((ours or 0) - (independent or 0)) + return {"check": f"semantic.{key} == {contract['method']} ({date_from}..{date_to}, team={team_id})", + "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "independent": independent} + + +def validate(pre=None): + """Module-convention validate(): run every declared contract over YTD, all-BU + per-BU.""" + import core.periods as P + yf, yt = P.ytd() + out = [] + for key, m in _model()["metrics"].items(): + if not m.get("validate"): + continue + out.append(validate_metric(key, yf, yt, None)) + topic = _model()["topics"][m["topic"]] + if (topic.get("store") or {}).get("team_col"): # company-level topics have no BU runs + for tid in O.TEAM_IDS: + out.append(validate_metric(key, yf, yt, tid)) + return out + + +# ---------------------------------------------------------------- store path (OM-2) +# The compiler behind run_semantic_query: registered metrics + whitelisted dims → DuckDB SQL over +# the tenant store (harness/datastore.py). EVERY identifier comes from the versioned model files +# (trusted); every VALUE is parameterized — the small model only ever picks keys, so there is no +# injection surface. Live XML-RPC stays the validate() path (the store never validates itself). + +def _store_con(): + import harness.datastore as DS + if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently + raise ModelError("the data cache is still warming up after a restart (a one-time first " + "sync) — the dashboards work now from live data; ask me again in a " + "moment and I'll have it") + return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention + + +def _measure_sql(m, alias): + """SQL for a base measure. Supports FILTERED measures (`store_filter_sql` — the Omni + filtered-measure concept: sum(CASE WHEN … )) and `negate: true` (e.g. GL income, stored + credit-negative, reported positive).""" + agg, f = m.get("agg"), m.get("field") + flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None + if agg == "sum": + core = f"sum(CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END)" if flt else f"sum({alias}.{f})" + elif agg == "count_distinct": + core = (f"count(DISTINCT CASE WHEN {flt} THEN {alias}.{f} END)" if flt + else f"count(DISTINCT {alias}.{f})") + elif agg == "count": + core = f"count(CASE WHEN {flt} THEN 1 END)" if flt else "count(*)" + else: + raise ModelError(f"metric {m['key']!r}: agg {agg!r} has no store expression") + return f"-({core})" if m.get("negate") else core + + +def _expand_measures(keys): + """Expand ratio/derived metrics into the base measures SQL must aggregate, keeping the + requested keys for post-compute. Returns (base_keys, requested_keys).""" + mts = _model()["metrics"] + base, seen = [], set() + + def need(k): + if k in seen: + return + seen.add(k) + m = mts.get(k) + if not m: + raise ModelError(f"unknown metric {k!r}") + if m["agg"] == "ratio": + need(m["numerator"]); need(m["denominator"]) + elif m["agg"] == "derived": + import ast as _ast + for node in _ast.walk(_ast.parse(m["expr"], mode="eval")): + if isinstance(node, _ast.Name): + need(node.id) + else: + if k not in base: + base.append(k) + for k in keys: + need(k) + return base, list(keys) + + +def _post_compute(row, key): + m = _model()["metrics"][key] + if m["agg"] == "ratio": + num, den = _post_compute(row, m["numerator"]), _post_compute(row, m["denominator"]) + return (num / den) if den else 0.0 + if m["agg"] == "derived": + return _eval_expr(m["expr"], lambda k: _post_compute(row, k)) + return row.get(key) or 0.0 + + +# A metric's declared `format` -> the grid field type the compiler and the cells both read. +# One mapping, so a new metric cannot arrive as an untyped column. +_FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"} + + +def _clean_tree(tree, cols): + """Run a filter tree through the SAME validator the client's view state goes through. + + `filter_sql`'s documented input contract is "feed me CLEANED trees" — it is faithful to the + TS engine rather than fail-closed on garbage, because rejecting garbage is the validator's + job. The UI path always cleans (view state is sanitised on the way into the store), but + store_query/store_rows are callable directly, and an uncleaned tree is where the two + diverge: `{"value": null}` is ACTIVE in TS (and throws on a text field) and INACTIVE here. + + Normalising at the entry point makes that unreachable rather than merely unlikely, and + brings the depth / width / node-count caps to the server side too. `aios_grid` is a leaf + module with zero imports of its own, so this cannot cycle. + """ + from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD, + MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS) + + # REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per + # level, total nodes and depth by DROPPING the excess — correct on the client, where the + # cap is a DoS guard on a per-row render loop and a dropped condition is the lesser evil. + # On the server it is not: dropping AND-conditions WIDENS the result, and store_rows then + # reports a row_count and scope-wide totals that look authoritative for a query the caller + # never asked for. That is precisely the silent [:N] that [[no-unverifiable-aggregates]] + # forbids. A caller that exceeds the caps gets an error, not a quietly different answer. + total = 0 + + def _walk(nodes, depth): + nonlocal total + nodes = list(nodes or []) + if len(nodes) > MAX_FILTER_SIBLINGS: + raise ModelError(f"filter tree has {len(nodes)} conditions at one level; the limit " + f"is {MAX_FILTER_SIBLINGS}. Refusing rather than dropping the " + f"excess, which would silently widen the result.") + for node in nodes: + if not isinstance(node, dict): + continue + total += 1 + if isinstance(node.get('children'), list): + if depth >= MAX_FILTER_DEPTH: + raise ModelError(f"filter tree nests deeper than {MAX_FILTER_DEPTH} levels; " + f"refusing rather than dropping the deepest group.") + _walk(node['children'], depth + 1) + + _walk(tree, 1) + if total > MAX_FILTER_NODES: + raise ModelError(f"filter tree has {total} nodes; the limit is {MAX_FILTER_NODES}. " + f"Refusing rather than truncating, which would silently widen it.") + + # CG-8. A MEASURE condition ("Sales in the last 90 days > 5,000") is answered by + # harness/measure_filter.py, which resolves it to a SET OF CUSTOMER IDS. It has no meaning + # on this path, and both ways it could arrive here are silent: + # - on a topic without that column, `clean_filter_tree` DROPS it as unknown — widening the + # query while store_rows reports an authoritative row_count for something nobody asked + # for (the same class as the sibling/depth caps above); + # - on sales_lines at row grain, `revenue` IS a real column, so it would compile as a + # per-LINE predicate with the window silently discarded — a different question answered + # confidently, which is worse than dropping it. + # The discriminator is the `window`, not the column name: no column condition has one. + windowed = [] + cohorts = [] + + def _find(nodes): + for node in nodes or []: + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + _find(node['children']) + elif node.get('window') is not None: + windowed.append(str(node.get('colId'))) + elif node.get('colId') == _COHORT_FIELD: + cohorts.append(str(node.get('value'))) + + _find(tree) + # Owner item 5, and the same refusal for the same reason. A cohort leaf names a + # hand-curated set of CUSTOMERS held per user in the tenant store — this path knows nothing + # about it, and `clean_filter_tree` below would drop it as an unknown key, widening the + # query while store_rows still reported an authoritative row_count. If a caller ever needs + # it here, the fix is to pass the membership in, not to let it fall through. + if cohorts: + raise ModelError( + f"filter tree carries cohort-membership condition(s) {sorted(set(cohorts))} — " + f"cohort membership is host state, not a column of this topic. Refusing rather than " + f"dropping it, which would silently widen the result under an authoritative count.") + if windowed: + raise ModelError( + f"filter tree carries measure condition(s) over a date window {sorted(set(windowed))}" + f" — those are resolved to a set of ids by harness.measure_filter, not compiled into " + f"this query. Refusing rather than dropping or mis-compiling them, either of which " + f"would silently answer a different question under an authoritative count.") + return clean_filter_tree(tree, set(cols)) + + +def _measure_row_sql(m, alias): + """A sum-metric's per-ROW value: the same expression `_measure_sql` aggregates, without the + aggregate wrapper. At line grain `revenue` IS `l.price_subtotal` for that line — deriving it + from the metric rather than hand-writing the column is what keeps ONE definition of the + number (the drift the semantic layer exists to prevent). Only `sum` has a row value; a + count/count_distinct metric is a property of a SET, not of a row.""" + if m.get("agg") != "sum": + return None + f = m.get("field") + flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None + core = f"CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END" if flt else f"{alias}.{f}" + return f"-({core})" if m.get("negate") else core + + +def store_columns(topic, include_measures=True, grain="aggregate"): + """The `{colId: {sql, type, aggregate}}` spec `harness.filter_sql` compiles against (CG-1). + + `grain="row"` is the LINE-grain projection (CG-2): sum-metrics resolve to their own raw + field with `aggregate=False`, so a line table filters and sorts on real row values in WHERE + rather than needing HAVING. count-style metrics are dropped — they have no row value. + + This is the seam between the grid's field contract and the semantic model: the grid speaks + field KEYS, the compiler needs SQL expressions and a field TYPE, and only the model may say + what a key resolves to. Nothing about a topic leaks into filter_sql itself. + + -> the dim's NAME column (what the grid displays and what a user means when + they type "contains floral"), filtered PRE-aggregation in WHERE. Correct + even in a grouped query: every row of a group shares its group's name. + _id -> the raw id, for exact machine-keyed filtering. + date -> the topic's date column, so a grid can filter/sort it like any other field. + -> the aggregate expression, marked `aggregate` so the caller routes it to + HAVING rather than WHERE (see store_query — that path is not built yet). + """ + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + s = t["store"] + cols = {} + for key, d in (s.get("dims") or {}).items(): + cols[key] = {"sql": d.get("name_col") or d["col"], "type": "text", "aggregate": False} + # Only expose a separate id column when the dim actually HAS one. `city` is its own + # name (col == name_col), so a `city_id` would be a text column typed int — filtering + # it numerically would quietly compare 0 against 0 for every row. + if d.get("name_col") and d["name_col"] != d["col"]: + cols[f"{key}_id"] = {"sql": d["col"], "type": "int", "aggregate": False} + if s.get("date_col"): + cols["date"] = {"sql": s["date_col"], "type": "date", "aggregate": False} + if include_measures: + for k, m in _model()["metrics"].items(): + if m.get("topic") != topic or m.get("agg") in ("ratio", "derived"): + continue # ratio/derived are post-computed, not SQL + # The field TYPE comes from the metric's declared format, never a guess: `units` is + # format:int, and typing it currency would render 791,960 units as dollars. + ftype = _FORMAT_TYPE.get(m.get("format"), "currency") + if grain == "row": + row_sql = _measure_row_sql(m, s["alias"]) + if row_sql: + cols[k] = {"sql": row_sql, "type": ftype, "aggregate": False} + else: + cols[k] = {"sql": _measure_sql(m, s["alias"]), + "type": ftype, "aggregate": True} + return cols + + +def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None, + team_id=None, filters=None, sort=None, limit=1000, exclude_services=False, + filter_tree=None, filter_conj="and", today=None): + """The semantic query over the tenant store — the engine behind the Analyst's + run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model.""" + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + s = t["store"] + alias = s["alias"] + dims = s.get("dims") or {} + limit = max(1, min(int(limit or 1000), 5000)) + + base_keys, requested = _expand_measures(list(measures or [])) + if not base_keys: + raise ModelError("at least one measure required") + # Cross-topic base metrics (e.g. aov = revenue ÷ orders, where orders lives on sales_orders): + # in a SCALAR query they resolve via a nested scalar query on their HOME topic (correct grain — + # never count(*) on the wrong table); in a GROUPED query they are refused (split the request). + foreign = [k for k in base_keys if _model()["metrics"][k]["topic"] != topic] + if foreign and (group_by or grain): + raise ModelError(f"metrics {foreign} belong to another topic — cross-topic measures are " + f"scalar-only; run them against their own topic when grouping") + base_keys = [k for k in base_keys if k not in foreign] + foreign_vals = {} + for k in foreign: + sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from, + date_to=date_to, team_id=team_id) + foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0 + if not base_keys: # purely foreign scalar request + return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)", + "measures": requested, "group_by": [], "grain": None} + + select, group_cols, params = [], [], [] + gb = [g for g in (group_by or []) if g] + for g in gb: + if g not in dims: + raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})") + d = dims[g] + select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}") + group_cols.append(d["col"]) + if d.get("name_col"): + select.append(f"max({d['name_col']}) AS {g}") + if grain: + if grain not in ("month", "week", "day"): + raise ModelError("grain must be month|week|day") + select.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP)) AS period") + group_cols.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP))") + for k in base_keys: + select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}") + + where = [render_scope(s["scope_sql"].strip())] + # TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes — + # datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date). + # Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' < + # '2025-07-01 00:00:00'), which silently dropped a whole day of GL ($49,467). Cast both sides. + if date_from: + where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") + params.append(f"{date_from} 00:00:00") + if date_to: + where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?") + params.append(f"{date_to} 23:59:59") + if team_id: + if not s.get("team_col"): + raise ModelError(f"topic {topic!r} is company-level — it has no business-unit filter") + where.append(f"{s['team_col']} = ?"); params.append(int(team_id)) + if exclude_services and s.get("service_filter"): + where.append(render_scope(s["service_filter"])) + for dim_key, vals in (filters or {}).items(): + if dim_key not in dims: + raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})") + # id dims filter by int; TEXT dims (e.g. city) filter by the value itself — either way + # every VALUE stays parameterized (the whitelist covers identifiers only) + def _coerce(v): + try: + return int(v) + except (TypeError, ValueError): + return str(v) + ids = [_coerce(v) for v in (vals if isinstance(vals, (list, tuple)) else [vals])] + where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})") + params.extend(ids) + + # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape; + # this is the nested, 11-operator contract the table UI emits. Compiled by + # harness/filter_sql, which is held in lock-step with the TS engine and the validator by + # aios-web/verify_filter_engine.py. Appended AFTER the dim filters so params stay in + # positional order with `where`. + if filter_tree: + from harness import filter_sql as _fs + cols = store_columns(topic) + pred = _fs.compile_filter_tree(_clean_tree(filter_tree, cols), filter_conj, cols, + today=today) + if pred is not None: + if pred.uses_aggregate: + bad = sorted(c for c in pred.columns_used if cols[c].get("aggregate")) + raise ModelError( + f"filter_tree references measure(s) {bad} — a measure filter has to land in " + f"HAVING, and that path is deliberately NOT built: CG-1 ships the WHERE path " + f"because its consumer (CG-2, the line-grain sales table) does not aggregate. " + f"Filter on dims, or aggregate first and filter the result.") + where.append(pred.sql) + params.extend(pred.params) + + sql = (f"SELECT {', '.join(select)} FROM {s['table']} {alias} {s.get('join','')} " + f"WHERE {' AND '.join(where)}") + if group_cols: + sql += f" GROUP BY {', '.join(group_cols)}" + if sort: + key = sort.lstrip("-") + if key not in base_keys + gb + (["period"] if grain else []): + raise ModelError(f"sort {sort!r} must reference a selected measure/dim") + sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}" + elif grain: + sql += " ORDER BY period" + sql += f" LIMIT {limit}" + + con = _store_con() + try: + cur = con.execute(sql, params) + cols = [d[0] for d in cur.description] + rows = [dict(zip(cols, r)) for r in cur.fetchall()] + finally: + con.close() + for row in rows: # ratio/derived post-compute per group + row.update(foreign_vals) # scalar cross-topic components + for k in requested: + if _model()["metrics"][k]["agg"] in ("ratio", "derived"): + row[k] = _post_compute(row, k) + if "period" in row and row["period"] is not None: + row["period"] = str(row["period"])[:10] + return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql, + "measures": requested, "group_by": gb, "grain": grain} + + +def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None, + filter_conj="and", search=None, sorts=None, limit=200, offset=0, + aggregates=None, exclude_services=False, id_sql=None, member_ids=None, + today=None): + """ROW-grain fetch — the line-grain counterpart to store_query's aggregate path (CG-2). + + store_query answers "what is the number"; this answers "which rows". At line grain the + payload is the binding constraint (254,837 sale_order_lines against 1,550 customers — the + whole book would be ~96 MB at the customer table's measured 377 B/row), so rows come back + WINDOWED. + + A window is only honest if the counts and totals are not windowed with it + ([[no-unverifiable-aggregates]]). So this returns THREE independent numbers, each from its + own query over the FULL scope, never from `len(rows)`: + + row_count rows matching the filter across the whole scope -> the N of "N of M" + total_count rows in the scope with no filter at all -> the M + aggregates the topic's OWN metric definitions, summed over the whole FILTERED scope + + That is the difference between a windowed fetch and a silent `[:N]` cap: the user is told + how many rows exist and shown totals for all of them, while receiving only the page they + can see. + + Filters/sorts/search compile through harness.filter_sql, so a line table means EXACTLY what + the client engine means at customer grain — that equivalence is what verify_filter_engine.py + exists to hold. + + Dates are returned already truncated to the 10-char ISO shape the filter compares against, + so what is displayed and what is filtered are the same string. Numerics come back RAW; the + payload layer rounds them (aios_grid._round), and the compiler mirrors that rounding. + """ + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + from harness import filter_sql as _fs + s = t["store"] + alias = s["alias"] + cols = store_columns(topic, grain="row") + id_sql = id_sql or f"{alias}.id" + limit = max(1, min(int(limit or 200), 5000)) + offset = max(0, int(offset or 0)) + + # --- the scope every one of the three queries shares ------------------------------- + scope, scope_params = [render_scope(s["scope_sql"].strip())], [] + if date_from: + scope.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") + scope_params.append(f"{date_from} 00:00:00") + if date_to: + scope.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?") + scope_params.append(f"{date_to} 23:59:59") + if team_id: + if not s.get("team_col"): + raise ModelError(f"topic {topic!r} is company-level — it has no business-unit filter") + scope.append(f"{s['team_col']} = ?") + scope_params.append(int(team_id)) + if exclude_services and s.get("service_filter"): + scope.append(render_scope(s["service_filter"])) + + # --- the user's narrowing, on top of the scope -------------------------------------- + narrow, narrow_params = [], [] + pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols, + today=today, + member_ids=member_ids or None, id_sql=id_sql) + if pred is not None: + if pred.uses_aggregate: # cannot happen at grain="row"; guard anyway + raise ModelError(f"row-grain filter referenced an aggregate: {sorted(pred.columns_used)}") + narrow.append(pred.sql) + narrow_params.extend(pred.params) + spred = _fs.compile_search(search, cols) + if spred is not None: + narrow.append(spred.sql) + narrow_params.extend(spred.params) + + frm = f"{s['table']} {alias} {s.get('join', '')}" + scope_where = " AND ".join(scope) + all_where = " AND ".join(scope + narrow) + con = _store_con() + try: + # 1. the WINDOW of rows + select = [f"{id_sql} AS _rid"] + [ + (f"SUBSTR(CAST({c['sql']} AS VARCHAR), 1, 10) AS {k}" if c["type"] == "date" + else f"{c['sql']} AS {k}") + for k, c in cols.items()] + order = _fs.compile_order_by(sorts, cols, tiebreak_sql=id_sql) or f"{id_sql} ASC" + sql = (f"SELECT {', '.join(select)} FROM {frm} WHERE {all_where} " + f"ORDER BY {order} LIMIT {limit} OFFSET {offset}") + cur = con.execute(sql, scope_params + narrow_params) + names = [d[0] for d in cur.description] + rows = [dict(zip(names, r)) for r in cur.fetchall()] + + # 2. the two counts — over the WHOLE scope, never len(rows) + count_sql = f"SELECT count(*) FROM {frm} WHERE {all_where}" + row_count = con.execute(count_sql, scope_params + narrow_params).fetchone()[0] + total_count = con.execute(f"SELECT count(*) FROM {frm} WHERE {scope_where}", + scope_params).fetchone()[0] + + # 3. the aggregates — the topic's OWN metric definitions over the whole FILTERED scope + mts = _model()["metrics"] + keys = [k for k in (aggregates if aggregates is not None + else [k for k, m in mts.items() + if m.get("topic") == topic and m.get("agg") == "sum"])] + aggs = {} + if keys: + for k in keys: + if k not in mts or mts[k].get("topic") != topic: + raise ModelError(f"aggregate {k!r} is not a metric of topic {topic!r}") + agg_sql = ("SELECT " + ", ".join(f"{_measure_sql(mts[k], alias)} AS {k}" for k in keys) + + f" FROM {frm} WHERE {all_where}") + cur = con.execute(agg_sql, scope_params + narrow_params) + aggs = dict(zip([d[0] for d in cur.description], cur.fetchone())) + finally: + con.close() + + return { + "topic": topic, + "rows": rows, + "columns": {k: {"type": c["type"]} for k, c in cols.items()}, + "window": {"offset": offset, "limit": limit, "returned": len(rows)}, + "row_count": row_count, # N — matches the filter, across the WHOLE scope + "total_count": total_count, # M — the unfiltered scope + "aggregates": aggs, # over the whole FILTERED scope, not the window + "sql": sql, + "count_sql": count_sql, + } + + +def store_field_values(topic, dim, search=None, limit=25): + """Resolve real filterable values for a dim (the Analyst's get_field_values tool — fixes + 'NYC Florist' vs the actual record name BEFORE querying).""" + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + d = (t["store"].get("dims") or {}).get(dim) + if not d: + raise ModelError(f"unknown dim {dim!r} for topic {topic!r}") + if not d.get("name_col"): + raise ModelError(f"dim {dim!r} has no name column (filter by id)") + s = t["store"] + sql = (f"SELECT DISTINCT {d['col']} AS id, {d['name_col']} AS name " + f"FROM {s['table']} {s['alias']} {s.get('join','')} " + f"WHERE {render_scope(s['scope_sql'].strip())}") + params = [] + if search: + sql += f" AND {d['name_col']} ILIKE ?" + params.append(f"%{search}%") + sql += f" ORDER BY name LIMIT {max(1, min(int(limit), 100))}" + con = _store_con() + try: + return [{"id": r[0], "name": r[1]} for r in con.execute(sql, params).fetchall()] + finally: + con.close() + + +def store_parity(date_from=None, date_to=None): + """THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope.""" + import core.periods as P + if not date_from: + date_from, date_to = P.ytd() + out = [] + for key in ("revenue", "margin", "units", "orders", "customers", "returns", "invoiced"): + m = _model()["metrics"][key] + # ⚠ A COMPANY-LEVEL topic is checked at company scope ONLY. `store_query` RAISES when a + # team is asked of a topic with no team_col, and the live builder refuses too — so + # looping the BUs here would fail the gate for a topic that is correct. Parity for a + # company-level number means company-level parity; pretending otherwise would either + # crash or, worse, compare two numbers that silently ignored the team. + _store = (_model()["topics"].get(m["topic"]) or {}).get("store") or {} + scopes = (None, *O.TEAM_IDS) if _store.get("team_col") else (None,) + for tid in scopes: + live = metric(key, date_from, date_to, tid) + res = store_query(m["topic"], [key], date_from=date_from, date_to=date_to, team_id=tid) + ours = res["rows"][0][key] if res["rows"] else 0 + gap = abs((ours or 0) - (live or 0)) + out.append({"check": f"store.{key} == live.{key} (team={tid})", + "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live}) + # The agent dim (2026-07-17): store agent-filtered revenue must equal live revenue over the + # agent's FIRST-agent book — a fully independent path (live partner read + line-domain sum) + # against the store's res_partner.agent_id attribution. Largest book = the sharpest check. + recs = O.search_read("res.partner", [("active", "in", [True, False]), + ("agent_ids", "!=", False)], ["agent_ids"], limit=100000) + books = {} + for r in recs: + a = (r.get("agent_ids") or [None])[0] + if a: + books.setdefault(a, []).append(r["id"]) + if books: + top = max(books, key=lambda k: len(books[k])) + live = O.sum_field("sale.order.line", + O.sale_line_domain(date_from, date_to, partner_ids=books[top]), + "price_subtotal") + res = store_query("sales_lines", ["revenue"], date_from=date_from, date_to=date_to, + filters={"agent": [top]}) + ours = res["rows"][0]["revenue"] if res["rows"] else 0 + gap = abs((ours or 0) - (live or 0)) + out.append({"check": f"store.revenue[agent={top}] == live book sum (first-agent, " + f"{len(books[top])} customers)", + "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live}) + return out diff --git a/royalimports_os/harness/skills/__init__.py b/royalimports_os/harness/skills/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8d59efbb28eee0df4170bf77cb4e8f15c74be80f --- /dev/null +++ b/royalimports_os/harness/skills/__init__.py @@ -0,0 +1 @@ +"""Portable skills — written once against the canonical model, run on any compatible connector.""" diff --git a/royalimports_os/harness/skills/sales.py b/royalimports_os/harness/skills/sales.py new file mode 100644 index 0000000000000000000000000000000000000000..63fcd76c400895287fb095906dda56861f1ae097 --- /dev/null +++ b/royalimports_os/harness/skills/sales.py @@ -0,0 +1,168 @@ +"""Sales skill — PORTABLE. The de-Odoo'd replacement for modules/sales.py: every metric is expressed +against the canonical model and runs on any connector with the SALES capability (INVOICING for +returns). No model names, no `amount_untaxed`, no Odoo — the connector translates. + +`validate(tenant)` parity-checks each metric against the current Odoo module, so a port only replaces +the original once it ties out to the cent (same discipline as the modules' own validate()). + +Date windows come from core.periods (calendar math is source-agnostic). +""" +from __future__ import annotations +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # royalimports_os/ +import core.periods as P +from harness.canonical import Capability, Entity, Query, Filter +from harness.base import Tenant + +REQUIRES = (Capability.SALES,) + + +# ------------------------------------------------------------------ tenant scope -> canonical filters +def _scope(tenant: Tenant, business_unit: str | None): + """The client's scope CONFIG (was hard-coded Odoo constants) as canonical filters — valid on both + SALES_ORDER and SALES_ORDER_LINE (status/business_unit/customer exist on both).""" + cfg = tenant.config + fs = [] + if cfg.get('confirmed_statuses'): + fs.append(Filter('status', 'in', list(cfg['confirmed_statuses']))) + bus = cfg.get('business_units') or {} + if business_unit and business_unit in bus: + fs.append(Filter('business_unit', 'in', [bus[business_unit]])) + elif bus: + fs.append(Filter('business_unit', 'in', list(bus.values()))) + if cfg.get('excluded_customer_ids'): + fs.append(Filter('customer', 'not_in', list(cfg['excluded_customer_ids']))) + return tuple(fs) + + +def _sum(conn, entity, measure, df, dt, filters): + return sum(r[measure] for r in conn.aggregate( + Query(entity, (measure,), 'sum', date_from=df, date_to=dt, filters=filters))) + + +def _count(conn, entity, df, dt, filters): + return sum(r['count'] for r in conn.aggregate( + Query(entity, (), 'count', date_from=df, date_to=dt, filters=filters))) + + +# ------------------------------------------------------------------ metrics (portable) +def revenue(tenant, date_from, date_to, business_unit=None): + return _sum(tenant.source(Capability.SALES), Entity.SALES_ORDER, 'amount', + date_from, date_to, _scope(tenant, business_unit)) + + +def headline(tenant, business_unit=None, t=None): + conn = tenant.source(Capability.SALES) + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + sc = _scope(tenant, business_unit) + rev = _sum(conn, Entity.SALES_ORDER, 'amount', yf, yt, sc) + rev_ly = _sum(conn, Entity.SALES_ORDER, 'amount', lf, lt, sc) + orders = _count(conn, Entity.SALES_ORDER, yf, yt, sc) + custs = conn.distinct_count(Query(Entity.SALES_ORDER, date_from=yf, date_to=yt, filters=sc), 'customer') + return {'ytd_revenue': rev, 'ytd_revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly), + 'ytd_orders': orders, 'ytd_customers': custs, 'aov': (rev / orders) if orders else 0.0} + + +def monthly_trend(tenant, n=13, business_unit=None, t=None): + conn = tenant.source(Capability.SALES) + t = t or P.today() + first = P.month_starts(n, t)[0][1] + span_from = f"{int(first[:4]) - 1:04d}{first[4:]}" # a year before the first shown month + q = Query(Entity.SALES_ORDER, ('amount',), 'sum', grain='month', + date_from=span_from, date_to=t.isoformat(), filters=_scope(tenant, business_unit)) + m = {r['period']: r['amount'] for r in conn.aggregate(q)} + rows = [] + for ym, _s, _e in P.month_starts(n, t): + y, mo = int(ym[:4]) - 1, int(ym[5:7]) + this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0) + rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) + return rows + + +def top_customers(tenant, limit=25, business_unit=None, t=None): + conn = tenant.source(Capability.SALES) + yf, yt = P.ytd(t or P.today()) + q = Query(Entity.SALES_ORDER, ('amount',), 'sum', group_by=('customer',), + date_from=yf, date_to=yt, filters=_scope(tenant, business_unit)) + rows = [{'pid': r['customer']['id'], 'customer': r['customer']['name'], + 'revenue': r['amount'], 'orders': r['count']} + for r in conn.aggregate(q) if r.get('customer')] + return sorted(rows, key=lambda x: -x['revenue'])[:limit] + + +def top_skus(tenant, limit=25, business_unit=None, t=None): + conn = tenant.source(Capability.SALES) + yf, yt = P.ytd(t or P.today()) + q = Query(Entity.SALES_ORDER_LINE, ('amount', 'qty', 'margin'), 'sum', group_by=('product',), + date_from=yf, date_to=yt, filters=_scope(tenant, business_unit)) + rows = [] + for r in conn.aggregate(q): + if not r.get('product'): + continue + lines = r['count'] or 0 + rows.append({'product': r['product']['name'], 'pid': r['product']['id'], 'revenue': r['amount'], + 'qty': r['qty'], 'margin': r['margin'], + 'profit_per_order': (r['margin'] / lines) if lines else 0.0}) + return sorted(rows, key=lambda x: -x['revenue'])[:limit] + + +def by_category(tenant, limit=20, business_unit=None, t=None): + conn = tenant.source(Capability.SALES) + yf, yt = P.ytd(t or P.today()) + q = Query(Entity.SALES_ORDER_LINE, ('amount',), 'sum', group_by=('category',), + date_from=yf, date_to=yt, filters=_scope(tenant, business_unit)) + rows = [{'category': r['category'], 'revenue': r['amount']} for r in conn.aggregate(q)] + return sorted(rows, key=lambda x: -x['revenue'])[:limit] + + +def returns_monthly(tenant, n=13, t=None): + """Returns (credit notes) per month — needs INVOICING, consolidated (no BU split).""" + conn = tenant.source(Capability.INVOICING) + t = t or P.today() + first = P.month_starts(n, t)[0][1] + span_from = f"{int(first[:4]) - 1:04d}{first[4:]}" + ex = tenant.config.get('excluded_customer_ids') or [] + filters = [Filter('doc_type', 'eq', 'out_refund'), Filter('status', 'eq', 'posted')] + if ex: + filters.append(Filter('customer', 'not_in', list(ex))) + q = Query(Entity.INVOICE, ('amount',), 'sum', grain='month', + date_from=span_from, date_to=t.isoformat(), filters=tuple(filters)) + m = {r['period']: -r['amount'] for r in conn.aggregate(q)} # out_refund amount is negative -> flip + rows = [] + for ym, _s, _e in P.month_starts(n, t): + y, mo = int(ym[:4]) - 1, int(ym[5:7]) + this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0) + rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) + return rows + + +# ------------------------------------------------------------------ parity validation (port gate) +def validate(tenant, t=None): + """Reconcile the ported skill to the current Odoo module — a port ships only when it ties out.""" + import modules.sales as M + t = t or P.today() + yf, yt = P.ytd(t) + checks = [] + + def chk(name, a, b, tol=1.0): + checks.append({'check': name, 'a': round(a, 2), 'b': round(b, 2), + 'gap': round(a - b, 2), 'ok': abs(a - b) <= tol}) + + h, mh = headline(tenant), M.headline() + chk('headline YTD revenue == module', h['ytd_revenue'], mh['ytd_revenue']) + chk('headline YTD orders == module', h['ytd_orders'], mh['ytd_orders']) + chk('headline YTD customers == module', h['ytd_customers'], mh['ytd_customers']) + chk('Σ top_customers == module', sum(r['revenue'] for r in top_customers(tenant, 10**9)), + sum(r['revenue'] for r in M.top_customers(limit=10**9))) + chk('Σ by_category == module', sum(r['revenue'] for r in by_category(tenant, 10**9)), + sum(r['revenue'] for r in M.by_category(limit=10**9))) + chk('Σ top_skus revenue == module', sum(r['revenue'] for r in top_skus(tenant, 10**9)), + sum(r['revenue'] for r in M.top_skus(limit=10**9))) + chk('monthly_trend Σ this-yr == module', sum(r['revenue'] for r in monthly_trend(tenant)), + sum(r['revenue'] for r in M.monthly_trend())) + chk('returns_monthly Σ this-yr == module', sum(r['revenue'] for r in returns_monthly(tenant)), + sum(r['revenue'] for r in M.returns_monthly())) + return checks diff --git a/royalimports_os/harness/tables.py b/royalimports_os/harness/tables.py new file mode 100644 index 0000000000000000000000000000000000000000..1a59557ad10411007456c3e22231b45427c640e9 --- /dev/null +++ b/royalimports_os/harness/tables.py @@ -0,0 +1,129 @@ +"""harness/tables.py — a topic-backed TABLE for the grid (CG-2). + +The grid's first table (customers) is built from `modules.customer_data.pool()`: one row per +customer, the whole book shipped as component args, filtered client-side. That works at 1,550 +rows and does not work at line grain — `sales_lines` is 201,558 rows / ~89 MB. + +This module builds the SECOND table: rows come from `semantic.store_rows`, so the fetch is +WINDOWED and the filtering is SQL. Its whole job is to hand the grid a payload in the same +host-neutral shape the React component already speaks, plus the one thing a windowed table must +carry that a whole-book table never needed: + + counts = {matched, total, shown, windowed} + +`matched` and `total` come from their own queries over the FULL scope; `shown` is the size of +this window. The component renders "showing SHOWN of MATCHED (filtered from TOTAL)" from those +numbers and NEVER from `rows.length` — which is the difference between an honest window and a +silent `[:N]` wearing a total ([[no-unverifiable-aggregates]]). + +The field CONTRACT is derived from the topic, not hand-written: labels and types come from the +model's dims and metrics, so a new metric cannot arrive as an untyped, unlabelled column and the +grid can never disagree with the semantic layer about what a column means. +""" +from harness import semantic as _sem + +#: Topic-backed tables the grid can open. `key` is the storage key its saved views live under — +#: separate from the customer table's, so the two tables' views can never collide. +TOPIC_TABLES = { + "sales_lines": { + "label": "Sales lines", + "storage_key": "sales_lines_table_workspace", + # the columns worth defaulting to visible, in order; everything else stays available + "default_visible": ["date", "order_partner", "product", "category", "state", "agent", + "team", "revenue", "units", "margin"], + # a line-grain table is unusable sorted by nothing in particular + "default_sorts": [{"colId": "date", "dir": "desc"}], + }, +} + +#: how many rows one window carries. Small enough that the payload stays a few hundred KB at +#: ~440 B/row, large enough to scroll. The honest counts make the window visible rather than +#: silent, so this is a performance knob and not a correctness one. +PAGE_ROWS = 200 + + +def _labels(topic): + """colId -> human label, taken from the MODEL (dim labels, metric labels).""" + t = _sem.topics()[topic] + out = {} + for key, d in ((t.get("store") or {}).get("dims") or {}).items(): + out[key] = d.get("label") or key.replace("_", " ").title() + out[f"{key}_id"] = f"{out[key]} id" + out["date"] = "Date" + for k, m in _sem.metrics().items(): + if m.get("topic") == topic: + out[k] = m.get("label") or k + return out + + +def table_fields(topic): + """The grid field contract for a topic, derived from the semantic model. + + Every column is `source='odoo'` — read-only. A line is a fact from the ERP; unlike the + customer table there is no overlay stratum here (notes/tags hang off a CUSTOMER, not off an + individual order line), so the two-strata split simply does not arise. + """ + cfg = TOPIC_TABLES[topic] + cols = _sem.store_columns(topic, grain="row") + labels = _labels(topic) + order = [k for k in cfg["default_visible"] if k in cols] + order += [k for k in cols if k not in order] + return [{ + "key": k, + "label": labels.get(k, k.replace("_", " ").title()), + "type": cols[k]["type"], + "source": "odoo", + "default": k in cfg["default_visible"], + } for k in order] + + +def table_payload(topic, config=None, offset=0, limit=None, team_id=None, + date_from=None, date_to=None): + """Fetch ONE window of a topic-backed table plus its scope-wide counts and totals. + + `config` is a grid ViewConfig (filters / filterConj / sorts / search) — the same shape the + customer table persists, so a saved view means the same thing on either table. + + Returns the component payload: {fields, rows, counts, aggregates, storageKey}. + """ + if topic not in TOPIC_TABLES: + raise _sem.ModelError(f"no topic-backed table registered for {topic!r}") + cfg = TOPIC_TABLES[topic] + config = config or {} + limit = PAGE_ROWS if limit is None else limit + + got = _sem.store_rows( + topic, + filter_tree=config.get("filters") or [], + filter_conj=config.get("filterConj") or "and", + search=config.get("search") or None, + sorts=config.get("sorts") or cfg["default_sorts"], + member_ids=config.get("memberPids") or None, + limit=limit, offset=offset, + team_id=team_id, date_from=date_from, date_to=date_to, + ) + + rows = [] + for i, r in enumerate(got["rows"]): + # `pid` is the grid's row identity contract (selection, detail, overlay all key on it). + # At line grain that identity is the order LINE id, which store_rows returns as _rid. + row = {k: v for k, v in r.items() if k != "_rid"} + row["pid"] = r["_rid"] + rows.append(row) + + return { + "fields": table_fields(topic), + "rows": rows, + # THE honest window. `matched`/`total` are scope-wide queries; `shown` is this window. + # A caller must never recompute `matched` from len(rows). + "counts": { + "shown": len(rows), + "matched": got["row_count"], + "total": got["total_count"], + "windowed": True, + }, + "aggregates": got["aggregates"], # over the whole FILTERED scope, not the window + "storageKey": cfg["storage_key"], + "label": cfg["label"], + "window": got["window"], + } diff --git a/royalimports_os/harness/telemetry.py b/royalimports_os/harness/telemetry.py new file mode 100644 index 0000000000000000000000000000000000000000..6feb0eb8e989b03dc7fc8263da11709c9d266820 --- /dev/null +++ b/royalimports_os/harness/telemetry.py @@ -0,0 +1,94 @@ +"""harness/telemetry.py — usage telemetry v0 (OM-3; principle 7: the improvement loop is +structural). Append-only JSONL of what people actually do — pages opened, explore queries run, +Analyst questions asked — the observe-half of just-in-time modeling: what's used gets promoted +into the governed model; what's ignored gets pruned. + +Fail-silent by design: telemetry must never break a page or a query. Local file v0 +(data/store/usage.jsonl, tenant data, git-ignored); mined in-session for now, store-backed later. +""" +import json +import sys +import time +import traceback +from pathlib import Path + +PATH = Path(__file__).resolve().parents[1] / "data" / "store" / "usage.jsonl" +ERR_PATH = PATH.parent / "errors.jsonl" + + +def log(kind, **fields): + try: + PATH.parent.mkdir(parents=True, exist_ok=True) + with PATH.open("a", encoding="utf-8") as f: + f.write(json.dumps({"ts": time.strftime("%Y-%m-%d %H:%M:%S"), + "kind": kind, **fields}, default=str) + "\n") + except Exception: + pass + + +def error(where, exc=None, **fields): + """THE central error sink (owner rule 2026-07-23: ALL errors are logged). One JSONL line per + caught error (data/store/errors.jsonl — per-deployment, git-ignored) + a mirror line on + stderr so it also lands in the Space container logs. Call it from EVERY except block that + swallows or downgrades an exception — a failure with nowhere to land is how the app breaks + silently (the 2026-07-23 drawer fire). Never raises.""" + try: + rec = {"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "where": str(where)[:120], **fields} + if exc is not None: + rec["type"] = type(exc).__name__ + rec["msg"] = str(exc)[:400] + try: + tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__, + limit=8)) + if tb.strip(): + rec["trace"] = tb[-1800:] + except Exception: + pass + ERR_PATH.parent.mkdir(parents=True, exist_ok=True) + with ERR_PATH.open("a", encoding="utf-8") as f: + f.write(json.dumps(rec, default=str) + "\n") + print(f"[app-error] {rec['ts']} {rec['where']}: " + f"{rec.get('type', '')} {rec.get('msg', '')}", file=sys.stderr, flush=True) + except Exception: + pass + + +def read_errors(limit=500): + """Tail of the error log (newest last) — the Data Health admin section reads this.""" + if not ERR_PATH.exists(): + return [] + out = [] + try: + with ERR_PATH.open(encoding="utf-8") as f: + for line in f: + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + continue + except Exception: + return out + return out[-limit:] + + +def read(limit=5000): + if not PATH.exists(): + return [] + out = [] + with PATH.open(encoding="utf-8") as f: + for line in f: + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + continue + return out[-limit:] + + +def summary(): + """Counts by kind + the most-touched pages/topics — the promote-or-prune shortlist.""" + from collections import Counter + ev = read() + kinds = Counter(e.get("kind") for e in ev) + pages = Counter(e.get("page") for e in ev if e.get("kind") == "page") + topics = Counter(e.get("topic") for e in ev if e.get("topic")) + return {"events": len(ev), "by_kind": dict(kinds), + "top_pages": pages.most_common(10), "top_topics": topics.most_common(10)} diff --git a/royalimports_os/harness/tenants.py b/royalimports_os/harness/tenants.py new file mode 100644 index 0000000000000000000000000000000000000000..8ca76e27e1587d662b659fb2a2be8c520c3c3469 --- /dev/null +++ b/royalimports_os/harness/tenants.py @@ -0,0 +1,96 @@ +"""Tenant definitions — one entry per client of the platform. + +Royal Imports is the first tenant. Everything that was HARD-CODED in core/odoo.py (teams 5/6, the +GIFTWARE house account, the sale/done statuses) moves here as per-client CONFIG. A second client is +just another Tenant with its own connectors + config — the skills and UI don't change. + +In production this comes from a tenants store (DB / HF dataset), not a literal; kept literal here so +the pivot is legible and runnable today. +""" +from __future__ import annotations +from harness.base import Tenant +from harness.canonical import Capability +from harness.connectors.odoo import OdooConnector + + +def royal_imports(doc_mode: str = 'order') -> Tenant: + odoo = OdooConnector(doc_mode=doc_mode) + return Tenant( + key='royal-imports', name='Royal Imports', + sources={odoo.key: odoo}, + config={ + 'business_units': {'Fisch': 5, 'Royal': 6}, # canonical BU -> Odoo team_id + 'confirmed_statuses': ['sale', 'done'], + 'excluded_customer_ids': OdooConnector.excluded_customer_ids(), # GIFTWARE house account + 'channels': {'Amazon': 'GIFTWARE DEALS'}, # marketplace split (see Amazon channel memory) + 'branding': {'navy': '#1F4E78', 'gold': '#C8A24B', 'name': 'Royal Imports'}, + }) + + +class _EmptyConnector: + """A source that answers every canonical query with NOTHING — the X8 isolation fixture. + + Deliberately NOT a mock library object and NOT a subclass shortcut around `aggregate`: the + isolation proof has to run against the SAME code path a real tenant uses, so this answers the + real interface and returns real (empty) results. A test double that skipped the interface + would prove the double is isolated, not that the runtime is. + + An EMPTY pool is the strongest possible fixture for the assertion that matters: tenant B's + `/customers` must return ZERO rows. Any row at all in that response came from somewhere it + should not have, and there is no ambiguity about whose it is. + """ + key = 'empty' + name = 'Empty (QA)' + + def capabilities(self) -> set: + return set() + + def aggregate(self, q) -> list[dict]: + return [] + + def records(self, q) -> list[dict]: + return [] + + def distinct_count(self, q, fieldname: str) -> int: + return 0 + + def supports(self, cap: Capability) -> bool: + return False + + def entities(self) -> set: + return set() + + +def qa_tenant_b() -> Tenant: + """Tenant 'qa-b' — the SECOND tenant the isolation proof needs, with no Odoo behind it. + + D-2's assertion is "tenant B cannot reach tenant A", and that cannot be tested with one + tenant configured. This is the minimum second tenant: its own slug (so its own store + namespace and its own DuckDB path), no credentials, and a connector that returns nothing. + + ⚠ NOT registered by default. `harness.runtime` only admits it when `AIOS_ENABLE_QA_TENANT=1`, + so a production deployment has exactly one tenant and an attacker cannot mint a session + against a fixture. (X8 says "the registry gains qa-b"; gating it is an additive safety + amendment, recorded in the S1 mailbox 2026-07-30 — the isolation MECHANISM under test, the + store namespace and the cookie's tenant claim, is identical either way.) + """ + empty = _EmptyConnector() + return Tenant( + key='qa-b', name='QA Tenant B', + sources={empty.key: empty}, + config={ + 'business_units': {}, + 'confirmed_statuses': [], + 'excluded_customer_ids': [], + 'channels': {}, + 'branding': {'name': 'QA Tenant B'}, + }) + + +# Example of a *different-shape* client (accounting-only), to show the harness adapts. Uncomment and +# wire real QBO creds to light up ledger/AR/customer skills — and NOT the SKU/inventory ones. +# def demo_qbo_client() -> Tenant: +# from harness.connectors.quickbooks import QuickBooksConnector +# qb = QuickBooksConnector(realm_id='...', token='...') +# return Tenant(key='demo-qbo', name='Demo (QuickBooks)', sources={qb.key: qb}, +# config={'business_units': {}, 'confirmed_statuses': [], 'excluded_customer_ids': []}) diff --git a/royalimports_os/harness/tools.py b/royalimports_os/harness/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..18b308caaef52094d7275a949e31abadf7235d6e --- /dev/null +++ b/royalimports_os/harness/tools.py @@ -0,0 +1,510 @@ +"""harness/tools.py — the compounding TOOL REGISTRY (OM-4 spine, 2026-07-11). + +The formal tool surface the (small) model calls — the productization directive's "tools we keep +compounding". Every tool wraps the SEMANTIC layer (harness/semantic.py): the model navigates by +registry keys and recipe plans (model/skills/*.skill.yml — recipes and these tools version +TOGETHER), never by SQL. Adding a connector/topic extends what the SAME tools reach — that is the +compounding. Exported in OpenAI function-calling format (`openai_tools()`) — OpenRouter-compatible, +so any cheap model with tool-calling drives the platform. + +Correctness posture (Part VI of the plan): whitelisted keys only; values parameterized downstream; +every query result carries a result_id + drill note; artifact tools (save/compose/schedule/alert) +are explicit and confirm-gated by recipe. Errors return a uniform envelope the model can read. +""" +import json +import time +import uuid +from pathlib import Path + +import harness.semantic as SEM + +VIEWS_PATH = Path(__file__).resolve().parents[1] / "data" / "store" / "views.json" + +_RESULTS = {} # result_id -> query result (session-scoped working memory for chart tools) +_RESULTS_CAP = 40 + +# The exhaustive chart vocabulary (2026-07-16): every Zelazny comparison form has a kind, so the +# Analyst never lacks a shape. The model picks by the CHART PICKER guide (analyst.py) + the +# charting skill recipes; the platform owns every pixel (app._render_analyst_artifact). +CHART_KINDS = ( + "line", "bar", "area", "scatter", "kpi", "map", # the original six + "pie", "donut", # part-to-whole (≤6 slices) + "stacked_bar", "grouped_bar", "ranked_bar", "stacked_pct", # composition / rank forms + "combo", "yoy_bars", # level+rate; this-vs-last-year + "waterfall", "pareto", "histogram", "heatmap", "treemap", # bridge / concentration / distribution + "funnel", "bullet", "bubble", "sparkline", # stages / target / 3-measure / mini +) + +# Per-kind param contract (beyond x): what else the spec must carry to be renderable. +_KIND_NEEDS = { + "combo": ("y", "y2"), "bullet": ("y", "y2"), "bubble": ("y", "size"), + "heatmap": ("y", "value"), "histogram": (), # histogram bins x itself + "stacked_bar": ("y", "series"), "grouped_bar": ("y", "series"), + "stacked_pct": ("y", "series"), +} +_QUERY_KEYS = ("topic", "measures", "group_by", "grain", "date_from", "date_to", + "team_id", "filters", "sort", "limit", "exclude_services") + + +def _remember(res): + rid = uuid.uuid4().hex[:10] + _RESULTS[rid] = res + while len(_RESULTS) > _RESULTS_CAP: + _RESULTS.pop(next(iter(_RESULTS))) + return rid + + +def _ok(data): + return {"ok": True, "data": data} + + +def _err(msg): + return {"ok": False, "error": str(msg)[:400]} + + +# ------------------------------------------------------------------ schema tools + +def list_topics(): + """The 'what data exists' tool.""" + out = [] + for k, t in SEM.topics().items(): + out.append({"topic": k, "label": t.get("label"), "entity": t.get("entity"), + "grain": t.get("grain"), + "dims": list((t.get("store") or {}).get("dims") or {}), + "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]}) + return out + + +def describe_topic(topic): + """The schema-learning tool: scope, grain, dims, metrics w/ definitions, and ai_context.""" + t = SEM.topics().get(topic) + if not t: + raise SEM.ModelError(f"unknown topic {topic!r} (use list_topics)") + mets = {k: {"label": m.get("label"), "description": m.get("description"), + "format": m.get("format"), "ai_context": m.get("ai_context")} + for k, m in SEM.metrics().items() if m["topic"] == topic} + return {"topic": topic, "label": t.get("label"), "scope": t.get("scope"), + "grain": t.get("grain"), "ai_context": t.get("ai_context"), + "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()}, + "metrics": mets} + + +# ------------------------------------------------------------------ query tools + +def run_semantic_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None, + team_id=None, filters=None, sort=None, limit=1000, exclude_services=False): + # limit defaults HIGH (1000): transforms/charts operate on the FULL result while the model + # only ever sees rows[:100] — a small default silently truncated per-customer analytics + # (the BELLA FLORIST wrong-decliner incident, 2026-07-17). + res = SEM.store_query(topic, measures, group_by=group_by, grain=grain, date_from=date_from, + date_to=date_to, team_id=team_id, filters=filters, sort=sort, + limit=limit, exclude_services=exclude_services) + # Echo the full query onto the result: chart specs built from it carry the query, so a SAVED + # view is a re-runnable QUERY (the OM-3 viewer re-executes it live), never a stale snapshot. + res["query"] = {"topic": topic, "measures": list(measures or []), "group_by": group_by, + "grain": grain, "date_from": date_from, "date_to": date_to, "team_id": team_id, + "filters": filters, "sort": sort, "limit": limit, + "exclude_services": exclude_services} + rid = _remember(res) + # The EFFECTIVE window, stated by the platform — the model must repeat this, never guess + # (a query without dates covers all recorded history; there is no hidden default window). + if date_from and date_to: + window = f"{date_from} to {date_to}" + elif date_from or date_to: + window = f"{'from ' + date_from if date_from else 'through ' + date_to}" + else: + window = "ALL recorded history (no date filter was applied)" + out = {"result_id": rid, "rows": res["rows"][:100], "row_count": res["row_count"], + "measures": res["measures"], "group_by": res["group_by"], "grain": res["grain"], + "window": window, + "note": "every number here is drillable; cite result_id when charting"} + if res["row_count"] >= (limit or 1000): # surface every truncation (plan hard line) + out["warning"] = (f"TRUNCATED: the result hit limit={limit} — the full set is larger. " + "Re-run with a higher limit (max 5000) BEFORE ranking, comparing or " + "aggregating, or your answer will be computed on a partial set.") + return out + + +def get_field_values(topic, dim, search=None): + return SEM.store_field_values(topic, dim, search=search) + + +# ------------------------------------------------------------------ transform tool (governed) + +def transform_result(result_id, transforms): + """Apply governed ANALYTICS TRANSFORMS to a query result -> a NEW result_id to chart/table. + The chain is recorded on the derived result, so saved views replay query -> transforms live.""" + res = _RESULTS.get(result_id) + if not res: + raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first") + import harness.transforms as TR + rows, applied = TR.apply(res, transforms, run_query=_run_query) + new = {**res, "rows": rows, "row_count": len(rows), + "transforms": (res.get("transforms") or []) + applied} + rid = _remember(new) + return {"result_id": rid, "rows": rows[:100], "row_count": len(rows), + "columns": sorted(rows[0]) if rows else [], + "note": "derived result — chart THIS result_id to show the transform"} + + +def _run_query(q): + return SEM.store_query(**{k: q.get(k) for k in _QUERY_KEYS if q.get(k) is not None}) + + +# ------------------------------------------------------------------ viz tools (emit OUR specs) + +def make_chart(result_id, kind, x, y=None, title=None, series=None, y2=None, size=None, + value=None, facet=None): + """Returns a validated CHART SPEC the platform renders with its own primitives (design system + enforced — the model never emits HTML/vega). Extra encodings per kind: combo/bullet need y2 + (line/target), bubble needs size, heatmap needs value (the colour measure); facet (a dim + column) turns line|bar|area|scatter into small multiples.""" + res = _RESULTS.get(result_id) + if not res: + raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first") + if kind not in CHART_KINDS: + raise SEM.ModelError(f"kind must be one of {CHART_KINDS}") + cols = set(res["rows"][0]) if res["rows"] else set() + if y is None and kind != "histogram": + raise SEM.ModelError(f"kind={kind!r} needs y (only histogram bins x by itself)") + for ref, nm in ((x, "x"), (y, "y"), (series, "series"), (y2, "y2"), (size, "size"), + (value, "value"), (facet, "facet")): + if ref and ref not in cols: + raise SEM.ModelError(f"{nm}={ref!r} not in result columns {sorted(cols)}") + given = {"y": y, "y2": y2, "size": size, "value": value, "series": series} + missing = [p for p in _KIND_NEEDS.get(kind, ()) if not given.get(p)] + if missing: + raise SEM.ModelError(f"kind={kind!r} also needs {missing} " + f"(pick from result columns {sorted(cols)})") + if facet and kind not in ("line", "bar", "area", "scatter"): + raise SEM.ModelError("facet (small multiples) works with line|bar|area|scatter only") + if kind == "yoy_bars" and f"{y}_ly" not in cols: + raise SEM.ModelError(f"yoy_bars needs a {y}_ly column — run transform_result " + "[{'op':'yoy'}] on the result first") + spec = {"kind": kind, "x": x, "y": y, "series": series, + "title": title or f"{y or x} by {x}", "result_id": result_id, + "query": res.get("query"), "rows": res["rows"]} + for k, v in (("y2", y2), ("size", size), ("value", value), ("facet", facet), + ("transforms", res.get("transforms"))): + if v: + spec[k] = v + out = {"chart": spec} + if kind in ("pie", "donut") and len(res["rows"]) > 6: + out["note"] = (f"{len(res['rows'])} slices — the platform will show the top 5 plus an " + "'Other' bucket; for a cleaner story run transform_result top_n first") + return out + + +def make_table(result_id, columns=None, title=None): + """First-class TABLE artifact: the exact rows, house-formatted (sortable, totals row, the + drill IS the table). columns (optional) picks and orders a subset.""" + res = _RESULTS.get(result_id) + if not res: + raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first") + rows = res["rows"] + if columns: + cols = set(rows[0]) if rows else set() + bad = [c for c in columns if c not in cols] + if bad: + raise SEM.ModelError(f"columns {bad} not in result columns {sorted(cols)}") + rows = [{c: r.get(c) for c in columns} for r in rows] + return {"table": {"kind": "table", "title": title, "columns": columns, + "result_id": result_id, "query": res.get("query"), + "transforms": res.get("transforms"), "rows": rows}} + + +def make_kpi_card(result_id, metric, compare_result_id=None): + res = _RESULTS.get(result_id) + if not res or not res["rows"]: + raise SEM.ModelError("result_id missing/empty — run a scalar run_semantic_query first") + val = res["rows"][0].get(metric) + if val is None: + raise SEM.ModelError(f"{metric!r} not in result") + card = {"kpi": {"metric": metric, "value": val, "result_id": result_id, + "query": res.get("query")}} + if compare_result_id and _RESULTS.get(compare_result_id, {}).get("rows"): + prev = _RESULTS[compare_result_id]["rows"][0].get(metric) + if prev: + card["kpi"]["delta_pct"] = (val - prev) / abs(prev) + card["kpi"]["compare_result_id"] = compare_result_id + card["kpi"]["compare_query"] = _RESULTS[compare_result_id].get("query") + return card + + +# ------------------------------------------------------------------ artifact tools (v0: local) + +def _load_views(): + if VIEWS_PATH.exists(): + return json.loads(VIEWS_PATH.read_text(encoding="utf-8")) + return {"views": {}, "dashboards": {}} + + +def _save_views(d): + VIEWS_PATH.parent.mkdir(parents=True, exist_ok=True) + VIEWS_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8") + + +def save_view(name, chart): + """Persist a chart/KPI spec (from make_chart / make_kpi_card) as a named view. Specs persist + WITH their semantic query and WITHOUT rows — the OM-3 viewer re-executes the query live, so a + saved view is always current, never a snapshot.""" + d = _load_views() + spec = chart.get("chart") or chart.get("kpi") or chart.get("table") or chart + spec = {k: v for k, v in spec.items() if k != "rows"} + if "kpi" in chart and not spec.get("kind"): + spec["kind"] = "kpi" + if "table" in chart and not spec.get("kind"): + spec["kind"] = "table" + if not spec.get("query"): + raise SEM.ModelError("spec carries no query — pass the exact object returned by " + "make_chart / make_kpi_card (from a fresh run_semantic_query)") + d["views"][name] = {"chart": spec, "saved_at": time.strftime("%Y-%m-%d %H:%M")} + _save_views(d) + return {"saved": name, "views": list(d["views"])} + + +def compose_dashboard(name, views): + """'Spawn a dashboard': compose saved views into a named dashboard spec (rendered at OM-3).""" + d = _load_views() + missing = [v for v in views if v not in d["views"]] + if missing: + raise SEM.ModelError(f"unknown views {missing} — save_view them first") + d["dashboards"][name] = {"views": views, "created_at": time.strftime("%Y-%m-%d %H:%M")} + _save_views(d) + return {"dashboard": name, "views": views} + + +# ------------------------------------------------------------------ the GAP LOOP (rung 4) + +GAP_KINDS = ("dimension", "metric", "transform", "chart_kind", "data_source", "other") +GAPS_KEY = "analyst_gaps" +GAPS_CAP = 500 + + +def report_gap(kind, missing, question, workaround=None): + """Log a CAPABILITY GAP: the model determined (after checking the schema) that no registered + dim/metric/transform/kind can answer. The entry lands in telemetry AND the durable store — + the admin Gaps view aggregates them into the platform build backlog. This is how every + honest 'I can't' becomes the next dim, transform, or recipe.""" + if kind not in GAP_KINDS: + raise SEM.ModelError(f"kind must be one of {GAP_KINDS}") + entry = {"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "kind": kind, + "missing": str(missing)[:120], "question": str(question)[:300], + "workaround": (str(workaround)[:200] if workaround else None)} + import harness.telemetry as TEL + TEL.log("analyst_gap", **{("gap_kind" if k == "kind" else k): v for k, v in entry.items()}) + try: # durable copy — async, the chatlog write posture + import threading + + import core.store as store + if store.available(): + def _fn(data): + data = list(data or []) + data.append(entry) + return data[-GAPS_CAP:] + threading.Thread(target=lambda: store.update(GAPS_KEY, _fn), + daemon=True, name="analyst-gap").start() + except Exception: + pass + return {"logged": True, + "note": "gap recorded for the platform backlog — now tell the user in ONE sentence " + "what is missing and offer the nearest ask that IS answerable"} + + +# ------------------------------------------------------------------ workspace tools (OM-3/P11) + +def list_workspace(): + """Everything in the tenant workspace (uniform modular objects) + available templates.""" + import harness.workspace as W + return {"objects": W.items(), "templates": W.templates()} + + +def instantiate_template(filename, new_name=None): + """Stamp a tenant-agnostic template into this workspace as NEW objects (never overwrites).""" + import harness.workspace as W + return W.instantiate_template(filename, new_name) + + +def update_view(name, changes): + """Patch an existing view (spec and/or query) — validated by re-execution before saving.""" + import harness.views as V + return V.update_view(name, changes) + + +def update_workbook(name, views=None, new_name=None): + """Recompose and/or rename a workbook (dashboard); renames follow into schedules.""" + import harness.views as V + return V.update_dashboard(name, views_list=views, new_name=new_name) + + +def delete_object(kind, name): + """Delete any workspace object. DESTRUCTIVE — the recipe requires explicit confirmation.""" + import harness.workspace as W + W.delete(kind, name) + return {"deleted": f"{kind} · {name}"} + + +# ------------------------------------------------------------------ registry + dispatch + +def _p(props, required): + return {"type": "object", "properties": props, "required": required} + + +TOOLS = { + "list_topics": {"fn": lambda **kw: list_topics(), + "description": "List the datasets (topics) available: their metrics, dims, and grain. Start here.", + "parameters": _p({}, [])}, + "describe_topic": {"fn": lambda **kw: describe_topic(kw["topic"]), + "description": "Full schema of one topic: scope rules, metric definitions, dims, and the business context you must respect.", + "parameters": _p({"topic": {"type": "string"}}, ["topic"])}, + "get_field_values": {"fn": lambda **kw: get_field_values(kw["topic"], kw["dim"], kw.get("search")), + "description": "Resolve real filter values (ids+names) for a dim. ALWAYS use before filtering by a typed name.", + "parameters": _p({"topic": {"type": "string"}, "dim": {"type": "string"}, + "search": {"type": "string"}}, ["topic", "dim"])}, + "run_semantic_query": {"fn": lambda **kw: run_semantic_query(**kw), + "description": "Run a governed query: registered measures over a topic, optional group_by dims / time grain / filters. The ONLY way to read data.", + "parameters": _p({"topic": {"type": "string"}, + "measures": {"type": "array", "items": {"type": "string"}}, + "group_by": {"type": "array", "items": {"type": "string"}}, + "grain": {"type": "string", "enum": ["month", "week", "day"]}, + "date_from": {"type": "string"}, "date_to": {"type": "string"}, + "team_id": {"type": "integer"}, + "filters": {"type": "object"}, + "sort": {"type": "string"}, "limit": {"type": "integer"}, + "exclude_services": {"type": "boolean"}}, + ["topic", "measures"])}, + "transform_result": {"fn": lambda **kw: transform_result(kw["result_id"], kw["transforms"]), + "description": "Apply governed analytics transforms to a result -> a NEW result_id (a " + "CHAINABLE list of {op, ...}). The Tableau-class analytics library — pick " + "op names (full catalog + recipes in the analytics skill). Families: " + "ordering/rank (sort, head, bottom_n, rank, rank_pct, ntile, top_n, " + "add_total) · part-to-whole (share_of_total, cum_share) · running/moving " + "(running_total/avg/max/min, running_count, moving_average/sum/median, " + "rolling_std) · period-over-period (diff, pct_change, lag, lead, " + "diff_from_first, index_to_100, percent_of_max, compare) · distribution/" + "stats (bin, describe, zscore, outliers, winsorize, clip, normalize, " + "correlate, weighted_average, safe_ratio, product) · business (abc_classify, " + "concentration, contribution_to_change, rfm, funnel_rates) · modeling " + "(trend_line, regression, cagr, growth_rate) · reference lines as columns " + "(reference_line, reference_band, target_line, xmr_limits) · reshape (pivot, " + "unpivot, filter_rows, dedupe, resample) · re-query windows (yoy, ytd, " + "rolling, forecast). ADDITIVITY LAW: accumulating ops " + "(running_total/share_of_total/cum_share/moving_sum/abc_classify/" + "concentration) work on ADDITIVE measures (revenue/units/margin/orders); " + "for a cumulative/trailing DISTINCT count (customers) or ratio use ytd/" + "rolling (they re-query) — never running_total. NEVER compute any of these " + "yourself; transform, then chart/table the new result_id.", + "parameters": _p({"result_id": {"type": "string"}, + "transforms": {"type": "array", "items": {"type": "object"}}}, + ["result_id", "transforms"])}, + "make_chart": {"fn": lambda **kw: make_chart(**kw), + "description": "Turn a query result into a platform chart. kinds: line|bar|area|scatter|" + "map|pie|donut|stacked_bar|grouped_bar|ranked_bar|stacked_pct|combo|" + "yoy_bars|waterfall|pareto|histogram|heatmap|treemap|funnel|bullet|bubble|" + "sparkline. x/y/series must be result columns. Extra encodings: combo " + "(bars y + line y2, dual axis), bullet (value y vs target y2), bubble " + "(scatter + size), heatmap (dims x,y + colour value), histogram (bins x, " + "no y), facet (a dim column -> small multiples of line|bar|area|scatter). " + "yoy_bars needs the yoy transform first. kind='map' plots customers " + "geographically: x = the customer dim, dot size = y.", + "parameters": _p({"result_id": {"type": "string"}, "kind": {"type": "string", "enum": list(CHART_KINDS)}, + "x": {"type": "string"}, "y": {"type": "string"}, + "title": {"type": "string"}, "series": {"type": "string"}, + "y2": {"type": "string"}, "size": {"type": "string"}, + "value": {"type": "string"}, "facet": {"type": "string"}}, + ["result_id", "kind", "x"])}, + "make_table": {"fn": lambda **kw: make_table(kw["result_id"], kw.get("columns"), + kw.get("title")), + "description": "Turn a query result into a first-class TABLE artifact (house-formatted, " + "totals row, drillable). Use when the user wants exact figures, many " + "columns, or a list — not a shape. columns (optional) picks and orders.", + "parameters": _p({"result_id": {"type": "string"}, + "columns": {"type": "array", "items": {"type": "string"}}, + "title": {"type": "string"}}, ["result_id"])}, + "make_kpi_card": {"fn": lambda **kw: make_kpi_card(**kw), + "description": "Turn a scalar query result into a KPI card; optional compare_result_id adds a YoY delta.", + "parameters": _p({"result_id": {"type": "string"}, "metric": {"type": "string"}, + "compare_result_id": {"type": "string"}}, ["result_id", "metric"])}, + "report_gap": {"fn": lambda **kw: report_gap(kw["kind"], kw["missing"], kw["question"], + kw.get("workaround")), + "description": "LAST RESORT — log a capability gap. Call ONLY after list_topics/" + "describe_topic confirm that NO registered dimension, metric, transform " + "or chart kind can answer the user's question (e.g. stock on hand, " + "which has no topic). NOT a gap: YoY/decline/growth compares " + "(transform_result yoy), rankings/top-N, shares, running totals, " + "distributions — those are ANSWERABLE via transform_result. Then tell " + "the user plainly what is missing and offer the nearest answerable ask. " + "NEVER call this for something the tools support, and NEVER guess " + "instead of calling it.", + "parameters": _p({"kind": {"type": "string", "enum": list(GAP_KINDS)}, + "missing": {"type": "string", + "description": "what does not exist, short (e.g. 'inventory/stock-on-hand topic')"}, + "question": {"type": "string", + "description": "the user's question, verbatim"}, + "workaround": {"type": "string", + "description": "the nearest answerable alternative you offered"}}, + ["kind", "missing", "question"])}, + "save_view": {"fn": lambda **kw: save_view(kw["name"], kw["chart"]), + "description": "Save a chart as a named view (confirm with the user first).", + "parameters": _p({"name": {"type": "string"}, "chart": {"type": "object"}}, ["name", "chart"])}, + "compose_dashboard": {"fn": lambda **kw: compose_dashboard(kw["name"], kw["views"]), + "description": "Compose saved views into a named dashboard (confirm with the user first).", + "parameters": _p({"name": {"type": "string"}, + "views": {"type": "array", "items": {"type": "string"}}}, ["name", "views"])}, + "list_workspace": {"fn": lambda **kw: list_workspace(), + "description": "List the tenant workspace: every saved view/dashboard/alert/report " + "(modular objects) plus available templates. Use when the user asks what " + "exists, wants to reuse/manage artifacts, or before composing.", + "parameters": _p({}, [])}, + "instantiate_template": {"fn": lambda **kw: instantiate_template(kw["filename"], + kw.get("new_name")), + "description": "Stamp a tenant-agnostic template (from list_workspace) into the " + "workspace as NEW objects — never overwrites. Confirm with the user first.", + "parameters": _p({"filename": {"type": "string"}, "new_name": {"type": "string"}}, + ["filename"])}, + "update_view": {"fn": lambda **kw: update_view(kw["name"], kw.get("changes")), + "description": "UPDATE an existing saved view: patch spec keys (title/kind/x/y/series) " + "and/or 'query' subkeys (measures, group_by, grain, date_from, date_to, " + "team_id, filters, sort, limit; null REMOVES a key). The patched query is " + "re-executed before saving — invalid updates are rejected. Confirm first.", + "parameters": _p({"name": {"type": "string"}, "changes": {"type": "object"}}, + ["name", "changes"])}, + "update_workbook": {"fn": lambda **kw: update_workbook(kw["name"], kw.get("views"), + kw.get("new_name")), + "description": "UPDATE a workbook (dashboard): recompose its views (list = new display " + "order; add/remove by including/omitting) and/or rename it. Confirm first.", + "parameters": _p({"name": {"type": "string"}, + "views": {"type": "array", "items": {"type": "string"}}, + "new_name": {"type": "string"}}, ["name"])}, + "delete_object": {"fn": lambda **kw: delete_object(kw["kind"], kw["name"]), + "description": "DELETE a workspace object (view|dashboard|alert|report). DESTRUCTIVE — " + "requires the user's explicit confirmation in this conversation first.", + "parameters": _p({"kind": {"type": "string", + "enum": ["view", "dashboard", "alert", "report"]}, + "name": {"type": "string"}}, ["kind", "name"])}, +} + + +def openai_tools(): + """The registry in OpenAI function-calling format (OpenRouter-compatible).""" + return [{"type": "function", + "function": {"name": k, "description": v["description"], "parameters": v["parameters"]}} + for k, v in TOOLS.items()] + + +def dispatch(name, arguments): + """Uniform tool execution for the Analyst loop: JSON-safe result or a readable error the + model can act on. Never raises.""" + t = TOOLS.get(name) + if not t: + return _err(f"unknown tool {name!r} (tools: {list(TOOLS)})") + try: + args = json.loads(arguments) if isinstance(arguments, str) else dict(arguments or {}) + missing = [r for r in t["parameters"].get("required", []) if r not in args] + if missing: + return _err(f"missing required arguments: {missing}") + return _ok(t["fn"](**args)) + except SEM.ModelError as e: + return _err(e) + except Exception as e: + return _err(f"{type(e).__name__}: {e}") diff --git a/royalimports_os/harness/transforms.py b/royalimports_os/harness/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..a3a133b8e3c75a2590596a38fa87acac79855bb1 --- /dev/null +++ b/royalimports_os/harness/transforms.py @@ -0,0 +1,1287 @@ +"""harness/transforms.py — governed ANALYTICS TRANSFORMS over query results (OM-4). + +The model never touches SQL or raw math: it requests named transforms on a result_id +(tools.transform_result) and the platform computes them here, deterministically, over the exact +rows the governed query returned. The chain is RECORDED on the derived result ("transforms"), so +a saved view replays query -> transforms at render time (harness/views.run_view) and stays live. + +This is the AIOS Analyst's TABLE-CALCULATION + ANALYTICS library — the open-source Tableau-class +capability set (Tableau table calcs + Analytics pane, Power BI DAX quick measures, pandas/polars +window ops, scipy/statsmodels stats) mapped onto ONE governed dispatcher. Grounding brief: +.claude/wiki/research/analytics-tools.md. Design laws (2026-07-18): + + 1. ONE tool, many ops. Every capability is a pure function rows->rows (or rows->summary) in the + OPS registry, dispatched through tools.transform_result — so the small model's prompt carries + ONE tool + a skill file, never 50 signatures. Ops CHAIN. + 2. ADDITIVITY IS LAW (no-unverifiable-aggregates). Additivity comes from the metric's `agg` in + the semantic model, NOT the column name: sum/count are additive; count_distinct and ratio are + NOT. Accumulating ops (running_total, cum_share, share_of_total, moving_sum) REFUSE a known + non-additive measure with a readable error; collapse ops (top_n's Other, add_total, pivot) + land it as None, never a fabricated sum. A cumulative/rolling DISTINCT count must re-run the + governed query per widening window (the ytd/rolling ops) — never sum displayed values. + 3. RE-AGGREGATION belongs in the QUERY, not here. Transforms are row-wise / window / reshape. + To regroup or re-aggregate, run a new run_semantic_query with a different group_by — that keeps + the parity proof. pivot is a pure RESHAPE (collision -> error, never a hidden sum). + 4. NO black-box models. Trend/forecast are transparent least-squares / seasonal-naive, fully + hand-computable and unit-tested; we do not pull in Prophet/Merlion/k-means (our own AVOID + briefs). Every op has a hand-computed unit test (_demo_analytics.py) — that suite is the ship + gate, not the provider-flaky eval gate. + 5. Reference/trend/forecast LINES are delivered as ADDED COLUMNS and drawn with the existing + combo/line chart kinds — the renderer stays untouched. +""" +import ast +import datetime as dt +import math + +import harness.semantic as SEM + + +# ================================================================== additivity (law #2) + +# Column-name suffixes that are inherently non-additive regardless of the base metric (ratios, +# indices, standardized/derived series that must never be re-summed). +_NON_ADDITIVE_SUFFIXES = ( + "_pct", "_ratio", "_share", "_share_pct", "_yoy_pct", "_cum_pct", "_pct_change", + "_pct_of_max", "_rank_pct", "_zscore", "_norm", "_idx", "_index100", "_wavg", + "_running_avg", "_ref", "_band_lo", "_band_hi", "_trend", "_ytd", +) +_NON_ADDITIVE_NAMES = {"aov", "margin_pct"} +_INHERIT_SUFFIXES = ("_ly", "_delta") # _ly / _delta inherit 's additivity + + +def _metric_agg(key): + try: + m = SEM.metrics().get(key) + except Exception: + m = None + return m.get("agg") if m else None + + +def _base_metric(col): + base = col + for suf in _INHERIT_SUFFIXES: + if base.endswith(suf): + return base[: -len(suf)] + return base + + +def known_non_additive(col): + """True when we KNOW a column must not be summed across rows/groups: a non-additive suffix, + a registered ratio/count_distinct metric, or a ratio/count_distinct-derived _ly/_delta. Unknown + numeric columns default to additive (we only block what we can prove wrong).""" + if col.endswith(_NON_ADDITIVE_SUFFIXES) or col.endswith("_z") or col in _NON_ADDITIVE_NAMES: + return True + return _metric_agg(_base_metric(col)) in ("count_distinct", "ratio") + + +def _is_additive(col): + return not known_non_additive(col) + + +# ================================================================== small helpers + +def _numeric(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def _num_cols(rows): + return [k for k in (rows[0] if rows else {}) if any(_numeric(r.get(k)) for r in rows)] + + +def _text_cols(rows): + nums = set(_num_cols(rows)) + return [k for k in (rows[0] if rows else {}) if k not in nums] + + +def _need_col(rows, col, what): + if not rows: + raise SEM.ModelError("cannot transform an empty result") + if col not in rows[0]: + raise SEM.ModelError(f"{what}={col!r} not in result columns {sorted(rows[0])}") + + +def _need_additive(rows, of, op): + _need_col(rows, of, "of") + if known_non_additive(of): + raise SEM.ModelError( + f"{op} sums {of!r} across rows, but {of!r} is a distinct-count or ratio measure and is " + f"NOT additive — summing it would over-count. For a cumulative/rolling distinct count " + f"use the 'ytd' or 'rolling' op (they re-run the query per window); otherwise pick an " + f"additive measure (revenue, units, margin, orders, cogs).") + + +def _vals(rows, col): + return [r.get(col) for r in rows if _numeric(r.get(col))] + + +def _mean(v): + return (sum(v) / len(v)) if v else None + + +def _median(v): + s = sorted(v) + n = len(s) + if not n: + return None + return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2.0 + + +def _std(v): + """Sample standard deviation (ddof=1); None for n<2.""" + n = len(v) + if n < 2: + return None + m = sum(v) / n + return math.sqrt(sum((x - m) ** 2 for x in v) / (n - 1)) + + +def _percentile(v, p): + """Linear-interpolation percentile, p in [0,100] (the numpy 'linear' method).""" + s = sorted(v) + n = len(s) + if not n: + return None + if n == 1: + return float(s[0]) + k = (n - 1) * (p / 100.0) + lo = math.floor(k) + hi = math.ceil(k) + if lo == hi: + return float(s[int(k)]) + return s[lo] + (s[hi] - s[lo]) * (k - lo) + + +# ================================================================== ordering / rank + +def sort_rows(rows, by, direction="desc"): + """Sort rows by a column, Nones last either way; works for numeric and text columns.""" + _need_col(rows, by, "by") + present = [r for r in rows if r.get(by) is not None] + absent = [r for r in rows if r.get(by) is None] + return sorted(present, key=lambda r: r[by], reverse=(direction != "asc")) + absent + + +def head(rows, n=20): + """Keep the FIRST n rows in the current order — pairs with sort for 'most negative first' + asks, where top_n (largest-by-value) is the wrong shape.""" + return rows[: max(1, min(int(n or 20), 500))] + + +def bottom_n(rows, by, n=10): + """The N SMALLEST rows by `by` (sort ascending + take N) — Tableau's 'Bottom N'.""" + _need_col(rows, by, "by") + return sort_rows(rows, by, "asc")[: max(1, min(int(n or 10), 500))] + + +def rank_rows(rows, by, direction="desc", out="rank"): + """Sort by `by` and add a 1-based rank column (ties keep row order — 'first' method).""" + ordered = sort_rows(rows, by, direction) + return [{**r, out: i + 1} for i, r in enumerate(ordered)] + + +def rank_pct(rows, by, direction="desc", out="rank_pct"): + """Percentile rank (0-100): position among the rows after sorting by `by`. Top row = 100 + (desc) — 'this row beats X% of the rest'.""" + ordered = sort_rows(rows, by, direction) + n = len([r for r in ordered if r.get(by) is not None]) + res = [] + for i, r in enumerate(ordered): + pr = (100.0 * (n - 1 - i) / (n - 1)) if (n > 1 and r.get(by) is not None) else ( + 100.0 if r.get(by) is not None else None) + res.append({**r, out: pr}) + return res + + +def ntile(rows, by, tiles=4, direction="asc", out="ntile"): + """Assign each row to one of `tiles` equal-count buckets by `by` (quartile=4, decile=10). + direction='asc' -> tile 1 is the smallest values (Tableau/pandas qcut convention).""" + _need_col(rows, by, "by") + tiles = max(2, min(int(tiles or 4), 100)) + ordered = sort_rows(rows, by, direction) + present = [r for r in ordered if r.get(by) is not None] + n = len(present) + res = [] + for i, r in enumerate(ordered): + if r.get(by) is None: + res.append({**r, out: None}) + else: + res.append({**r, out: min(tiles, int(i * tiles / n) + 1)}) + return res + + +# ================================================================== top-N / composition + +def top_n(rows, by, n=10, other=True, other_label="Other"): + """Keep the N largest rows by `by`; the rest collapse into ONE labelled bucket (additive + columns summed, non-additive columns None — never a fake average). other=False just truncates, + and the caller must surface the cut (the tool notes it).""" + _need_col(rows, by, "by") + n = max(1, min(int(n or 10), 500)) + ordered = sort_rows(rows, by, "desc") + head_rows, tail = ordered[:n], ordered[n:] + if not tail or not other: + return head_rows + bucket = _collapse(rows, tail, f"{other_label} ({len(tail)})") + return head_rows + [bucket] + + +def add_total(rows, label="Total"): + """Append a grand-total row: additive columns summed, non-additive columns None (honest), + the first text column = `label`. Same discipline as make_table's totals row, but in-data so a + chart can show it.""" + if not rows: + raise SEM.ModelError("cannot total an empty result") + return rows + [_collapse(rows, rows, label)] + + +def _collapse(all_rows, subset, label): + bucket = {} + for k in all_rows[0]: + vals = [t.get(k) for t in subset if _numeric(t.get(k))] + if vals and _is_additive(k): + bucket[k] = sum(vals) + elif vals: + bucket[k] = None + else: + bucket[k] = "" + texts = _text_cols(all_rows) + if texts: + bucket[texts[0]] = label + return bucket + + +# ================================================================== part-to-whole / cumulative + +def share_of_total(rows, of, out=None): + """Add `_share_pct` (0-100): each row's share of the column total across THESE rows.""" + _need_additive(rows, of, "share_of_total") + out = out or f"{of}_share_pct" + total = sum(r.get(of) or 0 for r in rows) + return [{**r, out: (100.0 * (r.get(of) or 0) / total) if total else None} for r in rows] + + +def cum_share(rows, of, out=None): + """Pareto prep: sort desc by `of`, add cumulative share % (0-100). The 'top X carry Y%' read.""" + _need_additive(rows, of, "cum_share") + out = out or f"{of}_cum_pct" + ordered = sort_rows(rows, of, "desc") + total = sum(r.get(of) or 0 for r in ordered) + run, res = 0.0, [] + for r in ordered: + run += r.get(of) or 0 + res.append({**r, out: (100.0 * run / total) if total else None}) + return res + + +# ================================================================== running / moving (window) + +def running_total(rows, of, out=None): + """Add `_running`: cumulative sum in the rows' current order (sort first if needed).""" + _need_additive(rows, of, "running_total") + out = out or f"{of}_running" + run, res = 0.0, [] + for r in rows: + run += r.get(of) or 0 + res.append({**r, out: run}) + return res + + +def running_avg(rows, of, out=None): + """Add `_running_avg`: expanding (cumulative) mean in row order — a smoothing, valid on + any numeric column (it never claims a total).""" + _need_col(rows, of, "of") + out = out or f"{of}_running_avg" + tot, cnt, res = 0.0, 0, [] + for r in rows: + v = r.get(of) + if _numeric(v): + tot += v + cnt += 1 + res.append({**r, out: (tot / cnt) if cnt else None}) + return res + + +def running_extreme(rows, of, kind="max", out=None): + """Add `_running_max` / `_running_min`: the cumulative max/min so far, in row order.""" + _need_col(rows, of, "of") + out = out or f"{of}_running_{kind}" + best, res = None, [] + for r in rows: + v = r.get(of) + if _numeric(v): + best = v if best is None else (max(best, v) if kind == "max" else min(best, v)) + res.append({**r, out: best}) + return res + + +def moving_average(rows, of, window=3, out=None): + """Add `_ma`: trailing moving average in row order; the first window-1 rows get + None (a partial-window average reads as a level change and lies). A smoothing — any numeric.""" + _need_col(rows, of, "of") + window = max(2, min(int(window or 3), 24)) + out = out or f"{of}_ma{window}" + vals = [r.get(of) or 0 for r in rows] + res = [] + for i, r in enumerate(rows): + ma = sum(vals[i - window + 1:i + 1]) / window if i >= window - 1 else None + res.append({**r, out: ma}) + return res + + +def moving_sum(rows, of, window=3, out=None): + """Add `_msum`: trailing moving SUM (additive measures only) — the first window-1 + rows get None.""" + _need_additive(rows, of, "moving_sum") + window = max(2, min(int(window or 3), 24)) + out = out or f"{of}_msum{window}" + vals = [r.get(of) or 0 for r in rows] + res = [] + for i, r in enumerate(rows): + ms = sum(vals[i - window + 1:i + 1]) if i >= window - 1 else None + res.append({**r, out: ms}) + return res + + +def moving_median(rows, of, window=3, out=None): + """Add `_mmed`: trailing moving MEDIAN (robust smoothing; outlier-resistant).""" + _need_col(rows, of, "of") + window = max(2, min(int(window or 3), 24)) + out = out or f"{of}_mmed{window}" + res = [] + for i, r in enumerate(rows): + if i >= window - 1: + win = [rows[j].get(of) for j in range(i - window + 1, i + 1) if _numeric(rows[j].get(of))] + res.append({**r, out: _median(win) if win else None}) + else: + res.append({**r, out: None}) + return res + + +def rolling_std(rows, of, window=3, out=None): + """Add `_rstd`: trailing-window SAMPLE standard deviation — demand/sales + VOLATILITY over time (feeds control-limit / safety-stock work). First window-1 rows None.""" + _need_col(rows, of, "of") + window = max(2, min(int(window or 3), 24)) + out = out or f"{of}_rstd{window}" + res = [] + for i, r in enumerate(rows): + if i >= window - 1: + win = [rows[j].get(of) for j in range(i - window + 1, i + 1) if _numeric(rows[j].get(of))] + res.append({**r, out: _std(win)}) + else: + res.append({**r, out: None}) + return res + + +def running_count(rows, out="running_count"): + """Add `running_count`: cumulative 1-based ROW count in the current order (cumulative # of + orders/SKUs/whatever the rows are). Counts rows — always safe (never a distinct-measure sum).""" + return [{**r, out: i + 1} for i, r in enumerate(rows)] + + +# ================================================================== period-over-period (row-wise) + +def diff(rows, of, out=None): + """Add `_diff`: value minus the previous row's value (first row None). Row-wise — safe on + any numeric column (period-over-period change, incl. of a ratio).""" + _need_col(rows, of, "of") + out = out or f"{of}_diff" + res, prev = [], None + for r in rows: + v = r.get(of) + res.append({**r, out: (v - prev) if _numeric(v) and _numeric(prev) else None}) + prev = v + return res + + +def pct_change(rows, of, out=None): + """Add `_pct_change` (0-100 signed): percent change from the previous row. prev=0 -> None.""" + _need_col(rows, of, "of") + out = out or f"{of}_pct_change" + res, prev = [], None + for r in rows: + v = r.get(of) + pc = (100.0 * (v - prev) / abs(prev)) if _numeric(v) and _numeric(prev) and prev else None + res.append({**r, out: pc}) + prev = v + return res + + +def lag(rows, of, k=1, out=None): + """Add `_lag`: the value from k rows earlier (Tableau LOOKUP / pandas shift).""" + _need_col(rows, of, "of") + k = max(1, min(int(k or 1), 100)) + out = out or f"{of}_lag{k}" + vals = [r.get(of) for r in rows] + return [{**r, out: (vals[i - k] if i - k >= 0 else None)} for i, r in enumerate(rows)] + + +def lead(rows, of, k=1, out=None): + """Add `_lead`: the value from k rows later.""" + _need_col(rows, of, "of") + k = max(1, min(int(k or 1), 100)) + out = out or f"{of}_lead{k}" + vals = [r.get(of) for r in rows] + n = len(rows) + return [{**r, out: (vals[i + k] if i + k < n else None)} for i, r in enumerate(rows)] + + +def diff_from_first(rows, of, out=None): + """Add `_vs_first`: value minus the FIRST row's value (Tableau 'difference from first').""" + _need_col(rows, of, "of") + out = out or f"{of}_vs_first" + base = next((r.get(of) for r in rows if _numeric(r.get(of))), None) + return [{**r, out: (r.get(of) - base) if _numeric(r.get(of)) and _numeric(base) else None} + for r in rows] + + +def index_to_100(rows, of, out=None): + """Add `_idx`: rebase the series to 100 at the first value (index-to-100 — compare shapes + of series at different levels). first=0 -> None.""" + _need_col(rows, of, "of") + out = out or f"{of}_idx" + base = next((r.get(of) for r in rows if _numeric(r.get(of))), None) + return [{**r, out: (100.0 * r.get(of) / base) if _numeric(r.get(of)) and base else None} + for r in rows] + + +def percent_of_max(rows, of, out=None): + """Add `_pct_of_max` (0-100): each row as a % of the largest value ('how far below best').""" + _need_col(rows, of, "of") + out = out or f"{of}_pct_of_max" + vals = _vals(rows, of) + mx = max(vals) if vals else None + return [{**r, out: (100.0 * r.get(of) / mx) if _numeric(r.get(of)) and mx else None} + for r in rows] + + +def compare(rows, a, b, how="diff", out=None): + """Column-wise comparison of two EXISTING columns per row (e.g. revenue vs revenue_ly already + in the result): how='diff' (a-b), 'pct' (100*(a-b)/|b|), or 'ratio' (a/b). Distinct from `diff` + (which is cross-row on ONE column). Safe on any columns; b=0 -> None for pct/ratio.""" + _need_col(rows, a, "a") + _need_col(rows, b, "b") + if how not in ("diff", "pct", "ratio"): + raise SEM.ModelError("how must be diff|pct|ratio") + out = out or f"{a}_vs_{b}_{how}" + + def _cmp(x, y): + if not (_numeric(x) and _numeric(y)): + return None + if how == "diff": + return x - y + if not y: + return None + return (100.0 * (x - y) / abs(y)) if how == "pct" else (x / y) + return [{**r, out: _cmp(r.get(a), r.get(b))} for r in rows] + + +# ================================================================== distribution / stats + +def bin_values(rows, of, bins=10): + """Equal-width histogram buckets over `of` across these rows -> one row per bucket: + {bucket, bucket_lo, bucket_hi, count, _sum}. Empty buckets kept (an honest gap).""" + _need_col(rows, of, "of") + bins = max(2, min(int(bins or 10), 50)) + vals = _vals(rows, of) + if not vals: + raise SEM.ModelError(f"no numeric values in {of!r} to bin") + lo, hi = min(vals), max(vals) + if lo == hi: + return [{"bucket": f"{lo:,.4g}", "bucket_lo": lo, "bucket_hi": hi, + "count": len(vals), f"{of}_sum": sum(vals)}] + width = (hi - lo) / bins + out = [] + for i in range(bins): + b_lo, b_hi = lo + i * width, lo + (i + 1) * width + hit = [v for v in vals if (b_lo <= v < b_hi) or (i == bins - 1 and v == hi)] + out.append({"bucket": f"{b_lo:,.4g} to {b_hi:,.4g}", "bucket_lo": b_lo, "bucket_hi": b_hi, + "count": len(hit), f"{of}_sum": sum(hit)}) + return out + + +def describe(rows, of): + """Summary statistics of `of` -> ONE row: count, mean, median, std, min, p25, p75, max + (+ sum when the measure is additive). The pandas .describe() of a column.""" + _need_col(rows, of, "of") + v = _vals(rows, of) + if not v: + raise SEM.ModelError(f"no numeric values in {of!r} to describe") + row = {"stat_of": of, "count": len(v), "mean": _mean(v), "median": _median(v), + "std": _std(v), "min": min(v), "p25": _percentile(v, 25), + "p75": _percentile(v, 75), "max": max(v)} + if _is_additive(of): + row["sum"] = sum(v) + return [row] + + +def zscore(rows, of, out=None): + """Add `_zscore`: standardized value (v-mean)/std (sample std). Constant column -> None.""" + _need_col(rows, of, "of") + out = out or f"{of}_zscore" + v = _vals(rows, of) + m, s = _mean(v), _std(v) + return [{**r, out: ((r.get(of) - m) / s if _numeric(r.get(of)) and s else None)} for r in rows] + + +def outliers(rows, of, method="zscore", k=None, out=None): + """Add `_outlier` (bool): flag statistical outliers. method='zscore' (|z|>k, default 3) or + 'iqr' (outside [Q1-k*IQR, Q3+k*IQR], default k=1.5 — Tukey's fences).""" + _need_col(rows, of, "of") + out = out or f"{of}_outlier" + v = _vals(rows, of) + if method == "iqr": + k = 1.5 if k is None else float(k) + q1, q3 = _percentile(v, 25), _percentile(v, 75) + iqr = (q3 - q1) if (q1 is not None and q3 is not None) else None + lo = (q1 - k * iqr) if iqr is not None else None + hi = (q3 + k * iqr) if iqr is not None else None + return [{**r, out: (bool(r.get(of) < lo or r.get(of) > hi) + if _numeric(r.get(of)) and lo is not None else None)} for r in rows] + k = 3.0 if k is None else float(k) + m, s = _mean(v), _std(v) + return [{**r, out: (abs((r.get(of) - m) / s) > k if _numeric(r.get(of)) and s else None)} + for r in rows] + + +def winsorize(rows, of, p=5, out=None): + """Add `_winsor`: `of` clipped to its [p, 100-p] percentiles (tame outliers before a mean + or chart without dropping rows).""" + _need_col(rows, of, "of") + p = max(0.0, min(float(p or 5), 49.0)) + out = out or f"{of}_winsor" + v = _vals(rows, of) + lo, hi = _percentile(v, p), _percentile(v, 100 - p) + return [{**r, out: (min(max(r.get(of), lo), hi) if _numeric(r.get(of)) else None)} for r in rows] + + +def clip(rows, of, lo=None, hi=None, out=None): + """Add `_clip`: `of` clamped to [lo, hi] (either bound optional).""" + _need_col(rows, of, "of") + if lo is None and hi is None: + raise SEM.ModelError("clip needs lo and/or hi") + out = out or f"{of}_clip" + + def _c(x): + if not _numeric(x): + return None + if lo is not None: + x = max(x, lo) + if hi is not None: + x = min(x, hi) + return x + return [{**r, out: _c(r.get(of))} for r in rows] + + +def normalize(rows, of, out=None): + """Add `_norm` (0-1): min-max scale of `of`. Constant column -> 0.0 for all.""" + _need_col(rows, of, "of") + out = out or f"{of}_norm" + v = _vals(rows, of) + lo, hi = (min(v), max(v)) if v else (None, None) + span = (hi - lo) if (lo is not None) else None + return [{**r, out: ((r.get(of) - lo) / span if span else 0.0) if _numeric(r.get(of)) else None} + for r in rows] + + +def correlate(rows, x, y): + """Pearson correlation between two columns -> ONE row {x, y, pearson_r, n}. r in [-1,1]; + 'do these two measures move together?'. Zero-variance column -> r None.""" + _need_col(rows, x, "x") + _need_col(rows, y, "y") + pairs = [(r[x], r[y]) for r in rows if _numeric(r.get(x)) and _numeric(r.get(y))] + n = len(pairs) + if n < 2: + return [{"x": x, "y": y, "pearson_r": None, "n": n}] + xs, ys = [p[0] for p in pairs], [p[1] for p in pairs] + mx, my = _mean(xs), _mean(ys) + cov = sum((a - mx) * (b - my) for a, b in pairs) + sx = math.sqrt(sum((a - mx) ** 2 for a in xs)) + sy = math.sqrt(sum((b - my) ** 2 for b in ys)) + r = (cov / (sx * sy)) if (sx and sy) else None + return [{"x": x, "y": y, "pearson_r": r, "n": n}] + + +def weighted_average(rows, of, weight): + """Weighted mean of `of` by `weight` -> ONE row {_wavg, weight_col, n}. The correct way to + average a per-unit figure (e.g. price weighted by units) — never a mean of means.""" + _need_col(rows, of, "of") + _need_col(rows, weight, "weight") + num = sum((r[of] * r[weight]) for r in rows if _numeric(r.get(of)) and _numeric(r.get(weight))) + den = sum(r[weight] for r in rows if _numeric(r.get(of)) and _numeric(r.get(weight))) + return [{f"{of}_wavg": (num / den) if den else None, "weight_col": weight, + "n": sum(1 for r in rows if _numeric(r.get(of)) and _numeric(r.get(weight)))}] + + +def safe_ratio(rows, numerator, denominator, out="ratio"): + """Add a row-wise ratio `numerator/denominator` (denominator 0 -> None). Build an ad-hoc rate + the semantic layer doesn't predefine, correctly guarded.""" + _need_col(rows, numerator, "numerator") + _need_col(rows, denominator, "denominator") + return [{**r, out: ((r[numerator] / r[denominator]) + if _numeric(r.get(numerator)) and r.get(denominator) else None)} + for r in rows] + + +def product(rows, a, b, out="product"): + """Add a row-wise product `a*b` (e.g. price * quantity).""" + _need_col(rows, a, "a") + _need_col(rows, b, "b") + return [{**r, out: (r[a] * r[b] if _numeric(r.get(a)) and _numeric(r.get(b)) else None)} + for r in rows] + + +# ================================================================== business templates + +def abc_classify(rows, of, a=80, b=95): + """Pareto ABC classification: sort desc by `of`, add `_cum_pct` and `abc_class` (A = the + vital few up to a% of the total, B up to b%, C the long tail). The classic 80/20 inventory / + customer / SKU segmentation.""" + _need_additive(rows, of, "abc_classify") + a, b = float(a), float(b) + ordered = sort_rows(rows, of, "desc") + total = sum(r.get(of) or 0 for r in ordered) + run, res = 0.0, [] + for r in ordered: + run += r.get(of) or 0 + cum = (100.0 * run / total) if total else None + cls = None if cum is None else ("A" if cum <= a else ("B" if cum <= b else "C")) + res.append({**r, f"{of}_cum_pct": cum, "abc_class": cls}) + return res + + +def concentration(rows, of): + """Concentration statistics of `of` across these rows -> ONE row: HHI (Herfindahl-Hirschman + Index, 0-10000), Gini (0-1), and top-1/5/10 share %. 'How concentrated is the book?'.""" + _need_additive(rows, of, "concentration") + vals = [r.get(of) or 0 for r in rows if _numeric(r.get(of))] + vals = [v for v in vals if v > 0] + n = len(vals) + total = sum(vals) + if not total: + return [{"of": of, "n": n, "hhi": None, "gini": None, + "top1_share_pct": None, "top5_share_pct": None, "top10_share_pct": None}] + shares = [v / total for v in vals] + hhi = sum(s * s for s in shares) * 10000.0 + asc = sorted(vals) + gini = (2.0 * sum((i + 1) * x for i, x in enumerate(asc))) / (n * total) - (n + 1.0) / n + desc = sorted(vals, reverse=True) + + def topk(k): + return 100.0 * sum(desc[:k]) / total + return [{"of": of, "n": n, "hhi": hhi, "gini": gini, "top1_share_pct": topk(1), + "top5_share_pct": topk(5), "top10_share_pct": topk(10)}] + + +def contribution_to_change(rows, of): + """Given a `_delta` column (run yoy first), add `_delta_share_pct`: each row's share of + the TOTAL change (who drove the movement — the bridge / contribution decomposition).""" + delta = f"{of}_delta" + _need_col(rows, delta, "of (expected _delta from yoy)") + total = sum(r.get(delta) or 0 for r in rows) + return [{**r, f"{of}_delta_share_pct": (100.0 * (r.get(delta) or 0) / total) if total else None} + for r in rows] + + +def rfm(rows, recency, frequency, monetary, tiles=5): + """RFM scoring: quintile-score each customer on Recency (LOWER days = better), Frequency and + Monetary (higher = better), 1..tiles. Adds r_score/f_score/m_score, rfm_cell ('545'), + rfm_score (sum) and rfm_segment (Champions / Loyal / Potential / At Risk / Lost / Others).""" + for c, nm in ((recency, "recency"), (frequency, "frequency"), (monetary, "monetary")): + _need_col(rows, c, nm) + tiles = max(2, min(int(tiles or 5), 10)) + + def _score(col, reverse): + # reverse=True -> smaller value scores higher (recency). Bucket by sorted position. + ordered = sort_rows(rows, col, "asc") + present = [r for r in ordered if r.get(col) is not None] + n = len(present) + sc = {} + for i, r in enumerate(ordered): + if r.get(col) is None: + sc[id(r)] = None + else: + t = min(tiles, int(i * tiles / n) + 1) + sc[id(r)] = (tiles + 1 - t) if reverse else t + return sc + rs, fs, ms = _score(recency, True), _score(frequency, False), _score(monetary, False) + out = [] + for r in rows: + rr, ff, mm = rs[id(r)], fs[id(r)], ms[id(r)] + seg = _rfm_segment(rr, ff, mm, tiles) + out.append({**r, "r_score": rr, "f_score": ff, "m_score": mm, + "rfm_cell": (f"{rr}{ff}{mm}" if None not in (rr, ff, mm) else None), + "rfm_score": (rr + ff + mm if None not in (rr, ff, mm) else None), + "rfm_segment": seg}) + return out + + +def _rfm_segment(r, f, m, tiles): + if None in (r, f, m): + return None + hi = tiles - 1 + lo = 2 + if r >= hi and f >= hi: + return "Champions" + if f >= hi: + return "Loyal" + if r >= hi: + return "Recent / Promising" + if r <= lo and f >= 3: + return "At Risk" + if r <= lo and f <= lo: + return "Lost" + return "Others" + + +def funnel_rates(rows, of): + """Stage conversion: over ordered stage rows carrying a count `of`, add `_step_pct` (vs the + previous stage) and `_overall_pct` (vs the first stage). The funnel drop-off read.""" + _need_col(rows, of, "of") + first = next((r.get(of) for r in rows if _numeric(r.get(of))), None) + res, prev = [], None + for r in rows: + v = r.get(of) + step = (100.0 * v / prev) if _numeric(v) and _numeric(prev) and prev else None + overall = (100.0 * v / first) if _numeric(v) and first else None + res.append({**r, f"{of}_step_pct": step, f"{of}_overall_pct": overall}) + prev = v + return res + + +# ================================================================== modeling (transparent) + +def _ols(xs, ys): + """Ordinary least squares y = slope*x + intercept over paired numerics -> (slope, intercept, + r2). None,None,None if degenerate.""" + pts = [(x, y) for x, y in zip(xs, ys) if _numeric(x) and _numeric(y)] + n = len(pts) + if n < 2: + return None, None, None + mx = sum(p[0] for p in pts) / n + my = sum(p[1] for p in pts) / n + sxx = sum((p[0] - mx) ** 2 for p in pts) + sxy = sum((p[0] - mx) * (p[1] - my) for p in pts) + if not sxx: + return None, None, None + slope = sxy / sxx + intercept = my - slope * mx + syy = sum((p[1] - my) ** 2 for p in pts) + ss_res = sum((y - (slope * x + intercept)) ** 2 for x, y in pts) + r2 = (1 - ss_res / syy) if syy else None + return slope, intercept, r2 + + +def trend_line(rows, of, out=None): + """Add `_trend`: the linear least-squares fitted value (a straight trend over the rows' + order) — draw it over `of` with a combo/line chart. The Analytics-pane trend line.""" + _need_col(rows, of, "of") + out = out or f"{of}_trend" + xs = list(range(len(rows))) + ys = [r.get(of) for r in rows] + slope, intercept, _ = _ols(xs, ys) + if slope is None: + return [{**r, out: None} for r in rows] + return [{**r, out: slope * i + intercept} for i, r in enumerate(rows)] + + +def regression(rows, x, y): + """Linear regression of `y` on `x` -> ONE row {slope, intercept, r2, n}. 'Is there a + relationship, how strong?' (r2 near 1 = tight fit).""" + _need_col(rows, x, "x") + _need_col(rows, y, "y") + slope, intercept, r2 = _ols([r.get(x) for r in rows], [r.get(y) for r in rows]) + n = sum(1 for r in rows if _numeric(r.get(x)) and _numeric(r.get(y))) + return [{"x": x, "y": y, "slope": slope, "intercept": intercept, "r2": r2, "n": n}] + + +def cagr(rows, of): + """Compound annual (per-period) growth rate first->last -> ONE row {cagr_pct, periods}. + ((last/first)^(1/periods) - 1) * 100. Needs first > 0.""" + _need_col(rows, of, "of") + vals = [r.get(of) for r in rows if _numeric(r.get(of))] + if len(vals) < 2: + return [{"of": of, "cagr_pct": None, "periods": max(0, len(vals) - 1)}] + first, last = vals[0], vals[-1] + p = len(vals) - 1 + c = ((last / first) ** (1.0 / p) - 1) * 100.0 if first > 0 and last > 0 else None + return [{"of": of, "cagr_pct": c, "periods": p}] + + +def growth_rate(rows, of): + """Total growth first->last -> ONE row {growth_pct, first, last}. (last-first)/|first| * 100.""" + _need_col(rows, of, "of") + vals = [r.get(of) for r in rows if _numeric(r.get(of))] + if len(vals) < 2: + return [{"of": of, "growth_pct": None, "first": (vals[0] if vals else None), + "last": (vals[-1] if vals else None)}] + first, last = vals[0], vals[-1] + g = (100.0 * (last - first) / abs(first)) if first else None + return [{"of": of, "growth_pct": g, "first": first, "last": last}] + + +# ================================================================== reference annotations (columns) + +def reference_line(rows, of, stat="mean", out=None): + """Add a constant column = a summary stat of `of` (mean|median|min|max), so a combo/line chart + can draw the reference line. The Analytics-pane average/median/constant line.""" + _need_col(rows, of, "of") + out = out or f"{of}_ref" + v = _vals(rows, of) + val = {"mean": _mean(v), "median": _median(v), "min": (min(v) if v else None), + "max": (max(v) if v else None)}.get(stat) + if stat not in ("mean", "median", "min", "max"): + raise SEM.ModelError("stat must be mean|median|min|max") + return [{**r, out: val} for r in rows] + + +def reference_band(rows, of, method="stddev", k=1): + """Add `_band_lo`/`_band_hi` constant columns for a shaded reference band. method + 'stddev' -> mean ± k·std; method 'percentile' -> the k-th and (100-k)-th percentiles.""" + _need_col(rows, of, "of") + v = _vals(rows, of) + if method == "percentile": + lo, hi = _percentile(v, float(k)), _percentile(v, 100 - float(k)) + elif method == "stddev": + m, s = _mean(v), _std(v) + lo = (m - float(k) * s) if (m is not None and s is not None) else None + hi = (m + float(k) * s) if (m is not None and s is not None) else None + else: + raise SEM.ModelError("method must be stddev|percentile") + return [{**r, f"{of}_band_lo": lo, f"{of}_band_hi": hi} for r in rows] + + +def target_line(rows, value, out="target"): + """Add a constant `target` column = value — for a bullet chart (actual vs target) or a goal + line on a combo chart.""" + if not _numeric(value): + raise SEM.ModelError("target_line needs a numeric value") + return [{**r, out: value} for r in rows] + + +def xmr_limits(rows, of): + """Wheeler XmR (process-behaviour) control limits on `of` in row order, as constant columns: + `_center` (mean), `_ucl`/`_lcl` (center ± 2.66·mR-bar, mR-bar = mean moving range), + and `_signal` (bool: this point is outside the limits). The HONEST, business-native + alternative to mean±k·std for spotting real signals in a noisy monthly series — the same XmR + the warehouse/expenses modules use. Draw center/ucl/lcl as reference lines over `of`.""" + _need_col(rows, of, "of") + vals = [r.get(of) for r in rows] + nums = [v for v in vals if _numeric(v)] + center = _mean(nums) + mr = [abs(vals[i] - vals[i - 1]) for i in range(1, len(vals)) + if _numeric(vals[i]) and _numeric(vals[i - 1])] + mrbar = _mean(mr) + ucl = (center + 2.66 * mrbar) if (center is not None and mrbar is not None) else None + lcl = (center - 2.66 * mrbar) if (center is not None and mrbar is not None) else None + out = [] + for r in rows: + v = r.get(of) + sig = (bool(v > ucl or v < lcl) if _numeric(v) and ucl is not None else None) + out.append({**r, f"{of}_center": center, f"{of}_ucl": ucl, f"{of}_lcl": lcl, + f"{of}_signal": sig}) + return out + + +# ================================================================== reshape (pure, no hidden agg) + +def pivot(rows, index, column, value): + """Long -> wide: one row per `index`, one column per distinct `column` value, cell = `value`. + A PURE reshape — if two source rows share an (index, column) pair it RAISES (never a hidden + sum; regroup in the query instead). Missing cells are None.""" + for c, nm in ((index, "index"), (column, "column"), (value, "value")): + _need_col(rows, c, nm) + cols, order, out_map = set(), [], {} + for r in rows: + idx, col = r.get(index), r.get(column) + cols.add(col) + if idx not in out_map: + out_map[idx] = {index: idx} + order.append(idx) + cell = str(col) + if cell in out_map[idx]: + raise SEM.ModelError( + f"pivot collision: {index}={idx!r} has two rows for {column}={col!r} — pivot cannot " + f"aggregate (that would hide a sum); regroup the query so (index, column) is unique") + out_map[idx][cell] = r.get(value) + col_names = [str(c) for c in sorted(cols, key=lambda z: (z is None, z))] + return [{index: out_map[idx][index], **{cn: out_map[idx].get(cn) for cn in col_names}} + for idx in order] + + +def unpivot(rows, keep, columns, var_name="metric", value_name="value"): + """Wide -> long (melt): for each row, emit one output row per `columns` entry carrying the + `keep` columns plus (`var_name`, `value_name`). The inverse of pivot.""" + keep = keep if isinstance(keep, (list, tuple)) else [keep] + columns = columns if isinstance(columns, (list, tuple)) else [columns] + for c in list(keep) + list(columns): + _need_col(rows, c, "column") + out = [] + for r in rows: + base = {k: r.get(k) for k in keep} + for c in columns: + out.append({**base, var_name: c, value_name: r.get(c)}) + return out + + +def filter_rows(rows, col, cmp, value): + """Keep rows where `col` `cmp` `value`. cmp in >, >=, <, <=, ==, !=, contains. The 'having' + clause over a result (e.g. keep customers with revenue_yoy_pct < 0). (Named `cmp`, not `op`, + to avoid colliding with the transform dispatch key.)""" + _need_col(rows, col, "col") + ops = {">": lambda a, b: a > b, ">=": lambda a, b: a >= b, "<": lambda a, b: a < b, + "<=": lambda a, b: a <= b, "==": lambda a, b: a == b, "!=": lambda a, b: a != b, + "contains": lambda a, b: str(b).lower() in str(a).lower()} + if cmp not in ops: + raise SEM.ModelError(f"cmp must be one of {sorted(ops)}") + fn = ops[cmp] + out = [] + for r in rows: + v = r.get(col) + try: + if cmp in (">", ">=", "<", "<=") and not _numeric(v): + continue + if fn(v, value): + out.append(r) + except TypeError: + continue + return out + + +def dedupe(rows, by=None): + """Keep the FIRST row per distinct key. by = a column or list of columns (default: the whole + row). Distinct rows, order-preserving.""" + if by is None: + keys = list(rows[0]) if rows else [] + else: + keys = by if isinstance(by, (list, tuple)) else [by] + for k in keys: + _need_col(rows, k, "by") + seen, out = set(), [] + for r in rows: + key = tuple(r.get(k) for k in keys) + if key not in seen: + seen.add(key) + out.append(r) + return out + + +def resample(rows, grain): + """Reindex a time series to a COMPLETE period spine (grain=month|week|day) from the first to + the last period, inserting a row for every MISSING period with None measures (never + interpolated). Run this BEFORE running_total / moving_* / diff on a sparse or filtered date + spine — otherwise those window ops silently skip the gaps and lie. The single most-flagged + correctness guard in the tool surveys (asfreq/reindex).""" + if grain not in ("month", "week", "day"): + raise SEM.ModelError("resample grain must be month|week|day") + _need_col(rows, "period", "period") + present = {str(r.get("period"))[:10]: r for r in rows if r.get("period") is not None} + if not present: + return rows + other_cols = [c for c in rows[0] if c != "period"] + keys = sorted(present) + p = keys[0] + last = keys[-1] + out, guard = [], 0 + while True: + norm = _period_start(p, grain) if grain == "month" else str(p)[:10] + if norm in present: + out.append(present[norm]) + else: + out.append({"period": norm, **{c: None for c in other_cols}}) + if norm >= last: + break + p = _step_period(p, grain, 1) + guard += 1 + if guard > 5000: # bounded (loop-library discipline) + break + return out + + +# ================================================================== re-query family (widening window) + +def _period_start(period, grain): + if grain == "month": + return f"{str(period)[:7]}-01" + return str(period)[:10] + + +def _period_end(period, grain): + p = str(period) + if grain == "month": + y, m = int(p[:4]), int(p[5:7]) + ny, nm = (y + 1, 1) if m == 12 else (y, m + 1) + return (dt.date(ny, nm, 1) - dt.timedelta(days=1)).isoformat() + if grain == "week": + return (dt.date.fromisoformat(p[:10]) + dt.timedelta(days=6)).isoformat() + return p[:10] + + +def _step_period(period, grain, n): + p = str(period) + if grain == "month": + idx = int(p[:4]) * 12 + (int(p[5:7]) - 1) + n + return f"{idx // 12:04d}-{idx % 12 + 1:02d}-01" + if grain == "week": + return (dt.date.fromisoformat(p[:10]) + dt.timedelta(weeks=n)).isoformat() + return (dt.date.fromisoformat(p[:10]) + dt.timedelta(days=n)).isoformat() + + +def _require_series(res, op): + grain = res.get("grain") + if not grain: + raise SEM.ModelError(f"{op} needs a time grain (run the query with grain=month|week|day)") + if res.get("group_by"): + raise SEM.ModelError(f"{op} works on a SINGLE time series — drop group_by (per-group " + "widening windows need one query per group)") + return grain + + +def _scalar(result, col): + rows = result.get("rows") or [] + return rows[0].get(col) if rows else None + + +def ytd(res, run_query, of): + """CORRECT cumulative year-to-date for ANY measure (incl. distinct counts / ratios): re-runs + the governed query over [Jan 1 .. each period end] and reads `of`. Never sums displayed values + (which would over-count a distinct customer count). Adds `_ytd`.""" + rows = res.get("rows") or [] + grain = _require_series(res, "ytd") + _need_col(rows, "period", "period") + _need_col(rows, of, "of") + q = dict(res.get("query") or {}) + ordered = sorted(rows, key=lambda r: str(r.get("period") or "")) + out = [] + for r in ordered: + p = r["period"] + wq = {**q, "grain": None, "group_by": None, + "date_from": f"{str(p)[:4]}-01-01", "date_to": _period_end(p, grain)} + out.append({**r, f"{of}_ytd": _scalar(run_query(wq), of)}) + return out + + +def rolling(res, run_query, of, window=3): + """CORRECT trailing-window value for ANY measure (incl. distinct counts): re-runs the governed + query over the trailing `window` periods and reads `of` (trailing-N distinct customers, T12M + revenue, ...). Adds `_roll`.""" + rows = res.get("rows") or [] + grain = _require_series(res, "rolling") + window = max(2, min(int(window or 3), 36)) + _need_col(rows, "period", "period") + _need_col(rows, of, "of") + q = dict(res.get("query") or {}) + ordered = sorted(rows, key=lambda r: str(r.get("period") or "")) + out = [] + for r in ordered: + p = r["period"] + start = _period_start(_step_period(p, grain, -(window - 1)), grain) + wq = {**q, "grain": None, "group_by": None, + "date_from": start, "date_to": _period_end(p, grain)} + out.append({**r, f"{of}_roll{window}": _scalar(run_query(wq), of)}) + return out + + +def forecast(res, run_query, of, periods=3, method="linear", season=12): + """Append `periods` future rows with a `_forecast` value. method='linear' extends the + least-squares trend; 'seasonal_naive' repeats the value from `season` periods ago. Transparent + and deterministic (no ML lib). Needs a time grain (to label future periods).""" + rows = res.get("rows") or [] + grain = _require_series(res, "forecast") + periods = max(1, min(int(periods or 3), 24)) + _need_col(rows, "period", "period") + _need_col(rows, of, "of") + ordered = sorted(rows, key=lambda r: str(r.get("period") or "")) + hist = [r.get(of) for r in ordered] + out = [{**r, f"{of}_forecast": None} for r in ordered] + if out: # bridge: last actual seeds the line + out[-1][f"{of}_forecast"] = ordered[-1].get(of) + last_p = ordered[-1]["period"] if ordered else None + if method == "seasonal_naive": + season = max(1, int(season or 12)) + for i in range(1, periods + 1): + src = len(hist) - season + (i - 1) + val = hist[src] if 0 <= src < len(hist) else None + out.append({"period": _step_period(last_p, grain, i), f"{of}_forecast": val}) + else: + slope, intercept, _ = _ols(list(range(len(hist))), hist) + for i in range(1, periods + 1): + val = (slope * (len(hist) - 1 + i) + intercept) if slope is not None else None + out.append({"period": _step_period(last_p, grain, i), f"{of}_forecast": val}) + return out + + +# ================================================================== yoy (re-query) + +def _shift_year(iso, delta): + y, rest = iso[:4], iso[4:] + shifted = f"{int(y) + delta}{rest}" + if shifted.endswith("-02-29"): # leap-day clamp + shifted = shifted[:-2] + "28" + return shifted + + +def yoy_compare(res, run_query): + """Same-period-last-year compare: re-run the source GOVERNED query shifted -1 year and join + on (period shifted +1y, *group dims). Adds `_ly`, `_delta` ($ change) and `_yoy_pct` + (0-100) per measure — sort by `_delta` asc for 'who dropped the most'. Unmatched periods + keep None — a partial-vs-full compare is never faked.""" + q = dict(res.get("query") or {}) + if not (q.get("date_from") and q.get("date_to")): + raise SEM.ModelError("yoy needs an explicit date_from/date_to on the source query") + ly_q = {**q, "date_from": _shift_year(q["date_from"], -1), + "date_to": _shift_year(q["date_to"], -1)} + ly_rows = run_query(ly_q)["rows"] + gb = list(res.get("group_by") or []) + period_col = "period" if res.get("grain") else None + keys = ([period_col] if period_col else []) + gb + measures = [m for m in (res.get("measures") or []) if res["rows"] and m in res["rows"][0]] + + def _key(row, shift_period): + parts = [] + for k in keys: + v = str(row.get(k) or "") + if k == period_col and shift_period and len(v) >= 4: + v = _shift_year(v, +1) + parts.append(v) + return tuple(parts) + + ly_map = {} + for r in ly_rows: + ly_map[_key(r, True)] = r + out = [] + for r in res["rows"]: + prev = ly_map.get(_key(r, False), {}) + row = dict(r) + for m in measures: + pv = prev.get(m) + row[f"{m}_ly"] = pv + cur = r.get(m) + row[f"{m}_delta"] = (cur - pv) if _numeric(pv) and _numeric(cur) else None + row[f"{m}_yoy_pct"] = (100.0 * (cur - pv) / abs(pv) + if _numeric(pv) and pv and _numeric(cur) else None) + out.append(row) + return out, ly_q + + +def _yoy(res, run_query): + out, _ = yoy_compare(res, run_query) + return out + + +# ================================================================== registry + replay + +def _op(fn, params=(), required=(), needs_query=False): + return {"fn": fn, "params": set(params), "required": set(required), "needs_query": needs_query} + + +OPS = { + # ordering / rank / top-N + "sort": _op(sort_rows, {"by", "direction"}, {"by"}), + "head": _op(head, {"n"}), + "bottom_n": _op(bottom_n, {"by", "n"}, {"by"}), + "rank": _op(rank_rows, {"by", "direction", "out"}, {"by"}), + "rank_pct": _op(rank_pct, {"by", "direction", "out"}, {"by"}), + "ntile": _op(ntile, {"by", "tiles", "direction", "out"}, {"by"}), + "top_n": _op(top_n, {"by", "n", "other", "other_label"}, {"by"}), + "add_total": _op(add_total, {"label"}), + # part-to-whole / cumulative + "share_of_total": _op(share_of_total, {"of", "out"}, {"of"}), + "cum_share": _op(cum_share, {"of", "out"}, {"of"}), + # running / moving windows + "running_total": _op(running_total, {"of", "out"}, {"of"}), + "running_avg": _op(running_avg, {"of", "out"}, {"of"}), + "running_max": _op(lambda rows, of, out=None: running_extreme(rows, of, "max", out), + {"of", "out"}, {"of"}), + "running_min": _op(lambda rows, of, out=None: running_extreme(rows, of, "min", out), + {"of", "out"}, {"of"}), + "moving_average": _op(moving_average, {"of", "window", "out"}, {"of"}), + "moving_sum": _op(moving_sum, {"of", "window", "out"}, {"of"}), + "moving_median": _op(moving_median, {"of", "window", "out"}, {"of"}), + "rolling_std": _op(rolling_std, {"of", "window", "out"}, {"of"}), + "running_count": _op(running_count, {"out"}), + # period-over-period (row-wise) + "diff": _op(diff, {"of", "out"}, {"of"}), + "pct_change": _op(pct_change, {"of", "out"}, {"of"}), + "lag": _op(lag, {"of", "k", "out"}, {"of"}), + "lead": _op(lead, {"of", "k", "out"}, {"of"}), + "diff_from_first": _op(diff_from_first, {"of", "out"}, {"of"}), + "index_to_100": _op(index_to_100, {"of", "out"}, {"of"}), + "percent_of_max": _op(percent_of_max, {"of", "out"}, {"of"}), + "compare": _op(compare, {"a", "b", "how", "out"}, {"a", "b"}), + # distribution / stats + "bin": _op(bin_values, {"of", "bins"}, {"of"}), + "describe": _op(describe, {"of"}, {"of"}), + "zscore": _op(zscore, {"of", "out"}, {"of"}), + "outliers": _op(outliers, {"of", "method", "k", "out"}, {"of"}), + "winsorize": _op(winsorize, {"of", "p", "out"}, {"of"}), + "clip": _op(clip, {"of", "lo", "hi", "out"}, {"of"}), + "normalize": _op(normalize, {"of", "out"}, {"of"}), + "correlate": _op(correlate, {"x", "y"}, {"x", "y"}), + "weighted_average": _op(weighted_average, {"of", "weight"}, {"of", "weight"}), + "safe_ratio": _op(safe_ratio, {"numerator", "denominator", "out"}, {"numerator", "denominator"}), + "product": _op(product, {"a", "b", "out"}, {"a", "b"}), + # business templates + "abc_classify": _op(abc_classify, {"of", "a", "b"}, {"of"}), + "concentration": _op(concentration, {"of"}, {"of"}), + "contribution_to_change": _op(contribution_to_change, {"of"}, {"of"}), + "rfm": _op(rfm, {"recency", "frequency", "monetary", "tiles"}, + {"recency", "frequency", "monetary"}), + "funnel_rates": _op(funnel_rates, {"of"}, {"of"}), + # modeling (transparent) + "trend_line": _op(trend_line, {"of", "out"}, {"of"}), + "regression": _op(regression, {"x", "y"}, {"x", "y"}), + "cagr": _op(cagr, {"of"}, {"of"}), + "growth_rate": _op(growth_rate, {"of"}, {"of"}), + # reference annotations (added columns) + "reference_line": _op(reference_line, {"of", "stat", "out"}, {"of"}), + "reference_band": _op(reference_band, {"of", "method", "k"}, {"of"}), + "target_line": _op(target_line, {"value", "out"}, {"value"}), + "xmr_limits": _op(xmr_limits, {"of"}, {"of"}), + # reshape (pure) + "pivot": _op(pivot, {"index", "column", "value"}, {"index", "column", "value"}), + "unpivot": _op(unpivot, {"keep", "columns", "var_name", "value_name"}, {"keep", "columns"}), + "filter_rows": _op(filter_rows, {"col", "cmp", "value"}, {"col", "cmp", "value"}), + "dedupe": _op(dedupe, {"by"}), + "resample": _op(resample, {"grain"}, {"grain"}), + # re-query family (widening / trailing windows, correct for non-additive) + "yoy": _op(_yoy, needs_query=True), + "ytd": _op(ytd, {"of"}, {"of"}, needs_query=True), + "rolling": _op(rolling, {"of", "window"}, {"of"}, needs_query=True), + "forecast": _op(forecast, {"of", "periods", "method", "season"}, {"of"}, needs_query=True), +} + + +def apply(res, ops, run_query=None): + """Apply an op CHAIN to a result dict; returns (rows, applied_ops). Ops are validated against + the registry (whitelisted names + params only). `run_query(query_dict) -> result` powers the + re-query family (yoy/ytd/rolling/forecast); both the live tool and the saved-view replay inject + their own.""" + if not isinstance(ops, list) or not ops: + raise SEM.ModelError("transforms must be a non-empty list of {op, ...} objects") + rows = list(res.get("rows") or []) + applied = [] + for spec in ops: + if not isinstance(spec, dict) or "op" not in spec: + raise SEM.ModelError(f"each transform needs an 'op' key: {spec!r}") + name = spec["op"] + entry = OPS.get(name) + if not entry: + raise SEM.ModelError(f"unknown transform {name!r} (transforms: {sorted(OPS)})") + params = {k: v for k, v in spec.items() if k != "op"} + bad = set(params) - entry["params"] + if bad: + raise SEM.ModelError(f"{name}: unknown params {sorted(bad)} " + f"(allowed: {sorted(entry['params'])})") + missing = entry["required"] - set(params) + if missing: + raise SEM.ModelError(f"{name}: missing required params {sorted(missing)}") + if entry["needs_query"]: + if run_query is None: + raise SEM.ModelError(f"{name} is unavailable here (no query runner)") + # every re-query op takes (res_with_current_rows, run_query, **params) + rows = entry["fn"]({**res, "rows": rows}, run_query, **params) + else: + rows = entry["fn"](rows, **params) + applied.append({"op": name, **params}) + return rows, applied diff --git a/royalimports_os/harness/views.py b/royalimports_os/harness/views.py new file mode 100644 index 0000000000000000000000000000000000000000..e3a983cea85acf83606dd62eb314df082191bb0a --- /dev/null +++ b/royalimports_os/harness/views.py @@ -0,0 +1,198 @@ +"""harness/views.py — saved views + spawned dashboards: the store and the RE-RUNNER (OM-3 viewer). + +The Analyst's save_view/compose_dashboard tools persist chart SPECS + their semantic QUERY (never +rows) into data/store/views.json. This module is the read side: load the specs, re-execute each +view's governed query against the tenant store at render time, and hand the page a fresh artifact +dict in exactly the shape `_render_analyst_artifact` already renders. The viewer therefore shows +LIVE (store-fresh) numbers, not the numbers from whenever the view was saved — same discipline as +every dashboard (a saved view is a query, not a snapshot). + +Legacy note: views saved before 2026-07-11 carry no 'query' (only a session-scoped result_id) and +cannot be re-run — run_view raises a readable error the page surfaces per view. +""" +import json +import time + +import harness.semantic as SEM +from harness.tools import VIEWS_PATH + +_QUERY_KEYS = ("topic", "measures", "group_by", "grain", "date_from", "date_to", + "team_id", "filters", "sort", "limit", "exclude_services") + + +def load(): + if VIEWS_PATH.exists(): + return json.loads(VIEWS_PATH.read_text(encoding="utf-8")) + return {"views": {}, "dashboards": {}} + + +def _save(d): + VIEWS_PATH.parent.mkdir(parents=True, exist_ok=True) + VIEWS_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8") + + +def views(): + return load().get("views", {}) + + +def dashboards(): + return load().get("dashboards", {}) + + +def delete_view(name): + d = load() + d.get("views", {}).pop(name, None) + for dash in d.get("dashboards", {}).values(): + dash["views"] = [v for v in dash.get("views", []) if v != name] + _save(d) + + +def delete_dashboard(name): + d = load() + d.get("dashboards", {}).pop(name, None) + _save(d) + + +# ------------------------------------------------------------------ UPDATE (owner directive +# 2026-07-12: the Analyst can CREATE **and UPDATE** each workbook in the workspace) + +_QUERY_PATCH_KEYS = ("measures", "group_by", "grain", "date_from", "date_to", "team_id", + "filters", "sort", "limit", "exclude_services", "topic") +_SPEC_PATCH_KEYS = ("title", "kind", "x", "y", "series", "metric", + "y2", "size", "value", "facet", "columns", "transforms") + + +def update_view(name, changes): + """Patch a saved view's chart spec and/or query, VALIDATED BY RE-EXECUTION before saving — + an update that cannot run does not land. `changes` may contain spec keys (title/kind/x/y/ + series/metric) and/or a `query` dict of query-key patches (a None value REMOVES the key).""" + d = load() + v = d.get("views", {}).get(name) + if not v: + raise SEM.ModelError(f"unknown view {name!r} (use list_workspace)") + spec = dict(v.get("chart") or {}) + q = dict(spec.get("query") or {}) + if not q: + raise SEM.ModelError(f"view {name!r} carries no query — rebuild it instead") + changes = dict(changes or {}) + qpatch = changes.pop("query", None) or {} + bad = [k for k in changes if k not in _SPEC_PATCH_KEYS] + bad += [k for k in qpatch if k not in _QUERY_PATCH_KEYS] + if bad: + raise SEM.ModelError(f"unknown patch keys {bad} (spec: {list(_SPEC_PATCH_KEYS)}; " + f"query: {list(_QUERY_PATCH_KEYS)})") + for k, val in qpatch.items(): + if val is None: + q.pop(k, None) + else: + q[k] = val + res = _run_query(q) # validation: it must RUN + new_spec = {**spec, **changes, "query": q} + rows = res["rows"] + if new_spec.get("transforms"): # the transform chain must replay too + import harness.transforms as TR + rows, _ = TR.apply({**res, "query": q}, new_spec["transforms"], run_query=_run_query) + cols = set(rows[0]) if rows else set() + for ref in ("x", "y", "series", "y2", "size", "value", "facet"): + if new_spec.get(ref) and cols and new_spec[ref] not in cols: + raise SEM.ModelError(f"{ref}={new_spec[ref]!r} not in the patched result columns " + f"{sorted(cols)} — patch {ref} too") + d["views"][name] = {"chart": new_spec, + "saved_at": time.strftime("%Y-%m-%d %H:%M")} + _save(d) + return {"updated": name, "row_count": res["row_count"], + "query": q, "note": "patched query re-ran successfully before saving"} + + +def update_dashboard(name, views_list=None, new_name=None): + """Update a workbook (dashboard): recompose its views (order = display order) and/or rename + it. Renames follow through to scheduled reports that reference it.""" + d = load() + dash = d.get("dashboards", {}).get(name) + if dash is None: + raise SEM.ModelError(f"unknown dashboard {name!r} (use list_workspace)") + if views_list is not None: + missing = [v for v in views_list if v not in d.get("views", {})] + if missing: + raise SEM.ModelError(f"unknown views {missing} — save_view them first") + dash["views"] = list(views_list) + dash["updated_at"] = time.strftime("%Y-%m-%d %H:%M") + if new_name and new_name != name: + if new_name in d.get("dashboards", {}) or new_name in d.get("views", {}): + raise SEM.ModelError(f"{new_name!r} already exists — pick another name") + d["dashboards"][new_name] = d["dashboards"].pop(name) + try: # follow the rename into schedules + import harness.routines as R + rt = R.load() + if name in rt.get("reports", {}): + rt["reports"][new_name] = rt["reports"].pop(name) + R._save(rt) + except Exception: + pass + name = new_name + _save(d) + return {"dashboard": name, "views": d["dashboards"][name]["views"]} + + +def _run_query(q): + return SEM.store_query(**{k: q.get(k) for k in _QUERY_KEYS if q.get(k) is not None}) + + +def run_view(name): + """Re-execute one saved view's governed query — then REPLAY its recorded transform chain — + and return a fresh artifact dict ({'chart': {..., 'rows': [...]}}, {'table': {...}} or + {'kpi': {...}}) for the house renderer.""" + v = views().get(name) + if not v: + raise SEM.ModelError(f"unknown view {name!r}") + spec = v.get("chart") or {} + q = spec.get("query") + if not q: + raise SEM.ModelError(f"view {name!r} was saved without its query (pre-2026-07-11) — " + "ask the Analyst to rebuild and re-save it") + res = _run_query(q) + if spec.get("kind") == "kpi" or (spec.get("metric") and spec.get("kind") != "table"): + metric = spec.get("metric") or spec.get("y") # KPI-shaped view + val = (res["rows"][0].get(metric) if res["rows"] else None) + kpi = {"metric": metric, "value": val or 0} + cq = spec.get("compare_query") + if cq: + prev_rows = _run_query(cq)["rows"] + prev = prev_rows[0].get(metric) if prev_rows else None + if prev: + kpi["delta_pct"] = ((val or 0) - prev) / abs(prev) + return {"kpi": kpi} + rows = res["rows"] + if spec.get("transforms"): # a saved view replays its transforms + import harness.transforms as TR + rows, _ = TR.apply({**res, "query": q}, spec["transforms"], run_query=_run_query) + if spec.get("kind") == "table": + return {"table": {**{k: spec.get(k) for k in ("kind", "title", "columns")}, + "rows": rows}} + return {"chart": {**{k: spec.get(k) for k in ("kind", "x", "y", "series", "title", + "y2", "size", "value", "facet")}, + "rows": rows}} + + +def run_dashboard(name): + """All of a dashboard's views, freshly re-queried. Returns [(view_name, artifact|None, err)].""" + dash = dashboards().get(name) + if dash is None: + raise SEM.ModelError(f"unknown dashboard {name!r}") + out = [] + for vn in dash.get("views", []): + try: + out.append((vn, run_view(vn), None)) + except Exception as e: + out.append((vn, None, str(e)[:200])) + return out + + +def store_freshness(): + """Newest sync checkpoint across entities — the staleness footer ('data as of …').""" + try: + import harness.datastore as DS + ts = [s.get("updated") for s in DS.status().values() if s.get("updated")] + return max(ts) if ts else None + except Exception: + return None diff --git a/royalimports_os/harness/windows.py b/royalimports_os/harness/windows.py new file mode 100644 index 0000000000000000000000000000000000000000..97bcadca2d8ec6e25e7d24418440719344346602 --- /dev/null +++ b/royalimports_os/harness/windows.py @@ -0,0 +1,390 @@ +"""harness/windows.py — the DATE WINDOW vocabulary (CG-7, owner item 2, 2026-07-26). + +A filter condition on a measure reads: + + Where [Sales] [in the last 90 days] [>] [5,000] + +The middle bracket is this module. It turns a window SPEC — a small JSON object a saved view can +persist — into a concrete `(date_from, date_to)` pair of ISO dates. + +WHY IT IS ITS OWN MODULE, AND WHY IT TAKES `today` AS AN ARGUMENT +----------------------------------------------------------------------------------------------- +Two engines must agree on what "this quarter" means: this one (which compiles the SQL) and +`customer-grid/windows.ts` (which renders the label the user reads and the bounds the client +engine would use). If they disagree by one day, the grid shows a number the label denies, and +nothing errors. `aios-web/verify_windows.py` holds them in lock-step over a fixed set of probe +dates, exactly as verify_filter_engine.py does for the filter tree. + +`today` is a PARAMETER, never `date.today()` read inside. Three reasons, all learned: + - a gate cannot compare two engines on "now" — the two runs are milliseconds apart and land on + different sides of midnight roughly once every few thousand runs, which is the worst kind of + flake because it looks like a real divergence + - a window resolved during a request must not shift between the count query and the row query + - the tenant's day boundary is the tenant's, not the server's + +DESIGN NOTES THAT ARE EASY TO GET WRONG +----------------------------------------------------------------------------------------------- + - Bounds are INCLUSIVE at both ends and are DATES, not timestamps. `store_query` casts both + sides to TIMESTAMP and appends 23:59:59 to the upper bound, so a bare date is correct there; + the client compares the 10-char ISO prefix, so a bare date is correct there too. + - "Last N days" INCLUDES today. Airtable's "the past week" means the last 7 days up to and + including now, not the 7 days before yesterday. Off-by-one here silently drops today's + orders from every "recent" filter — invisible until someone asks why a sale they just + entered is missing. + - Week starts MONDAY (ISO 8601). Stated rather than defaulted: Python's weekday() is + Monday=0 and JavaScript's getDay() is Sunday=0, so the two engines disagree unless one of + them is explicitly corrected. That correction is the single most likely divergence in this + file, and the gate probes a Sunday and a Monday for exactly that reason. + - A `custom` window with only one bound is legal and means open-ended on the other side. + - An UNRESOLVABLE window returns None rather than a default. A window that quietly becomes + "all time" would widen a filter while still reporting an authoritative count. +""" +import datetime as _dt + +#: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts; +#: verify_windows.py asserts the two lists are identical, because a kind the client can emit and +#: the server cannot resolve is a filter that silently stops narrowing. +WINDOW_KINDS = ( + "all_time", + "today", + "yesterday", + "this_week", + "last_week", + "this_month", + "last_month", + "this_quarter", + "last_quarter", + "this_year", + "last_year", + "ytd", + "ytd_last_year", + "ltm", + "past_week", + "past_month", + "past_year", + "last_n_days", + "next_n_days", + "custom", +) + +#: Kinds that carry an integer `n`. +N_KINDS = frozenset({"last_n_days", "next_n_days"}) + +#: How each kind reads in a sentence. The UI renders these; they are here so the label and the +#: arithmetic cannot drift apart in the one place a user would never think to check. +WINDOW_LABELS = { + "all_time": "all time", + "today": "today", + "yesterday": "yesterday", + "this_week": "this week", + "last_week": "last week", + "this_month": "this month", + "last_month": "last month", + "this_quarter": "this quarter", + "last_quarter": "last quarter", + "this_year": "this year", + "last_year": "last year", + "ytd": "year to date", + # "last year to date" (LYTD), not "year to date, last year": the longer phrasing CLIPPED to + # "year to date, last ye" in the 146px window select — and this vocabulary is CLOSED, so + # the rule for it is that it stays readable rather than that it fits. Seen in a live + # screenshot of the owner's own comparison; every assertion in that run was green. + "ytd_last_year": "last year to date", + "ltm": "the last 12 months", + "past_week": "the past week", + "past_month": "the past month", + "past_year": "the past year", + "last_n_days": "the last {n} days", + "next_n_days": "the next {n} days", + "custom": "a custom range", +} + +MAX_N = 3650 # ten years; a bound, not a business rule + +# --- the DATE-VALUE anchors (owner item 3, 2026-07-26) ---------------------------------------- +# The second half of a date CONDITION, as distinct from a measure's window: +# +# Where [Last order] [is before] [one month ago] +# ^ op ^ THIS +# +# A window answers "over what period do I sum"; an anchor answers "which single date am I +# comparing against". Both live in this module for one reason: they are the same two-engine +# contract, resolved from the same `today`, and a gate that holds one in lock-step and not the +# other would leave half the sentence free to drift. + +#: Anchor modes, in the order the picker offers them. `exact` is the historical behaviour — a +#: rule with NO mode is an `exact` rule whose value is an ISO date, which is what every view +#: saved before this change carries. +ANCHOR_MODES = ( + "today", + "yesterday", + "one_week_ago", + "one_month_ago", + "n_days_ago", + "exact", +) + +#: Modes that carry NO value. ⚠ This set is load-bearing far outside this module: a rule whose +#: value is blank normally reads as INACTIVE, and an inactive rule is IGNORED — which WIDENS the +#: result under a count nobody would doubt. Every activeness check (`isRuleActive` in TS, +#: `filter_sql.is_rule_active`, and the validator's value handling) has to know these four are +#: active with an empty value. +ANCHOR_VALUE_FREE = frozenset({"today", "yesterday", "one_week_ago", "one_month_ago"}) + +ANCHOR_LABELS = { + "today": "today", + "yesterday": "yesterday", + "one_week_ago": "one week ago", + "one_month_ago": "one month ago", + "n_days_ago": "{n} days ago", + "exact": "an exact date", +} + + +def _iso(d): + return d.isoformat() + + +def _month_start(d): + return d.replace(day=1) + + +def _month_end(d): + return _next_month(d.replace(day=1)) - _dt.timedelta(days=1) + + +def _next_month(d): + return (d.replace(day=28) + _dt.timedelta(days=4)).replace(day=1) + + +def _quarter_start(d): + return _dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1) + + +def _days_in_month(y, m): + return (_dt.date(y + (m == 12), 1 if m == 12 else m + 1, 1) - _dt.timedelta(days=1)).day + + +def _shift_months(d, n): + """`d` moved `n` months, CLAMPING the day to the target month's length. + + Jan 31 back one month is Dec 31, but Mar 31 back one month is Feb 28 (or 29) — there is no + Feb 31 to land on. Both engines must clamp identically or "the past month" differs by up to + three days for a third of the calendar; the gate probes a 31st and a leap day for exactly it. + """ + total = (d.year * 12 + (d.month - 1)) + n + y, m = divmod(total, 12) + m += 1 + return _dt.date(y, m, min(d.day, _days_in_month(y, m))) + + +def _parse_date(v): + """Accept 'YYYY-MM-DD' (and tolerate a longer ISO timestamp by taking its date part).""" + if isinstance(v, _dt.date): + return v + s = str(v or "").strip()[:10] + if not s: + return None + try: + return _dt.date.fromisoformat(s) + except ValueError: + return None + + +def normalize(spec): + """Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None. + + Fail-closed on the KIND (an unknown kind is not a window), tolerant on the rest — the same + split `clean_filter_tree` uses, so one malformed window cannot cost a user their saved view. + """ + if not isinstance(spec, dict): + return None + kind = spec.get("kind") + if kind not in WINDOW_KINDS: + return None + out = {"kind": kind} + if kind in N_KINDS: + try: + n = int(spec.get("n")) + except (TypeError, ValueError): + return None + if n < 1 or n > MAX_N: + return None + out["n"] = n + if kind == "custom": + f, t = _parse_date(spec.get("from")), _parse_date(spec.get("to")) + if f is None and t is None: + return None # a custom range with no bounds is not a window + if f is not None and t is not None and f > t: + f, t = t, f # the builder cannot enforce order; the engine can + if f is not None: + out["from"] = _iso(f) + if t is not None: + out["to"] = _iso(t) + return out + + +def resolve(spec, today): + """Window spec + the tenant's today -> `(date_from, date_to)`, both inclusive ISO dates. + + Either side may be None, meaning open-ended. Returns None when the spec is not a window at + all — callers MUST treat that as "this condition cannot be evaluated" and refuse, never as + "no window", which would silently widen the result to all time. + """ + spec = normalize(spec) + if spec is None: + return None + kind = spec["kind"] + d = _parse_date(today) + if d is None: + raise ValueError("resolve() needs an explicit `today` — see this module's docstring") + + if kind == "all_time": + return (None, None) + if kind == "today": + return (_iso(d), _iso(d)) + if kind == "yesterday": + y = d - _dt.timedelta(days=1) + return (_iso(y), _iso(y)) + + # ISO 8601: the week starts MONDAY. Python's weekday() is already Monday=0; the TS mirror + # has to correct getDay(), which is Sunday=0. The gate probes both a Sunday and a Monday. + if kind == "this_week": + start = d - _dt.timedelta(days=d.weekday()) + return (_iso(start), _iso(start + _dt.timedelta(days=6))) + if kind == "last_week": + start = d - _dt.timedelta(days=d.weekday() + 7) + return (_iso(start), _iso(start + _dt.timedelta(days=6))) + + if kind == "this_month": + return (_iso(_month_start(d)), _iso(_month_end(d))) + if kind == "last_month": + prev = _month_start(d) - _dt.timedelta(days=1) + return (_iso(_month_start(prev)), _iso(prev)) + + if kind == "this_quarter": + qs = _quarter_start(d) + qe = _month_end(_dt.date(qs.year, qs.month + 2, 1)) + return (_iso(qs), _iso(qe)) + if kind == "last_quarter": + prev_end = _quarter_start(d) - _dt.timedelta(days=1) + qs = _quarter_start(prev_end) + return (_iso(qs), _iso(prev_end)) + + if kind == "this_year": + return (_iso(_dt.date(d.year, 1, 1)), _iso(_dt.date(d.year, 12, 31))) + if kind == "last_year": + return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_dt.date(d.year - 1, 12, 31))) + if kind == "ytd": + # year-to-date ENDS TODAY, unlike this_year which runs to Dec 31. The distinction is the + # whole reason both exist: comparing "this year" against last year double-counts the + # months that have not happened yet. + return (_iso(_dt.date(d.year, 1, 1)), _iso(d)) + if kind == "ytd_last_year": + # SAME PERIOD last year — Jan 1 LY through today's month/day LY. This is the honest + # partner of `ytd`, and it is not a nicety: it mirrors `core.periods.ytd_last_year` + # EXACTLY (including the Feb 29 -> Feb 28 clamp), which is what `pool()` computes the + # `revenue_ly` column from. `Sales[ytd] < Sales[ytd_last_year]` therefore reproduces the + # retired `at_risk > 0` condition rather than approximating it. Comparing `ytd` against + # `last_year` instead would pit seven months against twelve. + return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_shift_months(d, -12))) + if kind == "ltm": + # the last twelve months INCLUDING today: 365 days back, both ends inclusive + return (_iso(d - _dt.timedelta(days=364)), _iso(d)) + + # "The past week/month/year" are ROLLING and end TODAY — Airtable's wording, and NOT the + # calendar kinds above: `last_week` is the previous Monday-Sunday, `past_week` is the seven + # days ending today. Both are offered because a user means different things by them, and + # picking one to serve both would silently answer the other question. + # past_week = the same rule as last_n_days(7) — one period back, PLUS ONE DAY, so today + # is included and the span is exactly 7 days rather than 8. + # past_year agrees with `ltm` except across a leap day, where the calendar shift keeps the + # same month/day and the 364-day subtraction cannot. + if kind == "past_week": + return (_iso(d - _dt.timedelta(days=6)), _iso(d)) + if kind == "past_month": + return (_iso(_shift_months(d, -1) + _dt.timedelta(days=1)), _iso(d)) + if kind == "past_year": + return (_iso(_shift_months(d, -12) + _dt.timedelta(days=1)), _iso(d)) + + if kind == "last_n_days": + # INCLUSIVE of today — "the last 7 days" is today and the 6 before it, not the 7 before + # yesterday. Getting this wrong drops today's orders from every recent-activity filter. + return (_iso(d - _dt.timedelta(days=spec["n"] - 1)), _iso(d)) + if kind == "next_n_days": + return (_iso(d), _iso(d + _dt.timedelta(days=spec["n"] - 1))) + + if kind == "custom": + return (spec.get("from"), spec.get("to")) + raise AssertionError(f"unhandled window kind {kind!r}") # unreachable; WINDOW_KINDS is closed + + +def label(spec): + """How a window reads in the condition sentence ('in the last 90 days').""" + spec = normalize(spec) + if spec is None: + return "an invalid range" + return WINDOW_LABELS[spec["kind"]].format(n=spec.get("n")) + + +def resolve_anchor(mode, value, today): + """A date condition's right-hand side -> ONE inclusive ISO date, or None. + + `mode` is an entry of ANCHOR_MODES; None/'' means `exact`, which is what every view saved + before anchors existed carries (op + an ISO date in `value`). `value` supplies the number for + `n_days_ago` and the date for `exact`, and is ignored by the four value-free modes. + + Returns None when the anchor cannot be resolved — an unknown mode, a non-numeric N, a value + that is not a date. The caller must then treat the CONDITION as unanswerable and match + nothing. It must NOT fall through to "no condition": that widens the result while the count + beside it still looks authoritative, which is the whole reason this returns None rather than + a best guess (same contract as `resolve()` above). + + `n_days_ago` accepts n = 0 where a WINDOW requires n >= 1. A zero-day window is empty and + could only be a mistake; "0 days ago" is today, which is a date a person can mean. + """ + m = mode or "exact" + if m not in ANCHOR_MODES: + return None + d = _parse_date(today) + if d is None: + raise ValueError("resolve_anchor() needs an explicit `today` — see this module's " + "docstring on why the clock is never read here") + if m == "today": + return _iso(d) + if m == "yesterday": + return _iso(d - _dt.timedelta(days=1)) + if m == "one_week_ago": + # A week ago is a DATE (today minus 7), not the past-week RANGE. The two read almost + # identically in English and mean different things in a comparison. + return _iso(d - _dt.timedelta(days=7)) + if m == "one_month_ago": + return _iso(_shift_months(d, -1)) + if m == "n_days_ago": + s = str("" if value is None else value).strip() + # `[0-9]` not `\d`: a Python \d matches every Unicode decimal digit, so a fullwidth + # '30' would resolve here and be rejected by the TS mirror's Number() — the exact + # two-engine divergence filter_sql.to_num was rewritten to avoid. + if not s or not all("0" <= ch <= "9" for ch in s): + return None + n = int(s) + # No `n < 0` guard: the digit test above already excludes a sign, so it would be + # unreachable. The negative control proved that — removing it changed nothing, which is + # how dead code hides in a defensive-looking line. The CAP is the live guard. + if n > MAX_N: + return None + return _iso(d - _dt.timedelta(days=n)) + parsed = _parse_date(value) # exact + return None if parsed is None else _iso(parsed) + + +def anchor_label(mode, value): + """How an anchor reads in the condition sentence ('one month ago', '30 days ago').""" + m = mode or "exact" + if m not in ANCHOR_MODES: + return "an invalid date" + if m == "n_days_ago": + return ANCHOR_LABELS[m].format(n=str("" if value is None else value).strip() or "N") + if m == "exact": + return str("" if value is None else value).strip() or ANCHOR_LABELS[m] + return ANCHOR_LABELS[m] diff --git a/royalimports_os/harness/workspace.py b/royalimports_os/harness/workspace.py new file mode 100644 index 0000000000000000000000000000000000000000..aba71cccd19acdf5f5c7691911f7730f4a6c9a99 --- /dev/null +++ b/royalimports_os/harness/workspace.py @@ -0,0 +1,226 @@ +"""harness/workspace.py — the unified WORKSPACE object model (owner directive 2026-07-12: +"every single time we create a module or analytics from the AIOS, it should be modular — a +workspace"). + +Every artifact AIOS creates — a view, a dashboard, an alert, a scheduled report — is a MODULAR, +self-contained workspace object: it carries everything needed to re-run itself (the governed +query / metric reference), renders through the house primitives, composes into bigger objects +(views → dashboards → schedules), and exports as a TENANT-AGNOSTIC template (queries reference +topic/metric KEYS; scope lives in the model, so a template stamps onto any tenant whose model +has those keys — the same principle that made model/ itself the template library). + +This module is the facade: one uniform envelope + one listing/manage/export surface over the +underlying stores (views.json, routines.json). It does not replace them — it makes them one +workspace. +""" +import json +import time +from pathlib import Path + +import harness.views as V +import harness.routines as R +import harness.semantic as SEM + +TEMPLATES_DIR = Path(__file__).resolve().parents[1] / "model" / "templates" + +# query keys that are tenant/context-specific and must NOT travel in a template +_STRIP = ("team_id", "filters", "date_from", "date_to") + + +def items(): + """Every workspace object, uniform envelope: {kind, name, detail, created_at}.""" + out = [] + vs, ds = V.views(), V.dashboards() + dash_members = {vn for d in ds.values() for vn in d.get("views", [])} + for name, v in vs.items(): + spec = v.get("chart") or {} + q = spec.get("query") or {} + out.append({"kind": "view", "name": name, + "detail": f"{q.get('topic', '?')} · {', '.join(q.get('measures') or [])}" + + (" · in a dashboard" if name in dash_members else ""), + "created_at": v.get("saved_at", "")}) + for name, d in ds.items(): + out.append({"kind": "dashboard", "name": name, + "detail": f"{len(d.get('views', []))} view(s): " + + ", ".join(d.get("views", [])[:4]), + "created_at": d.get("created_at", "")}) + rt = R.load() + for key, a in rt.get("alerts", {}).items(): + out.append({"kind": "alert", "name": key, + "detail": f"{a.get('metric')} · {a.get('mode')}" + + (f" · team {a['team_id']}" if a.get("team_id") else ""), + "created_at": a.get("created_at", "")}) + for key, r in rt.get("reports", {}).items(): + out.append({"kind": "report", "name": key, + "detail": f"{r.get('cadence')} · {r.get('channel')}", + "created_at": r.get("created_at", "")}) + for key, i in (rt.get("insights") or {}).items(): + out.append({"kind": "insight", "name": key, + "detail": f"{i.get('cadence')} · \"{(i.get('prompt') or '')[:60]}\"", + "created_at": i.get("created_at", "")}) + return sorted(out, key=lambda x: x.get("created_at") or "", reverse=True) + + +def delete(kind, name): + if kind == "view": + V.delete_view(name) + elif kind == "dashboard": + V.delete_dashboard(name) + elif kind == "alert": + R.delete_alert(name) + elif kind == "report": + R.delete_report(name) + elif kind == "insight": + R.delete_insight(name) + else: + raise SEM.ModelError(f"unknown workspace kind {kind!r}") + + +# ------------------------------------------------------------------ templates (tenant-agnostic) + +def _templatize_view(name): + v = V.views().get(name) + if not v: + raise SEM.ModelError(f"unknown view {name!r}") + spec = dict(v.get("chart") or {}) + q = dict(spec.get("query") or {}) + if not q: + raise SEM.ModelError(f"view {name!r} carries no query — cannot templatize") + stripped = {k: q.pop(k) for k in _STRIP if k in q} + stripped = {k: v for k, v in stripped.items() if v} # record only real tenant baggage + q["window"] = "ytd" # instantiation default; tenant picks at stamp time + spec["query"] = q + return {"name": name, "spec": spec, + "stripped": sorted(stripped)} # honest record of what was tenant-specific + + +def export_template(kind, name): + """Export a view or dashboard as a tenant-agnostic template JSON in model/templates/. + Dashboards embed their views' templates — one self-contained modular unit.""" + if kind == "view": + tpl = {"template": "view", "exported_at": time.strftime("%Y-%m-%d %H:%M"), + **_templatize_view(name)} + elif kind == "dashboard": + d = V.dashboards().get(name) + if d is None: + raise SEM.ModelError(f"unknown dashboard {name!r}") + tpl = {"template": "dashboard", "name": name, + "exported_at": time.strftime("%Y-%m-%d %H:%M"), + "views": [_templatize_view(vn) for vn in d.get("views", [])]} + else: + raise SEM.ModelError("only views and dashboards templatize (alerts/reports are one-liners " + "— recreate from the metric key)") + TEMPLATES_DIR.mkdir(parents=True, exist_ok=True) + safe = "".join(c if c.isalnum() or c in "-_ " else "_" for c in name).strip().replace(" ", "-") + path = TEMPLATES_DIR / f"{kind}-{safe}.json" + path.write_text(json.dumps(tpl, indent=1), encoding="utf-8") + return str(path) + + +PROPOSALS_DIR = Path(__file__).resolve().parents[1] / "model" / "proposals" + + +def promote(kind, name, note=""): + """Nominate a workspace object for PROMOTION into the governed model (principle 3: model + just-in-time, promote what's proven). v0: writes a reviewable proposal file under + model/proposals/ — the next Claude session is the Modeler: it reviews the proposal, lands + the actual model diff (a metric, a skill recipe, or a curated template), and deletes the + proposal. Git is the branch mode; nothing merges without that review.""" + if kind == "view": + v = V.views().get(name) + if not v: + raise SEM.ModelError(f"unknown view {name!r}") + spec = v.get("chart") or {} + payload = {"kind": "view", "name": name, "spec": spec} + q = spec.get("query") or {} + hint = (f"candidate: a named metric or skill recipe on topic {q.get('topic')!r} " + f"(measures {q.get('measures')}, group_by {q.get('group_by')})") + elif kind == "dashboard": + d = V.dashboards().get(name) + if d is None: + raise SEM.ModelError(f"unknown dashboard {name!r}") + payload = {"kind": "dashboard", "name": name, "views": d.get("views", []), + "view_specs": {vn: (V.views().get(vn) or {}).get("chart") + for vn in d.get("views", [])}} + hint = "candidate: a curated template in model/templates/ or a skill recipe" + else: + raise SEM.ModelError("only views and dashboards can be nominated") + try: + import harness.telemetry as TEL + usage = TEL.summary() + except Exception: + usage = {} + PROPOSALS_DIR.mkdir(parents=True, exist_ok=True) + safe = "".join(c if c.isalnum() or c in "-_ " else "_" for c in name).strip().replace(" ", "-") + path = PROPOSALS_DIR / f"{time.strftime('%Y%m%d-%H%M')}-{kind}-{safe}.json" + path.write_text(json.dumps({ + "proposed_at": time.strftime("%Y-%m-%d %H:%M"), + "note": note, "suggested": hint, "usage_at_nomination": usage, **payload, + "reviewer_instructions": ( + "MODELER REVIEW (next Claude session): decide metric / skill-recipe / template / " + "reject; land the model diff in model/*.yml with a validate contract where " + "applicable; re-run the eval gate; DELETE this proposal file; log the decision."), + }, indent=1), encoding="utf-8") + return str(path) + + +def proposals(): + if not PROPOSALS_DIR.exists(): + return [] + return sorted(p.name for p in PROPOSALS_DIR.glob("*.json")) + + +def templates(): + if not TEMPLATES_DIR.exists(): + return [] + return sorted(p.name for p in TEMPLATES_DIR.glob("*.json")) + + +def _ytd(): + from datetime import date + t = date.today() + return t.replace(month=1, day=1).isoformat(), t.isoformat() + + +def instantiate_template(filename, new_name=None): + """Stamp a template into THIS tenant's workspace: window resolves to YTD now; tenant scope + comes from the model (topics/tenant.yml), so no per-tenant edits are needed.""" + path = TEMPLATES_DIR / filename + if not path.exists(): + raise SEM.ModelError(f"unknown template {filename!r}") + tpl = json.loads(path.read_text(encoding="utf-8")) + df, dt_ = _ytd() + + def _stamp_view(vtpl, vname): + spec = dict(vtpl["spec"]) + q = dict(spec.get("query") or {}) + q.pop("window", None) + q["date_from"], q["date_to"] = df, dt_ + spec["query"] = q + spec["title"] = vname + import harness.tools as T + T.save_view(vname, {"chart": spec}) + return vname + + def _fresh(name): + """Never clobber an existing object — suffix until free.""" + vs, ds = V.views(), V.dashboards() + cand, i = name, 2 + while cand in vs or cand in ds: + cand = f"{name} ({i})" + i += 1 + return cand + + if tpl.get("template") == "view": + vname = _fresh(new_name or tpl["name"]) + return {"created": "view", "name": _stamp_view(tpl, vname)} + if tpl.get("template") == "dashboard": + dname = _fresh(new_name or tpl["name"]) + # namespace member views under the new dashboard so instantiation NEVER overwrites + # existing views (modularity means copies, not aliasing) + vnames = [_stamp_view(vt, _fresh(f"{dname} · {vt['name']}")) + for vt in tpl.get("views", [])] + import harness.tools as T + T.compose_dashboard(dname, vnames) + return {"created": "dashboard", "name": dname, "views": vnames} + raise SEM.ModelError(f"bad template file {filename!r}") diff --git a/royalimports_os/model/metrics/gl.yml b/royalimports_os/model/metrics/gl.yml new file mode 100644 index 0000000000000000000000000000000000000000..b82a3b8d1d9ba18a660d9fce536e3dc952f6c4dc --- /dev/null +++ b/royalimports_os/model/metrics/gl.yml @@ -0,0 +1,36 @@ +# Metrics: gl — raw-GL basis metrics (topic gl_lines), matching modules/expenses.py definitions +# EXACTLY (the parity anchor): opex = Σ balance of posted lines to expense-type accounts +# ('expense','expense_depreciation'), COGS ('expense_direct_cost') EXCLUDED; gl_revenue = income +# accounts negated. Filtered measures use store_filter_sql (aliases: l=line, m=move, a=account). +topic: gl_lines +metrics: + - key: opex + label: Operating expense + agg: sum + field: balance + store_filter_sql: "a.account_type IN ('expense', 'expense_depreciation')" + format: usd + description: "Posted operating expense from the GL (all expense-type accounts; COGS excluded) — the Expenses dashboard's basis." + ai_context: "Company-level; includes payroll JEs/depreciation/fees that never hit vendor bills. Watch close-lag: the newest month is often still being booked." + validate: + method: gl_opex + note: "Store-path opex must equal the live Odoo aggregate (posted, expense-type accounts) to the cent." + + - key: gl_cogs + label: COGS (GL) + agg: sum + field: balance + store_filter_sql: "a.account_type = 'expense_direct_cost'" + format: usd + description: "Cost of goods sold on the GL basis (expense_direct_cost accounts)." + ai_context: "Distinct from the sales-margin COGS (revenue - margin on order lines) — different basis, reconcile deliberately, never mix silently." + + - key: gl_revenue + label: Revenue (GL) + agg: sum + field: balance + negate: true + store_filter_sql: "a.account_type IN ('income', 'income_other')" + format: usd + description: "GL-basis revenue: income-type account balances, negated to read positively." + ai_context: "ALL-channel and accrual-basis — will not equal wholesale order revenue (sales_lines); that difference is structural (Amazon channel + timing), not an error." diff --git a/royalimports_os/model/metrics/invoice_lines.yml b/royalimports_os/model/metrics/invoice_lines.yml new file mode 100644 index 0000000000000000000000000000000000000000..0af67a191e51649fd6122de3940b1ef0aa3e8f69 --- /dev/null +++ b/royalimports_os/model/metrics/invoice_lines.yml @@ -0,0 +1,105 @@ +# Metrics: invoice_lines — the invoice-LINE lens, with per-line agent allocation (2026-07-28). +# +# These make the owner's agent questions resolvable by the Analyst / metric dictionary / grid +# instead of an ad-hoc script. Slice any of them by the `allocation`, `agent`, `agent_role` or +# `team` dims on the topic. +topic: invoice_lines +metrics: + - key: invoiced_line_sales_gross + label: Invoiced line sales (gross) + agg: sum + field: price_subtotal + store_filter_sql: "l.move_type = 'out_invoice'" + format: usd + description: "Untaxed invoice-line subtotals, invoices only (before credit notes)." + ai_context: >- + A component of `invoiced_line_sales`, not an answer on its own — it ignores credit notes. + + - key: invoiced_line_refunds + label: Invoiced line refunds + agg: sum + field: price_subtotal + store_filter_sql: "l.move_type = 'out_refund'" + format: usd + description: "Untaxed credit-note line subtotals, reported POSITIVE." + ai_context: >- + Credit-note `price_subtotal` is stored POSITIVE (unlike the document-level + amount_untaxed_signed on topic credit_notes), so this metric carries no negation and + `invoiced_line_sales` subtracts it. + + - key: invoiced_line_sales + label: Invoiced line sales (net) + agg: derived + expr: "invoiced_line_sales_gross - invoiced_line_refunds" + format: usd + description: >- + THE invoice-line revenue measure: untaxed product-line subtotals of posted customer + invoices, net of credit notes. + ai_context: >- + Use this — not the gross component — whenever asked for invoice-line sales. Net of credit + notes. This is NOT `revenue` (order-line based, topic sales_lines); the two differ by + unbilled orders and billing timing, and on Fisch 2025 they run ~2% apart, which is normal + and must not be "reconciled" away. + validate: + method: line_vs_move_grain + note: >- + Independent check: the sum of these product-line subtotals must equal the sum of + amount_untaxed_signed over the SAME posted customer moves read straight from Odoo. + Verified 2026-07-28 all-scope 2025: 8,955,776.63 - 149,961.44 = 8,805,815.19 exactly. + Catches a lost credit-note negation and a wrong display_type filter, neither of which + self-consistency can see. + + - key: agent_allocated_sales + label: Sales allocated to an agent + agg: sum + field: price_subtotal + store_filter_sql: "l.move_type = 'out_invoice' AND ca.is_true_agent" + format: usd + description: >- + Invoice-line sales whose line carries a REAL agent (res.partner.agent = True) — excludes + the internal salespeople who also carry commission lines. + ai_context: >- + Gross of credit notes by design (refund lines rarely carry commission rows); for the netted + view slice `invoiced_line_sales` by the `allocation` dim instead, which is MECE. + ⚠ This is "sales ALLOCATED to an agent", NOT "sales BY customers who have an agent" — 403 + of Fisch's 518 agent customers are mixed, so the two differ by ~$192k on 2025. + + - key: unallocated_line_sales + label: Sales not allocated to an agent + agg: sum + field: price_subtotal + store_filter_sql: "l.move_type = 'out_invoice' AND ca.is_true_agent IS NOT TRUE" + format: usd + description: >- + Invoice-line sales with no REAL agent on the line — the strict reading, which counts + salesperson-only lines as unallocated. + ai_context: >- + Two defensible readings and this is the WIDER one (salesperson-covered business counts as + unallocated). The narrower "no commission row at all" figure is this metric sliced to + allocation = 'none'. On Fisch 2025 the two are $1,447,579.59 vs $1,086,381.35 — a material + gap, so state which reading you gave. + + - key: agent_line_customers + label: Customers with an agent on the line + agg: count_distinct + field: partner_id + store_filter_sql: "ca.is_true_agent" + format: int + description: "Distinct customers with a real agent on at least one invoice line." + ai_context: >- + Fisch 2025 = 518. Counting customers, not revenue: a customer counts here if ANY line + carries an agent, so pairing this with `agent_allocated_sales` mixes two grains — pair it + with the customers' TOTAL sales if the question is "what do agent-assigned customers + generate" ($3,187,571 territory), and say which you did. + + - key: commission_amount + label: Commission accrued + agg: sum + field: amount + format: usd + topic: commission_lines + description: "Commission accrued on invoice lines (the OCA sale-commission `amount`)." + ai_context: >- + Resolved on topic `commission_lines` (one row per line × agent) NOT on invoice_lines, + because a line with two agents carries two commission amounts and the deduplicated join on + invoice_lines would drop one. Never sum commission through invoice_lines. diff --git a/royalimports_os/model/metrics/receivables.yml b/royalimports_os/model/metrics/receivables.yml new file mode 100644 index 0000000000000000000000000000000000000000..f982b8f2e7d1934d6ebe4f43e30e53b50aaae23a --- /dev/null +++ b/royalimports_os/model/metrics/receivables.yml @@ -0,0 +1,30 @@ +# Metrics: receivables (topic receivables) — AR exposure, matching the Collections conventions +# (amount_residual_signed nets credit notes; overdue = past invoice_date_due as of today). +topic: receivables +metrics: + - key: ar_outstanding + label: AR outstanding + agg: sum + field: amount_residual_signed + format: usd + description: "Total unpaid customer balance: signed residual over posted invoices and credit notes (credit notes net against invoices)." + ai_context: "Ask WITHOUT a date window for the current balance (a window filters by invoice date, not by 'as of')." + validate: + method: ar_outstanding + note: "Store-path outstanding must equal the live Odoo aggregate over the same domain, to the cent." + + - key: ar_overdue + label: AR overdue + agg: sum + field: amount_residual_signed + store_filter_sql: "CAST(m.invoice_date_due AS DATE) < CURRENT_DATE AND m.payment_state != 'paid'" + format: usd + description: "Unpaid balance on documents past their due date as of today." + ai_context: "Overdue is judged against today's date at query time; pairs with the Collections worklist for who-to-call." + + - key: open_invoices + label: Open invoices + agg: count + store_filter_sql: "m.payment_state IN ('not_paid', 'partial') AND m.move_type = 'out_invoice'" + format: int + description: "Count of posted customer invoices not yet fully paid." diff --git a/royalimports_os/model/metrics/returns.yml b/royalimports_os/model/metrics/returns.yml new file mode 100644 index 0000000000000000000000000000000000000000..d348189304e42ebedeaa9d342c44340345b5e3a9 --- /dev/null +++ b/royalimports_os/model/metrics/returns.yml @@ -0,0 +1,55 @@ +# Metrics: returns — the credit-note lens, defined once (2026-07-27, owner's ask). +# +# `modules/returns.py` has read these numbers from Odoo since it shipped; this file makes them +# MODEL metrics so the Analyst, the metric dictionary and the customer-grid condition builder all +# resolve the same definition instead of three. +topic: credit_notes +metrics: + - key: returns + label: Returns $ + agg: sum + field: amount_untaxed_signed + negate: true + format: usd + description: >- + Untaxed value of posted customer credit notes (refunds), reported POSITIVE. + ai_context: >- + ⚠ NEGATE IS LOAD-BEARING. `amount_untaxed_signed` is negative on a credit note, so the raw + sum is negative and this metric flips it. A returns figure that arrives negative means the + negation was lost somewhere — and it will read as "no returns" rather than as an error. + Company-level: credit notes are not business-unit tagged (they all carry team_id 1), so + never attribute a returns number to Fisch or Royal. Attribute by CUSTOMER instead. + validate: + method: returns_sign + note: >- + Independent check: the metric must equal the ABSOLUTE untaxed total of the same posted + credit notes read straight from Odoo, and must be positive. Catches a lost or doubled + negation, which self-consistency cannot see. + + - key: invoiced + label: Invoiced $ + topic: customer_invoices + agg: sum + field: amount_untaxed_signed + format: usd + description: >- + Untaxed value of posted customer invoices — the denominator the business rates returns + against. + ai_context: >- + NOT `revenue`. `revenue` is order-LINE subtotals of confirmed sales orders; this is posted + billing documents, and the two differ by unbilled orders and by billing timing. Use + `revenue` for "sales" and this only as the return-rate denominator. + + - key: returns_pct + label: Return % + agg: ratio + numerator: returns + denominator: invoiced + format: pct + description: "Returns as a share of invoiced sales — the rate modules/returns.py reports." + ai_context: >- + A FRACTION (0-1), like every other pct metric here — 0.0159 is 1.59%. The condition builder + divides a typed percentage by 100 before comparing (`input_scale`), so a user types 5 and + means 5%. Guarded against zero invoiced (returns 0). Cross-topic: the numerator is on + credit_notes and the denominator on customer_invoices, so it is resolved per customer by + computing both and combining, never by one query. diff --git a/royalimports_os/model/metrics/sales.yml b/royalimports_os/model/metrics/sales.yml new file mode 100644 index 0000000000000000000000000000000000000000..01eb6f34a1461c6d1e0124fab1807d63c2886092 --- /dev/null +++ b/royalimports_os/model/metrics/sales.yml @@ -0,0 +1,84 @@ +# Metrics: sales — the core wholesale metrics, defined ONCE (OM-0). Every surface (pages, the +# metric dictionary, the Analyst, MCP) resolves these by key through harness/semantic.py. +# Fields per metric: +# key/label/description — identity + the human definition (visibility = trust) +# agg + field — sum | count_distinct over the topic's entity +# agg: ratio — numerator/denominator are metric KEYS (resolved recursively) +# agg: derived + expr — arithmetic over metric keys (safe parser; +,-,*,/ and parens only) +# format — usd | int | pct (rendering hint for surfaces) +# ai_context — what a small model must know to use the metric correctly +# validate — the INDEPENDENT Odoo cross-check contract (named method implemented in +# harness/semantic.py _VALIDATORS; a metric that can't tie out says so) +topic: sales_lines +metrics: + - key: revenue + label: Revenue + agg: sum + field: price_subtotal + format: usd + description: "Untaxed revenue of confirmed wholesale order lines (the owner's 'sales' number)." + ai_context: "Always untaxed; excludes Amazon (GIFTWARE DEALS) and unconfirmed orders. Filter one BU via team_id (Fisch=5, Royal=6)." + validate: + method: order_level_revenue + note: "Σ line price_subtotal must equal Σ parent-order amount_untaxed under the same scope, to the cent (line vs order basis — the built-in cross-check)." + + - key: units + label: Units sold + agg: sum + field: product_uom_qty + format: int + description: "Total quantity across confirmed wholesale order lines." + ai_context: "Mixed UoMs are summed as ordered quantity; for weight/case analysis convert per product UoM first." + + - key: margin + label: Gross margin $ + agg: sum + field: margin + format: usd + description: "Line revenue minus line cost (Odoo Margin module), summed." + ai_context: "margin is read_group-aggregatable (Margin module installed). COGS = revenue - margin. purchase_price is per-UNIT cost — never sum it as a total." + + - key: cogs + label: COGS + agg: derived + expr: "revenue - margin" + format: usd + description: "Cost of goods sold, derived: revenue minus gross margin." + ai_context: "Derived, not pulled — Odoo carries cost on lines as margin; COGS is the difference." + + - key: margin_pct + label: Gross margin % + agg: ratio + numerator: margin + denominator: revenue + format: pct + description: "Gross margin as a share of revenue." + ai_context: "Guarded against zero revenue (returns 0). Compare across BUs/categories at the same scope only." + + - key: orders + label: Orders + topic: sales_orders + agg: count + format: int + description: "Confirmed wholesale orders in the window (order-header count — what the scorecards show)." + ai_context: "Order-level count; slightly higher than distinct-orders-from-lines because a few confirmed orders carry zero lines (a data-health artifact the reconciliation contract counts exactly)." + validate: + method: order_count + note: "Order-level count must equal distinct line-parent orders + line-less orders, exactly (surfaces empty orders instead of hiding them in a tolerance)." + + - key: customers + label: Active customers + agg: count_distinct + field: order_partner_id + format: int + description: "Distinct customers with at least one confirmed wholesale order line in the window." + ai_context: "Customer = the order's partner. Agent attribution uses res.partner.agent_ids, NOT the order user_id." + + - key: aov + label: Average order value + agg: ratio + numerator: revenue + denominator: orders + format: usd + description: "Revenue per distinct order." + ai_context: "Ratio of two registered metrics at identical scope; never average per-order averages." diff --git a/royalimports_os/model/skills/analytics.skill.yml b/royalimports_os/model/skills/analytics.skill.yml new file mode 100644 index 0000000000000000000000000000000000000000..c481cffad883374ea062182e9cd1a1eba91903ce --- /dev/null +++ b/royalimports_os/model/skills/analytics.skill.yml @@ -0,0 +1,171 @@ +# In-product SKILL: the ANALYTICS TRANSFORM LIBRARY (v1, 2026-07-18) — the Tableau-class +# table-calc + analytics capability set, mapped onto the ONE governed transform_result tool. +# Grounding: .claude/wiki/research/analytics-tools.md. Every op here EXISTS in harness/transforms.py +# (OPS) and is unit-proven in _demo_analytics.py — recipes and code version together. +# +# HOW IT WORKS: run_semantic_query -> transform_result([{op, ...}, ...]) -> make_chart|make_table. +# Ops CHAIN (each takes the previous derived result). The model picks op names + params; the +# platform computes deterministically over the exact governed rows and every derived result still +# drills to its numbers. NEVER do arithmetic in your head — there is an op for it. +key: analytics +label: Analytics transforms (table calcs, stats, business templates) + +# ========================================================================= THE ADDITIVITY LAW +# The ONE rule that must never be broken: revenue, units, margin, cogs, orders are ADDITIVE — you +# may sum/accumulate them. customers (a DISTINCT count) and margin_pct / aov (RATIOS) are NOT +# additive — summing them over months or groups OVER-COUNTS and is forbidden. The accumulating ops +# (running_total, share_of_total, cum_share, moving_sum, abc_classify, concentration) REFUSE a +# non-additive measure with an error. For a CUMULATIVE or TRAILING distinct count / ratio, use the +# re-query ops ytd / rolling (they re-run the governed query per window — correct by construction), +# never running_total/moving_sum. + +# ========================================================================= THE OP CATALOG (families) +ops: + ordering_rank: > + sort{by,direction} · head{n} · bottom_n{by,n} · rank{by,direction} · rank_pct{by,direction} + (percentile rank 0-100) · ntile{by,tiles,direction} (quartile=4/decile=10 buckets) · + top_n{by,n,other} (N largest + honest Other) · add_total{label} (grand-total row). + part_to_whole: "share_of_total{of} (adds _share_pct) · cum_share{of} (pareto prep, sorted). ADDITIVE-only." + running_moving: > + running_total{of} (ADDITIVE-only) · running_avg{of} · running_max{of} · running_min{of} · + running_count{} (cumulative # of rows) · moving_average{of,window} · moving_sum{of,window} + (ADDITIVE-only) · moving_median{of,window} · rolling_std{of,window} (trailing volatility). + running_avg/max/min and the moving averages/medians/std are smoothings — valid on any measure. + period_over_period: > + diff{of} (vs previous row, one column) · pct_change{of} · lag{of,k} · lead{of,k} · + diff_from_first{of} · index_to_100{of} (rebase a series to 100) · percent_of_max{of} · + compare{a,b,how:diff|pct|ratio} (between TWO existing columns, e.g. revenue vs revenue_ly). + All row-wise — safe on any measure (a period-over-period change of a ratio is fine). + distribution_stats: > + bin{of,bins} (histogram buckets) · describe{of} (count/mean/median/std/min/p25/p75/max[/sum]) · + zscore{of} · outliers{of,method:zscore|iqr,k} · winsorize{of,p} · clip{of,lo,hi} · + normalize{of} (0-1) · correlate{x,y} (Pearson r) · weighted_average{of,weight} · + safe_ratio{numerator,denominator} (row-wise, guarded) · product{a,b}. + business_templates: > + abc_classify{of,a=80,b=95} (Pareto A/B/C tiers, ADDITIVE-only) · concentration{of} + (HHI+Gini+top1/5/10 share, ADDITIVE-only) · contribution_to_change{of} (needs _delta from + yoy — who drove the change) · rfm{recency,frequency,monetary,tiles=5} · funnel_rates{of} + (stage step% + overall%). + modeling: > + trend_line{of} (adds _trend fitted line to draw over the series) · regression{x,y} + (slope/intercept/r2) · cagr{of} (compound per-period growth) · growth_rate{of} (total first->last). + Transparent least-squares only — no black-box models. + reference_annotations: > + reference_line{of,stat:mean|median|min|max} (adds _ref constant) · reference_band{of, + method:stddev|percentile,k} (adds _band_lo/_hi) · target_line{value} · xmr_limits{of} + (Wheeler control chart: adds _center/_ucl/_lcl + _signal bool — the business-native + way to flag a REAL signal vs noise in a monthly series). Draw the constant columns as extra + series on a combo/line chart (the Analytics-pane average/reference/target/control line). + reshape: > + pivot{index,column,value} (long->wide crosstab; RAISES on collision — regroup the query, never + a hidden sum) · unpivot{keep,columns} (wide->long melt) · filter_rows{col,cmp,value} + (cmp: >,>=,<,<=,==,!=,contains — a 'having' clause) · dedupe{by} · resample{grain} (fill a + GAPPY date spine with None rows — run BEFORE window ops on sparse/filtered time series). + requery_windows: > + yoy{} (adds _ly/_delta/_yoy_pct; needs date_from/date_to) · ytd{of} (CORRECT cumulative + year-to-date, incl. distinct counts) · rolling{of,window} (CORRECT trailing-N-period value, + incl. distinct counts — T12M revenue, trailing-3-mo customers) · forecast{of,periods,method: + linear|seasonal_naive} (appends future rows with _forecast). These re-run the governed + query per window — the ONLY correct path for cumulative/trailing NON-additive measures. + +# ========================================================================= RECIPES (compose ops) +recipes: + - ask: "80/20 — which [customers|SKUs] carry the book? / ABC classification / concentration" + plan: > + run_semantic_query(group_by=[dim], measures=[revenue], sort=-revenue, limit=1000) -> + transform_result([{op: abc_classify, of: revenue}]) -> make_table (A/B/C tiers + cum %). + For a single concentration read: transform_result([{op: concentration, of: revenue}]) -> + make_kpi/table (HHI, Gini, top-5 share). For the picture: make_chart(kind=pareto). + guard: "abc_classify / concentration are ADDITIVE-only (revenue/units/margin) — never on customers." + + - ask: "cumulative distinct customers YTD / running unique customers this year" + plan: > + run_semantic_query(topic=sales_lines, measures=[customers], grain=month, date_from, date_to) + -> transform_result([{op: ytd, of: customers}]) -> make_chart(kind=line, x=period, + y=customers_ytd) + guard: > + NEVER running_total on customers (distinct counts don't sum). ytd re-runs the query + Jan1->each month end — the correct cumulative distinct count. Same law: rolling for trailing. + + - ask: "trailing-12-month / LTM [revenue|customers] (smooth out seasonality)" + plan: > + run_semantic_query(measures=[revenue], grain=month, date_from, date_to) -> + transform_result([{op: rolling, of: revenue, window: 12}]) -> make_chart(kind=line, + x=period, y=revenue_roll12) + guard: "rolling re-queries the trailing window — correct for customers (distinct) too, unlike moving_sum." + + - ask: "forecast / project the next [3] months of [revenue]" + plan: > + run_semantic_query(measures=[revenue], grain=month, date_from, date_to) -> + transform_result([{op: forecast, of: revenue, periods: 3}]) -> make_chart(kind=line, + x=period, y=revenue, y2=revenue_forecast) or combo + guard: > + transparent linear/seasonal-naive projection, NOT a statistical model — say "a simple trend + projection, not a forecast model". Use method=seasonal_naive when the series is seasonal and + you have >= one full cycle of history. + + - ask: "add an average line / target line / ±1 std band to a chart" + plan: > + transform_result([{op: reference_line, of: revenue, stat: mean}]) then make_chart(kind=combo, + y=revenue, y2=revenue_ref). Target: [{op: target_line, value: 500000}] -> make_chart( + kind=bullet, y=revenue, y2=target). Band: [{op: reference_band, of: revenue, method: stddev, + k: 1}] -> table/chart with revenue_band_lo/_hi. + guard: "the reference is drawn as an extra series/column — never annotate by editing the answer text." + + - ask: "who DECLINED the most YoY / biggest movers, and who drove the total change" + plan: > + run_semantic_query(group_by=[order_partner], measures=[revenue], team_id?, date_from, date_to, + limit=1000) -> transform_result([{op: yoy}, {op: contribution_to_change, of: revenue}, + {op: sort, by: revenue_delta, direction: asc}, {op: head, n: 20}]) -> + make_table(columns=[order_partner, revenue, revenue_ly, revenue_delta, revenue_delta_share_pct]) + guard: "yoy needs date_from/date_to; contribution_to_change needs the _delta yoy adds first." + + - ask: "distribution / outliers of [order value|customer size]" + plan: > + run_semantic_query(group_by=[entity], measures=[revenue], limit=1000) -> + transform_result([{op: describe, of: revenue}]) -> make_table (or {op: outliers, of: revenue, + method: iqr} then filter_rows revenue_outlier == true, or {op: bin} -> histogram) + guard: "describe/outliers operate on per-entity values — group by the entity first, not a trend." + + - ask: "crosstab / pivot [revenue] by [month] x [business unit]" + plan: > + run_semantic_query(measures=[revenue], grain=month, group_by=[team]) -> + transform_result([{op: pivot, index: period, column: team, value: revenue}]) -> make_table + guard: > + pivot is a pure reshape — if it RAISES a collision, the query grain is too coarse (two rows + per cell); regroup the query. It never hides a sum. + + - ask: "is [X] correlated with [Y]? / relationship between two measures" + plan: > + run_semantic_query(group_by=[entity], measures=[X, Y], limit=1000) -> + transform_result([{op: correlate, x: X, y: Y}]) -> report r; for the picture make_chart( + kind=scatter, x=X, y=Y). regression{x,y} adds slope/intercept/r2. + guard: "correlation is not causation — say so; name the outliers rather than hiding them." + + - ask: "is [this month's revenue] a REAL signal or just noise? / control chart / XmR" + plan: > + run_semantic_query(measures=[revenue], grain=month, date_from, date_to) -> + transform_result([{op: xmr_limits, of: revenue}]) -> make_chart(kind=combo, x=period, + y=revenue, y2=revenue_ucl) and report the _signal points as the real signals + guard: > + XmR (mean ± 2.66·mean-moving-range) is how this business separates signal from noise + (warehouse/expenses use it). A point is only a SIGNAL if _signal is true — don't call a + within-limits wiggle a trend. + + - ask: "[any trend/cumulative question on a filtered or sparse monthly series]" + plan: > + run_semantic_query(measures=[...], grain=month, ...) -> transform_result([{op: resample, + grain: month}, {op: running_total, of: revenue}]) # resample FIRST + guard: > + if a month has no rows (filtered scope, a quiet SKU), the window ops (running_total, + moving_average, diff, ytd) would silently skip the gap and misalign. resample inserts the + missing months as None first so the series is honest. + +context: > + This library is the analytics half of the tool registry (charting.skill.yml is the viz half). + The additivity law is absolute: additive = revenue/units/margin/cogs/orders; NOT additive = + customers (distinct) and margin_pct/aov (ratios). When an accumulating op errors on additivity, + switch to the re-query op (ytd/rolling) — do not work around it. rfm needs recency (days since + last order), which is not yet a semantic metric — frequency=orders and monetary=revenue exist, + so if asked for RFM today, deliver F+M scoring and report recency as the missing input (a gap). + Every derived result keeps its result_id and drills to rows; saved views replay the whole chain. diff --git a/royalimports_os/model/skills/charting.skill.yml b/royalimports_os/model/skills/charting.skill.yml new file mode 100644 index 0000000000000000000000000000000000000000..b8b467e25f66d5f8cb4c42777cdb455887e46965 --- /dev/null +++ b/royalimports_os/model/skills/charting.skill.yml @@ -0,0 +1,108 @@ +# In-product SKILL: charting + analytics recipes (v1, 2026-07-16) — the visualization half of the +# tool registry. Teaches the small model WHICH form answers WHICH comparison (Zelazny: the form +# follows the comparison) and HOW to reach it: run_semantic_query -> transform_result (optional, +# governed) -> make_chart | make_table. The model NEVER computes derived numbers itself and NEVER +# emits HTML/vega — it picks a kind; the platform owns the pixels and every chart drills to its +# exact rows (result_id). +# +# This file versions TOGETHER with harness/tools.py (CHART_KINDS + transform ops) — a kind or op +# named here must exist there, and vice versa. +key: charting +label: Charts, tables, and analytics transforms + +kinds: [line, bar, area, scatter, map, pie, donut, stacked_bar, grouped_bar, ranked_bar, + stacked_pct, combo, yoy_bars, waterfall, pareto, histogram, heatmap, treemap, funnel, + bullet, bubble, sparkline] +transforms: "the full 60-op analytics library (table calcs, stats, business templates, re-query + windows) lives in analytics.skill.yml — this file covers the viz forms they feed" + +recipes: + - ask: "top [N] [customers|products|states] as a chart" + plan: > + run_semantic_query(group_by=[dim], sort=-measure, limit=high) -> + transform_result([{op: top_n, by: measure, n: N, other: false}]) -> + make_chart(kind=ranked_bar, x=dim, y=measure) + guard: > + for a RANKING use other:false (a giant 'Other' bar crushes the scale — the platform + captions what was cut); use other:true when the chart claims a share of the whole + (donut, stacked). Never truncate without the transform. + + - ask: "how concentrated is [revenue|margin]? / do a few customers carry the book?" + plan: > + run_semantic_query(group_by=[dim], sort=-measure, limit=high) -> + make_chart(kind=pareto, x=dim, y=measure) # platform draws bars + cumulative-% line + guard: "pareto reads concentration; for a plain ranking use ranked_bar instead" + + - ask: "share/mix of [revenue] by [category|BU] — part-to-whole" + plan: > + run_semantic_query(group_by=[dim]) -> if >6 groups: transform_result top_n -> + make_chart(kind=donut|pie, x=dim, y=measure). Mix OVER TIME: group_by dim + grain -> + make_chart(kind=stacked_bar, x=period, y=measure, series=dim); if the question is about + SHARES not levels -> stacked_pct. Hierarchical share -> treemap. + guard: "pie/donut only <=6 slices; percentages must come from the rows, never estimated" + + - ask: "this year vs last year [by month] — YoY comparison" + plan: > + run_semantic_query(grain=month, date_from/date_to = this year) -> + transform_result([{op: yoy}]) -> make_chart(kind=yoy_bars, x=period, y=revenue) + guard: > + yoy re-runs the SAME governed query shifted -1 year; needs explicit date_from/date_to. + The platform prints the YoY % on the chart — never hand-compute deltas. + + - ask: "what drove the change / bridge [revenue] from X to Y" + plan: > + get a result whose rows are SIGNED contributions (e.g. yoy transform then compute per-group + delta is NOT available — query each component) -> make_chart(kind=waterfall, x=label, + y=amount). The platform appends the Total bar. + guard: "steps must sum to the change being explained; if they don't, say what's missing" + + - ask: "distribution — how are [order values|customer sizes] spread?" + plan: > + run_semantic_query(group_by=[entity dim], limit=high) -> + make_chart(kind=histogram, x=measure) | or transform_result([{op: bin, of: measure, + bins: 12}]) -> make_chart(kind=bar, x=bucket, y=count) when you want the buckets as rows + guard: "histogram bins per-entity values — group by the entity first, never bin a trend" + + - ask: "is [X] related to [Y]? (two measures per entity)" + plan: > + run_semantic_query(group_by=[entity], measures=[X, Y]) -> make_chart(kind=scatter, x=X, + y=Y); a third measure sizes the dots -> kind=bubble, size=Z + guard: "state that correlation is visual, not causal; outliers get named in the answer" + + - ask: "level AND rate together (GM$ + GM%, revenue + orders)" + plan: > + run_semantic_query(grain=month, measures=[level, rate]) -> + make_chart(kind=combo, x=period, y=level, y2=rate) + guard: "y = the dollar level (bars, left); y2 = the rate/count (line, right)" + + - ask: "smoothed trend / running total / share columns" + plan: > + transform_result ops: moving_average{of, window} (adds _maN) · running_total{of} · + share_of_total{of} · rank{by} — then chart the derived result (e.g. line of the _ma3 + column, or make_table with the share column) + guard: "the first window-1 moving-average points are None by design — say so if asked" + + - ask: "pipeline/stage conversion; actual vs target" + plan: > + funnel: rows in stage order -> make_chart(kind=funnel, x=stage, y=value). + target: make_chart(kind=bullet, x=label, y=actual, y2=target) + guard: "funnels need a real stage sequence; never reorder stages by size" + + - ask: "same mini-chart per [BU|category] — compare shapes side by side" + plan: > + run_semantic_query(grain=month, group_by=[dim]) -> + make_chart(kind=line, x=period, y=measure, facet=dim) + guard: "facet shares scales across panels so shapes compare honestly" + + - ask: "show me the numbers / a list / exact figures" + plan: > + run_semantic_query(...) -> make_table(result_id, columns=[...], title=?) + guard: > + make_table renders house-formatted with a totals row — prefer it over pasting rows into + the answer text whenever there are more than ~5 rows or 3 columns + +context: > + Every chart/table keeps its result_id and semantic query — the platform renders a + "Data behind" drill for each one, and saved views re-run query + transforms live. If a form + needs a column the result lacks, transform first (the error message names valid columns). + When two forms could work, pick the simpler; never decorate. diff --git a/royalimports_os/model/skills/sales.skill.yml b/royalimports_os/model/skills/sales.skill.yml new file mode 100644 index 0000000000000000000000000000000000000000..a5020ff69510e49a9c7014a7c64c59393c05564b --- /dev/null +++ b/royalimports_os/model/skills/sales.skill.yml @@ -0,0 +1,184 @@ +# In-product SKILL: sales analysis recipes (v0, 2026-07-11) — the first "strong skills for +# small-model AI" (the productization directive; omni-adoption addendum §3). +# +# WHAT THIS IS: short task recipes a SMALL model loads before acting. Each recipe maps a user ask +# to an explicit TOOL-CALL PLAN over the semantic layer — the model navigates by recipe + changes +# parameters, it never invents joins, scope, or SQL. This file is ALSO the binding spec for the +# OM-4 tool registry: the tools named here (run_semantic_query, make_chart, make_kpi_card, +# get_field_values, compose_dashboard) are the signatures OM-4 +# implements — recipes and tools ship and version TOGETHER (drift between them = a build error). +# +# Recipe fields: ask (the intent pattern, [brackets] = user-changeable parameters) · plan (the +# tool calls, in order) · params (what the model may vary + allowed values) · guard (what it must +# never do here). Every numeric answer inherits the platform rule: it must carry its drill link. +key: sales +label: Sales analysis +topic: sales_lines # default topic; recipes may name another (sales_orders) +metrics: [revenue, units, margin, cogs, margin_pct, orders, customers, aov] + +recipes: + - ask: "revenue trend [monthly|weekly] [this year|LTM|a date range] [for Fisch|Royal|both]" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue], grain=month|week, + date_from=?, date_to=?, team_id=5|6|null) -> make_chart(kind=line, x=grain, y=revenue) + params: {grain: [month, week], team_id: [5, 6, null], window: "any date range"} + guard: "one BU per series; if the user asks 'both', emit two series (team_id=5 and 6), never a mixed line" + + - ask: "top [N] customers [by revenue|margin] [window] [BU]" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue, margin], group_by=order_partner, + date_from=?, date_to=?, team_id=?, sort=-revenue, limit=N) -> make_chart(kind=bar) + params: {N: "5-50 (default 15)", sort: [revenue, margin]} + guard: "group_by uses DIRECT m2o fields only (order_partner, product) — dot-path groupings fail on this platform" + + - ask: "top [N] products/SKUs [window] [BU]" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue, units, margin], group_by=product, + date_from=?, date_to=?, team_id=?, sort=-revenue, limit=N, exclude_services=true) + -> make_chart(kind=bar) + guard: "always exclude_services=true for SKU rankings (Delivery Charges pollutes top movers)" + + - ask: "what gross margin [%] did we run [window]? (NO business-unit words in the ask)" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue, margin, margin_pct], + date_from=?, date_to=?) # NO group_by — a plain margin question wants the CONSOLIDATED figure + guard: "only split by BU when the user says 'by business unit' / 'per BU' / 'Fisch vs Royal'" + + - ask: "gross margin [%] BY BUSINESS UNIT / per BU / Fisch vs Royal [window]" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue, margin, margin_pct], + group_by=team, date_from=?, date_to=?) -> make_chart(kind=bar, y=margin_pct) + guard: "margin_pct is a ratio — computed at scope, never averaged from sub-rows" + + - ask: "how are sales doing [window]? / sales KPIs" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue, margin_pct, orders, aov], + date_from=?, date_to=?, team_id=?) -> make_kpi_card(each metric, with same-window-last-year + deltas via a second query shifted -1 year) + guard: "deltas compare IDENTICAL windows year-over-year; never partial-vs-full periods" + + - ask: "revenue by state / by product category / where do we sell the most" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue], group_by=[state|category], + sort=-revenue) -> make_chart(kind=bar, x=state|category, y=revenue) + guard: > + 'state' = the customer's billing state; 'category' = the product's category, where the + root bucket literally named "All" is UNCATEGORIZED product — say so when it ranks + + - ask: "top customers BASED IN / located in [New York|a state|a city]" + plan: > + get_field_values(topic=sales_lines, dim=state|city, search=) -> run_semantic_query( + measures=[revenue], group_by=[order_partner], filters={state|city: [ids]}, sort=-revenue, + limit=N, date_from=?, date_to=?) -> make_table or ranked_bar + guard: > + location = the state/city dims, NEVER a customer-name search ('WEST NEW YORK FLORIST' is + not necessarily in New York). "New York" is ambiguous — check BOTH: if the state and the + city both match, ask which one (or default to the state and say so). City master data is + MESSY (case/trailing-space variants: 'BROOKLYN'/'Brooklyn'/'Brooklyn ') — include EVERY + matching variant from get_field_values in the filter. Always pass an explicit window and + state it. + + - ask: "[salesperson/agent]'s customers / sales / top customers for [Naomi|an agent] [window]" + plan: > + get_field_values(topic=sales_lines, dim=agent, search=) -> run_semantic_query( + topic=sales_lines, measures=[revenue], group_by=[order_partner], filters={agent: [id]}, + date_from=?, date_to=?, sort=-revenue, limit=N) -> make_table or ranked_bar + guard: > + agent = the CUSTOMER's assigned sales agent (book attribution: every order of that + customer counts toward their agent) — resolve the agent's real name via get_field_values + on the agent dim FIRST, never from customer names. For the agent's TOTAL (not per + customer) drop the group_by. State the window. + + - ask: "revenue [or orders] BY AGENT / how did the agents do / rank the agents [window]" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue, margin], group_by=[agent], + date_from=?, date_to=?, sort=-revenue) -> make_chart(kind=ranked_bar) or make_table + guard: > + the NULL agent group is customers with NO agent assigned — present it as "(no agent)" + and keep it in totals (it is usually large: house accounts). Order counts by agent run + on topic=sales_orders with the same agent dim. + + - ask: "which customers are DECLINING / negative YoY / dropped the most [BU] [window]" + plan: > + run_semantic_query(measures=[revenue], group_by=[order_partner], team_id=?, date_from=?, + date_to=?, limit=1000) -> transform_result([{op: yoy}, {op: sort, by: revenue_delta, + direction: asc}, {op: head, n: 20}]) -> make_table(columns=[order_partner, revenue, + revenue_ly, revenue_delta, revenue_yoy_pct]) + guard: > + YoY per customer comes ONLY from the yoy transform (adds revenue_ly, revenue_delta, + revenue_yoy_pct per row) — never join two queries by hand. sort revenue_delta asc + head + = biggest $ decline first. yoy needs explicit date_from/date_to on the source query. + + - ask: "export to Excel / send me a spreadsheet / CSV" + plan: > + run_semantic_query(...) -> make_table(result_id, title=?) — then tell the user: the table + below has a 'Download CSV (opens in Excel)' button under it + guard: > + you cannot SEND files or email; the platform renders a download button under every chart's + data expander and every table. Never claim to have sent anything. + + - ask: "map my customers / where are my [top] customers [by revenue]" + plan: > + run_semantic_query(topic=sales_lines, measures=[revenue], group_by=[order_partner], + sort=-revenue, limit=20..100) -> make_chart(kind=map, x=order_partner, y=revenue) + guard: > + the platform places each customer at its geocoded address, dot size = the measure; rows + without a geocoded address are skipped and counted on screen — no need to fetch + coordinates yourself + + - ask: "compare this year to last year [monthly] [BU]" + plan: > + run_semantic_query(... grain=month, window=this year) + run_semantic_query(... window=last + year shifted) -> make_chart(kind=line, two series: this year, last year) + + - ask: "which customer/product values are valid? (before filtering by name)" + plan: > + get_field_values(topic=sales_lines, field=order_partner|product, search=?) — resolve the + user's spelling to the real record BEFORE querying ('NYC Florist' vs the actual name) + guard: "never filter on a free-typed name; always resolve via get_field_values first" + + - ask: "save this as a dashboard / add to my dashboard" + plan: > + save_view(each query above, name=?) -> compose_dashboard(views=[...], name=?) + guard: > + pass save_view the EXACT object make_chart/make_kpi_card returned (it carries the query — + saved views RE-RUN live; a spec without its query is rejected). After compose_dashboard, + tell the user the dashboard is now in the sidebar Workflows menu as 'Saved · ' + and stays current (every open re-runs the queries). + + - ask: "change/edit that view or workbook — different window, add a breakdown, rename, reorder" + plan: > + list_workspace() to find the exact name -> update_view(name, changes={title?/kind?/x?/y?/ + series?, query: {date_from?/date_to?/group_by?/grain?/filters?/limit?/...}}) for a view | + update_workbook(name, views=[new order], new_name=?) for a workbook (dashboard) + guard: > + UPDATE tools modify existing objects — state exactly what will change and confirm first; + update_view re-runs the patched query before saving, so report its row_count back. To + REMOVE a query key (e.g. drop a group_by), pass it as null. + + - ask: "delete that view/dashboard/alert/report" + plan: "list_workspace() to resolve the exact kind+name -> delete_object(kind, name)" + guard: > + DESTRUCTIVE — never call delete_object unless the user explicitly confirmed THE NAMED + object in this conversation; deleting a view also removes it from workbooks that use it + + - ask: "what dashboards/views/alerts do I have? / reuse that template" + plan: > + list_workspace() -> answer from the objects list; to reuse a template: + instantiate_template(filename, new_name=?) — it stamps NEW objects, never overwrites + guard: > + instantiate_template is an ARTIFACT tool — confirm with the user first; afterwards point + them to the new dashboard in the sidebar Workflows menu ('Saved · ') + + - ask: "send me this every [Monday|month-start] / watch this KPI / alert me about X" + plan: > + (retired 2026-07-23 with the Routines module) say scheduled reports and KPI alerts are not + offered right now; offer the live alternative — save the view/dashboard so it re-runs + fresh on every open, and answer the underlying question NOW with run_semantic_query + guard: "never claim a schedule or alert was created; the schedule_report/create_alert/schedule_insight tools no longer exist" + +context: > + All recipes inherit the sales_lines topic scope (confirmed wholesale orders, untaxed amounts, + Amazon/GIFTWARE excluded, BUs strictly isolated — see the topic's ai_context). Order COUNTS come + from the sales_orders topic (order grain). Every number answered must link its drill-down rows; + if a query returns something that cannot be drilled, say so instead of presenting it. diff --git a/royalimports_os/model/tenant.yml b/royalimports_os/model/tenant.yml new file mode 100644 index 0000000000000000000000000000000000000000..98987a59a38a2431b595062c4a4670d965de2bac --- /dev/null +++ b/royalimports_os/model/tenant.yml @@ -0,0 +1,13 @@ +# Tenant configuration — the ONE place a customer's specifics live (OM-2 parameterization, +# 2026-07-11). Topics reference these as {param} placeholders in store scope_sql; everything else +# in model/ is TENANT-AGNOSTIC (= the template library, task #13). Onboarding a new customer = +# write THIS file (+ .env credentials) — the topics/metrics/skills stamp in unchanged. +key: royal_imports +label: Royal Imports / Fisch Floral Supply +params: + # Business units: the sales teams that constitute the wholesale business (BU strict isolation). + wholesale_team_ids: [5, 6] + team_names: {5: Fisch, 6: Royal} + # Channel exclusions: partners excluded from WHOLESALE scope (Amazon rides through this partner; + # all-channel topics — GL, receivables — deliberately do NOT apply it). + excluded_partner_name_like: "GIFTWARE DEALS%" diff --git a/royalimports_os/model/topics/commission_lines.yml b/royalimports_os/model/topics/commission_lines.yml new file mode 100644 index 0000000000000000000000000000000000000000..9ae8ba80dad2050578d3a16609ea7d869ec1c2e5 --- /dev/null +++ b/royalimports_os/model/topics/commission_lines.yml @@ -0,0 +1,66 @@ +# Topic: commission_lines — the raw commission rows, one per (invoice line × agent). +# +# Separate from `invoice_lines` on purpose. `invoice_lines` joins the commission table +# DEDUPLICATED (at most one agent per line) so that revenue never double-counts. That is right +# for revenue and WRONG for commission: a line with two agents accrues two commission amounts, +# and the deduplicated join would drop one. So commission money is summed here, at its own grain. +# +# Use `invoice_lines` for revenue-by-agent; use this ONLY for commission accrued / settlement. +key: commission_lines +label: Commission lines (agent × invoice line) +entity: account.invoice.line.agent + +# STORE-ONLY: no live-path domain builder. + +scope: + documents: >- + every commission row, including those attached to non-product lines. ⚠ 96,936 of the 147,160 + rows with a 2025 invoice_date hang off COGS lines rather than product lines, so a row count + here is NOT a count of agent-attributed sales lines. + basis: "commission amount in company currency (`amount`)" + date_field: "invoice_date" + +grain: "one row per (invoice line × agent); a line with two agents produces two rows" + +store: + table: account_invoice_line_agent + alias: a + # object_id -> account_move_line (the LINE), invoice_id -> account_move (the DOCUMENT). + # Two similarly named FKs; joining the wrong one silently changes the grain. + join: >- + LEFT JOIN res_partner ap ON ap.id = a.agent_id + LEFT JOIN account_move_line l ON l.id = a.object_id + LEFT JOIN sale_order_line sol ON sol.id = l.sale_line_id + LEFT JOIN sale_order so ON so.id = sol.order_id + date_col: "a.invoice_date" + team_col: "so.team_id" + scope_sql: "1 = 1" + dims: + commission_name: {col: "a.agent_id", name_col: "a.agent_name", label: "Commission name"} + plan: {col: "a.commission_id", name_col: "a.commission_name", label: "Commission plan"} + settled: {col: "a.settled", name_col: "CASE WHEN a.settled THEN 'Settled' ELSE 'Unsettled' END", label: "Settled"} + agent_role: + col: "CASE WHEN COALESCE(ap.agent, FALSE) THEN 'agent' ELSE 'salesperson' END" + name_col: "CASE WHEN COALESCE(ap.agent, FALSE) THEN 'Agent' ELSE 'Salesperson (not an agent)' END" + label: "Role" + line_type: + col: "l.display_type" + name_col: "l.display_type" + label: "Invoice line type" + team: + col: "so.team_id" + name_col: "CASE so.team_id WHEN 5 THEN 'Fisch' WHEN 6 THEN 'Royal' WHEN 7 THEN 'Giftware Deals (Amazon)' WHEN 1 THEN 'Sales (generic)' ELSE 'No sale order' END" + label: "Business unit" + +ai_context: > + Raw commission rows from the OCA sale-commission module — one row per invoice line per agent. + Use ONLY for commission money ("what commission did X accrue", settlement status, plan mix). + For revenue by agent use topic `invoice_lines`, which joins this table deduplicated so revenue + is not double-counted. + ⚠ ROW COUNTS HERE ARE NOT SALES-LINE COUNTS: commission rows attach to COGS lines as well as + product lines (96,936 of 147,160 rows dated 2025 are COGS). Filter `line_type` = 'product' + before comparing anything here to a sales figure. + ⚠ A NAME HERE IS NOT NECESSARILY AN AGENT — "Anna" and "Shantal Erlich" are internal + salespeople (res.partner.agent = False) who carry commission rows. Use the `agent_role` dim to + separate them; never call them agents. + Commission PLANS observed: 6.5% (Martin Pasternak), 8%, 5%, 2%, 1%, and a named "Jacob" plan. diff --git a/royalimports_os/model/topics/credit_notes.yml b/royalimports_os/model/topics/credit_notes.yml new file mode 100644 index 0000000000000000000000000000000000000000..68d2f82301cbeb8a048cd8d8db78e553050b1c3f --- /dev/null +++ b/royalimports_os/model/topics/credit_notes.yml @@ -0,0 +1,48 @@ +# Topic: credit_notes — posted customer CREDIT NOTES (returns), at document grain. +# +# Why its own topic rather than a filtered measure on a shared one: a FILTERED measure +# (`store_filter_sql`) exists only on the store path, and `semantic.metric()` resolves a sum by +# handing the metric's `field` to `O.sum_field(entity, domain, field)` with no room for a +# per-measure filter. A metric that can be computed on one path and not the other cannot be +# parity-checked — and parity is the admission bar for filtering on it. Two topics with two +# DOMAINS give both paths the same scope by construction. +key: credit_notes +label: Customer credit notes (returns) +entity: account.move +domain_builder: credit_note_domain + +scope: + documents: "posted customer credit notes only (move_type = out_refund, state = posted)" + channels: >- + COMPANY-LEVEL. Credit notes carry team_id = 1 for everything, so a business-unit filter on + the document is meaningless — modules/returns.py says the same and attributes by the + CUSTOMER instead. A BU-scoped caller is refused rather than given a company number wearing + their name. + excluded: >- + Nothing. modules/returns.py — the number the business already reads — does not exclude the + GIFTWARE DEALS partner the wholesale sales topics do. Matching it keeps one definition of + "returns"; the customer-grid pool is wholesale anyway, so the intersection makes the + difference unobservable there. + basis: "untaxed document amounts (amount_untaxed_signed, NEGATED — see the metric)" + date_field: "invoice_date" + +grain: "one row per posted credit note; time-filterable by invoice_date" + +store: + table: account_move + alias: m + join: "LEFT JOIN res_partner rp ON rp.id = m.partner_id" + date_col: "m.invoice_date" + scope_sql: "m.state = 'posted' AND m.move_type = 'out_refund'" + dims: + partner: {col: "m.partner_id", name_col: "m.partner_name", label: "Customer"} + +ai_context: > + Posted customer credit notes — refunds/returns. ⚠ `amount_untaxed_signed` is NEGATIVE on a + credit note (verified on the tenant store 2026-07-27: all 3,006 posted refunds are negative, + summing to -358,141.70), so the `returns` metric declares `negate: true` and reports a POSITIVE + amount. A returns figure that comes out negative means the negation was lost, and it will read + as "we have no returns" rather than as an error. This topic is COMPANY-LEVEL: credit notes are + not business-unit tagged, so never present a returns number as belonging to Fisch or Royal. + Odoo has no return-REASON field, so concentration (which customers, which SKUs) is the only + diagnostic available. diff --git a/royalimports_os/model/topics/customer_invoices.yml b/royalimports_os/model/topics/customer_invoices.yml new file mode 100644 index 0000000000000000000000000000000000000000..10878e21e73528f5bf88a07de133fcf227a286fe --- /dev/null +++ b/royalimports_os/model/topics/customer_invoices.yml @@ -0,0 +1,38 @@ +# Topic: customer_invoices — posted customer INVOICES, at document grain. +# +# It exists to be the DENOMINATOR of the return rate. `modules/returns.py` rates refunds against +# INVOICED totals, not against order-line revenue, and that is the number the business already +# reads — so Return % uses the same denominator rather than inventing a second one that would be +# close enough to look right and different enough to argue about. +# +# ⚠ This is NOT `revenue`. `revenue` is order-LINE subtotals of confirmed sales orders (topic +# sales_lines); this is posted invoice documents. They differ by unbilled orders, by billing +# timing, and by anything invoiced outside the wholesale sales scope. +key: customer_invoices +label: Customer invoices +entity: account.move +domain_builder: customer_invoice_domain + +scope: + documents: "posted customer invoices only (move_type = out_invoice, state = posted)" + channels: "COMPANY-LEVEL — see credit_notes; the same reasoning applies to the denominator." + excluded: "Nothing, matching credit_notes and modules/returns.py." + basis: "untaxed document amounts (amount_untaxed_signed, already positive for an invoice)" + date_field: "invoice_date" + +grain: "one row per posted customer invoice; time-filterable by invoice_date" + +store: + table: account_move + alias: m + join: "LEFT JOIN res_partner rp ON rp.id = m.partner_id" + date_col: "m.invoice_date" + scope_sql: "m.state = 'posted' AND m.move_type = 'out_invoice'" + dims: + partner: {col: "m.partner_id", name_col: "m.partner_name", label: "Customer"} + +ai_context: > + Posted customer invoices, the denominator of the return rate. `amount_untaxed_signed` is + already POSITIVE here (unlike a credit note), so this metric carries no negation. Do not use + this as "sales" — that is `revenue` on sales_lines, which is order-line based; this is billing. + Company-level: invoices in this topic are not business-unit filterable. diff --git a/royalimports_os/model/topics/gl_lines.yml b/royalimports_os/model/topics/gl_lines.yml new file mode 100644 index 0000000000000000000000000000000000000000..5362dd9d10e54d45dca2aabd541986c4f74b4707 --- /dev/null +++ b/royalimports_os/model/topics/gl_lines.yml @@ -0,0 +1,34 @@ +# Topic: gl_lines — the RAW general-ledger dataset (posted journal lines). Company-level (no BU +# filter — GL isn't team-scoped). BASIS RULE (the glaccount-vs-account lesson, now model law): +# this is the RAW-GL basis; NEVER mix it with the management-basis P&L in one analysis. +key: gl_lines +label: General ledger (raw, posted) +entity: account.move.line +# store-only topic: no live-path domain builder yet (lands with the Expenses retrofit); +# store_query() is authoritative; the parity check below compares against a DIRECT live aggregate +scope: + posted_only: "journal entries with state = posted only" + basis: "RAW GL — debit-positive balance; expenses positive, income negative (negated in metrics)" + level: "company-level (HQ) — no business-unit split" + date_field: "accounting date (account.move.line.date)" +grain: "one row per posted journal line; time-filterable by accounting date" + +store: + table: account_move_line + alias: l + join: "JOIN account_move m ON m.id = l.move_id JOIN account_account a ON a.id = l.account_id" + date_col: "l.date" + scope_sql: "m.state = 'posted'" + dims: + account: {col: "l.account_id", name_col: "l.account_name", label: "GL account"} + account_type: {col: "a.account_type", name_col: null, label: "Account type"} + partner: {col: "l.partner_id", name_col: "l.partner_name", label: "Partner"} + +ai_context: > + The raw posted general ledger. Use for operating-expense analysis (opex), GL-basis revenue, and + ledger drill-downs — the same basis as the Expenses dashboard. Expense-type accounts + ('expense', 'expense_depreciation') carry opex; 'expense_direct_cost' is COGS and is EXCLUDED + from opex by definition. Income accounts store credit-negative balances; the gl_revenue metric + negates them to read positively. This topic is company-level: business-unit filters do not + apply here (use sales_lines for BU questions). The newest month is often still being booked — + a sharply lower final month usually means an OPEN period, not a cost win (close-lag). diff --git a/royalimports_os/model/topics/invoice_lines.yml b/royalimports_os/model/topics/invoice_lines.yml new file mode 100644 index 0000000000000000000000000000000000000000..7959f2927fe55f22c0a9dadf4236b78b097988bf --- /dev/null +++ b/royalimports_os/model/topics/invoice_lines.yml @@ -0,0 +1,141 @@ +# Topic: invoice_lines — posted customer-invoice lines at LINE grain, carrying the per-line +# COMMISSION agent (the OCA sale-commission module). +# +# WHY THIS EXISTS (owner ask, 2026-07-28): "which FISCH customers are assigned to an Agent based +# on the invoice line, what do they generate, and what sales are NOT allocated to an agent." +# Neither `sales_lines` (order grain, customer-master agent) nor `customer_invoices` (document +# grain, no agent at all) can answer that — the agent lives on the invoice LINE, and the +# unallocated bucket is precisely the lines that have no commission row. +# +# ⚠ THIS IS A SECOND AGENT SOURCE AND IT DISAGREES WITH `sales_lines.agent`. That one is the +# CUSTOMER-MASTER book (res.partner.agent_ids); this one is per-line commission attribution. +# They name different people. Always state which source a number came from. +key: invoice_lines +label: Customer invoice lines (with commission agent) +entity: account.move.line + +# STORE-ONLY: no live-path domain builder. The live equivalent is a dot-path domain on +# account.move.line; it is deliberately not bound here because the agent join has no live analogue +# that stays honest about the unallocated bucket. + +scope: + documents: "posted customer invoices AND credit notes (move_type out_invoice/out_refund, parent_state posted)" + lines: >- + PRODUCT lines only (display_type = 'product'). Commission rows attach to 'cogs' lines too — + 96,936 of the 147,160 rows with a 2025 invoice_date. ⚠ MEASURED 2026-07-29, and NOT what it + first appears: dropping this filter does NOT move revenue or the customer count, because + non-product lines ('cogs', 'payment_term', 'line_note') all carry price_subtotal = 0 and + those customers already have product lines. The ~80% inflation is real for commission ROW + COUNTS (147,160 vs 50,191 product-line rows) and would corrupt any count-of-LINES measure — + it is a GRAIN guard, not a money guard. Keep it; just don't claim it protects the totals. + channels: >- + business unit comes from the ORIGINATING SALE ORDER (Fisch=5, Royal=6), because + account.move carries no BU (every invoice sits on team 1 "Sales"). Lines with no sale order + (direct invoices) have a NULL business unit and are KEPT, not dropped. + excluded: >- + Nothing is excluded by partner. GIFTWARE DEALS (Amazon) sits on its own team (7), so a + Fisch or Royal filter already excludes it — no partner-name exclusion needed here. + basis: "untaxed line subtotals (price_subtotal); credit-note lines are NEGATED by the metric" + date_field: "line date (= invoice date)" + +grain: >- + one row per posted customer-invoice product line. The commission table is one row per + (line × agent), so it is joined DEDUPLICATED to at most one row per line — otherwise the 74 + two-agent lines would double-count revenue. + +store: + table: account_move_line + alias: l + # LEFT JOIN from the LINE side is load-bearing: the whole point is the lines with NO commission + # row. An inner join would silently delete the unallocated bucket — the second half of the + # question — and still look correct. + # + # ca = at most ONE commission row per line, preferring a TRUE agent over a salesperson, then + # the lowest agent id for determinism. + join: >- + LEFT JOIN res_partner rp ON rp.id = l.partner_id + LEFT JOIN sale_order_line sol ON sol.id = l.sale_line_id + LEFT JOIN sale_order so ON so.id = sol.order_id + LEFT JOIN ( + SELECT object_id, agent_id, agent_name, is_true_agent, is_converted_salesman + FROM ( + SELECT a.object_id, a.agent_id, a.agent_name, + COALESCE(ap.agent, FALSE) AS is_true_agent, + COALESCE(ap.salesman_as_agent, FALSE) AS is_converted_salesman, + ROW_NUMBER() OVER (PARTITION BY a.object_id + ORDER BY COALESCE(ap.agent, FALSE) DESC, a.agent_id) AS rn + FROM account_invoice_line_agent a + LEFT JOIN res_partner ap ON ap.id = a.agent_id + ) ranked WHERE rn = 1 + ) ca ON ca.object_id = l.id + date_col: "l.date" + team_col: "so.team_id" + scope_sql: >- + l.parent_state = 'posted' AND l.move_type IN ('out_invoice','out_refund') + AND l.display_type = 'product' + dims: + partner: {col: "l.partner_id", name_col: "l.partner_name", label: "Customer"} + product: {col: "l.product_id", name_col: "l.product_name", label: "Product"} + team: + col: "so.team_id" + name_col: "CASE so.team_id WHEN 5 THEN 'Fisch' WHEN 6 THEN 'Royal' WHEN 7 THEN 'Giftware Deals (Amazon)' WHEN 1 THEN 'Sales (generic)' ELSE 'No sale order' END" + label: "Business unit" + # THE agent dim — real agents ONLY. A salesperson-only line lands here as NULL, which is + # correct: it is not allocated to an agent. + agent: + col: "CASE WHEN ca.is_true_agent THEN ca.agent_id END" + name_col: "CASE WHEN ca.is_true_agent THEN ca.agent_name END" + label: "Agent (invoice line)" + # Whoever is on the commission line, agent or not — for commission/payout questions. + commission_name: + col: "ca.agent_id" + name_col: "ca.agent_name" + label: "Commission name (agent OR salesperson)" + agent_role: + col: "CASE WHEN ca.agent_id IS NULL THEN 'none' WHEN ca.is_true_agent THEN 'agent' ELSE 'salesperson' END" + name_col: "CASE WHEN ca.agent_id IS NULL THEN 'Nobody' WHEN ca.is_true_agent THEN 'Agent' ELSE 'Salesperson (not an agent)' END" + label: "Role on the line" + # The partition the owner asked for. These three values are MECE and sum to the total. + allocation: + col: "CASE WHEN ca.is_true_agent THEN 'agent' WHEN ca.agent_id IS NOT NULL THEN 'salesperson' ELSE 'none' END" + name_col: "CASE WHEN ca.is_true_agent THEN 'Allocated to an agent' WHEN ca.agent_id IS NOT NULL THEN 'Salesperson only' ELSE 'Not allocated' END" + label: "Agent allocation" + # The OUTSTANDING OWNER RULING (2026-07-28): 4 of the 11 agents carry salesman_as_agent = + # True ("convert salesman into agent"). Exposed as a dim so BOTH readings are queryable and + # neither is baked into the definition of `agent`. + agent_origin: + col: "CASE WHEN ca.is_true_agent AND ca.is_converted_salesman THEN 'converted' WHEN ca.is_true_agent THEN 'external' END" + name_col: "CASE WHEN ca.is_true_agent AND ca.is_converted_salesman THEN 'Agent (converted salesman)' WHEN ca.is_true_agent THEN 'Agent (external)' END" + label: "Agent origin" + +ai_context: > + Posted customer-invoice PRODUCT lines with the per-line commission agent. Use this topic for: + "which customers are assigned to an agent based on the invoice line", "what sales are NOT + allocated to an agent", commission questions, and any agent question phrased about invoices or + billing. Use `sales_lines` instead for "sales" in the ordinary sense (confirmed order lines) and + for book/relationship ownership. + ⚠ TWO AGENT SOURCES, DIFFERENT PEOPLE — say which you used. Here = per-line commission + (account.invoice.line.agent). There (`sales_lines.agent`) = the customer-master book + (res.partner.agent_ids). JACOB BRISK, Samuel Shif, Yuda Stern and Billy Petruccelli appear ONLY + on invoice lines; Moishe Rubenstein, Avi Ash and Sang Ching ONLY on the customer master. + ⚠ A NAME ON A COMMISSION LINE IS NOT NECESSARILY AN AGENT. "Anna" and "Shantal Erlich" are + internal SALESPEOPLE (res.partner.agent = False, each with an Odoo login, salesperson of record + on 1,253 / 184 confirmed orders) who carry commission lines anyway. The `agent` dim already + excludes them — they show up under `commission_name` with agent_role 'salesperson'. NEVER + report them as agents; if a question says "agent", they are not one. + THE ALLOCATION PARTITION: the `allocation` dim is MECE and sums to the total — 'Allocated to an + agent' / 'Salesperson only' / 'Not allocated'. When asked "what is not allocated to an agent", + give the strict figure (salesperson + none) AND say what the narrower no-commission-at-all + figure is, because the two readings differ materially and the questioner may mean either. + BUSINESS UNIT comes from the originating sale order, so a line with no sale order has a NULL + business unit — present that as "no sale order (direct invoice)" and keep it in company totals. + Fisch is team 5. CREDIT NOTES are included and the revenue metric negates them; a positive + refund total means the negation was lost. + MEASURED BASELINE for FISCH 2025 (use to sanity-check any answer): total $4,443,244.34 over + 68,540 lines and 979 customers, splitting exactly into $2,995,664.75 allocated to agents, + $361,198.24 salesperson-only, and $1,086,381.35 not allocated. 518 customers have a real agent + on at least one line. Martin Pasternak alone is $2,155,660.64. + A CUSTOMER IS NOT ALL-OR-NOTHING: 403 of those 518 customers are MIXED (some lines carry an + agent, some do not), so "sales BY agent-assigned customers" ($3,187,771.31) is a different + number from "sales ALLOCATED to an agent" ($2,995,664.75). Answer whichever was asked and say + which one it is; they do not subtract to each other. diff --git a/royalimports_os/model/topics/receivables.yml b/royalimports_os/model/topics/receivables.yml new file mode 100644 index 0000000000000000000000000000000000000000..d8078184c2bdc9e57d5296e10ac2744baffc26f9 --- /dev/null +++ b/royalimports_os/model/topics/receivables.yml @@ -0,0 +1,31 @@ +# Topic: receivables — customer invoices/credit notes and what remains unpaid. Company-level +# (AR is consolidated — the Collections page convention). Store-only topic (live path = the AR +# module's own domains; the validator compares against a direct live aggregate). +key: receivables +label: Receivables (customer invoices) +entity: account.move + +scope: + posted_only: "posted customer documents only (out_invoice + out_refund)" + basis: "amount_residual_signed nets credit notes (+ for invoices, − for refunds)" + level: "company-level (HQ) — receivables are consolidated, no BU split" + date_field: "invoice date; overdue judged by invoice_date_due vs today" +grain: "one row per posted customer invoice/credit note" + +store: + table: account_move + alias: m + join: "" + date_col: "m.invoice_date" + scope_sql: "m.state = 'posted' AND m.move_type IN ('out_invoice', 'out_refund')" + dims: + partner: {col: "m.partner_id", name_col: "m.partner_name", label: "Customer"} + payment_state: {col: "m.payment_state", name_col: null, label: "Payment state"} + +ai_context: > + Customer receivables at document grain. "Outstanding" = the signed residual (credit notes net + against invoices). "Overdue" = residual on documents past their due date as of today. AR is + company-level: do not BU-filter. For who-to-call prioritization the platform's Collections + worklist is the richer surface; this topic answers totals, aging-style questions, and + per-customer exposure. Omit date filters when asking "what is outstanding NOW" — a date window + restricts by INVOICE date, which is usually not what an outstanding-balance question means. diff --git a/royalimports_os/model/topics/sales_lines.yml b/royalimports_os/model/topics/sales_lines.yml new file mode 100644 index 0000000000000000000000000000000000000000..e0b67f7b7d04b4408e62fc5fd8fde8793bafbf5b --- /dev/null +++ b/royalimports_os/model/topics/sales_lines.yml @@ -0,0 +1,106 @@ +# Topic: sales_lines — the wholesale order-line dataset (OM-0, the first topic of the semantic +# layer; see .claude/wiki/research/omni-adoption.md). A topic = the curated, scoped dataset that +# metrics, saved views and the AI Analyst all resolve against — ONE definition of what the +# data MEANS, consumed by every surface. +key: sales_lines +label: Sales lines (wholesale) +entity: sale.order.line + +# Scope binding — the ONE proven scope implementation lives in core/odoo.sale_line_domain (the +# same code every validated module uses; parity by construction). The scope: block below is the +# DECLARATIVE statement of what that builder bakes in — for humans, the metric dictionary, and the +# AI's context. When OM-2 retargets topics at the DuckDB store, these semantics become executable +# data; until then the builder is authoritative and this block must be kept true to it. +domain_builder: sale_line_domain +scope: + confirmed_only: "order state in (sale, done) — quotes and cancellations excluded" + channels: "wholesale teams only: Fisch=5, Royal=6; team_id parameter narrows to one business unit" + excluded: "partner GIFTWARE DEALS (= the Amazon channel) is excluded from wholesale scope" + basis: "line-level; monetary amounts are UNTAXED (price_subtotal)" + date_field: "order date (date_order on the parent sale.order)" + +grain: "one row per confirmed order line; time-filterable by order date; BU-filterable by team_id" + +# Executable store binding (OM-2): how this topic compiles against the DuckDB tenant store +# (harness/datastore.py tables). scope_sql MUST stay semantically identical to what the +# domain_builder bakes in — the OM-2 parity gate (store path == live path to the cent) enforces it. +store: + table: sale_order_line + alias: l + join: >- + JOIN sale_order o ON o.id = l.order_id + LEFT JOIN product_product pp ON pp.id = l.product_id + LEFT JOIN res_partner rp ON rp.id = l.order_partner_id + LEFT JOIN res_partner ag ON ag.id = rp.agent_id + date_col: "o.date_order" + team_col: "o.team_id" + scope_sql: >- + o.state IN ('sale','done') AND o.team_id IN ({wholesale_team_ids}) + AND l.order_partner_id NOT IN + (SELECT id FROM res_partner WHERE name LIKE {excluded_partner_name_like}) + service_filter: "l.product_id NOT IN (SELECT id FROM product_product WHERE type = 'service')" + dims: + order_partner: {col: "l.order_partner_id", name_col: "l.order_partner_name", label: "Customer"} + product: {col: "l.product_id", name_col: "l.product_name", label: "Product"} + # team names via CASE (crm_team is not synced; ids are baked tenant scope like the team filter) + team: {col: "o.team_id", + name_col: "CASE o.team_id WHEN 5 THEN 'Fisch' WHEN 6 THEN 'Royal' ELSE CAST(o.team_id AS VARCHAR) END", + label: "Business unit"} + # 2026-07-12: slicing dims via the synced masters (m2o fields land as _id + _name columns) + category: {col: "pp.categ_id", name_col: "pp.categ_name", label: "Product category"} + state: {col: "rp.state_id", name_col: "rp.state_name", label: "Customer state"} + city: {col: "rp.city", name_col: "rp.city", label: "Customer city"} + # 2026-07-17: the salesperson/agent dimension (the #1 logged Analyst gap). Agent = the + # CUSTOMER's assigned sales agent (res.partner.agent_ids[0] — the Customers-module + # convention, MECE), synced as res_partner.agent_id; names via self-join alias ag. + # + # ⚠ 2026-07-28: this is the CUSTOMER-MASTER (book) source, and it is ONE OF THREE. A second, + # per-INVOICE-LINE source exists (`account.invoice.line.agent`, the OCA sale-commission + # module) and names a DIFFERENT population — see topic `invoice_lines`. This dim is VERIFIED + # CLEAN: all 10 partners linked through agent_ids carry res.partner.agent = True, and 0 of + # 2,056 customers have a non-agent as agent_ids[0]. Do NOT "fix" it by reading the commission + # table, which mixes agents and salespeople. + agent: {col: "rp.agent_id", name_col: "ag.name", label: "Agent"} + +ai_context: > + The wholesale sales dataset of Royal Imports / Fisch Floral Supply, at order-LINE grain. + "Revenue" here always means untaxed line subtotals of CONFIRMED orders (state sale/done) for the + two wholesale business units (Fisch=5, Royal=6) with the GIFTWARE DEALS partner (the Amazon + channel) excluded — this is the number the owner calls "sales". The Margin module is installed: + each line carries `margin` (revenue minus cost) and `purchase_price` (per-UNIT cost, not a + total); COGS = revenue − margin. Business units are strictly isolated: a query scoped to one + team must never include the other. For inventory/SKU-lifecycle analysis use an ALL-channel + topic instead (Amazon included) — using this wholesale topic there false-flags Amazon + bestsellers as dead. Services (e.g. Delivery Charges) pollute SKU-level views; exclude + product type 'service' when ranking products. Geographic questions ("revenue by state", + "where do we sell") use the `state` dim — the customer's billing state; "based in " + uses the `city` dim. NEVER answer a location question by searching customer NAMES for the + place ("WEST NEW YORK FLORIST" is not necessarily in New York) — resolve values with + get_field_values on state/city. SALESPERSON/AGENT questions (e.g. "Naomi's customers", + "revenue by agent") use the `agent` dim: the agent is the CUSTOMER's assigned sales agent + (book attribution — every order of that customer counts toward their agent), not a per-order + salesperson. Resolve agent names with get_field_values on agent before filtering; customers + with no assigned agent appear as a NULL agent group — present it as "no agent assigned" and + keep it in totals. Never approximate an agent from customer names. + ⚠ THERE ARE TWO AGENT SOURCES IN THIS ODOO AND THEY DISAGREE — always say which one you used. + (a) THIS dim = the customer-master book (res.partner.agent_ids). (b) Topic `invoice_lines` = + the per-INVOICE-LINE commission agent (account.invoice.line.agent, the OCA sale-commission + module). They name DIFFERENT PEOPLE: JACOB BRISK, Samuel Shif, Yuda Stern and Billy Petruccelli + appear ONLY on invoice lines; Moishe Rubenstein, Avi Ash and Sang Ching appear ONLY on the + customer master. Martin Pasternak dominates invoice-line revenue while Naomi Linnell Rivera + holds the largest customer-master book — so "revenue by agent" gets a materially different + ranking depending on the source. Use THIS dim for "whose book is this customer in / who owns + the relationship"; use `invoice_lines` for "what was actually credited to an agent on the + billing", for commission questions, and for "what sales are NOT allocated to an agent" + (this topic cannot answer that — book attribution has no unallocated concept beyond a NULL agent). + ⚠ A NAME ON A COMMISSION LINE IS NOT NECESSARILY AN AGENT. The flag is res.partner.agent + (bool). "Anna" and "Shantal Erlich" are internal SALESPEOPLE (agent = False, each with an Odoo + user login, and the salesperson of record on 1,253 / 184 confirmed orders) who nonetheless + carry commission lines. They are correctly ABSENT from this dim. If the owner asks about + "agents", never report them as agents — report them separately as salespeople. Category + questions use the `category` dim (the product's category). DATA CAVEAT: many products sit in the root + category literally named "All" (master-data hygiene) — when ranking categories, mention that + "All" is the uncategorized bucket, not a real category. The `state` dim can be NULL (customer + has no address on file — present it as "no state on file", keep it in totals) and duplicate + state records exist (two ids can share one name, e.g. New Jersey twice) — list them as the + data shows and note the duplication rather than silently merging. diff --git a/royalimports_os/model/topics/sales_orders.yml b/royalimports_os/model/topics/sales_orders.yml new file mode 100644 index 0000000000000000000000000000000000000000..743e386ea621c566a5e36ee1a8ad6724cc873374 --- /dev/null +++ b/royalimports_os/model/topics/sales_orders.yml @@ -0,0 +1,43 @@ +# Topic: sales_orders — the wholesale ORDER-level dataset (header grain). Sibling of sales_lines: +# same business scope, order grain — for counts and order-level amounts (amount_untaxed). The two +# grains reconcile by contract (line Σ == order Σ; order count == line-parents + line-less orders). +key: sales_orders +label: Sales orders (wholesale) +entity: sale.order +domain_builder: sale_order_domain +scope: + confirmed_only: "state in (sale, done) — quotes and cancellations excluded" + channels: "wholesale teams only: Fisch=5, Royal=6; team_id parameter narrows to one business unit" + excluded: "house/GIFTWARE accounts excluded (same exclusions as sales_lines)" + basis: "order-header level; order revenue = amount_untaxed" + date_field: "date_order" +grain: "one row per confirmed order; time-filterable by date_order; BU-filterable by team_id" + +store: + table: sale_order + alias: o + join: >- + LEFT JOIN res_partner rp ON rp.id = o.partner_id + LEFT JOIN res_partner ag ON ag.id = rp.agent_id + date_col: "o.date_order" + team_col: "o.team_id" + scope_sql: >- + o.state IN ('sale','done') AND o.team_id IN ({wholesale_team_ids}) + AND o.partner_id NOT IN + (SELECT id FROM res_partner WHERE name LIKE {excluded_partner_name_like}) + dims: + partner: {col: "o.partner_id", name_col: "o.partner_name", label: "Customer"} + # team names via CASE (crm_team is not synced; ids are baked tenant scope like the team filter) + team: {col: "o.team_id", + name_col: "CASE o.team_id WHEN 5 THEN 'Fisch' WHEN 6 THEN 'Royal' ELSE CAST(o.team_id AS VARCHAR) END", + label: "Business unit"} + # agent = the customer's assigned sales agent (res_partner.agent_id, first of agent_ids — + # same attribution as sales_lines; added 2026-07-17 with the m2m sync) + agent: {col: "rp.agent_id", name_col: "ag.name", label: "Agent"} +ai_context: > + Order-HEADER grain of the same wholesale scope as sales_lines. Use for order counts and + order-level averages. Note: a small number of confirmed orders carry zero lines (data-health + artifact) — order-level counts therefore run slightly higher than distinct-order counts derived + from lines; the reconciliation contract accounts for them exactly. The `agent` dim carries the + same customer-book attribution as sales_lines (the customer's assigned agent) — use it for + "how many orders did write"; NULL agent = customer with no agent assigned. diff --git a/royalimports_os/modules/agent.py b/royalimports_os/modules/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..9b9a019c369b108514d24439e3b19bc1c79ba352 --- /dev/null +++ b/royalimports_os/modules/agent.py @@ -0,0 +1,277 @@ +"""Agent module — per-agent (res.partner.agent_ids) analytics. + +An *agent* owns a **book** of customers (the same attribute the Customers module slices by). This +module reports that book the way Sales/Customers/SKU report the whole company: a period scorecard +with custom date windows (Today / WTD / Last week / MTD / QTD / YTD / any custom range), a sales +trend, returns, top SKUs (with profit/order) and the FULL customer list — INCLUDING inactive +accounts (no recent orders) so a rep sees who they've stopped selling to. + +Scope: reuses the Sales `order_domain` (Fisch+Royal, excluded accounts removed, state sale/done) +so numbers tie to every other module. Returns are consolidated (credit notes aren't BU-tagged); +everything else is BU-filterable via team_id. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P +import modules.sales as sales_mod +import modules.customers as cust_mod + + +def options(t=None, team_id=None): + """Agent names with book activity — for the page/drawer picker.""" + return cust_mod.agent_options(t, team_id) + + +def _book(name): + """frozenset of every partner id assigned to the agent (incl. inactive). None only for 'All'.""" + return cust_mod.agent_partner_ids(name) + + +def _rev(date_from, date_to, team_id, book): + return O.sum_field('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), + 'amount_untaxed') + + +def _orders(date_from, date_to, team_id, book): + return O.get_odoo().search_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book)) + + +def _custs(date_from, date_to, team_id, book): + return O.distinct_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), + 'partner_id') + + +# ------------------------------------------------------------ period scorecard (custom dating) +# Today / WTD / Last week / MTD / QTD / YTD — each carries its window so the UI can decompose it +# and so "what did this agent sell this/last week" is a click, not a date-math exercise. +_PERIODS = [('Today', 'today'), ('Week to date', 'wtd'), ('Last week', 'lwk'), + ('Month to date', 'mtd'), ('Quarter to date', 'qtd'), ('Year to date', 'ytd')] + + +def _window(key, t): + if key == 'today': + return P._d(t), P._d(t) + if key == 'lwk': # the full prior Mon–Sun week + f, _tt = P.wtd(t) + start = dt.date.fromisoformat(f) - dt.timedelta(days=7) + return start.isoformat(), (dt.date.fromisoformat(f) - dt.timedelta(days=1)).isoformat() + return {'wtd': P.wtd, 'mtd': P.mtd, 'qtd': P.qtd, 'ytd': P.ytd}[key](t) + + +def scorecard(name, t=None, team_id=None): + """Book revenue (+ YoY same-period), orders for Today / WTD / Last week / MTD / QTD / YTD.""" + t = t or P.today() + book = _book(name) + out = [] + for label, key in _PERIODS: + f, tt = _window(key, t) + wk = key in ('today', 'wtd', 'lwk') # weekday-align the short windows' LY compare + cf, ct = P.shift_year(f, tt, weeks=wk) + rev, rev_ly = _rev(f, tt, team_id, book), _rev(cf, ct, team_id, book) + out.append({'key': key, 'label': label, 'date_from': f, 'date_to': tt, 'cmp_from': cf, 'cmp_to': ct, + 'revenue': rev, 'revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly), + 'orders': _orders(f, tt, team_id, book)}) + return out + + +def headline(name, date_from, date_to, team_id=None): + """Book KPIs for an ARBITRARY window (custom dating): revenue + YoY (same window LY), orders, + active customers, AOV and returns $ / return rate.""" + book = _book(name) + cf, ct = P.shift_year(date_from, date_to, weeks=False) + rev, rev_ly = _rev(date_from, date_to, team_id, book), _rev(cf, ct, team_id, book) + orders = _orders(date_from, date_to, team_id, book) + ret = sales_mod._returns_amt(date_from, date_to, book) + return {'date_from': date_from, 'date_to': date_to, 'cmp_from': cf, 'cmp_to': ct, + 'revenue': rev, 'revenue_ly': rev_ly, + 'yoy_pct': P.yoy_pct(rev, rev_ly), 'orders': orders, + 'customers': _custs(date_from, date_to, team_id, book), 'aov': (rev / orders) if orders else 0.0, + 'returns': ret, 'return_rate_pct': (ret / rev * 100.0) if rev else 0.0} + + +# ------------------------------------------------------------ sales trend +def _book_monthly_rev(book, date_from, date_to, team_id=None): + g = O.read_group('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), + ['amount_untaxed:sum'], ['date_order:month'], lazy=False) + out = {} + for r in g: + ym = ((r.get('__range') or {}).get('date_order:month') or {}).get('from', '')[:7] + if ym: + out[ym] = r.get('amount_untaxed') or 0.0 + return out + + +def monthly(name, n=13, t=None, team_id=None): + """Book sales per month vs the same month last year (one month-grouped query over 2 years).""" + t = t or P.today() + book = _book(name) + mrev = _book_monthly_rev(book, dt.date(t.year - 2, t.month, 1).isoformat(), t.isoformat(), team_id) + rows = [] + for ym, _s, _e in P.month_starts(n, t): + y, m = int(ym[:4]) - 1, int(ym[5:7]) + this, last = mrev.get(ym, 0.0), mrev.get(f'{y:04d}-{m:02d}', 0.0) + rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) + return rows + + +# ------------------------------------------------------------ full customer book (incl. inactive) +def customers(name, t=None, team_id=None): + """EVERY customer in the agent's book, including inactive accounts (no YTD/LY orders) — those + show $0 with status 'Inactive'/'Dormant'. Each row is clickable to the customer drawer and + carries recency so a rep can see who's gone quiet. Sorted by YTD revenue desc (inactive last).""" + t = t or P.today() + book = _book(name) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = cust_mod._cust_rev(yf, yt, team_id, book) + last = cust_mod._cust_rev(lf, lt, team_id, book) + lastord = cust_mod._last_order_dates(None, None, team_id, book) # all-time last order = recency + ids = list(book) if book is not None else list(set(this) | set(last)) + attrs = cust_mod._partner_attrs(set(ids)) + namemap = {r['id']: r.get('name') for r in O.search_read('res.partner', [('id', 'in', ids)], ['name'])} + rows = [] + for p in ids: + tr = this.get(p, {}).get('rev', 0.0) + lr = last.get(p, {}).get('rev', 0.0) + a = attrs.get(p, {}) + lo = lastord.get(p, '') + recency = (t - dt.date.fromisoformat(lo)).days if lo else None + status = 'Active' if tr > 0 else ('Dormant' if (lr > 0 or lo) else 'Inactive') + rows.append({'pid': p, 'customer': namemap.get(p) or (this.get(p) or last.get(p) or {}).get('name', '?'), + 'rev_ytd': tr, 'rev_ly': lr, 'change': tr - lr, 'yoy_pct': P.yoy_pct(tr, lr), + 'orders': this.get(p, {}).get('orders', 0), 'last_order': lo, 'recency_days': recency, + 'status': status, 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'), + 'agent': a.get('agent', name)}) + rows.sort(key=lambda r: (r['rev_ytd'] <= 0, -r['rev_ytd'], -(r['rev_ly']))) + return rows + + +# ------------------------------------------------------------ top SKUs (with profit/order) +def top_skus(name, t=None, team_id=None, top=30): + """The book's top SKUs YTD (line-level), each with revenue, units, margin and profit/order.""" + t = t or P.today() + book = _book(name) + if book is not None and not book: + return [] + yf, yt = P.ytd(t) + lex = [('order_partner_id', 'in', list(book))] if book is not None else None + return sales_mod.decompose(yf, yt, team_id, line_extra=lex, top=top)['skus'] + + +# ------------------------------------------------------------ returns (book-scoped, consolidated) +def returns_trend(name, n=13, t=None): + return sales_mod.returns_monthly(n, t, partner_ids=_book(name)) + + +def returns_headline(name, t=None): + return sales_mod.returns_headline(t, partner_ids=_book(name)) + + +# ------------------------------------------------------------ all-agents rollup (the page table) +def _returns_by_agent(t=None): + """{agent_name: returns$ YTD} — credit notes mapped to each customer's agent.""" + by_p = sales_mod.returns_by_partner(t) + attrs = cust_mod._partner_attrs(list(by_p)) + agg = {} + for pid, amt in by_p.items(): + a = (attrs.get(pid) or {}).get('agent') or '(none)' + agg[a] = agg.get(a, 0.0) + amt + return agg + + +def rollup(t=None, team_id=None): + """Every agent ranked by YTD book revenue (+ YoY, customers, orders) with returns $ and return + rate. Reuses the Customers MECE agent rollup, so Σ(agents) == total YTD revenue.""" + rows = cust_mod.by_dimension('agent', t, team_id=team_id) + ret = _returns_by_agent(t) + for r in rows: + r['agent'] = r['group'] + r['returns'] = ret.get(r['group'], 0.0) + r['return_rate_pct'] = (r['returns'] / r['revenue'] * 100.0) if r.get('revenue') else 0.0 + return rows + + +# ------------------------------------------------------------ VALIDATION +def validate(t=None, team_id=None): + """Reconcile the agent rollup to Odoo. (1) Σ(agent book revenue) == total YTD revenue — the + rollup is MECE over customers. (2) A sampled agent's scorecard YTD == its headline YTD.""" + t = t or P.today() + yf, yt = P.ytd(t) + checks = [] + rows = rollup(t, team_id=team_id) + agent_sum = sum(r['revenue'] for r in rows) + total = O.sum_field('sale.order', sales_mod.order_domain(yf, yt, team_id), 'amount_untaxed') + checks.append({'check': 'YTD revenue: Σ(agent book) == total', 'a': round(agent_sum, 2), + 'b': round(total, 2), 'gap': round(agent_sum - total, 2), + 'ok': abs(agent_sum - total) <= max(1.0, 0.001 * (total or 1))}) + # sampled agent: scorecard YTD == headline YTD for the same window + sample = next((r['agent'] for r in rows if r['agent'] not in ('(none)',)), None) + if sample: + sc_ytd = next((s['revenue'] for s in scorecard(sample, t, team_id) if s['key'] == 'ytd'), 0.0) + hl = headline(sample, yf, yt, team_id)['revenue'] + checks.append({'check': f'Agent "{sample}": scorecard YTD == headline YTD', 'a': round(sc_ytd, 2), + 'b': round(hl, 2), 'gap': round(sc_ytd - hl, 2), 'ok': abs(sc_ytd - hl) <= 1.0}) + return checks + + +# ------------------------------------------------------------ INVOICE-LINE ATTRIBUTION (2026-07-28) +# A SECOND agent source. Everything above this line attributes by BOOK — the customer's assigned +# agent (res.partner.agent_ids) — over confirmed SALES ORDERS. This section attributes per INVOICE +# LINE, from the OCA sale-commission module, via the semantic layer (topics invoice_lines / +# commission_lines). The two disagree on purpose and answer different questions: +# +# book -> "whose customer is this / who owns the relationship" (order basis) +# invoice -> "what was actually credited to an agent on the billing" + the ONLY source that can +# say what is NOT allocated to an agent (invoice basis) +# +# ⚠ A NAME ON A COMMISSION LINE IS NOT NECESSARILY AN AGENT — `res.partner.agent` is the flag. +# "Anna" and "Shantal Erlich" are internal SALESPEOPLE who carry commission lines; the `agent` +# dim excludes them and `include_salespeople` folds them back in as a clearly-labelled variant. +# See [[invoice-line-agent-commission]]. + +_ALLOC_LABEL = {'agent': 'Allocated to an agent', 'salesperson': 'Salesperson only', + 'none': 'Not allocated'} + + +def invoice_line_rollup(t=None, team_id=None, include_salespeople=False): + """Per-name invoice-line revenue + the MECE allocation split, for a YTD window. + + Returns {'by_agent': [...], 'allocation': [...], 'total': float, 'allocated': float, + 'unallocated': float, 'basis': str} — or {'error': msg} when the tenant store is not + ready (this path is store-only; there is no live fallback that stays honest about the + unallocated bucket). + """ + import harness.semantic as S + t = t or P.today() + yf, yt = P.ytd(t) + dim = 'commission_name' if include_salespeople else 'agent' + try: + by = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=[dim], + date_from=yf, date_to=yt, team_id=team_id, limit=200).get('rows') or [] + alloc = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=['allocation'], + date_from=yf, date_to=yt, team_id=team_id, limit=10).get('rows') or [] + tot = (S.store_query('invoice_lines', ['invoiced_line_sales'], date_from=yf, date_to=yt, + team_id=team_id).get('rows') or [{}])[0].get('invoiced_line_sales') or 0.0 + except Exception as e: # store not ready / model error — say so, don't fake + return {'error': str(e)} + # store_query row shape: the DIM KEY carries the display NAME and `_id` the raw value + # (allocation -> 'Salesperson only', allocation_id -> 'salesperson'). Reading `_name` + # returns None for every row and silently renders the whole table as "no agent". + rows = [{'agent': (r.get(dim) or '(no agent on the line)'), + 'revenue': r.get('invoiced_line_sales') or 0.0} for r in by] + rows.sort(key=lambda r: -r['revenue']) + amap = {r.get('allocation_id') or 'none': (r.get('invoiced_line_sales') or 0.0) for r in alloc} + allocated = amap.get('agent', 0.0) + return { + 'by_agent': rows, + 'allocation': [{'bucket': _ALLOC_LABEL[k], 'revenue': amap.get(k, 0.0)} + for k in ('agent', 'salesperson', 'none') if k in amap or True], + 'total': tot, 'allocated': allocated, 'unallocated': tot - allocated, + 'strict_none': amap.get('none', 0.0), 'salesperson_only': amap.get('salesperson', 0.0), + 'window': (yf, yt), + 'basis': ('invoice line · commission names incl. salespeople' if include_salespeople + else 'invoice line · real agents only'), + } diff --git a/royalimports_os/modules/ar.py b/royalimports_os/modules/ar.py new file mode 100644 index 0000000000000000000000000000000000000000..7cc4fa9d3b91d8efa8583d969e52b5ab1c0e8177 --- /dev/null +++ b/royalimports_os/modules/ar.py @@ -0,0 +1,514 @@ +"""Collections / AR module — receivables, aging, DSO, exposures, follow-up status, +and the reconciliation gap (open-invoice total vs Odoo's partner receivable). + +AR is company-level (account.move), not team-scoped. Excluded accounts removed. Aging is built +from open customer invoices/credit notes (signed residual nets credit notes), so buckets +sum to the open-doc AR total. DSO uses LTM gross invoiced sales. +""" +import sys +import math +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P +import core.store as store + +BUCKETS = ['Current', '1-30', '31-60', '61-90', '90+'] + +#: Store key for the NIGHTLY days-to-pay snapshot (owner, 2026-07-27). +#: +#: `days_to_pay()` costs ~300s — it walks three years of reconciled receivable lines and their +#: full-reconcile groups to find each invoice's SETTLEMENT date. That is fine for a page somebody +#: opens on purpose, and unacceptable inside `customer_data.pool()`, which every Customer-table +#: render and every Space container start depends on. So the expensive half runs on a schedule +#: and the table reads the ANSWER. +#: +#: Shape: {'computed': 'YYYY-MM-DD HH:MM', 'years': 3, 'days': {'': avg_days|None}}. +#: JSON object keys are strings, which is why the reader coerces back to int. +DTP_KEY = 'ar_days_to_pay' +#: Refresh when the snapshot is older than this. 20h rather than 24 so a daily cadence never +#: skips a day by drifting a few minutes later each time. +DTP_MAX_AGE_HOURS = 20 + + +def days_to_pay_snapshot(): + """`{pid: avg_days}` from the last snapshot — CHEAP, and it never raises. + + Returns `{}` when the snapshot is absent, unreadable or malformed. A blank column is the + correct degradation: the alternative is a Customer table that will not render because a + derived statistic is missing, and `days_to_pay` is a statistic, not a fact the page is about. + """ + try: + snap = store.get(DTP_KEY) or {} + days = snap.get('days') or {} + out = {} + for k, v in days.items(): + try: + out[int(k)] = None if v is None else float(v) + except (TypeError, ValueError): + continue + return out + except Exception: + return {} + + +def days_to_pay_computed_at(): + """When the snapshot was taken ('' if there is none) — so the column can say how old it is.""" + try: + return str((store.get(DTP_KEY) or {}).get('computed') or '') + except Exception: + return '' + + +def refresh_days_to_pay_snapshot(max_age_hours=DTP_MAX_AGE_HOURS, t=None, force=False): + """Recompute + persist the snapshot IF it is stale. Returns what it did, and never raises. + + Called from the app's existing background store-sync thread, so the ~300s cost is paid off + the render path by whichever container happens to be awake. Deliberately NOT called from + `pool()`: a lazy compute would just move the five minutes onto whoever opened the page first. + + ⚠ Writes through `store.put` on a key nothing else owns. It is a DERIVED artifact — losing it + costs one recompute and nothing else, which is why it is safe to overwrite wholesale rather + than read-modify-write. + """ + if not store.available(): + return {'skipped': 'no store'} + try: + if not force: + at = days_to_pay_computed_at() + if at: + age = dt.datetime.now() - dt.datetime.strptime(at, '%Y-%m-%d %H:%M') + if age < dt.timedelta(hours=max_age_hours): + return {'skipped': 'fresh', 'age_h': round(age.total_seconds() / 3600, 1)} + except Exception: + pass # an unparseable stamp means "recompute", never "crash" + try: + res = days_to_pay(t) + days = {str(r['pid']): r.get('avg_days') for r in (res.get('_all_rows') or [])} + store.put(DTP_KEY, {'computed': dt.datetime.now().strftime('%Y-%m-%d %H:%M'), + 'years': 3, 'days': days}) + return {'refreshed': len(days)} + except Exception as e: + return {'error': str(e)[:200]} + + +def _bucket(days): + if days <= 0: + return 'Current' + if days <= 30: + return '1-30' + if days <= 60: + return '31-60' + if days <= 90: + return '61-90' + return '90+' + + +def _open_docs(t=None): + o = O.get_odoo() + dom = [('move_type', 'in', ['out_invoice', 'out_refund']), ('state', '=', 'posted'), + ('payment_state', 'in', ['not_paid', 'partial'])] + ex = O.excluded_partner_ids() + if ex: + dom.append(('partner_id', 'not in', list(ex))) + docs = o.search_read('account.move', dom, + ['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due', + 'amount_total', 'amount_residual_signed']) + today = t or P.today() + for d in docs: + due = d.get('invoice_date_due') + try: + d['days_overdue'] = max((today - dt.date.fromisoformat(due)).days, 0) if due else 0 + except Exception: + d['days_overdue'] = 0 + d['bucket'] = _bucket(d['days_overdue']) + d['open'] = d.get('amount_residual_signed') or 0.0 + return docs + + +def _ltm_invoiced(t=None): + """LTM gross invoiced sales (out_invoice − out_refund, amount_total) for DSO.""" + o = O.get_odoo() + lf, lt = P.ltm(t) + base = [('state', '=', 'posted'), ('invoice_date', '>=', lf), ('invoice_date', '<=', lt)] + ex = O.excluded_partner_ids() + if ex: + base.append(('partner_id', 'not in', list(ex))) + inv = O.sum_field('account.move', base + [('move_type', '=', 'out_invoice')], 'amount_total') + ref = O.sum_field('account.move', base + [('move_type', '=', 'out_refund')], 'amount_total') + return inv - ref + + +def summary(t=None): + docs = _open_docs(t) + total_ar = sum(d['open'] for d in docs) + overdue = sum(d['open'] for d in docs if d['days_overdue'] > 0) + ltm_sales = _ltm_invoiced(t) + dso = (total_ar / (ltm_sales / 365.0)) if ltm_sales else None + # reconciliation gap vs Odoo partner receivable (company-level) + ex = set(O.excluded_partner_ids()) + partner_recv = sum(p['credit'] for p in O.search_read('res.partner', + [('credit', '>', 0)] + ([('id', 'not in', list(ex))] if ex else []), ['credit'])) + n_customers = len({O.m2o_id(d['partner_id']) for d in docs}) + return { + 'total_ar_open': total_ar, + 'overdue': overdue, + 'pct_overdue': (overdue / total_ar * 100) if total_ar else 0, + 'dso_days': dso, + 'ltm_invoiced': ltm_sales, + 'open_customers': n_customers, + 'partner_receivable': partner_recv, + 'reconciliation_gap': partner_recv - total_ar, + } + + +def aging(t=None): + docs = _open_docs(t) + out = {b: 0.0 for b in BUCKETS} + for d in docs: + out[d['bucket']] += d['open'] + total = sum(out.values()) or 1.0 + return [{'bucket': b, 'amount': out[b], 'pct': out[b] / total * 100} for b in BUCKETS] + + +def top_exposures(t=None, limit=30): + docs = _open_docs(t) + by = {} + for d in docs: + pid = O.m2o_id(d['partner_id']) + nm = O.m2o_name(d['partner_id']) + e = by.setdefault(pid, {'customer': nm, 'open': 0.0, 'overdue': 0.0, 'oldest': 0, 'docs': 0}) + e['open'] += d['open'] + e['docs'] += 1 + if d['days_overdue'] > 0: + e['overdue'] += d['open'] + e['oldest'] = max(e['oldest'], d['days_overdue']) + rows = sorted(by.values(), key=lambda x: -x['overdue']) + return rows[:limit] + + +def followup_status_mix(t=None): + """Mix of res.partner.followup_status among customers carrying receivable. + followup_status is a non-stored computed field, so we count client-side.""" + ex = set(O.excluded_partner_ids()) + dom = [('credit', '>', 1)] + ([('id', 'not in', list(ex))] if ex else []) + parts = O.search_read('res.partner', dom, ['followup_status']) + counts = {} + for p in parts: + s = (p.get('followup_status') or '—').replace('_', ' ') + counts[s] = counts.get(s, 0) + 1 + return [{'status': k, 'customers': v} for k, v in + sorted(counts.items(), key=lambda kv: -kv[1])] + + +def reconciliation_flags(t=None, limit=30, threshold=50.0): + """Customers whose Odoo receivable (credit) diverges from their open-doc total — + indicates unapplied payments/credits. Surfaced during the collections work.""" + docs = _open_docs(t) + open_by = {} + for d in docs: + pid = O.m2o_id(d['partner_id']) + open_by[pid] = open_by.get(pid, 0.0) + d['open'] + ex = set(O.excluded_partner_ids()) + parts = O.search_read('res.partner', + [('credit', '>', 0)] + ([('id', 'not in', list(ex))] if ex else []), + ['name', 'credit']) + rows = [] + for p in parts: + gap = (p['credit'] or 0) - open_by.get(p['id'], 0.0) + if abs(gap) > threshold: + rows.append({'customer': p['name'], 'odoo_receivable': p['credit'], + 'open_docs': open_by.get(p['id'], 0.0), 'gap': gap}) + rows.sort(key=lambda x: -abs(x['gap'])) + return rows[:limit] + + +def credit_exposure(t=None, grace_days=5, limit=40): + """Forward credit exposure per customer (OCA account_financial_risk semantics, re-implemented): + draft invoices + open-not-overdue + overdue-past-grace residuals + confirmed-uninvoiced order + value, vs the partner credit limit (when set). The number to check BEFORE accepting the next + PO from a shaky dealer. Company-level, GIFTWARE excluded.""" + docs = _open_docs(t) + ex = O.excluded_partner_ids() + exdom = [('partner_id', 'not in', list(ex))] if ex else [] + + per = {} + + def _e(pid, name): + return per.setdefault(pid, {'pid': pid, 'customer': name, 'draft': 0.0, 'open': 0.0, + 'overdue': 0.0, 'uninvoiced': 0.0, 'credit_limit': 0.0}) + + for d in docs: + e = _e(O.m2o_id(d['partner_id']), O.m2o_name(d['partner_id'])) + if d['days_overdue'] > grace_days: + e['overdue'] += d['open'] + else: + e['open'] += d['open'] + + for r in O.search_read('account.move', + [('move_type', 'in', ['out_invoice', 'out_refund']), + ('state', '=', 'draft')] + exdom, + ['partner_id', 'amount_total_signed']): + if r.get('partner_id'): + _e(O.m2o_id(r['partner_id']), O.m2o_name(r['partner_id']))['draft'] += \ + r.get('amount_total_signed') or 0.0 + + uexdom = [('order_partner_id', 'not in', list(ex))] if ex else [] + for g in O.read_group('sale.order.line', + [('order_id.state', 'in', ['sale', 'done']), + ('untaxed_amount_to_invoice', '!=', 0)] + uexdom, + ['untaxed_amount_to_invoice'], ['order_partner_id']): + p = g.get('order_partner_id') + if p: + _e(O.m2o_id(p), O.m2o_name(p))['uninvoiced'] += \ + g.get('untaxed_amount_to_invoice') or 0.0 + + try: # credit_limit only exists/means something when the feature is enabled — degrade quietly + for r in O.search_read('res.partner', [('id', 'in', list(per.keys()))], ['credit_limit']): + if per.get(r['id']) is not None: + per[r['id']]['credit_limit'] = r.get('credit_limit') or 0.0 + except Exception: + pass + + rows_all = [] + for e in per.values(): + e['exposure'] = e['draft'] + e['open'] + e['overdue'] + e['uninvoiced'] + e['headroom'] = (e['credit_limit'] - e['exposure']) if e['credit_limit'] else None + e['flag'] = bool(e['credit_limit']) and e['exposure'] > e['credit_limit'] + rows_all.append(e) + rows_all.sort(key=lambda x: -x['exposure']) + # Display rows drop ~zero exposures; _all_rows keeps EVERY customer (incl. net-credit + # balances) — validation sums must run over the unfiltered set or the filter biases them. + rows = [r for r in rows_all if abs(r['exposure']) > 1] + return {'rows': rows[:limit], 'n_flagged': sum(1 for r in rows if r['flag']), + 'total_exposure': sum(r['exposure'] for r in rows), + 'grace_days': grace_days, '_all_rows': rows_all} + + +def days_to_pay(t=None, years=3, limit=30): + """Per-customer average days from invoice to SETTLEMENT (OCA partner_time_to_pay semantics): + settlement date = the date the receivable line became fully reconciled (max counterpart line + date in its full-reconcile group), NOT the payment document date. Windows: lifetime (bounded + to `years`), last calendar year, this year — by invoice date. Joined with current open AR so + the table reads 'who owes us AND how do they behave'.""" + o = O.get_odoo() + today = t or P.today() + since = (today - dt.timedelta(days=365 * years)).isoformat() + ex = O.excluded_partner_ids() + dom = [('move_id.move_type', '=', 'out_invoice'), ('move_id.state', '=', 'posted'), + ('account_id.account_type', '=', 'asset_receivable'), + ('full_reconcile_id', '!=', False), ('date', '>=', since)] + if ex: + dom.append(('partner_id', 'not in', list(ex))) + inv_lines = O.search_read('account.move.line', dom, + ['partner_id', 'date', 'full_reconcile_id', 'move_id']) + + # Payment-term days per term id: prefer the term lines' `days`; fall back to parsing the name. + term_days = {} + try: + terms = O.search_read('account.payment.term', [], ['name']) + tl = O.search_read('account.payment.term.line', [], ['payment_id', 'days']) + by_term = {} + for l in tl: + k = O.m2o_id(l.get('payment_id')) + if k is not None: + by_term[k] = max(by_term.get(k, 0), l.get('days') or 0) + import re as _re + for tm in terms: + d = by_term.get(tm['id']) + if d is None: + m = _re.search(r'(\d+)', tm['name'] or '') + d = int(m.group(1)) if m and 'day' in (tm['name'] or '').lower() else 0 + term_days[tm['id']] = {'name': tm['name'], 'days': d} + except Exception: + term_days = {} + + # Per-invoice terms + totals (12m window for the free-credit estimate). + y365_iso = (today - dt.timedelta(days=365)).isoformat() + move_ids = list({O.m2o_id(l['move_id']) for l in inv_lines if l.get('move_id')}) + move_info = {} + for i in range(0, len(move_ids), 5000): + for mv in O.search_read('account.move', [('id', 'in', move_ids[i:i + 5000])], + ['invoice_payment_term_id', 'amount_total', 'invoice_date', + 'partner_id']): + move_info[mv['id']] = mv + + fr_ids = list({O.m2o_id(l['full_reconcile_id']) for l in inv_lines + if l.get('full_reconcile_id')}) + settle = {} + for i in range(0, len(fr_ids), 5000): + chunk = fr_ids[i:i + 5000] + try: # grouped max-date per reconcile (one call per chunk) + for g in O.read_group('account.move.line', + [('full_reconcile_id', 'in', chunk)], + ['date:max'], ['full_reconcile_id']): + k = O.m2o_id(g.get('full_reconcile_id')) + d = g.get('date') or g.get('date:max') or '' + d = str(d)[:10] + if k and d: + settle[k] = max(settle.get(k, ''), d) + except Exception: # fallback: raw lines + for l in o.search_read('account.move.line', + [('full_reconcile_id', 'in', chunk)], + ['full_reconcile_id', 'date']): + k = O.m2o_id(l['full_reconcile_id']) + d = str(l.get('date') or '')[:10] + if k and d: + settle[k] = max(settle.get(k, ''), d) + + y0 = dt.date(today.year, 1, 1).isoformat() + ly0 = dt.date(today.year - 1, 1, 1).isoformat() + per = {} + violations = 0 + for l in inv_lines: + k = O.m2o_id(l.get('full_reconcile_id')) + inv_d = str(l.get('date') or '')[:10] + pay_d = settle.get(k, '') + if not inv_d or not pay_d: + continue + days = (dt.date.fromisoformat(pay_d) - dt.date.fromisoformat(inv_d)).days + if days < 0: + violations += 1 + continue + pid = O.m2o_id(l['partner_id']) + e = per.setdefault(pid, {'pid': pid, 'customer': O.m2o_name(l['partner_id']), + 'all': [], 'ly': [], 'ytd': []}) + e['all'].append(days) + if inv_d >= y0: + e['ytd'].append(days) + elif inv_d >= ly0: + e['ly'].append(days) + + open_by = {} + for d in _open_docs(t): + pid = O.m2o_id(d['partner_id']) + oe = open_by.setdefault(pid, {'open': 0.0, 'overdue': 0.0}) + oe['open'] += d['open'] + if d['days_overdue'] > 0: + oe['overdue'] += d['open'] + + def _avg(v): + return (sum(v) / len(v)) if v else None + + # Terms + 12m invoiced per partner (dominant term by invoiced $) → compliance/free-credit. + pt_terms, pt_inv12 = {}, {} + for mid, mv in move_info.items(): + pid = O.m2o_id(mv.get('partner_id')) + tid = O.m2o_id(mv.get('invoice_payment_term_id')) + amt = mv.get('amount_total') or 0.0 + d = pt_terms.setdefault(pid, {}) + d[tid] = d.get(tid, 0.0) + amt + if str(mv.get('invoice_date') or '')[:10] >= y365_iso: + pt_inv12[pid] = pt_inv12.get(pid, 0.0) + amt + + rows = [] + total_free_credit = 0.0 + for pid, e in per.items(): + ob = open_by.get(pid, {'open': 0.0, 'overdue': 0.0}) + a_ly, a_ytd = _avg(e['ly']), _avg(e['ytd']) + a_all = _avg(e['all']) + dom_tid = max(pt_terms.get(pid, {None: 0}).items(), key=lambda kv: kv[1])[0] + tinfo = term_days.get(dom_tid, {'name': '(none)', 'days': 0}) + eff = a_ytd if a_ytd is not None else a_all + excess = max(0.0, (eff or 0) - tinfo['days']) if eff is not None else None + inv12 = pt_inv12.get(pid, 0.0) + free_credit = (excess / 365.0 * inv12) if excess else 0.0 + total_free_credit += free_credit or 0.0 + rows.append({'pid': pid, 'customer': e['customer'], 'open': ob['open'], 'overdue': ob['overdue'], + 'paid_invoices': len(e['all']), 'avg_days': a_all, + 'avg_days_ly': a_ly, 'avg_days_ytd': a_ytd, + 'improvement_days': (a_ly - a_ytd) if (a_ly is not None and a_ytd is not None) else None, + 'terms': tinfo['name'], 'term_days': tinfo['days'], + 'excess_days': excess, 'free_credit': free_credit, + 'invoiced_12m': inv12}) + rows.sort(key=lambda x: -x['open']) + return {'rows': rows[:limit], 'n_customers': len(per), 'violations': violations, + 'since': since, 'total_free_credit': total_free_credit, + '_rowsum_inv12': sum(pt_inv12.values()), '_all_rows': rows} + + +def credit_limit_proposals(exposure, behavior): + """PROPOSED credit limits for customers trading on open terms with NO limit set (only 22 of + 3,595 partners have one — the field is effectively unmaintained). Trade-credit heuristic: + limit ≈ (12m invoiced / 365) × (term days + 30 review buffer), tiered by observed payment + behavior (pays within terms +5d → ×1.25 · chronic 15d+ overrun → ×0.75), rounded UP to $500, + floor $1,000. READ-ONLY: an export worklist for the owner to enter in Odoo — once limits are + in, the exposure table's breach flag and Odoo's own credit hold both come alive.""" + beh = {r['pid']: r for r in behavior.get('_all_rows', []) if r.get('pid')} + out = [] + for e in exposure.get('_all_rows', []): + if e.get('credit_limit') or e.get('exposure', 0) <= 0: + continue + b = beh.get(e['pid']) + if not b or (b.get('invoiced_12m') or 0) <= 0: + continue + base = b['invoiced_12m'] / 365.0 * ((b.get('term_days') or 0) + 30) + exd = b.get('excess_days') + tier = ('on-time' if (exd is not None and exd <= 5) + else ('slow' if (exd or 0) > 15 else 'normal')) + mult = {'on-time': 1.25, 'normal': 1.0, 'slow': 0.75}[tier] + prop = max(1000.0, math.ceil(base * mult / 500.0) * 500.0) + out.append({'pid': e['pid'], 'customer': e['customer'], + 'invoiced_12m': b['invoiced_12m'], 'term_days': b.get('term_days') or 0, + 'excess_days': exd, 'tier': tier, 'exposure': e['exposure'], + 'proposed_limit': prop, + 'over_proposed': e['exposure'] > prop}) + out.sort(key=lambda r: -r['exposure']) + return {'rows': out, 'n': len(out), + 'n_over': sum(1 for r in out if r['over_proposed']), + 'over_value': sum(r['exposure'] - r['proposed_limit'] + for r in out if r['over_proposed']), + 'with_limit': [e for e in exposure.get('_all_rows', []) if e.get('credit_limit')]} + + +def validate(t=None, exposure=None, behavior=None): + """exposure/behavior: pass precomputed credit_exposure()/days_to_pay() results to avoid + recomputing the heavy pulls when the caller (the page bundle) already has them; validate.py + calls with no args and computes everything itself.""" + docs = _open_docs(t) + checks = [] + total = sum(d['open'] for d in docs) + bucket_sum = sum(a['amount'] for a in aging(t)) + checks.append({ + 'check': 'AR aging: Σ(buckets) == total open AR', + 'a': round(bucket_sum, 2), 'b': round(total, 2), + 'gap': round(bucket_sum - total, 2), + 'ok': abs(bucket_sum - total) <= 1.0}) + + exp_sum = sum(e['open'] for e in top_exposures(t, limit=10**9)) + checks.append({ + 'check': 'AR: Σ(per-customer open) == total open AR', + 'a': round(exp_sum, 2), 'b': round(total, 2), + 'gap': round(exp_sum - total, 2), + 'ok': abs(exp_sum - total) <= 1.0}) + + # Credit exposure: per-customer (open + overdue) row-build vs an independent read_group sum + # of amount_residual_signed over the same unpaid-invoice domain. The two sides are separate + # RPC snapshots on a LIVE ledger — a payment applied between them shifts the total — so the + # sides are pulled adjacently and the check carries an explicit live-drift tolerance. A + # structural bug (double-count, dropped partner, wrong bucket) shows as a %-level gap, far + # beyond it. + ex = O.excluded_partner_ids() + dom = [('move_type', 'in', ['out_invoice', 'out_refund']), ('state', '=', 'posted'), + ('payment_state', 'in', ['not_paid', 'partial'])] + if ex: + dom.append(('partner_id', 'not in', list(ex))) + resid = O.sum_field('account.move', dom, 'amount_residual_signed') + ce = exposure or credit_exposure(t) + ce_open = sum(r['open'] + r['overdue'] for r in ce['_all_rows']) + tol = max(500.0, abs(resid) * 0.001) + checks.append({ + 'check': 'Exposure: Σ(open+overdue per customer) == read_group Σ residual (live-drift tol)', + 'a': round(ce_open, 2), 'b': round(resid, 2), + 'gap': round(ce_open - resid, 2), + 'ok': abs(ce_open - resid) <= tol}) + + # Days-to-pay: settlement can never precede the invoice line date, and coverage exists. + dtp = behavior or days_to_pay(t) + checks.append({ + 'check': 'Days-to-pay: settlement>=invoice violations == 0 (and customers covered > 0)', + 'a': dtp['violations'], 'b': 0, + 'gap': dtp['violations'], + 'ok': dtp['violations'] == 0 and dtp['n_customers'] > 0}) + return checks diff --git a/royalimports_os/modules/assortment.py b/royalimports_os/modules/assortment.py new file mode 100644 index 0000000000000000000000000000000000000000..1e860daeae838253b21ffeff66bd51ac50bc43bc --- /dev/null +++ b/royalimports_os/modules/assortment.py @@ -0,0 +1,156 @@ +"""Assortment module — facet-level performance from the curated product taxonomy the team +already maintains IN Odoo (x_main/x_sub/x_color/x_material/x_occasion/x_collection + +x_studio_season), which no other module used until 2026-07-05. + +Governing question (dashboard standard): WHICH PARTS OF THE ASSORTMENT EARN THEIR KEEP — AND IS +THE SEASONAL BUY READY? Revenue/margin/YoY per facet value (BU-scoped via the standard wholesale +line domain), plus season readiness: units the coming 6 months sold LAST year per season tag vs +units on hand today. validate() reconciles facet partitions to independent totals. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P + +FACETS = [('x_main', 'Main category'), ('x_sub', 'Sub-category'), ('x_color', 'Color'), + ('x_material', 'Material'), ('x_occasion', 'Occasion'), + ('x_collection', 'Collection'), ('x_studio_season', 'Season')] +UNTAGGED = '(untagged)' + + +def _products(): + """pid -> {code, facets..., on_hand, cost}. One pull, all facet fields.""" + fields = ['default_code', 'qty_available', 'standard_price'] + [f for f, _ in FACETS] + out = {} + for p in O.search_read('product.product', + [('default_code', '!=', False), ('active', 'in', [True, False])], + fields): + out[p['id']] = p + return out + + +def _rev_by_product(df, dt_, team_id=None): + """product_id -> {rev, qty, margin} over the window, standard wholesale scope.""" + out = {} + for g in O.read_group('sale.order.line', O.sale_line_domain(df, dt_, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], + ['product_id'], lazy=False): + if g.get('product_id'): + out[O.m2o_id(g['product_id'])] = { + 'rev': g.get('price_subtotal') or 0.0, + 'qty': g.get('product_uom_qty') or 0.0, + 'margin': g.get('margin') or 0.0} + return out + + +def build(team_id=None, t=None): + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + prods = _products() + now = _rev_by_product(yf, yt, team_id) + ly = _rev_by_product(lf, lt, team_id) + + # ---- facet rollups (every dimension partitions the same YTD revenue) ------------------- + dims = {} + for fkey, flabel in FACETS: + agg = {} + for pid, r in now.items(): + p = prods.get(pid) + val = ((p or {}).get(fkey) or UNTAGGED) if p else UNTAGGED + val = str(val).strip() or UNTAGGED + e = agg.setdefault(val, {'value': val, 'rev': 0.0, 'rev_ly': 0.0, 'margin': 0.0, + 'qty': 0.0, 'skus': set()}) + e['rev'] += r['rev'] + e['margin'] += r['margin'] + e['qty'] += r['qty'] + e['skus'].add(pid) + for pid, r in ly.items(): + p = prods.get(pid) + val = ((p or {}).get(fkey) or UNTAGGED) if p else UNTAGGED + val = str(val).strip() or UNTAGGED + agg.setdefault(val, {'value': val, 'rev': 0.0, 'rev_ly': 0.0, 'margin': 0.0, + 'qty': 0.0, 'skus': set()})['rev_ly'] += r['rev'] + rows = [] + for e in agg.values(): + e['n_skus'] = len(e['skus']) + del e['skus'] + e['gm_pct'] = (e['margin'] / e['rev'] * 100) if e['rev'] else None + e['yoy_pct'] = ((e['rev'] - e['rev_ly']) / e['rev_ly'] * 100) if e['rev_ly'] else None + e['yoy_abs'] = e['rev'] - e['rev_ly'] + rows.append(e) + rows.sort(key=lambda x: -x['rev']) + dims[fkey] = {'label': flabel, 'rows': rows} + + # ---- coverage: how much of revenue is on FACETED (x_main-tagged) SKUs ------------------ + total_rev = sum(r['rev'] for r in now.values()) + faceted_rev = sum(r['rev'] for pid, r in now.items() + if prods.get(pid, {}).get('x_main')) + n_faceted = sum(1 for p in prods.values() if p.get('x_main')) + + # ---- season readiness: units sold in [today, +180d] LAST YEAR per season vs on hand ---- + nf, ntt = (t - dt.timedelta(days=365)).isoformat(), (t + dt.timedelta(days=180) - dt.timedelta(days=365)).isoformat() + ahead_ly = _rev_by_product(nf, ntt, team_id) + seasons = {} + for pid, r in ahead_ly.items(): + p = prods.get(pid) + s = str((p or {}).get('x_studio_season') or '').strip() + if not s: + continue + e = seasons.setdefault(s, {'season': s, 'demand_units_ly': 0.0, 'demand_rev_ly': 0.0, + 'on_hand_units': 0.0, 'on_hand_value': 0.0, 'skus': set()}) + e['demand_units_ly'] += r['qty'] + e['demand_rev_ly'] += r['rev'] + e['skus'].add(pid) + for pid, p in prods.items(): + s = str(p.get('x_studio_season') or '').strip() + if s and s in seasons: + seasons[s]['on_hand_units'] += p.get('qty_available') or 0.0 + seasons[s]['on_hand_value'] += (p.get('qty_available') or 0.0) * (p.get('standard_price') or 0.0) + season_rows = [] + for e in seasons.values(): + e['n_skus'] = len(e['skus']) + e['codes'] = sorted((prods.get(pid, {}).get('default_code') or '').strip() + for pid in e['skus'] if prods.get(pid, {}).get('default_code')) + del e['skus'] + e['cover_pct'] = (e['on_hand_units'] / e['demand_units_ly'] * 100) if e['demand_units_ly'] else None + season_rows.append(e) + season_rows.sort(key=lambda x: -x['demand_rev_ly']) + + # suspect facet hygiene: values carried by <3 SKUs across ALL products (typos like 'vgsd') + suspects = [] + for fkey, flabel in FACETS: + vals = {} + for p in prods.values(): + v = str(p.get(fkey) or '').strip() + if v: + vals[v] = vals.get(v, 0) + 1 + suspects += [{'facet': flabel, 'value': v, 'skus': n} for v, n in vals.items() if n < 3] + + return {'dims': dims, 'season': season_rows, 'suspects': sorted(suspects, key=lambda x: x['skus']), + 'total_rev': total_rev, 'faceted_rev': faceted_rev, + 'faceted_share': (faceted_rev / total_rev * 100) if total_rev else None, + 'n_faceted': n_faceted, 'n_products': len(prods), + 'ytd': (yf, yt), 'ly': (lf, lt)} + + +def validate(t=None, team_id=None, pre=None): + t = t or P.today() + b = pre or build(team_id, t) + yf, yt = b['ytd'] + total = O.sum_field('sale.order.line', O.sale_line_domain(yf, yt, team_id), 'price_subtotal') + checks = [] + for fkey, flabel in FACETS[:2]: # every dimension partitions the SAME revenue; check two + s = sum(r['rev'] for r in b['dims'][fkey]['rows']) + checks.append({'check': f'Assortment: Σ({flabel} facet rev) == total line revenue (YTD)', + 'a': round(s, 2), 'b': round(total, 2), 'gap': round(s - total, 2), + 'ok': abs(s - total) <= max(1.0, abs(total) * 0.001)}) + fs = b['faceted_rev'] + sum(r['rev'] for r in b['dims']['x_main']['rows'] + if r['value'] == UNTAGGED) + checks.append({'check': 'Assortment: faceted rev + untagged bucket == total (partition)', + 'a': round(fs, 2), 'b': round(sum(r['rev'] for r in b['dims']['x_main']['rows']), 2), + 'gap': round(fs - sum(r['rev'] for r in b['dims']['x_main']['rows']), 2), + 'ok': abs(fs - sum(r['rev'] for r in b['dims']['x_main']['rows'])) <= 1.0}) + return checks diff --git a/royalimports_os/modules/backorders.py b/royalimports_os/modules/backorders.py new file mode 100644 index 0000000000000000000000000000000000000000..cf16f4939657d1bc903dbdde7b1921e9b77e087f --- /dev/null +++ b/royalimports_os/modules/backorders.py @@ -0,0 +1,191 @@ +"""Backorders — what we owe customers, aged (the mirror image of AR aging). + +Every confirmed-but-undelivered sale line, aged against the order's promise date +(commitment_date, else the earliest open outgoing picking's scheduled date), valued at the +line's discounted unit price, joined to supply (on-hand + earliest inbound PO ETA) so each +row carries ONE verb action: Ship (stock is on hand) / Expedite the PO (inbound exists) / +Call the customer (no supply in sight). Line-level fill canon: ≥95% industry bar. + +Scope: wholesale teams (5/6), GIFTWARE/Amazon excluded — a backorder board is about promises +to dealers. BU-filterable. +""" +import datetime as dt + +import core.odoo as O +import core.periods as P +import modules.procurement as proc + +BUCKETS = ['not due', '0-7', '8-21', '22-45', '45+', 'no promise'] +_OPEN_PICK_DOMAIN = [('state', 'in', ['confirmed', 'assigned', 'waiting']), + ('picking_type_id.code', '=', 'outgoing')] + + +def _chunk(ids, n=2000): + ids = list(ids) + for i in range(0, len(ids), n): + yield ids[i:i + n] + + +def _open_pickings(): + """Open outgoing pickings → {origin(SO name): earliest scheduled date}.""" + picks = O.search_read('stock.picking', list(_OPEN_PICK_DOMAIN), + ['origin', 'scheduled_date']) + sched = {} + for p in picks: + o = (p.get('origin') or '').strip() + if not o: + continue + d = str(p.get('scheduled_date') or '')[:10] + if d and (o not in sched or d < sched[o]): + sched[o] = d + return picks, sched + + +def board(team_id=None, t=None): + """The full open-line board + bucket rollup. Returns dict(rows, buckets, orders, coverage).""" + t = t or P.today() + today = t + picks, sched = _open_pickings() + + # the sale orders behind those pickings, in wholesale scope + orders = [] + names = list(sched) + ex = O.excluded_partner_ids() + for ch in _chunk(names, 500): + dom = [('name', 'in', ch), ('state', 'in', ['sale', 'done'])] + dom.append(('team_id', '=', team_id) if team_id is not None + else ('team_id', 'in', O.TEAM_IDS)) + if ex: + dom.append(('partner_id', 'not in', list(ex))) + orders += O.search_read('sale.order', dom, + ['name', 'partner_id', 'team_id', 'date_order', + 'commitment_date']) + by_oid = {o['id']: o for o in orders} + + # their lines, open qty computed locally (a domain cannot compare two fields) + lines = [] + for ch in _chunk(list(by_oid), 500): + lines += O.search_read('sale.order.line', + [('order_id', 'in', ch), ('display_type', '=', False), + ('product_id', '!=', False)], + ['order_id', 'order_partner_id', 'product_id', + 'product_uom_qty', 'qty_delivered', 'price_unit', + 'price_subtotal', 'name']) + open_lines = [] + for r in lines: + openq = (r.get('product_uom_qty') or 0.0) - (r.get('qty_delivered') or 0.0) + if openq <= 1e-3: + continue + qty = r.get('product_uom_qty') or 0.0 + unit = (r.get('price_subtotal') or 0.0) / qty if qty else (r.get('price_unit') or 0.0) + r['open_qty'] = openq + r['open_value'] = openq * unit + open_lines.append(r) + + # supply joins: on-hand + earliest inbound ETA per product + pids = list({O.m2o_id(r['product_id']) for r in open_lines}) + onhand, codes = {}, {} + for ch in _chunk(pids): + for p in O.search_read('product.product', + [('id', 'in', ch), ('active', 'in', [True, False])], + ['qty_available', 'default_code']): + onhand[p['id']] = p.get('qty_available') or 0.0 + codes[p['id']] = (p.get('default_code') or '').strip() + _all_pol, pol = proc._open_po_lines() + inbound = {} + for l in pol: + pid = O.m2o_id(l.get('product_id')) + if not pid: + continue + cur = inbound.setdefault(pid, {'qty': 0.0, 'eta': None}) + cur['qty'] += l.get('open_qty') or 0.0 + eta = l.get('expected') or '' + if eta and (cur['eta'] is None or eta < cur['eta']): + cur['eta'] = eta + + rows = [] + n_promised = 0 + for r in open_lines: + oid = O.m2o_id(r['order_id']) + o = by_oid.get(oid) or {} + promise = str(o.get('commitment_date') or '')[:10] or sched.get(o.get('name', ''), '') + if str(o.get('commitment_date') or '')[:10]: + n_promised += 1 + if promise: + days = (today - dt.date.fromisoformat(promise)).days + bucket = ('not due' if days < 0 else '0-7' if days <= 7 + else '8-21' if days <= 21 else '22-45' if days <= 45 else '45+') + else: + days, bucket = None, 'no promise' + pid_prod = O.m2o_id(r['product_id']) + oh = onhand.get(pid_prod, 0.0) + inb = inbound.get(pid_prod) + if oh >= r['open_qty']: + action = 'Ship — stock on hand' + elif inb and inb.get('eta'): + action = f"Expedite PO — ETA {inb['eta']}" + else: + action = 'Call customer — no supply inbound' + rows.append({ + 'pid': O.m2o_id(r['order_partner_id']), + 'customer': O.m2o_name(r['order_partner_id']), + 'order': o.get('name', ''), + 'bu': O.TEAM_NAMES.get(O.m2o_id(o.get('team_id')), '?'), + 'sku': codes.get(pid_prod, ''), 'product': O.m2o_name(r['product_id']), + 'open_qty': r['open_qty'], 'open_value': r['open_value'], + 'promise': promise or '—', 'days_past': days if days is not None else '', + 'bucket': bucket, 'on_hand': oh, + 'inbound_eta': (inb or {}).get('eta') or '', + 'action': action, + }) + rows.sort(key=lambda x: -x['open_value']) + bucket_rollup = [] + for bu in sorted({r['bu'] for r in rows}): + for b in BUCKETS: + v = sum(r['open_value'] for r in rows if r['bu'] == bu and r['bucket'] == b) + if v: + bucket_rollup.append({'bu': bu, 'bucket': b, 'value': v}) + return { + 'rows': rows, + 'buckets': bucket_rollup, + 'n_orders': len({r['order'] for r in rows}), + 'open_value': sum(r['open_value'] for r in rows), + 'past_value': sum(r['open_value'] for r in rows + if r['bucket'] in ('0-7', '8-21', '22-45', '45+')), + 'promise_coverage': (n_promised / len(open_lines) * 100.0) if open_lines else 0.0, + 'n_pickings': len(picks), + } + + +def validate(t=None, team_id=None): + """(1) the picking pull is complete (search_count ties the row pull — truncation guard); + (2) the local open-qty arithmetic ties Odoo's own aggregates: for the pulled order set, + Σ(product_uom_qty) − Σ(qty_delivered) via server-side read_group == Σ of our per-row + open_qty (same universe, independent arithmetic path).""" + b = board(team_id, t) + n_count = O.get_odoo().search_count('stock.picking', list(_OPEN_PICK_DOMAIN)) + checks = [{'check': 'open outgoing pickings — pull complete', + 'a': b['n_pickings'], 'b': n_count, + 'gap': b['n_pickings'] - n_count, 'ok': b['n_pickings'] == n_count}] + oids = sorted({r['order'] for r in b['rows']}) + if oids: + dom = [('order_id.name', 'in', oids[:500]), ('display_type', '=', False), + ('product_id', '!=', False)] + try: + g = O.read_group('sale.order.line', dom, + ['product_uom_qty:sum', 'qty_delivered:sum'], [], lazy=False) + srv_open = ((g[0].get('product_uom_qty') or 0.0) + - (g[0].get('qty_delivered') or 0.0)) if g else 0.0 + # NET open recomputed row-level (includes over-shipped negatives, which the board + # excludes) so both sides measure the same universe + rl = O.search_read('sale.order.line', dom, ['product_uom_qty', 'qty_delivered']) + ours = sum((l.get('product_uom_qty') or 0.0) - (l.get('qty_delivered') or 0.0) + for l in rl) + checks.append({'check': 'open qty — server aggregate vs row arithmetic', + 'a': round(ours, 2), 'b': round(srv_open, 2), + 'gap': round(ours - srv_open, 2), + 'ok': abs(ours - srv_open) < 1.0}) + except Exception as e: + checks.append({'check': 'open qty aggregate (read_group)', 'a': '—', + 'b': str(e)[:60], 'gap': '', 'ok': False}) + return checks diff --git a/royalimports_os/modules/bookings.py b/royalimports_os/modules/bookings.py new file mode 100644 index 0000000000000000000000000000000000000000..1cc3a839c6a9e264df56e3e768ed06694aaa350a --- /dev/null +++ b/royalimports_os/modules/bookings.py @@ -0,0 +1,137 @@ +"""Order Book — pre-season booking coverage (the forward view sitting unused in sale.order). + +Giftware pre-books: confirmed orders carry FUTURE delivery windows (commitment_date), so the +order book for the season is known months ahead. The curve compares THIS year's cumulative +booked $ for season delivery against last year's same-week curve — 20% under with 12 weeks to +go = cut the container program NOW; over = expand POs / push laggard agents. + +Season window: deliveries Aug 1 – Dec 31 (the Q4 giftware peak; env-overridable later if the +business adds a spring season). x-axis = weeks before Oct 1 (nominal peak). +""" +import datetime as dt + +import core.odoo as O +import core.periods as P + +SEASON_FROM = (8, 1) # deliveries from Aug 1 +SEASON_TO = (12, 31) # through Dec 31 +PEAK = (10, 1) # nominal peak for the weeks-before axis + + +def _season_orders(year, team_id=None, cutoff=None): + """Confirmed orders with commitment_date inside `year`'s season window, booked up to + `cutoff` (order date). Returns [(order_date, amount)].""" + ex = O.excluded_partner_ids() + dom = [('state', 'in', ['sale', 'done']), + ('commitment_date', '>=', f'{year}-{SEASON_FROM[0]:02d}-{SEASON_FROM[1]:02d} 00:00:00'), + ('commitment_date', '<=', f'{year}-{SEASON_TO[0]:02d}-{SEASON_TO[1]:02d} 23:59:59')] + dom.append(('team_id', '=', team_id) if team_id is not None + else ('team_id', 'in', O.TEAM_IDS)) + if cutoff: + dom.append(('date_order', '<=', f'{cutoff} 23:59:59')) + if ex: + dom.append(('partner_id', 'not in', list(ex))) + rows = O.search_read('sale.order', dom, ['date_order', 'amount_untaxed']) + return [(str(r.get('date_order') or '')[:10], r.get('amount_untaxed') or 0.0) + for r in rows if r.get('date_order')] + + +def curve(team_id=None, t=None, years_back=2): + """Cumulative booked-$ curves, one per season year, aligned on weeks-before-peak. + The current year's curve stops at today; prior years run to their season end.""" + t = t or P.today() + ty = t.year + out_rows, totals = [], {} + for yr in range(ty - years_back, ty + 1): + peak = dt.date(yr, *PEAK) + cutoff = t.isoformat() if yr == ty else None + orders = _season_orders(yr, team_id, cutoff) + orders.sort() + cum = 0.0 + weekly = {} + for d, amt in orders: + cum += amt + wk = (peak - dt.date.fromisoformat(d)).days // 7 + weekly[wk] = cum # last write per week = cumulative at week end + for wk, v in sorted(weekly.items(), reverse=True): + out_rows.append({'year': str(yr), 'weeks_before_peak': -wk, 'booked': v}) + totals[str(yr)] = cum + # same-week-LY comparison for the headline + wk_now = (dt.date(ty, *PEAK) - t).days // 7 + ly_same = 0.0 + for r in out_rows: + if r['year'] == str(ty - 1) and r['weeks_before_peak'] <= -wk_now: + ly_same = max(ly_same, r['booked']) + return {'rows': out_rows, 'totals': totals, + 'ty': str(ty), 'ly': str(ty - 1), + 'booked_ty': totals.get(str(ty), 0.0), 'ly_same_week': ly_same, + 'weeks_to_peak': wk_now} + + +def by_category(team_id=None, t=None): + """Booked $ per product category, this season TY vs LY-as-of-the-same-date. + (read_group cannot group by a dot-path — group by product, map to category locally.)""" + t = t or P.today() + ty = t.year + per_prod = {} + for yr, cut in ((ty, t.isoformat()), + (ty - 1, t.replace(year=ty - 1).isoformat())): + ex = O.excluded_partner_ids() + dom = [('order_id.state', 'in', ['sale', 'done']), + ('order_id.commitment_date', '>=', + f'{yr}-{SEASON_FROM[0]:02d}-{SEASON_FROM[1]:02d} 00:00:00'), + ('order_id.commitment_date', '<=', + f'{yr}-{SEASON_TO[0]:02d}-{SEASON_TO[1]:02d} 23:59:59'), + ('order_id.date_order', '<=', f'{cut} 23:59:59'), + ('product_id', '!=', False)] + dom.append(('order_id.team_id', '=', team_id) if team_id is not None + else ('order_id.team_id', 'in', O.TEAM_IDS)) + if ex: + dom.append(('order_partner_id', 'not in', list(ex))) + for g in O.read_group('sale.order.line', dom, ['price_subtotal:sum'], + ['product_id'], lazy=False): + pid = O.m2o_id(g.get('product_id')) + if not pid: + continue + e = per_prod.setdefault(pid, {'ty': 0.0, 'ly': 0.0}) + e['ty' if yr == ty else 'ly'] += g.get('price_subtotal') or 0.0 + cats = {} + pids = list(per_prod) + for i in range(0, len(pids), 2000): + for p in O.search_read('product.product', + [('id', 'in', pids[i:i + 2000]), + ('active', 'in', [True, False])], ['categ_id']): + cats[p['id']] = O.m2o_name(p.get('categ_id')) or '(none)' + out = {} + for pid, v in per_prod.items(): + cat = cats.get(pid, '(none)') + e = out.setdefault(cat, {'category': cat, 'ty': 0.0, 'ly': 0.0}) + e['ty'] += v['ty'] + e['ly'] += v['ly'] + rows = list(out.values()) + for r in rows: + r['delta_pct'] = ((r['ty'] / r['ly'] - 1) * 100.0) if r['ly'] else None + rows.sort(key=lambda x: -x['ty']) + return rows + + +def validate(t=None, team_id=None): + """The client-side cumulative end point ties one server-side aggregate over the exact + same domain (sum_field) — the independent arithmetic path.""" + t = t or P.today() + b = curve(team_id, t) + ty = int(b['ty']) + ex = O.excluded_partner_ids() + dom = [('state', 'in', ['sale', 'done']), + ('commitment_date', '>=', f'{ty}-{SEASON_FROM[0]:02d}-{SEASON_FROM[1]:02d} 00:00:00'), + ('commitment_date', '<=', f'{ty}-{SEASON_TO[0]:02d}-{SEASON_TO[1]:02d} 23:59:59'), + ('date_order', '<=', f'{t.isoformat()} 23:59:59')] + dom.append(('team_id', '=', team_id) if team_id is not None + else ('team_id', 'in', O.TEAM_IDS)) + if ex: + dom.append(('partner_id', 'not in', list(ex))) + srv = O.sum_field('sale.order', dom, 'amount_untaxed') + return [{'check': f'{ty} season booked $ — client cumulative vs server sum', + 'a': round(b['booked_ty'], 2), 'b': round(srv, 2), + 'gap': round(b['booked_ty'] - srv, 2), + 'ok': abs(b['booked_ty'] - srv) < 1.0}] diff --git a/royalimports_os/modules/cohort.py b/royalimports_os/modules/cohort.py new file mode 100644 index 0000000000000000000000000000000000000000..274bf0218d6d4c0e0e81dab59b21434a23b7feaf --- /dev/null +++ b/royalimports_os/modules/cohort.py @@ -0,0 +1,171 @@ +"""Cohort — a FIXED set of customers the user curates by hand (owner item 6, 2026-07-26). + +The difference from the Data module, and the only one: **a Cohort's membership is not a query.** + + Data one row per customer across the whole scoped book. Which rows you see is whatever the + view's FILTER matches, so the set moves when the data moves. + Cohort a list of customer ids somebody chose. You can still filter, sort, group and add + columns exactly as on Data — but those only narrow what is DISPLAYED. The set itself + changes only when a person adds or removes a row. + +That distinction is the whole point: a formula list silently re-populates ("who is overdue +today"), a cohort does not ("the 40 accounts we agreed to call in Q3"). If a cohort could drift, +it would not be worth having — you could just save a view. + +Rows come from `modules.customer_data.pool()` — the SAME validated metric surface the Data module +renders, so a customer's numbers cannot differ between the two pages. This module owns only +MEMBERSHIP. + +Persistence: HF store key `customer_cohorts`, shaped {username: {cohort_id: {...}}}. Per-user +like the saved views it sits beside, and written through `store.update` (strict +read-modify-write — never get+put around a mutation; that is the restart-wipe bug). + +No validate(): there is no Odoo aggregate to reconcile a hand-picked list against. Its counts +derive from the rows it renders (rule 8b), and the metrics themselves are already validated by +the customers module they come from. +""" +import datetime as _dt + +import core.store as store + +KEY = 'customer_cohorts' + +#: A cohort is a list of ids, not a query — but ids are cheap and a runaway list would bloat the +#: per-user store blob, which is read on every page render. Generous enough to never bind in +#: practice (the whole book is ~1,550) and small enough to bound the payload. +MAX_MEMBERS = 5000 +MAX_COHORTS = 100 + + +def _now(): + return _dt.datetime.now().strftime('%Y-%m-%d %H:%M') + + +def all_for(username): + """{cohort_id: {name, members, note, created, updated}} for one user. + + Returns {} when the store is unavailable rather than raising: a cohort list that cannot load + should degrade to "no cohorts", never to a broken page. + """ + try: + return dict((store.get(KEY) or {}).get(username, {}) or {}) + except Exception: + return {} + + +def _clean_members(raw, allowed_pids=None): + """Ints only, de-duplicated, ORDER PRESERVED, and intersected with what this user may see. + + ⚠ The `allowed_pids` intersection is a PERMISSION boundary, not tidying. A cohort is stored + per user but the ids in it are customer ids: if an agent's scope narrows (or a cohort is ever + shared), replaying stored ids unchecked would render rows outside their book. Filtering on + READ means a scope change takes effect immediately, without rewriting anyone's stored data. + """ + out, seen = [], set() + for v in list(raw or [])[:MAX_MEMBERS * 2]: + try: + pid = int(v) + except (TypeError, ValueError): + continue + if pid in seen: + continue + if allowed_pids is not None and pid not in allowed_pids: + continue + seen.add(pid) + out.append(pid) + return out[:MAX_MEMBERS] + + +def visible(username, allowed_pids=None): + """Every cohort, with membership already scoped to what this user may open.""" + return {cid: {**c, 'members': _clean_members(c.get('members'), allowed_pids)} + for cid, c in all_for(username).items()} + + +def new_id(name): + """A stable id from the name plus a suffix, so two cohorts called "Q3 calls" can coexist.""" + slug = ''.join(ch if ch.isalnum() else '_' for ch in str(name).lower()).strip('_')[:40] + return f"cohort_{slug or 'list'}_{_dt.datetime.now().strftime('%H%M%S')}" + + +def create(username, name, members=(), note=''): + """Create a cohort and return its id. Raises ValueError on a blank name or when the per-user + cap is reached — refusing loudly beats silently dropping somebody's list.""" + name = str(name or '').strip()[:120] + if not name: + raise ValueError('a cohort needs a name') + existing = all_for(username) + if len(existing) >= MAX_COHORTS: + raise ValueError(f'you already have {MAX_COHORTS} cohorts — delete one first') + cid = new_id(name) + record = {'name': name, 'members': _clean_members(members), 'note': str(note or '')[:2000], + 'created': _now(), 'updated': _now()} + + def _up(data): + data.setdefault(username, {})[cid] = record + return data + store.update(KEY, _up) + return cid + + +def _mutate(username, cohort_id, change): + def _up(data): + book = data.setdefault(username, {}) + c = book.get(cohort_id) + if c is None: + return data # deleted concurrently — nothing to do, not an error + change(c) + c['updated'] = _now() + return data + store.update(KEY, _up) + + +def add_members(username, cohort_id, pids): + """Union new ids into a cohort, PRESERVING the existing order and appending the new ones. + + Additive on purpose: "add to list" from a view can be clicked twice, and the second click + must be a no-op rather than a reorder or a duplicate. + """ + def _change(c): + c['members'] = _clean_members(list(c.get('members') or []) + list(pids or [])) + _mutate(username, cohort_id, _change) + + +def remove_members(username, cohort_id, pids): + drop = {int(p) for p in (pids or []) if str(p).lstrip('-').isdigit()} + + def _change(c): + c['members'] = [p for p in _clean_members(c.get('members')) if p not in drop] + _mutate(username, cohort_id, _change) + + +def rename(username, cohort_id, name): + name = str(name or '').strip()[:120] + if not name: + raise ValueError('a cohort needs a name') + _mutate(username, cohort_id, lambda c: c.__setitem__('name', name)) + + +def set_note(username, cohort_id, note): + _mutate(username, cohort_id, lambda c: c.__setitem__('note', str(note or '')[:2000])) + + +def delete(username, cohort_id): + def _up(data): + (data.get(username) or {}).pop(cohort_id, None) + return data + store.update(KEY, _up) + + +def rows_for(cohort, pool_rows): + """The cohort's rows, in MEMBERSHIP order, from an already-built pool. + + Membership order, not pool order: the user built this list, and the order they built it in is + information. Ids with no matching pool row are dropped silently — a customer can fall out of + the 24-month pool without the cohort being wrong, and showing a blank row would be worse than + showing fewer. `missing_count` lets the page say so honestly rather than hiding it. + """ + by_pid = {r['pid']: r for r in pool_rows} + members = _clean_members(cohort.get('members')) + rows = [by_pid[p] for p in members if p in by_pid] + return rows, len(members) - len(rows) diff --git a/royalimports_os/modules/collections_exec.py b/royalimports_os/modules/collections_exec.py new file mode 100644 index 0000000000000000000000000000000000000000..6f962e487e51947e23e0901ccfbc162984629d8a --- /dev/null +++ b/royalimports_os/modules/collections_exec.py @@ -0,0 +1,376 @@ +"""Collections executive module — the LIVE Biweekly Executive Collection Report. + +Replicates the Excel "Royal Imports - Collection Report - Biweekly Executive" (Summary tab time-series ++ Overdue (FFS)/(RI) lists) but computed live from Odoo (read-only). + +AR as-of any date D = the receivable-account LEDGER balance: Σ balance of posted receivable move lines +dated <= D (customer payments post a credit to the receivable account on the payment date, so the +running account balance IS the AR). Validated to ~1% of the hardcoded report; today's value ties to +Σ amount_residual exactly. BU split is by the CUSTOMER (res.partner.team_id 5=Fisch/FFS, 6=Royal/RI), +NOT the invoice team (most invoices carry a generic team). Current aging uses exact per-invoice +amount_residual; historical aging is a FIFO approximation conserved toward the netting total (the +report's exact buckets come from Odoo's specific reconciliation, which the API cannot reconstruct — +account.partial.reconcile.max_date errors server-side). READ-ONLY. +""" +import sys +import datetime as dt +from collections import defaultdict +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P + +FFS, RI = 5, 6 +BU_OF = {FFS: 'FFS', RI: 'RI'} +AGE = ['At Date', '01-30', '31-60', '61-90', '91-120', '>120'] +# Overdue-list aging (the report folds not-due + 0-30 days into "At Date"; overdue starts at 31 days). +AGE_LIST = ['At Date', '31-60', '61-90', '91-120', '>120'] + +# Cash doesn't settle same-day, so the report's "as of" / ending period lags real-time by this many +# days: the last few days of receipts haven't cleared in the bank yet and would understate collections +# and overstate receivables. The ending period is therefore today - SETTLE_OFFSET_DAYS (real-time). +SETTLE_OFFSET_DAYS = 7 + +# Targets from the report (col U). (goal_value, 'lower'|'higher' is-better) +GOALS = { + 'recv_FFS': (427708.0, 'lower'), 'recv_RI': (150789.0, 'lower'), + 'dso': (30.0, 'lower'), 'dso_FFS': (30.0, 'lower'), 'dso_RI': (30.0, 'lower'), + 'odpct_FFS': (10.0, 'lower'), 'odpct_RI': (10.0, 'lower'), 'odpct_total': (10.0, 'lower'), +} + + +def _d(s): + try: + return dt.date.fromisoformat(str(s)[:10]) + except Exception: + return None + + +def _as_of(t=None): + """The report's as-of / ending-period date: real-time minus the settlement lag (default), or an + explicit override if a caller passes one. Using plain 'today' would understate the most recent + receipts (cash takes ~SETTLE_OFFSET_DAYS to clear), so the ending period is offset back by it.""" + return t or (P.today() - dt.timedelta(days=SETTLE_OFFSET_DAYS)) + + +def _bucket(days): + if days <= 0: + return 'At Date' + if days <= 30: + return '01-30' + if days <= 60: + return '31-60' + if days <= 90: + return '61-90' + if days <= 120: + return '91-120' + return '>120' + + +def _lbucket(days): + """Overdue-list bucket — not-due or within 30 days is 'At Date'; overdue starts at 31.""" + if days <= 30: + return 'At Date' + if days <= 60: + return '31-60' + if days <= 90: + return '61-90' + if days <= 120: + return '91-120' + return '>120' + + +def periods(t=None, n=14, step=14): + """n biweekly periods ending at t (latest last). Each {label, end(date), beg(date), year, month}.""" + t = _as_of(t) + out = [] + for i in range(n): + end = t - dt.timedelta(days=step * i) + out.append({'label': end.strftime('%m/%d'), 'end': end, 'beg': end - dt.timedelta(days=step - 1), + 'year': end.year, 'month': end.month}) + return list(reversed(out)) + + +def pull(t=None): + """The heavy one-time read: receivable ledger (invoice lines + reduction lines) + partner master. + Records are normalized to plain dicts with parsed date strings + the customer's BU.""" + rec = [('account_id.account_type', '=', 'asset_receivable'), ('parent_state', '=', 'posted')] + inv = O.search_read('account.move.line', + rec + [('move_id.move_type', 'in', ['out_invoice', 'out_refund'])], + ['date', 'date_maturity', 'balance', 'amount_residual', 'partner_id'], limit=300000) + red = O.search_read('account.move.line', + rec + [('move_id.move_type', 'not in', ['out_invoice', 'out_refund'])], + ['date', 'balance', 'partner_id'], limit=300000) + ex = set(O.excluded_partner_ids()) + pids = sorted({O.m2o_id(r['partner_id']) for r in inv + red + if r.get('partner_id') and O.m2o_id(r['partner_id']) not in ex}) + parts = {p['id']: p for p in O.search_read('res.partner', [('id', 'in', pids)], + ['name', 'team_id', 'user_id', 'property_payment_term_id'])} + + def bu(pid): + return BU_OF.get(O.m2o_id(parts.get(pid, {}).get('team_id'))) + I, R = [], [] + for r in inv: + pid = O.m2o_id(r['partner_id']) + if pid in ex or pid not in parts: + continue + I.append({'pid': pid, 'bu': bu(pid), 'date': str(r.get('date') or '')[:10], + 'due': str(r.get('date_maturity') or r.get('date') or '')[:10], + 'bal': r.get('balance') or 0.0, 'resid': r.get('amount_residual') or 0.0}) + for r in red: + pid = O.m2o_id(r['partner_id']) + if pid in ex or pid not in parts: + continue + R.append({'pid': pid, 'bu': bu(pid), 'date': str(r.get('date') or '')[:10], + 'bal': r.get('balance') or 0.0}) + return {'inv': I, 'red': R, 'parts': parts, + 'pulled_at': dt.datetime.now().strftime('%Y-%m-%d %H:%M')} + + +# ---------------------------------------------------------------- as-of-date primitives +def _recv_by_partner(data, D): + """AR netting as-of D, per partner = Σ(invoice + reduction balances dated <= D).""" + bal = defaultdict(float) + for r in data['inv']: + if r['date'] <= D: + bal[r['pid']] += r['bal'] + for r in data['red']: + if r['date'] <= D: + bal[r['pid']] += r['bal'] + return bal + + +def _agg_bu(by_partner, data): + out = {'FFS': 0.0, 'RI': 0.0, 'total': 0.0} + for pid, v in by_partner.items(): + bu = BU_OF.get(O.m2o_id(data['parts'].get(pid, {}).get('team_id'))) + if bu: + out[bu] += v + out['total'] += v + return out + + +def _aging(data, D, recv=None, exact=False): + """Aging buckets ($) by BU as-of D, + overdue total by BU. exact=True uses live per-invoice + amount_residual (current period); else FIFO (payments applied oldest-due-first). When `recv` + (netting by BU) is given, the buckets are conserved to it — the difference (unapplied payments / + credits sitting in the receivable account, not tied to an invoice) lands in 'At Date'.""" + Dd = _d(D) + inv_by = defaultdict(list) + red_by = defaultdict(float) + for r in data['inv']: + if r['date'] <= D: + inv_by[r['pid']].append(r) + if not exact: + for r in data['red']: + if r['date'] <= D: + red_by[r['pid']] += r['bal'] + out = {'FFS': {b: 0.0 for b in AGE}, 'RI': {b: 0.0 for b in AGE}} + overdue = {'FFS': 0.0, 'RI': 0.0} + for pid, invs in inv_by.items(): + bu = invs[0]['bu'] + if bu not in out: + continue + if exact: + for r in invs: + amt = r['resid'] + if abs(amt) < 1e-6: + continue + days = (Dd - (_d(r['due']) or Dd)).days + out[bu][_bucket(days)] += amt + if days > 30 and amt > 0: + overdue[bu] += amt + else: + pay = -red_by.get(pid, 0.0) + pos = [] + for r in invs: + if r['bal'] >= 0: + pos.append([r['due'], r['bal']]) + else: + pay += -r['bal'] # credit notes act like reductions + pos.sort(key=lambda x: x[0]) # oldest due first + for it in pos: + if pay <= 0: + break + take = min(pay, it[1]) + it[1] -= take + pay -= take + for due, amt in pos: + if amt < 1e-6: + continue + days = (Dd - (_d(due) or Dd)).days + out[bu][_bucket(days)] += amt + if days > 30 and amt > 0: + overdue[bu] += amt + if recv is not None: + for bu in ('FFS', 'RI'): + out[bu]['At Date'] += recv.get(bu, 0.0) - sum(out[bu].values()) + return out, overdue + + +def _ltm_invoiced(data, D): + """Net invoiced (out_invoice − out_refund) in the trailing 365 days ending D, by BU — for DSO.""" + lo = (_d(D) - dt.timedelta(days=365)).isoformat() + out = {'FFS': 0.0, 'RI': 0.0, 'total': 0.0} + for r in data['inv']: + if lo < r['date'] <= D: + if r['bu']: + out[r['bu']] += r['bal'] + out['total'] += r['bal'] + return out + + +def _flows(data, beg, end): + """New sales (net invoiced) + new collection (payments) in [beg, end], by BU.""" + b, e = beg.isoformat(), end.isoformat() + sales = {'FFS': 0.0, 'RI': 0.0, 'total': 0.0} + coll = {'FFS': 0.0, 'RI': 0.0, 'total': 0.0} + for r in data['inv']: + if b <= r['date'] <= e and r['bu']: + sales[r['bu']] += r['bal'] + sales['total'] += r['bal'] + for r in data['red']: + if b <= r['date'] <= e and r['bu']: + coll[r['bu']] += -r['bal'] + coll['total'] += -r['bal'] + return sales, coll + + +def _last_payment(data): + lp = {} + for r in data['red']: + if r['bal'] < 0 and r['date']: + if r['pid'] not in lp or r['date'] > lp[r['pid']]: + lp[r['pid']] = r['date'] + return lp + + +def _idle(data, D, lp, recv_pp): + Dd = _d(D) + out = {'FFS': {'IDLE3': 0, 'IDLE6': 0, 'IDLE12': 0, 'total': 0}, + 'RI': {'IDLE3': 0, 'IDLE6': 0, 'IDLE12': 0, 'total': 0}} + for pid, bal in recv_pp.items(): + if bal <= 1: + continue + bu = BU_OF.get(O.m2o_id(data['parts'].get(pid, {}).get('team_id'))) + if bu not in out: + continue + last = lp.get(pid) + months = ((Dd - _d(last)).days / 30.0) if last else 999 + tier = 'IDLE12' if months > 12 else 'IDLE6' if months > 6 else 'IDLE3' if months > 3 else None + if tier: + out[bu][tier] += 1 + out[bu]['total'] += 1 + return out + + +def _is_cod(term_name): + return any(k in (term_name or '').lower() for k in ('immediate', 'cod', 'cash')) + + +def _credit_terms(data, recv_pp): + out = {'FFS': {'COD': 0, 'TOP': 0, 'total': 0}, 'RI': {'COD': 0, 'TOP': 0, 'total': 0}} + for pid, bal in recv_pp.items(): + if bal <= 1: + continue + p = data['parts'].get(pid, {}) + bu = BU_OF.get(O.m2o_id(p.get('team_id'))) + if bu not in out: + continue + out[bu]['COD' if _is_cod(O.m2o_name(p.get('property_payment_term_id'))) else 'TOP'] += 1 + out[bu]['total'] += 1 + return out + + +# ---------------------------------------------------------------- assembly +def summary(data, t=None): + """The biweekly time-series — a list of per-period metric dicts (latest last).""" + t = _as_of(t) + lp = _last_payment(data) + pers = periods(t) + rows = [] + for i, pr in enumerate(pers): + D = pr['end'].isoformat() + recv_pp = _recv_by_partner(data, D) + recv = _agg_bu(recv_pp, data) + aging, overdue = _aging(data, D, recv, exact=(i == len(pers) - 1)) + ltm = _ltm_invoiced(data, D) + sales, coll = _flows(data, pr['beg'], pr['end']) + dso = {bu: (recv[bu] / (ltm[bu] / 365.0)) if ltm.get(bu) else None for bu in ('FFS', 'RI', 'total')} + rows.append({'period': pr, 'recv': recv, 'overdue': overdue, 'aging': aging, 'ltm': ltm, + 'dso': dso, 'sales': sales, 'coll': coll, + 'idle': _idle(data, D, lp, recv_pp), 'terms': _credit_terms(data, recv_pp)}) + for i, r in enumerate(rows): # change in receivable vs prior period end + prev = rows[i - 1]['recv'] if i else None + r['change'] = {bu: (r['recv'][bu] - prev[bu]) if prev else None for bu in ('FFS', 'RI', 'total')} + return rows + + +def overdue_list(data, team, t=None): + """Current (as-of t) ranked overdue customers for a BU (5=FFS / 6=RI), exact per-invoice.""" + t = _as_of(t) + Dd = t + bu = BU_OF.get(team) + lp = _last_payment(data) + by = {} + for r in data['inv']: + if r['bu'] != bu: + continue + amt = r['resid'] + if abs(amt) < 1e-6: + continue + e = by.setdefault(r['pid'], dict({'pid': r['pid'], 'total': 0.0, 'overdue': 0.0}, **{b: 0.0 for b in AGE_LIST})) + e['total'] += amt + days = (Dd - (_d(r['due']) or Dd)).days + e[_lbucket(days)] += amt + if days > 30 and amt > 0: + e['overdue'] += amt + tot_od = sum(e['overdue'] for e in by.values() if e['overdue'] > 0) or 1.0 + rows = [] + for pid, e in by.items(): + if e['overdue'] <= 0: + continue + p = data['parts'].get(pid, {}) + last = lp.get(pid) + months = ((Dd - _d(last)).days / 30.0) if last else 999 + idle = 'IDLE12' if months > 12 else 'IDLE6' if months > 6 else 'IDLE3' if months > 3 else 'Active' + term = O.m2o_name(p.get('property_payment_term_id')) + rows.append(dict({'customer': p.get('name') or '(unknown)', 'odoo_id': pid, + 'sales_rep': O.m2o_name(p.get('user_id')) or '', + 'top': 'COD' if _is_cod(term) else (term or 'TOP'), + 'overdue': e['overdue'], 'overdue_pct': e['overdue'] / tot_od * 100, + 'idle': idle, 'total_recv': e['total']}, **{b: e[b] for b in AGE_LIST})) + rows.sort(key=lambda x: -x['overdue']) + for i, r in enumerate(rows, 1): + r['no'] = i + return rows + + +def goal_status(key, value): + """('goal', 'Achieved'|'Unmet') for a metric vs the report's hardcoded target, or (None, None).""" + g = GOALS.get(key) + if not g or value is None: + return None, None + target, better = g + ok = (value <= target) if better == 'lower' else (value >= target) + return target, ('Achieved' if ok else 'Unmet') + + +def validate(data, t=None): + """Aging buckets (conserved) tie to BU receivable; the Overdue list total ties to the summary's + current overdue (the two independent code paths agree).""" + t = _as_of(t) + D = t.isoformat() + recv = _agg_bu(_recv_by_partner(data, D), data) + aging, overdue = _aging(data, D, recv, exact=True) + ag_total = sum(aging['FFS'].values()) + sum(aging['RI'].values()) + checks = [{'check': 'Aging buckets (FFS+RI) == receivable (FFS+RI)', + 'a': round(ag_total, 2), 'b': round(recv['FFS'] + recv['RI'], 2), + 'gap': round(ag_total - (recv['FFS'] + recv['RI']), 2), + 'ok': abs(ag_total - (recv['FFS'] + recv['RI'])) <= 1.0}] + od_list = sum(r['overdue'] for r in overdue_list(data, FFS, t)) + sum(r['overdue'] for r in overdue_list(data, RI, t)) + od_sum = overdue['FFS'] + overdue['RI'] + checks.append({'check': 'Overdue list total == summary current overdue', + 'a': round(od_list, 2), 'b': round(od_sum, 2), 'gap': round(od_list - od_sum, 2), + 'ok': abs(od_list - od_sum) <= 1.0}) + return checks diff --git a/royalimports_os/modules/collections_send.py b/royalimports_os/modules/collections_send.py new file mode 100644 index 0000000000000000000000000000000000000000..90f9c3eb963038450485bceedb713cb05c8b573e --- /dev/null +++ b/royalimports_os/modules/collections_send.py @@ -0,0 +1,263 @@ +"""Collections statements — the send layer behind the Collections page's Statements section. + +Folded in from the standalone collections_app (2026-07-05). THE one sanctioned exception to the +app's read-only-on-Odoo rule, unchanged from the standalone tool: WRITE is whitelisted to exactly +one operation — + + create on mail.mail (queueing an outbound statement email) + +A mail.mail record with state='outgoing' is picked up by Odoo's "Mail: Email Queue Manager" cron +(every ~15 min) and delivered through the company's Office 365 relay. Because we set model/res_id, +each sent statement also appears in the customer's chatter in Odoo — the audit log lives where AR +already works. This module has its OWN narrow XML-RPC client; the app-wide odoo_client stays +hard-blocking on all writes. The UI gates the section to admin users; SAFE_MODE is enforced HERE +in the data layer so the UI cannot bypass it. + +Env (Space secrets / .env): ODOO_URL, ODOO_DB, ODOO_USER, ODOO_API_KEY +Optional: SAFE_MODE (default ON), SAFE_RECIPIENTS, SENDER_NAME, SENDER_EMAIL, REPLY_TO, +COMPANY_NAME, ROYAL_MAIL_SERVER_ID, ROYAL_AUTHOR_ID +""" +import os +import datetime as dt +import xmlrpc.client +from pathlib import Path + +try: # self-contained: load credentials from the app root .env if present (HF uses Secrets) + from dotenv import load_dotenv + load_dotenv(Path(__file__).resolve().parents[1] / '.env') +except Exception: + pass + +WRITE_WHITELIST = {('mail.mail', 'create')} + +EXCLUDE_NAMES = {'GIFTWARE DEALS'} # the Amazon channel — not part of Fisch or Royal collections +DOMAIN = [('followup_reminder_type', '=', 'automatic'), ('credit', '>', 1)] + +COMPANY = os.environ.get('COMPANY_NAME', 'Royal Imports') + +# --- Sender identity (statements go out AS Royal Imports) --- +# Odoo routes outbound mail to the matching ir.mail_server by from_filter, and the Office 365 +# relay only accepts sends as its authenticated address. The "Office 365 - Royal" server (id 2) +# authenticates as contact@royalimports.com with from_filter='contact@royalimports.com' — so the +# From MUST be that address for delivery to succeed. Friendly display name; replies routed to AR. +SENDER_NAME = os.environ.get('SENDER_NAME', 'Royal Imports Accounts Receivable') +SENDER_EMAIL = os.environ.get('SENDER_EMAIL', 'contact@royalimports.com') +REPLY_TO = os.environ.get('REPLY_TO', 'accounting@royalimports.com') +ROYAL_MAIL_SERVER_ID = int(os.environ.get('ROYAL_MAIL_SERVER_ID', '2')) +ROYAL_AUTHOR_ID = int(os.environ.get('ROYAL_AUTHOR_ID', '8978')) # "Royal Imports" partner + +SENDER_DISPLAY = f'"{SENDER_NAME}" <{SENDER_EMAIL}>' + + +class WriteBlocked(RuntimeError): + pass + + +class SafeModeBlocked(RuntimeError): + pass + + +# --- Testing guardrail ------------------------------------------------------- +# While SAFE_MODE is on, NO email can be queued to any address outside the allow-list — enforced +# here in the data layer so the UI cannot bypass it. Default: ON. To go live for real customers, +# set the Space secret SAFE_MODE=0. +SAFE_MODE = os.environ.get('SAFE_MODE', '1').strip().lower() not in ('0', 'false', 'no', '') +SAFE_RECIPIENTS = {a.strip().lower() for a in + os.environ.get('SAFE_RECIPIENTS', 'farhan@teamroyalimports.com').split(',') + if a.strip()} + + +def safe_recipient_ok(addr): + return (not SAFE_MODE) or (str(addr or '').strip().lower() in SAFE_RECIPIENTS) + + +class Odoo: + """Narrow client: read anything, write ONLY the whitelisted mail.mail create.""" + + def __init__(self): + self.url = os.environ.get('ODOO_URL', '').rstrip('/') + self.db = os.environ.get('ODOO_DB', '') + self.user = os.environ.get('ODOO_USER', '') + self.key = os.environ.get('ODOO_API_KEY', '') + missing = [k for k, v in [('ODOO_URL', self.url), ('ODOO_DB', self.db), + ('ODOO_USER', self.user), ('ODOO_API_KEY', self.key)] if not v] + if missing: + raise RuntimeError(f"Missing secrets: {', '.join(missing)}") + common = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/common') + self.uid = common.authenticate(self.db, self.user, self.key, {}) + if not self.uid: + raise RuntimeError('Odoo authentication failed') + self.models = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/object') + + def _exec(self, model, method, args, kwargs=None): + mutating = method in ('write', 'create', 'unlink', 'copy') or \ + any(method.startswith(p) for p in ('action_', 'button_', 'do_', 'send_', + 'set_', 'update_', 'process_')) + if mutating and (model, method) not in WRITE_WHITELIST: + raise WriteBlocked(f'{method} on {model} is not allowed from this app') + return self.models.execute_kw(self.db, self.uid, self.key, + model, method, args, kwargs or {}) + + def search_read(self, model, domain=None, fields=None, limit=None, order=None): + kw = {'fields': fields or []} + if limit is not None: + kw['limit'] = limit + if order: + kw['order'] = order + return self._exec(model, 'search_read', [domain or []], kw) + + def queue_mail(self, payload): + """The single allowed write: queue an outbound email.""" + return self._exec('mail.mail', 'create', [payload]) + + +# --------------------------------------------------------------- data builders +def load_collection_list(odoo): + """The saved follow-up filter (Reminders=Automatic, Receivable>$1), GIFTWARE excluded, + with priority tiers.""" + partners = odoo.search_read('res.partner', DOMAIN, + ['name', 'credit', 'total_overdue', 'followup_status', + 'followup_next_action_date', 'followup_responsible_id', + 'email', 'phone', 'mobile']) + partners = [p for p in partners if (p['name'] or '').strip().upper() not in EXCLUDE_NAMES] + pids = [p['id'] for p in partners] + + docs = odoo.search_read('account.move', + [('move_type', 'in', ['out_invoice', 'out_refund']), ('state', '=', 'posted'), + ('payment_state', 'in', ['not_paid', 'partial']), ('partner_id', 'in', pids)], + ['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due', + 'amount_total', 'amount_residual_signed']) + + today = dt.date.today() + by_partner = {} + for d in docs: + pid = d['partner_id'][0] + due = d.get('invoice_date_due') + try: + days = (today - dt.date.fromisoformat(due)).days if due else 0 + except Exception: + days = 0 + d['days_overdue'] = max(days, 0) + d['open_signed'] = d['amount_residual_signed'] + by_partner.setdefault(pid, []).append(d) + + rows = [] + for p in partners: + pid = p['id'] + odoo_overdue = float(p.get('total_overdue') or 0) + docs_p = sorted(by_partner.get(pid, []), key=lambda x: x.get('invoice_date_due') or '') + inv_overdue = sum(d['open_signed'] for d in docs_p if d['days_overdue'] > 0) + oldest = max((d['days_overdue'] for d in docs_p), default=0) + gap = inv_overdue - odoo_overdue + reconcile = abs(gap) > 50 + + if odoo_overdue >= 5000 or (odoo_overdue > 0 and oldest > 90): + tier = 'A-Urgent' + elif odoo_overdue >= 1000 or (odoo_overdue > 0 and oldest > 30): + tier = 'B-Active' + elif odoo_overdue > 0: + tier = 'C-Light' + else: + tier = 'Monitor' + + rows.append({ + 'partner_id': pid, + 'Customer': p['name'], + 'Tier': tier, + 'Overdue': odoo_overdue, + 'Receivable': float(p.get('credit') or 0), + 'Oldest (days)': oldest, + 'Open Docs': len(docs_p), + 'Email': p.get('email') or '', + 'Phone': p.get('phone') or p.get('mobile') or '', + 'Status': (p.get('followup_status') or '').replace('_', ' '), + 'Reconcile?': 'YES' if reconcile else '', + '_docs': docs_p, + }) + tier_rank = {'A-Urgent': 0, 'B-Active': 1, 'C-Light': 2, 'Monitor': 3} + rows.sort(key=lambda r: (tier_rank[r['Tier']], -r['Overdue'])) + return rows + + +# --------------------------------------------------------------- statement email +DEFAULT_SUBJECT = 'Statement of Account — {company} — {month}' +DEFAULT_INTRO = ( + 'Dear {customer},

' + 'Please find below your current statement of account with {company}. ' + 'According to our records, the following invoices remain open:' +) +DEFAULT_FOOTER = ( + 'If you have already sent payment, please disregard this notice — and thank you. ' + 'For any questions about an invoice, simply reply to this email.

' + 'Thank you for your business,
{company} — Accounts Receivable' +) + + +def render_statement_html(row, intro_tpl=DEFAULT_INTRO, footer_tpl=DEFAULT_FOOTER): + month = dt.date.today().strftime('%B %Y') + intro = intro_tpl.format(customer=row['Customer'], company=COMPANY, month=month) + footer = footer_tpl.format(customer=row['Customer'], company=COMPANY, month=month) + + lines = [] + total_open = 0.0 + for d in row['_docs']: + kind = 'Credit Note' if d['move_type'] == 'out_refund' else 'Invoice' + amt = d['open_signed'] + total_open += amt + overdue_txt = f"{d['days_overdue']}d overdue" if d['days_overdue'] > 0 else 'current' + color = '#C0392B' if d['days_overdue'] > 0 else '#1F4E78' + lines.append( + f"{d['name']} ({kind})" + f"{d.get('invoice_date') or ''}" + f"{d.get('invoice_date_due') or ''}" + f"{overdue_txt}" + f"${amt:,.2f}") + + table = ( + "" + "" + "" + "" + "" + "" + "" + + ''.join(lines) + + f"" + f"" + f"" + f"" + "
DocumentDateDueStatusOpen Balance
Total open${total_open:,.2f}
Of which overdue${row['Overdue']:,.2f}
") + + return (f"
" + f"{intro}{table}{footer}
") + + +def queue_statement(odoo, row, subject_tpl=DEFAULT_SUBJECT, + intro_tpl=DEFAULT_INTRO, footer_tpl=DEFAULT_FOOTER, + override_to=None): + """Queue one statement email in Odoo. Returns mail.mail id. + override_to: send to a different address (used by the test-send button).""" + to = override_to or row['Email'] + if not to: + raise ValueError(f"{row['Customer']} has no email address") + # Hard guardrail — refuse any recipient outside the allow-list while SAFE_MODE is on. + if not safe_recipient_ok(to): + raise SafeModeBlocked( + f"Guardrail ON: refusing to email {to}. Only {', '.join(sorted(SAFE_RECIPIENTS))} " + f"is allowed right now. (Set SAFE_MODE=0 to send to real customers.)") + month = dt.date.today().strftime('%B %Y') + subject = subject_tpl.format(customer=row['Customer'], company=COMPANY, month=month) + payload = { + 'subject': subject, + 'body_html': render_statement_html(row, intro_tpl, footer_tpl), + 'email_to': to, + 'email_from': SENDER_DISPLAY, # From: Royal Imports + 'reply_to': REPLY_TO, # replies -> AR + 'mail_server_id': ROYAL_MAIL_SERVER_ID, # force Royal O365 relay + 'author_id': ROYAL_AUTHOR_ID, # clean attribution in chatter + 'state': 'outgoing', + 'auto_delete': False, + 'model': 'res.partner', + 'res_id': row['partner_id'], + } + return odoo.queue_mail(payload) diff --git a/royalimports_os/modules/complexity.py b/royalimports_os/modules/complexity.py new file mode 100644 index 0000000000000000000000000000000000000000..92dd89ae87ad67e79cf9c88505fb8f1c07dbd908 --- /dev/null +++ b/royalimports_os/modules/complexity.py @@ -0,0 +1,270 @@ +"""SKU Complexity — the BCG tail-rationalization view (cost-waste brief rec 2). + +The canon point: revenue-proportional allocation OVERSTATES tail-SKU profitability — a $4 SKU +that generates 200 order lines, 40 picks and 3 returns costs the same to HANDLE as a $400 one. +This module re-costs every SKU by its ACTIVITY over the LTM, all-channel (the Amazon +channel-scope rule: a wholesale-only view would false-flag Amazon sellers). + +Two-tier verdict — the kill test must not rest on an estimate: + KILL CANDIDATE gm − carrying < 0 (loses money on HARD costs alone: GM minus 25%/yr + of its current inventory book value) + REVIEW hard-positive but gm − carrying − activity_cost < 0 + (underwater once the POOLED activity rate applies — + an estimate, labeled as such) + KEEP covers both. +Pooled activity rate = LTM opex (expense-type bill lines, from modules/spend.spend_cube) ÷ total +activity units (SO lines + PO lines + picks + invoice lines + return lines) — self-consistent +with the GL, an AVERAGE (includes fixed rent/insurance), therefore an upper bound; the math is +shown in the page's verify expander. Guardrails the owner should apply before killing: basket +role (does it pull baskets?) and the count-trust set — both live in other modules; the export +carries the columns to join. +""" +import core.odoo as O +import core.periods as P +import modules.spend as spend_mod +import modules.customers as cust_mod + +CARRY_RATE = 0.25 # $/yr carrying per $ of inventory book value (APQC 20-30% norm) + + +def _chunk(ids, n=2000): + ids = list(ids) + for i in range(0, len(ids), n): + yield ids[i:i + n] + + +def _count_by_product(model, domain): + out = {} + for g in O.read_group(model, domain, ['id'], ['product_id'], lazy=False): + pid = O.m2o_id(g.get('product_id')) + if pid: + out[pid] = g.get('__count') or 0 + return out + + +def build(t=None): + t = t or P.today() + lf, lt = P.ltm(t) + ex = O.excluded_partner_ids() + + # sales: lines, qty, revenue, margin — ALL channels (no team filter), house accounts + # excluded, PHYSICAL products only (service/delivery pseudo-products are not SKUs and + # must not appear on a kill-list) + phys = ('product_id.type', '=', 'product') + sol_dom = [('state', 'in', ['sale', 'done']), ('product_id', '!=', False), phys, + ('order_id.date_order', '>=', f'{lf} 00:00:00'), + ('order_id.date_order', '<=', f'{lt} 23:59:59')] + if ex: + sol_dom.append(('order_partner_id', 'not in', list(ex))) + sales = {} + for g in O.read_group('sale.order.line', sol_dom, + ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], + ['product_id'], lazy=False): + pid = O.m2o_id(g.get('product_id')) + if pid: + sales[pid] = {'so_lines': g.get('__count') or 0, + 'rev': g.get('price_subtotal') or 0.0, + 'qty': g.get('product_uom_qty') or 0.0, + 'gm': g.get('margin') or 0.0} + + po_lines = _count_by_product('purchase.order.line', + [('state', 'in', ['purchase', 'done']), + ('product_id', '!=', False), phys, + ('order_id.date_order', '>=', f'{lf} 00:00:00'), + ('order_id.date_order', '<=', f'{lt} 23:59:59')]) + picks = _count_by_product('stock.move', + [('state', '=', 'done'), ('product_id', '!=', False), phys, + ('date', '>=', f'{lf} 00:00:00'), + ('date', '<=', f'{lt} 23:59:59'), + ('picking_id.picking_type_id.code', '=', 'outgoing')]) + inv_lines = _count_by_product('account.move.line', + [('move_id.move_type', '=', 'out_invoice'), + ('parent_state', '=', 'posted'), + ('product_id', '!=', False), phys, + ('move_id.invoice_date', '>=', lf), + ('move_id.invoice_date', '<=', lt)]) + ret_lines = _count_by_product('account.move.line', + [('move_id.move_type', '=', 'out_refund'), + ('parent_state', '=', 'posted'), + ('product_id', '!=', False), phys, + ('move_id.invoice_date', '>=', lf), + ('move_id.invoice_date', '<=', lt)]) + + # current inventory book value per product (valuation layers sum = current book) + book = {} + for g in O.read_group('stock.valuation.layer', [], ['value:sum'], ['product_id'], + lazy=False): + pid = O.m2o_id(g.get('product_id')) + if pid: + book[pid] = g.get('value') or 0.0 + + # the pooled activity rate — opex (expense-type) ÷ total activity units, GL-consistent + cube = spend_mod.spend_cube(t) + opex_pool = cube['spend_total'] + pids = set(sales) | set(po_lines) | set(picks) | set(inv_lines) | set(ret_lines) | \ + {p for p, v in book.items() if abs(v) > 1} + total_units = sum(sales.get(p, {}).get('so_lines', 0) + po_lines.get(p, 0) + + picks.get(p, 0) + inv_lines.get(p, 0) + ret_lines.get(p, 0) + for p in pids) + rate = (opex_pool / total_units) if total_units else 0.0 + + meta = {} + for ch in _chunk(list(pids)): + for p in O.search_read('product.product', + [('id', 'in', ch), ('active', 'in', [True, False])], + ['default_code', 'name', 'categ_id']): + meta[p['id']] = p + + rows = [] + for pid in pids: + s = sales.get(pid, {'so_lines': 0, 'rev': 0.0, 'qty': 0.0, 'gm': 0.0}) + units = (s['so_lines'] + po_lines.get(pid, 0) + picks.get(pid, 0) + + inv_lines.get(pid, 0) + ret_lines.get(pid, 0)) + bv = max(book.get(pid, 0.0), 0.0) + carrying = bv * CARRY_RATE + activity = units * rate + adj_hard = s['gm'] - carrying + adj_full = adj_hard - activity + if adj_hard < 0 and (bv > 0 or s['rev'] > 0): + verdict = 'KILL CANDIDATE' + elif adj_full < 0: + verdict = 'REVIEW' + else: + verdict = 'KEEP' + m = meta.get(pid, {}) + rows.append({'pid': pid, 'code': (m.get('default_code') or '').strip() or f'#{pid}', + 'product': m.get('name') or '', 'category': O.m2o_name(m.get('categ_id')), + 'rev': s['rev'], 'gm': s['gm'], 'so_lines': s['so_lines'], + 'po_lines': po_lines.get(pid, 0), 'picks': picks.get(pid, 0), + 'ret_lines': ret_lines.get(pid, 0), 'units_activity': units, + 'book_value': bv, 'carrying': carrying, 'activity_cost': activity, + 'adj_hard': adj_hard, 'adj_full': adj_full, 'verdict': verdict}) + rows.sort(key=lambda x: x['adj_full']) + + # whale curve: cumulative FULL-adjusted profit by descending adj_full rank + ranked = sorted(rows, key=lambda x: -x['adj_full']) + cum, whale = 0.0, [] + for i, r in enumerate(ranked, 1): + cum += r['adj_full'] + if i % max(1, len(ranked) // 200) == 0 or i == len(ranked): + whale.append({'rank': i, 'cum_profit': cum}) + peak = max((w['cum_profit'] for w in whale), default=0.0) + + n_kill = sum(1 for r in rows if r['verdict'] == 'KILL CANDIDATE') + n_rev = sum(1 for r in rows if r['verdict'] == 'REVIEW') + return { + 'rows': rows, 'whale': whale, 'peak_profit': peak, + 'final_profit': cum, 'n_skus': len(rows), + 'n_kill': n_kill, 'n_review': n_rev, + 'kill_book_value': sum(r['book_value'] for r in rows + if r['verdict'] == 'KILL CANDIDATE'), + 'kill_carrying': sum(r['carrying'] for r in rows if r['verdict'] == 'KILL CANDIDATE'), + 'rate': rate, 'opex_pool': opex_pool, 'total_units': total_units, + 'window': (lf, lt), + } + + +def impact(pre, t=None, verdict='KILL CANDIDATE'): + """Who feels it if we kill: LTM revenue on the verdict SKUs by CUSTOMER (with their + share-of-book, so a dependency reads differently from a nuisance) and rolled up by AGENT. + Σ(customer stake) == Σ(agent stake) == Σ(verdict SKUs' revenue) — the ties are asserted + in validate(). Same scope as build(): all channels, physical products, house excluded.""" + t = t or P.today() + lf, lt = pre['window'] + ex = O.excluded_partner_ids() + kill_pids = [r['pid'] for r in pre['rows'] if r['verdict'] == verdict] + + base = [('state', 'in', ['sale', 'done']), ('product_id', '!=', False), + ('product_id.type', '=', 'product'), + ('order_id.date_order', '>=', f'{lf} 00:00:00'), + ('order_id.date_order', '<=', f'{lt} 23:59:59')] + if ex: + base.append(('order_partner_id', 'not in', list(ex))) + + # revenue on the verdict SKUs per (customer × SKU) — chunked (grouped reads over large + # id-domains echo the domain per group → server MemoryError) + per_cust = {} + for ch in _chunk(kill_pids, 400): + for g in O.read_group('sale.order.line', base + [('product_id', 'in', ch)], + ['price_subtotal:sum'], ['order_partner_id', 'product_id'], + lazy=False): + pid = O.m2o_id(g.get('order_partner_id')) + if not pid: + continue + e = per_cust.setdefault(pid, {'pid': pid, + 'customer': O.m2o_name(g.get('order_partner_id')), + 'rev_stake': 0.0, 'skus': set()}) + e['rev_stake'] += g.get('price_subtotal') or 0.0 + e['skus'].add(O.m2o_id(g.get('product_id'))) + + # each affected customer's TOTAL book (same scope) for the share-of-book column + book = {} + for g in O.read_group('sale.order.line', base, ['price_subtotal:sum'], + ['order_partner_id'], lazy=False): + pid = O.m2o_id(g.get('order_partner_id')) + if pid: + book[pid] = g.get('price_subtotal') or 0.0 + + attrs = cust_mod._partner_attrs(list(per_cust)) + by_customer = [] + for e in per_cust.values(): + total = book.get(e['pid'], 0.0) + by_customer.append({'pid': e['pid'], 'customer': e['customer'], + 'agent': (attrs.get(e['pid']) or {}).get('agent') or '(none)', + 'rev_stake': e['rev_stake'], 'n_skus': len(e['skus']), + 'book_rev': total, + 'share_pct': (e['rev_stake'] / total * 100) if total else None}) + by_customer.sort(key=lambda x: -x['rev_stake']) + + by_agent = {} + for r in by_customer: + a = by_agent.setdefault(r['agent'], {'agent': r['agent'], 'customers': 0, + 'rev_stake': 0.0, 'book_rev': 0.0}) + a['customers'] += 1 + a['rev_stake'] += r['rev_stake'] + a['book_rev'] += r['book_rev'] + agents = [{**a, 'share_pct': (a['rev_stake'] / a['book_rev'] * 100) + if a['book_rev'] else None} for a in by_agent.values()] + agents.sort(key=lambda x: -x['rev_stake']) + return {'by_customer': by_customer, 'by_agent': agents, + 'stake_total': sum(r['rev_stake'] for r in by_customer), + 'n_customers': len(by_customer), 'verdict': verdict} + + +def validate(t=None, team_id=None, pre=None): + """Revenue and margin tie the server aggregates over the same domain; book value ties the + server SVL sum (the GL-05000 figure).""" + t = t or P.today() + b = pre or build(t) + lf, lt = b['window'] + ex = O.excluded_partner_ids() + dom = [('state', 'in', ['sale', 'done']), ('product_id', '!=', False), + ('product_id.type', '=', 'product'), + ('order_id.date_order', '>=', f'{lf} 00:00:00'), + ('order_id.date_order', '<=', f'{lt} 23:59:59')] + if ex: + dom.append(('order_partner_id', 'not in', list(ex))) + checks = [] + srv_rev = O.sum_field('sale.order.line', dom, 'price_subtotal') + a_rev = sum(r['rev'] for r in b['rows']) + checks.append({'check': 'complexity: Σ SKU revenue == server Σ (all-channel LTM)', + 'a': round(a_rev, 2), 'b': round(srv_rev, 2), + 'gap': round(a_rev - srv_rev, 2), + 'ok': abs(a_rev - srv_rev) <= max(1.0, srv_rev * 0.001)}) + srv_bv = O.sum_field('stock.valuation.layer', [], 'value') + a_bv = sum(r['book_value'] for r in b['rows']) + checks.append({'check': 'complexity: Σ SKU book value == server Σ valuation layers ' + '(negatives clamped per SKU — gap = clamp effect)', + 'a': round(a_bv, 2), 'b': round(srv_bv, 2), + 'gap': round(a_bv - srv_bv, 2), + 'ok': a_bv >= srv_bv - 1.0}) + imp = b.get('impact') + if imp: + kill_rev = sum(r['rev'] for r in b['rows'] if r['verdict'] == imp['verdict']) + for key in ('by_customer', 'by_agent'): + s = sum(r['rev_stake'] for r in imp[key]) + checks.append({'check': f'kill-impact {key} == Σ(kill SKUs revenue)', + 'a': round(s, 2), 'b': round(kill_rev, 2), + 'gap': round(s - kill_rev, 2), + 'ok': abs(s - kill_rev) <= max(1.0, kill_rev * 0.001)}) + return checks \ No newline at end of file diff --git a/royalimports_os/modules/customer_data.py b/royalimports_os/modules/customer_data.py new file mode 100644 index 0000000000000000000000000000000000000000..3b357e17ac024d2a37cae51b8d9d6ff3701791de --- /dev/null +++ b/royalimports_os/modules/customer_data.py @@ -0,0 +1,532 @@ +"""Customer List — build, filter and save customer lists (replaces My Day, owner IA 2026-07-23). + +The page is a customer-list WORKBENCH in two parts: + 1. BUILDER — the whole scoped book, one row per customer with a consistent metric surface + (YTD / LY revenue, at-risk $, cadence, overdue days, estimated missed $ …): filter, sort, + SELECT customers and add them to a list. + 2. LISTS — saved lists whose membership is a visible, editable FORMULA (a rule set over the + metric columns) plus hand-picked members. 'Call list' and 'Win-back' ship as TEMPLATES — + the exact formulas the old My Day queues used — and the user can change or reset them. + +Composes already-validated customers-module functions; no separate validate() — the page's +counts derive FROM the lists it renders (rule 8b). Persistence: HF store key 'customer_lists' +({username: {list_name: {rules, sort, members, note}}}); templates are virtual until edited. + +The user↔agent link (users.py 'agent' field) scopes the pool to that agent's book; users with +no link (owner/CFO/admin) see the whole book. +""" +import core.odoo as O +import core.periods as P +import core.store as store +import core.table_store as table_store +import modules.ar as ar +import modules.customers as cust +import modules.map as map_mod +import modules.sales as S + +LIMIT = 500 # honest builder display cap — always shown with the full count, never silent +KEY = 'customer_lists' +TABLE_KEY = 'customer_table_workspace' + +# ------------------------------------------------------------------ the metric pool +# Field metadata for the rule builder: key -> (label, kind). 'num' fields take >=/<=/>/=', '<=', '>', '<', '=') +TEXT_OPS = ('contains', '=') + + +def agent_pids(agent_name): + """Partner ids of an agent's book (None = unscoped/whole book).""" + return cust.agent_partner_ids(agent_name) if agent_name else None + + +def _ar_attrs(t=None): + """{pid: {ar_open, ar_overdue, ar_exposure}} — the AR / credit-exposure family. + + Composed from `modules/ar.credit_exposure()`, NOT re-derived: it is already reconciled to + Odoo by `ar.validate()`, so the Customer table and the Collections page cannot disagree about + what a customer owes. Read from the UNFILTERED `_all_rows` — `rows` is a display-truncated + top-40, and a column built off that would silently blank everybody else. + + ⚠ `credit_limit` is deliberately NOT a column: 20 of 1,548 customers have one. EXPOSURE is + the number that answers "how much are we out on this customer", and it needs no limit. + + ⚠ `days_to_pay` is NOT COMPUTED here — it is READ from a nightly snapshot. MEASURED + 2026-07-27: `ar.credit_exposure()` costs 9s; `ar.days_to_pay()` costs **301s**, because + settlement date means walking three years of reconciled receivable lines and their + full-reconcile groups. Computing it in `pool()` would add five minutes to every Customer-table + build and every Space container start; computing it LAZILY would just move those five minutes + onto whoever opened the page first. So the expensive half runs on the app's existing + background store-sync thread (`ar.refresh_days_to_pay_snapshot`) and this reads the answer + out of the store for nothing. A missing snapshot yields a blank column, never a slow page. + """ + out = {} + try: + exp = ar.credit_exposure(t) + for r in exp.get('_all_rows') or []: + out[r['pid']] = { + 'ar_open': r.get('open', 0.0) or 0.0, + 'ar_overdue': r.get('overdue', 0.0) or 0.0, + 'ar_exposure': r.get('exposure', 0.0) or 0.0, + } + except Exception: + pass + for pid, avg in ar.days_to_pay_snapshot().items(): + # A customer with an AR row but no settled invoice in the window keeps `None` — blank, + # not 0. "Pays in 0 days" is a claim, and a false one. + out.setdefault(pid, {})['days_to_pay'] = avg + return out + + +def _mix_attrs(pids=None, t=None): + """{pid: {top_category, sku_count, top_sku, top_category_pct}} — the product-mix family. + + ONE read_group over sale.order.line by (partner, product) across the LTM window, then the + product->category map, then aggregation in Python. Line grain is the only grain that can + answer "what does this customer buy", and it is deliberately ALL-CHANNEL scope-wise for the + same reason inventory analysis is (see [[ri-channel-scope-amazon]]): a customer's product mix + is a fact about the customer, not about a sales team. + + `top_category_pct` is that category's share of the customer's LTM line revenue — the number + that says whether "top" means dominant or merely first. + + ⚠ There is NO `team_id` parameter, and its absence is the point rather than an omission: for + a BU-scoped viewer the revenue columns beside this one ARE team-scoped while these are not, + so "top category" can name products bought through a channel that viewer's revenue figures + exclude. That is the same trade the inventory modules make, and it is the right one — a + customer's product mix is a fact about the customer. + """ + t = t or P.today() + mf, mt = P.ltm(t) + # ⚠ `pids` NARROWS THE RESULT, NEVER THE DOMAIN — and that is not a style choice. Putting + # the caller's ~1,500 partner ids into an `in` clause on a LINE-grain read_group makes Odoo + # answer **502 Bad Gateway**; the same query unfiltered returns in 22s. Measured 2026-07-27, + # after the filtered form took the live Customer page past 20 MINUTES without rendering. + # The grouped result is per-partner anyway, so dropping the extras in Python costs nothing. + dom = [('order_id.state', 'in', ['sale', 'done']), + ('order_id.date_order', '>=', str(mf)), ('order_id.date_order', '<=', str(mt))] + keep = set(pids) if pids else None + rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'], + ['order_partner_id', 'product_id'], lazy=False) + prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')} + prod_ids.discard(None) + cats = {} + plist = list(prod_ids) + for i in range(0, len(plist), 5000): + for p in O.search_read('product.product', [('id', 'in', plist[i:i + 5000])], ['categ_id']): + # `categ_id`'s display name is the full PATH ("All / Foams & Finishes / Styrofoam"). + # The column takes the LEAF, because a column and a group label have to be readable — + # and the path's own root ("All") carries no information. ⚠ Two categories that share + # a leaf name would merge in a group-by; measured against this catalogue that does not + # happen, and readability is worth more than defending against a rename. + full = O.m2o_name(p.get('categ_id')) or '' + cats[p['id']] = (full.split(' / ')[-1].strip() or '(none)') if full else '(none)' + + per = {} + for r in rows: + pid = O.m2o_id(r.get('order_partner_id')) + prod = O.m2o_id(r.get('product_id')) + if pid is None or prod is None or (keep is not None and pid not in keep): + continue + val = r.get('price_subtotal', 0.0) or 0.0 + e = per.setdefault(pid, {'skus': set(), 'by_cat': {}, 'by_sku': {}, 'total': 0.0}) + e['skus'].add(prod) + e['by_cat'][cats.get(prod, '(none)')] = e['by_cat'].get(cats.get(prod, '(none)'), 0.0) + val + e['by_sku'][prod] = e['by_sku'].get(prod, 0.0) + val + e['total'] += val + + top_prod_ids = {max(e['by_sku'], key=e['by_sku'].get) for e in per.values() if e['by_sku']} + names = {} + tlist = list(top_prod_ids) + for i in range(0, len(tlist), 5000): + for p in O.search_read('product.product', [('id', 'in', tlist[i:i + 5000])], ['name']): + names[p['id']] = p.get('name') or '' + + out = {} + for pid, e in per.items(): + cat = max(e['by_cat'], key=e['by_cat'].get) if e['by_cat'] else '(none)' + sku = max(e['by_sku'], key=e['by_sku'].get) if e['by_sku'] else None + out[pid] = { + 'top_category': cat, + 'top_category_pct': (e['by_cat'][cat] / e['total']) if e['total'] else None, + 'sku_count': len(e['skus']), + 'top_sku': names.get(sku, '(none)') if sku is not None else '(none)', + } + return out + + +def _salesperson_attrs(pids=None, t=None): + """{pid: salesperson} — the DOMINANT order-taker over LTM, by order count. + + ⚠ Salesperson is NOT the Agent, and they are not interchangeable (owner, 2026-07-27): + the SALESPERSON is whoever entered and took the order, the AGENT is the person the customer + is assigned to and who earns the commission — including when the customer orders through the + office and somebody else keys it in. So this reads `sale.order.user_id` (the ORDER's taker, + fully populated: 20 people over $6.2M LTM) and NOT `res.partner.user_id`, which is a + different, near-empty field. + + A customer can have several over a year, so the column takes the one who took the MOST of + their orders — a single value the table can group and filter cleanly. Ties break toward the + larger revenue. + """ + t = t or P.today() + mf, mt = P.ltm(t) + # Same rule as `_mix_attrs`: the pid list narrows the RESULT, not the domain. Order grain is + # far smaller than line grain so this one never actually 502'd, but a caller's id list has no + # business being pushed into a query whose grouping already answers per partner. + keep = set(pids) if pids else None + dom = S.order_domain(str(mf), str(mt), None) + rows = O.read_group('sale.order', dom, ['amount_untaxed:sum'], + ['partner_id', 'user_id'], lazy=False) + per = {} + for r in rows: + pid = O.m2o_id(r.get('partner_id')) + if pid is None or (keep is not None and pid not in keep): + continue + who = O.m2o_name(r.get('user_id')) or '(none)' + n, rev = r.get('__count', 0) or 0, r.get('amount_untaxed', 0.0) or 0.0 + cur = per.get(pid) + if cur is None or (n, rev) > (cur[1], cur[2]): + per[pid] = (who, n, rev) + return {pid: v[0] for pid, v in per.items()} + + +def pool(agent_name=None, team_id=None): + """One row per customer across the scoped book: the union of YTD buyers, same-period-LY + buyers and anyone with an order in the 24-month cadence window — so lapsed/win-back + accounts are in the pool, not just YTD actives. Reuses the customers module's validated + building blocks (_cust_rev / _cadence_bulk / _partner_attrs) so every metric matches the + Customers page definitions; est_missed is the Call-list formula (min(cycles missed, 3) × AOV).""" + return _pool_build(agent_name, team_id, limit=None)['rows'] + + +def pool_first(agent_name=None, team_id=None, limit=100): + """Wave-7 W1 (C1 as amended): the cold-load FAST SLICE — the pool's first `limit` rows in + its own default order + the true total, built from the CHEAP families only (revenue ×3, + cadence, status, est_missed, partner attributes ≈ 12s measured). The AR / product-mix / + salesperson / coords columns stay BLANK until the full build swaps in: `_mix_attrs` alone + is a whole-book 20s read_group whose domain must never be pid-narrowed (502 — see its + docstring), so "all columns on 100 rows" measured 44.6s vs 45.1s full — no win. A partial + payload is a UI PHASE, never a reporting basis — validate() reconciles the full pool only.""" + return _pool_build(agent_name, team_id, limit=limit, fast=True) + + +def _pool_build(agent_name, team_id, limit, fast=False): + t = P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + mf, mt = P.ltm(t) + pids = agent_pids(agent_name) + this = cust._cust_rev(yf, yt, team_id, pids) + last = cust._cust_rev(lf, lt, team_id, pids) + ltm = cust._cust_rev(mf, mt, team_id, pids) + cad = cust._cadence_bulk(t, team_id, agent_pids=pids) + all_pids = set(this) | set(last) | set(cad) + total = len(all_pids) + if limit is not None and total > limit: + # the SAME ordering the assembled pool ships (see the sort below) — the slice must be + # the first page of the very list the full build renders, or the swap-in reshuffles. + ranked = sorted(all_pids, + key=lambda p: -((this.get(p) or {}).get('rev', 0.0) + + (last.get(p) or {}).get('rev', 0.0))) + all_pids = set(ranked[:limit]) + attrs = cust._partner_attrs(list(all_pids)) + if fast: + # C1 fast phase: the slow families stay empty — their columns render blank behind the + # toolbar's partial indicator until the full build swaps in. + ar_a, mix_a, sp_a, geo_a = {}, {}, {}, {} + else: + # The three families the owner asked for (2026-07-27). Each degrades to {} rather than + # raising: a customer table that cannot render because AR is momentarily unreachable is + # a worse failure than one with a blank column. + ar_a = _ar_attrs(t) + mix_a = _mix_attrs(all_pids, t) + sp_a = _salesperson_attrs(all_pids, t) + # W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache, + # read-only reuse (coords_for never geocodes); missing → null lat/lon, and the + # client's map view discloses the count. + geo_a = map_mod.coords_for(list(all_pids)) + # customers whose only activity sits in the older half of the 24-month cadence window + # appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly + # the win-back targets this union exists for, would render as '?') + unnamed = [pid for pid in all_pids + if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name') + or (ltm.get(pid) or {}).get('name'))] + extra_names = ({r['id']: r['name'] for r in + O.search_read('res.partner', [('id', 'in', unnamed)], ['name'])} + if unnamed else {}) + rows = [] + for pid in all_pids: + tv = this.get(pid) or {} + lv = last.get(pid) or {} + c = cad.get(pid) or {} + rev, ly = tv.get('rev', 0.0), lv.get('rev', 0.0) + gap = c.get('typical_gap_days') + overdue = c.get('overdue_days') + aov = c.get('aov', 0.0) + est = (min(overdue / gap, 3.0) * aov) if (gap and overdue and overdue > 0) else 0.0 + status = ('New' if rev > 0 and ly <= 0 else + 'Lost' if ly > 0 and rev <= 0 else + 'Declining' if 0 < rev < ly else + 'Growing' if rev > 0 else 'Dormant') + a = attrs.get(pid) or {} + rows.append({ + 'pid': pid, + 'customer': tv.get('name') or lv.get('name') + or (ltm.get(pid) or {}).get('name') or extra_names.get(pid) or '?', + 'status': status, + 'agent': a.get('agent', '(none)'), 'city': a.get('city', '(none)'), + 'state': a.get('state', '(none)'), + # Owner 2026-07-27, the partner-attribute family. `.get(..., default)` is not + # defensive habit here: search_read returned 1,548 of 1,550 pool pids — two partners + # in the sales history are archived or deleted — so a customer with NO attrs row is + # a real case, and it must render as '(none)' rather than KeyError the whole page. + 'country': a.get('country', '(none)'), 'zip': a.get('zip', '(none)'), + 'payment_terms': a.get('payment_terms', '(none)'), + 'customer_since': a.get('customer_since', ''), + # Row-level datum for the `created_time` field type (wave-5 item 11) — rides every + # row like pid, no column of its own until a user creates one. + '_created': a.get('created_at', ''), + 'tags': a.get('tags', '(none)'), 'pricelist': a.get('pricelist', '(none)'), + # AR / credit exposure — composed from the already-reconciled ar.py blocks, so the + # Customer table and the Collections page cannot disagree. + 'ar_open': (ar_a.get(pid) or {}).get('ar_open', 0.0), + 'ar_overdue': (ar_a.get(pid) or {}).get('ar_overdue', 0.0), + 'ar_exposure': (ar_a.get(pid) or {}).get('ar_exposure', 0.0), + 'days_to_pay': (ar_a.get(pid) or {}).get('days_to_pay'), + # Product mix, LTM, line grain, ALL-channel. + 'top_category': (mix_a.get(pid) or {}).get('top_category', '(none)'), + 'top_category_pct': (mix_a.get(pid) or {}).get('top_category_pct'), + 'sku_count': (mix_a.get(pid) or {}).get('sku_count', 0), + 'top_sku': (mix_a.get(pid) or {}).get('top_sku', '(none)'), + # The ORDER-TAKER, not the agent. Different people, different questions. + 'salesperson': sp_a.get(pid, '(none)'), + # W11: nullable — the Map view pins what it can and counts what it cannot. + 'lat': (geo_a.get(pid) or {}).get('lat'), + 'lon': (geo_a.get(pid) or {}).get('lon'), + 'revenue_ytd': rev, 'revenue_ly': ly, + 'yoy_pct': P.yoy_pct(rev, ly), + 'at_risk': max(ly - rev, 0.0), + 'ltm_rev': (ltm.get(pid) or {}).get('rev', 0.0), + 'orders_24m': c.get('n_orders', 0), 'aov': aov, + 'last_order': c.get('last_order', ''), + 'days_since': c.get('days_since'), + 'typical_gap_days': gap, 'overdue_days': overdue, + 'est_missed': est, + }) + for r in rows: + r['odoo_status'] = 'Active' + # I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo. + # Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction, + # so letting it write would mark ~1,450 living customers archived on every cold load. + if not fast: + rows, total = _reconcile_ledger(rows, total, agent_name, team_id) + rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly'])) + return {'rows': rows, 'total': total} + + +LEDGER_KEY = 'customer_ledger' + + +def _reconcile_ledger(rows, total, agent_name, team_id): + """Remember every customer we have ever seen, and keep serving the ones Odoo stopped + returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately — + so stored in our platform even if deleted in Odoo"). + + Why a ledger at all: this pool is built from SALES HISTORY, not from a partner list, so a + partner deleted in Odoo usually keeps appearing (their order lines remain) — until the day + the lines go too, when the row would silently vanish along with every note, tag and cohort + membership attached to it. Nothing else in the app would notice. The ledger is the only + record that the customer ever existed, which is why it starts recording now rather than + when the surfacing is finished: history you did not write down is not recoverable later. + + ⚠ SCOPE-SAFE. The ledger is written ONLY from an UNSCOPED build (no agent, no BU). A scoped + build legitimately sees a fraction of the book, and reconciling from it would mark every + customer outside that scope 'Archived' for everyone — the failure would look exactly like + the data loss this is meant to prevent. Scoped builds READ the ledger and resurrect nothing + they cannot prove they should see. + """ + try: + ledger = dict(store.get(LEDGER_KEY) or {}) + except Exception: + return rows, total # a ledger we cannot read must not break the table + today = str(P.today()) + live = {r['pid'] for r in rows} + unscoped = agent_name is None and team_id is None + + if unscoped: + for r in rows: + e = ledger.get(str(r['pid'])) or {} + e.update({'name': r['customer'], 'last_seen': today}) + e.setdefault('first_seen', today) + ledger[str(r['pid'])] = e + + # Rows the ledger knows and this build did not return. On an unscoped build that means + # "gone from Odoo"; on a scoped one it usually just means "not in this agent's book", so + # only the unscoped build may surface them. + revived = [] + if unscoped: + for key, e in ledger.items(): + try: + pid = int(key) + except (TypeError, ValueError): + continue + if pid in live: + continue + revived.append({ + 'pid': pid, 'customer': e.get('name') or '?', 'status': 'Dormant', + 'odoo_status': 'Archived', + # Everything else is genuinely UNKNOWN now — the record is gone. Blank is the + # honest rendering; carrying the last-known numbers forward would state figures + # as current that nothing can reconcile ([[no-unverifiable-aggregates]]). + 'agent': '(none)', 'city': '(none)', 'state': '(none)', 'country': '(none)', + 'zip': '(none)', 'payment_terms': '(none)', 'customer_since': '', + '_created': '', 'tags': '(none)', 'pricelist': '(none)', + 'ar_open': 0.0, 'ar_overdue': 0.0, 'ar_exposure': 0.0, 'days_to_pay': None, + 'top_category': '(none)', 'top_category_pct': None, 'sku_count': 0, + 'top_sku': '(none)', 'salesperson': '(none)', 'lat': None, 'lon': None, + 'revenue_ytd': 0.0, 'revenue_ly': 0.0, 'yoy_pct': None, 'at_risk': 0.0, + 'ltm_rev': 0.0, 'orders_24m': 0, 'aov': 0.0, 'last_order': '', + 'days_since': None, 'typical_gap_days': None, 'overdue_days': None, + 'est_missed': 0.0, + }) + if store.available(): + try: + store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async') + except Exception: + pass # recording is best-effort; the table is not + return rows + revived, total + len(revived) + + +# ------------------------------------------------------------------ the rule engine +# A list's formula = [{'field','op','value'}, ...] over FIELDS + a 'sort' key ('-x' = desc). +# The two templates ARE the old My Day queue formulas, expressed as visible rules. +# +# 'note' is a SEED, not the source of truth (owner 2026-07-25). It is what a list reads +# BEFORE anyone edits it; the moment a user saves a description the stored one wins, and +# an empty stored description stays empty — the template must never resurrect. That +# precedence lives in aios_grid.views_from_defs (a saved view replaces its template-derived +# one wholesale) and is pinned by _qa_grid_view_note.py. Edit the prose here only to change +# what a NEW user starts with. +TEMPLATES = { + 'Call list': { + 'rules': [{'field': 'orders_24m', 'op': '>=', 'value': 3}, + {'field': 'overdue_days', 'op': '>', 'value': 0}, + {'field': 'days_since', 'op': '<=', 'value': 365}], + 'sort': '-est_missed', + 'note': 'Customers overdue against their OWN reorder cadence (3+ orders, a real gap ' + 'pattern, ordered within 365d), ranked by estimated missed revenue.'}, + 'Win-back': { + 'rules': [{'field': 'revenue_ly', 'op': '>=', 'value': 2000}, + {'field': 'at_risk', 'op': '>', 'value': 0}], + 'sort': '-at_risk', + 'note': 'Bought materially last year (over $2,000), down or gone this year, ranked by ' + 'dollars at risk.'}, +} + + +def _match(row, rule): + """One rule against one row. None/missing numeric values fail every numeric comparison + (a customer with no cadence is never 'overdue'); text ops are case-insensitive.""" + field, op, val = rule.get('field'), rule.get('op'), rule.get('value') + if field not in FIELDS: + return True # unknown field → rule is inert, not a crash + v = row.get(field) + if FIELDS[field][1] == 'text': + s, q = str(v or '').lower(), str(val or '').lower() + return q in s if op == 'contains' else s == q + try: + v, val = float(v), float(val) + except (TypeError, ValueError): + return False + return {'>=': v >= val, '<=': v <= val, '>': v > val, '<': v < val, + '=': v == val}.get(op, False) + + +def apply_rules(rows, rules, sort=None): + """Filter the pool through a formula (AND of all rules) and apply the list's sort.""" + out = [r for r in rows if all(_match(r, ru) for ru in (rules or []))] + if sort: + key = sort.lstrip('-') + out.sort(key=lambda r: (r.get(key) is None, + -(r.get(key) or 0) if sort.startswith('-') else (r.get(key) or 0)) + if FIELDS.get(key, ('', 'num'))[1] == 'num' + else str(r.get(key) or '').lower(), reverse=False) + return out + + +# ------------------------------------------------------------------ persistence (HF store) +def saved_lists(username): + """{list_name: {'rules','sort','members','note'}} for one user ({} when none / store down).""" + try: + return (store.get(KEY) or {}).get(username, {}) or {} + except Exception: + return {} + + +def save_list(username, name, definition): + def _up(d): + d.setdefault(username, {})[name] = definition + return d + store.update(KEY, _up) + + +def delete_list(username, name): + def _up(d): + (d.get(username) or {}).pop(name, None) + return d + store.update(KEY, _up) + + +# ------------------------------------------------------------------ Airtable-style table workspace +# This is intentionally separate from `customer_lists`: a LIST owns membership/formula +# semantics, while a VIEW owns presentation/query state (filters, multi-sort, grouping, +# visible fields, widths, color, row height). List views can reference the same formula +# without conflating the two persistence contracts. +# +# The store logic MOVED to core/table_store.py 2026-07-27 (the table-page factory): these +# names are the Customer table's instance of the generic per-object workspace store, kept as +# module functions so every existing caller (app.py's host loop, QA gates) is untouched. +# A NEW table object gets its own `core.table_store.make('_table_workspace')`. +_TSTORE = table_store.make(TABLE_KEY) + +table_workspace = _TSTORE.workspace +shared_table_views = _TSTORE.shared_views # wave-9 I17: views shared WITH this viewer +shared_table_view = _TSTORE.shared_view # ...and one raw, for authorisation only +save_table_view = _TSTORE.save_view +delete_table_view = _TSTORE.delete_view +save_table_field = _TSTORE.save_field +delete_table_field = _TSTORE.delete_field +duplicate_table_field = _TSTORE.duplicate_field +patch_table_overlay = _TSTORE.patch_overlay +save_table_folders = _TSTORE.save_folders + + +# ------------------------------------------------------------------ digest compatibility +def queues(agent_name=None, team_id=None, limit=200): + """The two canonical morning lists (call list + win-back) for the daily digest email — + unchanged formulas via the customers module (the digest always uses the CANONICAL templates, + not a user's edited copy, so every rep's email means the same thing).""" + pids = agent_pids(agent_name) + return { + 'calls': cust.contact_recommendations(team_id=team_id, limit=limit, agent_pids=pids), + 'risk': cust.at_risk(team_id=team_id, limit=limit, agent_pids=pids), + } diff --git a/royalimports_os/modules/customers.py b/royalimports_os/modules/customers.py new file mode 100644 index 0000000000000000000000000000000000000000..73a58b1b5165c7bd3479f806dd1a545caeef8c5a --- /dev/null +++ b/royalimports_os/modules/customers.py @@ -0,0 +1,1430 @@ +"""Customers module — the "why" behind the brand divergence (Fisch −5.5% vs Royal +40%). + +Centerpiece is the **customer revenue bridge**: it decomposes the YoY revenue change into +New (+), Expansion (+), Contraction (−) and Lost (−). That decomposition both answers the +business question and self-validates — the four components must reconcile last-period total +to this-period total exactly (built into validate()). + +Plus a MECE value-tier segmentation (LTM), an at-risk / win-back list ranked by dollars at +stake, and new/lost customer lists. All order-level (sale.order), RI+FFS scope, excluded +accounts removed — reusing the Sales module's order_domain so scope is identical everywhere. +""" +import sys +import datetime as dt +import statistics +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P +import modules.sales as sales_mod + + +# ---- per-customer grouped reads: STORE-backed (OM-2 retrofit 2026-07-12) with LIVE fallback -- +# _cust_rev/_last_order_dates are the choke points behind the Customers bundle, the Map's slow +# wave-1 and the Customer List queues. SQL mirrors sales.order_domain incl. the agent filter +# (partner_ids; an EMPTY set matches no orders, same as the live domain). validate() stays live. +USE_STORE = True + + +def _cust_group_store(date_from, date_to, team_id, agent_pids, select): + import harness.datastore as DS + params, w = [], ["state IN ('sale','done')"] + teams = sales_mod.TEAMS(team_id) + w.append("team_id IN (" + ",".join("?" * len(teams)) + ")") + params += list(teams) + if date_from: + w.append("CAST(date_order AS TIMESTAMP) >= CAST(? AS TIMESTAMP)") + params.append(f"{date_from} 00:00:00") + if date_to: + w.append("CAST(date_order AS TIMESTAMP) <= CAST(? AS TIMESTAMP)") + params.append(f"{date_to} 23:59:59") + ex = O.excluded_partner_ids() + if ex: + w.append("partner_id NOT IN (" + ",".join("?" * len(ex)) + ")") + params += list(ex) + if agent_pids is not None: + if not agent_pids: + return [] # empty set matches no orders + w.append("partner_id IN (" + ",".join("?" * len(agent_pids)) + ")") + params += list(agent_pids) + if O.doc_mode() == 'invoice': + w.append("invoice_status = 'invoiced'") + return DS.ro_con().execute( + f"SELECT o.partner_id, coalesce(p.name, '#' || o.partner_id), {select} " + "FROM sale_order o LEFT JOIN res_partner p ON p.id = o.partner_id " + "WHERE " + " AND ".join(w) + " GROUP BY 1, 2", params).fetchall() + + +def _cust_rev(date_from, date_to, team_id=None, agent_pids=None): + """{partner_id: {'name', 'rev', 'orders'}} over a window (RI+FFS, excluded accounts removed). + agent_pids (frozenset|None) restricts to one Agent's customers — the module-wide Agent filter.""" + if USE_STORE: + try: + rows = _cust_group_store(date_from, date_to, team_id, agent_pids, + 'sum(amount_untaxed), count(*)') + return {r[0]: {'name': r[1], 'rev': r[2] or 0.0, 'orders': r[3]} + for r in rows if r[0]} + except Exception: + pass + g = O.read_group('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=agent_pids), + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + out = {} + for r in g: + pid = O.m2o_id(r.get('partner_id')) + if not pid: + continue + out[pid] = {'name': O.m2o_name(r.get('partner_id')), + 'rev': r.get('amount_untaxed') or 0.0, + 'orders': r.get('__count') or 0} + return out + + +# ----------------------------------------------------------- agent filter (module-wide) +# Agent = res.partner.agent_ids (the assigned sales agent on a customer; the same attribute the +# rollups slice by). The Customer module can be filtered to one agent's whole book: we resolve the +# agent name → the set of partner ids assigned to them, then pass it as agent_pids everywhere. +def agent_options(t=None, team_id=None): + """Agent names that have customers with YTD activity in scope — for the module filter dropdown. + Reuses the agent rollup (cheap, cached at the app layer). '(none)' = unassigned customers.""" + return [r['group'] for r in by_dimension('agent', t, team_id=team_id)] + + +def agent_partner_ids(agent_name): + """frozenset of partner ids assigned to `agent_name` via res.partner.agent_ids. None when no + agent is selected ('All agents'); an EMPTY frozenset (matches nobody) when the agent has no + customers. '(none)' resolves to customers with no agent assigned. + + INCLUDES INACTIVE/archived partners (active in [True,False]) — the agent's book is the whole + book (dormant + archived accounts too), which is what the Agents page promises AND what an + AGENT-LOGIN user must see as their complete, isolated book. Active-only used to drop archived + customers entirely (e.g. an agent whose whole small book is archived resolved to an EMPTY book — + the fails-closed trap). Resolves the agent name to EVERY matching partner id (not just the + first) so a same-name collision can't silently mis-scope the book (owner 2026-07-21).""" + if not agent_name or agent_name in ('All agents', 'All'): + return None + _all = [('active', 'in', [True, False])] + if agent_name == '(none)': + rows = O.search_read('res.partner', [('customer_rank', '>', 0), + ('agent_ids', '=', False)] + _all, ['id'], limit=100000) + return frozenset(r['id'] for r in rows) + ag = O.search_read('res.partner', [('name', '=', agent_name)] + _all, ['id'], limit=10) + if not ag: + return frozenset() + rows = O.search_read('res.partner', [('agent_ids', 'in', [a['id'] for a in ag])] + _all, + ['id'], limit=100000) + return frozenset(r['id'] for r in rows) + + +# ----------------------------------------------------------- revenue bridge +def revenue_bridge(t=None, team_id=None, agent_pids=None): + """Decompose YTD-vs-same-period-LY revenue change into New/Expansion/Contraction/Lost.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + tset, lset = set(this), set(last) + + new_ids = tset - lset + lost_ids = lset - tset + both = tset & lset + + new_rev = sum(this[p]['rev'] for p in new_ids) + lost_rev = sum(last[p]['rev'] for p in lost_ids) + expansion = sum(this[p]['rev'] - last[p]['rev'] for p in both if this[p]['rev'] > last[p]['rev']) + contraction = sum(this[p]['rev'] - last[p]['rev'] for p in both if this[p]['rev'] < last[p]['rev']) + + this_total = sum(v['rev'] for v in this.values()) + last_total = sum(v['rev'] for v in last.values()) + return { + 'last_total': last_total, + 'this_total': this_total, + 'change': this_total - last_total, + 'new': {'rev': new_rev, 'n': len(new_ids)}, + 'expansion': {'rev': expansion, 'n': sum(1 for p in both if this[p]['rev'] > last[p]['rev'])}, + 'contraction': {'rev': contraction, 'n': sum(1 for p in both if this[p]['rev'] < last[p]['rev'])}, + 'lost': {'rev': -lost_rev, 'n': len(lost_ids)}, + 'retained_n': len(both), + 'active_this': len(tset), + 'active_last': len(lset), + } + + +def bridge_by_brand(t=None, agent_pids=None): + rows = [] + for tid in O.TEAM_IDS: + b = revenue_bridge(t, team_id=tid, agent_pids=agent_pids) + rows.append({'brand': O.TEAM_NAMES[tid], **{ + 'last': b['last_total'], 'this': b['this_total'], 'change': b['change'], + 'new': b['new']['rev'], 'expansion': b['expansion']['rev'], + 'contraction': b['contraction']['rev'], 'lost': b['lost']['rev'], + 'new_n': b['new']['n'], 'lost_n': b['lost']['n']}}) + return rows + + +def bridge_component_customers(component, t=None, team_id=None, agent_pids=None): + """The customers contributing to one revenue-bridge component (New / Expansion / Contraction / + Lost), each with LY, YTD, the bridge delta and share of that component — for the click-through + drawer off the Revenue-bridge chart.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + tset, lset = set(this), set(last) + comp = (component or '').strip().lower() + rows = [] + if comp == 'new': + rows = [{'pid': p, 'customer': this[p]['name'], 'rev_ly': 0.0, 'rev_ytd': this[p]['rev'], + 'change': this[p]['rev']} for p in (tset - lset)] + elif comp == 'lost': + rows = [{'pid': p, 'customer': last[p]['name'], 'rev_ly': last[p]['rev'], 'rev_ytd': 0.0, + 'change': -last[p]['rev']} for p in (lset - tset)] + elif comp in ('expansion', 'contraction'): + up = comp == 'expansion' + for p in (tset & lset): + d = this[p]['rev'] - last[p]['rev'] + if (d > 0) == up and d != 0: + rows.append({'pid': p, 'customer': this[p]['name'], 'rev_ly': last[p]['rev'], + 'rev_ytd': this[p]['rev'], 'change': d}) + denom = sum(abs(r['change']) for r in rows) or 1.0 + for r in rows: + r['at_risk'] = max(0.0, r['rev_ly'] - r['rev_ytd']) + r['pct_of_component'] = abs(r['change']) / denom * 100.0 + rows.sort(key=lambda r: -abs(r['change'])) + _attach_attrs(rows) + return {'component': component, 'rows': rows, 'n': len(rows), + 'total': sum(r['change'] for r in rows)} + + +def period_customers(ym, t=None, team_id=None, agent_pids=None): + """Customers who purchased in calendar month `ym` (YYYY-MM) this year — that month's revenue, + the same month last year, order count, and each customer's YTD-vs-LY at-risk for context. + Powers the click-through drawer off the Monthly (this-year vs last-year) bar chart.""" + t = t or P.today() + y, m = int(ym[:4]), int(ym[5:7]) + + def _bounds(yr): + s = dt.date(yr, m, 1) + e = dt.date(yr + (m // 12), (m % 12) + 1, 1) - dt.timedelta(days=1) + return s.isoformat(), e.isoformat() + ts, te = _bounds(y) + ls, le = _bounds(y - 1) + this = _cust_rev(ts, te, team_id, agent_pids) + last = _cust_rev(ls, le, team_id, agent_pids) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + ytd = _cust_rev(yf, yt, team_id, agent_pids) + lytd = _cust_rev(lf, lt, team_id, agent_pids) + rows = [{'pid': p, 'customer': v['name'], 'rev_this': v['rev'], 'orders': v['orders'], + 'rev_ly': last.get(p, {}).get('rev', 0.0), + 'at_risk': max(0.0, lytd.get(p, {}).get('rev', 0.0) - ytd.get(p, {}).get('rev', 0.0))} + for p, v in this.items()] + rows.sort(key=lambda r: -r['rev_this']) + _attach_attrs(rows) + return {'month': ym, 'rows': rows, 'n': len(rows), + 'rev_this': sum(r['rev_this'] for r in rows), + 'rev_ly': sum(r['rev_ly'] for r in rows)} + + +def monthly_kpis(t=None, team_id=None, n_months=24): + """Monthly time-series for the acquisition/engagement trends at the top of the page: order + count, AOV, distinct purchasing customers, and newly-acquired customers (first-ever order that + month).""" + t = t or P.today() + months = P.month_starts(n_months, t) + win_start = months[0][1] + dom = sales_mod.order_domain(win_start, t.isoformat(), team_id) + + def _ym(r): + rng = (r.get('__range') or {}).get('date_order:month') or {} + return (rng.get('from') or '')[:7] + mrev, morders = {}, {} + for r in O.read_group('sale.order', dom, ['amount_untaxed:sum'], ['date_order:month'], lazy=False): + ym = _ym(r) + if ym: + mrev[ym] = r.get('amount_untaxed') or 0.0 + morders[ym] = r.get('__count') or 0 + mbuyers = {} + for r in O.read_group('sale.order', dom, ['__count'], ['date_order:month', 'partner_id'], lazy=False): + ym = _ym(r) + if ym: + mbuyers[ym] = mbuyers.get(ym, 0) + 1 + # new customers: first-ever order month per partner (all history), bucketed into our window + mnew = {} + for r in O.read_group('sale.order', sales_mod.order_domain('2000-01-01', t.isoformat(), team_id), + ['date_order:min'], ['partner_id'], lazy=False): + ym = str(r.get('date_order') or '')[:7] + if ym: + mnew[ym] = mnew.get(ym, 0) + 1 + out = [] + for ym, _s, _e in months: + orders = morders.get(ym, 0) + out.append({'month': ym, 'orders': orders, 'revenue': mrev.get(ym, 0.0), + 'aov': (mrev.get(ym, 0.0) / orders) if orders else 0.0, + 'buyers': mbuyers.get(ym, 0), 'new_customers': mnew.get(ym, 0)}) + return out + + +def customer_trends(t=None, team_id=None, agent_pids=None): + """12 months aligned THIS-year vs same-month-LAST-year, with both the monthly value and the + running CUMULATIVE (this vs last) for orders, AOV, purchasing customers (distinct) and new + customers. Powers the Trends chart's Monthly-bars / Cumulative-line toggle.""" + t = t or P.today() + months = P.month_starts(24, t) # 24 ascending: prior 12 then recent 12 + win_start = months[0][1] + dom = sales_mod.order_domain(win_start, t.isoformat(), team_id, partner_ids=agent_pids) + + def _ym(r): + rng = (r.get('__range') or {}).get('date_order:month') or {} + return (rng.get('from') or '')[:7] + mrev, morders, mbuyers = {}, {}, {} + for r in O.read_group('sale.order', dom, ['amount_untaxed:sum'], ['date_order:month'], lazy=False): + ym = _ym(r) + if ym: + mrev[ym] = r.get('amount_untaxed') or 0.0 + morders[ym] = r.get('__count') or 0 + for r in O.read_group('sale.order', dom, ['__count'], ['date_order:month', 'partner_id'], lazy=False): + ym = _ym(r) + pid = O.m2o_id(r.get('partner_id')) + if ym and pid: + mbuyers.setdefault(ym, set()).add(pid) + mnew = {} + for r in O.read_group('sale.order', sales_mod.order_domain('2000-01-01', t.isoformat(), team_id, partner_ids=agent_pids), + ['date_order:min'], ['partner_id'], lazy=False): + ym = str(r.get('date_order') or '')[:7] + if ym: + mnew[ym] = mnew.get(ym, 0) + 1 + + recent = [m[0] for m in months[12:24]] # the 12 most recent months (ascending) + prior = [m[0] for m in months[0:12]] # the same 12 months a year earlier + out = [] + cr_t = co_t = cn_t = 0.0 + cr_l = co_l = cn_l = 0.0 + cb_t, cb_l = set(), set() + for i in range(12): + ty, ly = recent[i], prior[i] + ot, rt, nt = morders.get(ty, 0), mrev.get(ty, 0.0), mnew.get(ty, 0) + ol, rl, nl = morders.get(ly, 0), mrev.get(ly, 0.0), mnew.get(ly, 0) + bt, bl = mbuyers.get(ty, set()), mbuyers.get(ly, set()) + co_t += ot; cr_t += rt; cn_t += nt; cb_t |= bt + co_l += ol; cr_l += rl; cn_l += nl; cb_l |= bl + out.append({ + 'month': ty, 'month_ly': ly, + 'orders': ot, 'orders_ly': ol, + 'aov': (rt / ot if ot else 0.0), 'aov_ly': (rl / ol if ol else 0.0), + 'buyers': len(bt), 'buyers_ly': len(bl), + 'new_customers': nt, 'new_customers_ly': nl, + 'orders_cum': co_t, 'orders_cum_ly': co_l, + 'aov_cum': (cr_t / co_t if co_t else 0.0), 'aov_cum_ly': (cr_l / co_l if co_l else 0.0), + 'buyers_cum': len(cb_t), 'buyers_cum_ly': len(cb_l), + 'new_customers_cum': cn_t, 'new_customers_cum_ly': cn_l, + }) + return out + + +# ----------------------------------------------------------- segmentation (MECE) +TIERS = [('Whale (≥$25k)', 25000), ('Large ($10–25k)', 10000), + ('Mid ($2–10k)', 2000), ('Small (<$2k)', 0)] + + +def _tier(rev): + for name, lo in TIERS: + if rev >= lo: + return name + return TIERS[-1][0] + + +def segments(t=None, team_id=None, agent_pids=None): + """MECE value-tier segmentation over LTM revenue (each active customer in one tier).""" + t = t or P.today() + lf, lt = P.ltm(t) + cust = _cust_rev(lf, lt, team_id, agent_pids) + agg = {name: {'segment': name, 'customers': 0, 'revenue': 0.0} for name, _ in TIERS} + for v in cust.values(): + a = agg[_tier(v['rev'])] + a['customers'] += 1 + a['revenue'] += v['rev'] + total_rev = sum(a['revenue'] for a in agg.values()) or 1.0 + total_n = sum(a['customers'] for a in agg.values()) or 1 + rows = [] + for name, _ in TIERS: + a = agg[name] + a['rev_share'] = a['revenue'] / total_rev * 100 + a['cust_share'] = a['customers'] / total_n * 100 + a['avg_rev'] = a['revenue'] / a['customers'] if a['customers'] else 0.0 + rows.append(a) + return rows + + +def tier_customers(tier, t=None, team_id=None, agent_pids=None): + """The customers in ONE LTM value tier — each with LTM revenue, order count, units bought and + average order value — for the Value-tiers drill. Tier membership matches segments() exactly + (same LTM window + _tier).""" + t = t or P.today() + lf, lt = P.ltm(t) + cust = _cust_rev(lf, lt, team_id, agent_pids) + pids = [pid for pid, v in cust.items() if _tier(v['rev']) == tier] + if not pids: + return [] + units = {} + for r in O.read_group('sale.order.line', O.sale_line_domain(lf, lt, team_id, partner_ids=pids), + ['product_uom_qty:sum'], ['order_partner_id'], lazy=False): + pid = O.m2o_id(r.get('order_partner_id')) + if pid: + units[pid] = r.get('product_uom_qty') or 0.0 + pdata = {p['id']: p for p in O.search_read('res.partner', [('id', 'in', pids)], ['city', 'state_id'])} + rows = [] + for pid in pids: + v = cust[pid] + o = v.get('orders', 0) + p = pdata.get(pid, {}) + rows.append({'pid': pid, 'customer': v['name'], 'city': p.get('city') or '(none)', + 'state': O.m2o_name(p.get('state_id')) or '(none)', 'agent': '(none)', + 'rev_ltm': v['rev'], 'orders': o, 'units': units.get(pid, 0.0), + 'aov': (v['rev'] / o) if o else 0.0}) + rows.sort(key=lambda x: -x['rev_ltm']) + return rows + + +# ----------------------------------------------------------- at-risk / win-back +def at_risk(t=None, team_id=None, limit=30, min_prior=2000.0, agent_pids=None): + """Customers who bought materially last year but are down or gone this year, ranked by + dollars at risk (last − this). The win-back target list.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + rows = [] + for pid, lv in last.items(): + if lv['rev'] < min_prior: + continue + tv = this.get(pid, {'rev': 0.0}) + down = lv['rev'] - tv['rev'] + if down <= 0: + continue + rows.append({'pid': pid, 'customer': lv['name'], 'rev_ly': lv['rev'], 'rev_ytd': tv['rev'], + 'orders': tv.get('orders', 0), 'at_risk': down, + 'status': 'Lost' if pid not in this else 'Declining'}) + rows.sort(key=lambda x: -x['at_risk']) + return rows[:limit] + + +def _first_order_dates(team_id=None, t=None, agent_pids=None): + """{pid: first-ever confirmed order date (ISO)} — one all-history read_group(min).""" + t = t or P.today() + g = O.read_group('sale.order', sales_mod.order_domain('2000-01-01', t.isoformat(), team_id, partner_ids=agent_pids), + ['date_order:min'], ['partner_id'], lazy=False) + out = {} + for r in g: + pid = O.m2o_id(r.get('partner_id')) + if pid and r.get('date_order'): + out[pid] = str(r['date_order'])[:10] + return out + + +def new_customers(t=None, team_id=None, limit=30, agent_pids=None): + """Customers acquired or reactivated this year (active this YTD, not in same-period LY), each + tagged New (first-ever order this year) vs Reactivated (ordered in a prior year, had lapsed).""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + cand = [pid for pid in this if pid not in last] + firsts = _first_order_dates(team_id, t, agent_pids) if cand else {} + rows = [] + for pid in cand: + v = this[pid] + first = firsts.get(pid, '') + status = 'New' if first[:4] == str(t.year) else 'Reactivated' + rows.append({'pid': pid, 'customer': v['name'], 'rev_ytd': v['rev'], 'orders': v['orders'], + 'first_order': first, 'status': status}) + rows.sort(key=lambda x: -x['rev_ytd']) + return rows[:limit] + + +def lists_bundle(t=None, team_id=None, agent_pids=None): + """At-risk, New/reactivated and Follow-up lists, all enriched with a CONSISTENT insight column + set — YTD $, YoY %, orders, last order, days overdue vs the customer's own cadence, agent — + plus each list's own metric (at-risk $ / status / est. missed $).""" + t = t or P.today() + cad = _cadence_bulk(t, team_id, agent_pids=agent_pids) + ar = at_risk(t, team_id, agent_pids=agent_pids) + nw = new_customers(t, team_id, agent_pids=agent_pids) + fu = contact_recommendations(t, team_id, agent_pids=agent_pids) + pids = {r['pid'] for r in ar} | {r['pid'] for r in nw} | {r['pid'] for r in fu} + attrs = _partner_attrs(list(pids)) + + def enrich(rows): + for r in rows: + c = cad.get(r['pid'], {}) + r['last_order'] = r.get('last_order') or c.get('last_order', '') + r['overdue_days'] = c.get('overdue_days') + r['agent'] = (attrs.get(r['pid']) or {}).get('agent', '(none)') + if 'rev_ytd' not in r and 'ltm_rev' in r: + r['rev_ytd'] = r['ltm_rev'] + if 'yoy_pct' not in r: + r['yoy_pct'] = P.yoy_pct(r.get('rev_ytd', 0.0), r.get('rev_ly', 0.0)) + return rows + return {'at_risk': enrich(ar), 'new': enrich(nw), 'followups': enrich(fu)} + + +def _cadence_bulk(t=None, team_id=None, months=24, agent_pids=None): + """{pid: {n_orders, last_order, typical_gap_days, days_since, overdue_days, aov}} from ONE + read_group over the window (count + min/max date + revenue per partner). Bulk approximation of + per-customer cadence — the basis for the follow-up list and the lists' cadence columns.""" + t = t or P.today() + months = max(1, months) + start = dt.date(t.year - (months // 12) - (1 if t.month <= (months % 12) else 0), + ((t.month - 1 - (months % 12)) % 12) + 1, 1) + g = O.read_group('sale.order', sales_mod.order_domain(start.isoformat(), t.isoformat(), team_id, partner_ids=agent_pids), + ['amount_untaxed:sum', 'mind:min(date_order)', 'maxd:max(date_order)'], + ['partner_id'], lazy=False) + out = {} + for r in g: + pid = O.m2o_id(r.get('partner_id')) + if not pid: + continue + n = r.get('__count') or 0 + rev = r.get('amount_untaxed') or 0.0 + try: + dmin = dt.date.fromisoformat(str(r.get('mind'))[:10]) + dmax = dt.date.fromisoformat(str(r.get('maxd'))[:10]) + except (TypeError, ValueError): + continue + span = (dmax - dmin).days + gap = (span / (n - 1)) if n >= 2 and span > 0 else None + days_since = (t - dmax).days + overdue = (days_since - gap) if gap else None + out[pid] = {'n_orders': n, 'last_order': dmax.isoformat(), 'typical_gap_days': gap, + 'days_since': days_since, 'overdue_days': overdue, 'aov': (rev / n) if n else 0.0} + return out + + +def contact_recommendations(t=None, team_id=None, limit=40, months=24, agent_pids=None): + """Prioritised follow-up list: customers overdue against their OWN purchase cadence, ranked by + estimated missed revenue (how far past due × their average order value). Reps' call list.""" + t = t or P.today() + cad = _cadence_bulk(t, team_id, months, agent_pids=agent_pids) + attrs = _partner_attrs(list(cad)) + names = {} + # names from the LTM revenue map (cheap, already scoped) + lf, lt = P.ltm(t) + rev_map = _cust_rev(lf, lt, team_id, agent_pids) + rows = [] + for pid, c in cad.items(): + gap = c['typical_gap_days'] + if c['n_orders'] < 3 or not gap or gap <= 0: + continue # need a real, repeated cadence + overdue = c['overdue_days'] + if overdue is None or overdue <= 0: + continue # only those past due + if c['days_since'] > 365: + continue # long-dead → not a cadence follow-up + ltm_rev = rev_map.get(pid, {}).get('rev', 0.0) + cycles_missed = overdue / gap # how many reorder cycles past due + # cap at 3 cycles so a long-churned account (belongs in win-back) doesn't dominate the + # "call them now" list; this rewards valuable regulars who are RECENTLY due. + est_missed = min(cycles_missed, 3.0) * c['aov'] + rows.append({ + 'pid': pid, 'customer': rev_map.get(pid, {}).get('name', '?'), + 'last_order': c['last_order'], 'typical_gap_days': gap, 'overdue_days': overdue, + 'ltm_rev': ltm_rev, 'orders': c['n_orders'], 'est_missed': est_missed, + 'agent': (attrs.get(pid) or {}).get('agent', '(none)'), + }) + rows.sort(key=lambda x: -x['est_missed']) + return rows[:limit] + + +# ====================================================================== EXPLORER +# Granular (per-customer drill-down) + sum-level (filter/group by city, state, etc). +# All revenue is order-level (sale.order, amount_untaxed) in the same RI+FFS scope as the +# rest of Sales, so every rollup sums back to the headline number (proven in validate()). + +# Group-by dimensions for the sum-level rollups (key → display label). +DIMENSIONS = { + 'city': 'City', 'state': 'State / Region', 'country': 'Country', + 'agent': 'Agent', 'segment': 'Value tier', +} + + +def _partner_attrs(pids): + """{pid: {city, state, country, agent, zip, payment_terms, customer_since, tags, pricelist}} + — the customer attributes the sum-level rollups slice by and the Customer table displays. + + Agent is res.partner.agent_ids (the assigned sales agent; ~one per customer), NOT Odoo's + user_id 'salesperson' — MEASURED 2026-07-27 at 26 of 1,548 (2%), which is why agent_ids has + always been the field here. Odoo's `credit_limit` is not read for the same reason: 20 of + 1,548 (1%). Credit EXPOSURE comes from AR (modules/collections.py), not from that field. + + Every value collapses to '(none)' when blank (MECE), so a group-by has no null bucket and a + filter has something to match. ⚠ `zip` is TEXT, never numeric: postal codes carry leading + zeros, and 01730 read as a number is 1730 — a different town. + """ + pids = list(pids) + if not pids: + return {} + rows = O.search_read('res.partner', [('id', 'in', pids)], + ['city', 'state_id', 'country_id', 'agent_ids', 'zip', + 'property_payment_term_id', 'create_date', 'category_id', + 'property_product_pricelist']) + aids = {a for r in rows for a in (r.get('agent_ids') or [])} + anames = ({p['id']: p['name'] for p in O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])} + if aids else {}) + # Tags are m2m: ids on the partner, names in res.partner.category. One extra read for the + # whole set rather than one per customer. + tids = {t for r in rows for t in (r.get('category_id') or [])} + tnames = ({c['id']: c['name'] for c in + O.search_read('res.partner.category', [('id', 'in', list(tids))], ['name'])} + if tids else {}) + out = {} + for r in rows: + ag = r.get('agent_ids') or [] + city = (r.get('city') or '').strip() + tags = [tnames.get(t) for t in (r.get('category_id') or []) if tnames.get(t)] + # create_date is a DATETIME ('2024-01-15 10:23:45'); the column is a DATE, and the grid + # compares dates ISO-LEXICALLY, so a trailing time would sort and filter as text noise. + since = (r.get('create_date') or '') + out[r['id']] = { + 'city': city.title() if city else '(none)', + 'state': O.m2o_name(r.get('state_id')) or '(none)', + 'country': O.m2o_name(r.get('country_id')) or '(none)', + 'agent': (anames.get(ag[0]) or '(none)') if ag else '(none)', + 'zip': (r.get('zip') or '').strip() or '(none)', + 'payment_terms': O.m2o_name(r.get('property_payment_term_id')) or '(none)', + 'customer_since': str(since)[:10] if since else '', + # The FULL datetime, for the grid's `created_time` field type (wave-5 item 11) — + # customer_since above stays the DATE the date-typed column sorts/filters on. + 'created_at': str(since) if since else '', + 'tags': ', '.join(tags) if tags else '(none)', + 'pricelist': O.m2o_name(r.get('property_product_pricelist')) or '(none)', + } + return out + + +def _attach_attrs(rows, key='pid'): + """Enrich a list of customer rows (each carrying a partner id under `key`) in place with the + standardized city / state / agent fields — so every customer list (page or drawer) can show the + same filters/columns. Returns the same list.""" + pids = [r[key] for r in rows if r.get(key) is not None] + if not pids: + return rows + attrs = _partner_attrs(pids) + for r in rows: + a = attrs.get(r.get(key), {}) + r.setdefault('city', a.get('city', '(none)')) + r.setdefault('state', a.get('state', '(none)')) + r.setdefault('agent', a.get('agent', '(none)')) + return rows + + +def _last_order_dates(date_from, date_to, team_id=None, agent_pids=None): + """{pid: last order date (ISO)} within the window — for the directory's recency column.""" + if USE_STORE: + try: + rows = _cust_group_store(date_from, date_to, team_id, agent_pids, + 'max(date_order)') + return {r[0]: str(r[2])[:10] for r in rows if r[0] and r[2]} + except Exception: + pass + g = O.read_group('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=agent_pids), + ['date_order:max'], ['partner_id'], lazy=False) + out = {} + for r in g: + pid = O.m2o_id(r.get('partner_id')) + if pid and r.get('date_order'): + out[pid] = str(r['date_order'])[:10] + return out + + +def directory(t=None, team_id=None, limit=None, agent_pids=None): + """Per-customer table (YTD): revenue, YoY, orders, AOV, last order, city/state/salesperson. + Basis for the drill-down picker and (indirectly) the rollups. Sorted by revenue desc.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + attrs = _partner_attrs(set(this) | set(last)) + lastord = _last_order_dates(yf, yt, team_id, agent_pids) + rows = [] + for pid, v in this.items(): + a = attrs.get(pid, {}) + rev, orders = v['rev'], v['orders'] + ly = last.get(pid, {}).get('rev', 0.0) + rows.append({ + 'pid': pid, 'customer': v['name'], + 'revenue': rev, 'revenue_ly': ly, 'yoy_pct': P.yoy_pct(rev, ly), + 'orders': orders, 'aov': (rev / orders) if orders else 0.0, + 'last_order': lastord.get(pid, ''), + 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'), + 'agent': a.get('agent', '(none)'), + }) + rows.sort(key=lambda x: -x['revenue']) + return rows[:limit] if limit else rows + + +def by_dimension(dim, t=None, team_id=None, agent_pids=None): + """Sum-level rollup: YTD revenue (vs LY) grouped by a customer attribute. dim is a key of + DIMENSIONS. MECE — each active customer lands in exactly one group, so Σ(groups) == total + YTD revenue (verified in validate()). When consolidated (team_id=None) each group also + carries its Fisch + Royal revenue, mirroring the HQ rollup down to each brand. Sorted desc.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + attrs = _partner_attrs(set(this) | set(last)) + split = team_id is None + f_rev = _cust_rev(yf, yt, 5, agent_pids) if split else {} + r_rev = _cust_rev(yf, yt, 6, agent_pids) if split else {} + + def keyfor(pid, rev): + if dim == 'segment': + return _tier(rev) + return (attrs.get(pid) or {}).get(dim) or '(none)' + + agg = {} + + def bucket(k): + return agg.setdefault(k, {'group': k, 'revenue': 0.0, 'revenue_ly': 0.0, + 'fisch': 0.0, 'royal': 0.0, 'customers': 0, 'orders': 0}) + for pid, v in this.items(): + d = bucket(keyfor(pid, v['rev'])) + d['revenue'] += v['rev']; d['customers'] += 1; d['orders'] += v['orders'] + if split: + d['fisch'] += f_rev.get(pid, {}).get('rev', 0.0) + d['royal'] += r_rev.get(pid, {}).get('rev', 0.0) + for pid, v in last.items(): + bucket(keyfor(pid, v['rev']))['revenue_ly'] += v['rev'] + rows = list(agg.values()) + for d in rows: + d['yoy_pct'] = P.yoy_pct(d['revenue'], d['revenue_ly']) + d['avg_per_customer'] = d['revenue'] / d['customers'] if d['customers'] else 0.0 + rows.sort(key=lambda x: -x['revenue']) + return rows + + +def _order_dom(pid, date_from, date_to, team_id=None): + return sales_mod.order_domain(date_from, date_to, team_id) + [('partner_id', '=', pid)] + + +def _line_dom(pid, date_from, date_to, team_id=None): + return O.sale_line_domain(date_from, date_to, team_id, extra=[('order_partner_id', '=', pid)]) + + +def _monthly_rev(pid, date_from, date_to, team_id=None): + """{'YYYY-MM': revenue} over a window via ONE month-grouped read_group (was 26 point queries).""" + g = O.read_group('sale.order', _order_dom(pid, date_from, date_to, team_id), + ['amount_untaxed:sum'], ['date_order:month'], lazy=False) + out = {} + for r in g: + rng = (r.get('__range') or {}).get('date_order:month') or {} + ym = (rng.get('from') or '')[:7] + if ym: + out[ym] = r.get('amount_untaxed') or 0.0 + return out + + +def customer_detail(pid, t=None, team_id=None, n_months=13, top=12): + """Granular drill-down for one customer: profile, RFM-style KPIs, monthly YoY trend, top SKUs + + categories bought (LTM, with margin), and recent orders. The ~14 independent Odoo reads run + CONCURRENTLY (O.parallel_map) so a cold drawer loads in ~1-2s instead of ~6s.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + mf, mt = P.ltm(t) + range_start = dt.date(t.year - 2, t.month, 1).isoformat() + # Fisch/Royal mix only when consolidated (team_id=None). When a single BU is selected we never + # compute or expose the other BU — strict isolation for per-BU permissioning. + split = team_id is None + r = O.parallel_map({ + 'prof': lambda: O.search_read('res.partner', [('id', '=', pid)], ['name', 'email', 'phone']), + 'attrs': lambda: _partner_attrs([pid]), + 'ytd_rev': lambda: O.sum_field('sale.order', _order_dom(pid, yf, yt, team_id), 'amount_untaxed'), + 'ytd_rev_ly': lambda: O.sum_field('sale.order', _order_dom(pid, lf, lt, team_id), 'amount_untaxed'), + 'ytd_fisch': (lambda: O.sum_field('sale.order', _order_dom(pid, yf, yt, 5), 'amount_untaxed')) if split else (lambda: None), + 'ytd_royal': (lambda: O.sum_field('sale.order', _order_dom(pid, yf, yt, 6), 'amount_untaxed')) if split else (lambda: None), + 'ltm_rev': lambda: O.sum_field('sale.order', _order_dom(pid, mf, mt, team_id), 'amount_untaxed'), + 'orders_ltm': lambda: O.get_odoo().search_count('sale.order', _order_dom(pid, mf, mt, team_id)), + 'orders_ytd': lambda: O.get_odoo().search_count('sale.order', _order_dom(pid, yf, yt, team_id)), + 'lastrow': lambda: O.search_read('sale.order', _order_dom(pid, None, None, team_id), ['date_order'], order='date_order desc', limit=1), + 'firstrow': lambda: O.search_read('sale.order', _order_dom(pid, None, None, team_id), ['date_order'], order='date_order asc', limit=1), + 'mrev': lambda: _monthly_rev(pid, range_start, t.isoformat(), team_id), + 'lg': lambda: O.read_group('sale.order.line', _line_dom(pid, mf, mt, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], ['product_id'], lazy=False), + 'recent': lambda: O.search_read('sale.order', _order_dom(pid, None, None, team_id), + ['name', 'date_order', 'amount_untaxed', 'state'], order='date_order desc', limit=10), + }) + prof = (r['prof'] or [{}])[0] + a = r['attrs'].get(pid, {}) + ytd_rev, ytd_rev_ly, ltm_rev = r['ytd_rev'], r['ytd_rev_ly'], r['ltm_rev'] + ytd_fisch, ytd_royal = r['ytd_fisch'], r['ytd_royal'] + orders_ltm, orders_ytd = r['orders_ltm'], r['orders_ytd'] + last_order = str(r['lastrow'][0]['date_order'])[:10] if r['lastrow'] else None + first_order = str(r['firstrow'][0]['date_order'])[:10] if r['firstrow'] else None + recency = (t - dt.date.fromisoformat(last_order)).days if last_order else None + + mrev = r['mrev'] + monthly = [] + for ym, start, end in P.month_starts(n_months, t): + y, m = int(ym[:4]) - 1, int(ym[5:7]) + monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0), + 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)}) + + lg = r['lg'] + skus = [{'product': O.m2o_name(x.get('product_id')), 'revenue': x.get('price_subtotal') or 0.0, + 'qty': x.get('product_uom_qty') or 0.0, 'margin': x.get('margin') or 0.0} + for x in lg if x.get('product_id')] + skus.sort(key=lambda x: -x['revenue']) + line_rev = sum(s['revenue'] for s in skus) + margin = sum(s['margin'] for s in skus) + + cat = sales_mod._product_cat() + catagg = {} + for x in lg: + prodid = O.m2o_id(x.get('product_id')) + if not prodid: + continue + c = cat.get(prodid, '(uncategorized)') + catagg[c] = catagg.get(c, 0.0) + (x.get('price_subtotal') or 0.0) + cats = sorted([{'category': k, 'revenue': v} for k, v in catagg.items()], + key=lambda x: -x['revenue'])[:10] + + recent_rows = [{'order': x.get('name'), 'date': str(x.get('date_order'))[:10], + 'amount': x.get('amount_untaxed') or 0.0, + 'status': 'Confirmed' if x.get('state') in ('sale', 'done') else x.get('state')} + for x in r['recent']] + + return { + 'pid': pid, 'name': prof.get('name') or '(unknown)', + 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'), + 'country': a.get('country', '(none)'), 'agent': a.get('agent', '(none)'), + 'email': prof.get('email') or '', 'phone': prof.get('phone') or '', + 'ytd_rev': ytd_rev, 'ytd_rev_ly': ytd_rev_ly, 'yoy_pct': P.yoy_pct(ytd_rev, ytd_rev_ly), + 'ytd_fisch': ytd_fisch, 'ytd_royal': ytd_royal, + 'ltm_rev': ltm_rev, 'orders_ytd': orders_ytd, 'orders_ltm': orders_ltm, + 'aov_ltm': (ltm_rev / orders_ltm) if orders_ltm else 0.0, + 'first_order': first_order, 'last_order': last_order, 'recency_days': recency, + 'ltm_margin': margin, 'ltm_gm_pct': (margin / line_rev * 100) if line_rev else 0.0, + 'monthly': monthly, 'top_skus': skus[:top], 'top_categories': cats, + 'recent_orders': recent_rows, + } + + +def customer_drawer_bundle(pid, t=None, team_id=None): + """Everything the customer drawer's first paint needs, in as few round-trips as possible: + customer_detail (itself parallel) then the other three pulls CONCURRENTLY. One cached unit, so + switching drawer sections never re-hits Odoo. customer_affinity (the look-alike co-buyer scan — + 3 dependent heavy reads, 4-8s) and customer_stockout stay LAZY, loaded only by their own + sections, so a cold drawer opens in ~2-3s regardless of how big the customer is.""" + detail = customer_detail(pid, t=t, team_id=team_id) + decomp, winback, cadence = O.parallel([ + lambda: customer_yoy_decomp(pid, t, team_id), + lambda: customer_winback(pid, t, team_id), + lambda: customer_cadence(pid, t, team_id), + ]) + return {'detail': detail, 'decomp': decomp, 'winback': winback, 'cadence': cadence} + + +def customer_stockout(pid, t=None, team_id=None, top=25): + """Stockout exposure for one customer: of the SKUs they buy, which are currently OUT / LOW on + hand, and how much of their YoY $ decline sits on SKUs that are now out of stock (a likely + stockout-driven loss) vs other causes. On-hand is a CURRENT snapshot (no historical stock), so + 'decline on a now-out SKU' is a strong proxy, not proof, of a stockout cause.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + + def skumap(df, dtt): + # storable goods only — services (delivery charges etc.) can't "stock out" + dom = O.sale_line_domain(df, dtt, team_id, extra=[('order_partner_id', '=', pid), + ('product_id.type', '!=', 'service')]) + g = O.read_group('sale.order.line', dom, + ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False) + return {O.m2o_id(r['product_id']): {'name': O.m2o_name(r.get('product_id')), + 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0} + for r in g if r.get('product_id')} + this, last = skumap(yf, yt), skumap(lf, lt) + pids = list(set(this) | set(last)) + empty = {'rows': [], 'n_out': 0, 'n_low': 0, 'rev_at_risk': 0.0, 'risk_pct': 0.0, + 'drop_stockout': 0.0, 'drop_other': 0.0, 'total_drop': 0.0} + if not pids: + return empty + q = O.read_group('stock.quant', [('location_id.usage', '=', 'internal'), ('product_id', 'in', pids)], + ['quantity:sum'], ['product_id'], lazy=False) + onhand = {O.m2o_id(r['product_id']): (r.get('quantity') or 0.0) for r in q if r.get('product_id')} + codemap = {} + for pr in O.search_read('product.product', [('id', 'in', pids)], ['default_code']): + codemap[pr['id']] = str(pr['default_code']).strip() if pr.get('default_code') else None + rows, drop_stockout, drop_other = [], 0.0, 0.0 + for p in pids: + tr = this.get(p, {}).get('rev', 0.0) + lr = last.get(p, {}).get('rev', 0.0) + oh = onhand.get(p, 0.0) + ly_qty = last.get(p, {}).get('qty', 0.0) + out = oh <= 0 + low = (not out) and oh < max(1.0, ly_qty * 0.25) # under ~a quarter of their annual usage + change = tr - lr + rows.append({'sku': (this.get(p) or last.get(p) or {}).get('name', '?'), 'code': codemap.get(p), + 'on_hand': oh, 'ly_rev': lr, 'ytd_rev': tr, 'change': change, + 'status': 'OUT' if out else ('LOW' if low else 'OK')}) + if change < 0: + if out: + drop_stockout += -change + else: + drop_other += -change + rows.sort(key=lambda r: ({'OUT': 0, 'LOW': 1, 'OK': 2}[r['status']], -r['ly_rev'])) + this_total = sum(v['rev'] for v in this.values()) or 1.0 + rev_at_risk = sum(r['ytd_rev'] for r in rows if r['status'] in ('OUT', 'LOW')) + return {'rows': rows[:top], 'n_out': sum(1 for r in rows if r['status'] == 'OUT'), + 'n_low': sum(1 for r in rows if r['status'] == 'LOW'), + 'rev_at_risk': rev_at_risk, 'risk_pct': rev_at_risk / this_total * 100, + 'drop_stockout': drop_stockout, 'drop_other': drop_other, + 'total_drop': drop_stockout + drop_other} + + +# ====================================================================== ADVANCED FILTERS +# "Find customers who bought X" — resolve a SKU query / category to the set of partner ids that +# purchased it, so the directory can be filtered by purchase behaviour (not just attributes). + +def _buyer_window(t): + """24-month look-back for 'who bought this' — catches recent AND lapsed buyers (win-back).""" + t = t or P.today() + return (t - dt.timedelta(days=730)).isoformat(), t.isoformat() + + +def sku_buyers(query, t=None, team_id=None): + """Set of partner ids who bought any SKU whose code/name matches `query` (last 24 months). + Returns None when the query is blank (= no filter).""" + q = (query or '').strip() + # A 1-char ilike matches half the catalogue — treat it as "still typing" (no filter yet). + if len(q) < 2: + return None + df, dtt = _buyer_window(t) + prods = O.search_read('product.product', ['|', ('default_code', 'ilike', q), ('name', 'ilike', q)], + ['id'], limit=3000) + pids = [p['id'] for p in prods] + if not pids: + return set() + g = O.read_group('sale.order.line', + O.sale_line_domain(df, dtt, team_id, extra=[('product_id', 'in', pids)]), + ['order_partner_id'], ['order_partner_id'], lazy=False) + return {O.m2o_id(r.get('order_partner_id')) for r in g if r.get('order_partner_id')} + + +def category_buyers(category, t=None, team_id=None): + """Set of partner ids who bought from a main category (last 24 months). None = no filter.""" + if not category or category == '(any)': + return None + df, dtt = _buyer_window(t) + catmap = sales_mod._product_cat() + prod_ids = [pid for pid, c in catmap.items() if c == category] + if not prod_ids: + return set() + g = O.read_group('sale.order.line', + O.sale_line_domain(df, dtt, team_id, extra=[('product_id', 'in', prod_ids)]), + ['order_partner_id'], ['order_partner_id'], lazy=False) + return {O.m2o_id(r.get('order_partner_id')) for r in g if r.get('order_partner_id')} + + +def main_categories(): + """Sorted list of main category names (for the 'bought in category' selector).""" + return sorted(set(sales_mod._product_cat().values())) + + +# ====================================================================== WIN-BACK / SKU MOVES +def customer_winback(pid, t=None, team_id=None, top=10): + """SKU-level YoY moves for one customer (this YTD vs same period last year): + losers — SKUs down YoY, each with its share of the customer's TOTAL gross loss (pct_of_loss) + gainers — SKUs up YoY (bought more) + lapsed — losers gone to zero this year (the re-pitch hooks) + lapsed_categories — category-level gaps + total_loss / this_total / last_total / net_change + 'change' is signed (this − last); negative = decline so the UI tints it red.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + + def sku_map(df, dtt): + g = O.read_group('sale.order.line', _line_dom(pid, df, dtt, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False) + return {O.m2o_id(r['product_id']): {'name': O.m2o_name(r.get('product_id')), + 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0} + for r in g if r.get('product_id')} + + this, last = sku_map(yf, yt), sku_map(lf, lt) + # product_id -> SKU code, so each row can deep-link to its SKU drawer (code is how the SKU + # views key products; merged-by-code duplicates resolve to the same drawer) + pids = list(set(this) | set(last)) + codemap = {} + if pids: + for pr in O.search_read('product.product', [('id', 'in', pids)], ['default_code']): + codemap[pr['id']] = str(pr['default_code']).strip() if pr.get('default_code') else None + rows = [] + for p in (set(this) | set(last)): + tv, lv = (this.get(p) or {}), (last.get(p) or {}) + tr, lr = tv.get('rev', 0.0), lv.get('rev', 0.0) + tq, lq = tv.get('qty', 0.0), lv.get('qty', 0.0) + name = (this.get(p) or last.get(p) or {}).get('name') or '(?)' + p_ly = (lr / lq) if lq else None # realised $/unit last year + p_ytd = (tr / tq) if tq else None # realised $/unit this year + rows.append({'sku': name, 'code': codemap.get(p), 'ly_rev': lr, 'ytd_rev': tr, + 'change': tr - lr, 'qty_ly': lq, 'qty_ytd': tq, 'qty_change': tq - lq, + 'price_ly': p_ly, 'price_ytd': p_ytd, + 'price_chg_pct': (((p_ytd - p_ly) / p_ly * 100) if (p_ly and p_ytd) else None), + # split the $ change into volume vs price effects (sum to change) + 'vol_effect': ((tq - lq) * p_ly) if p_ly is not None else (tr - lr), + 'price_effect': ((p_ytd - p_ly) * tq) if (p_ly is not None and p_ytd is not None) else 0.0}) + total_loss = sum(-r['change'] for r in rows if r['change'] < 0) + for r in rows: + r['pct_of_loss'] = (-r['change'] / total_loss * 100) if (r['change'] < 0 and total_loss) else 0.0 + losers = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change']) + gainers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change']) + lapsed = [r for r in losers if r['ytd_rev'] == 0] + + cat = sales_mod._product_cat() + + def cat_rev(m): + agg = {} + for p, v in m.items(): + agg[cat.get(p, '(uncategorized)')] = agg.get(cat.get(p, '(uncategorized)'), 0.0) + v['rev'] + return agg + tcat, lcat = cat_rev(this), cat_rev(last) + lapsed_categories = sorted([{'category': c, 'ly_rev': lcat[c], 'ytd_rev': tcat.get(c, 0.0), + 'change': tcat.get(c, 0.0) - lcat[c]} + for c in lcat if lcat[c] > tcat.get(c, 0.0)], + key=lambda x: x['change']) + this_total = sum(v['rev'] for v in this.values()) + last_total = sum(v['rev'] for v in last.values()) + # 'all_skus' feeds the in-drawer SKU list; cap it so a mega-account doesn't ship thousands + # of rows to the browser (the list is searchable/filtered, top-by-revenue is what matters). + all_skus = sorted(rows, key=lambda x: -max(x['ytd_rev'], x['ly_rev']))[:500] + return { + 'losers': losers[:top], 'gainers': gainers[:top], 'lapsed': lapsed[:top], + 'lapsed_categories': lapsed_categories[:6], 'all_skus': all_skus, 'n_skus': len(rows), + 'n_lapsed': len(lapsed), 'n_losers': len(losers), 'n_gainers': len(gainers), + 'total_loss': total_loss, 'lost_dollars': total_loss, + 'this_total': this_total, 'last_total': last_total, 'net_change': this_total - last_total, + } + + +def customer_yoy_decomp(pid, t=None, team_id=None): + """Decompose the customer's YoY sales change into a volume (order count) effect and a price + (basket size / AOV) effect. Identity: ΔSales = ΔOrders·AOV_last + Orders_this·ΔAOV.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this_sales = O.sum_field('sale.order', _order_dom(pid, yf, yt, team_id), 'amount_untaxed') + last_sales = O.sum_field('sale.order', _order_dom(pid, lf, lt, team_id), 'amount_untaxed') + this_orders = O.get_odoo().search_count('sale.order', _order_dom(pid, yf, yt, team_id)) + last_orders = O.get_odoo().search_count('sale.order', _order_dom(pid, lf, lt, team_id)) + this_aov = (this_sales / this_orders) if this_orders else 0.0 + last_aov = (last_sales / last_orders) if last_orders else 0.0 + volume_effect = (this_orders - last_orders) * last_aov + price_effect = this_orders * (this_aov - last_aov) + return { + 'this_sales': this_sales, 'last_sales': last_sales, 'd_sales': this_sales - last_sales, + 'sales_yoy_pct': P.yoy_pct(this_sales, last_sales), + 'this_orders': this_orders, 'last_orders': last_orders, 'd_orders': this_orders - last_orders, + 'this_aov': this_aov, 'last_aov': last_aov, 'd_aov': this_aov - last_aov, + 'aov_yoy_pct': P.yoy_pct(this_aov, last_aov), + 'volume_effect': volume_effect, 'price_effect': price_effect, + } + + +def customer_affinity(pid, t=None, team_id=None, top=12, max_cobuyers=60): + """'Customers who buy similar products also buy …'. Finds the customers who bought this + customer's SKUs (co-buyers), then ranks the OTHER SKUs those co-buyers buy (that this customer + doesn't) by spend among them — a cross-sell list. LTM window + capped co-buyers to stay fast.""" + df, dtt = P.ltm(t) + g = O.read_group('sale.order.line', _line_dom(pid, df, dtt, team_id), + ['price_subtotal:sum'], ['product_id'], lazy=False) + mine = {O.m2o_id(r['product_id']) for r in g if r.get('product_id')} + if not mine: + return {'recs': [], 'n_cobuyers': 0, 'mine': 0} + cg = O.read_group('sale.order.line', + O.sale_line_domain(df, dtt, team_id, extra=[('product_id', 'in', list(mine))]), + ['price_subtotal:sum'], ['order_partner_id'], lazy=False) + cobuyers = sorted([(O.m2o_id(r['order_partner_id']), r.get('price_subtotal') or 0.0) + for r in cg if r.get('order_partner_id') and O.m2o_id(r['order_partner_id']) != pid], + key=lambda x: -x[1])[:max_cobuyers] + cob_ids = [c[0] for c in cobuyers] + if not cob_ids: + return {'recs': [], 'n_cobuyers': 0, 'mine': len(mine)} + # Single groupby over the co-buyers' lines (one aggregated row per product) — fast. + pg = O.read_group('sale.order.line', + O.sale_line_domain(df, dtt, team_id, extra=[('order_partner_id', 'in', cob_ids)]), + ['price_subtotal:sum'], ['product_id'], lazy=False) + recs = sorted([{'sku': O.m2o_name(r['product_id']), 'pid': O.m2o_id(r['product_id']), + 'rev': r.get('price_subtotal') or 0.0, 'orders': r.get('__count') or 0} + for r in pg if r.get('product_id') and O.m2o_id(r['product_id']) not in mine], + key=lambda x: -x['rev'])[:top] + if recs: + codemap = {} + for pr in O.search_read('product.product', [('id', 'in', [r['pid'] for r in recs])], ['default_code']): + codemap[pr['id']] = str(pr['default_code']).strip() if pr.get('default_code') else None + for r in recs: + r['code'] = codemap.get(r['pid']) + return {'recs': recs, 'n_cobuyers': len(cob_ids), 'mine': len(mine)} + + +# ====================================================================== CADENCE / CHURN / BENCHMARK +def _order_dates(pid, team_id=None): + """Recent confirmed order dates (date objects), ascending. Capped at the 3000 most recent so a + very high-volume buyer can't hang the drawer — cadence/frequency only need recent orders.""" + rows = O.search_read('sale.order', _order_dom(pid, None, None, team_id), + ['date_order'], order='date_order desc', limit=3000) + return sorted(dt.date.fromisoformat(str(r['date_order'])[:10]) for r in rows if r.get('date_order')) + + +def customer_cadence(pid, t=None, team_id=None): + """Reorder rhythm from the gaps between orders: typical gap (median of recent), predicted next + order, days overdue vs that rhythm, and whether the rhythm is slowing (last-4 vs prior-4 gap).""" + t = t or P.today() + dates = _order_dates(pid, team_id) + n = len(dates) + out = {'n_orders': n, 'median_gap_days': None, 'last_order': dates[-1].isoformat() if dates else None, + 'predicted_next': None, 'overdue_days': None, 'drift_pct': None, + 'recent_gap': None, 'prior_gap': None, 'dates': [d.isoformat() for d in dates]} + if n < 2: + return out + gaps = [(dates[i] - dates[i - 1]).days for i in range(1, n)] + median_gap = statistics.median(gaps[-8:]) + last = dates[-1] + predicted = last + dt.timedelta(days=round(median_gap)) + recent_gap = statistics.mean(gaps[-4:]) if len(gaps) >= 4 else statistics.mean(gaps) + prior_gap = statistics.mean(gaps[-8:-4]) if len(gaps) >= 8 else None + out.update({'median_gap_days': median_gap, 'predicted_next': predicted.isoformat(), + 'overdue_days': (t - predicted).days, 'recent_gap': recent_gap, 'prior_gap': prior_gap, + 'drift_pct': ((recent_gap - prior_gap) / prior_gap * 100) if prior_gap else None}) + return out + + +def customer_churn_score(pid, t=None, team_id=None, cad=None, sales_yoy=None): + """0–100 churn-risk score (higher = more at risk): overdue-vs-cadence (50%) + frequency decay + last-90 vs prior-90 (30%) + YoY sales trend (20%). Bucketed Healthy/Watch/At-risk/Critical.""" + t = t or P.today() + cad = cad or customer_cadence(pid, t, team_id) + mg = cad.get('median_gap_days') + if mg and cad.get('overdue_days') is not None: + s_overdue = min(1.0, max(0.0, cad['overdue_days'] / mg) / 2.0) # 2 cycles late = max + else: + s_overdue = 0.5 + dates = [dt.date.fromisoformat(d) for d in cad.get('dates', [])] + last90 = sum(1 for d in dates if (t - d).days <= 90) + prior90 = sum(1 for d in dates if 90 < (t - d).days <= 180) + if prior90 == 0: + # recent activity but no 90–180d baseline (reactivated / new) → cautious mid risk, not zero + s_freq = 0.6 if last90 == 0 else 0.3 + else: + s_freq = min(1.0, max(0.0, (prior90 - last90) / prior90)) + if sales_yoy is None: + sales_yoy = customer_yoy_decomp(pid, t, team_id)['sales_yoy_pct'] + s_yoy = 0.5 if sales_yoy is None else min(1.0, max(0.0, -sales_yoy / 50.0)) # -50% YoY = max + score = 100 * (0.5 * s_overdue + 0.3 * s_freq + 0.2 * s_yoy) + bucket = ('Critical' if score >= 70 else 'At-risk' if score >= 45 + else 'Watch' if score >= 25 else 'Healthy') + driver = max([('overdue rhythm', s_overdue), ('fewer recent orders', s_freq), + ('falling spend', s_yoy)], key=lambda x: x[1])[0] + return {'score': round(score), 'bucket': bucket, 'driver': driver} + + +def _pctile(values, x): + """Percentile rank (0–100) of x within values — mean/midpoint method (ties count as half), so + the median of a peer set lands at the 50th percentile.""" + if not values: + return None + below = sum(1 for v in values if v < x) + equal = sum(1 for v in values if v == x) + return (below + 0.5 * equal) / len(values) * 100 + + +def rank_contribution(directory_rows, pid): + """Rank by YTD revenue + % of BU YTD — pure, from the already-cached directory list.""" + total = sum(r['revenue'] for r in directory_rows) or 1.0 + ranked = sorted(directory_rows, key=lambda r: -r['revenue']) + rank = next((i + 1 for i, r in enumerate(ranked) if r['pid'] == pid), None) + me = next((r for r in directory_rows if r['pid'] == pid), None) + return {'rank': rank, 'n': len(directory_rows), + 'pct_of_bu': (me['revenue'] / total * 100) if me else None} + + +def peer_benchmark(directory_rows, pid): + """Percentile rank vs same value-tier peers on revenue, AOV, frequency, YoY — pure.""" + me = next((r for r in directory_rows if r['pid'] == pid), None) + if not me: + return None + tier = _tier(me['revenue']) + peers = [r for r in directory_rows if _tier(r['revenue']) == tier] + yoy_vals = [r['yoy_pct'] for r in peers if r['yoy_pct'] is not None] + return {'cohort_n': len(peers), 'tier': tier, + 'rev_pctile': _pctile([r['revenue'] for r in peers], me['revenue']), + 'aov_pctile': _pctile([r['aov'] for r in peers], me['aov']), + 'freq_pctile': _pctile([r['orders'] for r in peers], me['orders']), + 'yoy_pctile': (_pctile(yoy_vals, me['yoy_pct']) if me['yoy_pct'] is not None else None)} + + +def customer_whitespace(mine_categories, company_categories, top=6): + """Pure: breadth (# categories bought / company total) + the biggest company categories this + customer buys $0 of, ranked by company revenue. `company_categories` = sales.by_category rows.""" + skip = {'(uncategorized)', 'All'} + mine = {c for c in mine_categories if c not in skip} + cats = [c for c in company_categories if c['category'] not in skip and c['revenue'] > 0] + total = len(cats) + ws = sorted([c for c in cats if c['category'] not in mine], key=lambda x: -x['revenue'])[:top] + return {'breadth_x': len(mine), 'breadth_y': total, 'whitespace': ws} + + +# ====================================================================== PAGE-LEVEL: NRR / MIGRATION +def nrr(t=None, team_id=None, agent_pids=None): + """Net Revenue Retention from the revenue bridge (existing book only; new logos excluded).""" + b = revenue_bridge(t, team_id, agent_pids=agent_pids) + last = b['last_total'] or 1.0 + ending = b['last_total'] + b['expansion']['rev'] + b['contraction']['rev'] + b['lost']['rev'] + return {'nrr_pct': ending / last * 100, 'starting': b['last_total'], + 'expansion': b['expansion']['rev'], 'contraction': b['contraction']['rev'], + 'churned': b['lost']['rev'], 'ending_existing': ending, 'new': b['new']['rev']} + + +def tier_migration(t=None, team_id=None, agent_pids=None): + """Value-tier flows LTM vs prior-LTM: upgraded / held / downgraded / new / lapsed + net $.""" + lf, lt = P.ltm(t) + pf, pt = P.prior_ltm(t) + this = _cust_rev(lf, lt, team_id, agent_pids) + last = _cust_rev(pf, pt, team_id, agent_pids) + order = {name: i for i, (name, _) in enumerate(TIERS)} # 0 = Whale (top) … 3 = Small + flows = {k: {'flow': k, 'n': 0, 'net': 0.0} for k in + ['Upgraded', 'Held', 'Downgraded', 'New / reactivated', 'Lapsed']} + for pid in set(this) | set(last): + tr = this.get(pid, {}).get('rev', 0.0) + lr = last.get(pid, {}).get('rev', 0.0) + if pid in this and pid in last: + k = ('Upgraded' if order[_tier(tr)] < order[_tier(lr)] + else 'Downgraded' if order[_tier(tr)] > order[_tier(lr)] else 'Held') + elif pid in this: + k = 'New / reactivated' + else: + k = 'Lapsed' + flows[k]['n'] += 1 + flows[k]['net'] += (tr - lr) + return {'flows': list(flows.values())} + + +# ====================================================================== GROUP DRAWER (rollup → drawer) +def group_detail(dim, value, t=None, team_id=None, n_months=13, top=15, agent_pids=None): + """For one rollup group (e.g. dim='city', value='Brooklyn'): KPIs, monthly trend, the customers + in the group (clickable), and the top SKUs sold there. dim is a DIMENSIONS key.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + attrs = _partner_attrs(set(this) | set(last)) + + def keyfor(pid): + if dim == 'segment': + return _tier(this.get(pid, {}).get('rev', last.get(pid, {}).get('rev', 0.0))) + return (attrs.get(pid) or {}).get(dim) or '(none)' + pids = [pid for pid in (set(this) | set(last)) if keyfor(pid) == value] + if not pids: + return None + + # KPI totals and the customer list cover EVERY member of the group (cheap, in-memory). + rev_ytd = sum(this.get(p, {}).get('rev', 0.0) for p in pids) + rev_ly = sum(last.get(p, {}).get('rev', 0.0) for p in pids) + custs = sorted([{'pid': p, 'customer': (this.get(p) or last.get(p) or {}).get('name', '?'), + 'revenue': this.get(p, {}).get('rev', 0.0), 'orders': this.get(p, {}).get('orders', 0), + 'yoy_pct': P.yoy_pct(this.get(p, {}).get('rev', 0.0), last.get(p, {}).get('rev', 0.0)), + 'city': (attrs.get(p) or {}).get('city', '(none)'), + 'state': (attrs.get(p) or {}).get('state', '(none)'), + 'agent': (attrs.get(p) or {}).get('agent', '(none)')} + for p in pids], key=lambda x: -x['revenue']) + + # The trend + top-SKU read_groups put every pid in an IN(...) clause, so a huge group + # (a big city, the '(none)' bucket) would build a slow, oversized query. Cap those two + # queries to the group's top buyers by revenue; KPIs/customer list above stay complete. + QCAP = 300 + capped = len(pids) > QCAP + qpids = sorted( + pids, key=lambda p: -max(this.get(p, {}).get('rev', 0.0), last.get(p, {}).get('rev', 0.0)) + )[:QCAP] if capped else pids + + range_start = dt.date(t.year - 2, t.month, 1).isoformat() + g = O.read_group('sale.order', + sales_mod.order_domain(range_start, t.isoformat(), team_id) + [('partner_id', 'in', qpids)], + ['amount_untaxed:sum'], ['date_order:month'], lazy=False) + mrev = {} + for r in g: + rng = (r.get('__range') or {}).get('date_order:month') or {} + ym = (rng.get('from') or '')[:7] + if ym: + mrev[ym] = r.get('amount_untaxed') or 0.0 + monthly = [] + for ym, _s, _e in P.month_starts(n_months, t): + y, m = int(ym[:4]) - 1, int(ym[5:7]) + monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0), + 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)}) + + mf, mt = P.ltm(t) + lg = O.read_group('sale.order.line', + O.sale_line_domain(mf, mt, team_id, extra=[('order_partner_id', 'in', qpids)]), + ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], ['product_id'], lazy=False) + skus = sorted([sales_mod._sku_profit( + {'product': O.m2o_name(r.get('product_id')), 'pid': O.m2o_id(r.get('product_id')), + 'rev': r.get('price_subtotal') or 0.0, 'revenue': r.get('price_subtotal') or 0.0, + 'qty': r.get('product_uom_qty') or 0.0, + 'margin': r.get('margin') or 0.0, 'lines': r.get('__count') or 0}) + for r in lg if r.get('product_id')], + key=lambda x: -x['rev'])[:top] + if skus: # attach SKU code for deep-linking each to its SKU drawer + scode = {} + for pr in O.search_read('product.product', [('id', 'in', [s['pid'] for s in skus])], ['default_code']): + scode[pr['id']] = str(pr['default_code']).strip() if pr.get('default_code') else None + for s in skus: + s['code'] = scode.get(s['pid']) + return {'dim': dim, 'value': value, 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, + 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly), 'n_customers': len([p for p in pids if p in this]), + 'n_total': len(pids), 'monthly': monthly, 'top_skus': skus, 'customers': custs, + 'capped': capped, 'qcap': QCAP} + + +# ----------------------------------------------------------- KPI-card customer sets +_KPI_SETS = { + 'active': 'Active customers (YTD)', 'new': 'New / reactivated (YTD)', + 'lost': 'Lost customers (YTD)', 'retained': 'Retained customers (YTD)', + 'existing': 'Existing book (NRR base)', +} + + +def customer_set(kind, t=None, team_id=None, agent_pids=None): + """Standardized customer list behind a headline KPI card (Active / New / Lost / Retained / + Existing-book), each row carrying the same shape (pid, customer, rev_ytd, rev_ly, change, + yoy_pct, orders, city, state, agent) so the drawer renders them with one standardized view.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _cust_rev(yf, yt, team_id, agent_pids) + last = _cust_rev(lf, lt, team_id, agent_pids) + tset, lset = set(this), set(last) + ids = {'active': tset, 'new': tset - lset, 'lost': lset - tset, + 'retained': tset & lset, 'existing': lset}.get(kind, tset) + attrs = _partner_attrs(list(ids)) + rows = [] + for p in ids: + tr = this.get(p, {}).get('rev', 0.0) + lr = last.get(p, {}).get('rev', 0.0) + a = attrs.get(p, {}) + rows.append({'pid': p, 'customer': (this.get(p) or last.get(p) or {}).get('name', '?'), + 'rev_ytd': tr, 'rev_ly': lr, 'change': tr - lr, 'yoy_pct': P.yoy_pct(tr, lr), + 'orders': this.get(p, {}).get('orders', 0), + 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'), + 'agent': a.get('agent', '(none)')}) + rows.sort(key=lambda r: -(r['rev_ly'] if kind == 'lost' else r['rev_ytd'])) + return {'kind': kind, 'label': _KPI_SETS.get(kind, kind), 'rows': rows, 'n': len(rows), + 'rev_ytd': sum(r['rev_ytd'] for r in rows), 'rev_ly': sum(r['rev_ly'] for r in rows)} + + +# ----------------------------------------------------------- VALIDATION +def validate(t=None, team_id=None): + """Reconcile every headline number to an independent Odoo aggregate. + + When team_id is set (a single BU is selected) the checks run SCOPED to that BU so the + validation panel never exposes other-BU figures — BU isolation holds even here. The two + cross-BU brand-mirror checks (#2, #4b) only make sense consolidated, so they run only + when team_id is None. + """ + t = t or P.today() + checks = [] + + # 1. Bridge identity: last + new + expansion + contraction + lost == this + b = revenue_bridge(t, team_id=team_id) + recon = (b['last_total'] + b['new']['rev'] + b['expansion']['rev'] + + b['contraction']['rev'] + b['lost']['rev']) + checks.append({'check': 'Revenue bridge reconciles last→this (New+Exp+Contr+Lost)', + 'a': round(recon, 2), 'b': round(b['this_total'], 2), + 'gap': round(recon - b['this_total'], 2), + 'ok': abs(recon - b['this_total']) <= 1.0}) + + # 2. Σ(brand this_total) == company this_total (consolidated only — cross-BU) + if team_id is None: + brand_this = sum(r['this'] for r in bridge_by_brand(t)) + checks.append({'check': 'Σ(brand YTD) == company YTD', + 'a': round(brand_this, 2), 'b': round(b['this_total'], 2), + 'gap': round(brand_this - b['this_total'], 2), + 'ok': abs(brand_this - b['this_total']) <= 1.0}) + + # 3. Σ(segment LTM rev) == total LTM rev + lf, lt = P.ltm(t) + seg_sum = sum(s['revenue'] for s in segments(t, team_id=team_id)) + ltm_total = sum(v['rev'] for v in _cust_rev(lf, lt, team_id=team_id).values()) + checks.append({'check': 'Σ(segment LTM rev) == total LTM rev', + 'a': round(seg_sum, 2), 'b': round(ltm_total, 2), + 'gap': round(seg_sum - ltm_total, 2), + 'ok': abs(seg_sum - ltm_total) <= 1.0}) + + # 4. Sum-level rollup is MECE: Σ(by_dimension('agent')) == total YTD revenue + yf, yt = P.ytd(t) + ytd_total = sum(v['rev'] for v in _cust_rev(yf, yt, team_id=team_id).values()) + roll = by_dimension('agent', t, team_id=team_id) + dim_sum = sum(r['revenue'] for r in roll) + checks.append({'check': 'Σ(agent rollup) == total YTD revenue', + 'a': round(dim_sum, 2), 'b': round(ytd_total, 2), + 'gap': round(dim_sum - ytd_total, 2), + 'ok': abs(dim_sum - ytd_total) <= 1.0}) + + # 4b. Brand mirror reconciles: Σ(Fisch)+Σ(Royal) == total YTD (consolidated only) + if team_id is None: + brand_sum = sum(r['fisch'] + r['royal'] for r in roll) + checks.append({'check': 'Rollup brand mirror: Σ(Fisch)+Σ(Royal) == total YTD', + 'a': round(brand_sum, 2), 'b': round(ytd_total, 2), + 'gap': round(brand_sum - ytd_total, 2), + 'ok': abs(brand_sum - ytd_total) <= 1.0}) + + # 5. Drill-down scope holds: top customer's LTM order-level rev == line-level rev + top = directory(t, team_id=team_id, limit=1) + if top: + pid = top[0]['pid'] + mf, mt = P.ltm(t) + ord_rev = O.sum_field('sale.order', _order_dom(pid, mf, mt), 'amount_untaxed') + line_rev = O.sum_field('sale.order.line', _line_dom(pid, mf, mt), 'price_subtotal') + checks.append({'check': 'Drill-down: top customer LTM order==line revenue', + 'a': round(ord_rev, 2), 'b': round(line_rev, 2), + 'gap': round(ord_rev - line_rev, 2), + 'ok': abs(ord_rev - line_rev) <= max(1.0, 0.005 * ord_rev)}) + + # 6. YoY decomposition identity: volume effect + price effect == Δsales (drawer math) + if top: + dc = customer_yoy_decomp(top[0]['pid'], t, team_id=team_id) + recon = dc['volume_effect'] + dc['price_effect'] + checks.append({'check': 'Drawer: volume + price effect == Δsales (top customer)', + 'a': round(recon, 2), 'b': round(dc['d_sales'], 2), + 'gap': round(recon - dc['d_sales'], 2), + 'ok': abs(recon - dc['d_sales']) <= max(1.0, 0.005 * abs(dc['d_sales']) + 1)}) + + # 7. NRR ties to the bridge: ending-existing == last + expansion + contraction + churned + nr = nrr(t, team_id=team_id) + nrr_recon = nr['starting'] + nr['expansion'] + nr['contraction'] + nr['churned'] + checks.append({'check': 'NRR: ending-existing == start + exp + contr + churn', + 'a': round(nrr_recon, 2), 'b': round(nr['ending_existing'], 2), + 'gap': round(nrr_recon - nr['ending_existing'], 2), + 'ok': abs(nrr_recon - nr['ending_existing']) <= 1.0}) + + # 8. Churn score bounded [0,100] with a valid bucket (drawer pill) + if top: + cs = customer_churn_score(top[0]['pid'], t, team_id=team_id) + ok = 0 <= cs['score'] <= 100 and cs['bucket'] in ('Healthy', 'Watch', 'At-risk', 'Critical') + checks.append({'check': f"Churn score in [0,100] with valid bucket ({cs['bucket']})", + 'a': cs['score'], 'b': cs['score'], 'gap': 0, 'ok': ok}) + + # 9. Tier migration is MECE: Σ(flow customers) == distinct customers active in either window + tm = tier_migration(t) + lf, lt = P.ltm(t) + pf, pt = P.prior_ltm(t) + active = len(set(_cust_rev(lf, lt)) | set(_cust_rev(pf, pt))) + flow_n = sum(f['n'] for f in tm['flows']) + checks.append({'check': 'Tier migration: Σ(flow customers) == active (LTM ∪ prior-LTM)', + 'a': flow_n, 'b': active, 'gap': flow_n - active, 'ok': flow_n == active}) + return checks diff --git a/royalimports_os/modules/dictionary.py b/royalimports_os/modules/dictionary.py new file mode 100644 index 0000000000000000000000000000000000000000..3dbf203f7a179c611c932210430b9e01601e02b0 --- /dev/null +++ b/royalimports_os/modules/dictionary.py @@ -0,0 +1,51 @@ +"""Metric Dictionary — the semantic layer's module adapter (OM-0c, 2026-07-11). + +The truth lives in harness/semantic.py + model/*.yml (topics + metrics, one governed definition +per metric). This thin adapter exists so the app conventions hold: registry row (validate=True, +nav=False — the page is a sidebar sentinel like Feedback) → validate.py auto-runs the semantic +layer's independent reconciliation contracts alongside every module's checks. + +modules→harness is sanctioned HERE (the dictionary IS the platform layer's surface); the reverse +lazy import inside semantic.py (order-scope builder) is tracked for the OM-2 lift into core/store. +""" +import harness.semantic as S + + +def topics(): + return S.topics() + + +def metrics(): + return S.metrics() + + +def describe(key): + return S.describe(key) + + +def metric(key, date_from=None, date_to=None, team_id=None): + return S.metric(key, date_from, date_to, team_id) + + +def formula(m): + """Human-readable formula for a metric definition (the dictionary's Formula column).""" + agg = m.get("agg") + if agg == "ratio": + return f"{m['numerator']} ÷ {m['denominator']}" + if agg == "derived": + return m.get("expr", "") + if agg == "count": + return "count(rows)" + if agg == "count_distinct": + return f"distinct {m['field']}" + return f"Σ {m.get('field', '?')}" + + +def run_reconciliation(): + """Run every declared independent contract live against Odoo (YTD, all-BU + per-BU).""" + return S.validate() + + +def validate(pre=None): + """validate.py convention: the dictionary's checks ARE the semantic layer's contracts.""" + return S.validate() diff --git a/royalimports_os/modules/digest.py b/royalimports_os/modules/digest.py new file mode 100644 index 0000000000000000000000000000000000000000..c5e742c32eeba1cc97ab884504e8db8f49524b6b --- /dev/null +++ b/royalimports_os/modules/digest.py @@ -0,0 +1,224 @@ +"""Daily agent digest — "Your book this morning": one email per user, their queue only. + +Delivery reuses the ONE sanctioned Odoo write path (modules/collections_send.py: mail.mail +create through the Office 365 relay) including its SAFE_MODE allow-list — while the guardrail +is ON no digest can leave the allow-list. Content rules (adoption canon, engagement-adoption +brief): ≤10 lines, only non-empty sections, every line deep-links back to the exact row +(?src=digest), and the email is SKIPPED entirely when there is nothing to act on — silence +keeps opens high. Internal mail: no model/res_id (these must NOT land in customer chatter). + +Scheduling: app.py starts one daemon ticker (in the cache_resource singleton) that calls +run_due() every few minutes; run_due sends at most once per (weekday, user) after +DIGEST_UTC_HOUR (default 11:00 UTC = 7am ET in summer), idempotent across Space restarts via +the store key 'digest_log' (empty-skips are logged too, so a quiet day isn't rebuilt all day). + +BU isolation carries into email: a single-BU user's digest is scoped to that BU's team_id. +""" +import os +import html as _html +import datetime as dt + +import core.store as store +import core.users as users +import core.links as links +import modules.collections_send as cs +import modules.customer_data as md # call-list / win-back queues (ex-myday, ex-customer_list) +import modules.tasks as tasks +import modules.customers as cust + +KEY = 'digest_log' +UTC_HOUR = int(os.environ.get('DIGEST_UTC_HOUR', '11')) +WEEKDAYS_ONLY = os.environ.get('DIGEST_WEEKDAYS_ONLY', '1').strip().lower() \ + not in ('0', 'false', 'no', '') +ENABLED = os.environ.get('DIGEST_ENABLED', '1').strip().lower() not in ('0', 'false', 'no', '') + +_NAVY, _GOLD, _MUTED = '#5B8FD9', '#D9A93A', '#6B7280' + + +def _team_for(user): + """Single-BU users get a digest scoped to that BU — isolation carries into email.""" + bus = (user or {}).get('bus', 'all') + if isinstance(bus, (list, tuple)) and len(bus) == 1: + return bus[0] + return None + + +def _m(x): + try: + return f"${float(x):,.0f}" + except (TypeError, ValueError): + return '—' + + +def _line(text, href, meta=''): + t = _html.escape(str(text)) + m = f" {_html.escape(str(meta))}" if meta else '' + return (f"" + f"{t}" + f"{m}") + + +def _sec(title): + return (f"{_html.escape(title)}") + + +def build(user, t=None): + """(subject, html, n_actionable) for one user. n_actionable == 0 → skip the send.""" + t = t or dt.date.today() + uname = user.get('username', '') + agent = user.get('agent') + team_id = _team_for(user) + pids = md.agent_pids(agent) + + today = t.isoformat() + my = [x for x in tasks.for_owner(uname)] if store.available() else [] + due = [x for x in my if (x.get('due') or '9999-12-31') <= today] + q = md.queues(agent, team_id, limit=25) + calls, risk = q['calls'][:5], q['risk'][:3] + + # yesterday's orders for the book (one read_group; skip silently on a transport error — + # the digest must never fail because one section couldn't be pulled) + yday = (t - dt.timedelta(days=1)).isoformat() + y_rev = y_orders = 0 + try: + rev_map = cust._cust_rev(yday, yday, team_id, pids) + y_rev = sum(v.get('rev', 0.0) for v in rev_map.values()) + y_orders = sum(v.get('orders', 0) for v in rev_map.values()) + except Exception: + pass + + n = len(due) + len(calls) + len(risk) + if n == 0 and y_orders == 0: + return None, None, 0 + + rows = [] + if due: + rows.append(_sec(f'Tasks due ({len(due)})')) + for x in due[:5]: + ent = x.get('entity') or {} + href = links.deeplink('customer_data', src='digest') + if ent.get('kind') and ent.get('id') is not None: + href = links.deeplink('customer_data', ent['kind'], ent['id'], src='digest') + meta = f"due {x.get('due') or '—'}" + rows.append(_line(x.get('title', 'Task'), href, meta)) + if calls: + rows.append(_sec('Call today — overdue vs their own cadence')) + for r in calls: + href = links.deeplink('customer_data', 'customer', r['pid'], src='digest') + meta = (f"{int(r.get('overdue_days') or 0)}d past cycle · " + f"est. missed {_m(r.get('est_missed'))}") + rows.append(_line(r.get('customer', '?'), href, meta)) + if risk: + rows.append(_sec('Win-back — down vs last year')) + for r in risk: + href = links.deeplink('customer_data', 'customer', r['pid'], src='digest') + meta = f"{_m(r.get('at_risk'))} at risk · {r.get('status', '')}" + rows.append(_line(r.get('customer', '?'), href, meta)) + if y_orders: + rows.append(_sec('Yesterday')) + rows.append(_line(f"{y_orders} orders · {_m(y_rev)}", + links.deeplink('sales', src='digest'))) + + scope = f"{agent}'s book" if agent else 'the whole book' + list_url = links.deeplink('customer_data', src='digest') + html_body = f""" +
+
+
Your book this morning
+
{t.strftime('%A, %B %d')} · {scope}
+
+ {''.join(rows)}
+ +
+ Internal daily digest from Loopable. + Sent only on days with something to act on. +
+
""" + subject = f"Your book this morning — {n} to act on · {t.strftime('%b %d')}" + return subject, html_body, n + + +def send_one(user, odoo=None, override_to=None): + """Build + queue one digest. Returns a short result string (never raises past itself).""" + to = override_to or user.get('email') + if not to: + return 'no-email' + subject, body, n = build(user) + if n == 0: + return 'empty-skip' + if not cs.safe_recipient_ok(to): + return 'safe-mode-blocked' + odoo = odoo or cs.Odoo() + mid = odoo.queue_mail({ + 'subject': subject, 'body_html': body, + 'email_to': to, 'email_from': cs.SENDER_DISPLAY, 'reply_to': cs.REPLY_TO, + 'mail_server_id': cs.ROYAL_MAIL_SERVER_ID, 'author_id': cs.ROYAL_AUTHOR_ID, + 'state': 'outgoing', 'auto_delete': False, + # This Odoo runs the custom rt_multi_dba module whose mail-create hook browses + # env[mail.model].browse(mail.res_id) — a mail WITHOUT model/res_id crashes with + # KeyError: False (verified 2026-07-05). Anchor internal mail on the house partner + # (Royal Imports, cs.ROYAL_AUTHOR_ID) so nothing lands in a CUSTOMER's chatter. + 'model': 'res.partner', 'res_id': cs.ROYAL_AUTHOR_ID, + }) + return f'queued:{mid}' + + +def run_due(force_user=None, override_to=None): + """The ticker entry point. Normal path: after UTC_HOUR on weekdays, send each active + emailed user their digest at most once per day (dedup via digest_log, including + empty-skips). force_user bypasses the schedule/dedup (the 'Send my digest now' button) + but NEVER the SAFE_MODE guardrail; forced sends are not logged (the 7am run still runs).""" + if not store.available(): + return {} + now = dt.datetime.utcnow() + today = now.date().isoformat() + if force_user is None: + if not ENABLED: + return {} + if WEEKDAYS_ONLY and now.weekday() >= 5: + return {} + if now.hour < UTC_HOUR: + return {} + try: + reg = users.registry() + except Exception: + return {} + already = (store.get(KEY) or {}).get(today, {}) if force_user is None else {} + results = {} + odoo = None + for un, u in sorted(reg.items()): + if force_user is not None and un != force_user: + continue + if not u.get('active', True) or not u.get('email'): + continue + if force_user is None and un in already: + continue + pub = users._public(un, u) + try: + if odoo is None: + odoo = cs.Odoo() + results[un] = send_one(pub, odoo, override_to=override_to) + except cs.SafeModeBlocked: + results[un] = 'safe-mode-blocked' + except Exception as e: + results[un] = f'error:{str(e)[:80]}' + if force_user is None and results: + stamp = now.strftime('%H:%M') + + def _rec(d): + day = d.setdefault(today, {}) + for un, r in results.items(): + day[un] = {'at': stamp, 'result': r} + return d + try: + store.update(KEY, _rec) + except Exception: + pass + return results + + +def today_log(): + return (store.get(KEY) or {}).get(dt.date.today().isoformat(), {}) diff --git a/royalimports_os/modules/expenses.py b/royalimports_os/modules/expenses.py new file mode 100644 index 0000000000000000000000000000000000000000..e75a634577d69cba60d614d95d629ce8814d6aa1 --- /dev/null +++ b/royalimports_os/modules/expenses.py @@ -0,0 +1,402 @@ +"""modules/expenses.py — Expenses: the operating-expense analytics dashboard (the CFO +cost-control surface). + +The company's operating cost BASE, read straight from the general ledger: every posted line to an +expense-type account (account_type in {'expense','expense_depreciation'}) over time. COGS +('expense_direct_cost') is the product side and is EXCLUDED — this is opex, not cost of sales. +Because it reads the GL directly (not just vendor bills) it also captures payroll journal entries, +depreciation and bank/merchant fees that never pass through account.move in_invoice. + +This is the MONITOR / DIAGNOSE complement to the Spend & Payables WORKFLOW (which recovers +vendor-bill waste). It answers one executive question: *what is our operating cost base, how is it +trending against revenue, what grew, and where did a category spike?* It NAMES the categories to +look at; it never performs an action. + + 1. TREND + LEVERAGE — monthly opex and opex-as-%-of-revenue (the operating-leverage picture: + are costs growing faster than the top line?). Revenue denominator = income-type GL accounts + (all channels), so the ratio is internally consistent and independently reconcilable. + 2. YoY BRIDGE — prior-LTM opex → per-category movers → current-LTM opex (what grew / fell). + 3. COST SPIKES — a Wheeler XmR (individuals) control chart per category flags the months a + category ran above its own upper control limit: a diagnostic watch list, worst first. + 4. FIXED VS VARIABLE — categories split by month-to-month coefficient of variation + coverage: + fixed/recurring (rent, wages, subscriptions) vs variable (where in-year cuts actually land). + 5. DIRECTORY — every expense account with LTM $, share, YoY, % of revenue and type, + each drilling to the raw ledger lines behind it (no unverifiable aggregates). + +Company-level: opex is shared overhead, not cleanly BU-splittable → an HQ module that ignores the +brand selector. READ-ONLY. +""" +import os + +import core.odoo as O +import core.periods as P + +EXP_TYPES = ('expense', 'expense_depreciation') # opex — COGS 'expense_direct_cost' excluded +INC_TYPES = ('income',) # operating revenue (the leverage denominator) + +XMR_CONST = 2.66 # individuals-chart control-limit constant (Wheeler): UCL = mean + 2.66·mR̄ +MIN_MONTHS = 6 # minimum completed months of history before an XmR flag is trustworthy +MATERIAL_LTM = 5000.0 # ignore sub-$5k/yr accounts for spike/leverage noise (still in the directory) +FIXED_CV = 0.35 # monthly coefficient-of-variation below this + broad coverage = fixed/recurring +FIXED_COVERAGE = 9 # months (of the completed window) with activity to count as recurring +BRIDGE_TOP = 5 # named category movers in the YoY bridge (rest roll into "Other"; + # kept small so the waterfall's x-axis labels stay legible) + + +def _odoo_base(): + return os.environ.get('ODOO_URL', '').rstrip('/') + + +def _short_lbl(s, n=16): + s = str(s or '') + return s if len(s) <= n else s[:n - 1] + '…' + + +# ---- data primitives: STORE-backed (OM-2 retrofit 2026-07-12) with LIVE fallback ------------- +# build() reads the tenant DuckDB store (kept minutes-fresh by the app's auto-sync) — the page's +# ~30 monthly read_groups become millisecond SQL. validate() STAYS on live Odoo, so the existing +# to-the-cent contract doubles as the retrofit's standing parity proof. Any store problem +# (missing file, cold table) falls back to the live reads — the page never breaks on a cold store. +USE_STORE = True + + +def _store_con(): + import harness.datastore as DS + return DS.ro_con() # per-thread cached read-only (shared retrofit primitive) + + +def _accounts_live(): + rows = O.search_read('account.account', + [('account_type', 'in', list(EXP_TYPES) + list(INC_TYPES))], + ['code', 'name', 'account_type']) + exp = {r['id']: {'code': r.get('code') or '', 'name': r.get('name') or ''} + for r in rows if r.get('account_type') in EXP_TYPES} + inc_ids = [r['id'] for r in rows if r.get('account_type') in INC_TYPES] + return exp, inc_ids + + +def _accounts_store(): + types = list(EXP_TYPES) + list(INC_TYPES) + rows = _store_con().execute( + 'SELECT id, code, name, account_type FROM account_account ' + 'WHERE account_type IN (' + ','.join('?' * len(types)) + ')', types).fetchall() + exp = {r[0]: {'code': r[1] or '', 'name': r[2] or ''} for r in rows if r[3] in EXP_TYPES} + inc_ids = [r[0] for r in rows if r[3] in INC_TYPES] + return exp, inc_ids + + +def _accounts(): + """{id: {code, name}} for expense-type accounts, and the list of income-account ids.""" + if USE_STORE: + try: + return _accounts_store() + except Exception: + pass + return _accounts_live() + + +def _acct_balances_live(acc_ids, date_from, date_to): + g = O.read_group('account.move.line', + [('account_id', 'in', list(acc_ids)), ('parent_state', '=', 'posted'), + ('date', '>=', date_from), ('date', '<=', date_to)], + ['balance:sum'], ['account_id'], lazy=False) + return {O.m2o_id(r['account_id']): (r.get('balance') or 0.0) + for r in g if r.get('account_id')} + + +def _acct_balances_store(acc_ids, date_from, date_to): + ids = list(acc_ids) + rows = _store_con().execute( + 'SELECT l.account_id, sum(l.balance) FROM account_move_line l ' + 'JOIN account_move m ON m.id = l.move_id ' + "WHERE m.state = 'posted' AND l.account_id IN (" + ','.join('?' * len(ids)) + ') ' + 'AND CAST(l.date AS TIMESTAMP) >= ? AND CAST(l.date AS TIMESTAMP) <= ? ' + 'GROUP BY 1', ids + [str(date_from), str(date_to)]).fetchall() + return {r[0]: (r[1] or 0.0) for r in rows} + + +def _acct_balances(acc_ids, date_from, date_to): + """{account_id: Σ balance} over posted lines in the window (debit-positive: expense accounts + read positive, income accounts read negative).""" + if not acc_ids: + return {} + if USE_STORE: + try: + return _acct_balances_store(acc_ids, date_from, date_to) + except Exception: + pass + return _acct_balances_live(acc_ids, date_from, date_to) + + +def _income_total(inc_ids, date_from, date_to): + """Operating revenue over the window (income accounts carry credit balances → negate).""" + if not inc_ids: + return 0.0 + bal = _acct_balances(inc_ids, date_from, date_to) + return -sum(bal.values()) + + +def _xmr(series): + """Wheeler individuals-chart limits over a numeric series. Returns (mean, ucl, lcl, mrbar) or + None when there isn't enough history.""" + xs = list(series) + if len(xs) < MIN_MONTHS: + return None + mean = sum(xs) / len(xs) + mrs = [abs(xs[i] - xs[i - 1]) for i in range(1, len(xs))] + mrbar = (sum(mrs) / len(mrs)) if mrs else 0.0 + return mean, mean + XMR_CONST * mrbar, mean - XMR_CONST * mrbar, mrbar + + +def _cv_coverage(series): + """(coefficient of variation, months-with-activity) over a completed monthly series.""" + xs = list(series) + n = len(xs) + cov = sum(1 for x in xs if abs(x) > 0.005) + if n < 2: + return None, cov + mean = sum(xs) / n + if mean <= 0: + return None, cov + sd = (sum((x - mean) ** 2 for x in xs) / n) ** 0.5 + return sd / mean, cov + + +def build(t=None): + """The full Expenses bundle: scorecard inputs, monthly trend + leverage, YoY bridge steps, + the category directory, the XmR spike watch list and the fixed/variable split.""" + t = t or P.today() + lf, lt = P.ltm(t) + pf, pt = P.prior_ltm(t) + exp, inc_ids = _accounts() + exp_ids = list(exp.keys()) + + # per-account LTM & prior-LTM opex, + LTM/prior revenue — four independent aggregates, parallel + ltm_by, prior_by, rev_ltm, rev_prior, inc_ltm_by = O.parallel([ + lambda: _acct_balances(exp_ids, lf, lt), + lambda: _acct_balances(exp_ids, pf, pt), + lambda: _income_total(inc_ids, lf, lt), + lambda: _income_total(inc_ids, pf, pt), + lambda: _acct_balances(inc_ids, lf, lt), + ]) + opex_ltm = sum(ltm_by.values()) + opex_prior = sum(prior_by.values()) + + # monthly matrix (24 calendar months, ascending; last row = current partial month) + months = P.month_starts(24, t) + mres = O.parallel([ + (lambda mf=mf, mt=mt: (_acct_balances(exp_ids, mf, mt), _income_total(inc_ids, mf, mt))) + for (_lbl, mf, mt) in months]) + cur_ym = f'{t.year}-{t.month:02d}' + monthly, month_acct = [], {aid: [] for aid in exp_ids} + for (lbl, _mf, _mt), (opx, rev) in zip(months, mres): + total = sum(opx.values()) + monthly.append({'month': lbl, 'opex': total, 'revenue': rev, + 'ratio': (total / rev * 100) if rev else None, + 'partial': lbl == cur_ym, 'closing': False}) + for aid in exp_ids: + month_acct[aid].append(opx.get(aid, 0.0)) + # 3-month moving average on the opex column series (for a quiet trend line) + op_series = [m['opex'] for m in monthly] + for i, m in enumerate(monthly): + w = op_series[max(0, i - 2):i + 1] + m['ma3'] = sum(w) / len(w) + + complete_idx = [i for i, (lbl, _mf, _mt) in enumerate(months) if lbl != cur_ym] + # Month-end close lags: the newest completed calendar month is often still being booked + # (vendor bills / payroll JEs / marketplace fee settlements not yet posted), which shows up as + # a near-empty month. Detect the trailing contiguous under-booked months (< CLOSE_FRAC of the + # completed-month median) and anchor month-level analysis on the last FULLY-BOOKED month, so a + # −90% "drop" that is really an open period never reads as a cost win. LTM/YoY stay rolling + # (they tie to the GL) and are captioned as-booked. + CLOSE_FRAC = 0.5 + comp_all = [monthly[i]['opex'] for i in complete_idx] + _srt = sorted(comp_all) + med = _srt[len(_srt) // 2] if _srt else 0.0 + closing_idx = set() + for i in reversed(complete_idx): + if med > 0 and monthly[i]['opex'] < CLOSE_FRAC * med: + closing_idx.add(i) + else: + break + for i in closing_idx: + monthly[i]['closing'] = True + closed_idx = [i for i in complete_idx if i not in closing_idx] + last_c = closed_idx[-1] if closed_idx else (complete_idx[-1] if complete_idx else None) + last_month = months[last_c][0] if last_c is not None else cur_ym + + # ---- category directory + fixed/variable classification ---- + rows, flags = [], [] + for aid in exp_ids: + ltm_v = ltm_by.get(aid, 0.0) + pr_v = prior_by.get(aid, 0.0) + if abs(ltm_v) < 0.005 and abs(pr_v) < 0.005: + continue # never active in either window + comp = [month_acct[aid][i] for i in closed_idx] + cv, cov = _cv_coverage(comp) + ftype = ('Fixed/recurring' + if (cov >= FIXED_COVERAGE and cv is not None and cv < FIXED_CV) + else 'Variable') + rows.append({'aid': aid, 'code': exp[aid]['code'], 'account': exp[aid]['name'], + 'ltm': ltm_v, 'prior': pr_v, 'yoy_d': ltm_v - pr_v, + 'yoy_pct': P.yoy_pct(ltm_v, pr_v), + 'pct_opex': (ltm_v / opex_ltm * 100) if opex_ltm else 0.0, + 'pct_rev': (ltm_v / rev_ltm * 100) if rev_ltm else 0.0, + 'type': ftype}) + # ---- XmR spike flag (latest completed month above the account's own UCL) ---- + stat = _xmr(comp) + if stat and last_c is not None: + mean, ucl, _lcl, mrbar = stat + latest = month_acct[aid][last_c] + if latest > ucl and mrbar > 0 and ltm_v >= MATERIAL_LTM: + flags.append({'aid': aid, 'code': exp[aid]['code'], 'account': exp[aid]['name'], + 'month': last_month, 'value': latest, 'mean': mean, 'ucl': ucl, + 'exceed': latest - ucl}) + rows.sort(key=lambda r: -r['ltm']) + flags.sort(key=lambda x: -x['exceed']) + + fixed_ltm = sum(r['ltm'] for r in rows if r['type'] == 'Fixed/recurring') + var_ltm = sum(r['ltm'] for r in rows if r['type'] == 'Variable') + + # ---- income (revenue) breakdown — the denominator's verify list ---- + inc_names = {} + for r in O.search_read('account.account', [('id', 'in', inc_ids)], ['name']): + inc_names[r['id']] = r.get('name') or '' + inc_rows = [{'account': inc_names.get(aid, ''), 'ltm': -bal} + for aid, bal in inc_ltm_by.items()] + inc_rows.sort(key=lambda r: -r['ltm']) + + # ---- YoY bridge: prior LTM → top movers → other → current LTM ---- + movers = sorted([r for r in rows if r.get('yoy_d')], key=lambda r: -abs(r['yoy_d'])) + top = movers[:BRIDGE_TOP] + steps = [{'label': 'Prior LTM', 'amount': round(opex_prior, 0), 'kind': 'total'}] + used, seen = 0.0, set() + for r in top: + d = r['yoy_d'] + used += d + lbl = _short_lbl(r['account']) # the waterfall x-axis is categorical — + if lbl in seen: # two similarly-named accounts must not collide + lbl = f"{_short_lbl(r['account'], 11)} {r['code']}" + seen.add(lbl) + steps.append({'label': lbl, 'amount': round(d, 0), + 'kind': 'down' if d > 0 else 'up'}) # cost up = red(bad); down = green(good) + other = (opex_ltm - opex_prior) - used + if abs(other) > 1: + steps.append({'label': 'Other', 'amount': round(other, 0), + 'kind': 'down' if other > 0 else 'up'}) + steps.append({'label': 'Current LTM', 'amount': round(opex_ltm, 0), 'kind': 'total'}) + + # ---- scorecard scalars ---- + opex_g = P.yoy_pct(opex_ltm, opex_prior) + rev_g = P.yoy_pct(rev_ltm, rev_prior) + opex_ratio = (opex_ltm / rev_ltm * 100) if rev_ltm else None + prior_ratio = (opex_prior / rev_prior * 100) if rev_prior else None + ratio_delta = (opex_ratio - prior_ratio) if (opex_ratio is not None and prior_ratio is not None) else None + + comp_totals = [monthly[i]['opex'] for i in closed_idx] + latest_full = comp_totals[-1] if comp_totals else 0.0 + trail = comp_totals[-13:-1] # the 12 closed months before the latest + trail_avg = (sum(trail) / len(trail)) if trail else 0.0 + latest_vs_avg = P.yoy_pct(latest_full, trail_avg) + + top5_share = (sum(r['ltm'] for r in rows[:5]) / opex_ltm * 100) if opex_ltm else 0.0 + top_cat = rows[0] if rows else None + grower = max(rows, key=lambda r: (r.get('yoy_d') or -1e18)) if rows else None + n_up_fast = sum(1 for r in rows + if r.get('yoy_pct') is not None and rev_g is not None + and r['yoy_pct'] > rev_g and r['yoy_d'] > 0 and r['ltm'] > MATERIAL_LTM) + + return { + 'window': (lf, lt), 'last_month': last_month, + 'n_closing': len(closing_idx), + 'closing_months': [months[i][0] for i in sorted(closing_idx)], + 'opex_ltm': opex_ltm, 'opex_prior': opex_prior, 'opex_g': opex_g, + 'rev_ltm': rev_ltm, 'rev_prior': rev_prior, 'rev_g': rev_g, + 'opex_ratio': opex_ratio, 'prior_ratio': prior_ratio, 'ratio_delta': ratio_delta, + 'latest_full': latest_full, 'trail_avg': trail_avg, 'latest_vs_avg': latest_vs_avg, + 'top5_share': top5_share, 'top_cat': top_cat, 'grower': grower, 'n_up_fast': n_up_fast, + 'n_accounts': len(rows), + 'monthly': monthly, 'steps': steps, 'rows': rows, 'flags': flags, 'inc_rows': inc_rows, + 'fixed_ltm': fixed_ltm, 'var_ltm': var_ltm, + } + + +def account_detail(aid, t=None): + """Raw-GL drill for one expense account: 15-month balance trend with its own control band, + LTM vs prior-LTM totals, the recent posted ledger lines behind it, and an Odoo deep-link. + Ties to the directory's numbers (same GL basis) — not the management-basis 'account' drawer.""" + t = t or P.today() + acc = O.search_read('account.account', [('id', '=', aid)], ['code', 'name', 'account_type']) + if not acc: + return None + acc = acc[0] + months = P.month_starts(15, t) + cur_ym = f'{t.year}-{t.month:02d}' + + def _m(mf, mt): + return O.sum_field('account.move.line', + [('account_id', '=', aid), ('parent_state', '=', 'posted'), + ('date', '>=', mf), ('date', '<=', mt)], 'balance') + lf, lt = P.ltm(t) + pf, pt = P.prior_ltm(t) + vals = O.parallel( + [(lambda mf=mf, mt=mt: _m(mf, mt)) for (_lbl, mf, mt) in months] + + [lambda: _m(lf, lt), lambda: _m(pf, pt)]) + mvals, ltm_v, prior_v = vals[:-2], vals[-2], vals[-1] + + stat = _xmr([v for (lbl, _mf, _mt), v in zip(months, mvals) if lbl != cur_ym]) + mean = ucl = None + if stat: + mean, ucl = stat[0], stat[1] + trend = [{'month': lbl, 'amount': v, 'partial': lbl == cur_ym, + 'over': bool(ucl is not None and v > ucl and lbl != cur_ym)} + for (lbl, _mf, _mt), v in zip(months, mvals)] + + base = _odoo_base() + jl = O.search_read('account.move.line', + [('account_id', '=', aid), ('parent_state', '=', 'posted')], + ['date', 'move_id', 'partner_id', 'name', 'balance'], + order='date desc', limit=40) + lines = [{'date': l.get('date') or '', 'entry': O.m2o_name(l.get('move_id')), + 'partner': O.m2o_name(l.get('partner_id')) or '', 'label': l.get('name') or '', + 'amount': round(l.get('balance') or 0.0, 2), + 'link': (f"{base}/web#id={O.m2o_id(l['move_id'])}&model=account.move&view_type=form" + if base and l.get('move_id') else None)} + for l in jl] + return {'aid': aid, 'code': acc.get('code') or '', 'name': acc.get('name') or '', + 'ltm': ltm_v, 'prior': prior_v, 'yoy_pct': P.yoy_pct(ltm_v, prior_v), + 'avg_month': (ltm_v / 12.0), 'mean': mean, 'ucl': ucl, + 'trend': trend, 'lines': lines, + 'odoo_link': (f"{base}/web#id={aid}&model=account.account&view_type=form" if base else None)} + + +def validate(t=None, team_id=None, pre=None): + """(1) client opex LTM ties an independent server Σ balance over the same expense accounts; + (2) client revenue LTM ties the independent server income Σ; (3) the directory rows re-add to + the opex headline (nothing dropped). Company-level — team_id ignored (opex is consolidated).""" + t = t or P.today() + b = pre or build(t) + lf, lt = b['window'] + exp, inc_ids = _accounts() + exp_ids = list(exp.keys()) + + srv_opex = O.sum_field('account.move.line', + [('account_id', 'in', exp_ids), ('parent_state', '=', 'posted'), + ('date', '>=', lf), ('date', '<=', lt)], 'balance') + srv_rev = _income_total(inc_ids, lf, lt) + dir_sum = sum(r['ltm'] for r in b['rows']) + + checks = [ + {'check': 'operating expense LTM — client Σ == server Σ balance', + 'a': round(b['opex_ltm'], 2), 'b': round(srv_opex, 2), + 'gap': round(b['opex_ltm'] - srv_opex, 2), + 'ok': abs(b['opex_ltm'] - srv_opex) <= max(1.0, abs(srv_opex) * 0.001)}, + {'check': 'operating revenue LTM — client Σ == server income Σ', + 'a': round(b['rev_ltm'], 2), 'b': round(srv_rev, 2), + 'gap': round(b['rev_ltm'] - srv_rev, 2), + 'ok': abs(b['rev_ltm'] - srv_rev) <= max(1.0, abs(srv_rev) * 0.001)}, + {'check': 'category directory Σ == opex headline (nothing dropped)', + 'a': round(dir_sum, 2), 'b': round(b['opex_ltm'], 2), + 'gap': round(dir_sum - b['opex_ltm'], 2), 'ok': abs(dir_sum - b['opex_ltm']) < 1.0}, + ] + return checks diff --git a/royalimports_os/modules/feedback.py b/royalimports_os/modules/feedback.py new file mode 100644 index 0000000000000000000000000000000000000000..9b0814710e20dc50555d42b71e391bf7fbb74327 --- /dev/null +++ b/royalimports_os/modules/feedback.py @@ -0,0 +1,68 @@ +"""Feedback module — feature requests / feedback / bug reports from every user. + +Sits OUTSIDE the Dashboards/Workflows nav groups (its own sidebar entry). Submissions accumulate +in the platform's writable HF Dataset store (core/store.py, key 'feedback') so they survive +restarts and are shared across users/instances. The record shape is deliberately structured so an +AI staging pass can later triage items into the build workflow (type/module/importance/status + +free-text details). No Odoo involvement at all — this module never touches ERP data. + +Statuses: New (just submitted) -> Staged (AI/owner queued it for a build cycle) -> Planned -> +Done | Declined. Triage is admin-only; submitting is open to every signed-in user. +""" +import time +import datetime as dt + +import core.store as store + +KEY = 'feedback' +TYPES = ['Feature request', 'Feedback', 'Bug'] +IMPORTANCE = ['Must have', 'Nice to have', 'Idea'] +STATUSES = ['New', 'Staged', 'Planned', 'Done', 'Declined'] + + +def available(): + return store.available() + + +def items(fresh=True): + """All submissions, newest first. Lenient read (display only).""" + data = store.get(KEY, fresh=fresh) or {} + out = list(data.get('items', [])) + out.sort(key=lambda r: r.get('ts', ''), reverse=True) + return out + + +def add(user, name, ftype, module, title, details, importance): + """Append one submission (strict read-modify-write — a transient read error aborts, + never clobbers). Returns the new id.""" + rec = { + 'id': f'fb-{int(time.time() * 1000)}-{user}', + 'ts': dt.datetime.now().strftime('%Y-%m-%d %H:%M'), + 'user': user, 'name': name, + 'type': ftype, 'module': module, + 'title': (title or '').strip(), + 'details': (details or '').strip(), + 'importance': importance, + 'status': 'New', 'note': '', + } + store.update(KEY, lambda d: {**d, 'items': d.get('items', []) + [rec]}) + return rec['id'] + + +def set_fields(fid, **fields): + """Update triage fields (status / note) on one submission by id.""" + def _fn(d): + for it in d.get('items', []): + if it.get('id') == fid: + it.update(fields) + return d + store.update(KEY, _fn) + + +def counts(rows=None): + """Status -> count, for the header KPIs.""" + rows = items() if rows is None else rows + out = {s: 0 for s in STATUSES} + for r in rows: + out[r.get('status', 'New')] = out.get(r.get('status', 'New'), 0) + 1 + return out diff --git a/royalimports_os/modules/financial.py b/royalimports_os/modules/financial.py new file mode 100644 index 0000000000000000000000000000000000000000..016f1401c833d9a05c938ead52582bf12396ff7a --- /dev/null +++ b/royalimports_os/modules/financial.py @@ -0,0 +1,344 @@ +"""Financial module — gross margin (the bridge between Sales revenue and Inventory cost), +margin by brand / category / SKU, margin trend & YoY, margin-dilutive SKUs, and the +cash-conversion-cycle capstone (DSO + DIO − DPO) that ties the AR + Inventory + Financial +working-capital picture together. + +Margin source: Odoo's Margin module is installed, so sale.order.line carries `margin` +(= price_subtotal − purchase_price×qty) and `purchase_price` (snapshot cost at sale). +Both are read_group-aggregatable (verified). COGS is derived as revenue − margin. + +Data-quality caveats surfaced and reported, not hidden: + - ~4% of sales-line revenue runs through zero-cost lines → their margin reads as 100% + and slightly overstates GM. Quantified in `uncosted_share()` and the validation panel. + - product.list_price is uniformly 1.0 in this Odoo (pricing lives in pricelists), so + "discount vs list" is not computable here — price realization is expressed as realized + GM% instead. (A pricelist-join version is on the backlog.) +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P +import modules.inventory as inv_mod +import modules.ar as ar_mod + + +# ---------- core margin window ---------- +def _window(date_from, date_to, team_id=None): + """Revenue, margin, derived COGS and GM% for a sale-line window (RI+FFS scope).""" + dom = O.sale_line_domain(date_from=str(date_from), date_to=str(date_to), team_id=team_id) + rev = O.sum_field('sale.order.line', dom, 'price_subtotal') + margin = O.sum_field('sale.order.line', dom, 'margin') + cogs = rev - margin + return { + 'revenue': rev, + 'margin': margin, + 'cogs': cogs, + 'gm_pct': (margin / rev * 100) if rev else 0.0, + } + + +def headline(t=None, team_id=None): + """YTD gross margin vs same-period last year (consistent with the Sales headline).""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _window(yf, yt, team_id) + last = _window(lf, lt, team_id) + lf2, lt2 = P.ltm(t) + ltm = _window(lf2, lt2, team_id) + return { + 'window': f'{yf} → {yt}', + 'revenue': this['revenue'], + 'revenue_ly': last['revenue'], + 'revenue_yoy_pct': P.yoy_pct(this['revenue'], last['revenue']), + 'cogs': this['cogs'], + 'cogs_ly': last['cogs'], + 'cogs_yoy_pct': P.yoy_pct(this['cogs'], last['cogs']), + 'gm_dollars': this['margin'], + 'gm_pct': this['gm_pct'], + 'gm_pct_ly': last['gm_pct'], + 'gm_pp_delta': this['gm_pct'] - last['gm_pct'], + 'gm_dollars_ly': last['margin'], + 'gm_dollars_yoy_pct': P.yoy_pct(this['margin'], last['margin']), + 'ltm_revenue': ltm['revenue'], + 'ltm_gm_dollars': ltm['margin'], + 'ltm_gm_pct': ltm['gm_pct'], + } + + +def by_brand(t=None): + """Fisch vs Royal: revenue, GM$, GM% (YTD) + same-period-LY for variance columns.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + rows = [] + for tid in O.TEAM_IDS: + w = _window(yf, yt, team_id=tid) + ly = _window(lf, lt, team_id=tid) + rows.append({'brand': O.TEAM_NAMES[tid], 'revenue': w['revenue'], + 'gm_dollars': w['margin'], 'gm_pct': w['gm_pct'], + 'gm_dollars_ly': ly['margin'], 'gm_change': w['margin'] - ly['margin'], + 'gm_pct_ly': ly['gm_pct'], 'gm_pp_delta': w['gm_pct'] - ly['gm_pct']}) + return sorted(rows, key=lambda r: -r['gm_dollars']) + + +def _product_cat_map(): + """product_id → main category name (reusing inventory's category resolver).""" + o = O.get_odoo() + catmap = inv_mod._cat_main_map() + prods = o.search_read('product.product', [('default_code', '!=', False)], + ['id', 'categ_id']) + return {p['id']: (catmap.get(O.m2o_id(p.get('categ_id'))) or '(uncategorized)') + for p in prods} + + +def _per_product(date_from, date_to, team_id=None): + """read_group sale lines by product → rev, margin, qty (YTD window).""" + o = O.get_odoo() + dom = O.sale_line_domain(date_from=str(date_from), date_to=str(date_to), team_id=team_id) + g = o.read_group('sale.order.line', domain=dom, + fields=['price_subtotal:sum', 'margin:sum', 'product_uom_qty:sum'], + groupby=['product_id'], lazy=False) + out = [] + for r in g: + pid = O.m2o_id(r.get('product_id')) + if not pid: + continue + out.append({'product_id': pid, 'name': O.m2o_name(r.get('product_id')), + 'revenue': r.get('price_subtotal') or 0.0, + 'margin': r.get('margin') or 0.0, + 'qty': r.get('product_uom_qty') or 0.0}) + return out + + +def by_category(t=None, limit=20, team_id=None): + """Where the gross-profit dollars come from, by main category (YTD) — with same-period-LY + so the page can show each category's GM$ change and pp drift (the mix/variance read).""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + catmap = _product_cat_map() + agg = {} + for r in _per_product(yf, yt, team_id): + cat = catmap.get(r['product_id'], '(uncategorized)') + a = agg.setdefault(cat, {'category': cat, 'revenue': 0.0, 'gm_dollars': 0.0, + 'revenue_ly': 0.0, 'gm_dollars_ly': 0.0}) + a['revenue'] += r['revenue'] + a['gm_dollars'] += r['margin'] + for r in _per_product(lf, lt, team_id): + cat = catmap.get(r['product_id'], '(uncategorized)') + a = agg.setdefault(cat, {'category': cat, 'revenue': 0.0, 'gm_dollars': 0.0, + 'revenue_ly': 0.0, 'gm_dollars_ly': 0.0}) + a['revenue_ly'] += r['revenue'] + a['gm_dollars_ly'] += r['margin'] + for a in agg.values(): + a['gm_pct'] = (a['gm_dollars'] / a['revenue'] * 100) if a['revenue'] else 0.0 + a['gm_pct_ly'] = (a['gm_dollars_ly'] / a['revenue_ly'] * 100) if a['revenue_ly'] else None + a['gm_change'] = a['gm_dollars'] - a['gm_dollars_ly'] + a['gm_pp_delta'] = (a['gm_pct'] - a['gm_pct_ly']) if a['gm_pct_ly'] is not None else None + return sorted(agg.values(), key=lambda x: -x['gm_dollars'])[:limit] + + +def _shift_years(d, years): + """Same calendar date shifted by whole years (Feb-29 → Feb-28).""" + d = dt.date.fromisoformat(str(d)) + try: + return d.replace(year=d.year + years).isoformat() + except ValueError: + return d.replace(year=d.year + years, day=28).isoformat() + + +# Bridge cutoff presets: label + window fn. The compare window is ALWAYS the same dates shifted +# back one year (for LTM that IS the prior 12 months), so seasonality never distorts the bridge. +BRIDGE_BASES = { + 'ytd': ('YTD', P.ytd), + 'qtd': ('QTD', P.qtd), + 'mtd': ('MTD', P.mtd), + 'ltm': ('LTM', P.ltm), +} + + +def margin_bridge(t=None, team_id=None, basis='ytd'): + """The gross-margin bridge — decomposes ΔGM$ (the `basis` window vs the same window one year + earlier) into per-SKU effects that sum EXACTLY (validated to the cent): + volume = (Q1−Q0)·(P0−C0) price = (P1−P0)·Q1 cost = −(C1−C0)·Q1 (SKUs in both periods) + new = GM1 of SKUs with no prior-window sales lost = −GM0 of SKUs gone this window + basis ∈ BRIDGE_BASES: 'ytd' (default), 'qtd', 'mtd', 'ltm' — the owner-selectable cutoff. + P/C are realized $-per-unit from the window aggregates (Odoo margin basis). Zero-qty lines + (services, adjustments) can't carry a unit price — their ΔGM lands in `other` so the bridge + still ties. The classic FP&A "why did margin move" chart (IBCS/PVM canon).""" + t = t or P.today() + _lbl, _win = BRIDGE_BASES.get(basis, BRIDGE_BASES['ytd']) + yf, yt = _win(t) + lf, lt = _shift_years(yf, -1), _shift_years(yt, -1) + this = {r['product_id']: r for r in _per_product(yf, yt, team_id)} + last = {r['product_id']: r for r in _per_product(lf, lt, team_id)} + vol = prc = cst = new = lost = other = 0.0 + fx = [] # per-SKU effect rows — the drill-down behind each bridge bar + for pid, r1 in this.items(): + r0 = last.get(pid) + if r0 is None: + new += r1['margin'] + fx.append({'name': r1['name'], 'bucket': 'New SKUs', 'volume': 0.0, 'price': 0.0, + 'cost': 0.0, 'change': r1['margin']}) + continue + q0, q1 = r0['qty'], r1['qty'] + if q0 > 0 and q1 > 0: + p0, c0 = r0['revenue'] / q0, (r0['revenue'] - r0['margin']) / q0 + p1, c1 = r1['revenue'] / q1, (r1['revenue'] - r1['margin']) / q1 + v, pr, ct = (q1 - q0) * (p0 - c0), (p1 - p0) * q1, -(c1 - c0) * q1 + vol += v + prc += pr + cst += ct + fx.append({'name': r1['name'], 'bucket': 'Continuing', 'volume': v, 'price': pr, + 'cost': ct, 'change': r1['margin'] - r0['margin']}) + else: # unit price undefined on either side → exact residual bucket + other += r1['margin'] - r0['margin'] + fx.append({'name': r1['name'], 'bucket': 'Other', 'volume': 0.0, 'price': 0.0, + 'cost': 0.0, 'change': r1['margin'] - r0['margin']}) + for pid, r0 in last.items(): + if pid not in this: + lost += -r0['margin'] + fx.append({'name': r0['name'], 'bucket': 'Lost SKUs', 'volume': 0.0, 'price': 0.0, + 'cost': 0.0, 'change': -r0['margin']}) + gm0 = sum(r['margin'] for r in last.values()) + gm1 = sum(r['margin'] for r in this.values()) + fx.sort(key=lambda r: -abs(r['change'])) + return {'gm_ly': gm0, 'gm_ytd': gm1, 'volume': vol, 'price': prc, 'cost': cst, + 'new': new, 'lost': lost, 'other': other, + 'delta': gm1 - gm0, + 'sku_effects': fx[:600], + 'basis': basis, 'basis_label': _lbl, + 'cmp_label': 'prior LTM' if basis == 'ltm' else 'LY', + 'date_from': str(yf), 'date_to': str(yt), 'cmp_from': str(lf), 'cmp_to': str(lt), + 'window': f'{yf} → {yt}', 'cmp_window': f'{lf} → {lt}', + 'ties': abs((vol + prc + cst + new + lost + other) - (gm1 - gm0)) <= 1.0} + + +def margin_trend(n=13, t=None, team_id=None): + """Monthly revenue, GM$ and GM% over the last n months (trend + seasonality).""" + t = t or P.today() + out = [] + for label, mf, mt in P.month_starts(n, t): + w = _window(mf, mt, team_id) + out.append({'month': label, 'revenue': w['revenue'], + 'gm_dollars': w['margin'], 'gm_pct': w['gm_pct']}) + return out + + +def low_margin_skus(t=None, limit=25, min_rev=2000.0, team_id=None): + """Margin-dilutive SKUs: meaningful YTD revenue but low realized GM%. Zero-cost + (uncosted) lines are flagged separately — their 100% GM is a data gap, not real.""" + t = t or P.today() + yf, yt = P.ytd(t) + rows = [] + for r in _per_product(yf, yt, team_id): + if r['revenue'] < min_rev: + continue + gm_pct = (r['margin'] / r['revenue'] * 100) if r['revenue'] else 0.0 + rows.append({'name': r['name'], 'revenue': r['revenue'], + 'gm_dollars': r['margin'], 'gm_pct': gm_pct, + 'uncosted': abs(r['margin'] - r['revenue']) < 0.01}) + real = [r for r in rows if not r['uncosted']] + return sorted(real, key=lambda x: x['gm_pct'])[:limit] + + +def uncosted_share(t=None, team_id=None): + """Share of YTD revenue running through zero-cost lines (margin overstated).""" + t = t or P.today() + yf, yt = P.ytd(t) + dom = O.sale_line_domain(date_from=str(yf), date_to=str(yt), team_id=team_id) + rev_all = O.sum_field('sale.order.line', dom, 'price_subtotal') + rev_zero = O.sum_field('sale.order.line', dom + [('purchase_price', '=', 0)], 'price_subtotal') + return {'rev_all': rev_all, 'rev_uncosted': rev_zero, + 'pct': (rev_zero / rev_all * 100) if rev_all else 0.0} + + +def cash_conversion_cycle(t=None): + """CCC = DIO + DSO − DPO (days). Directional, company-wide. + + Throughput basis = LTM purchases (vendor bills net of refunds), the most consistent + company-wide proxy for COGS available without a P&L-account reconciliation. DSO comes + from the AR module (revenue-based, solid). The dominant driver — inventory days — is + robust to the exact basis, which is the point. + """ + t = t or P.today() + o = O.get_odoo() + lf, lt = P.ltm(t) + base = [('state', '=', 'posted'), ('invoice_date', '>=', str(lf)), ('invoice_date', '<=', str(lt))] + purch = (O.sum_field('account.move', base + [('move_type', '=', 'in_invoice')], 'amount_total') + - O.sum_field('account.move', base + [('move_type', '=', 'in_refund')], 'amount_total')) + daily = (purch / 365.0) if purch else None + + ap_rows = o.search_read('account.move', + [('move_type', 'in', ['in_invoice', 'in_refund']), + ('state', '=', 'posted'), + ('payment_state', 'in', ['not_paid', 'partial'])], + ['amount_residual_signed']) + ap_open = abs(sum(r.get('amount_residual_signed') or 0.0 for r in ap_rows)) + + inv_value = inv_mod.summary(t)['total_inv_value'] + dso = ar_mod.summary(t)['dso_days'] + dio = (inv_value / daily) if daily else None + dpo = (ap_open / daily) if daily else None + ccc = (dio + dso - dpo) if (dio is not None and dso is not None and dpo is not None) else None + return { + 'ltm_purchases': purch, + 'inventory_value': inv_value, + 'ap_open': ap_open, + 'dio_days': dio, + 'dso_days': dso, + 'dpo_days': dpo, + 'ccc_days': ccc, + 'basis': 'LTM purchases (vendor bills net of refunds) as COGS/throughput proxy', + } + + +def validate(t=None, team_id=None): + """Reconcile margin metrics to independent Odoo aggregates. When team_id is set (a single BU + selected) every check runs SCOPED to that BU, so the validation panel never reconciles + against — or exposes — the other BU's numbers. The cross-BU brand-mirror check (#1) only + makes sense consolidated, so it runs only when team_id is None.""" + t = t or P.today() + yf, yt = P.ytd(t) + o = O.get_odoo() + checks = [] + + total = _window(yf, yt, team_id) + # 1. Σ(brand margin) == total margin (consolidated only — cross-BU) + if team_id is None: + brand_sum = sum(b['gm_dollars'] for b in by_brand(t)) + checks.append({'check': 'Margin: Σ(brand) == total (YTD)', + 'a': round(brand_sum, 2), 'b': round(total['margin'], 2), + 'gap': round(brand_sum - total['margin'], 2), + 'ok': abs(brand_sum - total['margin']) <= 1.0}) + + # 2. Σ(per-product margin) == total margin (grouping integrity) + prod_sum = sum(r['margin'] for r in _per_product(yf, yt, team_id)) + checks.append({'check': 'Margin: Σ(per-product) == total (YTD)', + 'a': round(prod_sum, 2), 'b': round(total['margin'], 2), + 'gap': round(prod_sum - total['margin'], 2), + 'ok': abs(prod_sum - total['margin']) <= 1.0}) + + # 2b. The margin bridge decomposition sums exactly to ΔGM (volume+price+cost+new+lost+other) + br = margin_bridge(t, team_id=team_id) + _parts = br['volume'] + br['price'] + br['cost'] + br['new'] + br['lost'] + br['other'] + checks.append({'check': 'Margin bridge: Σ(effects) == ΔGM YTD vs LY', + 'a': round(_parts, 2), 'b': round(br['delta'], 2), + 'gap': round(_parts - br['delta'], 2), + 'ok': abs(_parts - br['delta']) <= 1.0}) + + # 3. Odoo line-level margin identity: margin == price_subtotal − purchase_price×qty + dom = O.sale_line_domain(date_from=str(yf), date_to=str(yt), team_id=team_id) + smp = o.search_read('sale.order.line', dom, + ['price_subtotal', 'purchase_price', 'product_uom_qty', 'margin'], limit=1) + if smp: + r = smp[0] + recomputed = (r.get('price_subtotal') or 0) - (r.get('purchase_price') or 0) * (r.get('product_uom_qty') or 0) + checks.append({'check': 'Margin identity: price_subtotal − cost×qty (sample line)', + 'a': round(r.get('margin') or 0, 2), 'b': round(recomputed, 2), + 'gap': round((r.get('margin') or 0) - recomputed, 2), + 'ok': abs((r.get('margin') or 0) - recomputed) <= 0.05}) + return checks diff --git a/royalimports_os/modules/health.py b/royalimports_os/modules/health.py new file mode 100644 index 0000000000000000000000000000000000000000..ae447edb486de983034dfee6732fc0f95ac940d7 --- /dev/null +++ b/royalimports_os/modules/health.py @@ -0,0 +1,559 @@ +"""Data Health module — read-only month-end close & reconciliation checks against Odoo. + +Built for the bookkeeper / accountant workflow: pick a period, clear unposted items, +reconcile AR and AP, review accruals & cut-off (unbilled revenue, goods-received-not-invoiced), +and catch errors (below-cost sales, duplicate bills, negative stock, costing gaps). + +Each check returns the same shape — category / title / severity / scope / count / dollar +impact / a recommended fix / sample rows — so the page can group and render them uniformly. +Checks take a (date_from, date_to) window: + - scope 'period' → transactions dated inside the window (what happened this month) + - scope 'asof' → open balances / aging as of date_to (month-end position) + - scope 'current' → live snapshot, period-independent (e.g. on-hand stock) +READ-ONLY — nothing is ever written. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P +import modules.ar as ar_mod + +CATEGORIES = ['Posting & completeness', 'Accruals & cut-off', 'Receivables (AR)', + 'Payables (AP)', 'Inventory & COGS', 'Data hygiene'] + + +def _check(key, category, title, severity, scope, count, impact, impact_kind, fix, + sample=None, cols=None): + return {'key': key, 'category': category, 'title': title, 'severity': severity, + 'scope': scope, 'count': count, 'impact': impact, 'impact_kind': impact_kind, + 'fix': fix, 'sample': sample or [], 'cols': cols or {}} + + +def _days(a, b): + try: + return (dt.date.fromisoformat(str(a)[:10]) - dt.date.fromisoformat(str(b)[:10])).days + except Exception: + return 0 + + +def _ex(): + ex = O.excluded_partner_ids() + return list(ex) if ex else [] + + +def _sum(model, dom, field): + """sum_field guarded against empty result sets — Odoo's read_group returns None for a + sum over zero rows, which XML-RPC can't marshal. Return 0.0 instead of erroring.""" + o = O.get_odoo() + return O.sum_field(model, dom, field) if o.search_count(model, dom) else 0.0 + + +# ---- Posting & completeness ------------------------------------------------- +def _draft_moves(df, dt_): + o = O.get_odoo() + dom = [('state', '=', 'draft'), ('move_type', 'in', + ['out_invoice', 'in_invoice', 'out_refund', 'in_refund', 'entry']), + ('date', '>=', df), ('date', '<=', dt_)] + n = o.search_count('account.move', dom) + rows = o.search_read('account.move', dom, + ['name', 'move_type', 'partner_id', 'date', 'amount_total'], + limit=25, order='date desc') + TYPE = {'out_invoice': 'cust invoice', 'in_invoice': 'vendor bill', + 'out_refund': 'cust credit', 'in_refund': 'vendor credit', 'entry': 'journal'} + sample = [{'entry': r.get('name') or '(draft)', 'type': TYPE.get(r['move_type'], r['move_type']), + 'partner': O.m2o_name(r.get('partner_id')), 'date': str(r.get('date'))[:10], + 'amount': r.get('amount_total') or 0} for r in rows] + return _check('draft', CATEGORIES[0], 'Draft / unposted entries in period', + 'high' if n else 'low', 'period', n, None, None, + 'Post or delete these before closing — draft invoices/bills/journals are not in the books yet.', + sample, {'amount': 'money'}) + + +# ---- Accruals & cut-off ----------------------------------------------------- +def _dni_lines(df, dt_): + """Delivered-not-invoiced at LINE level (OCA account_cutoff_picking semantics, re-implemented): + per sale line, (qty_delivered − qty_invoiced) > 0 valued at the line's effective unit price. + Unlike order-level invoice_status (which counts whole orders and, under order-basis invoicing, + goods not yet shipped), this measures exactly the shipped-but-unbilled quantity. Domains can't + compare two fields, so candidates (qty_delivered > 0) are pulled and reconciled client-side.""" + o = O.get_odoo() + dom = [('order_id.state', 'in', ['sale', 'done']), ('order_id.team_id', 'in', O.TEAM_IDS), + ('qty_delivered', '>', 0), + ('order_id.date_order', '>=', f'{df} 00:00:00'), + ('order_id.date_order', '<=', f'{dt_} 23:59:59')] + if _ex(): + dom.append(('order_partner_id', 'not in', _ex())) + lines = o.search_read('sale.order.line', dom, + ['order_id', 'order_partner_id', 'qty_delivered', 'qty_invoiced', + 'product_uom_qty', 'price_unit', 'price_subtotal']) + out = [] + for r in lines: + gap = (r.get('qty_delivered') or 0) - (r.get('qty_invoiced') or 0) + if gap <= 1e-3: + continue + qty = r.get('product_uom_qty') or 0 + unit = (r['price_subtotal'] / qty) if qty else (r.get('price_unit') or 0) + r['_dni_qty'] = gap + r['_dni_val'] = gap * unit + out.append(r) + return lines, out + + +def _unbilled_revenue(df, dt_): + lines, dni = _dni_lines(df, dt_) + total = sum(r['_dni_val'] for r in dni) + by_order = {} + for r in dni: + onm = O.m2o_name(r['order_id']) + e = by_order.setdefault(onm, {'order': onm, 'customer': O.m2o_name(r['order_partner_id']), + 'unbilled': 0.0}) + e['unbilled'] += r['_dni_val'] + sample = sorted(by_order.values(), key=lambda x: -x['unbilled'])[:25] + return _check('unbilled', CATEGORIES[1], 'Delivered but not invoiced (revenue to go bill)', + 'high' if total > 25000 else 'medium', 'period', len(by_order), total, 'dollar', + 'Shipped quantity exceeds invoiced quantity on these orders — bill the gap now ' + '(or accrue it at cut-off). Line-level: partial invoices are netted correctly.', + sample, {'unbilled': 'money'}) + + +def _grni(df, dt_): + """Goods received not invoiced — accrue the unbilled vendor cost at close.""" + o = O.get_odoo() + lines = o.search_read('purchase.order.line', + [('order_id.invoice_status', '=', 'to invoice'), ('qty_received', '>', 0)], + ['qty_received', 'qty_invoiced', 'price_unit', 'product_id', 'order_id']) + by = {} + total = 0.0 + for r in lines: + d = (r.get('qty_received') or 0) - (r.get('qty_invoiced') or 0) + if d <= 0: + continue + val = d * (r.get('price_unit') or 0) + total += val + po = O.m2o_name(r.get('order_id')) + e = by.setdefault(po, {'po': po, 'accrual': 0.0}) + e['accrual'] += val + rows = sorted(by.values(), key=lambda x: -x['accrual'])[:25] + return _check('grni', CATEGORIES[1], 'Goods received, not yet invoiced (GRNI accrual)', + 'high' if total > 25000 else 'medium', 'current', len(by), total, 'dollar', + 'Accrue this vendor cost at period close (Dr inventory/COGS, Cr GRNI) until the bills arrive.', + rows, {'accrual': 'money'}) + + +# ---- Receivables (AR) ------------------------------------------------------- +def _ancient_ar(df, dt_): + o = O.get_odoo() + cutoff = (dt.date.fromisoformat(dt_) - dt.timedelta(days=365)).isoformat() + dom = [('move_type', '=', 'out_invoice'), ('state', '=', 'posted'), + ('payment_state', 'in', ['not_paid', 'partial']), ('invoice_date_due', '<', cutoff)] + if _ex(): + dom.append(('partner_id', 'not in', _ex())) + n = o.search_count('account.move', dom) + amt = _sum('account.move', dom, 'amount_residual_signed') + rows = o.search_read('account.move', dom, + ['name', 'partner_id', 'invoice_date_due', 'amount_residual_signed'], + limit=25, order='invoice_date_due asc') + sample = [{'invoice': r['name'], 'customer': O.m2o_name(r['partner_id']), + 'days_overdue': _days(dt_, r['invoice_date_due']), + 'open': r.get('amount_residual_signed') or 0} for r in rows] + return _check('ancient_ar', CATEGORIES[2], 'Invoices 365+ days overdue (write-off candidates)', + 'high' if amt > 25000 else 'medium', 'asof', n, amt, 'dollar', + 'Decide: escalate, settle, or write off. Provision the doubtful portion at close.', + sample, {'open': 'money', 'days_overdue': ('int', 'days overdue')}) + + +def _ar_recon(df, dt_): + rows = ar_mod.reconciliation_flags(limit=25) + allr = ar_mod.reconciliation_flags(limit=10 ** 9) + amt = sum(abs(r['gap']) for r in allr) + return _check('ar_recon', CATEGORIES[2], 'AR sub-ledger vs partner balance mismatches', + 'medium', 'current', len(allr), amt, 'dollar', + 'Reconcile unapplied payments/credits so the AR sub-ledger ties to the partner balance.', + rows, {'odoo_receivable': 'money', 'open_docs': 'money', 'gap': 'money'}) + + +def _stale_ar(df, dt_): + o = O.get_odoo() + cutoff = (dt.date.fromisoformat(dt_) - dt.timedelta(days=180)).isoformat() + g = o.read_group('sale.order', + [('state', 'in', ['sale', 'done']), ('team_id', 'in', O.TEAM_IDS), + ('date_order', '>=', f'{cutoff} 00:00:00'), ('date_order', '<=', f'{dt_} 23:59:59')], + ['partner_id'], ['partner_id'], lazy=False) + active = {O.m2o_id(r['partner_id']) for r in g if r.get('partner_id')} + parts = o.search_read('res.partner', [('credit', '>', 500)] + + ([('id', 'not in', _ex())] if _ex() else []), ['name', 'credit']) + stale = [p for p in parts if p['id'] not in active] + amt = sum(p['credit'] for p in stale) + sample = sorted([{'customer': p['name'], 'owes': p['credit']} for p in stale], + key=lambda x: -x['owes'])[:25] + return _check('stale_ar', CATEGORIES[2], 'Customers owing money but quiet 180+ days', + 'high' if amt > 25000 else 'medium', 'asof', len(stale), amt, 'dollar', + 'Push collection and a win-back; provision if uncollectible.', sample, {'owes': 'money'}) + + +# ---- Payables (AP) ---------------------------------------------------------- +def _ap_overdue(df, dt_): + o = O.get_odoo() + dom = [('move_type', '=', 'in_invoice'), ('state', '=', 'posted'), + ('payment_state', 'in', ['not_paid', 'partial']), ('invoice_date_due', '<', dt_)] + n = o.search_count('account.move', dom) + amt = abs(_sum('account.move', dom, 'amount_residual_signed')) + rows = o.search_read('account.move', dom, + ['name', 'partner_id', 'invoice_date_due', 'amount_residual_signed', 'ref'], + limit=25, order='invoice_date_due asc') + sample = [{'bill': r.get('ref') or r['name'], 'vendor': O.m2o_name(r['partner_id']), + 'days_overdue': _days(dt_, r['invoice_date_due']), + 'open': abs(r.get('amount_residual_signed') or 0)} for r in rows] + return _check('ap_overdue', CATEGORIES[3], 'Vendor bills overdue (as of period end)', + 'medium', 'asof', n, amt, 'dollar', + 'Schedule/clear overdue payables; confirm none are duplicates before paying.', + sample, {'open': 'money', 'days_overdue': ('int', 'days overdue')}) + + +def _duplicate_bills(df, dt_): + o = O.get_odoo() + dom = [('move_type', '=', 'in_invoice'), ('state', '=', 'posted'), + ('invoice_date', '>=', df), ('invoice_date', '<=', dt_), ('ref', '!=', False)] + rows = o.search_read('account.move', dom, ['name', 'partner_id', 'ref', 'amount_total', 'invoice_date']) + seen = {} + for r in rows: + k = (O.m2o_id(r.get('partner_id')), str(r.get('ref')).strip().lower(), round(r.get('amount_total') or 0, 2)) + seen.setdefault(k, []).append(r) + dups = [v for v in seen.values() if len(v) > 1] + sample = sorted([{'vendor': O.m2o_name(v[0]['partner_id']), 'ref': v[0]['ref'], + 'amount': v[0]['amount_total'] or 0, 'copies': len(v)} for v in dups], + key=lambda x: -x['amount'])[:25] + amt = sum((len(v) - 1) * (v[0]['amount_total'] or 0) for v in dups) + return _check('dup_bills', CATEGORIES[3], 'Possible duplicate vendor bills (same vendor/ref/amount)', + 'high' if dups else 'low', 'period', len(dups), amt, 'dollar', + 'Review before paying — duplicate bills cause double payment.', sample, {'amount': 'money'}) + + +# ---- Inventory & COGS ------------------------------------------------------- +def _negative_margin(df, dt_): + o = O.get_odoo() + dom = O.sale_line_domain(df, dt_, extra=[('margin', '<', 0)]) + n = o.search_count('sale.order.line', dom) + loss = _sum('sale.order.line', dom, 'margin') + g = o.read_group('sale.order.line', dom, ['margin:sum', 'product_id'], ['product_id'], lazy=False) + rows = sorted([{'product': O.m2o_name(r['product_id']), 'margin_lost': r.get('margin') or 0} + for r in g if r.get('product_id')], key=lambda x: x['margin_lost'])[:25] + return _check('neg_margin', CATEGORIES[4], 'Sales below cost (negative margin) in period', + 'high' if loss < -5000 else 'medium', 'period', n, loss, 'dollar', + 'Reprice or stop selling these SKUs; check for costing errors driving false losses.', + rows, {'margin_lost': 'money'}) + + +def _negative_stock(df, dt_): + o = O.get_odoo() + dom = [('location_id.usage', '=', 'internal'), ('quantity', '<', 0)] + n = o.search_count('stock.quant', dom) + rows = o.search_read('stock.quant', dom, ['product_id', 'quantity'], limit=25, order='quantity asc') + sample = [{'product': O.m2o_name(r['product_id']), 'on_hand': r['quantity']} for r in rows] + return _check('neg_stock', CATEGORIES[4], 'Negative on-hand stock (impossible quantities)', + 'high' if n > 20 else 'medium', 'current', n, None, None, + 'Fix receipts/adjustments — negative on-hand distorts inventory valuation and COGS.', + sample, {'on_hand': 'num'}) + + +def _product_master(df, dt_): + o = O.get_odoo() + prods = o.search_read('product.product', [('active', '=', True), ('default_code', '!=', False)], + ['id', 'default_code', 'name', 'standard_price', 'type', 'sale_ok', 'categ_id']) + q = o.read_group('stock.quant', [('location_id.usage', '=', 'internal')], + ['product_id', 'quantity:sum'], ['product_id'], lazy=False) + onhand = {O.m2o_id(r['product_id']): (r.get('quantity') or 0.0) for r in q if r.get('product_id')} + roots = {c['id'] for c in o.search_read('product.category', [('parent_id', '=', False)], ['id'])} + uncosted, dup, uncat = [], {}, [] + for p in prods: + oh = onhand.get(p['id'], 0.0) + if p.get('type') == 'product' and oh > 0 and (p.get('standard_price') or 0) <= 0: + uncosted.append({'sku': p['default_code'], 'product': p['name'], 'on_hand': oh}) + dup.setdefault(str(p['default_code']).strip(), []).append(p['name']) + if p.get('sale_ok') and O.m2o_id(p.get('categ_id')) in roots: + uncat.append({'sku': p['default_code'], 'product': p['name']}) + dups = [{'sku': k, 'records': len(v)} for k, v in dup.items() if len(v) > 1] + c_unc = _check('uncosted', CATEGORIES[4], 'In-stock SKUs with zero cost (valuation gap)', + 'medium', 'current', len(uncosted), None, None, + 'Set standard cost — these read as $0 inventory and distort margin and valuation.', + sorted(uncosted, key=lambda x: -x['on_hand'])[:25], {'on_hand': 'num'}) + c_dup = _check('dup_codes', CATEGORIES[5], 'Duplicate active SKU codes', 'medium', 'current', + len(dups), None, None, 'Merge/retire duplicates — shared codes double-count.', + sorted(dups, key=lambda x: -x['records'])[:25], {'records': 'int'}) + c_cat = _check('uncat', CATEGORIES[5], 'Sellable products with no real category', 'low', 'current', + len(uncat), None, None, 'Assign a category so by-category reporting works.', uncat[:25], {}) + return [c_unc, c_dup, c_cat] + + +def _orders_no_rep(df, dt_): + o = O.get_odoo() + dom = [('state', 'in', ['sale', 'done']), ('team_id', 'in', O.TEAM_IDS), ('user_id', '=', False), + ('date_order', '>=', f'{df} 00:00:00'), ('date_order', '<=', f'{dt_} 23:59:59')] + n = o.search_count('sale.order', dom) + amt = _sum('sale.order', dom, 'amount_untaxed') + rows = o.search_read('sale.order', dom, ['name', 'partner_id', 'amount_untaxed'], + limit=25, order='amount_untaxed desc') + sample = [{'order': r['name'], 'customer': O.m2o_name(r['partner_id']), 'amount': r['amount_untaxed']} + for r in rows] + return _check('no_rep', CATEGORIES[5], 'Orders with no salesperson assigned', 'low', 'period', + n, amt, 'dollar', 'Assign a salesperson for correct attribution/commissions.', + sample, {'amount': 'money'}) + + +# ---- orchestration ---------------------------------------------------------- +def _overdue_activities(df, dt_): + """Scheduled activities (mail.activity) past their deadline — at probe time ALL 811 open + activities were overdue: the activity system is dead-lettered, so nothing scheduled there + can be trusted as a reminder.""" + o = O.get_odoo() + today = P.today().isoformat() + n_open = o.search_count('mail.activity', []) + over = O.search_read('mail.activity', [('date_deadline', '<', today)], + ['user_id', 'date_deadline', 'res_model', 'summary']) + by_user = {} + for a in over: + u = O.m2o_name(a.get('user_id')) or '(unassigned)' + e = by_user.setdefault(u, {'user': u, 'overdue': 0, 'oldest': today}) + e['overdue'] += 1 + d = str(a.get('date_deadline') or today)[:10] + if d < e['oldest']: + e['oldest'] = d + sample = sorted(by_user.values(), key=lambda x: -x['overdue'])[:15] + return _check('overdue_activities', CATEGORIES[5], 'Scheduled activities past deadline', + 'medium' if len(over) < 50 else 'high', 'current', len(over), None, 'count', + f'{len(over)} of {n_open} open activities are overdue - clear or delete them; ' + 'a reminder system where everything is late reminds nobody of anything.', + sample, {'overdue': 'int'}) + + +def _partner_tag_hygiene(df, dt_): + """Partner tags duplicate the BU concept (Fisch/Royal tags vs team_id) — flag duplicate tag + names and tag-vs-team mismatches (a Royal-tagged customer on the Fisch team).""" + o = O.get_odoo() + tags = O.search_read('res.partner.category', [], ['name']) + names = {} + for t in tags: + names.setdefault((t['name'] or '').strip().upper(), []).append(t['id']) + dups = {k: v for k, v in names.items() if len(v) > 1} + issues = [{'issue': f'duplicate tag name "{k}" ({len(v)} tags)', 'count': len(v)} for k, v in dups.items()] + mism = 0 + sample_m = [] + for tag_name, team in (('FISCH', 6), ('ROYAL', 5)): # tag says one BU, team says the OTHER + ids = [i for k, v in names.items() if k == tag_name for i in v] + if ids: + rows = O.search_read('res.partner', + [('category_id', 'in', ids), ('team_id', '=', team)], + ['name'], limit=10) + n = o.search_count('res.partner', [('category_id', 'in', ids), ('team_id', '=', team)]) + mism += n + sample_m += [{'issue': f'tagged {tag_name.title()} but on the other BU team', + 'partner': r['name']} for r in rows[:5]] + total = sum(i['count'] for i in issues) + mism + return _check('tag_hygiene', CATEGORIES[5], 'Partner tag hygiene (duplicates / BU mismatch)', + 'low', 'current', total, None, 'count', + 'Merge duplicate tags; align Fisch/Royal tags with the sales team (tags feed ' + 'segmentation - a mismatch silently mis-buckets the customer).', + issues + sample_m, {}) + + +def _no_terms_invoices(df, dt_): + """Posted customer invoices with NO payment terms — due date defaults silently and dunning + logic has nothing to anchor on.""" + o = O.get_odoo() + dom = [('move_type', '=', 'out_invoice'), ('state', '=', 'posted'), + ('invoice_payment_term_id', '=', False), + ('invoice_date', '>=', f'{df}'), ('invoice_date', '<=', f'{dt_}')] + if _ex(): + dom.append(('partner_id', 'not in', _ex())) + n = o.search_count('account.move', dom) + amt = _sum('account.move', dom, 'amount_total') + rows = O.search_read('account.move', dom, ['name', 'partner_id', 'invoice_date', 'amount_total'], + limit=15, order='amount_total desc') + sample = [{'invoice': r['name'], 'customer': O.m2o_name(r['partner_id']), + 'date': str(r['invoice_date'])[:10], 'amount': r['amount_total']} for r in rows] + return _check('no_terms', CATEGORIES[2], 'Invoices posted without payment terms', + 'low' if amt < 25000 else 'medium', 'period', n, amt, 'dollar', + 'Set a default payment term on these customers - no terms means the due date ' + 'and any dunning cadence are meaningless for them.', + sample, {'amount': 'money'}) + + +# Economic-nexus (Wayfair) SALES thresholds by state, 2026 — transaction-count tests are mostly +# repealed so only the revenue test is monitored. None = no state sales tax (NH/OR/MT/DE). +# NY is $500k AND 100 sales; AK is local-option (monitored at $100k). Source: Avalara/TaxJar +# state guides — VERIFY WITH THE CPA before registering anywhere; this is a radar, not advice. +NEXUS_THRESHOLDS = { + 'AL': 250000, 'AK': 100000, 'AZ': 100000, 'AR': 100000, 'CA': 500000, 'CO': 100000, + 'CT': 100000, 'DE': None, 'FL': 100000, 'GA': 100000, 'HI': 100000, 'ID': 100000, + 'IL': 100000, 'IN': 100000, 'IA': 100000, 'KS': 100000, 'KY': 100000, 'LA': 100000, + 'ME': 100000, 'MD': 100000, 'MA': 100000, 'MI': 100000, 'MN': 100000, 'MS': 250000, + 'MO': 100000, 'MT': None, 'NE': 100000, 'NV': 100000, 'NH': None, 'NJ': 100000, + 'NM': 100000, 'NY': 500000, 'NC': 100000, 'ND': 100000, 'OH': 100000, 'OK': 100000, + 'OR': None, 'PA': 100000, 'RI': 100000, 'SC': 100000, 'SD': 100000, 'TN': 100000, + 'TX': 500000, 'UT': 100000, 'VT': 100000, 'VA': 100000, 'WA': 100000, 'WV': 100000, + 'WI': 100000, 'WY': 100000, 'DC': 100000} + + +def nexus(t=None): + """Economic-nexus radar: trailing-12m invoiced revenue (posted invoices − credit notes) by + SHIP-TO state vs each state's Wayfair threshold. The company collects ZERO sales tax (all + resale-exempt) — crossing a threshold unnoticed creates back-liability. Trailing 12m is a + PROXY (states legally measure current/previous calendar year); status: OVER / >75% warming / + monitoring. Home state carries physical nexus regardless.""" + t = t or P.today() + d12 = (t - dt.timedelta(days=365)).isoformat() + + def _by_ship(mt, sign): + out = {} + for g in O.read_group('account.move', + [('move_type', '=', mt), ('state', '=', 'posted'), + ('invoice_date', '>=', d12)], + ['amount_untaxed:sum'], ['partner_shipping_id'], lazy=False): + pid = O.m2o_id(g.get('partner_shipping_id')) + if pid: + e = out.setdefault(pid, [0.0, 0]) + e[0] += sign * (g.get('amount_untaxed') or 0.0) + e[1] += g.get('__count') or 0 + return out + inv = _by_ship('out_invoice', 1) + for pid, (v, n) in _by_ship('out_refund', -1).items(): + e = inv.setdefault(pid, [0.0, 0]) + e[0] += v # refunds reduce state revenue; their doc count isn't a 'sale' + pids = list(inv.keys()) + pstate = {} + for i in range(0, len(pids), 5000): + for p in O.search_read('res.partner', [('id', 'in', pids[i:i + 5000])], + ['state_id', 'country_id']): + code = None + if p.get('state_id'): + # state m2o name is the full name; pull the code from the state record below + code = O.m2o_id(p['state_id']) + pstate[p['id']] = {'state_rid': code, + 'country': O.m2o_name(p.get('country_id')) or ''} + srids = list({v['state_rid'] for v in pstate.values() if v['state_rid']}) + scode = {} + for i in range(0, len(srids), 5000): + for s in O.search_read('res.country.state', [('id', 'in', srids[i:i + 5000])], + ['code', 'country_id']): + scode[s['id']] = {'code': s.get('code'), 'country': O.m2o_name(s.get('country_id'))} + per, unmapped_val, unmapped_n = {}, 0.0, 0 + for pid, (val, n) in inv.items(): + ps = pstate.get(pid) or {} + sc = scode.get(ps.get('state_rid')) or {} + code, ctry = sc.get('code'), (sc.get('country') or ps.get('country') or '') + if code and ('United States' in ctry or ctry == ''): + e = per.setdefault(code, {'state': code, 'revenue_12m': 0.0, 'n_invoices': 0}) + e['revenue_12m'] += val + e['n_invoices'] += n + else: + unmapped_val += val + unmapped_n += n + rows = [] + for e in per.values(): + th = NEXUS_THRESHOLDS.get(e['state']) + e['threshold'] = th + e['pct_of_threshold'] = (e['revenue_12m'] / th * 100) if th else None + e['status'] = ('no sales tax' if th is None + else 'OVER' if e['revenue_12m'] >= th + else 'warming' if e['revenue_12m'] >= th * 0.75 + else 'monitor') + rows.append(e) + rows.sort(key=lambda x: -(x['pct_of_threshold'] or 0)) + total = sum(e['revenue_12m'] for e in per.values()) + unmapped_val + return {'rows': rows, 'unmapped_value': unmapped_val, 'unmapped_n': unmapped_n, + 'n_over': sum(1 for r in rows if r['status'] == 'OVER'), + 'n_warming': sum(1 for r in rows if r['status'] == 'warming'), + '_total_built': total, '_d12': d12} + + +def nexus_validate(nx): + """Σ(state revenue) + unmapped == server net invoiced (partition, two aggregation paths).""" + d12 = nx['_d12'] + srv = (O.sum_field('account.move', + [('move_type', '=', 'out_invoice'), ('state', '=', 'posted'), + ('invoice_date', '>=', d12)], 'amount_untaxed') + - O.sum_field('account.move', + [('move_type', '=', 'out_refund'), ('state', '=', 'posted'), + ('invoice_date', '>=', d12)], 'amount_untaxed')) + return [{'check': 'Nexus: Σ(state revenue) + unmapped == server net invoiced (12m)', + 'a': round(nx['_total_built'], 2), 'b': round(srv, 2), + 'gap': round(nx['_total_built'] - srv, 2), + 'ok': abs(nx['_total_built'] - srv) <= max(1.0, abs(srv) * 0.001)}] + + +def _window(date_from=None, date_to=None, t=None): + t = t or P.today() + if date_from and date_to: + return date_from, date_to + return P.ytd(t) # default = year to date + + +def run_all(date_from=None, date_to=None, t=None): + df, dt_ = _window(date_from, date_to, t) + checks = [_draft_moves(df, dt_), _unbilled_revenue(df, dt_), _grni(df, dt_), + _ancient_ar(df, dt_), _ar_recon(df, dt_), _stale_ar(df, dt_), + _ap_overdue(df, dt_), _duplicate_bills(df, dt_), + _negative_margin(df, dt_), _negative_stock(df, dt_), _orders_no_rep(df, dt_), + _overdue_activities(df, dt_), _partner_tag_hygiene(df, dt_), + _no_terms_invoices(df, dt_)] + checks += _product_master(df, dt_) + sev = {'high': 0, 'medium': 1, 'low': 2} + return sorted([c for c in checks if c], + key=lambda c: (sev.get(c['severity'], 9), -abs(c['impact'] or 0))) + + +def by_category(date_from=None, date_to=None, t=None): + checks = run_all(date_from, date_to, t) + return {cat: [c for c in checks if c['category'] == cat] for cat in CATEGORIES} + + +def summarize(checks): + """Build the summary KPIs from an already-computed checks list (no extra Odoo calls).""" + flagged = [c for c in checks if c['count'] > 0] + return { + 'total_checks': len(checks), + 'issues': len(flagged), + 'high': sum(1 for c in flagged if c['severity'] == 'high'), + 'dollar_at_stake': sum(abs(c['impact']) for c in flagged if c['impact_kind'] == 'dollar'), + 'clean': len(checks) - len(flagged), + } + + +def summary(date_from=None, date_to=None, t=None): + return summarize(run_all(date_from, date_to, t)) + + +def validate(date_from=None, date_to=None, t=None): + """Light, independent reconciliations (no full run_all): aggregate vs row-by-row.""" + df, dt_ = _window(date_from, date_to, t) + o = O.get_odoo() + out = [] + + # DNI: row-by-row delivered/invoiced quantity sums vs independent read_group aggregates + # over the SAME candidate domain (two aggregation paths must agree). + dom = [('order_id.state', 'in', ['sale', 'done']), ('order_id.team_id', 'in', O.TEAM_IDS), + ('qty_delivered', '>', 0), + ('order_id.date_order', '>=', f'{df} 00:00:00'), + ('order_id.date_order', '<=', f'{dt_} 23:59:59')] + if _ex(): + dom.append(('order_partner_id', 'not in', _ex())) + lines, _ = _dni_lines(df, dt_) + row_qd = sum(r.get('qty_delivered') or 0 for r in lines) + row_qi = sum(r.get('qty_invoiced') or 0 for r in lines) + agg_qd = _sum('sale.order.line', dom, 'qty_delivered') + agg_qi = _sum('sale.order.line', dom, 'qty_invoiced') + out.append({'check': 'DNI: row Σ qty_delivered == read_group Σ (candidate lines, period)', + 'a': round(row_qd, 2), 'b': round(agg_qd, 2), 'gap': round(row_qd - agg_qd, 2), + 'ok': abs(row_qd - agg_qd) <= 0.01}) + out.append({'check': 'DNI: row Σ qty_invoiced == read_group Σ (candidate lines, period)', + 'a': round(row_qi, 2), 'b': round(agg_qi, 2), 'gap': round(row_qi - agg_qi, 2), + 'ok': abs(row_qi - agg_qi) <= 0.01}) + + sdom = [('location_id.usage', '=', 'internal'), ('quantity', '<', 0)] + n_neg = o.search_count('stock.quant', sdom) + ids = o.search_read('stock.quant', sdom, ['id']) + out.append({'check': 'Negative-stock: search_count == len(search_read)', + 'a': n_neg, 'b': len(ids), 'gap': n_neg - len(ids), 'ok': n_neg == len(ids)}) + return out diff --git a/royalimports_os/modules/inventory.py b/royalimports_os/modules/inventory.py new file mode 100644 index 0000000000000000000000000000000000000000..44c7b2479aecc3198417bb9e3932e234e699d9cf --- /dev/null +++ b/royalimports_os/modules/inventory.py @@ -0,0 +1,294 @@ +"""Inventory module — working capital tied up, dead stock, stockouts blocking revenue, +ABC, slow movers, turnover. + +Valuation: on-hand qty (stock.quant, internal locations) × standard cost. (stock.quant.value +is a non-stored computed field Odoo refuses to aggregate, so standard-cost valuation is used +and is independently cross-checked.) Sales velocity from sale.order.line over LTM (RI+FFS). +""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import pandas as pd +import core.odoo as O +import core.periods as P + + +def _cat_main_map(): + cats = O.search_read('product.category', [], ['id', 'complete_name']) + out = {} + for c in cats: + parts = [x.strip() for x in (c['complete_name'] or '').split('/')] + out[c['id']] = parts[1] if len(parts) >= 2 else (parts[0] if parts else None) + return out + + +def _build(t=None): + """One master frame: product × on-hand × cost × category × LTM sales.""" + t = t or P.today() + o = O.get_odoo() + + # On-hand qty per product (internal locations only) + q = o.read_group('stock.quant', [('location_id.usage', '=', 'internal')], + ['product_id', 'quantity:sum'], ['product_id'], lazy=False) + onhand = {O.m2o_id(r['product_id']): (r.get('quantity') or 0.0) + for r in q if r.get('product_id')} + + # Products (coded). 'type' lets us separate storable goods from services + # (delivery charges etc. are services — they can't "stock out"). + fields = ['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'] + prods = o.search_read('product.product', [('default_code', '!=', False)], fields) + catmap = _cat_main_map() + + # LTM sales per product (line level, RI+FFS scope) + lf, lt = P.ltm(t) + s = o.read_group('sale.order.line', O.sale_line_domain(lf, lt), + ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], + ['product_id'], lazy=False) + rev = {O.m2o_id(r['product_id']): (r.get('price_subtotal') or 0.0) for r in s if r.get('product_id')} + qty = {O.m2o_id(r['product_id']): (r.get('product_uom_qty') or 0.0) for r in s if r.get('product_id')} + + rows = [] + for p in prods: + pid = p['id'] + oh = onhand.get(pid, 0.0) + cost = p.get('standard_price') or 0.0 + rows.append({ + 'product_id': pid, + 'sku': str(p['default_code']).strip(), + 'name': p['name'], + 'type': p.get('type') or '', + 'storable': (p.get('type') == 'product'), + 'category': catmap.get(O.m2o_id(p.get('categ_id'))) or '(uncategorized)', + 'cost': cost, + 'on_hand': oh, + 'inv_value': oh * cost, + 'rev_ltm': rev.get(pid, 0.0), + 'qty_ltm': qty.get(pid, 0.0), + }) + return pd.DataFrame(rows), (lf, lt) + + +def summary(t=None): + df, (lf, lt) = _build(t) + in_stock = df[df['on_hand'] > 0] + dead = df[(df['inv_value'] >= 1000) & (df['rev_ltm'] < 100)] + # stockouts only make sense for STORABLE goods (services like delivery never stock) + stockout = df[(df['storable']) & (df['rev_ltm'] >= 5000) & (df['on_hand'] <= 0)] + uncosted = in_stock[in_stock['cost'] <= 0] + return { + 'ltm_window': f'{lf} → {lt}', + 'total_inv_value': float(df['inv_value'].sum()), + 'total_on_hand_units': float(df['on_hand'].sum()), + 'skus_in_stock': int((df['on_hand'] > 0).sum()), + 'dead_skus': int(len(dead)), + 'dead_value': float(dead['inv_value'].sum()), + 'stockout_skus': int(len(stockout)), + 'stockout_rev_ltm': float(stockout['rev_ltm'].sum()), + 'uncosted_in_stock_skus': int(len(uncosted)), + } + + +def by_category(t=None): + df, _ = _build(t) + g = df.groupby('category').agg( + inv_value=('inv_value', 'sum'), + rev_ltm=('rev_ltm', 'sum'), + skus=('product_id', 'count'), + in_stock=('on_hand', lambda s: int((s > 0).sum())), + ).reset_index() + # crude turnover proxy: LTM revenue / on-hand value (higher = faster) + g['turnover_x'] = g.apply(lambda r: (r['rev_ltm'] / r['inv_value']) if r['inv_value'] > 0 else None, axis=1) + return g.sort_values('inv_value', ascending=False).to_dict('records') + + +def dead_stock(t=None, limit=50): + df, _ = _build(t) + d = df[(df['inv_value'] >= 1000) & (df['rev_ltm'] < 100)] + return d.sort_values('inv_value', ascending=False).head(limit)[ + ['sku', 'name', 'category', 'on_hand', 'cost', 'inv_value', 'rev_ltm']].to_dict('records') + + +def stockouts(t=None, limit=50): + df, _ = _build(t) + s = df[(df['storable']) & (df['rev_ltm'] >= 5000) & (df['on_hand'] <= 0)] + return s.sort_values('rev_ltm', ascending=False).head(limit)[ + ['sku', 'name', 'category', 'on_hand', 'rev_ltm', 'qty_ltm']].to_dict('records') + + +def abc(t=None): + """ABC by LTM revenue contribution. A=top 80%, B=next 15%, C=last 5%.""" + df, _ = _build(t) + d = df[df['rev_ltm'] > 0].sort_values('rev_ltm', ascending=False).copy() + total = d['rev_ltm'].sum() or 1.0 + d['cum'] = d['rev_ltm'].cumsum() / total + def cls(c): + return 'A' if c <= 0.80 else ('B' if c <= 0.95 else 'C') + d['abc'] = d['cum'].apply(cls) + out = d.groupby('abc').agg(skus=('product_id', 'count'), + rev_ltm=('rev_ltm', 'sum'), + inv_value=('inv_value', 'sum')).reset_index() + return out.to_dict('records') + + +# ----------------------------------------------------------- deeper analytics +_CARRY_PCT = 25.0 # default annual carrying cost as a % of inventory value (capital + storage + obsolescence) +_COVERAGE_BUCKETS = [('Critical (<15d)', 0, 15), ('Lean (15–45d)', 15, 45), + ('Healthy (45–120d)', 45, 120), ('Ample (120–270d)', 120, 270), + ('Overstock (270–540d)', 270, 540), ('Excess (>540d)', 540, 1e15)] + + +def _enrich(df, target_days): + """Add daily run-rate, days-of-supply, COGS, and excess-vs-target columns.""" + df = df.copy() + daily = df['qty_ltm'] / 365.0 + df['daily_rate'] = daily + df['cogs_ltm'] = df['qty_ltm'] * df['cost'] + + def _dos(r): + if r['daily_rate'] > 0: + return r['on_hand'] / r['daily_rate'] + return float('inf') if r['on_hand'] > 0 else 0.0 + df['dos'] = df.apply(_dos, axis=1) + df['target_units'] = daily * target_days + df['excess_units'] = (df['on_hand'] - df['target_units']).clip(lower=0) + df['excess_value'] = df['excess_units'] * df['cost'] + return df + + +def analytics(t=None, target_days=180, carry_pct=_CARRY_PCT): + """Inventory health analytics from ONE build: overstock (excess vs a target days-of-supply), the + coverage-profile distribution, turnover, average days-of-supply, and the carrying cost of excess.""" + df, (lf, lt) = _build(t) + df = _enrich(df, target_days) + instock = df[(df['storable']) & (df['on_hand'] > 0)] + total_inv = float(df['inv_value'].sum()) + total_cogs = float(df['cogs_ltm'].sum()) + + # overstock = sells, but holds materially more than the target coverage + over = df[(df['storable']) & (df['on_hand'] > 0) & (df['qty_ltm'] > 0) + & (df['dos'] > target_days) & (df['excess_value'] > 0)] + over_value = float(over['excess_value'].sum()) + overrows = [{'sku': r['sku'], 'code': r['sku'], 'product': r['name'], 'category': r['category'], + 'on_hand': float(r['on_hand']), + 'dos': (None if r['dos'] == float('inf') else round(float(r['dos']), 0)), + 'excess_units': float(r['excess_units']), 'excess_value': float(r['excess_value']), + 'carry_cost': float(r['excess_value']) * (carry_pct / 100.0), 'rev_ltm': float(r['rev_ltm'])} + for _, r in over.sort_values('excess_value', ascending=False).head(80).iterrows()] + + dead = df[(df['inv_value'] >= 1000) & (df['rev_ltm'] < 100)] + + # coverage profile (in-stock storable, by days-of-supply) + prof = [] + for nm, lo, hi in _COVERAGE_BUCKETS: + sub = instock[(instock['qty_ltm'] > 0) & (instock['dos'] >= lo) & (instock['dos'] < hi)] + prof.append({'bucket': nm, 'skus': int(len(sub)), 'inv_value': float(sub['inv_value'].sum()), + 'units': float(sub['on_hand'].sum())}) + nosale = instock[instock['qty_ltm'] <= 0] + prof.append({'bucket': 'No recent sales', 'skus': int(len(nosale)), + 'inv_value': float(nosale['inv_value'].sum()), 'units': float(nosale['on_hand'].sum())}) + + healthy_val = sum(p['inv_value'] for p in prof if p['bucket'].startswith(('Healthy', 'Ample'))) + excess_val = sum(p['inv_value'] for p in prof if p['bucket'].startswith(('Overstock', 'Excess', 'No recent'))) + return { + 'window': f'{lf} → {lt}', 'target_days': target_days, 'carry_pct': carry_pct, + 'total_inv_value': total_inv, + 'turnover_x': (total_cogs / total_inv) if total_inv else None, + 'avg_dos': (total_inv / (total_cogs / 365.0)) if total_cogs else None, + 'overstock': overrows, 'overstock_value': over_value, 'overstock_skus': int(len(over)), + 'carrying_cost': over_value * (carry_pct / 100.0), + 'coverage': prof, 'healthy_value': healthy_val, 'excess_value_total': excess_val, + 'dead_value': float(dead['inv_value'].sum()), 'dead_skus': int(len(dead)), + } + + +def _bucket(dos_raw, on_hand, qty_ltm): + """Coverage-bucket label for one SKU from its days-of-supply (matches the coverage profile).""" + if on_hand <= 0: + return 'Out of stock' + if qty_ltm <= 0: + return 'No recent sales' + for nm, lo, hi in _COVERAGE_BUCKETS: + if lo <= dos_raw < hi: + return nm + return _COVERAGE_BUCKETS[-1][0] + + +# the coverage facet, worst→best, for the SKU directory filter +COVERAGE_LABELS = ['Out of stock'] + [b[0] for b in _COVERAGE_BUCKETS] + ['No recent sales'] + + +def sku_inventory(target_days=180, carry_pct=_CARRY_PCT, t=None): + """Per-SKU-code inventory position for the SKU module — keyed by SKU code (variants merged the + same way the SKU module merges them). CONSOLIDATED: on-hand stock is one physical warehouse, not + brand-tagged, so this is brand-independent. Returns {code: {name, category, storable, on_hand, + inv_value, unit_cost, qty_ltm, dos (None=never sells through), bucket, excess_units, excess_value, + carry_cost, turns, target_units, target_days}}.""" + df, _ = _build(t) + df = _enrich(df, target_days) + g = df.groupby('sku').agg( + name=('name', 'first'), category=('category', 'first'), storable=('storable', 'max'), + on_hand=('on_hand', 'sum'), inv_value=('inv_value', 'sum'), + cogs_ltm=('cogs_ltm', 'sum'), qty_ltm=('qty_ltm', 'sum'), rev_ltm=('rev_ltm', 'sum'), + ).reset_index() + out = {} + for r in g.itertuples(index=False): + daily = r.qty_ltm / 365.0 + dos_raw = (r.on_hand / daily) if daily > 0 else (float('inf') if r.on_hand > 0 else 0.0) + target_units = daily * target_days + excess_units = max(r.on_hand - target_units, 0.0) + # code-level unit cost: on-hand-weighted where in stock, else the LTM COGS unit cost + unit_cost = (r.inv_value / r.on_hand) if r.on_hand > 0 else ( + (r.cogs_ltm / r.qty_ltm) if r.qty_ltm > 0 else 0.0) + excess_value = excess_units * unit_cost + out[r.sku] = { + 'name': r.name, 'category': r.category, 'storable': bool(r.storable), + 'on_hand': float(r.on_hand), 'inv_value': float(r.inv_value), + 'unit_cost': float(unit_cost), 'qty_ltm': float(r.qty_ltm), + 'dos': (None if dos_raw == float('inf') else round(float(dos_raw), 0)), + 'bucket': _bucket(dos_raw, r.on_hand, r.qty_ltm), + 'excess_units': float(excess_units), 'excess_value': float(excess_value), + 'carry_cost': float(excess_value) * (carry_pct / 100.0), + 'turns': (float(r.cogs_ltm) / float(r.inv_value)) if r.inv_value > 0 else None, + 'target_units': float(target_units), 'target_days': target_days, + } + return out + + +def validate(t=None): + df, _ = _build(t) + o = O.get_odoo() + checks = [] + + # 1. On-hand units: Σ(per-product) == ungrouped Odoo total + g = o.read_group('stock.quant', [('location_id.usage', '=', 'internal')], + ['quantity:sum'], [], lazy=False) + odoo_total = (g[0].get('quantity') or 0.0) if g else 0.0 + # our df only covers coded products; compare coded-product on-hand to a coded-only Odoo total + coded_ids = set(df['product_id']) + ours = float(df['on_hand'].sum()) + checks.append({ + 'check': 'On-hand units (coded SKUs) ≤ Odoo total internal qty', + 'a': round(ours, 0), 'b': round(odoo_total, 0), + 'gap': round(odoo_total - ours, 0), + 'ok': ours <= odoo_total + 1}) # coded ≤ all (uncoded/raw exist) + + # 2. Σ(category inv_value) == total inv_value + cat_sum = sum(c['inv_value'] for c in by_category(t)) + tot = float(df['inv_value'].sum()) + checks.append({ + 'check': 'Inv value: Σ(category) == total', + 'a': round(cat_sum, 2), 'b': round(tot, 2), + 'gap': round(cat_sum - tot, 2), + 'ok': abs(cat_sum - tot) <= 1.0}) + + # 3. inv_value definition holds for a sampled SKU (qty×cost) + smp = df[df['on_hand'] > 0].head(1) + if len(smp): + r = smp.iloc[0] + recomputed = r['on_hand'] * r['cost'] + checks.append({ + 'check': f"inv_value == on_hand×cost (sample {r['sku']})", + 'a': round(float(r['inv_value']), 2), 'b': round(float(recomputed), 2), + 'gap': round(float(r['inv_value'] - recomputed), 2), + 'ok': abs(r['inv_value'] - recomputed) <= 0.01}) + return checks diff --git a/royalimports_os/modules/management.py b/royalimports_os/modules/management.py new file mode 100644 index 0000000000000000000000000000000000000000..9acfe0e8f684ac834481153bdd536857b1b9ad5c --- /dev/null +++ b/royalimports_os/modules/management.py @@ -0,0 +1,224 @@ +"""modules/management.py — Management Income Statements (FFS-M / RI-M / GD-M / BU-M), +reproduced from RAW ODOO for the Sep-2024 cutover onward. + +Domain logic only (pure: data in -> plain dicts out). The heavy lifting is in +core/management_pl.py (live clean-entity pull + `-M = clean + overlay`); this module shapes it +for the page and provides validate(). See MODEL_LOGIC_MAP.md §10. + +Company-level financial reporting with its own entity + month picker (the management basis spans +Fisch / Royal / Giftware + a consolidation — not the 2-team Fisch/Royal DBA brand), so it is an HQ +module that ignores the global brand selector. +""" +import json +import os + +from core import management_pl as MP +from core import odoo as O + +ENTITIES = [('FFS', 'Fisch (FFS)'), ('RI', 'Royal (RI)'), ('GD', 'Giftware (GD)'), ('BU', 'Consolidated')] +ENTITY_LABEL = dict(ENTITIES) +_EXCL_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data', 'mgmt_exclusions.json') +_SECTION_ORDER = ['income', 'other_income', 'cogs', 'expense', 'deprec', ''] +_SECTION_LABEL = {'income': 'Income', 'other_income': 'Other income', 'cogs': 'Cost of revenue', + 'expense': 'Operating expense', 'deprec': 'Depreciation', '': ''} + + +def months_available(): + """Months we can reproduce -M for = the months present in the extracted overlay + (Sep-2024 .. last reconciled month in the model). Ascending list of (year, month).""" + out = [] + for ym in sorted(MP._overlay().keys()): + y, m = ym.split('-') + out.append((int(y), int(m))) + return out + + +def summary_kpis(year, month, entity): + s = MP.statement(year, month, entity) + rev = s['income'] + return {'income': rev, 'gross_profit': s['gross_profit'], 'net_profit': s['net_profit'], + 'gross_margin': (s['gross_profit'] / rev * 100) if rev else 0.0, + 'net_margin': (s['net_profit'] / rev * 100) if rev else 0.0} + + +def statement(year, month, entity): + """Section subtotals (income / cogs / gross / opex / deprec / net) for one entity-month.""" + return MP.statement(year, month, entity) + + +def line_rows(year, month, entity): + """Account-level rows for the statement table: {section,code,account,amount}, P&L-ordered.""" + s = MP.statement(year, month, entity) + name_by_code = {a['code']: a['name'] for a in MP.account_master().values()} + rows = [{'section': _SECTION_LABEL.get(MP._bucket(code) or '', ''), + 'code': code, 'account': name_by_code.get(code, ''), 'amount': round(amt, 2)} + for code, amt in s['lines'].items()] + order = {lbl: i for i, lbl in enumerate(_SECTION_LABEL[k] for k in _SECTION_ORDER)} + rows.sort(key=lambda r: (order.get(r['section'], 9), str(r['code']))) + return rows + + +def _exclusions(): + """{entity: [{code,name,flag}]} — GL codes excluded from each entity's analytics (from the + model's -M tabs, via build_mgmt_overlay.py). flag=True = a real misclassification to flag.""" + try: + with open(_EXCL_PATH, encoding='utf-8') as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return {} + + +def _odoo_base(): + return os.environ.get('ODOO_URL', '').rstrip('/') + + +def _move_links(gid, ana, start, end): + """Deep-links to the Odoo journal entries behind a misclassified balance, so the bookkeeper can + open each and re-tag its analytic. Read-only: these are just URLs into Odoo's own UI.""" + base = _odoo_base() + if not base: + return [] + lines = O.search_read('account.analytic.line', + [('date', '>=', start), ('date', '<=', end), + ('general_account_id', '=', gid), ('account_id', '=', ana)], ['move_line_id']) + mlids = [l['move_line_id'][0] for l in lines if l.get('move_line_id')] + moves = [] + if mlids: + mls = O.search_read('account.move.line', [('id', 'in', mlids)], ['move_id']) + moves = sorted({ml['move_id'][0] for ml in mls if ml.get('move_id')}) + return [f"{base}/web#id={mid}&model=account.move&view_type=form" for mid in moves] + + +def classification_flags(year=None, month=None): + """Health check: any *flaggable* excluded GL code with a nonzero balance in its entity's analytic + is a misclassification (e.g. an Amazon cost booked to Fisch, or a mortgage to a BU instead of HQ). + EBMS / inventory adjustments are excluded by design and never flagged. Each flag carries the + offending amount + click-to-fix Odoo journal-entry links. Defaults to the latest month.""" + if year is None or month is None: + av = months_available() + if not av: + return [] + year, month = av[-1] + excl = _exclusions() + start, end = MP.month_bounds(year, month) + code_id = {a['code']: gid for gid, a in MP.account_master().items()} + flags = [] + for entity, items in excl.items(): + ana = MP.ENTITY_ANALYTIC.get(entity) + if ana is None: + continue + for it in items: + if not it.get('flag'): + continue + gid = code_id.get(it['code']) + if not gid: + continue + bal = O.sum_field('account.analytic.line', + [('date', '>=', start), ('date', '<=', end), + ('general_account_id', '=', gid), ('account_id', '=', ana)], 'amount') + if abs(bal) > 0.005: + flags.append({'entity': entity, 'entity_label': ENTITY_LABEL.get(entity, entity), + 'code': it['code'], 'account': it['name'], 'amount': round(bal, 2), + 'moves': _move_links(gid, ana, start, end)}) + return flags + + +def _recent_months(n, anchor=None): + """The last `n` (year, month) tuples ending at `anchor` (default = latest reproducible month, + else the current month).""" + import datetime + if anchor is None: + av = months_available() + anchor = av[-1] if av else (datetime.date.today().year, datetime.date.today().month) + y, m = anchor + out = [] + for _ in range(n): + out.append((y, m)); m -= 1 + if m == 0: + y, m = y - 1, 12 + return list(reversed(out)) + + +def account_detail(code, months=13): + """Canonical financial-account view (consolidated, brand-independent): one GL account's monthly + trend split by business unit (clean analytic basis, management display sign) + recent posted + journal lines + an Odoo deep-link. Same data wherever the account is opened from. None if unknown.""" + am = MP.account_master() + gid = next((i for i, a in am.items() if a['code'] == code), None) + if gid is None: + return None + acc = am[gid] + section = _SECTION_LABEL.get(MP._bucket(code) or '', '') or (acc.get('type') or '') + yms = _recent_months(months) + ent_keys = [('FFS', 'Fisch'), ('RI', 'Royal'), ('GD', 'GD'), ('HQ', 'HQ')] + trend = [] + for (y, m) in yms: + ce = MP.clean_entities(y, m) + row = {'month': f'{y}-{m:02d}'} + tot = 0.0 + for k, _lbl in ent_keys: + v = ce.get(k, {}).get(code, 0.0) + row[k] = round(v, 2); tot += v + tot += ce.get('Internal', {}).get(code, 0.0) + row['total'] = round(tot, 2) + trend.append(row) + + base = _odoo_base() + jl = O.search_read('account.move.line', [('account_id', '=', gid), ('parent_state', '=', 'posted')], + ['date', 'move_id', 'partner_id', 'name', 'balance'], order='date desc', limit=30) + lines = [] + for l in jl: + mid = l['move_id'][0] if l.get('move_id') else None + lines.append({'date': l.get('date') or '', 'entry': O.m2o_name(l.get('move_id')), + 'partner': O.m2o_name(l.get('partner_id')) or '', 'label': l.get('name') or '', + 'amount': round(l.get('balance') or 0.0, 2), + 'link': (f"{base}/web#id={mid}&model=account.move&view_type=form" if base and mid else None)}) + return {'code': code, 'name': acc.get('name', ''), 'type': acc.get('type'), 'section': section, + 'window': f"{yms[0][0]}-{yms[0][1]:02d} -> {yms[-1][0]}-{yms[-1][1]:02d}", + 'window_total': round(sum(r['total'] for r in trend), 2), + 'odoo_link': (f"{base}/web#id={gid}&model=account.account&view_type=form" if base else None), + 'trend': trend, 'lines': lines} + + +def validate(year=None, month=None): + """Reconcile the reproduced consolidated -M to INDEPENDENT Odoo aggregates (works on the Space, + no workbook): each headline = a separate read_group sum straight from Odoo. Defaults to the + latest reproducible month. Returns [{check,a,b,gap,ok}] per the royalimports_os module contract.""" + if year is None or month is None: + avail = months_available() + if not avail: + return [{'check': 'overlay present (data/mgmt_overlay.json)', 'a': 0, 'b': 1, 'gap': -1, 'ok': False}] + year, month = avail[-1] + checks = [] + start, end = MP.month_bounds(year, month) + clean = MP.clean_entities(year, month) + am = MP.account_master() + code_id = {a['code']: gid for gid, a in am.items()} + bu_ana = [MP.ENTITY_ANALYTIC[b] for b in MP.BUS] + + for code, label in (('51000', 'Product Sales'), ('600000', 'Product Cost')): + gid = code_id.get(code) + indep = 0.0 + if gid: + g = O.read_group('account.analytic.line', + domain=[('date', '>=', start), ('date', '<=', end), + ('general_account_id', '=', gid), ('account_id', 'in', bu_ana)], + fields=['amount:sum'], groupby=[], lazy=False) + indep = abs((g[0].get('amount') or 0.0) if g else 0.0) # display positive + repro = sum(clean.get(b, {}).get(code, 0.0) for b in MP.BUS) + gap = round(repro - indep, 2) + checks.append({'check': f'{code} {label} — consolidated clean vs direct Odoo sum', + 'a': round(repro, 2), 'b': round(indep, 2), 'gap': gap, 'ok': abs(gap) < 0.01}) + + # consolidation consistency: BU net == FFS + RI + GD net + bu_net = MP.statement(year, month, 'BU')['net_profit'] + parts = sum(MP.statement(year, month, b)['net_profit'] for b in MP.BUS) + gap = round(bu_net - parts, 2) + checks.append({'check': 'BU-M net profit == FFS-M + RI-M + GD-M', + 'a': round(bu_net, 2), 'b': round(parts, 2), 'gap': gap, 'ok': abs(gap) < 0.01}) + + # classification health: no excluded GL code should post to the wrong entity's analytic + nflag = len(classification_flags(year, month)) + checks.append({'check': 'No GL misclassifications (excluded codes in wrong business unit)', + 'a': nflag, 'b': 0, 'gap': nflag, 'ok': nflag == 0}) + return checks diff --git a/royalimports_os/modules/map.py b/royalimports_os/modules/map.py new file mode 100644 index 0000000000000000000000000000000000000000..e627e16180d3bda185a6c3b0dbe38bd7c7e59049 --- /dev/null +++ b/royalimports_os/modules/map.py @@ -0,0 +1,304 @@ +"""Map module — every customer pinned at their EXACT street address (not city-aggregated). + +Coordinates resolve in this order, per customer: + 1. Odoo's own partner_latitude / partner_longitude when present; + 2. else a geocode of the street address, cached in the writable HF store (`geocoords.json`, + keyed by partner id + an address hash so a changed address re-geocodes). Geocoding uses the + free US Census batch geocoder (these customers are almost all US/NY-metro florists). +Customers with no usable address (or a geocode that fails) fall to a list under the map. + +Odoo stays STRICTLY read-only — geocodes are cached in the platform's own store, never written back +to Odoo. The map's purpose is select-and-export (e.g. to assign agents): box/lasso-select on the map +builds an Excel list. Assigning agents happens in Odoo / via that export, never as an app write. +""" +import csv +import io +import hashlib + +import core.odoo as O +import core.periods as P +import core.store as store +import modules.sales as sales_mod +import modules.customers as cust_mod + +_GEO_NS = 'geocoords' # store file: {str(pid): {lat,lon,src,h,addr}} +_BASE_FROM = '2015-01-01' # "all-time" floor for the customer base +_CENSUS_URL = 'https://geocoding.geo.census.gov/geocoder/locations/addressbatch' +# Americas bounds (US + AK/HI + PR/VI + Canada + Caribbean/Guyana) — wide enough for every real +# customer, tight enough to still reject garbage coordinates (0,0 / Europe / Asia). +_LAT_LO, _LAT_HI, _LON_LO, _LON_HI = 6.0, 60.0, -160.0, -58.0 + + +def _today_iso(): + return P.today().isoformat() + + +def _norm_state(s): + return (s or '').replace(' (US)', '').strip() + + +def _addr_parts(p): + """(street, city, state, zip) cleaned from a res.partner row.""" + street = ' '.join(x for x in [(p.get('street') or '').strip(), + (p.get('street2') or '').strip()] if x) + return (street, (p.get('city') or '').strip(), + _norm_state(O.m2o_name(p.get('state_id'))), (p.get('zip') or '').strip()) + + +def _addr_str(p): + return ', '.join(x for x in _addr_parts(p) if x) + + +def _addr_hash(p): + return hashlib.sha1('|'.join(_addr_parts(p)).lower().encode('utf-8')).hexdigest()[:16] + + +def _has_coords(p): + la, lo = p.get('partner_latitude'), p.get('partner_longitude') + return bool(la) and bool(lo) and (abs(la) > 1e-6 or abs(lo) > 1e-6) + + +def _in_bounds(lat, lon): + return lat is not None and lon is not None and _LAT_LO <= lat <= _LAT_HI and _LON_LO <= lon <= _LON_HI + + +def _base_pids(team_id=None, agent_pids=None): + """Partner ids of every customer with a confirmed order in scope (team/agent), giftware + excluded — via the same order_domain the rest of the app uses.""" + g = O.read_group('sale.order', + sales_mod.order_domain(_BASE_FROM, _today_iso(), team_id, partner_ids=agent_pids), + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + return [O.m2o_id(r.get('partner_id')) for r in g if O.m2o_id(r.get('partner_id'))] + + +def _read_partners(pids): + """One res.partner read: address + Odoo coords + agent, with agent ids resolved to names.""" + if not pids: + return [], {} + # ('active','in',[True,False]) keeps archived partners that still carry orders, so the map + # covers every order customer and reconciles to the distinct-partner count. + parts = O.search_read('res.partner', [('id', 'in', list(pids)), ('active', 'in', [True, False])], + ['name', 'street', 'street2', 'city', 'state_id', 'zip', 'country_id', + 'partner_latitude', 'partner_longitude', 'agent_ids']) + aids = {a for p in parts for a in (p.get('agent_ids') or [])} + anames = ({x['id']: x['name'] for x in + O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])} if aids else {}) + return parts, anames + + +def customer_locations(t=None, team_id=None, agent_pids=None): + """The map dataset. Returns {'located', 'unlocated', 'stats', 'validation'}. + Each located row: pid, customer, lat, lon, source, address, city, state, zip, agent, + rev_ltm, rev_total, orders, last_order. Unlocated rows carry a 'reason' instead of lat/lon.""" + t = t or P.today() + lf, lt = P.ltm(t) + today = _today_iso() + dom = sales_mod.order_domain(_BASE_FROM, today, team_id, partner_ids=agent_pids) + # Wave 1 — five independent reads concurrently (this is the slow part): all-time revenue (which + # also yields the customer set), LTM revenue, last-order dates, the independent base count, and + # the geocode cache. Running them in parallel cuts a cold load from ~15s to ~5s. + all_rev, ltm_rev, last_ord, base_indep, cache = O.parallel([ + lambda: cust_mod._cust_rev(_BASE_FROM, today, team_id, agent_pids), + lambda: cust_mod._cust_rev(lf, lt, team_id, agent_pids), + lambda: cust_mod._last_order_dates(_BASE_FROM, today, team_id, agent_pids), + lambda: O.distinct_count('sale.order', dom, 'partner_id'), + lambda: (store.get(_GEO_NS) if store.available() else {}), + ]) + # Agent-scoped map = the agent's WHOLE assigned book (a "my customers" coverage view), NOT just + # the order-history subset. Active accounts with an address that simply haven't ordered yet are + # still the agent's customers and belong on their map — so the count reconciles to the Agents-page + # book (e.g. an agent's 114 assigned customers show 114, not the 80 who happen to have ordered). + # A never-ordered customer has no BU, so this only applies in the All-BU view; a specific-BU map + # stays order-based, since BU is derived from orders (owner 2026-07-21). + book_scoped = agent_pids is not None and team_id is None + pids = list(set(all_rev.keys()) | (set(agent_pids) if book_scoped else set())) + if not pids: + return {'located': [], 'unlocated': [], + 'stats': {'total': 0, 'located': 0, 'unlocated': 0, 'no_agent': 0, 'need_geocode': 0}, + 'validation': validate_rows(0, 0, 0, base_indep)} + parts, anames = _read_partners(pids) # Wave 2 — needs the customer ids from Wave 1 + + located, unlocated = [], [] + for p in parts: + pid = p['id'] + ag = p.get('agent_ids') or [] + row = { + 'pid': pid, 'customer': p.get('name') or f'#{pid}', + 'address': _addr_str(p), + 'city': ((p.get('city') or '').strip().title()) or '(none)', + 'state': _norm_state(O.m2o_name(p.get('state_id'))) or '(none)', + 'zip': (p.get('zip') or '').strip(), + 'agent': (anames.get(ag[0]) or '(none)') if ag else '(none)', + 'rev_ltm': (ltm_rev.get(pid) or {}).get('rev', 0.0), + 'rev_total': (all_rev.get(pid) or {}).get('rev', 0.0), + 'orders': (all_rev.get(pid) or {}).get('orders', 0), + 'last_order': last_ord.get(pid, ''), + } + lat = lon = src = None + if _has_coords(p): + lat, lon, src = p['partner_latitude'], p['partner_longitude'], 'odoo' + else: + c = cache.get(str(pid)) + if c and c.get('h') == _addr_hash(p) and c.get('lat') is not None: + lat, lon, src = c['lat'], c['lon'], c.get('src', 'geocode') + if _in_bounds(lat, lon): + located.append({**row, 'lat': round(lat, 6), 'lon': round(lon, 6), 'source': src}) + else: + _street, _city, _stt, _zip = _addr_parts(p) + geocodable = bool(_street and (_city or _zip)) + if not row['address']: + reason = 'no address in Odoo' + elif (cache.get(str(pid)) or {}).get('src') == 'census_nomatch': + reason = 'address could not be located' + elif geocodable: + reason = 'not geocoded yet' + else: + reason = 'address too incomplete to locate' + unlocated.append({**row, 'reason': reason}) + + located.sort(key=lambda r: -r['rev_ltm']) + unlocated.sort(key=lambda r: -r['rev_ltm']) + stats = { + 'total': len(parts), 'located': len(located), 'unlocated': len(unlocated), + 'no_agent': sum(1 for r in (located + unlocated) if r['agent'] == '(none)'), + 'need_geocode': sum(1 for r in unlocated if r['reason'] == 'not geocoded yet'), + } + # Reconcile to the set we set out to map: the agent's full book when book-scoped, else the + # independent distinct-order-customer count. + recon = len(pids) if book_scoped else base_indep + return {'located': located, 'unlocated': unlocated, 'stats': stats, + 'validation': validate_rows(len(parts), len(located), len(unlocated), recon, + bad=[r for r in located if not _in_bounds(r['lat'], r['lon'])])} + + +def coords_for(pids): + """{pid: {'lat','lon'}} for the given partners — Odoo's own coordinates, else the geocode + CACHE (read-only reuse; NEVER triggers geocoding). Wave-7 W11: the Customer table's Map + VIEW reads these off the pool rows; the standalone Map page retired the same day, and + this module survives as the geocode library.""" + if not pids: + return {} + try: + cache = store.get(_GEO_NS) if store.available() else {} + parts = O.search_read( + 'res.partner', [('id', 'in', list(pids)), ('active', 'in', [True, False])], + ['street', 'street2', 'city', 'state_id', 'zip', + 'partner_latitude', 'partner_longitude']) + except Exception: + return {} # a table that renders without pins beats a table that cannot render + out = {} + for p in parts: + lat = lon = None + if _has_coords(p): + lat, lon = p['partner_latitude'], p['partner_longitude'] + else: + c = cache.get(str(p['id'])) + if c and c.get('h') == _addr_hash(p) and c.get('lat') is not None: + lat, lon = c['lat'], c['lon'] + if _in_bounds(lat, lon): + out[p['id']] = {'lat': round(lat, 6), 'lon': round(lon, 6)} + return out + + +def validate_rows(total, n_loc, n_unloc, base_indep, bad=None): + """Reconcile the map dataset to an independent Odoo aggregate (distinct order partners).""" + checks = [{ + 'check': 'Map: located + unlocated == customers in scope', + 'a': n_loc + n_unloc, 'b': base_indep, 'gap': (n_loc + n_unloc) - base_indep, + 'ok': (n_loc + n_unloc) == base_indep, + }, { + 'check': 'Map: every plotted point within plausible bounds', + 'a': n_loc - len(bad or []), 'b': n_loc, 'gap': -len(bad or []), 'ok': not (bad or []), + }] + return checks + + +def validate(team_id=None): + """Registry hook for validate.py.""" + return customer_locations(team_id=team_id)['validation'] + + +# ---------------------------------------------------------------- geocoding (US Census batch) +def _geocodable_pids(team_id=None): + """Every customer that can appear on a map: order-customers (per the team filter) PLUS every + customer ASSIGNED to an agent. The latter is why an agent's never-ordered accounts can be + geocoded and pinned on their 'my customers' map (customer_locations now includes the full book); + without it those addresses would never enter the geocode queue.""" + pids = set(_base_pids(team_id)) + for p in O.search_read('res.partner', [('agent_ids', '!=', False), ('active', 'in', [True, False])], + ['id'], limit=100000): + pids.add(p['id']) + return list(pids) + + +def _pending_geocode(team_id=None): + """res.partner rows with an address, no Odoo coords, and no cache entry for that exact address.""" + pids = _geocodable_pids(team_id) + parts, _ = _read_partners(pids) + cache = store.get(_GEO_NS) if store.available() else {} + pend = [] + for p in parts: + if _has_coords(p): + continue + street, city, _state, zc = _addr_parts(p) + if not (street and (city or zc)): + continue # too little to geocode → stays unlocated + c = cache.get(str(p['id'])) + if c and c.get('h') == _addr_hash(p): + continue # already attempted for this exact address + pend.append(p) + return pend, cache + + +def geocode_missing(team_id=None, limit=None, _chunk=2000): + """Geocode pending addresses via the US Census batch geocoder; cache results (incl. no-match, + so we never retry the same address) in the HF store. Returns a summary dict. Read-only on Odoo.""" + if not store.available(): + return {'error': 'No writable store (HF_TOKEN missing).'} + try: + import requests + except ImportError: + return {'error': 'requests not installed'} + pend, cache = _pending_geocode(team_id) + if limit: + pend = pend[:limit] + if not pend: + return {'attempted': 0, 'matched': 0, 'failed': 0, 'cache_size': len(cache)} + matched = failed = 0 + for i in range(0, len(pend), _chunk): + batch = pend[i:i + _chunk] + buf = io.StringIO() + w = csv.writer(buf) + for p in batch: + street, city, state, zc = _addr_parts(p) + w.writerow([p['id'], street, city, state, zc]) + try: + resp = requests.post(_CENSUS_URL, + files={'addressFile': ('a.csv', buf.getvalue(), 'text/csv')}, + data={'benchmark': 'Public_AR_Current'}, timeout=300) + resp.raise_for_status() + except Exception as e: + store.put(_GEO_NS, cache) + return {'error': f'Census request failed: {e}', 'matched': matched, 'failed': failed} + by_pid = {p['id']: p for p in batch} + for row in csv.reader(io.StringIO(resp.text)): + if not row: + continue + try: + pid = int(row[0]) + except (ValueError, IndexError): + continue + p = by_pid.get(pid) + if p is None: + continue + h = _addr_hash(p) + if len(row) >= 6 and row[2] == 'Match' and row[5]: + lon, lat = row[5].split(',') + cache[str(pid)] = {'lat': float(lat), 'lon': float(lon), 'src': 'census', + 'h': h, 'addr': row[4]} + matched += 1 + else: + cache[str(pid)] = {'lat': None, 'lon': None, 'src': 'census_nomatch', 'h': h} + failed += 1 + store.put(_GEO_NS, cache) + return {'attempted': len(pend), 'matched': matched, 'failed': failed, 'cache_size': len(cache)} diff --git a/royalimports_os/modules/o2c.py b/royalimports_os/modules/o2c.py new file mode 100644 index 0000000000000000000000000000000000000000..eb3f0cc3ab6c6d4c7925724d92ac74c139fb2430 --- /dev/null +++ b/royalimports_os/modules/o2c.py @@ -0,0 +1,210 @@ +"""Order-to-Cash timing — where the days hide, by stage, each with its owner: + + order → ship (warehouse) sale.order.date_order → effective_date (delivery complete) + ship → invoice (admin — a FREE DSO leak; links to the DNI list in Data Health) + invoice → paid (collections) invoice date → full-reconcile settlement date + (same OCA partner_time_to_pay semantics as ar.days_to_pay) + +Plus the terms-gap rollup: per payment term, contractual days vs $-weighted actual days — +"are the TERMS wrong?" — with the stranded free-credit $ it represents (reuses the validated +per-customer machinery in modules/ar.days_to_pay: dominant term, excess days, free credit). +Medians (not means) per stage — one whale order must not move the story. +""" +import datetime as dt + +import core.odoo as O +import core.periods as P +import modules.ar as ar + + +def _median(v): + v = sorted(v) + n = len(v) + if not n: + return None + return v[n // 2] if n % 2 else (v[n // 2 - 1] + v[n // 2]) / 2.0 + + +def _d(s): + s = str(s or '')[:10] + try: + return dt.date.fromisoformat(s) + except ValueError: + return None + + +def stages(team_id=None, t=None): + """Per-BU stage medians + monthly trend over the LTM. Coverage counts are honest: + each stage only scores orders where both endpoints exist.""" + t = t or P.today() + lf, lt = P.ltm(t) + ex = O.excluded_partner_ids() + + dom = [('state', 'in', ['sale', 'done']), + ('date_order', '>=', f'{lf} 00:00:00'), ('date_order', '<=', f'{lt} 23:59:59')] + dom.append(('team_id', '=', team_id) if team_id is not None + else ('team_id', 'in', O.TEAM_IDS)) + if ex: + dom.append(('partner_id', 'not in', list(ex))) + orders = O.search_read('sale.order', dom, + ['name', 'date_order', 'effective_date', 'team_id']) + by_name = {o['name']: o for o in orders} + + # invoices LTM joined by exact invoice_origin (multi-origin merges skipped — counted) + inv_dom = [('move_type', '=', 'out_invoice'), ('state', '=', 'posted'), + ('invoice_date', '>=', lf)] + if ex: + inv_dom.append(('partner_id', 'not in', list(ex))) + invs = O.search_read('account.move', inv_dom, + ['invoice_origin', 'invoice_date', 'name']) + first_inv = {} + multi_origin = 0 + for iv in invs: + og = (iv.get('invoice_origin') or '').strip() + if not og: + continue + if ',' in og: + multi_origin += 1 + continue + d = str(iv.get('invoice_date') or '')[:10] + if og in by_name and d and (og not in first_inv or d < first_inv[og]): + first_inv[og] = d + + # invoice → settled (receivable full-reconcile max counterpart date), LTM invoices + pay_dom = [('move_id.move_type', '=', 'out_invoice'), ('move_id.state', '=', 'posted'), + ('account_id.account_type', '=', 'asset_receivable'), + ('full_reconcile_id', '!=', False), ('date', '>=', lf)] + if ex: + pay_dom.append(('partner_id', 'not in', list(ex))) + rec_lines = O.search_read('account.move.line', pay_dom, + ['date', 'full_reconcile_id']) + fr_ids = list({O.m2o_id(l['full_reconcile_id']) for l in rec_lines}) + settle = {} + # raw lines + local max — a grouped read here makes the server echo the full id-list + # domain back per group (__domain) and dies with a MemoryError at 5k-id chunks + for i in range(0, len(fr_ids), 5000): + for l in O.search_read('account.move.line', + [('full_reconcile_id', 'in', fr_ids[i:i + 5000])], + ['full_reconcile_id', 'date']): + k = O.m2o_id(l.get('full_reconcile_id')) + d = str(l.get('date') or '')[:10] + if k and d: + settle[k] = max(settle.get(k, ''), d) + # survivorship guard: a recent invoice only appears here if it is ALREADY reconciled, + # so the newest cohort is all fast payers. Score only invoices ≥120d old (they've had + # time to be slow); the honest label carries into the UI. + pay_cutoff = (t - dt.timedelta(days=120)).isoformat() + pay_days = [] + for l in rec_lines: + i_d, s_d = _d(l.get('date')), _d(settle.get(O.m2o_id(l['full_reconcile_id']))) + if i_d and s_d and s_d >= i_d and i_d.isoformat() <= pay_cutoff: + pay_days.append(((s_d - i_d).days, i_d)) + + # assemble per-order stage observations + obs = [] # {bu, month, ship_days?, inv_days?} + for o in orders: + od, sd = _d(o.get('date_order')), _d(o.get('effective_date')) + bu = O.TEAM_NAMES.get(O.m2o_id(o.get('team_id')), '?') + month = str(o.get('date_order') or '')[:7] + ship = (sd - od).days if od and sd and sd >= od else None + iv = _d(first_inv.get(o['name'])) + invd = (iv - sd).days if sd and iv and iv >= sd else None + obs.append({'bu': bu, 'month': month, 'ship': ship, 'inv': invd}) + + def _roll(rows, key): + vals = [r[key] for r in rows if r.get(key) is not None] + return _median(vals), len(vals) + + by_bu = [] + for bu in sorted({r['bu'] for r in obs}): + sub = [r for r in obs if r['bu'] == bu] + m_ship, n_ship = _roll(sub, 'ship') + m_inv, n_inv = _roll(sub, 'inv') + by_bu.append({'bu': bu, 'order_to_ship': m_ship, 'n_ship': n_ship, + 'ship_to_invoice': m_inv, 'n_inv': n_inv}) + m_pay = _median([d for d, _ in pay_days]) + + trend = [] + months = sorted({r['month'] for r in obs}) + for m in months: + sub = [r for r in obs if r['month'] == m] + s, ns = _roll(sub, 'ship') + v, nv = _roll(sub, 'inv') + p_vals = [d for d, i_d in pay_days if i_d.isoformat()[:7] == m] + if ns >= 10 and s is not None: + trend.append({'month': m, 'stage': 'order-to-ship', 'days': s}) + if nv >= 10 and v is not None: + trend.append({'month': m, 'stage': 'ship-to-invoice', 'days': v}) + if len(p_vals) >= 10: + trend.append({'month': m, 'stage': 'invoice-to-paid', 'days': _median(p_vals)}) + + ship_all, n_ship_all = _roll(obs, 'ship') + inv_all, n_inv_all = _roll(obs, 'inv') + return { + 'by_bu': by_bu, 'pay_median': m_pay, 'n_pay': len(pay_days), 'trend': trend, + 'ship_median': ship_all, 'n_ship': n_ship_all, + 'inv_median': inv_all, 'n_inv': n_inv_all, + 'n_orders': len(orders), 'n_invoices': len(invs), 'multi_origin': multi_origin, + 'window': (lf, lt), + } + + +def terms_gap(t=None): + """Per payment term: contractual days vs invoiced-$-weighted actual days-to-pay and the + stranded free-credit $ — rolled up from ar.days_to_pay's validated per-customer rows + (dominant term per customer).""" + dtp = ar.days_to_pay(t, limit=100000) + rows = dtp['rows'] if isinstance(dtp, dict) else dtp + per = {} + for r in rows: + term = r.get('terms') or '(none)' + eff = r.get('avg_days_ytd') if r.get('avg_days_ytd') is not None else r.get('avg_days') + if eff is None: + continue + e = per.setdefault(term, {'term': term, 'term_days': r.get('term_days') or 0, + 'w': 0.0, 'wd': 0.0, 'customers': 0, 'free_credit': 0.0, + 'open': 0.0}) + inv_w = max(r.get('open') or 0.0, 0.0) + 1.0 # weight by open AR (+1 floor so + # zero-AR customers still count) + e['w'] += inv_w + e['wd'] += eff * inv_w + e['customers'] += 1 + e['free_credit'] += r.get('free_credit') or 0.0 + e['open'] += r.get('open') or 0.0 + out = [] + for e in per.values(): + actual = e['wd'] / e['w'] if e['w'] else None + out.append({'term': e['term'], 'term_days': e['term_days'], + 'actual_days': actual, + 'gap_days': (actual - e['term_days']) if actual is not None else None, + 'customers': e['customers'], 'open_ar': e['open'], + 'free_credit': e['free_credit']}) + out.sort(key=lambda x: -(x['free_credit'] or 0)) + return out + + +def validate(t=None, team_id=None): + """Pull-completeness both sides: LTM order count and posted-invoice count vs + independent search_count on the same domains.""" + t = t or P.today() + lf, lt = P.ltm(t) + ex = O.excluded_partner_ids() + dom = [('state', 'in', ['sale', 'done']), + ('date_order', '>=', f'{lf} 00:00:00'), ('date_order', '<=', f'{lt} 23:59:59')] + dom.append(('team_id', '=', team_id) if team_id is not None + else ('team_id', 'in', O.TEAM_IDS)) + if ex: + dom.append(('partner_id', 'not in', list(ex))) + b = stages(team_id, t) + n_orders = O.get_odoo().search_count('sale.order', dom) + inv_dom = [('move_type', '=', 'out_invoice'), ('state', '=', 'posted'), + ('invoice_date', '>=', lf)] + if ex: + inv_dom.append(('partner_id', 'not in', list(ex))) + n_inv = O.get_odoo().search_count('account.move', inv_dom) + return [ + {'check': 'LTM orders — pull complete', 'a': b['n_orders'], 'b': n_orders, + 'gap': b['n_orders'] - n_orders, 'ok': b['n_orders'] == n_orders}, + {'check': 'LTM posted invoices — pull complete', 'a': b['n_invoices'], 'b': n_inv, + 'gap': b['n_invoices'] - n_inv, 'ok': b['n_invoices'] == n_inv}, + ] diff --git a/royalimports_os/modules/outreach.py b/royalimports_os/modules/outreach.py new file mode 100644 index 0000000000000000000000000000000000000000..573e01d6a06ccca3acede7d3644c1230a84f0e3f --- /dev/null +++ b/royalimports_os/modules/outreach.py @@ -0,0 +1,451 @@ +"""Outreach — the internal campaign engine (the HubSpot-parity build, crm-outreach brief v1). + +Select a segment of customers → render a personalized template per recipient → queue one +mail.mail each through the ONE sanctioned write path (modules/collections_send.py client: +mail.mail create only, SAFE_MODE allow-list enforced in the data layer) → read outcomes back +from mail.mail state → attribute orders placed within the window to the campaign (the thing +HubSpot can't do: revenue attribution straight off sale.order). + +Store keys: + suppression {email_lower: {reason: unsub|bounce|manual, source, at}} + outreach_templates {id: {name, class: marketing|transactional, subject, body_html, ...}} + campaigns {seq, items: {id: {name, template_id, segment, audience[], excluded[], + status draft|queued, scheduled_base, send_log[], sent_at, created_by}}} + +Compliance (CAN-SPAM — applies fully to B2B): + - marketing-class sends get the physical-address + unsubscribe footer appended ALWAYS; + - the suppression list is enforced HERE (same choke point as SAFE_MODE), not in the UI; + - unsubscribe v1 = reply-to-unsubscribe (explicitly valid) + List-Unsubscribe mailto header; + - frequency governor: max 1 marketing email per dealer per 7 days across ALL campaigns. +O365 limits: 30 msgs/min → sends are staggered via mail.mail.scheduled_date in 1-min buckets +of 25; 10k recipients/day is far above the dealer book. +""" +import os +import html as _html +import datetime as dt + +from jinja2 import Environment, StrictUndefined + +import core.store as store +import core.odoo as O +import core.periods as P +import modules.customers as cust +import modules.collections_send as cs + +K_SUPP = 'suppression' +K_TPL = 'outreach_templates' +K_CAMP = 'campaigns' + +BATCH_PER_MINUTE = 25 # O365 message rate limit is 30/min — stay under it +GOVERNOR_DAYS = 7 # max 1 marketing email per dealer per 7 days +ATTRIB_DAYS = 14 # orders within N days of send count toward the campaign + +_env = Environment(undefined=StrictUndefined, autoescape=False) + + +def _now(): + return dt.datetime.now().strftime('%Y-%m-%d %H:%M') + + +# ------------------------------------------------------------------ suppression +def suppression(): + return store.get(K_SUPP) or {} + + +def is_suppressed(email): + return str(email or '').strip().lower() in suppression() + + +def suppress(email, reason, source, by='?'): + e = str(email or '').strip().lower() + if not e: + return + + def _add(d): + d[e] = {'reason': reason, 'source': source, 'by': by, 'at': _now()} + return d + store.update(K_SUPP, _add) + + +def unsuppress(email): + e = str(email or '').strip().lower() + + def _rm(d): + d.pop(e, None) + return d + store.update(K_SUPP, _rm) + + +# ------------------------------------------------------------------ templates +DEFAULT_TEMPLATES = { + 'reorder-v1': { + 'name': 'Reorder reminder', 'class': 'marketing', + 'subject': 'Time to restock, {{ customer }}?', + 'body_html': ( + '

Hi {{ customer }},

' + '

It has been {{ days_since }} days since your last order with us' + '{% if last_order %} (placed {{ last_order }}){% endif %} — based on your usual ' + 'rhythm, you may be running low.

' + '{% if top_skus %}

Your recent favorites:

    ' + '{% for s in top_skus %}
  • {{ s }}
  • {% endfor %}
{% endif %}' + '

Reply to this email or reach out to {{ agent }} and we will get your next ' + 'order moving.

' + '

Thank you for your business,
{{ company }}

'), + }, + 'winback-v1': { + 'name': 'Win-back', 'class': 'marketing', + 'subject': 'We miss you at {{ company }}, {{ customer }}', + 'body_html': ( + '

Hi {{ customer }},

' + '

We noticed your orders have slowed this year and wanted to check in — our new ' + 'catalog has landed and we would love to get you back to the table.

' + '

Reply to this email or contact {{ agent }} to hear what is new.

' + '

Thank you,
{{ company }}

'), + }, +} + + +def templates(): + t = store.get(K_TPL) or {} + if not t: + stamped = {k: {**v, 'created_by': 'system', 'at': _now()} + for k, v in DEFAULT_TEMPLATES.items()} + try: + store.update(K_TPL, lambda d: (d.update(stamped), d)[1] if not d else d) + t = store.get(K_TPL) or stamped + except Exception: + t = stamped + return t + + +def save_template(tid, name, tclass, subject, body_html, by='?'): + tid = (tid or name).strip().lower().replace(' ', '-') + + def _set(d): + d[tid] = {'name': name, 'class': tclass, 'subject': subject, + 'body_html': body_html, 'created_by': by, 'at': _now()} + return d + store.update(K_TPL, _set) + return tid + + +def render(subject_tmpl, body_tmpl, ctx): + """StrictUndefined — a missing token fails the PREVIEW, never a live send.""" + return (_env.from_string(subject_tmpl).render(**ctx), + _env.from_string(body_tmpl).render(**ctx)) + + +# ------------------------------------------------------------------ company footer (CAN-SPAM) +_company_cache = {} + + +def company_address(): + """Physical postal address for the compliance footer. Env override, else res.company.""" + if 'addr' in _company_cache: + return _company_cache['addr'] + addr = os.environ.get('COMPANY_ADDRESS', '').strip() + if not addr: + try: + rows = O.search_read('res.company', [], ['name', 'street', 'street2', 'city', + 'state_id', 'zip']) + # multi-company instance: take the first company with a real street address + c = next((r for r in rows if r.get('street')), None) + if c: + bits = [c.get('street') or '', c.get('street2') or '', c.get('city') or '', + O.m2o_name(c.get('state_id')), c.get('zip') or ''] + addr = ', '.join(b for b in bits if b) + except Exception: + addr = '' + _company_cache['addr'] = addr + return addr + + +def marketing_footer(): + addr = company_address() + return (f"
{_html.escape(cs.COMPANY)}" + + (f" · {_html.escape(addr)}" if addr else '') + + "
You are receiving this because you are a wholesale customer of " + f"{_html.escape(cs.COMPANY)}. To stop receiving these emails, reply with " + "the word unsubscribe and we will remove you within 10 business days." + "
") + + +def wrap_body(inner_html, tclass): + shell = (f"
{inner_html}" + + (marketing_footer() if tclass == 'marketing' else '') + '
') + return shell + + +# ------------------------------------------------------------------ audiences +SEGMENT_SOURCES = { + 'reorder': 'Reorder due — overdue vs their own cadence', + 'winback': 'Win-back — bought LY, down or gone this year', + 'new': 'New / reactivated this year', +} + + +def build_audience(source, team_id=None, agent=None, limit=500): + """Segment spec → candidate rows [{pid, customer, metric, agent?}] (active-list semantics: + evaluated NOW; the campaign freezes a snapshot).""" + pids_scope = cust.agent_partner_ids(agent) if agent else None + if source == 'reorder': + rows = cust.contact_recommendations(team_id=team_id, limit=limit, agent_pids=pids_scope) + return [{'pid': r['pid'], 'customer': r['customer'], 'agent': r.get('agent', ''), + 'metric': r.get('est_missed', 0.0), 'metric_label': 'est. missed $'} + for r in rows] + if source == 'winback': + rows = cust.at_risk(team_id=team_id, limit=limit, agent_pids=pids_scope) + return [{'pid': r['pid'], 'customer': r['customer'], 'agent': '', + 'metric': r.get('at_risk', 0.0), 'metric_label': 'at-risk $'} for r in rows] + if source == 'new': + rows = cust.new_customers(team_id=team_id, limit=limit, agent_pids=pids_scope) + return [{'pid': r['pid'], 'customer': r['customer'], 'agent': '', + 'metric': r.get('rev_ytd', 0.0), 'metric_label': 'YTD $'} for r in rows] + raise ValueError(f'unknown segment source {source}') + + +def _partner_info(pids): + """{pid: {email, name, credit}} in one read.""" + if not pids: + return {} + rows = O.search_read('res.partner', [('id', 'in', list(pids))], + ['name', 'email', 'credit']) + return {r['id']: r for r in rows} + + +def _tokens_bulk(pids, team_id=None): + """Per-recipient template context, pulled in TWO read_groups (never per-recipient calls): + cadence (last order / days since) + top-3 SKUs by LTM revenue.""" + cad = cust._cadence_bulk(team_id=team_id) + lf, lt = P.ltm(P.today()) # ISO strings already (periods convention) + g = O.read_group('sale.order.line', + O.sale_line_domain(lf, lt, team_id, partner_ids=list(pids)), + ['price_subtotal:sum'], ['order_partner_id', 'product_id'], lazy=False) + by_pid = {} + for r in g: + pid = O.m2o_id(r.get('order_partner_id')) + name = O.m2o_name(r.get('product_id')) + if pid and name: + by_pid.setdefault(pid, []).append((r.get('price_subtotal') or 0.0, name)) + top = {pid: [n for _, n in sorted(v, reverse=True)[:3]] for pid, v in by_pid.items()} + out = {} + for pid in pids: + c = cad.get(pid, {}) + out[pid] = {'last_order': c.get('last_order') or '', + 'days_since': int(c.get('days_since') or 0), + 'top_skus': top.get(pid, [])} + return out + + +def _agent_disp(a): + """Agent token for templates — the '(none)' placeholder must never reach a customer.""" + a = str(a or '').strip() + return a if a and a.lower() != '(none)' else 'your account manager' + + +def _recent_marketing_sends(): + """{email_lower: last marketing send date iso} across all campaigns — the frequency governor.""" + out = {} + camps = (store.get(K_CAMP) or {}).get('items', {}) + for c in camps.values(): + if c.get('template_class') != 'marketing': + continue + day = (c.get('sent_at') or '')[:10] + if not day: + continue + for s in c.get('send_log', []): + e = str(s.get('email', '')).lower() + if e and (e not in out or out[e] < day): + out[e] = day + return out + + +# ------------------------------------------------------------------ campaigns +def campaigns(): + d = store.get(K_CAMP) or {} + return d.get('items', {}) + + +def create_campaign(name, template_id, segment_spec, audience_rows, by='?'): + """Freeze the audience snapshot with exclusions applied UP FRONT (each with its reason — + the excluded rows are shown, never silently dropped): no email / suppressed / marketing + governor (emailed < GOVERNOR_DAYS ago) / duplicate email within the audience.""" + tpl = templates().get(template_id) or {} + tclass = tpl.get('class', 'marketing') + info = _partner_info([r['pid'] for r in audience_rows]) + recent = _recent_marketing_sends() if tclass == 'marketing' else {} + supp = suppression() + cutoff = (dt.date.today() - dt.timedelta(days=GOVERNOR_DAYS)).isoformat() + seen = set() + audience, excluded = [], [] + for r in audience_rows: + p = info.get(r['pid']) or {} + email = str(p.get('email') or '').strip() + el = email.lower() + if not email: + excluded.append({**r, 'reason': 'no email address'}) + elif el in supp: + excluded.append({**r, 'reason': f"suppressed ({supp[el].get('reason', '?')})"}) + elif tclass == 'marketing' and recent.get(el, '') >= cutoff: + excluded.append({**r, 'reason': f'emailed within {GOVERNOR_DAYS}d (governor)'}) + elif el in seen: + excluded.append({**r, 'reason': 'duplicate email in audience'}) + else: + seen.add(el) + audience.append({**r, 'email': email}) + created = {} + + def _add(d): + d.setdefault('seq', 0) + d.setdefault('items', {}) + d['seq'] += 1 + cid = f"c{d['seq']}" + d['items'][cid] = { + 'id': cid, 'name': name, 'template_id': template_id, 'template_class': tclass, + 'segment': segment_spec, 'audience': audience, 'excluded': excluded, + 'status': 'draft', 'send_log': [], 'sent_at': None, + 'created_by': by, 'created_at': _now(), + } + created['id'] = cid + return d + store.update(K_CAMP, _add) + return created['id'], len(audience), len(excluded) + + +def preview_samples(campaign, n=3, team_id=None): + """Render the template for the first n recipients — StrictUndefined surfaces bad tokens + here, at preview time.""" + tpl = templates().get(campaign['template_id']) or {} + aud = campaign['audience'][:n] + toks = _tokens_bulk([r['pid'] for r in aud], team_id) + out = [] + for r in aud: + ctx = {'customer': r['customer'], 'agent': _agent_disp(r.get('agent')), + 'company': cs.COMPANY, 'open_balance': 0.0, **toks.get(r['pid'], {})} + subj, body = render(tpl.get('subject', ''), tpl.get('body_html', ''), ctx) + out.append({'to': r['email'], 'subject': subj, + 'body': wrap_body(body, tpl.get('class', 'marketing'))}) + return out + + +def send_campaign(cid, base_dt_utc=None, by='?', team_id=None): + """Queue every audience mail (personalized, one mail.mail per recipient, staggered + scheduled_date ≤ BATCH_PER_MINUTE/min). SAFE_MODE is checked per recipient HERE: while the + guardrail is ON, non-allow-listed recipients are SKIPPED (logged), so a test send to + yourself works without touching real customers. Returns (queued, skipped).""" + camps = campaigns() + c = camps.get(cid) + if not c or c.get('status') != 'draft': + raise ValueError('campaign missing or already sent') + tpl = templates().get(c['template_id']) or {} + tclass = tpl.get('class', 'marketing') + base = base_dt_utc or dt.datetime.utcnow() + toks = _tokens_bulk([r['pid'] for r in c['audience']], team_id) + info = _partner_info([r['pid'] for r in c['audience']]) + odoo = cs.Odoo() + log = [] + queued = skipped = 0 + for i, r in enumerate(c['audience']): + email = r['email'] + if is_suppressed(email): # re-check at send time + log.append({**r, 'result': 'suppressed'}) + skipped += 1 + continue + if not cs.safe_recipient_ok(email): + log.append({**r, 'result': 'safe-mode-skip'}) + skipped += 1 + continue + ctx = {'customer': r['customer'], 'agent': _agent_disp(r.get('agent')), + 'company': cs.COMPANY, + 'open_balance': float((info.get(r['pid']) or {}).get('credit') or 0.0), + **toks.get(r['pid'], {})} + try: + subj, body = render(tpl.get('subject', ''), tpl.get('body_html', ''), ctx) + when = base + dt.timedelta(minutes=queued // BATCH_PER_MINUTE) + payload = { + 'subject': subj, 'body_html': wrap_body(body, tclass), + 'email_to': email, + 'email_from': cs.SENDER_DISPLAY, 'reply_to': cs.REPLY_TO, + 'mail_server_id': cs.ROYAL_MAIL_SERVER_ID, 'author_id': cs.ROYAL_AUTHOR_ID, + 'state': 'outgoing', 'auto_delete': False, + 'scheduled_date': when.strftime('%Y-%m-%d %H:%M:%S'), + 'model': 'res.partner', 'res_id': r['pid'], # audit trail in chatter + 'headers': repr({'List-Unsubscribe': + f''}), + } + mid = odoo.queue_mail(payload) + log.append({**r, 'result': 'queued', 'mail_id': mid, + 'scheduled': when.strftime('%H:%M')}) + queued += 1 + except Exception as e: + log.append({**r, 'result': f'error: {str(e)[:90]}'}) + skipped += 1 + + def _fin(d): + item = d.get('items', {}).get(cid) + if item: + item['status'] = 'queued' + item['send_log'] = log + item['sent_at'] = _now() + item['sent_by'] = by + return d + store.update(K_CAMP, _fin) + return queued, skipped + + +def refresh_outcomes(cid): + """Read mail.mail state back for a campaign's queued mails (sent / exception + reason).""" + c = campaigns().get(cid) or {} + ids = [s['mail_id'] for s in c.get('send_log', []) if s.get('mail_id')] + if not ids: + return {} + rows = O.search_read('mail.mail', [('id', 'in', ids)], + ['state', 'failure_reason']) + by_id = {r['id']: r for r in rows} + + def _upd(d): + item = d.get('items', {}).get(cid) + if item: + for s in item.get('send_log', []): + r = by_id.get(s.get('mail_id')) + if r: + s['state'] = r.get('state') + fr = r.get('failure_reason') + if fr: + s['failure'] = str(fr)[:120] + return d + store.update(K_CAMP, _upd) + return by_id + + +def attribution(cid, days=ATTRIB_DAYS): + """Orders placed by audience customers within `days` of send → per-customer rows + totals. + Correlation window, honestly labeled — not causal ML.""" + c = campaigns().get(cid) or {} + sent = (c.get('sent_at') or '')[:10] + if not sent: + return {'rows': [], 'orders': 0, 'revenue': 0.0, 'window': days, 'from': None, 'to': None} + pids = [r['pid'] for r in c.get('audience', [])] + if not pids: + return {'rows': [], 'orders': 0, 'revenue': 0.0, 'window': days, 'from': sent, 'to': None} + end = (dt.date.fromisoformat(sent) + dt.timedelta(days=days)).isoformat() + g = O.read_group('sale.order', + [('partner_id', 'in', pids), ('state', 'in', ['sale', 'done']), + ('date_order', '>=', f'{sent} 00:00:00'), + ('date_order', '<=', f'{end} 23:59:59')], + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + names = {r['pid']: r['customer'] for r in c.get('audience', [])} + rows = [] + for r in g: + pid = O.m2o_id(r.get('partner_id')) + if not pid: + continue + rows.append({'pid': pid, 'customer': names.get(pid) or O.m2o_name(r.get('partner_id')), + 'orders': r.get('__count') or 0, + 'revenue': r.get('amount_untaxed') or 0.0}) + rows.sort(key=lambda x: -x['revenue']) + return {'rows': rows, 'orders': sum(x['orders'] for x in rows), + 'revenue': sum(x['revenue'] for x in rows), 'window': days, + 'from': sent, 'to': end} diff --git a/royalimports_os/modules/overstock.py b/royalimports_os/modules/overstock.py new file mode 100644 index 0000000000000000000000000000000000000000..f62543af508f7c99dd4847f26e424085101dda38 --- /dev/null +++ b/royalimports_os/modules/overstock.py @@ -0,0 +1,265 @@ +"""Overstock & dead-stock module (lives under Inventory). + +A deeper, book-value-accurate lens on trapped inventory capital than the standard-cost coverage +view in inventory.py. Three things: + + 1. DEEP per-SKU list — every in-stock SKU with its BOOK value (cumulative stock.valuation.layer, + which ties to GL account 05000), days-of-inventory (DSI = on-hand / last-3mo annualised + ALL-CHANNEL sales), a dead/excess/slow/healthy status, last-sold month and trailing units. + 2. SUMMARY — trapped capital = dead + excess; share of inventory; bucket distribution. + 3. ROLLING COHORT RECOVERY — each month-end classifies its dead cohort, then tracks it forward; + a SKU "graduates" only when DSI < 120 days. Quantifies how little dead stock self-clears. + +CONSOLIDATED (HQ): on-hand stock is one physical warehouse, not brand-tagged — ignores the DBA +filter, like the rest of Inventory. ALL-CHANNEL sales (no team/partner scope) are used on purpose: +a SKU that sells only through the Amazon/GIFTWARE channel is NOT dead. + +READ-ONLY. +""" +import sys +import calendar +import datetime +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O + +DEAD_UNITS_12M = 5 # < this many units sold (all channels) in trailing 12mo = dead +GRADUATE_DSI = 120 # days-of-inventory below this = healthy turnover ("graduated") +EXCESS_DSI = 365 # sells, but holds > 1 year of supply = excess/overstock +_VAL_FLOOR = 50.0 # ignore sub-$50 book scraps as noise + + +# --------------------------------------------------------------------------- date helpers +def _today(): + return datetime.date.today() + + +def _eom(y, m): + return datetime.date(y, m, calendar.monthrange(y, m)[1]) + + +def _seq(y0, m0, y1, m1): + out, y, m = [], y0, m0 + while (y, m) <= (y1, m1): + out.append((y, m)); y, m = (y + (m // 12)), (m % 12 + 1) + return out + + +# --------------------------------------------------------------------------- raw pulls +def _monthly_sales(yms): + """All-channel confirmed sales UNITS per product per (y,m). Parallelised — one read_group/month.""" + def _one(ym): + y, m = ym + f = f"{y}-{m:02d}-01 00:00:00"; t = f"{y}-{m:02d}-{calendar.monthrange(y, m)[1]} 23:59:59" + g = O.read_group('sale.order.line', + [('order_id.state', 'in', ['sale', 'done']), ('order_id.date_order', '>=', f), + ('order_id.date_order', '<=', t), ('product_id.type', '!=', 'service')], + ['product_uom_qty:sum', 'product_id'], ['product_id'], lazy=False) + return {O.m2o_id(r['product_id']): (r.get('product_uom_qty') or 0.0) for r in g if r.get('product_id')} + res = O.parallel([(lambda ym=ym: _one(ym)) for ym in yms]) + return dict(zip(yms, res)) + + +def _onhand(asof): + """Cumulative on-hand (qty, book value) per product from stock.valuation.layer as of `asof` (a + 'YYYY-MM-DD' date). Book value ties to GL 05000 (validated).""" + g = O.read_group('stock.valuation.layer', [('create_date', '<=', f'{asof} 23:59:59')], + ['value:sum', 'quantity:sum', 'product_id'], ['product_id'], lazy=False) + return {O.m2o_id(r['product_id']): ((r.get('quantity') or 0.0), (r.get('value') or 0.0)) + for r in g if r.get('product_id')} + + +def _cat_main(catid_map, categ): + cid = O.m2o_id(categ) + return catid_map.get(cid) or '(uncategorized)' + + +def _master(pids): + """Product code/name/category for a set of ids (incl. archived — re-SKUed items).""" + if not pids: + return {}, {} + prods = O.search_read('product.product', [('id', 'in', list(pids)), ('active', 'in', [True, False])], + ['default_code', 'name', 'categ_id']) + cats = O.search_read('product.category', [], ['id', 'complete_name']) + catmap = {} + for c in cats: + parts = [x.strip() for x in (c['complete_name'] or '').split('/')] + catmap[c['id']] = parts[1] if len(parts) >= 2 else (parts[0] if parts else None) + pm = {p['id']: {'sku': (p.get('default_code') or f"#{p['id']}"), 'name': p.get('name') or '', + 'category': _cat_main(catmap, p.get('categ_id'))} for p in prods} + return pm, catmap + + +# --------------------------------------------------------------------------- classification +def _dsi(on_hand, units_3m): + if units_3m > 0: + return on_hand / (units_3m / 91.25) + return None # no recent sales -> effectively infinite cover + + +def _status(units_12m, dsi): + if units_12m < DEAD_UNITS_12M: + return 'Dead (no sales 12mo)' + if dsi is None or dsi > EXCESS_DSI: + return 'Excess (>365d cover)' + if dsi > GRADUATE_DSI: + return 'Slow (120-365d)' + return 'Healthy (<120d)' + + +def deep(asof=None): + """The deep snapshot: per-SKU rows + summary KPIs + bucket distribution, as of `asof` (default today).""" + asof = asof or _today().isoformat() + y, m = int(asof[:4]), int(asof[5:7]) + yms = _seq(*(lambda d: (d.year, d.month))(_eom(y, m).replace(day=1) - datetime.timedelta(days=365)), + y, m) # 13 months incl. current + sales = _monthly_sales(yms) + onhand = _onhand(asof) + pids = [pid for pid, (q, v) in onhand.items() if q > 0.5 and v > 0] + pm, _ = _master(pids) + + def trailing(pid, n): + return sum(sales.get(ym, {}).get(pid, 0.0) for ym in yms[-n:]) + + def last_sold(pid): + for ym in reversed(yms): + if sales.get(ym, {}).get(pid, 0.0) > 0: + return f"{calendar.month_abbr[ym[1]]} {ym[0]}" + return '12+ mo ago' + + rows = [] + for pid in pids: + q, v = onhand[pid] + u12, u3 = trailing(pid, 12), trailing(pid, 3) + dsi = _dsi(q, u3) + meta = pm.get(pid, {}) + rows.append({ + 'product_id': pid, 'sku': meta.get('sku', f'#{pid}'), 'code': meta.get('sku', f'#{pid}'), + 'product': meta.get('name', ''), 'name': meta.get('name', ''), + 'category': meta.get('category', '(uncategorized)'), + 'on_hand': float(q), 'book_value': float(v), 'unit_cost': float(v / q) if q else 0.0, + 'units_12mo': float(u12), 'units_3mo': float(u3), + 'dsi': (None if dsi is None else round(dsi, 0)), 'last_sold': last_sold(pid), + 'status': _status(u12, dsi), + }) + rows.sort(key=lambda r: -r['book_value']) + + total = sum(r['book_value'] for r in rows) + def grp(pred): + sub = [r for r in rows if pred(r)] + return len(sub), sum(r['book_value'] for r in sub) + dn, dv = grp(lambda r: r['status'].startswith('Dead')) + en, ev = grp(lambda r: r['status'].startswith('Excess')) + sn, sv = grp(lambda r: r['status'].startswith('Slow')) + hn, hv = grp(lambda r: r['status'].startswith('Healthy')) + buckets = [{'status': lbl, 'skus': n, 'book_value': val, + 'share': (val / total * 100 if total else 0)} + for lbl, (n, val) in [('Dead (no sales 12mo)', (dn, dv)), ('Excess (>365d cover)', (en, ev)), + ('Slow (120-365d)', (sn, sv)), ('Healthy (<120d)', (hn, hv))]] + summary = { + 'asof': asof, 'window': f'{yms[0][0]}-{yms[0][1]:02d} → {yms[-1][0]}-{yms[-1][1]:02d}', + 'total_book': total, + 'trapped_value': dv + ev, 'trapped_skus': dn + en, 'trapped_share': ((dv + ev) / total * 100 if total else 0), + 'dead_value': dv, 'dead_skus': dn, 'excess_value': ev, 'excess_skus': en, + 'slow_value': sv, 'slow_skus': sn, 'healthy_value': hv, 'healthy_skus': hn, + } + return {'rows': rows, 'summary': summary, 'buckets': buckets} + + +# --------------------------------------------------------------------------- rolling cohort recovery +def cohort_recovery(asof=None): + """For each month-end (Jan-2025 → last full month) classify the dead cohort, then track it forward: + a SKU graduates when DSI < 120. Returns the average recovery curve + per-cohort summary.""" + asof = asof or _today() + if isinstance(asof, str): + asof = datetime.date.fromisoformat(asof) + last = asof.replace(day=1) - datetime.timedelta(days=1) # last complete month + cohorts = _seq(2025, 1, last.year, last.month) + smon = _seq(2024, 1, last.year, last.month) # trailing-12 needs a year of runway + sales = _monthly_sales(smon) + + def trail(pid, ym, n): + i = smon.index(ym) + return sum(sales.get(smon[j], {}).get(pid, 0.0) for j in range(max(0, i - n + 1), i + 1)) + + # on-hand (qty, value) at each cohort month-end, in parallel + ends = [_eom(y, m).isoformat() for (y, m) in cohorts] + states = dict(zip(cohorts, O.parallel([(lambda e=e: _onhand(e)) for e in ends]))) + + # per (cohort-month, pid): value, dead?, dsi + flat = {} + for ym in cohorts: + d = {} + for pid, (q, v) in states[ym].items(): + if q <= 0.5 or v <= 0: + continue + dsi = _dsi(q, trail(pid, ym, 3)) + d[pid] = (v, (v > _VAL_FLOOR and trail(pid, ym, 12) < DEAD_UNITS_12M), dsi) + flat[ym] = d + + idx = {c: i for i, c in enumerate(cohorts)} + def graduated_by(pid, c, k): + for t in cohorts[idx[c]:idx[c] + k + 1]: + st = flat[t].get(pid) + if st and st[2] is not None and st[2] < GRADUATE_DSI: + return True + return False + + curve_num, curve_den = {}, {} + rows = [] + for c in cohorts: + cv = {pid: flat[c][pid][0] for pid in flat[c] if flat[c][pid][1]} # dead pids -> entry value + tot = sum(cv.values()) + if not tot: + continue + maxk = len(cohorts) - 1 - idx[c] + for k in range(maxk + 1): + rec = sum(v for pid, v in cv.items() if graduated_by(pid, c, k)) + curve_num[k] = curve_num.get(k, 0.0) + rec + curve_den[k] = curve_den.get(k, 0.0) + tot + + def pct_at(k): + return (sum(v for pid, v in cv.items() if graduated_by(pid, c, k)) / tot * 100) if k <= maxk else None + stuck = sum(v for pid, v in cv.items() if not graduated_by(pid, c, maxk)) + rows.append({'cohort': f"{c[0]}-{c[1]:02d}", 'skus': len(cv), 'dead_value': tot, + 'rec_3mo': pct_at(3), 'rec_6mo': pct_at(6), + 'rec_today': (tot - stuck) / tot * 100, 'stuck_value': stuck}) + + curve = [{'months_since': k, 'recovered_pct': (curve_num[k] / curve_den[k] * 100 if curve_den[k] else 0)} + for k in sorted(curve_num)] + cd = {p['months_since']: p['recovered_pct'] for p in curve} + return {'curve': curve, 'cohorts': rows, + 'rec_3mo': cd.get(3), 'rec_6mo': cd.get(6), 'rec_12mo': cd.get(12), + 'latest_stuck': (rows[-1]['stuck_value'] if rows else 0.0)} + + +# --------------------------------------------------------------------------- validate +def validate(asof=None): + """Reconcile to independent Odoo aggregates.""" + asof = asof or _today().isoformat() + d = deep(asof) + checks = [] + + # 1. layer book value (in-stock SKUs) ties to GL account 05000 balance (the audited inventory line) + acc = O.search_read('account.account', [('code', '=', '05000')], ['id']) + if acc: + gl = O.sum_field('account.move.line', [('parent_state', '=', 'posted'), + ('account_id', '=', acc[0]['id']), ('date', '<=', asof)], 'balance') + ours = d['summary']['total_book'] + checks.append({'check': 'On-hand book value ≈ GL 05000 Inventory', + 'a': round(ours, 0), 'b': round(gl, 0), 'gap': round(ours - gl, 0), + 'ok': abs(ours - gl) <= max(2500.0, abs(gl) * 0.05)}) + + # 2. buckets partition the total book value exactly + bsum = sum(b['book_value'] for b in d['buckets']) + tot = d['summary']['total_book'] + checks.append({'check': 'Σ(status buckets) == total book value', 'a': round(bsum, 2), + 'b': round(tot, 2), 'gap': round(bsum - tot, 2), 'ok': abs(bsum - tot) <= 1.0}) + + # 3. trapped = dead + excess (definition holds) + s = d['summary'] + checks.append({'check': 'Trapped == dead + excess value', 'a': round(s['trapped_value'], 2), + 'b': round(s['dead_value'] + s['excess_value'], 2), + 'gap': round(s['trapped_value'] - s['dead_value'] - s['excess_value'], 2), + 'ok': abs(s['trapped_value'] - s['dead_value'] - s['excess_value']) <= 1.0}) + return checks diff --git a/royalimports_os/modules/pricecomp.py b/royalimports_os/modules/pricecomp.py new file mode 100644 index 0000000000000000000000000000000000000000..72b93b8a8486fd2ed7c4d7321cedd017c295610a --- /dev/null +++ b/royalimports_os/modules/pricecomp.py @@ -0,0 +1,174 @@ +"""Price compliance — selling below the customer's pricelist tier (pocket-price floor). + +Discount FIELDS are unused in this Odoo (verified dead), but reps can still key a low +price_unit directly — leakage that is invisible today. Per (customer × SKU) over the LTM: +realized unit price (Σ subtotal ÷ Σ qty — discount-true) vs the customer's applicable +pricelist tier (fixed-price rules, qty break matched on the average order quantity). +Exceptions below THRESHOLD of tier form the worklist, each valued at its annualized leak +(= (tier − realized) × LTM qty). Pricing canon: ~1% realization ≈ 8-9% operating profit. + +v1 simplifications (documented, deliberate): + - qty break matched on AVG order qty per (customer, SKU) — not per order line; + - fixed-price rules only (this instance prices via fixed tiers, like the legacy SKU app); + - LTM window, wholesale scope. +""" +import datetime as dt + +import core.odoo as O +import core.periods as P +import modules.customers as cust + +THRESHOLD = 0.97 # below 97% of tier = exception + + +def _rules(): + """All date-valid fixed-price pricelist rules, indexed by (pricelist, product-variant) + and (pricelist, product-template). Highest min_quantity ≤ qty wins at match time.""" + today = P.today().isoformat() + rows = O.search_read( + 'product.pricelist.item', + [('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product'])], + ['pricelist_id', 'product_id', 'product_tmpl_id', 'min_quantity', + 'fixed_price', 'date_start', 'date_end', 'applied_on']) + ok = [] + for r in rows: + ds = str(r.get('date_start') or '')[:10] + de = str(r.get('date_end') or '')[:10] + if (ds and ds > today) or (de and de < today): + continue + ok.append(r) + by_var, by_tmpl = {}, {} + for r in ok: + pl = O.m2o_id(r['pricelist_id']) + if r['applied_on'] == '0_product_variant' and r.get('product_id'): + by_var.setdefault((pl, O.m2o_id(r['product_id'])), []).append(r) + elif r.get('product_tmpl_id'): + by_tmpl.setdefault((pl, O.m2o_id(r['product_tmpl_id'])), []).append(r) + return ok, by_var, by_tmpl + + +def _tier_for(pl, pid, tmpl, qty, by_var, by_tmpl): + """The applicable fixed tier price: variant rules beat template rules; within a scope, + the highest qty break ≤ qty wins.""" + for cands in (by_var.get((pl, pid)), by_tmpl.get((pl, tmpl))): + if not cands: + continue + eligible = [r for r in cands if (r.get('min_quantity') or 0) <= max(qty, 1)] + if eligible: + best = max(eligible, key=lambda r: r.get('min_quantity') or 0) + return best.get('fixed_price') or None + return None + + +def matched_pairs(team_id=None, t=None): + """The shared substrate: every (customer × SKU) pair bought in the LTM with its applicable + fixed tier attached (tier=None when the customer has no pricelist / no rule matches). + Used by the compliance sweep here AND by the price corridor + pocket waterfall + (modules/reprice.py) — one definition of 'realized vs tier' everywhere.""" + t = t or P.today() + lf, lt = P.ltm(t) + + # realized price per (customer, SKU) — one server-side aggregate + g = O.read_group('sale.order.line', O.sale_line_domain(lf, lt, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum'], + ['order_partner_id', 'product_id'], lazy=False) + pairs = [] + for r in g: + pid = O.m2o_id(r.get('order_partner_id')) + prod = O.m2o_id(r.get('product_id')) + qty = r.get('product_uom_qty') or 0.0 + rev = r.get('price_subtotal') or 0.0 + n = r.get('__count') or 1 + if pid and prod and qty > 0 and rev > 0: + pairs.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')), + 'prod': prod, 'product': O.m2o_name(r.get('product_id')), + 'qty': qty, 'rev': rev, 'lines': n, + 'unit': rev / qty, 'avg_order_qty': qty / n}) + + # customer pricelists + product template/code lookups + pids = list({p['pid'] for p in pairs}) + plists = {} + for i in range(0, len(pids), 2000): + for r in O.search_read('res.partner', [('id', 'in', pids[i:i + 2000])], + ['property_product_pricelist']): + plists[r['id']] = O.m2o_id(r.get('property_product_pricelist')) + prods = list({p['prod'] for p in pairs}) + pinfo = {} + for i in range(0, len(prods), 2000): + for r in O.search_read('product.product', + [('id', 'in', prods[i:i + 2000]), + ('active', 'in', [True, False])], + ['product_tmpl_id', 'default_code']): + pinfo[r['id']] = {'tmpl': O.m2o_id(r.get('product_tmpl_id')), + 'code': (r.get('default_code') or '').strip()} + + all_rules, by_var, by_tmpl = _rules() + for p in pairs: + info = pinfo.get(p['prod'], {}) + p['sku'] = info.get('code', '') + pl = plists.get(p['pid']) + tier = (_tier_for(pl, p['prod'], info.get('tmpl'), p['avg_order_qty'], + by_var, by_tmpl) if pl else None) + p['tier'] = tier if (tier and tier > 0) else None + return {'pairs': pairs, 'checked': len(pairs), 'n_rules': len(all_rules), + 'window': (lf, lt)} + + +def build(team_id=None, t=None, pre=None): + """The compliance sweep. Returns dict(rows[exceptions], checked, matched, kpis…).""" + mp = pre or matched_pairs(team_id, t) + matched_list = [p for p in mp['pairs'] if p.get('tier')] + exceptions = [p for p in matched_list if p['unit'] / p['tier'] < THRESHOLD] + attrs = cust._partner_attrs(list({p['pid'] for p in exceptions})) + rows = [] + for p in exceptions: + tier = p['tier'] + rows.append({ + 'pid': p['pid'], 'customer': p['customer'], + 'sku': p['sku'], 'product': p['product'], + 'agent': (attrs.get(p['pid']) or {}).get('agent', '(none)'), + 'tier_price': tier, 'realized': p['unit'], + 'pct_of_tier': p['unit'] / tier * 100.0, + 'qty_ltm': p['qty'], 'rev_ltm': p['rev'], + 'leak': (tier - p['unit']) * p['qty'], 'lines': p['lines'], + }) + rows.sort(key=lambda x: -x['leak']) + return { + 'rows': rows, 'checked': mp['checked'], 'matched': len(matched_list), + 'n_rules': mp['n_rules'], + 'leak_total': sum(r['leak'] for r in rows), + 'exception_rate': (len(rows) / len(matched_list) * 100.0) if matched_list else 0.0, + 'window': mp['window'], + } + + +def validate(t=None, team_id=None): + """(1) rule pull complete vs search_count; (2) the TOP exception re-derived row-level: + its realized unit price recomputed from its raw order lines (independent path) must match + the read_group aggregate the sweep used.""" + b = build(team_id, t) + n = O.get_odoo().search_count( + 'product.pricelist.item', + [('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product'])]) + checks = [{'check': 'fixed pricelist rules — pull complete (pre date-filter)', + 'a': b['n_rules'], 'b': f'<= {n}', 'gap': '', + 'ok': b['n_rules'] <= n}] + if b['rows']: + top = b['rows'][0] + lf, lt = b['window'] + lines = O.search_read('sale.order.line', + O.sale_line_domain(lf, lt, team_id, + partner_ids=[top['pid']], + extra=[('product_id.default_code', '=', + top['sku'])]), + ['price_subtotal', 'product_uom_qty']) + qty = sum(l.get('product_uom_qty') or 0 for l in lines) + rev = sum(l.get('price_subtotal') or 0 for l in lines) + unit = rev / qty if qty else 0.0 + checks.append({'check': f"top exception realized price — {top['sku']}", + 'a': round(unit, 4), 'b': round(top['realized'], 4), + 'gap': round(unit - top['realized'], 4), + 'ok': abs(unit - top['realized']) < 0.01}) + return checks diff --git a/royalimports_os/modules/pricing.py b/royalimports_os/modules/pricing.py new file mode 100644 index 0000000000000000000000000000000000000000..c49f49988cc7f329afb253879efcc941eef8812f --- /dev/null +++ b/royalimports_os/modules/pricing.py @@ -0,0 +1,395 @@ +"""Pricing module — per-SKU economics for pricing decisions. + +For every SKU (LTM): units, revenue, unit cost, avg selling price, GROSS margin $/%, markup % +(= revenue/COGS − 1), and a fully-loaded NET margin after allocating operating expenses to the SKU. + +COST-TO-SKU ALLOCATION (tiered, channel-scoped, SELECTABLE driver): + COGS is per-SKU already (Odoo Margin module). Operating expenses live per analytic + (Fisch/Royal/Amazon/HQ). Each channel pool = its analytic opex; we distribute it across that + channel's SKUs by a chosen DRIVER, and HQ overhead across all SKUs by the same driver: + pool_C = channel_rate_C * Σ(channel revenue) # rate-based magnitude (robust to Amazon + opex(sku)= Σ_C pool_C * driverC(sku)/Σ driverC + pool_HQ * driver(sku)/Σ driver + # invoice revenue not fully visible per-SKU) + Driver options: + - 'cogs' : cost-weighted (default) — higher-cost items bear more overhead. + - 'cbm' : physical size — units × volume (m³); BIGGER/bulkier SKUs absorb more (storage/freight/ + FBA scale with size). Volume is on ~24% of SKUs in Odoo; the rest are imputed at the + category (else global) median volume. Weight is unusable (~0% populated). + - 'revenue': % of sale (reduces to channel_rate × revenue). + - 'units' : per-unit. + net(sku) = gross_margin(sku) − opex(sku). NET is a decision estimate; GROSS (margin/markup) is exact. + +Brand-filterable: team_id None = all channels (incl Amazon); 5 = Fisch, 6 = Royal (wholesale scope). +READ-ONLY. +""" +import sys +import statistics +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P + +_ANA = {1: 'Fisch', 2: 'Royal', 3: 'Amazon', 4: 'HQ', 5: 'Internal'} +# Period: calendar 2025 (not LTM) so the pricing P&L ties to the 2025 P&L / GL / vendor analyses. +_FY = ('2025-01-01', '2025-12-31') +_FY_LABEL = 'FY2025' +DRIVERS = {'cogs': 'COGS (cost-weighted)', 'cbm': 'CBM / volume (size)', 'revenue': 'Revenue', 'units': 'Units'} +_DRIVER_SHORT = {'cogs': 'COGS', 'cbm': 'CBM', 'units': 'Units', 'revenue': 'Revenue'} + + +def has_driver_data(row, driver): + """True if the SKU has REAL (not imputed / defaulted) data for the chosen allocation driver.""" + if driver == 'cbm': + return bool(row.get('vol_known')) # volume on file in Odoo (else category-median imputed) + if driver == 'cogs': + return (row.get('cogs') or 0) > 0 # has a real product cost (else uncosted) + if driver == 'units': + return (row.get('units') or 0) > 0 + return (row.get('revenue') or 0) > 0 # revenue: always present in the table + + +def rates(t=None): + """LTM opex rates per channel + HQ, from the analytic ledger. {channel: opex/revenue}.""" + lf, lt = _FY + o = O.get_odoo() + rev = {_ANA.get(O.m2o_id(r['account_id'])): (r['amount'] or 0.0) for r in o.read_group( + 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt), + ('general_account_id.account_type', 'in', ['income', 'income_other'])], + ['amount:sum', 'account_id'], ['account_id'], lazy=False) if O.m2o_id(r['account_id']) in _ANA} + opx = {_ANA.get(O.m2o_id(r['account_id'])): -(r['amount'] or 0.0) for r in o.read_group( + 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt), + ('general_account_id.account_type', 'in', ['expense', 'expense_depreciation'])], + ['amount:sum', 'account_id'], ['account_id'], lazy=False) if O.m2o_id(r['account_id']) in _ANA} + tot_rev = sum(rev.values()) or 1.0 + def rate(k): + return (opx.get(k, 0.0) / rev[k]) if rev.get(k) else 0.0 + return {'Fisch': rate('Fisch'), 'Royal': rate('Royal'), 'Amazon': rate('Amazon'), + 'HQ': (opx.get('HQ', 0.0) + opx.get('Internal', 0.0)) / tot_rev, + 'window': _FY_LABEL, '_rev': rev, '_opex': opx, '_tot_rev': tot_rev} + + +def _meta(pids): + prods = O.search_read('product.product', [('id', 'in', pids), ('active', 'in', [True, False])], + ['default_code', 'name', 'categ_id', 'volume']) + cats = {} + for c in O.search_read('product.category', [], ['id', 'complete_name']): + parts = [x.strip() for x in (c['complete_name'] or '').split('/')] + cats[c['id']] = parts[1] if len(parts) >= 2 else (parts[0] if parts else None) + # median volume per category + global, to impute the SKUs without volume on file + by_cat = {} + allv = [] + for p in prods: + v = p.get('volume') or 0.0 + if v > 0: + allv.append(v) + by_cat.setdefault(O.m2o_id(p.get('categ_id')), []).append(v) + gmed = statistics.median(allv) if allv else 0.0 + cmed = {c: statistics.median(vs) for c, vs in by_cat.items()} + out = {} + for p in prods: + cid = O.m2o_id(p.get('categ_id')) + v = p.get('volume') or 0.0 + out[p['id']] = {'sku': p.get('default_code') or f"#{p['id']}", 'name': p.get('name') or '', + 'category': cats.get(cid) or '(uncategorized)', + 'volume': v, 'vol_used': v if v > 0 else (cmed.get(cid) or gmed), 'vol_known': v > 0} + return out + + +def _build(team_id=None, driver='cogs', t=None): + t = t or P.today() + lf, lt = _FY + o = O.get_odoo() + rt = rates(t) + gift = list(O.excluded_partner_ids()) + win = [('order_id.date_order', '>=', f'{lf} 00:00:00'), ('order_id.date_order', '<=', f'{lt} 23:59:59')] + sbase = [('order_id.state', 'in', ['sale', 'done']), ('product_id.type', '!=', 'service')] + win + + def by_prod(extra, fields): + return {O.m2o_id(r['product_id']): r for r in o.read_group('sale.order.line', sbase + extra, + fields + ['product_id'], ['product_id'], lazy=False) if r.get('product_id')} + + if team_id in (5, 6): # one wholesale BU + allc = by_prod([('order_id.team_id', '=', team_id), ('order_partner_id', 'not in', gift)], + ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum']) + chan_of = {pid: ('Fisch' if team_id == 5 else 'Royal') for pid in allc} + chan_rev = {pid: {('Fisch' if team_id == 5 else 'Royal'): (a['price_subtotal'] or 0.0)} for pid, a in allc.items()} + else: # all channels + allc = by_prod([], ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum']) + rf = by_prod([('order_id.team_id', '=', 5), ('order_partner_id', 'not in', gift)], ['price_subtotal:sum']) + rr = by_prod([('order_id.team_id', '=', 6), ('order_partner_id', 'not in', gift)], ['price_subtotal:sum']) + ra = by_prod([('order_partner_id', 'in', gift)] if gift else [('id', '=', -1)], ['price_subtotal:sum']) + chan_rev = {pid: {'Fisch': (rf.get(pid, {}).get('price_subtotal') or 0.0), + 'Royal': (rr.get(pid, {}).get('price_subtotal') or 0.0), + 'Amazon': (ra.get(pid, {}).get('price_subtotal') or 0.0)} for pid in allc} + + meta = _meta(list(allc)) + # assemble per-SKU base + skus = {} + for pid, a in allc.items(): + rev = a['price_subtotal'] or 0.0 + if rev <= 0: + continue + units = a['product_uom_qty'] or 0.0 + gm = a['margin'] or 0.0 + cogs = rev - gm + m = meta.get(pid, {}) + skus[pid] = {'rev': rev, 'units': units, 'gm': gm, 'cogs': cogs, 'cr': chan_rev.get(pid, {}), + 'cbm': units * (m.get('vol_used') or 0.0), 'm': m} + + # driver value per SKU + its apportionment to each channel (by revenue mix) + def dval(s): + return {'cogs': s['cogs'], 'cbm': s['cbm'], 'units': s['units'], 'revenue': s['rev']}.get(driver, s['cogs']) + chans = ['Fisch', 'Royal', 'Amazon'] + pool = {C: rt[C] * sum(s['cr'].get(C, 0.0) for s in skus.values()) for C in chans} # rate × visible channel rev + pool_hq = rt['HQ'] * sum(s['rev'] for s in skus.values()) + sumdrvC = {C: sum(dval(s) * (s['cr'].get(C, 0.0) / s['rev']) for s in skus.values() if s['rev']) for C in chans} + sumdrv = sum(dval(s) for s in skus.values()) or 1.0 + + rows = [] + for pid, s in skus.items(): + load = pool_hq * (dval(s) / sumdrv) + for C in chans: + if sumdrvC[C] > 0 and s['rev']: + load += pool[C] * (dval(s) * (s['cr'].get(C, 0.0) / s['rev'])) / sumdrvC[C] + rev, gm, cogs, units = s['rev'], s['gm'], s['cogs'], s['units'] + net = gm - load + m = s['m'] + rows.append({ + 'product_id': pid, 'sku': m.get('sku', f'#{pid}'), 'code': m.get('sku', f'#{pid}'), + 'product': m.get('name', ''), 'name': m.get('name', ''), 'category': m.get('category', '(uncategorized)'), + 'units': units, 'revenue': rev, 'cogs': cogs, + 'unit_cost': (cogs / units) if units else 0.0, 'avg_price': (rev / units) if units else 0.0, + 'cbm_unit': m.get('volume', 0.0), 'cbm_used': m.get('vol_used', 0.0), 'cbm_total': s['cbm'], + 'vol_known': m.get('vol_known', False), 'cbm_src': ('on file' if m.get('vol_known') else 'imputed'), + 'gm_dollars': gm, 'gm_pct': (gm / rev * 100) if rev else 0.0, + 'markup_pct': (gm / cogs * 100) if cogs > 0 else None, + 'opex_load': load, 'opex_pct': (load / rev * 100) if rev else 0.0, + 'net_dollars': net, 'net_pct': (net / rev * 100) if rev else 0.0, + 'status': ('Below cost' if gm < 0 else 'Net-negative' if net < 0 else 'Thin (<10% net)' if (net / rev) < 0.10 else 'Healthy'), + }) + rows.sort(key=lambda r: -r['revenue']) + return rows, rt + + +def table(team_id=None, driver='cogs', t=None): + return _build(team_id, driver, t)[0] + + +def summary(team_id=None, driver='cogs', t=None, built=None): + rows, rt = built or _build(team_id, driver, t) + rev = sum(r['revenue'] for r in rows) + gm = sum(r['gm_dollars'] for r in rows) + net = sum(r['net_dollars'] for r in rows) + mk = [r['markup_pct'] for r in rows if r['markup_pct'] is not None] + vol_known = sum(1 for r in rows if r['vol_known']) + driver_known = sum(1 for r in rows if has_driver_data(r, driver)) + return { + 'window': rt['window'], 'skus': len(rows), 'revenue': rev, 'driver': driver, 'driver_label': DRIVERS.get(driver, driver), + 'driver_short': _DRIVER_SHORT.get(driver, driver), + 'gm_dollars': gm, 'gm_pct': (gm / rev * 100) if rev else 0.0, + 'net_dollars': net, 'net_pct': (net / rev * 100) if rev else 0.0, + 'avg_markup': (sum(mk) / len(mk)) if mk else 0.0, + 'below_cost_skus': sum(1 for r in rows if r['gm_dollars'] < 0), + 'net_negative_skus': sum(1 for r in rows if r['net_dollars'] < 0), + 'net_negative_rev': sum(r['revenue'] for r in rows if r['net_dollars'] < 0), + 'vol_coverage': (vol_known / len(rows) * 100) if rows else 0.0, + 'driver_known': driver_known, 'driver_coverage': (driver_known / len(rows) * 100) if rows else 0.0, + 'rates': {k: round(rt[k] * 100, 1) for k in ('Fisch', 'Royal', 'Amazon', 'HQ')}, + } + + +def cost_drift(t=None, team_id=None): + """Replacement-cost drift from confirmed PO lines: the net price paid per unit in the LAST + 12 months vs the 12 months BEFORE, per product, joined to what the SELL price did over the + same two windows (BU-scoped sell side; cost is company-wide). ERODING = cost up >5% while + the selling price followed by less than half — the margin leaks silently until repriced. + cost_impact_12m = (cost_now − cost_prior) × units sold last 12m (the annualized $ at stake).""" + t = t or P.today() + d24 = (t - dt.timedelta(days=730)).isoformat() + d12 = (t - dt.timedelta(days=365)).isoformat() + lines = O.search_read('purchase.order.line', + [('order_id.state', 'in', ('purchase', 'done')), + ('order_id.date_order', '>=', d24), + ('product_qty', '>', 0), ('price_unit', '>', 0)], + ['product_id', 'product_qty', 'product_uom_qty', 'price_subtotal', + 'order_id']) + oids = list({O.m2o_id(l['order_id']) for l in lines if l.get('order_id')}) + od = {} + for i in range(0, len(oids), 5000): + for o_ in O.search_read('purchase.order', [('id', 'in', oids[i:i + 5000])], ['date_order']): + od[o_['id']] = str(o_['date_order'])[:10] + cur, prior = {}, {} # pid -> [net spend, qty] + for l in lines: + pid = O.m2o_id(l.get('product_id')) + d = od.get(O.m2o_id(l.get('order_id'))) + if not pid or not d: + continue + e = (cur if d >= d12 else prior).setdefault(pid, [0.0, 0.0]) + e[0] += l.get('price_subtotal') or 0.0 + # BASE-UoM qty, so a piece→case purchase-UoM switch doesn't fake a price spike + e[1] += l.get('product_uom_qty') or l.get('product_qty') or 0.0 + both = [p for p in cur if p in prior and prior[p][1] > 0 and cur[p][1] > 0] + + def _sell(a, b_): + out = {} + for g in O.read_group('sale.order.line', O.sale_line_domain(a, b_, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], + lazy=False): + pid = O.m2o_id(g.get('product_id')) + if pid: + out[pid] = (g.get('price_subtotal') or 0.0, g.get('product_uom_qty') or 0.0) + return out + s_now, s_pri = _sell(d12, t.isoformat()), _sell(d24, d12) + + meta = {} + for i in range(0, len(both), 5000): + for p in O.search_read('product.product', + [('id', 'in', both[i:i + 5000]), ('active', 'in', [True, False])], + ['default_code', 'name', 'standard_price']): + meta[p['id']] = p + rows = [] + for pid in both: + c_now, c_pri = cur[pid][0] / cur[pid][1], prior[pid][0] / prior[pid][1] + if c_pri <= 0: + continue + rn, qn = s_now.get(pid, (0.0, 0.0)) + rp, qp = s_pri.get(pid, (0.0, 0.0)) + asp_now = rn / qn if qn else None + asp_pri = rp / qp if qp else None + p = meta.get(pid, {}) + rows.append({'pid': pid, 'code': (p.get('default_code') or '').strip(), + 'product': p.get('name') or '', + 'cost_prior': c_pri, 'cost_now': c_now, + 'drift_pct': (c_now / c_pri - 1) * 100, + 'buy_qty_12m': cur[pid][1], 'std_cost': p.get('standard_price') or 0.0, + 'asp_now': asp_now, 'asp_prior': asp_pri, + 'price_chg_pct': ((asp_now / asp_pri - 1) * 100) + if (asp_now and asp_pri) else None, + 'units_12m': qn, 'cost_impact_12m': (c_now - c_pri) * qn, + 'gm_pct_now': ((asp_now - c_now) / asp_now * 100) if asp_now else None}) + # a >3x (or <1/3) per-base-unit move is a UoM/master-data break, not market inflation — + # surfaced as its own data-quality list so it can't pollute the erosion signal + breaks = sorted((r for r in rows if not (1 / 3 <= (r['cost_now'] / r['cost_prior']) <= 3)), + key=lambda r: -abs(r['drift_pct'])) + broken = {r['pid'] for r in breaks} + eroding = sorted((r for r in rows + if r['pid'] not in broken + and r['drift_pct'] > 5 and (r['units_12m'] or 0) > 0 + and (r['price_chg_pct'] is None or r['price_chg_pct'] < r['drift_pct'] / 2)), + key=lambda r: -(r['cost_impact_12m'] or 0)) + improving = sorted((r for r in rows if r['pid'] not in broken + and r['drift_pct'] < -5 and (r['units_12m'] or 0) > 0), + key=lambda r: r['cost_impact_12m']) + return {'rows': rows, 'eroding': eroding, 'improving': improving, 'breaks': breaks, + 'n_products': len(rows), + 'erosion_total': sum(r['cost_impact_12m'] for r in eroding), + 'tailwind_total': sum(r['cost_impact_12m'] for r in improving), + '_cur_spend': sum(e[0] for e in cur.values()), + '_cur_domain_from': d12, 'windows': (d24, d12, t.isoformat())} + + +def cost_drift_validate(cd, t=None): + """The 12m PO spend our per-product weighting is built on == the server-side sum over the + identical domain (two independent aggregation paths).""" + t = t or P.today() + dom = [('order_id.state', 'in', ('purchase', 'done')), + ('order_id.date_order', '>=', cd['_cur_domain_from']), + ('product_qty', '>', 0), ('price_unit', '>', 0)] + srv = O.sum_field('purchase.order.line', dom, 'price_subtotal') + return [{'check': 'Cost drift: Σ(per-product 12m PO spend) == server Σ(line subtotal), same domain', + 'a': round(cd['_cur_spend'], 2), 'b': round(srv, 2), + 'gap': round(cd['_cur_spend'] - srv, 2), + 'ok': abs(cd['_cur_spend'] - srv) <= max(1.0, abs(srv) * 0.001)}] + + +def _pnl_entities(t=None): + """Actual LTM P&L per analytic entity (the basis the Management P&L is built from).""" + lf, lt = _FY + o = O.get_odoo() + def grp(types): + return {_ANA.get(O.m2o_id(r['account_id'])): (r['amount'] or 0.0) for r in o.read_group( + 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt), + ('general_account_id.account_type', 'in', types)], ['amount:sum', 'account_id'], ['account_id'], lazy=False) + if O.m2o_id(r['account_id']) in _ANA} + inc, cog, opx = grp(['income', 'income_other']), grp(['expense_direct_cost']), grp(['expense', 'expense_depreciation']) + return {k: {'revenue': inc.get(k, 0.0), 'cogs': -cog.get(k, 0.0), 'opex': -opx.get(k, 0.0)} for k in _ANA.values()} + + +def gl_pnl(t=None): + """Actual LTM P&L straight from the posted GL (the official books) — the independent reconciliation + target. Revenue − COGS − Opex = Net.""" + lf, lt = _FY + def s(types): + return O.sum_field('account.move.line', [('parent_state', '=', 'posted'), ('date', '>=', lf), + ('date', '<=', lt), ('account_id.account_type', 'in', types)], 'balance') + rev = -s(['income', 'income_other']) # income is credit → flip to positive + cogs = s(['expense_direct_cost']) + opex = s(['expense', 'expense_depreciation']) + return {'revenue': rev, 'cogs': cogs, 'gm': rev - cogs, 'opex': opex, 'net': rev - cogs - opex} + + +def reconcile(team_id=None, driver='cogs', t=None, built=None): + """Bridge the per-SKU P&L to the ACTUAL P&L: attributed SKUs + unattributed (Amazon-direct, not + booked per-SKU) = the displayed P&L. Ties by construction; the unattributed line is the residual.""" + rows, rt = built or _build(team_id, driver, t) + ent = _pnl_entities(t) + tot_rev = sum(e['revenue'] for e in ent.values()) or 1.0 + hq_rate = (ent['HQ']['opex'] + ent['Internal']['opex']) / tot_rev + if team_id in (5, 6): + k = 'Fisch' if team_id == 5 else 'Royal' + rev_t, cogs_t = ent[k]['revenue'], ent[k]['cogs'] + opex_t = ent[k]['opex'] + hq_rate * ent[k]['revenue'] # BU opex + its share of HQ + else: + rev_t = sum(e['revenue'] for e in ent.values()) + cogs_t = sum(e['cogs'] for e in ent.values()) + opex_t = sum(e['opex'] for e in ent.values()) # all opex incl HQ + net_t = rev_t - cogs_t - opex_t + rev_s = sum(r['revenue'] for r in rows) + gm_s = sum(r['gm_dollars'] for r in rows) + cogs_s, opex_s = rev_s - gm_s, sum(r['opex_load'] for r in rows) + net_s = gm_s - opex_s + una = {'revenue': rev_t - rev_s, 'cogs': cogs_t - cogs_s, 'gm': (rev_t - rev_s) - (cogs_t - cogs_s), + 'opex': opex_t - opex_s, 'net': net_t - net_s} + pnl = {'revenue': rev_t, 'cogs': cogs_t, 'gm': rev_t - cogs_t, 'opex': opex_t, 'net': net_t} + sku = {'revenue': rev_s, 'cogs': cogs_s, 'gm': gm_s, 'opex': opex_s, 'net': net_s} + return {'pnl': pnl, 'sku': sku, 'unattrib': una, 'ties': abs((net_s + una['net']) - net_t) < 1.0} + + +def page_data(team_id=None, driver='cogs', t=None): + """One build → rows + summary + reconciliation + validation (avoids rebuilding 4×).""" + built = _build(team_id, driver, t) + return {'rows': built[0], 'summary': summary(team_id, driver, t, built=built), + 'reconcile': reconcile(team_id, driver, t, built=built), + 'validation': validate(team_id, driver, t, built=built)} + + +def validate(team_id=None, driver='cogs', t=None, built=None): + rows, rt = built or _build(team_id, driver, t) + checks = [] + lf, lt = _FY + gift = list(O.excluded_partner_ids()) + dom = [('order_id.state', 'in', ['sale', 'done']), ('order_id.date_order', '>=', f'{lf} 00:00:00'), + ('order_id.date_order', '<=', f'{lt} 23:59:59'), ('product_id.type', '!=', 'service'), ('price_subtotal', '>', 0)] + if team_id in (5, 6): + dom += [('order_id.team_id', '=', team_id), ('order_partner_id', 'not in', gift)] + indep = O.sum_field('sale.order.line', dom, 'price_subtotal') + ours = sum(r['revenue'] for r in rows) + checks.append({'check': 'Σ per-SKU revenue == scoped FY2025 (positive lines)', 'a': round(ours, 0), + 'b': round(indep, 0), 'gap': round(ours - indep, 0), 'ok': abs(ours - indep) <= max(50.0, indep * 0.01)}) + if rows: + s = rows[0] + checks.append({'check': f"GM == revenue−COGS (sample {s['sku']})", 'a': round(s['gm_dollars'], 2), + 'b': round(s['revenue'] - s['cogs'], 2), 'gap': round(s['gm_dollars'] - (s['revenue'] - s['cogs']), 2), + 'ok': abs(s['gm_dollars'] - (s['revenue'] - s['cogs'])) <= 0.5}) + # THE finance check — per-SKU P&L reconciles to the actual P&L (and that target ties to the posted GL) + rc = reconcile(team_id, driver, t, built=(rows, rt)) + checks.append({'check': 'Attributed SKUs + unattributed net == P&L net (reconciles)', + 'a': round(rc['sku']['net'] + rc['unattrib']['net'], 0), 'b': round(rc['pnl']['net'], 0), + 'gap': round(rc['sku']['net'] + rc['unattrib']['net'] - rc['pnl']['net'], 0), 'ok': rc['ties']}) + if team_id is None: + gl = gl_pnl(t) + checks.append({'check': 'P&L target (analytic) == posted GL net (actual books)', + 'a': round(rc['pnl']['net'], 0), 'b': round(gl['net'], 0), + 'gap': round(rc['pnl']['net'] - gl['net'], 0), 'ok': abs(rc['pnl']['net'] - gl['net']) <= 2.0}) + return checks diff --git a/royalimports_os/modules/procurement.py b/royalimports_os/modules/procurement.py new file mode 100644 index 0000000000000000000000000000000000000000..8632737d8d1cfd4bd41ced18ece8dd5a782d92db --- /dev/null +++ b/royalimports_os/modules/procurement.py @@ -0,0 +1,540 @@ +"""Procurement module — biweekly buy list + the full Qualify-tab workflow, rebuilt from the Royal +Imports Inventory System (v24/v25). + +Master data (Supplier / Country / Lead / CBM / container / first cost) comes from the curated map +`procurement_suppliers.json` (~3,960 SKUs from the Inventory System Mastersheet). The team's live +edits (change vendor/country/cost) are overlaid in the UI from a `procurement_overrides` store, so a +re-vendored SKU regroups instantly without re-pulling Odoo. Alternative vendors + prices come from +Odoo `product.supplierinfo` (the cheapest-vendor view). Demand / on-hand / incoming POs are live +from Odoo (read-only). + +Reorder math (validated on the v24.4 Backend): + 8-month demand = units sold same 8-month window last year (all channels) + avg/day = 8M demand / window days + days of cover = (on-hand + incoming PO) / avg/day + reorder point = lead-time days * avg/day + Reorder QTY = ceil(reorder point * 1.05) [Baseline] + Reorder +trend = Reorder QTY * (YTD units this yr / last yr) + BUY TODAY = days of cover < lead time +READ-ONLY on Odoo; the team's Order Qty + master-data edits persist in the platform store. +""" +import sys +import json +import math +import calendar +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from functools import lru_cache +import core.odoo as O +import core.periods as P + +_HERE = Path(__file__).resolve().parent +_SAFETY = 0.05 +_PROJ_MONTHS = 8 +_NO_SVC = [('product_id.type', '!=', 'service')] + + +@lru_cache(maxsize=1) +def suppliers(): + """SKU(default_code) -> {desc, unit, qty_per_unit, first_cost, cbm, max_container, vendor, + country, lead} from the curated map.""" + f = _HERE.parent / 'procurement_suppliers.json' + try: + return json.loads(f.read_text(encoding='utf-8')) + except Exception: + return {} + + +def _add_months(d, n): + m = d.month - 1 + n + y = d.year + m // 12 + m = m % 12 + 1 + return dt.date(y, m, min(d.day, calendar.monthrange(y, m)[1])) + + +def _proj_window(t): + return t, _add_months(t, _PROJ_MONTHS) + + +def _demand_domain(a, b): + return [('order_id.state', 'in', ('sale', 'done')), ('order_id.date_order', '>=', a), + ('order_id.date_order', '<=', b)] + _NO_SVC + + +def _units(a, b): + g = O.read_group('sale.order.line', _demand_domain(a, b), ['product_uom_qty:sum'], + ['product_id'], lazy=False) + return {O.m2o_id(r['product_id']): (r.get('product_uom_qty') or 0.0) + for r in g if r.get('product_id')} + + +def _incoming_po(): + out = {} + for r in O.read_group('purchase.order.line', [('state', '=', 'purchase')], + ['product_qty:sum', 'qty_received:sum'], ['product_id'], lazy=False): + pid = O.m2o_id(r.get('product_id')) + q = (r.get('product_qty') or 0.0) - (r.get('qty_received') or 0.0) + if pid and q > 0: + out[pid] = q + return out + + +def _bought_12m(cutoff_iso): + """Units PURCHASED from suppliers in the last 12 months (confirmed POs), per product — the + reference for 'how much we actually bought last year' next to the projected need.""" + out = {} + for r in O.read_group('purchase.order.line', + [('order_id.state', 'in', ('purchase', 'done')), + ('order_id.date_order', '>=', cutoff_iso)], + ['product_qty:sum'], ['product_id'], lazy=False): + pid = O.m2o_id(r.get('product_id')) + if pid: + out[pid] = r.get('product_qty') or 0.0 + return out + + +def _supplierinfo_by_tmpl(): + """{template_id: [{'vendor','vendor_id','price'}...]} — Odoo's vendors per product, for the + multiple-vendor / cheapest-price views (Odoo prices are populated; lead times are not).""" + out = {} + for s in O.search_read('product.supplierinfo', [], ['product_tmpl_id', 'partner_id', 'price'], + limit=100000): + t = O.m2o_id(s.get('product_tmpl_id')) + if not t: + continue + out.setdefault(t, []).append({'vendor': O.m2o_name(s.get('partner_id')), + 'vendor_id': O.m2o_id(s.get('partner_id')), + 'price': s.get('price') or 0.0}) + return out + + +def _preorders(cutoff_iso): + """Per-product open customer demand (undelivered units on confirmed orders) + earliest need-by + date. STORED fields only — `qty_to_deliver` is a computed field and filtering/aggregating it forces + Odoo to compute across the whole line population (8+ min); instead sum the stored + `product_uom_qty - qty_delivered`. Bounded to orders since `cutoff_iso` so ancient never-closed + backorders don't dominate the figures. Returns ({pid: qty}, {pid: 'YYYY-MM-DD'}).""" + dom = [('order_id.state', '=', 'sale'), ('order_id.date_order', '>=', cutoff_iso)] + qty = {} + for r in O.read_group('sale.order.line', dom, ['product_uom_qty:sum', 'qty_delivered:sum'], + ['product_id'], lazy=False): + pid = O.m2o_id(r.get('product_id')) + d = (r.get('product_uom_qty') or 0.0) - (r.get('qty_delivered') or 0.0) + if pid and d > 0: + qty[pid] = d + # need-by: only never-shipped lines (qty_delivered is STORED, so this filter is cheap) AND only + # for products that actually have open qty — need_by is consumed ONLY when preorder_qty>0 (pid in + # qty), so scoping the pull to those product_ids is both correct and much faster (~849 of ~2,870 + # products → the search drops from ~8.4s to ~5s; the other ~2,000 products' rows were pure waste). + open_pids = list(qty) + if not open_pids: + return qty, {} + lines = O.search_read('sale.order.line', + dom + [('qty_delivered', '=', 0), ('product_id', 'in', open_pids)], + ['product_id', 'order_id'], limit=400000) + odate = {} + oids = list({O.m2o_id(l['order_id']) for l in lines if l.get('order_id')}) + for o in O.search_read('sale.order', [('id', 'in', oids)], ['commitment_date', 'date_order']): + d = o.get('commitment_date') or o.get('date_order') + if d: + odate[o['id']] = str(d)[:10] + need = {} + for l in lines: + pid, oid = O.m2o_id(l['product_id']), O.m2o_id(l['order_id']) + d = odate.get(oid) + if pid and d and (pid not in need or d < need[pid]): + need[pid] = d + return qty, need + + +def recommendations(t=None): + """Base per-SKU table (curated master data + live Odoo demand/stock/PO + alt-vendors). The UI + overlays the team's vendor/country/cost edits on top; reorder methods + CBM/container are here.""" + t = t or P.today() + sup = suppliers() + if not sup: + return [] + codeset = set(sup.keys()) + ps, pe = _proj_window(t) + window_days = max(1, (pe - ps).days) + d8a, d8b = _add_months(t, -12).isoformat(), _add_months(pe, -12).isoformat() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + prods, demand8, ytd_now, ytd_prev, inpo, si, preorders, bought = O.parallel([ + lambda: O.search_read('product.product', + [('default_code', '!=', False), ('active', 'in', [True, False])], + ['default_code', 'name', 'qty_available', 'standard_price', 'product_tmpl_id']), + lambda: _units(d8a, d8b), + lambda: _units(yf, yt), + lambda: _units(lf, lt), + _incoming_po, + _supplierinfo_by_tmpl, + lambda: _preorders(d8a), + lambda: _bought_12m(d8a), + ]) + pre_qty, pre_need = preorders + today_iso = t.isoformat() + by_code = {} + for p in prods: + c = (p.get('default_code') or '').strip() + if c and c in codeset and c not in by_code: + by_code[c] = p + if not by_code: + return [] + + rows = [] + for code, p in by_code.items(): + pid = p['id'] + meta = sup.get(code, {}) + lead = meta.get('lead') or 0 + tmpl = O.m2o_id(p.get('product_tmpl_id')) + alts = si.get(tmpl, []) + priced = sorted((a for a in alts if (a.get('price') or 0) > 0), key=lambda a: a['price']) + cheapest = priced[0] if priced else None + cost = (meta.get('first_cost') or (cheapest['price'] if cheapest else None) + or p.get('standard_price') or 0.0) + d8 = demand8.get(pid, 0.0) + avg_day = d8 / window_days if window_days else 0.0 + on_hand = p.get('qty_available') or 0.0 + po = inpo.get(pid, 0.0) + avail = on_hand + po + days_cover = (avail / avg_day) if avg_day > 0 else (None if avail <= 0 else 9999.0) + reorder_pt = lead * avg_day + rbase = math.ceil(reorder_pt * (1 + _SAFETY)) if reorder_pt > 0 else 0 + yn, yp = ytd_now.get(pid, 0.0), ytd_prev.get(pid, 0.0) + trend = (yn / yp) if yp > 0 else None + rtrend = int(round(rbase * trend)) if trend is not None else rbase + buy = (days_cover is not None and lead > 0 and avg_day > 0 and days_cover < lead) + rows.append({ + 'code': code, 'product': meta.get('desc') or p.get('name') or code, + 'unit': meta.get('unit') or '', 'qty_per_unit': meta.get('qty_per_unit'), + 'vendor': meta.get('vendor') or '(no supplier)', 'country': meta.get('country') or '', + 'lead': lead, 'on_hand': on_hand, 'incoming_po': po, 'demand_8m': d8, 'avg_day': avg_day, + 'bought_12m': bought.get(pid, 0.0), + 'days_cover': days_cover, 'reorder_baseline': rbase, 'reorder_trend': rtrend, + 'trend_pct': ((trend - 1) * 100) if trend is not None else None, + 'ytd_units': yn, 'ytd_units_ly': yp, 'unit_cost': cost, 'buy': buy, + 'cbm': meta.get('cbm'), 'max_container': meta.get('max_container'), + 'preorder_qty': pre_qty.get(pid, 0.0), + 'need_by': pre_need.get(pid) if pre_qty.get(pid, 0) > 0 else None, + 'overdue': bool(pre_qty.get(pid, 0) > 0 and pre_need.get(pid) and pre_need.get(pid) < today_iso), + 'email': meta.get('email'), 'city': meta.get('city'), 'street': meta.get('street'), + 'port': meta.get('port'), + 'n_vendors': len({a['vendor_id'] for a in alts if a.get('vendor_id')}), + 'cheapest_vendor': cheapest['vendor'] if cheapest else None, + 'cheapest_price': cheapest['price'] if cheapest else None, + 'alt_vendors': priced, + }) + return rows + + +def purchase_need_key(r): + """Sort key = 'what we need to purchase', most urgent first. SKUs flagged BUY (past the + reorder point) come before OK SKUs; within each, lower days-of-cover (closer to stocking out) + ranks higher; a bigger suggested reorder breaks ties. No-demand / fully-covered SKUs + (days_cover None or 9999) sink to the bottom. This is what orders the per-vendor line list so + that in 'All SKUs' mode the actionable SKUs float to the top while the whole catalogue stays + visible below.""" + dc = r.get('days_cover') + if dc is None or dc >= 9999: + dc = float('inf') + return (0 if r.get('buy') else 1, dc, -(r.get('reorder_trend') or 0)) + + +def by_supplier(recs): + """Group rows into per-vendor purchase orders (pure — call with overrides already applied). + Each vendor's lines are ordered by purchase need (buy-first, most urgent on top — see + purchase_need_key), so the buy list and the 'All SKUs per vendor' view surface what to order + without hunting; the full SKU set for the vendor is still present, just sorted underneath.""" + pos = {} + for r in recs: + v = r['vendor'] + po = pos.setdefault(v, {'vendor': v, 'country': r.get('country', ''), 'lead': r.get('lead', 0), + 'lines': [], 'skus': 0}) + po['lines'].append(r) + po['skus'] += 1 + for po in pos.values(): + po['lines'].sort(key=purchase_need_key) + return sorted(pos.values(), key=lambda x: (x['vendor'] or '~')) + + +def data_quality(): + sup = suppliers() + codes = list(sup.keys()) + cs = set(codes) + found = {c for c in ((p.get('default_code') or '').strip() + for p in O.search_read('product.product', + [('default_code', '!=', False), ('active', 'in', [True, False])], + ['default_code'])) if c in cs} + missing = [c for c in codes if c not in found] + no_vendor = [c for c, d in sup.items() if not d.get('vendor')] + no_lead = [c for c, d in sup.items() if not d.get('lead')] + return [ + {'issue': 'Curated SKU not found in Odoo (no live data)', 'count': len(missing), + 'fix': 'Check the SKU internal reference matches Odoo, or remove it from the mapping.', + 'sample': [{'default_code': c, 'name': sup.get(c, {}).get('desc')} for c in missing]}, + {'issue': 'No supplier in the mapping', 'count': len(no_vendor), + 'fix': 'Set the vendor on the SKU (Procurement page or Vendor module) so a PO can be grouped.', + 'sample': [{'default_code': c, 'name': sup.get(c, {}).get('desc')} for c in no_vendor]}, + {'issue': 'No lead time in the mapping', 'count': len(no_lead), + 'fix': 'Set the supplier lead time (days) — the buy trigger needs it.', + 'sample': [{'default_code': c, 'name': sup.get(c, {}).get('desc')} for c in no_lead]}, + ] + + +def _open_po_lines(): + """All confirmed-PO lines still (partly) undelivered: state='purchase', open qty > 0, with the + line's expected date. One pull powers the PO board, the per-SKU inbound match and the late + chase list. (Same open-qty semantics as _incoming_po — the two are cross-validated.)""" + lines = O.search_read('purchase.order.line', + [('state', '=', 'purchase'), ('product_qty', '>', 0)], + ['order_id', 'partner_id', 'product_id', 'product_qty', 'qty_received', + 'price_unit', 'price_subtotal', 'date_planned']) + out = [] + for r in lines: + open_qty = (r.get('product_qty') or 0) - (r.get('qty_received') or 0) + if open_qty <= 1e-3: + continue + qty = r.get('product_qty') or 0 + unit = (r['price_subtotal'] / qty) if qty else (r.get('price_unit') or 0) + r['open_qty'] = open_qty + r['open_value'] = open_qty * unit + r['expected'] = str(r.get('date_planned') or '')[:10] + out.append(r) + return lines, out + + +def open_pos(t=None): + """The open-PO board matched to procurement: order-level list of every confirmed PO with + undelivered quantity, per-SKU inbound (open qty + earliest ETA + late portion) keyed by + default_code so it joins the buy list, and the late-delivery chase list.""" + t = t or P.today() + today = t.isoformat() + _all, lines = _open_po_lines() + + pids = list({O.m2o_id(l['product_id']) for l in lines if l.get('product_id')}) + codes = {} + for i in range(0, len(pids), 5000): + for p in O.search_read('product.product', + [('id', 'in', pids[i:i + 5000]), ('active', 'in', [True, False])], + ['default_code']): + codes[p['id']] = (p.get('default_code') or '').strip() + + oids = list({O.m2o_id(l['order_id']) for l in lines if l.get('order_id')}) + odates = {} + for i in range(0, len(oids), 5000): + for o_ in O.search_read('purchase.order', [('id', 'in', oids[i:i + 5000])], + ['date_order', 'amount_total']): + odates[o_['id']] = {'date_order': str(o_.get('date_order') or '')[:10], + 'amount_total': o_.get('amount_total') or 0.0} + + by_po, by_sku, late_rows = {}, {}, [] + for l in lines: + po_id = O.m2o_id(l['order_id']) + po_nm = O.m2o_name(l['order_id']) + sup = O.m2o_name(l['partner_id']) + pid = O.m2o_id(l['product_id']) + code = codes.get(pid) or O.m2o_name(l['product_id']) + exp = l['expected'] + is_late = bool(exp and exp < today) + + e = by_po.setdefault(po_nm, { + 'po': po_nm, 'supplier': sup, + 'ordered': odates.get(po_id, {}).get('date_order', ''), + 'po_value': odates.get(po_id, {}).get('amount_total', 0.0), + 'expected': exp, 'open_lines': 0, 'open_value': 0.0, 'late_value': 0.0}) + e['open_lines'] += 1 + e['open_value'] += l['open_value'] + if exp and (not e['expected'] or exp < e['expected']): + e['expected'] = exp + if is_late: + e['late_value'] += l['open_value'] + + s = by_sku.setdefault(code, {'open_qty': 0.0, 'open_value': 0.0, 'eta': None, + 'late_qty': 0.0, 'pos': set()}) + s['open_qty'] += l['open_qty'] + s['open_value'] += l['open_value'] + s['pos'].add(po_nm) + if exp and (s['eta'] is None or exp < s['eta']): + s['eta'] = exp + if is_late: + s['late_qty'] += l['open_qty'] + + if is_late: + days_late = (t - dt.date.fromisoformat(exp)).days + late_rows.append({'supplier': sup, 'po': po_nm, 'sku': code, + 'product': O.m2o_name(l['product_id']), + 'open_qty': l['open_qty'], 'open_value': l['open_value'], + 'expected': exp, 'days_late': days_late}) + + for s in by_sku.values(): + s['pos'] = sorted(s['pos']) + pos = sorted(by_po.values(), key=lambda x: (x['expected'] or '9999')) + late_rows.sort(key=lambda x: -x['days_late']) + return { + 'pos': pos, 'by_sku': by_sku, 'late': late_rows, + 'n_pos': len(pos), 'open_value': sum(p['open_value'] for p in pos), + 'late_value': sum(p['late_value'] for p in pos), 'n_late_lines': len(late_rows), + } + + +def _pctl(sorted_vals, q): + return sorted_vals[min(len(sorted_vals) - 1, int(len(sorted_vals) * q))] if sorted_vals else None + + +def vendor_reliability(t=None): + """Vendor delivery truth from COMPLETED POs (24m): actual lead (date_order → effective_date) + and promise lateness (date_planned → effective_date), per vendor. purchase.order carries all + three dates, so no picking join is needed (the 'incoming picking' route is polluted by + customer returns — verified live: only 29% of incoming pickings match a PO). + The buy trigger runs on the CURATED nominal lead; this measures how wrong that is per vendor.""" + t = t or P.today() + cutoff = (t - dt.timedelta(days=730)).isoformat() + pos_ = O.search_read('purchase.order', + [('state', 'in', ('purchase', 'done')), ('date_order', '>=', cutoff), + ('effective_date', '!=', False)], + ['name', 'partner_id', 'date_order', 'date_planned', 'effective_date', + 'amount_total']) + # nominal lead per vendor = the most common curated per-SKU lead for that vendor (upper-name join) + nom = {} + for d in suppliers().values(): + v = (d.get('vendor') or '').strip().upper() + if v and d.get('lead'): + nom.setdefault(v, {}) + nom[v][d['lead']] = nom[v].get(d['lead'], 0) + 1 + nominal = {v: max(c.items(), key=lambda kv: kv[1])[0] for v, c in nom.items()} + + rows, per = [], {} + for p in pos_: + v = (O.m2o_name(p.get('partner_id')) or '(unknown)').strip() + o = dt.date.fromisoformat(str(p['date_order'])[:10]) + e = dt.date.fromisoformat(str(p['effective_date'])[:10]) + lead = (e - o).days + late = None + if p.get('date_planned'): + late = (e - dt.date.fromisoformat(str(p['date_planned'])[:10])).days + rows.append({'po': p['name'], 'vendor': v, 'ordered': o.isoformat(), + 'promised': str(p.get('date_planned') or '')[:10] or None, + 'received': e.isoformat(), 'lead_days': lead, 'days_late': late, + 'po_value': p.get('amount_total') or 0.0}) + s = per.setdefault(v, {'vendor': v, 'n_pos': 0, 'value': 0.0, 'leads': [], 'lates': []}) + s['n_pos'] += 1 + s['value'] += p.get('amount_total') or 0.0 + s['leads'].append(lead) + if late is not None: + s['lates'].append(late) + + vendors = [] + for s in per.values(): + s['leads'].sort() + s['lates'].sort() + vu = s['vendor'].upper() + v_nom = nominal.get(vu) + p90 = _pctl(s['leads'], 0.9) + vendors.append({'vendor': s['vendor'], 'n_pos': s['n_pos'], 'value': s['value'], + 'lead_med': _pctl(s['leads'], 0.5), 'lead_p90': p90, + 'nominal_lead': v_nom, + 'lead_gap': (p90 - v_nom) if (v_nom and p90 is not None) else None, + 'on_promise_pct': (sum(1 for d in s['lates'] if d <= 0) / len(s['lates']) * 100) + if s['lates'] else None, + 'late_med': _pctl(s['lates'], 0.5), 'late_p90': _pctl(s['lates'], 0.9)}) + vendors.sort(key=lambda x: -x['value']) + all_lates = sorted(r['days_late'] for r in rows if r['days_late'] is not None) + return {'vendors': vendors, 'rows': rows, 'n_pos': len(rows), + 'n_late': sum(1 for d in all_lates if d > 0), + 'late_share_pct': (sum(1 for d in all_lates if d > 0) / len(all_lates) * 100) + if all_lates else None, + 'late_med': _pctl(all_lates, 0.5), 'late_p90': _pctl(all_lates, 0.9), + 'n_vendors': len(vendors), + 'actual_p90_by_vendor': {v['vendor'].upper(): v['lead_p90'] for v in vendors + if v['n_pos'] >= 3 and v['lead_p90'] is not None}} + + +def buy_under_actual_lead(recs, vr): + """SKUs the nominal-lead trigger says are SAFE but that are already past the reorder point + under their vendor's ACTUAL p90 lead (vendors with 3+ completed POs). The honest delta — + the base buy list is untouched.""" + p90 = vr.get('actual_p90_by_vendor') or {} + out = [] + for r in recs: + if r.get('buy'): + continue + ap = p90.get((r.get('vendor') or '').strip().upper()) + if ap is None or r.get('days_cover') is None or r['days_cover'] >= 9999: + continue + eff = max(ap, r.get('lead') or 0) + if r.get('avg_day', 0) > 0 and r['days_cover'] < eff: + out.append({**r, 'actual_lead_p90': ap}) + out.sort(key=lambda x: (x['days_cover'] if x['days_cover'] is not None else 9999)) + return out + + +def summary(t=None): + t = t or P.today() + recs = recommendations(t) + ps, pe = _proj_window(t) + vr = vendor_reliability(t) + return { + 'recs': recs, 'n_skus': len(recs), + 'window_days': max(1, (pe - ps).days), 'safety_pct': _SAFETY * 100, + 'inbound': open_pos(t), + 'data_quality': data_quality(), + 'vendor_reliability': vr, + 'buy_flip': buy_under_actual_lead(recs, vr), + 'pulled_at': dt.datetime.now().strftime('%Y-%m-%d %H:%M'), + } + + +def vendor_book(t=None): + """Per-vendor rollup for the Vendor module: each vendor's SKU count, lead, country, demand value, + and the SKUs they supply. Plus the multiple-vendor / cheapest-price view.""" + recs = recommendations(t) + multi = [r for r in recs if (r.get('n_vendors') or 0) > 1] + return {'recs': recs, 'multi_vendor': multi, 'n_multi': len(multi)} + + +def validate(t=None, team_id=None): + t = t or P.today() + _, pe = _proj_window(t) + f, to = _add_months(t, -12).isoformat(), _add_months(pe, -12).isoformat() + g = O.read_group('sale.order.line', _demand_domain(f, to), ['product_uom_qty:sum'], + ['product_id'], lazy=False) + per = sum((r.get('product_uom_qty') or 0.0) for r in g if r.get('product_id')) + tot = O.sum_field('sale.order.line', _demand_domain(f, to), 'product_uom_qty') + checks = [{'check': '8-month demand: Σ(per-SKU units) == line-level units', + 'a': round(per, 2), 'b': round(tot, 2), 'gap': round(per - tot, 2), + 'ok': abs(per - tot) <= 1.0}] + + # Open POs: row-by-row NET (product_qty − qty_received over ALL confirmed lines — the same + # semantics as _incoming_po, over-receipts included) vs the _incoming_po read_group aggregate. + # Two aggregation paths, small live-drift tolerance (two RPC snapshots on a live ledger). + ib = open_pos(t) + all_lines, _ = _open_po_lines() + row_net = sum((l.get('product_qty') or 0) - (l.get('qty_received') or 0) for l in all_lines) + agg_net = sum(_incoming_po().values()) + checks.append({'check': 'Open POs: row-level Σ net qty == read_group Σ net qty (live-drift tol)', + 'a': round(row_net, 2), 'b': round(agg_net, 2), + 'gap': round(row_net - agg_net, 2), + 'ok': abs(row_net - agg_net) <= max(1.0, abs(agg_net) * 0.001)}) + + # Internal consistency: the late chase list is exactly the late portion of the SKU rollup. + late_val = sum(r['open_value'] for r in ib['late']) + sku_late_val_ok = sum(p['late_value'] for p in ib['pos']) + checks.append({'check': 'Open POs: Σ(late chase lines) == Σ(per-PO late value)', + 'a': round(late_val, 2), 'b': round(sku_late_val_ok, 2), + 'gap': round(late_val - sku_late_val_ok, 2), + 'ok': abs(late_val - sku_late_val_ok) <= 1.0}) + + # Vendor reliability: the per-vendor rollup partitions the PO rows exactly (counts AND value). + vr = vendor_reliability(t) + vn = sum(v['n_pos'] for v in vr['vendors']) + checks.append({'check': 'Vendor reliability: Σ(per-vendor POs) == PO rows pulled', + 'a': vn, 'b': vr['n_pos'], 'gap': vn - vr['n_pos'], + 'ok': vn == vr['n_pos']}) + vv = sum(v['value'] for v in vr['vendors']) + rv = sum(r['po_value'] for r in vr['rows']) + checks.append({'check': 'Vendor reliability: Σ(per-vendor PO value) == Σ(row PO value)', + 'a': round(vv, 2), 'b': round(rv, 2), 'gap': round(vv - rv, 2), + 'ok': abs(vv - rv) <= 1.0}) + return checks diff --git a/royalimports_os/modules/products.py b/royalimports_os/modules/products.py new file mode 100644 index 0000000000000000000000000000000000000000..b5b6f1435abd29d24aeea8498422793ee840a087 --- /dev/null +++ b/royalimports_os/modules/products.py @@ -0,0 +1,500 @@ +"""Products / SKU module — SKU health: YoY movers (risers & decliners), zombie SKUs +(catalog rot — sellable, formerly selling, now dead), new winners, coverage collapse +(SKUs losing customer breadth — the FFS-recovery early-warning signal), and velocity leaders. + +Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here. +Basket / co-purchase (MBA) is intentionally deferred — the existing client app computes it +runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the +project guardrails keep off the local PC. (See BACKLOG.) + +All line-level (sale.order.line), RI+FFS scope, excluded accounts removed — reusing sale_line_domain. +Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/ +window), so it's its own function the app calls lazily and caches. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from functools import lru_cache +import core.odoo as O +import core.periods as P +import modules.sales as sales_mod + +# SKU-health views are about real products — exclude service/delivery pseudo-SKUs +# (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby +# on the dot-path does not). Validation reconciles on this same non-service universe. +_NO_SVC = [('product_id.type', '!=', 'service')] + + +@lru_cache(maxsize=1) +def _code_map(): + """product_id → SKU code. Multiple product records can share a default_code (re-SKUing + /duplicates); keying by code merges them so a re-coded item doesn't read as a fake + decliner + fake riser. Products without a code fall back to a per-id key. Archived + products are INCLUDED — re-SKUing typically archives the old record and creates a new + one under the same code, and the old record still carries last-year sales.""" + prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'], + limit=50000) + return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") + for p in prods} + + +def _sku_rev(date_from, date_to, team_id=None): + """{sku_code: {'name','rev','qty','orders'}} over a window (services excluded, + duplicate product records merged by SKU code).""" + g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False) + codes = _code_map() + out = {} + for r in g: + pid = O.m2o_id(r.get('product_id')) + if not pid: + continue + key = codes.get(pid, f"pid:{pid}") + e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')), + 'rev': 0.0, 'qty': 0.0, 'orders': 0}) + e['rev'] += r.get('price_subtotal') or 0.0 + e['qty'] += r.get('product_uom_qty') or 0.0 + e['orders'] += r.get('__count') or 0 + return out + + +def yoy_movers(t=None, limit=20, team_id=None): + """Top SKU risers and decliners by YTD-vs-same-period-LY revenue change.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + rows = [] + for pid in set(this) | set(last): + tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')}) + lr = last.get(pid, {'rev': 0.0}) + name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or '' + rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'], + 'change': tr['rev'] - lr['rev']}) + risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit] + decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit] + return {'risers': risers, 'decliners': decliners} + + +def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None): + """Catalog rot: SKUs that sold materially last year but are ~dead this year.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + rows = [] + for pid, lr in last.items(): + if lr['rev'] < min_prior: + continue + tr = this.get(pid, {'rev': 0.0}) + if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior → not a zombie + continue + rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'], + 'lost': lr['rev'] - tr['rev']}) + rows.sort(key=lambda x: -x['lost']) + return rows[:limit] + + +def new_winners(t=None, limit=20, min_this=1500.0, team_id=None): + """SKUs that barely sold last year but are selling well this year.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + rows = [] + for pid, tr in this.items(): + if tr['rev'] < min_this: + continue + lr = last.get(pid, {'rev': 0.0}) + if lr['rev'] > 0.05 * tr['rev']: + continue + rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'], + 'gained': tr['rev'] - lr['rev']}) + rows.sort(key=lambda x: -x['gained']) + return rows[:limit] + + +def velocity_leaders(t=None, limit=25, team_id=None): + """Top SKUs by LTM unit velocity (units/month) and revenue.""" + t = t or P.today() + lf, lt = P.ltm(t) + sku = _sku_rev(lf, lt, team_id) + rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0, + 'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()] + rows.sort(key=lambda x: -x['units_ltm']) + return rows[:limit] + + +def _code_category(): + """code -> category name. Built from _code_map (pid->code) + the sales category map + (pid->category); the first record carrying a code sets that code's category.""" + codes = _code_map() + cats = sales_mod._product_cat() + out = {} + for pid, code in codes.items(): + if code not in out and pid in cats: + out[code] = cats[pid] + return out + + +def directory(t=None, team_id=None): + """Every SKU that sold this YTD or the same period last year, with the fields the SKU + drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the + customer directory — the full list (filtering happens in the UI), sorted by YTD revenue.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + code_cat = _code_category() + rows = [] + for code in set(this) | set(last): + tr = this.get(code, {}) + lr = last.get(code, {}) + rev_ytd = tr.get('rev', 0.0) + rev_ly = lr.get('rev', 0.0) + rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code, + 'category': code_cat.get(code, '(uncategorized)'), + 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly, + 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly), + 'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)}) + rows.sort(key=lambda r: -r['rev_ytd']) + return rows + + +def categories(t=None, team_id=None): + """Sorted distinct category names present in the SKU directory (for the drill-down filter).""" + return sorted({r['category'] for r in directory(t, team_id)}) + + +def _coverage(date_from, date_to, team_id=None): + """{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code. + (A customer buying two records sharing a code can count twice; duplicates are rare, and + code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)""" + g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC), + ['__count'], ['product_id', 'order_partner_id'], lazy=False) + codes = _code_map() + cov = {} + names = {} + for r in g: + pid = O.m2o_id(r.get('product_id')) + if not pid: + continue + key = codes.get(pid, f"pid:{pid}") + cov[key] = cov.get(key, 0) + 1 + names.setdefault(key, O.m2o_name(r.get('product_id'))) + return cov, names + + +def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None): + """SKUs that lost the most customer breadth YoY — early warning a SKU is dying even if + revenue hasn't fully cratered yet.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + cov_t, names_t = _coverage(yf, yt, team_id) + cov_l, names_l = _coverage(lf, lt, team_id) + rows = [] + for pid, lc in cov_l.items(): + if lc < min_prior_custs: + continue + tc = cov_t.get(pid, 0) + drop = lc - tc + if drop <= 0: + continue + rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '', + 'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop, + 'pct_drop': drop / lc * 100}) + rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop'])) + return rows[:limit] + + +# ====================================================================== SKU DRAWER (mirror of customer drawer) +def _sku_product_ids(code): + """All product.product ids sharing this SKU code (merged variants / archived records).""" + return [pid for pid, c in _code_map().items() if c == code] + + +def _sku_dom(pids, date_from, date_to, team_id=None): + return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC) + + +def _sku_name_category(pids): + rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id']) + name = rows[0]['name'] if rows else '(unknown)' + catmap = sales_mod._product_cat() + main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)') + return name, main + + +def _sku_buyers(pids, date_from, date_to, team_id=None): + """{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window.""" + g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False) + out = {} + for r in g: + pid = O.m2o_id(r.get('order_partner_id')) + if pid: + out[pid] = {'name': O.m2o_name(r.get('order_partner_id')), + 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0} + return out + + +def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None): + """KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code. + Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read.""" + t = t or P.today() + pids = _sku_product_ids(code) + if not pids: + return None + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + mf, mt = P.ltm(t) + name, category = _sku_name_category(pids) + + def s(df, dtt, field='price_subtotal'): + return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field) + rev_ytd, rev_ly = s(yf, yt), s(lf, lt) + qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty') + buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id)) + buyers_ly = len(_sku_buyers(pids, lf, lt, team_id)) + g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id), + ['price_subtotal:sum', 'margin:sum'], [], lazy=False) + line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0 + margin = (g[0].get('margin') if g else 0) or 0.0 + + # Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected + # on sale.order.line), in 2 reads instead of 26 point queries. + range_start = dt.date(t.year - 2, t.month, 1).isoformat() + lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id), + ['price_subtotal', 'order_id']) + oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')}) + odate = {} + for i in range(0, len(oids), 1000): + for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']): + if o.get('date_order'): + odate[o['id']] = str(o['date_order'])[:7] + mrev = {} + for line in lines: + ym = odate.get(O.m2o_id(line.get('order_id'))) + if ym: + mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0) + monthly = [] + for ym, start, end in P.month_starts(n_months, t): + y, m = int(ym[:4]) - 1, int(ym[5:7]) + monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0), + 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)}) + + allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id) + total = sum(v['rev'] for v in allsku.values()) or 1.0 + rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev'])) + if c == code), None) + return { + 'code': code, 'name': name, 'category': category, + 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly), + 'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly), + 'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly, + 'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin, + 'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100, + 'monthly': monthly, + } + + +def sku_buyer_bridge(code, t=None, team_id=None, top=12): + """Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list.""" + t = t or P.today() + pids = _sku_product_ids(code) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id) + tset, lset = set(this), set(last) + retained, new, churned = tset & lset, tset - lset, lset - tset + churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']} + for p in churned], key=lambda x: -x['ly_rev'])[:top] + return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)}, + 'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)}, + 'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)}, + 'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0, + 'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset), + 'this_total': sum(v['rev'] for v in this.values())} + + +def sku_concentration(code, t=None, team_id=None): + """Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM).""" + t = t or P.today() + mf, mt = P.ltm(t) + buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id) + revs = sorted([v['rev'] for v in buyers.values()], reverse=True) + total = sum(revs) or 1.0 + hhi = sum((r / total) ** 2 for r in revs) + return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0, + 'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0, + 'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0} + + +def sku_price_dispersion(code, t=None, team_id=None, cap=60): + """Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP + accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever.""" + t = t or P.today() + mf, mt = P.ltm(t) + g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False) + rows, tot_rev, tot_qty = [], 0.0, 0.0 + for r in g: + pid = O.m2o_id(r.get('order_partner_id')) + rev = r.get('price_subtotal') or 0.0 + qty = r.get('product_uom_qty') or 0.0 + if not pid or qty <= 0: + continue + rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')), + 'price': rev / qty, 'qty': qty, 'rev': rev}) + tot_rev += rev + tot_qty += qty + vwap = (tot_rev / tot_qty) if tot_qty else 0.0 + for r in rows: + r['recoverable'] = max(0.0, vwap - r['price']) * r['qty'] + rows.sort(key=lambda x: -x['recoverable']) + return {'vwap': vwap, 'n_buyers': len(rows), + 'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]} + + +def sku_top_buyers(code, t=None, team_id=None, top=15): + """Ranked buyers of this SKU (YTD) with YoY — clickable to open the customer drawer.""" + t = t or P.today() + pids = _sku_product_ids(code) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id) + rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'], + 'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()] + rows.sort(key=lambda x: -x['rev']) + return rows[:top] + + +_BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10–25k)', 10000.0), + ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)] + + +def sku_customer_analysis(code, t=None, team_id=None, top=15): + """WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL + spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend — who gets hurt + most if it stocks out / who to protect).""" + t = t or P.today() + lf, lt = P.ltm(t) + pids = _sku_product_ids(code) + empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0} + if not pids: + return empty + buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}} + if not buyers: + return empty + bpids = list(buyers) + g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)], + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')} + + def tier(rev): + for nm, lo in _BUYER_TIERS: + if rev >= lo: + return nm + return _BUYER_TIERS[-1][0] + seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS} + dep = [] + for p, v in buyers.items(): + ct = total.get(p, v['rev']) or v['rev'] + s = seg[tier(ct)] + s['buyers'] += 1 + s['sku_rev'] += v['rev'] + dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct, + 'dependency_pct': (v['rev'] / ct * 100) if ct else None}) + tot = sum(s['sku_rev'] for s in seg.values()) or 1.0 + segments = [] + for nm, _ in _BUYER_TIERS: + s = seg[nm] + s['rev_share'] = s['sku_rev'] / tot * 100 + s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0 + segments.append(s) + dep.sort(key=lambda x: -x['sku_rev']) + deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None] + return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers), + 'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0} + + +def sku_whitespace(code, t=None, team_id=None, top=15): + """Customers who buy this SKU's category but NOT this SKU — ranked prospect list.""" + import modules.customers as cust_mod + t = t or P.today() + pids = _sku_product_ids(code) + _name, category = _sku_name_category(pids) + cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set() + mf, mt = P.ltm(t) + prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id))) + if not prospects: + return {'category': category, 'rows': []} + g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)], + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']), + 'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')] + rows.sort(key=lambda x: -x['total_spend']) + return {'category': category, 'rows': rows[:top]} + + +def sku_drawer_bundle(code, t=None, team_id=None, allsku=None): + """The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate), + then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call).""" + detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku) + if detail is None: + return {'detail': None} + bridge, conc, price, buyers, white, ca = O.parallel([ + lambda: sku_buyer_bridge(code, t, team_id), + lambda: sku_concentration(code, t, team_id), + lambda: sku_price_dispersion(code, t, team_id), + lambda: sku_top_buyers(code, t, team_id), + lambda: sku_whitespace(code, t, team_id), + lambda: sku_customer_analysis(code, t, team_id), + ]) + return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price, + 'buyers': buyers, 'white': white, 'ca': ca} + + +def validate(t=None, team_id=None): + """Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU + selected) every check runs SCOPED to that BU, so the validation panel never reconciles + against — or exposes — the other BU's numbers. All checks here are BU-scopeable (no cross-BU + mirror), so team_id threads straight through.""" + t = t or P.today() + yf, yt = P.ytd(t) + checks = [] + sku = _sku_rev(yf, yt, team_id) + sku_sum = sum(v['rev'] for v in sku.values()) + line_total = O.sum_field('sale.order.line', + O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal') + checks.append({'check': 'SKU rev: Σ(per-SKU) == total line revenue, ex-services (YTD)', + 'a': round(sku_sum, 2), 'b': round(line_total, 2), + 'gap': round(sku_sum - line_total, 2), + 'ok': abs(sku_sum - line_total) <= 1.0}) + + m = yoy_movers(t, limit=10**9, team_id=team_id) + movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners']) + lf, lt = P.ytd_last_year(t) + last_total = O.sum_field('sale.order.line', + O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal') + checks.append({'check': 'SKU movers: Σ(Δ) == (YTD − LY) total', + 'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2), + 'gap': round(movers_sum - (line_total - last_total), 2), + 'ok': abs(movers_sum - (line_total - last_total)) <= 1.0}) + + # SKU drawer: buyer-bridge reconciles — retained + new buyer revenue == this-period SKU revenue + top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None + if top_code: + bb = sku_buyer_bridge(top_code, t, team_id=team_id) + recon = bb['retained']['rev'] + bb['new']['rev'] + checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue', + 'a': round(recon, 2), 'b': round(bb['this_total'], 2), + 'gap': round(recon - bb['this_total'], 2), + 'ok': abs(recon - bb['this_total']) <= 1.0}) + return checks diff --git a/royalimports_os/modules/reprice.py b/royalimports_os/modules/reprice.py new file mode 100644 index 0000000000000000000000000000000000000000..55925be809309769276b68d6ef0e2f6cd011aa55 --- /dev/null +++ b/royalimports_os/modules/reprice.py @@ -0,0 +1,409 @@ +"""Reprice lifecycle — the price-side discipline the pricing brief found missing +(.claude/wiki/research/pricing.md): rule hygiene BEFORE the reprice, realization tracking +AFTER it, and the between-waves views (pocket waterfall, peer price corridor). + +Everything reads sale.order.line + product.pricelist.item + product masters, read-only. +NO elasticity claims — rejected at wholesale data density (see the brief's anti-patterns). + +Windows: LTM for corridor/waterfall (matches pricecomp), all-channel sales for the DEAD-rule +flag (the Amazon channel-scope trap: a rule serving Amazon sales is not dead). +""" +import datetime as dt + +import core.odoo as O +import core.periods as P +import modules.pricecomp as pcomp +import modules.customers as cust_mod + +STALE_MONTHS = 24 +MIN_CORRIDOR_BUYERS = 8 +CORRIDOR_FLOOR_PCTL = 25 # floor = p25 of %-of-tier among the SKU's buyers +REALIZED_TARGET_PCT = 80.0 # an increase that landed <80% = worklist + + +def _chunk(ids, n=400): + ids = list(ids) + for i in range(0, len(ids), n): + yield ids[i:i + n] + + +def _all_channel_sold(lf, lt): + """{product_id: qty} confirmed sales, ALL channels (never flag an Amazon seller as dead).""" + out = {} + for g in O.read_group('sale.order.line', + [('state', 'in', ['sale', 'done']), ('product_id', '!=', False), + ('order_id.date_order', '>=', f'{lf} 00:00:00'), + ('order_id.date_order', '<=', f'{lt} 23:59:59')], + ['product_uom_qty:sum'], ['product_id'], lazy=False): + pid = O.m2o_id(g.get('product_id')) + if pid: + out[pid] = g.get('product_uom_qty') or 0.0 + return out + + +# ================================================================ 1. RULE HYGIENE +def rule_health(t=None): + """Every fixed pricelist rule flagged for the pre-reprice cleanup. One verb per flag: + STALE → reprice it · DEAD → archive it · BELOW-COST → raise it NOW (before the next + order books — the realized below-cost view only catches it after) · OVERLAP → resolve + precedence.""" + t = t or P.today() + lf, lt = P.ltm(t) + stale_cut = (t - dt.timedelta(days=STALE_MONTHS * 30)).isoformat() + today = t.isoformat() + + rules = O.search_read( + 'product.pricelist.item', + [('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product'])], + ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price', + 'min_quantity', 'date_start', 'date_end', 'write_date']) + + # resolve template rules to variants; pull cost + active per involved variant + tmpl_ids = {O.m2o_id(r['product_tmpl_id']) for r in rules + if r['applied_on'] == '1_product' and r.get('product_tmpl_id')} + var_ids = {O.m2o_id(r['product_id']) for r in rules + if r['applied_on'] == '0_product_variant' and r.get('product_id')} + by_tmpl, vinfo = {}, {} + for ch in _chunk(list(tmpl_ids) | set() if not tmpl_ids else list(tmpl_ids), 2000): + for p in O.search_read('product.product', + [('product_tmpl_id', 'in', ch), ('active', 'in', [True, False])], + ['product_tmpl_id', 'standard_price', 'active', + 'default_code']): + by_tmpl.setdefault(O.m2o_id(p['product_tmpl_id']), []).append(p) + vinfo[p['id']] = p + for ch in _chunk([v for v in var_ids if v not in vinfo], 2000): + for p in O.search_read('product.product', + [('id', 'in', ch), ('active', 'in', [True, False])], + ['product_tmpl_id', 'standard_price', 'active', + 'default_code']): + vinfo[p['id']] = p + + sold = _all_channel_sold(lf, lt) + + def _variants_of(r): + if r['applied_on'] == '0_product_variant': + v = vinfo.get(O.m2o_id(r.get('product_id'))) + return [v] if v else [] + return by_tmpl.get(O.m2o_id(r.get('product_tmpl_id')), []) + + # overlap detection: same (pricelist, target, qty break) with overlapping date windows + def _target_key(r): + return (O.m2o_id(r['pricelist_id']), r['applied_on'], + O.m2o_id(r.get('product_id')) or O.m2o_id(r.get('product_tmpl_id')), + r.get('min_quantity') or 0) + seen_keys = {} + for r in rules: + seen_keys.setdefault(_target_key(r), []).append(r) + + def _windows_overlap(a, b): + a0 = str(a.get('date_start') or '')[:10] or '0000' + a1 = str(a.get('date_end') or '')[:10] or '9999' + b0 = str(b.get('date_start') or '')[:10] or '0000' + b1 = str(b.get('date_end') or '')[:10] or '9999' + return a0 <= b1 and b0 <= a1 + + rows = [] + counts = {'stale': 0, 'dead': 0, 'below_cost': 0, 'overlap': 0} + below_cost_gap = 0.0 + for r in rules: + de = str(r.get('date_end') or '')[:10] + if de and de < today: + continue # already expired — not operating + variants = _variants_of(r) + v_sold = sum(sold.get(v['id'], 0.0) for v in variants) + any_active = any(v.get('active') for v in variants) + # cost basis: the CHEAPEST variant — flags only when price is below even that + costs = [v.get('standard_price') or 0.0 for v in variants + if (v.get('standard_price') or 0.0) > 0] + min_cost = min(costs) if costs else None + price = r.get('fixed_price') or 0.0 + code = next(((v.get('default_code') or '').strip() for v in variants + if (v.get('default_code') or '').strip()), '') + + flags = [] + if not variants or not any_active or (any_active and v_sold <= 0): + if not variants or not any_active: + flags.append(('dead', 'Archive rule — product archived/missing')) + elif v_sold <= 0: + flags.append(('dead', 'Archive rule — no sales in 12m (all channels)')) + if str(r.get('write_date') or '')[:10] < stale_cut and v_sold > 0: + flags.append(('stale', f'Reprice — untouched >{STALE_MONTHS}m, still selling')) + if min_cost is not None and price > 0 and price < min_cost and v_sold > 0: + flags.append(('below_cost', 'Raise NOW — tier below current unit cost')) + below_cost_gap += (min_cost - price) * v_sold + siblings = seen_keys.get(_target_key(r), []) + if len(siblings) > 1 and any(s is not r and _windows_overlap(r, s) + for s in siblings): + flags.append(('overlap', 'Resolve precedence — duplicate qty-break rule')) + if not flags: + continue + for kind, _v in flags: + counts[kind] += 1 + rows.append({ + 'rule_id': r['id'], 'pricelist': O.m2o_name(r['pricelist_id']), + 'sku': code, + 'product': O.m2o_name(r.get('product_id')) or O.m2o_name(r.get('product_tmpl_id')), + 'qty_break': r.get('min_quantity') or 0, 'price': price, + 'unit_cost': min_cost if min_cost is not None else '', + 'sold_12m': v_sold, 'last_touched': str(r.get('write_date') or '')[:10], + 'flags': ', '.join(k for k, _ in flags), + 'action': flags[0][1], + }) + rows.sort(key=lambda x: (0 if 'below_cost' in x['flags'] else 1, -(x['sold_12m'] or 0))) + return {'rows': rows, 'n_rules': len(rules), 'counts': counts, + 'below_cost_gap': below_cost_gap, 'window': (lf, lt)} + + +# ================================================================ 2. REALIZATION TRACKER +def realization_events(t=None, months_back=15, min_rules=15): + """Candidate reprice events = days on which many fixed rules were (re)written. Odoo keeps + no price history, so the event DATE is the anchor and the CURRENT rule price is the target.""" + t = t or P.today() + since = (t - dt.timedelta(days=months_back * 30)).isoformat() + rules = O.search_read('product.pricelist.item', + [('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product']), + ('write_date', '>=', since)], + ['write_date', 'product_id', 'product_tmpl_id']) + per_day = {} + for r in rules: + d = str(r.get('write_date') or '')[:10] + if d: + e = per_day.setdefault(d, {'n_rules': 0, 'targets': set()}) + e['n_rules'] += 1 + e['targets'].add(O.m2o_id(r.get('product_id')) + or O.m2o_id(r.get('product_tmpl_id'))) + out = [{'date': d, 'n_rules': v['n_rules'], 'n_products': len(v['targets'])} + for d, v in per_day.items() if v['n_rules'] >= min_rules] + out.sort(key=lambda x: x['date'], reverse=True) + return out + + +def _avg_price_by_product(prod_ids, a, b, team_id=None): + """{product_id: (avg unit price, qty)} over a window — chunked (large in-domains make + grouped reads echo the domain per group and MemoryError server-side).""" + out = {} + for ch in _chunk(prod_ids): + dom = O.sale_line_domain(a, b, team_id, extra=[('product_id', 'in', ch)]) + for g in O.read_group('sale.order.line', dom, + ['price_subtotal:sum', 'product_uom_qty:sum'], + ['product_id'], lazy=False): + pid = O.m2o_id(g.get('product_id')) + qty = g.get('product_uom_qty') or 0.0 + rev = g.get('price_subtotal') or 0.0 + if pid and qty > 0: + out[pid] = (rev / qty, qty) + return out + + +def realization(event_date, t=None, team_id=None, before_days=180, after_min_days=30): + """Did the increase STICK? For every product whose rule was written on `event_date`: + realized avg before (180d) vs after (event → today) vs the CURRENT rule target. + realization % = (after − before) / (target − before). <80% at scale = the worklist.""" + t = t or P.today() + ev = dt.date.fromisoformat(event_date) + if (t - ev).days < after_min_days: + return {'rows': [], 'note': f'event only {(t - ev).days}d old — too early to score', + 'event': event_date} + rules = O.search_read('product.pricelist.item', + [('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product']), + ('write_date', '>=', f'{event_date} 00:00:00'), + ('write_date', '<=', f'{event_date} 23:59:59')], + ['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price', + 'min_quantity']) + # base tier target per product (lowest qty break); template rules → variants + tmpl_ids = [O.m2o_id(r['product_tmpl_id']) for r in rules + if r['applied_on'] == '1_product' and r.get('product_tmpl_id')] + t2v = {} + for ch in _chunk(tmpl_ids, 2000): + for p in O.search_read('product.product', + [('product_tmpl_id', 'in', ch), ('active', 'in', [True, False])], + ['product_tmpl_id', 'default_code']): + t2v.setdefault(O.m2o_id(p['product_tmpl_id']), []).append(p) + target, codes = {}, {} + for r in sorted(rules, key=lambda x: x.get('min_quantity') or 0, reverse=True): + price = r.get('fixed_price') or 0.0 + if price <= 0: + continue + if r['applied_on'] == '0_product_variant' and r.get('product_id'): + target[O.m2o_id(r['product_id'])] = price # lowest break wins (reverse sort) + else: + for p in t2v.get(O.m2o_id(r.get('product_tmpl_id')), []): + target[p['id']] = price + codes[p['id']] = (p.get('default_code') or '').strip() + prods = list(target) + if not prods: + return {'rows': [], 'note': 'no fixed rules found on that date', 'event': event_date} + for ch in _chunk([p for p in prods if p not in codes], 2000): + for p in O.search_read('product.product', + [('id', 'in', ch), ('active', 'in', [True, False])], + ['default_code', 'name']): + codes[p['id']] = (p.get('default_code') or '').strip() + + b0 = (ev - dt.timedelta(days=before_days)).isoformat() + b1 = (ev - dt.timedelta(days=1)).isoformat() + before = _avg_price_by_product(prods, b0, b1, team_id) + after = _avg_price_by_product(prods, event_date, t.isoformat(), team_id) + + rows = [] + for pid in prods: + if pid not in before or pid not in after: + continue + p0, _q0 = before[pid] + p1, q1 = after[pid] + tgt = target[pid] + if tgt <= p0 * 1.005: + continue # not an increase vs realized base — skip + realized_pct = (p1 - p0) / (tgt - p0) * 100.0 + days_since = max((t - ev).days, 1) + run_rate = q1 / days_since * 365.0 + rows.append({'prod': pid, 'sku': codes.get(pid, f'#{pid}'), + 'before': p0, 'after': p1, 'target': tgt, + 'realized_pct': realized_pct, 'qty_after': q1, + 'missing': max(0.0, (tgt - p1)) * run_rate}) + rows.sort(key=lambda x: -x['missing']) + n = len(rows) + landed = sum(1 for r in rows if r['realized_pct'] >= REALIZED_TARGET_PCT) + return {'rows': rows, 'event': event_date, 'n_scored': n, + 'n_landed': landed, + 'landed_pct': (landed / n * 100.0) if n else None, + 'missing_total': sum(r['missing'] for r in rows), + 'note': None} + + +# ================================================================ 3. CORRIDOR + WATERFALL +def corridor(team_id=None, t=None, pre=None): + """Peer price corridor: within each SKU (≥MIN_CORRIDOR_BUYERS tier-matched buyers), the + floor = p{CORRIDOR_FLOOR_PCTL} of realized-%-of-tier. A customer below the floor AND below + 95% of tier is off-corridor even if 'on' their assigned tier — the tier ASSIGNMENT review. + Uplift = (floor − actual) × tier × qty.""" + mp = pre or pcomp.matched_pairs(team_id, t) + by_prod = {} + for p in mp['pairs']: + if p.get('tier'): + by_prod.setdefault(p['prod'], []).append(p) + flags, n_skus = [], 0 + for prod, plist in by_prod.items(): + if len(plist) < MIN_CORRIDOR_BUYERS: + continue + n_skus += 1 + ratios = sorted(x['unit'] / x['tier'] for x in plist) + floor = ratios[max(0, int(len(ratios) * CORRIDOR_FLOOR_PCTL / 100) - 1)] + for x in plist: + ratio = x['unit'] / x['tier'] + if ratio < floor and ratio < 0.95: + flags.append({'pid': x['pid'], 'customer': x['customer'], + 'sku': x['sku'], 'product': x['product'], + 'pct_of_tier': ratio * 100.0, 'floor_pct': floor * 100.0, + 'qty_ltm': x['qty'], + 'uplift': (floor - ratio) * x['tier'] * x['qty']}) + flags.sort(key=lambda x: -x['uplift']) + per_cust = {} + for f in flags: + e = per_cust.setdefault(f['pid'], {'pid': f['pid'], 'customer': f['customer'], + 'skus': 0, 'uplift': 0.0}) + e['skus'] += 1 + e['uplift'] += f['uplift'] + by_customer = sorted(per_cust.values(), key=lambda x: -x['uplift']) + return {'flags': flags, 'by_customer': by_customer, 'n_skus_scored': n_skus, + 'uplift_total': sum(f['uplift'] for f in flags), 'window': mp['window']} + + +def waterfall(team_id=None, t=None, pre=None): + """Honest pocket-price waterfall, LTM, tier-matched pairs only (coverage stated): + tier list value → − below-tier selling → + above-tier selling → invoice value + → − credit notes (company-level, not BU-taggable) → pocket. Stages with no data are + DROPPED, never imputed.""" + mp = pre or pcomp.matched_pairs(team_id, t) + lf, lt = mp['window'] + matched = [p for p in mp['pairs'] if p.get('tier')] + tier_value = sum(p['tier'] * p['qty'] for p in matched) + invoice_value = sum(p['rev'] for p in matched) + below = sum(max(0.0, (p['tier'] - p['unit'])) * p['qty'] for p in matched) + above = sum(max(0.0, (p['unit'] - p['tier'])) * p['qty'] for p in matched) + matched_rev_share = (invoice_value / + sum(p['rev'] for p in mp['pairs']) * 100.0) if mp['pairs'] else 0.0 + # credit notes, LTM, product lines (company-level — honesty note in the UI) + refunds = O.sum_field('account.move.line', + [('move_id.move_type', '=', 'out_refund'), + ('parent_state', '=', 'posted'), + ('move_id.invoice_date', '>=', lf), + ('move_id.invoice_date', '<=', lt), + ('product_id', '!=', False)], 'price_subtotal') + stages = [{'stage': 'Tier list value', 'amount': tier_value, 'kind': 'base'}] + if below > 0: + stages.append({'stage': 'Below-tier selling', 'amount': -below, 'kind': 'leak'}) + if above > 0: + stages.append({'stage': 'Above-tier selling', 'amount': above, 'kind': 'gain'}) + stages.append({'stage': 'Invoice value', 'amount': invoice_value, 'kind': 'subtotal'}) + if refunds: + stages.append({'stage': 'Credit notes (company)', 'amount': -refunds, 'kind': 'leak'}) + stages.append({'stage': 'Pocket value', 'amount': invoice_value - (refunds or 0.0), + 'kind': 'total'}) + return {'stages': stages, 'tier_value': tier_value, 'invoice_value': invoice_value, + 'below': below, 'above': above, 'refunds': refunds or 0.0, + 'matched_rev_share': matched_rev_share, 'window': (lf, lt)} + + +def below_tier_detail(team_id=None, t=None, pre=None): + """The waterfall's below-tier bar, drilled: every pair sold under its tier (the FULL gap, + not just the <97% compliance threshold), rolled up by customer / SKU / agent. Each rollup's + total reproduces the waterfall stage to the cent (rule 8b — no unverifiable aggregates).""" + mp = pre or pcomp.matched_pairs(team_id, t) + below = [p for p in mp['pairs'] if p.get('tier') and p['unit'] < p['tier']] + attrs = cust_mod._partner_attrs(list({p['pid'] for p in below})) + by_cust, by_sku, by_agent = {}, {}, {} + for p in below: + gap = (p['tier'] - p['unit']) * p['qty'] + c = by_cust.setdefault(p['pid'], {'pid': p['pid'], 'customer': p['customer'], + 'skus': 0, 'gap': 0.0, 'rev': 0.0}) + c['skus'] += 1 + c['gap'] += gap + c['rev'] += p['rev'] + s = by_sku.setdefault(p['prod'], {'code': p['sku'] or f"#{p['prod']}", + 'product': p['product'], 'customers': 0, + 'gap': 0.0, 'rev': 0.0}) + s['customers'] += 1 + s['gap'] += gap + s['rev'] += p['rev'] + a_name = (attrs.get(p['pid']) or {}).get('agent') or '(none)' + a = by_agent.setdefault(a_name, {'agent': a_name, 'pairs': 0, 'gap': 0.0}) + a['pairs'] += 1 + a['gap'] += gap + out = { + 'by_customer': sorted(by_cust.values(), key=lambda x: -x['gap']), + 'by_sku': sorted(by_sku.values(), key=lambda x: -x['gap']), + 'by_agent': sorted(by_agent.values(), key=lambda x: -x['gap']), + 'total': sum((p['tier'] - p['unit']) * p['qty'] for p in below), + 'n_pairs': len(below), + } + return out + + +# ================================================================ VALIDATION +def validate(t=None, team_id=None, pre_mp=None): + """(1) rule universe ties search_count; (2) waterfall arithmetic closes exactly: + tier − below + above == invoice (same pairs, two computation paths).""" + n = O.get_odoo().search_count('product.pricelist.item', + [('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product'])]) + rh = rule_health(t) + checks = [{'check': 'pricelist rules — pull complete', + 'a': rh['n_rules'], 'b': n, 'gap': rh['n_rules'] - n, + 'ok': rh['n_rules'] == n}] + wf = waterfall(team_id, t, pre=pre_mp) + lhs = wf['tier_value'] - wf['below'] + wf['above'] + checks.append({'check': 'waterfall closes: tier − below + above == invoice', + 'a': round(lhs, 2), 'b': round(wf['invoice_value'], 2), + 'gap': round(lhs - wf['invoice_value'], 2), + 'ok': abs(lhs - wf['invoice_value']) < 1.0}) + bd = below_tier_detail(team_id, t, pre=pre_mp) + for key in ('by_customer', 'by_sku', 'by_agent'): + s = sum(r['gap'] for r in bd[key]) + checks.append({'check': f'below-tier drill {key} == the waterfall bar', + 'a': round(s, 2), 'b': round(wf['below'], 2), + 'gap': round(s - wf['below'], 2), + 'ok': abs(s - wf['below']) < 1.0}) + return checks \ No newline at end of file diff --git a/royalimports_os/modules/returns.py b/royalimports_os/modules/returns.py new file mode 100644 index 0000000000000000000000000000000000000000..0991b506809f1bb6c91d0bfa33b859d4678e63d2 --- /dev/null +++ b/royalimports_os/modules/returns.py @@ -0,0 +1,238 @@ +"""Returns module — the credit-note lens nobody reads: 14% of billing documents are refunds. + +Governing question: WHERE DO RETURNS CONCENTRATE — WHICH SKUs (quality signal), WHICH +CUSTOMERS (behavior signal), WHICH CATEGORIES (product-line signal), WHICH SUPPLIERS +(sourcing-quality signal) AND WHICH AGENTS (book-behavior signal) — AND IS THE RATE MOVING? +Odoo has no return-reason field, so the concentration IS the diagnostic. Rates are always shown +next to raw $ — a big seller with average rate is noise; a small line with 4x the company rate +is the finding. + +Supplier attribution: curated procurement map (default_code) first, else the DOMINANT vendor +from confirmed PO history (most units bought), else '(no supplier data)'. Agent attribution: +the customer's res.partner agent — credit-note team_id is NOT trustworthy. + +Company-level (credit notes carry team_id=1 for everything). READ-ONLY. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P +import modules.customers as cust_mod +import modules.procurement as proc_mod + +_EX_MOVE = [] # posted customer docs only + + +def _mdom(move_type, a, b): + return [('move_type', '=', move_type), ('state', '=', 'posted'), + ('invoice_date', '>=', a), ('invoice_date', '<=', b)] + + +def _ldom(move_type, a, b): + return [('move_id.move_type', '=', move_type), ('parent_state', '=', 'posted'), + ('move_id.invoice_date', '>=', a), ('move_id.invoice_date', '<=', b), + ('product_id', '!=', False)] + + +def build(t=None): + t = t or P.today() + d12 = (t - dt.timedelta(days=365)).isoformat() + d13m = (t.replace(day=1) - dt.timedelta(days=380)).replace(day=1).isoformat() + ti = t.isoformat() + + # ---- headline + monthly (from the MOVES — count + untaxed value) ----------------------- + def _monthly(mt): + out = {} + for g in O.read_group('account.move', _mdom(mt, d13m, ti), + ['amount_untaxed:sum'], ['invoice_date:month'], lazy=False): + k = g.get('invoice_date:month') + out[str(k)] = {'value': g.get('amount_untaxed') or 0.0, 'n': g.get('__count') or 0} + return out + ref_m, inv_m = _monthly('out_refund'), _monthly('out_invoice') + months = sorted(set(ref_m) | set(inv_m), + key=lambda m: dt.datetime.strptime(m, '%B %Y')) + monthly = [] + for m in months: + r, i = ref_m.get(m, {'value': 0, 'n': 0}), inv_m.get(m, {'value': 0, 'n': 0}) + monthly.append({'month': m, 'refund_value': r['value'], 'n_refunds': r['n'], + 'invoice_value': i['value'], 'n_invoices': i['n'], + 'rate_pct': (r['value'] / i['value'] * 100) if i['value'] else None}) + + # ---- by customer (moves, 12m) ---------------------------------------------------------- + def _by_partner(mt): + out = {} + for g in O.read_group('account.move', _mdom(mt, d12, ti), + ['amount_untaxed:sum'], ['partner_id'], lazy=False): + pid = O.m2o_id(g.get('partner_id')) + if pid: + out[pid] = {'name': O.m2o_name(g.get('partner_id')), + 'value': g.get('amount_untaxed') or 0.0, 'n': g.get('__count') or 0} + return out + ref_c, inv_c = _by_partner('out_refund'), _by_partner('out_invoice') + by_customer = [] + for pid, r in ref_c.items(): + inv = inv_c.get(pid, {'value': 0.0, 'n': 0}) + by_customer.append({'pid': pid, 'customer': r['name'], 'ret_value': r['value'], + 'n_refunds': r['n'], 'invoiced': inv['value'], + 'rate_pct': (r['value'] / inv['value'] * 100) if inv['value'] else None}) + by_customer.sort(key=lambda x: -x['ret_value']) + + # ---- by SKU (refund LINES vs invoice LINES, 12m) ---------------------------------------- + def _by_product(mt): + out = {} + for g in O.read_group('account.move.line', _ldom(mt, d12, ti), + ['price_subtotal:sum', 'quantity:sum'], ['product_id'], lazy=False): + pid = O.m2o_id(g.get('product_id')) + if pid: + out[pid] = {'value': g.get('price_subtotal') or 0.0, + 'qty': g.get('quantity') or 0.0} + return out + ref_p, inv_p = _by_product('out_refund'), _by_product('out_invoice') + # meta for the UNION of refunded + sold products (category/supplier denominators need the + # invoice side too) + all_pids = list(set(ref_p) | set(inv_p)) + meta = {} + for i in range(0, len(all_pids), 5000): + for p in O.search_read('product.product', + [('id', 'in', all_pids[i:i + 5000]), + ('active', 'in', [True, False])], + ['default_code', 'name', 'categ_id']): + meta[p['id']] = p + by_sku = [] + for pid, r in ref_p.items(): + inv = inv_p.get(pid, {'value': 0.0, 'qty': 0.0}) + m = meta.get(pid, {}) + by_sku.append({'pid': pid, 'code': (m.get('default_code') or '').strip() or f'#{pid}', + 'product': m.get('name') or '', + 'ret_value': r['value'], 'ret_units': r['qty'], + 'sold_value': inv['value'], 'sold_units': inv['qty'], + 'rate_pct': (r['value'] / inv['value'] * 100) if inv['value'] else None}) + by_sku.sort(key=lambda x: -x['ret_value']) + + # ---- by CATEGORY (product-line signal: which kinds of products come back) --------------- + def _cat_of(pid): + return O.m2o_name((meta.get(pid) or {}).get('categ_id')) or '(none)' + cat = {} + for pid, r in ref_p.items(): + e = cat.setdefault(_cat_of(pid), {'ret_value': 0.0, 'ret_units': 0.0, + 'sold_value': 0.0, 'n_skus': 0}) + e['ret_value'] += r['value'] + e['ret_units'] += r['qty'] + e['n_skus'] += 1 + for pid, r in inv_p.items(): + e = cat.setdefault(_cat_of(pid), {'ret_value': 0.0, 'ret_units': 0.0, + 'sold_value': 0.0, 'n_skus': 0}) + e['sold_value'] += r['value'] + by_category = [{'category': k, **v, + 'rate_pct': (v['ret_value'] / v['sold_value'] * 100) + if v['sold_value'] else None} + for k, v in cat.items() if v['ret_value'] > 0] + by_category.sort(key=lambda x: -x['ret_value']) + + # ---- by SUPPLIER (sourcing-quality signal) ---------------------------------------------- + # dominant vendor per product from confirmed PO history (most units bought), one grouped read + po_vendor, _best_q = {}, {} + try: + for g in O.read_group('purchase.order.line', + [('order_id.state', 'in', ('purchase', 'done')), + ('product_id', '!=', False)], + ['product_qty:sum'], ['product_id', 'partner_id'], lazy=False): + pid = O.m2o_id(g.get('product_id')) + q = g.get('product_qty') or 0.0 + v = O.m2o_name(g.get('partner_id')) + if pid and v and q > _best_q.get(pid, 0.0): + _best_q[pid] = q + po_vendor[pid] = v + except Exception: + pass + sup_map = proc_mod.suppliers() + + def _vendor_of(pid): + code = ((meta.get(pid) or {}).get('default_code') or '').strip() + cur = sup_map.get(code) or {} + return (cur.get('vendor') or '').strip() or po_vendor.get(pid) or '(no supplier data)' + sup = {} + for pid, r in ref_p.items(): + e = sup.setdefault(_vendor_of(pid), {'ret_value': 0.0, 'ret_units': 0.0, + 'sold_value': 0.0, 'n_skus': 0}) + e['ret_value'] += r['value'] + e['ret_units'] += r['qty'] + e['n_skus'] += 1 + for pid, r in inv_p.items(): + e = sup.setdefault(_vendor_of(pid), {'ret_value': 0.0, 'ret_units': 0.0, + 'sold_value': 0.0, 'n_skus': 0}) + e['sold_value'] += r['value'] + by_supplier = [{'supplier': k, **v, + 'rate_pct': (v['ret_value'] / v['sold_value'] * 100) + if v['sold_value'] else None} + for k, v in sup.items() if v['ret_value'] > 0] + by_supplier.sort(key=lambda x: -x['ret_value']) + + # ---- by AGENT (book-behavior signal; agent = the customer's res.partner agent, NOT the + # credit-note team_id). Denominator = the agent's WHOLE invoiced book, not just refunders. + attrs = cust_mod._partner_attrs(list(set(ref_c) | set(inv_c))) + + def _agent_of(pid): + return (attrs.get(pid) or {}).get('agent') or '(none)' + ag = {} + for pid, r in ref_c.items(): + e = ag.setdefault(_agent_of(pid), {'ret_value': 0.0, 'n_refunds': 0, 'invoiced': 0.0, + 'top_customer': '', 'top_value': 0.0, 'top_pid': None}) + e['ret_value'] += r['value'] + e['n_refunds'] += r['n'] + if r['value'] > e['top_value']: + e['top_value'] = r['value'] + e['top_customer'] = r['name'] + e['top_pid'] = pid + for pid, r in inv_c.items(): + e = ag.setdefault(_agent_of(pid), {'ret_value': 0.0, 'n_refunds': 0, 'invoiced': 0.0, + 'top_customer': '', 'top_value': 0.0, 'top_pid': None}) + e['invoiced'] += r['value'] + by_agent = [{'agent': k, **v, + 'rate_pct': (v['ret_value'] / v['invoiced'] * 100) if v['invoiced'] else None} + for k, v in ag.items() if v['ret_value'] > 0] + by_agent.sort(key=lambda x: -x['ret_value']) + + # ---- headline --------------------------------------------------------------------------- + n_ref = sum(r['n'] for r in ref_c.values()) + ref_val = sum(r['value'] for r in ref_c.values()) + n_inv = sum(r['n'] for r in inv_c.values()) + inv_val = sum(r['value'] for r in inv_c.values()) + top10 = sum(r['ret_value'] for r in by_sku[:10]) + return {'monthly': monthly, 'by_customer': by_customer, 'by_sku': by_sku, + 'by_category': by_category, 'by_supplier': by_supplier, 'by_agent': by_agent, + 'n_refunds': n_ref, 'refund_value': ref_val, + 'n_invoices': n_inv, 'invoice_value': inv_val, + 'doc_rate_pct': (n_ref / n_inv * 100) if n_inv else None, + 'value_rate_pct': (ref_val / inv_val * 100) if inv_val else None, + 'top10_share_pct': (top10 / ref_val * 100) if ref_val else None, + 'window': (d12, ti)} + + +def validate(t=None, team_id=None, pre=None): + t = t or P.today() + b = pre or build(t) + d12, ti = b['window'] + checks = [] + srv_val = O.sum_field('account.move', _mdom('out_refund', d12, ti), 'amount_untaxed') + a = sum(r['ret_value'] for r in b['by_customer']) + checks.append({'check': 'Returns: Σ(per-customer refunds) == server Σ(credit-note untaxed)', + 'a': round(a, 2), 'b': round(srv_val, 2), 'gap': round(a - srv_val, 2), + 'ok': abs(a - srv_val) <= max(1.0, abs(srv_val) * 0.001)}) + srv_line = O.sum_field('account.move.line', _ldom('out_refund', d12, ti), 'price_subtotal') + a2 = sum(r['ret_value'] for r in b['by_sku']) + checks.append({'check': 'Returns: Σ(per-SKU refund lines) == server Σ(refund product lines)', + 'a': round(a2, 2), 'b': round(srv_line, 2), 'gap': round(a2 - srv_line, 2), + 'ok': abs(a2 - srv_line) <= max(1.0, abs(srv_line) * 0.001)}) + # the three new rollups are re-groupings of the SAME universes — they must tie exactly + for key, base, label in (('by_category', a2, 'per-SKU lines'), + ('by_supplier', a2, 'per-SKU lines'), + ('by_agent', sum(r['ret_value'] for r in b['by_customer']), + 'per-customer refunds')): + s = sum(r['ret_value'] for r in b[key]) + checks.append({'check': f'Returns: Σ({key}) == Σ({label})', + 'a': round(s, 2), 'b': round(base, 2), 'gap': round(s - base, 2), + 'ok': abs(s - base) <= 1.0}) + return checks diff --git a/royalimports_os/modules/sales.py b/royalimports_os/modules/sales.py new file mode 100644 index 0000000000000000000000000000000000000000..a5f4e647c4187106d35ce877d957c0f3f9eae345 --- /dev/null +++ b/royalimports_os/modules/sales.py @@ -0,0 +1,761 @@ +"""Sales module — revenue, YoY (same-period), seasonality, rep/customer/SKU breakdowns. + +Scope: confirmed orders (state sale/done) on Fisch+Royal teams, excluded accounts removed. +Order-level metrics come from sale.order (amount_untaxed); SKU-level from +sale.order.line (price_subtotal). Each public metric has a paired validate_* that +reconciles against an independent Odoo aggregate. +""" +import sys +from functools import lru_cache +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # royalimports_os/ +import core.odoo as O +import core.periods as P +import modules.inventory as inv_mod # reuse the main-category resolver + + +def order_domain(date_from=None, date_to=None, team_id=None, partner_ids=None): + """Confirmed sale.order domain (RI+FFS scope, house accounts excluded). partner_ids (a + collection, possibly empty) restricts to those customers — the Customer-module Agent filter + rides on this; None = no partner restriction (an empty collection matches no orders).""" + dom = [('state', 'in', ['sale', 'done']), ('team_id', 'in', TEAMS(team_id))] + if date_from: + dom.append(('date_order', '>=', f'{date_from} 00:00:00')) + if date_to: + dom.append(('date_order', '<=', f'{date_to} 23:59:59')) + ex = O.excluded_partner_ids() + if ex: + dom.append(('partner_id', 'not in', list(ex))) + if partner_ids is not None: + dom.append(('partner_id', 'in', list(partner_ids))) + if O.doc_mode() == 'invoice': + dom.append(('invoice_status', '=', 'invoiced')) + return dom + + +def TEAMS(team_id): + return [team_id] if team_id is not None else O.TEAM_IDS + + +# ---- order-level primitives: STORE-backed (OM-2 retrofit 2026-07-12) with LIVE fallback ------ +# headline/scorecard/trends make ~50 of these per build; the store serves them in milliseconds +# (kept minutes-fresh by the app's auto-sync). The SQL mirrors order_domain() EXACTLY — same +# states, teams, exclusions, date bounds and the Orders/Invoiced basis (invoice_status is +# synced). validate() stays on live Odoo = the standing parity proof. Any store problem falls +# back to the live reads. +USE_STORE = True + + +def _order_where_store(date_from, date_to, team_id): + """(where_sql, params) mirroring order_domain() for the store's sale_order table.""" + params, w = [], ["state IN ('sale','done')"] + teams = TEAMS(team_id) + w.append("team_id IN (" + ",".join("?" * len(teams)) + ")") + params += list(teams) + if date_from: + w.append("CAST(date_order AS TIMESTAMP) >= CAST(? AS TIMESTAMP)") + params.append(f"{date_from} 00:00:00") + if date_to: + w.append("CAST(date_order AS TIMESTAMP) <= CAST(? AS TIMESTAMP)") + params.append(f"{date_to} 23:59:59") + ex = O.excluded_partner_ids() + if ex: + w.append("partner_id NOT IN (" + ",".join("?" * len(ex)) + ")") + params += list(ex) + if O.doc_mode() == 'invoice': + w.append("invoice_status = 'invoiced'") + return " AND ".join(w), params + + +def _order_agg_store(expr, date_from, date_to, team_id): + import harness.datastore as DS + where, params = _order_where_store(date_from, date_to, team_id) + r = DS.ro_con().execute( + f"SELECT {expr} FROM sale_order WHERE {where}", params).fetchone() + return r[0] or 0 + + +def _order_partner_groups_store(date_from, date_to, team_id): + """[(partner_id, n_orders, revenue)] — feeds cadence + concentration.""" + import harness.datastore as DS + where, params = _order_where_store(date_from, date_to, team_id) + return DS.ro_con().execute( + f"SELECT partner_id, count(*), sum(amount_untaxed) FROM sale_order WHERE {where} " + "GROUP BY 1", params).fetchall() + + +def _orev(date_from, date_to, team_id=None): + if USE_STORE: + try: + return float(_order_agg_store('sum(amount_untaxed)', date_from, date_to, team_id)) + except Exception: + pass + return O.sum_field('sale.order', order_domain(date_from, date_to, team_id), 'amount_untaxed') + + +def _orders(date_from, date_to, team_id=None): + if USE_STORE: + try: + return int(_order_agg_store('count(*)', date_from, date_to, team_id)) + except Exception: + pass + return O.get_odoo().search_count('sale.order', order_domain(date_from, date_to, team_id)) + + +def _custs(date_from, date_to, team_id=None): + if USE_STORE: + try: + return int(_order_agg_store('count(DISTINCT partner_id)', + date_from, date_to, team_id)) + except Exception: + pass + return O.distinct_count('sale.order', order_domain(date_from, date_to, team_id), 'partner_id') + + +# ---------------------------------------------------------------- headline +def headline(t=None, team_id=None): + """Headline scoped to a DBA (team_id) or consolidated (None). by_team always shows both.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + rev = _orev(yf, yt, team_id) + rev_ly = _orev(lf, lt, team_id) + orders = _orders(yf, yt, team_id) + custs = _custs(yf, yt, team_id) + out = { + 'as_of': yt, + 'ytd_revenue': rev, + 'ytd_revenue_ly': rev_ly, + 'yoy_pct': P.yoy_pct(rev, rev_ly), + 'ytd_orders': orders, + 'ytd_customers': custs, + 'aov': (rev / orders) if orders else 0.0, + 'by_team': {}, + } + for tid in O.TEAM_IDS: + out['by_team'][O.TEAM_NAMES[tid]] = { + 'ytd': _orev(yf, yt, tid), + 'ytd_ly': _orev(lf, lt, tid), + } + return out + + +# ---------------------------------------------------------------- period scorecard +# (label, key, weekday-aligned-LY?) — Today/WTD compare to 52 weeks ago (same weekday); the +# month/quarter/year periods compare to the same calendar window last year. +_PERIODS = [('Today', 'today', True), ('Week to date', 'wtd', True), ('Month to date', 'mtd', False), + ('Quarter to date', 'qtd', False), ('Year to date', 'ytd', False)] + + +def _period_window(key, t): + import datetime as dt + if key == 'today': + return P._d(t), P._d(t) + return {'wtd': P.wtd, 'mtd': P.mtd, 'qtd': P.qtd, 'ytd': P.ytd}[key](t) + + +def period_scorecard(t=None, team_id=None): + """The headline period scorecard: revenue (+ YoY same-period), orders, AOV and active customers + for Today / WTD / MTD / QTD / YTD. Each entry carries its date window so the UI can make every + number click through to the decomposition drawer.""" + t = t or P.today() + out = [] + for label, key, wk in _PERIODS: + f, tt = _period_window(key, t) + cf, ct = P.shift_year(f, tt, weeks=wk) + rev = _orev(f, tt, team_id) + rev_ly = _orev(cf, ct, team_id) + orders = _orders(f, tt, team_id) + out.append({ + 'key': key, 'label': label, 'date_from': f, 'date_to': tt, + 'cmp_from': cf, 'cmp_to': ct, + 'revenue': rev, 'revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly), + 'orders': orders, 'customers': _custs(f, tt, team_id), + 'aov': (rev / orders) if orders else 0.0, + }) + return out + + +# ---------------------------------------------------------------- seasonality / trend +def weekly_trend(n_weeks=13, t=None, team_id=None): + """Per-week revenue for the last n_weeks (Mon–Sun) vs the same week 52 weeks earlier (weekday- + aligned YoY). Each row carries start/end so a clicked week decomposes to its exact window.""" + t = t or P.today() + rows = [] + for label, start, end in P.week_starts(n_weeks, t): + this = _orev(start, end, team_id) + cf, ct = P.shift_year(start, end, weeks=True) + last = _orev(cf, ct, team_id) + rows.append({'week': label, 'start': start, 'end': end, 'cmp_from': cf, 'cmp_to': ct, + 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) + return rows + + +def monthly_trend(n=13, t=None, team_id=None): + """Per-month revenue for the last n months + same month one year earlier (YoY).""" + t = t or P.today() + rows = [] + for ym, start, end in P.month_starts(n, t): + this = _orev(start, end, team_id) + # same month last year + y, m = int(ym[:4]) - 1, int(ym[5:7]) + import datetime as dt + ly_start = dt.date(y, m, 1).isoformat() + ly_end = (dt.date(y + (m // 12), (m % 12) + 1, 1) - dt.timedelta(days=1)).isoformat() + last = _orev(ly_start, ly_end, team_id) + rows.append({'month': ym, 'start': start, 'end': end, 'cmp_from': ly_start, 'cmp_to': ly_end, + 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) + return rows + + +# ---------------------------------------------------------------- breakdowns +def by_team(t=None): + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + return [{'team': O.TEAM_NAMES[tid], 'ytd': _orev(yf, yt, tid), + 'ytd_ly': _orev(lf, lt, tid), + 'yoy_pct': P.yoy_pct(_orev(yf, yt, tid), _orev(lf, lt, tid))} + for tid in O.TEAM_IDS] + + +def by_rep(t=None, limit=25, team_id=None): + t = t or P.today() + yf, yt = P.ytd(t) + g = O.read_group('sale.order', order_domain(yf, yt, team_id), + ['amount_untaxed:sum'], ['user_id'], lazy=False) + rows = [{'rep': O.m2o_name(r.get('user_id')) or '(none)', 'uid': O.m2o_id(r.get('user_id')), + 'revenue': r.get('amount_untaxed') or 0.0, + 'orders': r.get('__count') or r.get('user_id_count') or 0} + for r in g] + rows.sort(key=lambda x: -x['revenue']) + return rows[:limit] + + +def category_product_ids(category): + """The product ids belonging to a main category — for decomposing a category number.""" + return [pid for pid, c in _product_cat().items() if c == category] + + +def _top_customers_store(yf, yt, team_id, limit): + import harness.datastore as DS + params, w = [], ["state IN ('sale','done')"] + teams = TEAMS(team_id) + w.append("team_id IN (" + ",".join("?" * len(teams)) + ")") + params += list(teams) + w.append("CAST(date_order AS TIMESTAMP) >= CAST(? AS TIMESTAMP)") + params.append(f"{yf} 00:00:00") + w.append("CAST(date_order AS TIMESTAMP) <= CAST(? AS TIMESTAMP)") + params.append(f"{yt} 23:59:59") + ex = O.excluded_partner_ids() + if ex: + w.append("partner_id NOT IN (" + ",".join("?" * len(ex)) + ")") + params += list(ex) + if O.doc_mode() == 'invoice': + w.append("invoice_status = 'invoiced'") + rows = DS.ro_con().execute( + "SELECT o.partner_id, coalesce(p.name, '#' || o.partner_id), " + "sum(o.amount_untaxed), count(*) " + "FROM sale_order o LEFT JOIN res_partner p ON p.id = o.partner_id " + "WHERE " + " AND ".join(w) + " GROUP BY 1, 2 ORDER BY 3 DESC LIMIT ?", + params + [int(limit)]).fetchall() + return [{'pid': r[0], 'customer': r[1], 'revenue': r[2] or 0.0, 'orders': r[3]} + for r in rows if r[0]] + + +def top_customers(t=None, limit=25, team_id=None): + t = t or P.today() + yf, yt = P.ytd(t) + if USE_STORE: + try: + return _top_customers_store(yf, yt, team_id, limit) + except Exception: + pass + g = O.read_group('sale.order', order_domain(yf, yt, team_id), + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + rows = [{'pid': O.m2o_id(r.get('partner_id')), + 'customer': O.m2o_name(r.get('partner_id')), + 'revenue': r.get('amount_untaxed') or 0.0, + 'orders': r.get('__count') or 0} + for r in g if r.get('partner_id')] + rows.sort(key=lambda x: -x['revenue']) + return rows[:limit] + + +def _top_skus_store(yf, yt, team_id, limit): + """Mirrors sale_line_domain: confirmed states, order-side teams/dates/invoice-mode, + line-side partner exclusion, product_id set. Names + SKU codes join in-store (the live + path needs an extra search_read for codes).""" + import harness.datastore as DS + params, w = [], ["o.state IN ('sale','done')", "l.product_id IS NOT NULL"] + teams = TEAMS(team_id) + w.append("o.team_id IN (" + ",".join("?" * len(teams)) + ")") + params += list(teams) + w.append("CAST(o.date_order AS TIMESTAMP) >= CAST(? AS TIMESTAMP)") + params.append(f"{yf} 00:00:00") + w.append("CAST(o.date_order AS TIMESTAMP) <= CAST(? AS TIMESTAMP)") + params.append(f"{yt} 23:59:59") + ex = O.excluded_partner_ids() + if ex: + w.append("l.order_partner_id NOT IN (" + ",".join("?" * len(ex)) + ")") + params += list(ex) + if O.doc_mode() == 'invoice': + w.append("o.invoice_status = 'invoiced'") + rows = DS.ro_con().execute( + "SELECT l.product_id, coalesce(p.name, '#' || l.product_id), p.default_code, " + "sum(l.price_subtotal), sum(l.product_uom_qty), sum(l.margin), count(*) " + "FROM sale_order_line l JOIN sale_order o ON o.id = l.order_id " + "LEFT JOIN product_product p ON p.id = l.product_id " + "WHERE " + " AND ".join(w) + " GROUP BY 1, 2, 3 ORDER BY 4 DESC LIMIT ?", + params + [int(limit)]).fetchall() + return [_sku_profit({'product': r[1], 'pid': r[0], + 'code': (str(r[2]).strip() if r[2] else None), + 'revenue': r[3] or 0.0, 'qty': r[4] or 0.0, + 'margin': r[5] or 0.0, 'lines': r[6]}) + for r in rows if r[0]] + + +def top_skus(t=None, limit=25, team_id=None): + t = t or P.today() + yf, yt = P.ytd(t) + if USE_STORE: + try: + return _top_skus_store(yf, yt, team_id, limit) + except Exception: + pass + g = O.read_group('sale.order.line', O.sale_line_domain(yf, yt, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], ['product_id'], lazy=False) + rows = [_sku_profit({'product': O.m2o_name(r.get('product_id')), 'pid': O.m2o_id(r.get('product_id')), + 'revenue': r.get('price_subtotal') or 0.0, + 'qty': r.get('product_uom_qty') or 0.0, + 'margin': r.get('margin') or 0.0, 'lines': r.get('__count') or 0}) + for r in g if r.get('product_id')] + rows.sort(key=lambda x: -x['revenue']) + rows = rows[:limit] + if rows: # attach SKU code so the UI can deep-link each to its SKU drawer + codemap = {} + for pr in O.search_read('product.product', [('id', 'in', [r['pid'] for r in rows])], ['default_code']): + codemap[pr['id']] = str(pr['default_code']).strip() if pr.get('default_code') else None + for r in rows: + r['code'] = codemap.get(r['pid']) + return rows + + +def _sku_profit(s): + """Attach profit/order + margin% to a SKU row (uses the Margin module's `margin` sum). `lines` + (order lines) ~= the number of orders containing the SKU (one line per SKU per order in + practice), so profit/order = total margin ÷ lines.""" + lines = s.get('lines') or 0 + s['profit_per_order'] = (s['margin'] / lines) if lines else 0.0 + s['margin_pct'] = (s['margin'] / s['revenue'] * 100.0) if s.get('revenue') else 0.0 + return s + + +def _attach_sku_codes(rows, pid_key='pid'): + """Attach SKU `code` (default_code) to product rows so each can deep-link to its SKU drawer.""" + ids = [r[pid_key] for r in rows if r.get(pid_key)] + if not ids: + return rows + codemap = {} + for pr in O.search_read('product.product', [('id', 'in', ids)], ['default_code']): + codemap[pr['id']] = str(pr['default_code']).strip() if pr.get('default_code') else None + for r in rows: + r['code'] = codemap.get(r.get(pid_key)) + return rows + + +def decompose(date_from, date_to, team_id=None, line_extra=None, order_extra=None, + product_ids=None, compare=None, top=30): + """Universal decomposition of ANY sales number into its contributors over a window. Line-level + (sale.order.line) so it breaks down by customer, SKU and category consistently and ties to the + headline (line == order revenue, proven in validate()). + + line_extra extra sale.order.line domain clauses (e.g. a rep via order_id.user_id) + order_extra the order-level translation of the same scope (for the orders count) + product_ids restrict to a category's products + compare (cmp_from, cmp_to) for the same-period-last-year total (YoY headline) + + Returns totals + ranked `customers`, `skus`, `categories` (each with revenue + % share).""" + ex = list(line_extra or []) + if product_ids is not None: + ex.append(('product_id', 'in', list(product_ids))) + dom = O.sale_line_domain(date_from, date_to, team_id, extra=ex) + + gc = O.read_group('sale.order.line', dom, ['price_subtotal:sum', 'product_uom_qty:sum'], + ['order_partner_id'], lazy=False) + customers = [{'pid': O.m2o_id(r.get('order_partner_id')), 'customer': O.m2o_name(r.get('order_partner_id')), + 'revenue': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0, + 'lines': r.get('__count') or 0} for r in gc if r.get('order_partner_id')] + + gs = O.read_group('sale.order.line', dom, ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], + ['product_id'], lazy=False) + skus = [_sku_profit({'pid': O.m2o_id(r.get('product_id')), 'product': O.m2o_name(r.get('product_id')), + 'revenue': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0, + 'margin': r.get('margin') or 0.0, 'lines': r.get('__count') or 0}) + for r in gs if r.get('product_id')] + + cat = _product_cat() + cagg = {} + for s in skus: + c = cat.get(s['pid'], '(uncategorized)') + e = cagg.setdefault(c, {'category': c, 'revenue': 0.0, 'qty': 0.0, 'skus': 0}) + e['revenue'] += s['revenue']; e['qty'] += s['qty']; e['skus'] += 1 + categories = sorted(cagg.values(), key=lambda x: -x['revenue']) + + total = sum(c['revenue'] for c in customers) + units = sum(s['qty'] for s in skus) + for lst in (customers, skus, categories): + for r in lst: + r['pct'] = (r['revenue'] / total * 100.0) if total else 0.0 + customers.sort(key=lambda x: -x['revenue']) + skus.sort(key=lambda x: -x['revenue']) + _attach_sku_codes(skus) + + # orders count (order-level, single cheap aggregate) + AOV + odom = order_domain(date_from, date_to, team_id) + list(order_extra or []) + if product_ids is not None and not order_extra: + odom = odom + [('order_line.product_id', 'in', list(product_ids))] + orders = O.get_odoo().search_count('sale.order', odom) + + total_ly = None + if compare: + lex = list(line_extra or []) + if product_ids is not None: + lex.append(('product_id', 'in', list(product_ids))) + total_ly = O.sum_field('sale.order.line', + O.sale_line_domain(compare[0], compare[1], team_id, extra=lex), 'price_subtotal') + + return { + 'window': f'{date_from} → {date_to}', 'date_from': date_from, 'date_to': date_to, + 'total': total, 'total_ly': total_ly, 'yoy_pct': (P.yoy_pct(total, total_ly) if total_ly is not None else None), + 'orders': orders, 'units': units, 'n_customers': len(customers), 'n_skus': len(skus), + 'aov': (total / orders) if orders else 0.0, + 'customers': customers[:top], 'skus': skus[:top], 'categories': categories, + 'all_customers': customers, 'all_skus': skus, + } + + +_ORDER_STATE = {'draft': 'Quote', 'sent': 'Quote sent', 'sale': 'Confirmed', 'done': 'Locked', + 'cancel': 'Cancelled'} +_ORDER_INV = {'upselling': 'Upselling', 'invoiced': 'Invoiced', 'to invoice': 'To invoice', + 'no': 'Nothing to invoice'} + + +def orders_in_scope(date_from, date_to, team_id=None, line_extra=None, order_extra=None, + product_ids=None): + """The per-ORDER list behind any sales number over a window — the raw sale.order rows that make + up a decomposition, so a chart click drills all the way down to the individual orders (each + exportable to Excel). Revenue / units / margin are the IN-SCOPE line contribution (e.g. for a + clicked SKU, only that SKU's lines), so Σ(order revenue) ties to the decomposition headline. + Returns ALL orders (no silent cap — the export must be complete), newest-revenue first.""" + ex = list(line_extra or []) + if product_ids is not None: + ex.append(('product_id', 'in', list(product_ids))) + dom = O.sale_line_domain(date_from, date_to, team_id, extra=ex) + g = O.read_group('sale.order.line', dom, + ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], + ['order_id'], lazy=False) + by_id = {} + for r in g: + oid = O.m2o_id(r.get('order_id')) + if not oid: + continue + by_id[oid] = {'oid': oid, 'order': O.m2o_name(r.get('order_id')), + 'revenue': r.get('price_subtotal') or 0.0, + 'qty': r.get('product_uom_qty') or 0.0, + 'margin': r.get('margin') or 0.0, 'lines': r.get('__count') or 0} + if not by_id: + return [] + # one order-level pull for the human columns (date / customer / status) + for m in O.search_read('sale.order', [('id', 'in', list(by_id))], + ['name', 'date_order', 'partner_id', 'state', 'invoice_status']): + e = by_id.get(m['id']) + if not e: + continue + e['order'] = m.get('name') or e['order'] + e['date'] = (m.get('date_order') or '')[:10] + e['customer'] = O.m2o_name(m.get('partner_id')) + e['status'] = _ORDER_STATE.get(m.get('state'), m.get('state') or '') + e['invoiced'] = _ORDER_INV.get(m.get('invoice_status'), m.get('invoice_status') or '') + return sorted(by_id.values(), key=lambda x: -x['revenue']) + + +@lru_cache(maxsize=1) +def _product_cat(): + """product_id → main category name (reuses inventory's category resolver). + Cached for the process — the category taxonomy is structural and rarely changes; this avoids + re-reading the whole product master on every customer drill-down / win-back.""" + catmap = inv_mod._cat_main_map() + prods = O.search_read('product.product', [('default_code', '!=', False)], ['id', 'categ_id']) + return {p['id']: (catmap.get(O.m2o_id(p.get('categ_id'))) or '(uncategorized)') for p in prods} + + +def _line_rev_by_product_store(date_from, date_to, team_id): + """[(product_id, Σ price_subtotal)] mirroring sale_line_domain (order-side scope join, + line-side exclusions, product set).""" + import harness.datastore as DS + params, w = [], ["o.state IN ('sale','done')", "l.product_id IS NOT NULL"] + teams = TEAMS(team_id) + w.append("o.team_id IN (" + ",".join("?" * len(teams)) + ")") + params += list(teams) + w.append("CAST(o.date_order AS TIMESTAMP) >= CAST(? AS TIMESTAMP)") + params.append(f"{date_from} 00:00:00") + w.append("CAST(o.date_order AS TIMESTAMP) <= CAST(? AS TIMESTAMP)") + params.append(f"{date_to} 23:59:59") + ex = O.excluded_partner_ids() + if ex: + w.append("l.order_partner_id NOT IN (" + ",".join("?" * len(ex)) + ")") + params += list(ex) + if O.doc_mode() == 'invoice': + w.append("o.invoice_status = 'invoiced'") + return DS.ro_con().execute( + "SELECT l.product_id, sum(l.price_subtotal) FROM sale_order_line l " + "JOIN sale_order o ON o.id = l.order_id WHERE " + " AND ".join(w) + " GROUP BY 1", + params).fetchall() + + +def _cat_rev(date_from, date_to, cat, team_id=None): + """Revenue per main category over a window (line-level, brand-aware).""" + if USE_STORE: + try: + out = {} + for pid, amt in _line_rev_by_product_store(date_from, date_to, team_id): + c = cat.get(pid, '(uncategorized)') + out[c] = out.get(c, 0.0) + (amt or 0.0) + return out + except Exception: + pass + g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id), + ['price_subtotal:sum'], ['product_id'], lazy=False) + out = {} + for r in g: + pid = O.m2o_id(r.get('product_id')) + if not pid: + continue + c = cat.get(pid, '(uncategorized)') + out[c] = out.get(c, 0.0) + (r.get('price_subtotal') or 0.0) + return out + + +def by_category(t=None, limit=20, team_id=None): + """Revenue by main category, YTD vs same-period last year — which categories drive (or + drag) the number. Brand-filterable, so you can see e.g. which categories Fisch is losing.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + cat = _product_cat() + this = _cat_rev(yf, yt, cat, team_id) + last = _cat_rev(lf, lt, cat, team_id) + rows = [{'category': c, 'revenue': this.get(c, 0.0), 'revenue_ly': last.get(c, 0.0), + 'change': this.get(c, 0.0) - last.get(c, 0.0), + 'yoy_pct': P.yoy_pct(this.get(c, 0.0), last.get(c, 0.0))} + for c in (set(this) | set(last))] + rows.sort(key=lambda x: -x['revenue']) + return rows[:limit] + + +def cadence(t=None, team_id=None): + """Reorder behaviour over LTM: repeat-purchase rate, avg orders/customer, and the + order-frequency distribution. A wholesale-health signal (are customers coming back?).""" + t = t or P.today() + lf, lt = P.ltm(t) + counts = None + if USE_STORE: + try: + counts = [r[1] for r in _order_partner_groups_store(lf, lt, team_id) if r[0]] + except Exception: + counts = None + if counts is None: + g = O.read_group('sale.order', order_domain(lf, lt, team_id), + ['partner_id'], ['partner_id'], lazy=False) + counts = [r.get('__count') or 0 for r in g if r.get('partner_id')] + n = len(counts) + total_orders = sum(counts) + repeat = sum(1 for c in counts if c >= 2) + buckets = [('1 order', lambda c: c == 1), ('2-3 orders', lambda c: 2 <= c <= 3), + ('4-9 orders', lambda c: 4 <= c <= 9), ('10+ orders', lambda c: c >= 10)] + dist = [{'frequency': label, 'customers': sum(1 for c in counts if fn(c)), + 'pct': (sum(1 for c in counts if fn(c)) / n * 100) if n else 0.0} + for label, fn in buckets] + return { + 'customers': n, + 'total_orders': total_orders, + 'avg_orders': (total_orders / n) if n else 0.0, + 'repeat_customers': repeat, + 'repeat_rate': (repeat / n * 100) if n else 0.0, + 'distribution': dist, + 'ltm_window': f'{lf} → {lt}', + } + + +def concentration(t=None, team_id=None): + """Top-N customer share of YTD revenue.""" + t = t or P.today() + yf, yt = P.ytd(t) + revs = None + if USE_STORE: + try: + revs = sorted([r[2] or 0.0 for r in _order_partner_groups_store(yf, yt, team_id) + if r[0]], reverse=True) + except Exception: + revs = None + if revs is None: + g = O.read_group('sale.order', order_domain(yf, yt, team_id), + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + revs = sorted([r.get('amount_untaxed') or 0.0 for r in g if r.get('partner_id')], + reverse=True) + total = sum(revs) or 1.0 + def share(n): + return sum(revs[:n]) / total * 100.0 + return {'n_customers': len(revs), 'total': total, + 'top10_pct': share(10), 'top25_pct': share(25), + 'top50_pct': share(50), 'top100_pct': share(100)} + + +# ---------------------------------------------------------------- returns (credit notes) +# Returns = posted customer credit notes (account.move, move_type='out_refund'). They carry a +# partner and salesperson but sit on the generic 'Sales' team (not Fisch/Royal team 5/6), so +# returns are reported CONSOLIDATED and sliced by agent (via the customer's res.partner.agent_ids) +# and by period — never BU-split. amount_untaxed_signed is negative for out_refund; we flip it so +# a "return" reads as a positive dollar figure everywhere. +def returns_domain(date_from=None, date_to=None, partner_ids=None): + dom = [('move_type', '=', 'out_refund'), ('state', '=', 'posted')] + if date_from: + dom.append(('invoice_date', '>=', date_from)) + if date_to: + dom.append(('invoice_date', '<=', date_to)) + ex = O.excluded_partner_ids() + if ex: + dom.append(('partner_id', 'not in', list(ex))) + if partner_ids is not None: + dom.append(('partner_id', 'in', list(partner_ids))) + return dom + + +def _returns_amt(date_from, date_to, partner_ids=None): + """Total returns $ (positive) over a window.""" + v = O.sum_field('account.move', returns_domain(date_from, date_to, partner_ids), 'amount_untaxed_signed') + return -(v or 0.0) + + +def returns_monthly(n=13, t=None, partner_ids=None): + """Returns $ per month for the last n months + same month one year earlier (YoY). Shaped like + the revenue trend (`revenue`/`revenue_ly` keys) so it renders through the same chart_yoy_bars. + ONE month-grouped read over a ~2-year span (was 2n sequential sum queries).""" + t = t or P.today() + months = P.month_starts(n, t) + span_from = f"{int(months[0][0][:4]) - 1:04d}-{months[0][0][5:7]}-01" # 1 year before the first month + g = O.read_group('account.move', returns_domain(span_from, t.isoformat(), partner_ids), + ['amount_untaxed_signed:sum'], ['invoice_date:month'], lazy=False) + mret = {} + for r in g: + ym = ((r.get('__range') or {}).get('invoice_date:month') or {}).get('from', '')[:7] + if ym: + mret[ym] = -(r.get('amount_untaxed_signed') or 0.0) + rows = [] + for ym, start, end in months: + y, m = int(ym[:4]) - 1, int(ym[5:7]) + this, last = mret.get(ym, 0.0), mret.get(f'{y:04d}-{m:02d}', 0.0) + rows.append({'month': ym, 'start': start, 'end': end, + 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) + return rows + + +def returns_by_partner(t=None, partner_ids=None): + """{partner_id: returns$ YTD} (positive) — the raw per-customer credit-note total. The agent + rollup maps these to res.partner.agent_ids in the Customers module.""" + t = t or P.today() + yf, yt = P.ytd(t) + g = O.read_group('account.move', returns_domain(yf, yt, partner_ids), + ['amount_untaxed_signed:sum'], ['partner_id'], lazy=False) + return {O.m2o_id(r.get('partner_id')): -(r.get('amount_untaxed_signed') or 0.0) + for r in g if r.get('partner_id')} + + +def returns_headline(t=None, partner_ids=None): + """YTD returns $ + YoY + return rate (returns / gross revenue). Consolidated.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + ret = _returns_amt(yf, yt, partner_ids) + ret_ly = _returns_amt(lf, lt, partner_ids) + gross = _orev(yf, yt, None) if partner_ids is None else O.sum_field( + 'sale.order', order_domain(yf, yt, None, partner_ids=partner_ids), 'amount_untaxed') + return {'ytd': ret, 'ytd_ly': ret_ly, 'yoy_pct': P.yoy_pct(ret, ret_ly), + 'rate_pct': (ret / gross * 100.0) if gross else 0.0} + + +# ---------------------------------------------------------------- VALIDATION +def validate(t=None, team_id=None): + """Reconcile metrics against independent Odoo aggregates. Returns list of checks. + + When team_id is set (a single BU selected) every check runs SCOPED to that BU, so the + validation panel never reconciles against — or exposes — the other BU's numbers. The one + cross-BU check (#2, Σ teams == total) only makes sense consolidated, so it runs only when + team_id is None.""" + t = t or P.today() + yf, yt = P.ytd(t) + checks = [] + + # 1. Order-level revenue (sale.order) vs line-level revenue (sale.order.line) + order_rev = _orev(yf, yt, team_id) + line_rev = O.sum_field('sale.order.line', O.sale_line_domain(yf, yt, team_id), 'price_subtotal') + gap = order_rev - line_rev + checks.append({ + 'check': 'YTD revenue: order-level == line-level', + 'a': round(order_rev, 2), 'b': round(line_rev, 2), + 'gap': round(gap, 2), + 'ok': abs(gap) <= max(1.0, 0.001 * order_rev)}) + + # 2. Sum of per-team revenue == total (consolidated only — cross-BU) + if team_id is None: + team_sum = sum(_orev(yf, yt, tid) for tid in O.TEAM_IDS) + checks.append({ + 'check': 'YTD revenue: Σ(team) == total', + 'a': round(team_sum, 2), 'b': round(order_rev, 2), + 'gap': round(team_sum - order_rev, 2), + 'ok': abs(team_sum - order_rev) <= 1.0}) + + # 3. Sum of per-customer revenue == total + cust_sum = sum(c['revenue'] for c in top_customers(t, limit=10**9, team_id=team_id)) + checks.append({ + 'check': 'YTD revenue: Σ(customer) == total', + 'a': round(cust_sum, 2), 'b': round(order_rev, 2), + 'gap': round(cust_sum - order_rev, 2), + 'ok': abs(cust_sum - order_rev) <= 1.0}) + + # 4. Σ(category revenue) == line-level total (YTD) + cat_sum = sum(c['revenue'] for c in by_category(t, limit=10**9, team_id=team_id)) + checks.append({ + 'check': 'YTD revenue: Σ(category) == line-level total', + 'a': round(cat_sum, 2), 'b': round(line_rev, 2), + 'gap': round(cat_sum - line_rev, 2), + 'ok': abs(cat_sum - line_rev) <= 1.0}) + + # 6. Cadence: Σ(frequency-bucket customers) == total LTM customers + cad = cadence(t, team_id=team_id) + bucket_n = sum(d['customers'] for d in cad['distribution']) + checks.append({ + 'check': 'Cadence: Σ(frequency buckets) == LTM customers', + 'a': bucket_n, 'b': cad['customers'], + 'gap': bucket_n - cad['customers'], + 'ok': bucket_n == cad['customers']}) + + # 7. Decompose ties out: the YTD decomposition total == line-level YTD, and its customer / SKU / + # category breakdowns each sum back to that total (the drill-any-number drawer is trustworthy). + dec = decompose(yf, yt, team_id=team_id) + checks.append({ + 'check': 'Decompose YTD total == line-level revenue', + 'a': round(dec['total'], 2), 'b': round(line_rev, 2), + 'gap': round(dec['total'] - line_rev, 2), + 'ok': abs(dec['total'] - line_rev) <= max(1.0, 0.001 * line_rev)}) + cust_d = sum(c['revenue'] for c in dec['all_customers']) + sku_d = sum(s['revenue'] for s in dec['all_skus']) + cat_d = sum(c['revenue'] for c in dec['categories']) + checks.append({ + 'check': 'Decompose: Σ(customers)=Σ(SKUs)=Σ(categories)=total', + 'a': round(cust_d, 2), 'b': round(dec['total'], 2), + 'gap': round(max(abs(cust_d - dec['total']), abs(sku_d - dec['total']), abs(cat_d - dec['total'])), 2), + 'ok': all(abs(x - dec['total']) <= 1.0 for x in (cust_d, sku_d, cat_d))}) + + return checks diff --git a/royalimports_os/modules/spend.py b/royalimports_os/modules/spend.py new file mode 100644 index 0000000000000000000000000000000000000000..0569ceb81e5d52a6f9af393720777ca2ba098d02 --- /dev/null +++ b/royalimports_os/modules/spend.py @@ -0,0 +1,301 @@ +"""Spend & Payables — the expense-waste module (cost-waste brief recs 1/3/4): the first +systematic analysis of vendor bills on this platform. + + 1. TERMS CAPTURE — per supplier: effective terms (per-bill due-date gap — partner-level terms + are sparsely populated) vs actual bill→settlement days (payable full-reconcile counterpart + date). Paying EARLY = free cash surrendered (the mirror of the receivables free-credit + finding). China/import suppliers are deposit-based — DPO stretch is structurally capped, so + the book is SPLIT by supplier country; the verb is pay-to-TERMS, never pay-late. + 2. DUPLICATE BILLS — same supplier + same amount (±$1) within 45 days, or same non-empty + vendor ref. Classic AP recovery: 0.1-0.5% of spend. + 3. SPEND CUBE — NON-product bill lines (Stock Interim / inventory accounts excluded) by + GL account × supplier × month: category fragmentation (consolidate + rebid) and + MAVERICK % = lines that never touched a PO (no purchase_line_id). + 4. FREIGHT RECOVERY — outbound freight PAID (the 'Freight Out' expense account) vs freight + BILLED to customers (freight/shipping lines on customer invoices). NAW: distributors + recapture only ~70%. + +READ-ONLY. Supplier names normalized with a dependency-free suffix-strip (upgrade path: +RapidFuzz+cleanco per the brief). +""" +import re +import datetime as dt + +import core.odoo as O +import core.periods as P + +DUP_WINDOW_DAYS = 45 +EARLY_GRACE_DAYS = 5 # paying ≤5d early is noise, not a finding +FREIGHT_OUT_PAT = re.compile(r'freight out|freight-out', re.I) +FREIGHT_BILL_PAT = re.compile(r'freight|shipping|delivery', re.I) +# product-side interim/inventory accounts excluded from the OPEX cube +PRODUCT_ACC_PAT = re.compile(r'stock interim|inventory|goods received', re.I) + +_SUFFIX = re.compile(r'\b(inc|llc|ltd|corp|corporation|co|company|s\.?a\.?|gmbh)\b\.?', re.I) + + +def _norm_supplier(name): + s = _SUFFIX.sub('', str(name or '').upper()) + return re.sub(r'[^A-Z0-9 ]', '', s).strip() or str(name or '') + + +def _chunk(ids, n=2000): + ids = list(ids) + for i in range(0, len(ids), n): + yield ids[i:i + n] + + +def _bills(lf, lt): + return O.search_read('account.move', + [('move_type', '=', 'in_invoice'), ('state', '=', 'posted'), + ('invoice_date', '>=', lf), ('invoice_date', '<=', lt)], + ['partner_id', 'invoice_date', 'invoice_date_due', 'amount_total', + 'amount_untaxed', 'ref', 'payment_state', 'name']) + + +def terms_capture(t=None): + """Per-supplier payment discipline over the LTM + the duplicate-bill worklist.""" + t = t or P.today() + lf, lt = P.ltm(t) + bills = _bills(lf, lt) + + # settlement date per bill: payable line's full-reconcile group, max counterpart date + move_ids = [b['id'] for b in bills] + pay_lines = [] + for ch in _chunk(move_ids): + pay_lines += O.search_read('account.move.line', + [('move_id', 'in', ch), + ('account_id.account_type', '=', 'liability_payable'), + ('full_reconcile_id', '!=', False)], + ['move_id', 'full_reconcile_id']) + fr_by_move = {O.m2o_id(l['move_id']): O.m2o_id(l['full_reconcile_id']) for l in pay_lines} + settle = {} + for ch in _chunk(list(set(fr_by_move.values())), 4000): + for l in O.search_read('account.move.line', [('full_reconcile_id', 'in', ch)], + ['full_reconcile_id', 'date']): + k = O.m2o_id(l['full_reconcile_id']) + d = str(l.get('date') or '')[:10] + if k and d: + settle[k] = max(settle.get(k, ''), d) + + # supplier country for the import/domestic split + pids = list({O.m2o_id(b['partner_id']) for b in bills}) + country = {} + for ch in _chunk(pids): + for r in O.search_read('res.partner', [('id', 'in', ch)], ['country_id']): + country[r['id']] = O.m2o_name(r.get('country_id')) or '(none)' + + per = {} + for b in bills: + pid = O.m2o_id(b['partner_id']) + inv_d = str(b.get('invoice_date') or '')[:10] + due_d = str(b.get('invoice_date_due') or '')[:10] + if not pid or not inv_d: + continue + e = per.setdefault(pid, {'pid': pid, 'supplier': O.m2o_name(b['partner_id']), + 'country': country.get(pid, '(none)'), 'bills': 0, + 'spend': 0.0, 'terms': [], 'actual': [], + 'early_cash': 0.0, 'early_bills': 0}) + e['bills'] += 1 + e['spend'] += b.get('amount_total') or 0.0 + terms_days = None + if due_d and due_d >= inv_d: + terms_days = (dt.date.fromisoformat(due_d) - dt.date.fromisoformat(inv_d)).days + e['terms'].append(terms_days) + fr = fr_by_move.get(b['id']) + pay_d = settle.get(fr, '') if fr else '' + if pay_d and pay_d >= inv_d: + actual = (dt.date.fromisoformat(pay_d) - dt.date.fromisoformat(inv_d)).days + e['actual'].append(actual) + if terms_days is not None and actual < terms_days - EARLY_GRACE_DAYS: + early = terms_days - actual + e['early_cash'] += early / 365.0 * (b.get('amount_total') or 0.0) + e['early_bills'] += 1 + + def _med(v): + v = sorted(v) + return v[len(v) // 2] if v else None + rows = [] + for e in per.values(): + seg = ('(country unset)' if e['country'] == '(none)' + else 'import' if e['country'] not in ('United States', 'Canada') + else 'domestic') + rows.append({'pid': e['pid'], 'supplier': e['supplier'], 'country': e['country'], + 'segment': seg, + 'bills': e['bills'], 'spend': e['spend'], + 'terms_days': _med(e['terms']), 'actual_days': _med(e['actual']), + 'early_bills': e['early_bills'], 'early_cash': e['early_cash']}) + rows.sort(key=lambda x: -x['early_cash']) + + # duplicate-bill candidates + by_partner = {} + for b in bills: + by_partner.setdefault(O.m2o_id(b['partner_id']), []).append(b) + dups = [] + for pid, bs in by_partner.items(): + bs = sorted(bs, key=lambda x: str(x.get('invoice_date') or '')) + for i, a in enumerate(bs): + for c in bs[i + 1:]: + da = str(a.get('invoice_date') or '')[:10] + dc = str(c.get('invoice_date') or '')[:10] + if not da or not dc: + continue + gap = (dt.date.fromisoformat(dc) - dt.date.fromisoformat(da)).days + if gap > DUP_WINDOW_DAYS: + break + amt_a, amt_c = a.get('amount_total') or 0.0, c.get('amount_total') or 0.0 + same_ref = (a.get('ref') and a.get('ref') == c.get('ref')) + # same vendor ref = strong signal at any gap; without a ref, only a same-amount + # bill within 7 days counts (monthly recurring bills — rent, loans, standing + # POs — land ~28-31 days apart and are NOT duplicates) + if abs(amt_a - amt_c) <= 1.0 and amt_a > 100 and (same_ref or gap <= 7): + dups.append({'supplier': O.m2o_name(a['partner_id']), + 'bill_a': a['name'], 'bill_b': c['name'], + 'date_a': da, 'date_b': dc, 'amount': amt_a, + 'same_ref': 'yes' if same_ref else '', + 'ref': str(a.get('ref') or '')[:24]}) + dups.sort(key=lambda x: -x['amount']) + return {'rows': rows, 'dups': dups, + 'n_bills': len(bills), 'spend_total': sum(b.get('amount_total') or 0 for b in bills), + 'early_cash_total': sum(r['early_cash'] for r in rows), + 'dup_exposure': sum(d['amount'] for d in dups), + 'window': (lf, lt)} + + +def _bill_lines(lf, lt): + lines = [] + # account.move.line.display_type is a SELECTION ('product'/'tax'/'payment_term'/…), + # not a boolean — '= False' matches nothing (found live 2026-07-05) + got = O.search_read('account.move.line', + [('move_id.move_type', '=', 'in_invoice'), + ('parent_state', '=', 'posted'), + ('move_id.invoice_date', '>=', lf), + ('move_id.invoice_date', '<=', lt), + ('display_type', '=', 'product'), + ('price_subtotal', '!=', 0)], + ['partner_id', 'account_id', 'price_subtotal', 'purchase_line_id', + 'product_id', 'date']) + acc_ids = list({O.m2o_id(l['account_id']) for l in got if l.get('account_id')}) + atype = {} + for ch in _chunk(acc_ids): + for a in O.search_read('account.account', [('id', 'in', ch)], ['account_type']): + atype[a['id']] = a.get('account_type') or '' + for l in got: + l['account'] = O.m2o_name(l.get('account_id')) + l['atype'] = atype.get(O.m2o_id(l.get('account_id')), '') + lines.append(l) + return lines + + +def spend_cube(t=None): + """OPEX spend by GL category × supplier(normalized), with fragmentation and no-PO %. + Scope = expense-type accounts ONLY (account_type 'expense'/'expense_depreciation'): + inventory-interim (asset) and loan/advance (liability) bill lines are excluded by TYPE, + and COGS (expense_direct_cost) belongs to the product side. Note: this company runs no + PO process for indirect spend, so no-PO% ≈ 100% by construction — it becomes a signal + only if POs are ever adopted for indirect purchases.""" + t = t or P.today() + lf, lt = P.ltm(t) + lines = [l for l in _bill_lines(lf, lt) + if l.get('atype') in ('expense', 'expense_depreciation')] + cats, sup_cat = {}, {} + for l in lines: + acc = l['account'] + amt = l.get('price_subtotal') or 0.0 + sup = _norm_supplier(O.m2o_name(l.get('partner_id'))) + c = cats.setdefault(acc, {'category': acc, 'spend': 0.0, 'lines': 0, + 'maverick': 0.0, 'suppliers': set()}) + c['spend'] += amt + c['lines'] += 1 + c['suppliers'].add(sup) + if not l.get('purchase_line_id'): + c['maverick'] += amt + k = (acc, sup) + sup_cat[k] = sup_cat.get(k, 0.0) + amt + rows = [] + for c in cats.values(): + n_sup = len(c['suppliers']) + top = sorted(((s, v) for (a, s), v in sup_cat.items() if a == c['category']), + key=lambda kv: -kv[1]) + rows.append({'category': c['category'], 'spend': c['spend'], 'lines': c['lines'], + 'n_suppliers': n_sup, + 'maverick_pct': (c['maverick'] / c['spend'] * 100) if c['spend'] else 0.0, + 'top_supplier': top[0][0][:30] if top else '', + 'top_share_pct': (top[0][1] / c['spend'] * 100) if (top and c['spend']) else 0.0, + 'consolidate': ('yes' if n_sup >= 3 and c['spend'] > 10000 else '')}) + rows.sort(key=lambda x: -x['spend']) + return {'rows': rows, + 'spend_total': sum(r['spend'] for r in rows), + 'maverick_total': sum(r['spend'] * r['maverick_pct'] / 100 for r in rows), + 'frag_categories': [r for r in rows if r['consolidate']], + 'window': (lf, lt)} + + +def freight_recovery(t=None): + """Outbound freight paid (the 'Freight Out' expense account) vs freight billed to + customers (freight/shipping/delivery lines on out_invoices), monthly + the headline + recovery %. Company-level.""" + t = t or P.today() + lf, lt = P.ltm(t) + paid_lines = [l for l in _bill_lines(lf, lt) + if l['account'] and FREIGHT_OUT_PAT.search(l['account'])] + paid_m = {} + for l in paid_lines: + m = str(l.get('date') or '')[:7] + paid_m[m] = paid_m.get(m, 0.0) + (l.get('price_subtotal') or 0.0) + + billed = O.search_read('account.move.line', + [('move_id.move_type', '=', 'out_invoice'), + ('parent_state', '=', 'posted'), + ('move_id.invoice_date', '>=', lf), + ('move_id.invoice_date', '<=', lt), + ('display_type', '=', 'product')], + ['account_id', 'product_id', 'price_subtotal', 'date', + 'partner_id']) + billed_lines = [l for l in billed + if FREIGHT_BILL_PAT.search(O.m2o_name(l.get('product_id')) or '') + or FREIGHT_BILL_PAT.search(O.m2o_name(l.get('account_id')) or '')] + billed_m, billed_cust = {}, {} + for l in billed_lines: + m = str(l.get('date') or '')[:7] + amt = l.get('price_subtotal') or 0.0 + billed_m[m] = billed_m.get(m, 0.0) + amt + pid = O.m2o_id(l.get('partner_id')) + if pid: + billed_cust[pid] = billed_cust.get(pid, 0.0) + amt + months = sorted(set(paid_m) | set(billed_m)) + monthly = [{'month': m, 'paid': paid_m.get(m, 0.0), 'billed': billed_m.get(m, 0.0)} + for m in months] + paid_total = sum(paid_m.values()) + billed_total = sum(billed_m.values()) + return {'monthly': monthly, 'paid_total': paid_total, 'billed_total': billed_total, + 'recovery_pct': (billed_total / paid_total * 100) if paid_total else None, + 'n_billed_customers': len(billed_cust), + 'window': (lf, lt)} + + +def validate(t=None, team_id=None, pre=None): + """(1) bill headline ties the server sum over the same domain; (2) the cube total + + excluded product-account lines re-adds to the full bill-line total (nothing dropped + silently).""" + t = t or P.today() + tc = pre or terms_capture(t) + lf, lt = tc['window'] + srv = O.sum_field('account.move', + [('move_type', '=', 'in_invoice'), ('state', '=', 'posted'), + ('invoice_date', '>=', lf), ('invoice_date', '<=', lt)], + 'amount_total') + a = tc['spend_total'] + checks = [{'check': 'vendor-bill spend — client Σ == server Σ', + 'a': round(a, 2), 'b': round(srv, 2), 'gap': round(a - srv, 2), + 'ok': abs(a - srv) <= max(1.0, srv * 0.001)}] + all_lines = _bill_lines(lf, lt) + full = sum(l.get('price_subtotal') or 0 for l in all_lines) + cube = spend_cube(t) + excluded = sum(l.get('price_subtotal') or 0 for l in all_lines + if l.get('atype') not in ('expense', 'expense_depreciation')) + lhs = cube['spend_total'] + excluded + checks.append({'check': 'opex cube + non-expense-type lines == all bill lines', + 'a': round(lhs, 2), 'b': round(full, 2), 'gap': round(lhs - full, 2), + 'ok': abs(lhs - full) < 1.0}) + return checks \ No newline at end of file diff --git a/royalimports_os/modules/tasks.py b/royalimports_os/modules/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..90e60338597af63414d9e47e940518ca69d0b6ec --- /dev/null +++ b/royalimports_os/modules/tasks.py @@ -0,0 +1,213 @@ +"""Store-backed task / worklist lifecycle — the writable PROCESS layer over the read-only ERP +(the Palantir split: subject layer = Odoo, read-only; process layer = this, in the HF store). + +Store key 'tasks': {"seq": int, "items": {id: task}}. A task: + id 't' — stable + title verb-first next action ("Call Smith Floral about reorder") + detail one-line evidence/why (frozen at flag time for rule tasks) + entity {'kind': customer|sku|agent|account, 'id': …, 'label': …} | None + owner_user username in the users registry + due 'YYYY-MM-DD' | None + state open | done | snoozed | dismissed + source 'user' | 'rule:' + cond_key rule tasks: fingerprint of the flagging condition — a dismissed task is NOT + re-created until the condition RESETS (cond_key changes). The anti-fatigue rule. + dismiss_reason enumerated (required on dismiss); reviewed to tune/kill rules + outcome 'self-resolved' when a rule flag cleared on its own (auto-closed) + created_by / created_at / status_changed / snooze_until / history (append-only) + +All mutations go through store.update (strict read-modify-write — never get()+put()). +""" +import datetime as dt + +import core.store as store + +KEY = 'tasks' +DISMISS_REASONS = ['Not relevant', 'Already handled', 'Data is wrong', 'Duplicate', 'Other'] +STATES = ('open', 'done', 'snoozed', 'dismissed') + + +def _now(): + return dt.datetime.now().strftime('%Y-%m-%d %H:%M') + + +def _today(): + return dt.date.today().isoformat() + + +def _entity_key(entity): + if not entity: + return None + return f"{entity.get('kind')}:{entity.get('id')}" + + +def all_tasks(): + d = store.get(KEY) + return list((d.get('items') or {}).values()) + + +def effective_state(t, today=None): + """Snoozed past its wake date reads as open (Linear semantics — wake early on due).""" + s = t.get('state', 'open') + if s == 'snoozed' and (t.get('snooze_until') or '') <= (today or _today()): + return 'open' + return s + + +def for_owner(owner=None, states=('open',)): + """Open queue for a user (None = everyone), due-date order, snooze-awakened.""" + today = _today() + rows = [t for t in all_tasks() + if (owner is None or t.get('owner_user') == owner) + and effective_state(t, today) in states] + return sorted(rows, key=lambda t: (t.get('due') or '9999-12-31', t.get('id', ''))) + + +def counts_by_user(): + """{user: {'open': n, 'overdue': n, 'stale7': n}} — feeds the owner rollup + digests.""" + today = _today() + week_ago = (dt.date.today() - dt.timedelta(days=7)).isoformat() + out = {} + for t in all_tasks(): + if effective_state(t, today) != 'open': + continue + u = t.get('owner_user') or '—' + rec = out.setdefault(u, {'open': 0, 'overdue': 0, 'stale7': 0}) + rec['open'] += 1 + if (t.get('due') or '9999') < today: + rec['overdue'] += 1 + if (t.get('status_changed') or t.get('created_at') or '')[:10] <= week_ago: + rec['stale7'] += 1 + return out + + +def create(title, owner_user, created_by, detail='', entity=None, due=None, + source='user', cond_key=None): + """Create one task (human producer). Returns the new task id.""" + created = {} + + def _add(d): + d.setdefault('seq', 0) + d.setdefault('items', {}) + d['seq'] += 1 + tid = f"t{d['seq']}" + d['items'][tid] = { + 'id': tid, 'title': str(title).strip(), 'detail': str(detail or '').strip(), + 'entity': entity or None, 'owner_user': owner_user, 'due': due, + 'state': 'open', 'source': source, 'cond_key': cond_key, + 'dismiss_reason': None, 'outcome': None, + 'created_by': created_by, 'created_at': _now(), 'status_changed': _now(), + 'snooze_until': None, + 'history': [{'at': _now(), 'by': created_by, 'event': 'created'}], + } + created['id'] = tid + return d + store.update(KEY, _add) + return created.get('id') + + +def set_state(tid, state, user, dismiss_reason=None, snooze_until=None): + """Move a task through its lifecycle. Dismiss REQUIRES an enumerated reason.""" + if state not in STATES: + raise ValueError(f'bad state {state}') + if state == 'dismissed' and not dismiss_reason: + raise ValueError('dismiss requires a reason') + + def _set(d): + t = (d.get('items') or {}).get(tid) + if not t: + return d + t['state'] = state + t['status_changed'] = _now() + if state == 'dismissed': + t['dismiss_reason'] = dismiss_reason + if state == 'snoozed': + t['snooze_until'] = snooze_until + t.setdefault('history', []).append( + {'at': _now(), 'by': user, 'event': state, + **({'reason': dismiss_reason} if dismiss_reason else {}), + **({'until': snooze_until} if snooze_until else {})}) + return d + store.update(KEY, _set) + + +def sync_rule_tasks(rule_id, flags, default_owner=None): + """Rule producer (the nightly job): reconcile the store against the CURRENT flag set. + + flags: [{entity, title, detail, owner_user?, due?, cond_key}] — cond_key fingerprints the + condition (e.g. bucket/threshold hit), so: + - open/snoozed task for the same (rule, entity): refresh detail (evidence stays current); + - done/dismissed task with the SAME cond_key: skip (never re-nag until the condition resets); + - cond_key changed: create a NEW task (the condition reset and re-fired); + - open rule task whose entity no longer flags: auto-close as self-resolved (the + stale-list/paid-invoice trust-killer). + One store commit for the whole reconcile. + """ + src = f'rule:{rule_id}' + + def _sync(d): + d.setdefault('seq', 0) + d.setdefault('items', {}) + items = d['items'] + by_entity = {} + for t in items.values(): + if t.get('source') == src and t.get('entity'): + by_entity.setdefault(_entity_key(t['entity']), []).append(t) + seen = set() + for f in flags: + ek = _entity_key(f.get('entity')) + if not ek: + continue + seen.add(ek) + existing = sorted(by_entity.get(ek, []), key=lambda t: t['id']) + live = [t for t in existing if t.get('state') in ('open', 'snoozed')] + if live: + live[-1]['detail'] = f.get('detail', live[-1].get('detail')) + live[-1]['cond_key'] = f.get('cond_key', live[-1].get('cond_key')) + continue + closed_same = [t for t in existing + if t.get('cond_key') and t.get('cond_key') == f.get('cond_key')] + if closed_same: + continue # dismissed/done and unchanged — stay quiet + d['seq'] += 1 + tid = f"t{d['seq']}" + items[tid] = { + 'id': tid, 'title': f.get('title', rule_id), 'detail': f.get('detail', ''), + 'entity': f.get('entity'), 'owner_user': f.get('owner_user') or default_owner, + 'due': f.get('due'), 'state': 'open', 'source': src, + 'cond_key': f.get('cond_key'), 'dismiss_reason': None, 'outcome': None, + 'created_by': 'system', 'created_at': _now(), 'status_changed': _now(), + 'snooze_until': None, + 'history': [{'at': _now(), 'by': 'system', 'event': 'created'}], + } + for ek, ts in by_entity.items(): + if ek in seen: + continue + for t in ts: + if t.get('state') in ('open', 'snoozed'): + t['state'] = 'done' + t['outcome'] = 'self-resolved' + t['status_changed'] = _now() + t.setdefault('history', []).append( + {'at': _now(), 'by': 'system', 'event': 'self-resolved'}) + return d + store.update(KEY, _sync) + + +def rule_health(): + """fired / actioned(done by a person) / dismissed / self-resolved per rule — the noise budget. + <20% actioned = the rule is noise: tighten or kill (SRE actionability test).""" + out = {} + for t in all_tasks(): + src = t.get('source') or '' + if not src.startswith('rule:'): + continue + r = out.setdefault(src[5:], {'fired': 0, 'done': 0, 'dismissed': 0, 'self_resolved': 0}) + r['fired'] += 1 + if t.get('outcome') == 'self-resolved': + r['self_resolved'] += 1 + elif t.get('state') == 'done': + r['done'] += 1 + elif t.get('state') == 'dismissed': + r['dismissed'] += 1 + return out diff --git a/royalimports_os/modules/vendor.py b/royalimports_os/modules/vendor.py new file mode 100644 index 0000000000000000000000000000000000000000..9469417e9fc9f1efc37a67383fa14a13ae5f3b6f --- /dev/null +++ b/royalimports_os/modules/vendor.py @@ -0,0 +1,43 @@ +"""Vendor module — per-vendor rollup + the multiple-vendor / cheapest-price view. + +PURE: it takes the procurement recs (which already merge the curated supplier map + live Odoo +demand/stock + Odoo's alternative vendors via product.supplierinfo) and groups them by vendor, so +there is NO extra Odoo round-trip — the Procurement page and this page share one cached pull. Vendor +master-data edits (country / lead) persist through the procurement overrides store, applied to every +SKU of that vendor (the UI does the bulk write). READ-ONLY on Odoo. +""" + + +def rollup(recs): + """[{vendor, country, lead, n_skus, n_buy, demand_units, buy_value, lines}], by SKU count desc.""" + out = {} + for r in recs: + v = r.get('vendor') or '(no supplier)' + g = out.setdefault(v, {'vendor': v, 'country': r.get('country', ''), 'lead': r.get('lead', 0), + 'n_skus': 0, 'n_buy': 0, 'demand_units': 0.0, 'buy_value': 0.0, 'lines': []}) + g['n_skus'] += 1 + if r.get('buy'): + g['n_buy'] += 1 + g['demand_units'] += (r.get('demand_8m') or 0) + g['buy_value'] += (r.get('reorder_trend') or 0) * (r.get('unit_cost') or 0) + g['lines'].append(r) + if not g['country'] and r.get('country'): + g['country'] = r['country'] + if not g['lead'] and r.get('lead'): + g['lead'] = r['lead'] + return sorted(out.values(), key=lambda x: (-x['n_skus'], x['vendor'] or '~')) + + +def multi_vendor(recs): + """SKUs Odoo lists with >1 vendor — current vendor/cost vs the cheapest alternative (price-shop).""" + out = [] + for r in recs: + if (r.get('n_vendors') or 0) <= 1: + continue + cur = r.get('unit_cost') or 0 + cheapest = r.get('cheapest_price') + cheaper = bool(cheapest is not None and cur > 0 and cheapest < cur - 1e-9) + out.append({**r, 'current_cost': cur, + 'savings': (cur - cheapest) if cheaper else 0.0, 'cheaper': cheaper}) + out.sort(key=lambda r: -r['savings']) + return out diff --git a/royalimports_os/modules/warehouse.py b/royalimports_os/modules/warehouse.py new file mode 100644 index 0000000000000000000000000000000000000000..be73f2dff793f3e77cc281d21d23759e19b9b3e4 --- /dev/null +++ b/royalimports_os/modules/warehouse.py @@ -0,0 +1,936 @@ +"""Warehouse module — SKU movement & warehouse efficiency, live from stock.picking / stock.move +(read-only). Physical operations are consolidated (one network of warehouses, not BU-tagged). + +What the data supports (probed 2026-07-05): date_done is fully populated on done pickings, the +Ditmas Ave outbound runs an explicit Pick -> Pack -> Ship funnel (separate picking types), ~25k +pickings and ~221k moves a year, and inventory-adjustment moves (~10k/yr) give a shrinkage / +correction signal. Cycle times, on-time rates, throughput, backlog age and adjustment dollars are +all derivable; anything scanner-level (walk paths, per-picker rates) is NOT in the data and is +not claimed. + +All metrics are computed from ONE 12-month done-picking pull + small side pulls; validate() +reconciles row-built aggregates to independent read_group/search_count paths. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P + +ONTIME_GRACE_DAYS = 1 # shipped within scheduled + 1 day counts as on time +# Main-warehouse funnel + Putaway (the dock-to-stock proxy: receipts sit until put away). +_STAGE_NAMES = {'Pick': 'Pick', 'Pack': 'Pack', 'Delivery Orders': 'Ship', 'Putaway': 'Putaway'} + + +def _dtp(v): + try: + return dt.datetime.fromisoformat(str(v)[:19]) + except Exception: + return None + + +def _types(): + """picking_type_id -> {code, wh, name, stage} (stage set for the main-warehouse funnel).""" + out = {} + for t in O.search_read('stock.picking.type', [], ['name', 'code', 'warehouse_id']): + wh = O.m2o_name(t.get('warehouse_id')) or '' + stage = _STAGE_NAMES.get(t['name']) if 'Ditmas' in wh else None + out[t['id']] = {'code': t['code'], 'wh': wh, 'name': t['name'], 'stage': stage} + return out + + +def _done_pickings(y1): + return O.search_read('stock.picking', + [('state', '=', 'done'), ('date_done', '>=', y1)], + ['picking_type_id', 'scheduled_date', 'date_done', 'date', + 'backorder_id']) + + +def _open_pickings(): + return O.search_read('stock.picking', + [('state', 'in', ['confirmed', 'waiting', 'assigned'])], + ['picking_type_id', 'scheduled_date', 'date', 'origin', 'partner_id', + 'state']) + + +def _median(v): + if not v: + return None + s = sorted(v) + n = len(s) + return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2 + + +def _p90(v): + if not v: + return None + s = sorted(v) + return s[min(len(s) - 1, int(len(s) * 0.9))] + + +def _iso_week(d): + y, w, _ = d.isocalendar() + return f'{y}-W{w:02d}', w + + +def _xmr(series): + """Wheeler XmR (individuals) limits + the THREE detection rules — no more (extra rules = + alarm fatigue; see the warehouse research brief). series = [(label, value)] in time order. + Ported logic cribbed from xmrit (MIT); ~30 lines, no dependency.""" + vals = [v for _, v in series] + if len(vals) < 8: + return {'center': None, 'unpl': None, 'lnpl': None, 'points': [ + {'label': l, 'value': v, 'signal': ''} for l, v in series]} + center = sum(vals) / len(vals) + mrs = [abs(vals[i] - vals[i - 1]) for i in range(1, len(vals))] + mr_bar = sum(mrs) / len(mrs) if mrs else 0.0 + unpl = min(100.0, center + 2.66 * mr_bar) + lnpl = max(0.0, center - 2.66 * mr_bar) + pts = [] + for i, (l, v) in enumerate(series): + sig = '' + if v > unpl or v < lnpl: + sig = 'outside limits' # rule 1 + if not sig and i >= 7 and all((x - center) > 0 for _, x in series[i - 7:i + 1]): + sig = 'sustained shift (8 above center)' # rule 2 + if not sig and i >= 7 and all((x - center) < 0 for _, x in series[i - 7:i + 1]): + sig = 'sustained shift (8 below center)' + if not sig and i >= 3: + last4 = [x for _, x in series[i - 3:i + 1]] + near = sum(1 for x in last4 + if abs(x - center) > 0.5 * ((unpl - center) if x >= center else (center - lnpl))) + if near >= 3: + sig = 'moderate shift emerging' # rule 3 + pts.append({'label': l, 'value': v, 'signal': sig}) + return {'center': center, 'unpl': unpl, 'lnpl': lnpl, 'points': pts} + + +def build(t=None): + """The full warehouse-efficiency bundle from one done-picking pull + side pulls. + Pull window ~14 months so the weekly volume series has a same-week-last-year comparison.""" + t = t or P.today() + y1 = (t - dt.timedelta(days=430)).isoformat() + types = _types() + done = _done_pickings(y1) + open_ = _open_pickings() + _y365 = (t - dt.timedelta(days=365)).isoformat() + + # ---- monthly throughput by flow (12m) ---------------------------------- + monthly = {} + for r in done: + if str(r['date_done'])[:10] < _y365: + continue + ti = types.get(O.m2o_id(r['picking_type_id']), {}) + code = ti.get('code') or '?' + m = str(r['date_done'])[:7] + key = {'incoming': 'Receipts', 'outgoing': 'Deliveries', + 'internal': 'Internal'}.get(code, code) + if ti.get('name') == 'Returns': + key = 'Returns' + e = monthly.setdefault(m, {'month': m, 'Receipts': 0, 'Deliveries': 0, + 'Internal': 0, 'Returns': 0}) + e[key] = e.get(key, 0) + 1 + monthly_rows = sorted(monthly.values(), key=lambda x: x['month']) + + # ---- outbound funnel cycle times (main warehouse Pick -> Pack -> Ship, + Putaway) -------- + stages, stage_m = {}, {} + for r in done: + if str(r['date_done'])[:10] < _y365: + continue + ti = types.get(O.m2o_id(r['picking_type_id']), {}) + if not ti.get('stage'): + continue + cr, dd, sch = _dtp(r['date']), _dtp(r['date_done']), _dtp(r['scheduled_date']) + if not (cr and dd): + continue + cyc = (dd - cr).total_seconds() / 86400 + s = stages.setdefault(ti['stage'], {'stage': ti['stage'], 'n': 0, + 'cycle': [], 'vs_sched': []}) + s['n'] += 1 + s['cycle'].append(cyc) + if sch: + s['vs_sched'].append((dd - sch).total_seconds() / 86400) + stage_m.setdefault((ti['stage'], str(r['date_done'])[:7]), []).append(cyc) + stage_rows = [] + for name in ('Pick', 'Pack', 'Ship', 'Putaway'): + s = stages.get(name) + if s: + stage_rows.append({'stage': name, 'done_12m': s['n'], + 'median_days': round(_median(s['cycle']) or 0, 2), + 'p90_days': round(_p90(s['cycle']) or 0, 2), + 'vs_sched_median': round(_median(s['vs_sched']) or 0, 2)}) + stage_monthly = sorted( + ({'stage': k[0], 'month': k[1], 'median_days': round(_median(v) or 0, 2), + 'p90_days': round(_p90(v) or 0, 2), 'n': len(v)} for k, v in stage_m.items()), + key=lambda x: (x['month'], x['stage'])) + + # ---- on-time ship (outgoing): monthly + WEEKLY (XmR grain) + clean-order -- + y365 = (t - dt.timedelta(days=365)).isoformat() + d90 = (t - dt.timedelta(days=90)).isoformat() + ontime_m, weekly, clean_m, wk_vol = {}, {}, {}, {} + ship_cycles = [] + clean90 = {'n': 0, 'clean': 0, 'late': 0, 'bo': 0} + for r in done: + ti = types.get(O.m2o_id(r['picking_type_id']), {}) + if ti.get('code') != 'outgoing': + continue + dd, sch, cr = _dtp(r['date_done']), _dtp(r['scheduled_date']), _dtp(r['date']) + if not dd: + continue + dstr = str(r['date_done'])[:10] + wk_key, wk_no = _iso_week(dd.date()) + wk_vol.setdefault(wk_key, {'week': wk_key, 'wk_no': wk_no, 'year': dd.year, 'n': 0}) + wk_vol[wk_key]['n'] += 1 + late = not (sch and (dd - sch).total_seconds() / 86400 <= ONTIME_GRACE_DAYS) + bo = bool(r.get('backorder_id')) + if dstr < y365: + continue # older-than-12m rows only feed the LY weekly compare + m = dstr[:7] + e = ontime_m.setdefault(m, {'month': m, 'shipped': 0, 'on_time': 0, + 'backorders': 0, 'cycle': []}) + e['shipped'] += 1 + if not late: + e['on_time'] += 1 + if bo: + e['backorders'] += 1 + if cr: + c = (dd - cr).total_seconds() / 86400 + e['cycle'].append(c) + ship_cycles.append(c) + w = weekly.setdefault(wk_key, {'week': wk_key, 'n': 0, 'on_time': 0}) + w['n'] += 1 + if not late: + w['on_time'] += 1 + cm = clean_m.setdefault(m, {'month': m, 'On time & complete': 0, 'Late only': 0, + 'Backordered only': 0, 'Late + backordered': 0}) + cat = ('On time & complete' if not late and not bo else + 'Late only' if late and not bo else + 'Backordered only' if bo and not late else 'Late + backordered') + cm[cat] += 1 + if dstr >= d90: + clean90['n'] += 1 + clean90['clean'] += (0 if (late or bo) else 1) + clean90['late'] += (1 if late else 0) + clean90['bo'] += (1 if bo else 0) + ontime_rows = [] + for m in sorted(ontime_m): + e = ontime_m[m] + ontime_rows.append({'month': m, 'shipped': e['shipped'], + 'on_time_pct': e['on_time'] / e['shipped'] * 100 if e['shipped'] else None, + 'median_days': round(_median(e['cycle']) or 0, 2), + 'backorder_pct': e['backorders'] / e['shipped'] * 100 if e['shipped'] else None}) + n_out = sum(e['shipped'] for e in ontime_m.values()) + n_ontime = sum(e['on_time'] for e in ontime_m.values()) + + # weekly on-time series (complete weeks only — drop the in-progress week) -> XmR + cur_wk = _iso_week(t)[0] + wk_series = [(k, weekly[k]['on_time'] / weekly[k]['n'] * 100) + for k in sorted(weekly) if k != cur_wk and weekly[k]['n'] >= 5] + xmr = _xmr(wk_series[-52:]) + + # weekly volume: last 13 complete weeks vs same ISO week last year + vol_rows = [] + for k in sorted(wk_vol): + v = wk_vol[k] + if k == cur_wk or v['year'] < t.year - 1: + continue + if v['year'] == t.year: + ly = next((x for x in wk_vol.values() + if x['wk_no'] == v['wk_no'] and x['year'] == v['year'] - 1), None) + vol_rows.append({'week': f"W{v['wk_no']:02d}", 'shipped': v['n'], + 'shipped_ly': ly['n'] if ly else 0}) + vol_rows = vol_rows[-13:] + clean_rows = [clean_m[m] for m in sorted(clean_m)] + clean_pct_90d = (clean90['clean'] / clean90['n'] * 100) if clean90['n'] else None + + # ---- backlog (open pickings, oldest first) + expedite list -------------- + today = dt.datetime.combine(t, dt.time()) + ship_med = _median([s for s in ship_cycles]) or 1.0 + bl, expedite = {}, [] + for r in open_: + ti = types.get(O.m2o_id(r['picking_type_id']), {}) + label = f"{ti.get('wh', '?')}: {ti.get('name', '?')}" + sch = _dtp(r['scheduled_date']) or _dtp(r['date']) + age = (today - sch).days if sch else None + e = bl.setdefault(label, {'queue': label, 'open': 0, 'overdue': 0, 'oldest_days': 0}) + e['open'] += 1 + if age is not None and age > 0: + e['overdue'] += 1 + e['oldest_days'] = max(e['oldest_days'], age) + # Expedite: outgoing, past scheduled+grace, waited longer than the normal ship cycle. + # 'waiting' state = unreserved stock -> that's a PROCUREMENT gap, routed separately + # (don't tell the warehouse to expedite what it cannot pick). + if (ti.get('code') == 'outgoing' and age is not None + and age > ONTIME_GRACE_DAYS and age > ship_med): + expedite.append({'order': r.get('origin') or '', 'customer': O.m2o_name(r.get('partner_id')), + 'queue': label, 'scheduled': str(r.get('scheduled_date') or '')[:10], + 'days_late': age, + 'blocked_on_stock': r.get('state') == 'waiting'}) + backlog_rows = sorted(bl.values(), key=lambda x: (-x['oldest_days'], -x['open'])) + expedite.sort(key=lambda x: -x['days_late']) + + n_done_12m = sum(1 for r in done if str(r['date_done'])[:10] >= _y365) + return { + 'monthly': monthly_rows, 'stages': stage_rows, 'stage_monthly': stage_monthly, + 'ontime': ontime_rows, 'backlog': backlog_rows, 'expedite': expedite, + 'xmr': xmr, 'weekly_vol': vol_rows, 'clean': clean_rows, + 'clean_pct_90d': clean_pct_90d, + 'kpi': { + 'done_12m': n_done_12m, 'open_now': len(open_), + 'ontime_pct': (n_ontime / n_out * 100) if n_out else None, + 'ship_median_days': round(_median(ship_cycles) or 0, 2), + 'shipped_12m': n_out, + 'backlog_overdue': sum(e['overdue'] for e in bl.values()), + }, + '_n_out_rowpath': n_out, + } + + +def adjustments(t=None, top=15): + """Inventory-adjustment moves (to/from an 'inventory'-usage location) = the shrinkage / + correction signal. Valued from Odoo's stock.valuation.layer (ACTUAL historical cost) where a + layer exists for the move; current standard cost only as fallback — upgraded 2026-07-05 from + the all-approximation version. Monthly gains/losses + the most-corrected SKUs.""" + t = t or P.today() + y1 = (t - dt.timedelta(days=365)).isoformat() + gains = O.search_read('stock.move', + [('state', '=', 'done'), ('date', '>=', y1), + ('location_id.usage', '=', 'inventory')], + ['product_id', 'product_uom_qty', 'date']) + losses = O.search_read('stock.move', + [('state', '=', 'done'), ('date', '>=', y1), + ('location_dest_id.usage', '=', 'inventory')], + ['product_id', 'product_uom_qty', 'date']) + pids = list({O.m2o_id(r['product_id']) for r in gains + losses if r.get('product_id')}) + cost = {} + for i in range(0, len(pids), 5000): + for p in O.search_read('product.product', + [('id', 'in', pids[i:i + 5000]), ('active', 'in', [True, False])], + ['default_code', 'standard_price']): + cost[p['id']] = {'code': (p.get('default_code') or '').strip() or f"#{p['id']}", + 'cost': p.get('standard_price') or 0.0} + + # Actual valuation per move from stock.valuation.layer (Σ |value| per move; a move can carry + # several layers). Moves WITH a layer row use it — even a legitimate 0; only layerless moves + # fall back to qty × current standard cost. + mids = [m['id'] for m in gains + losses] + layer_val, layer_seen = {}, set() + for i in range(0, len(mids), 5000): + for l in O.search_read('stock.valuation.layer', + [('stock_move_id', 'in', mids[i:i + 5000])], + ['stock_move_id', 'value']): + mid = O.m2o_id(l['stock_move_id']) + layer_seen.add(mid) + layer_val[mid] = layer_val.get(mid, 0.0) + abs(l.get('value') or 0.0) + + d90 = (t - dt.timedelta(days=90)).isoformat() + monthly, per_sku = {}, {} + n_layer = n_fallback = 0 + for rows, sign, key in ((gains, +1, 'gains'), (losses, -1, 'losses')): + for r in rows: + pid = O.m2o_id(r['product_id']) + c = cost.get(pid, {'code': '?', 'cost': 0.0}) + if r['id'] in layer_seen: + val = layer_val.get(r['id'], 0.0) + n_layer += 1 + else: + val = (r.get('product_uom_qty') or 0.0) * c['cost'] + n_fallback += 1 + m = str(r['date'])[:7] + e = monthly.setdefault(m, {'month': m, 'gains': 0.0, 'losses': 0.0, 'moves': 0}) + e[key] += val + e['moves'] += 1 + s = per_sku.setdefault(pid, {'sku': c['code'], 'product': O.m2o_name(r['product_id']), + 'net_qty': 0.0, 'net_value': 0.0, 'moves': 0, + 'moves_90d': 0, 'gross_value': 0.0}) + s['net_qty'] += sign * (r.get('product_uom_qty') or 0.0) + s['net_value'] += sign * val + s['gross_value'] += abs(val) + s['moves'] += 1 + if str(r['date'])[:10] >= d90: + s['moves_90d'] += 1 + monthly_rows = sorted(monthly.values(), key=lambda x: x['month']) + all_sku = list(per_sku.values()) + top_rows = sorted(all_sku, key=lambda x: -abs(x['net_value']))[:top] + # Pareto: cumulative share of gross correction value (the "N SKUs = 80% of corrections" view) + by_gross = sorted(all_sku, key=lambda x: -x['gross_value']) + tot_gross = sum(s['gross_value'] for s in by_gross) or 1.0 + run = 0.0 + pareto = [] + for s in by_gross[:30]: + run += s['gross_value'] + pareto.append({'sku': s['sku'], 'gross_value': s['gross_value'], + 'cum_pct': run / tot_gross * 100}) + # Worklists (exception-first: every flag terminates in an actionable list) + cc_list = sorted((s for s in all_sku if s['moves_90d'] >= 3), + key=lambda x: (-x['moves_90d'], -abs(x['net_value']))) + churn_list = sorted((s for s in all_sku + if s['gross_value'] > 500 and s['gross_value'] > 2 * abs(s['net_value'])), + key=lambda x: -x['gross_value']) + return {'monthly': monthly_rows, 'top': top_rows, 'pareto': pareto, + # FULL worklists — the UI paginates; a silent [:40] cap made the headline counts + # unverifiable in-app (owner rule: every sum drills to its rows) + 'cycle_count': cc_list, 'churn': churn_list, + 'n_cycle_count': len(cc_list), 'n_churn': len(churn_list), + 'total_losses': sum(m['losses'] for m in monthly_rows), + 'total_gains': sum(m['gains'] for m in monthly_rows), + 'n_moves': sum(m['moves'] for m in monthly_rows), + 'n_layer_valued': n_layer, 'n_cost_fallback': n_fallback, + '_rowsum_loss_qty': sum((r.get('product_uom_qty') or 0.0) for r in losses), + '_rowsum_layer_abs': sum(layer_val.values()), + '_y1': y1} + + +def touches(t=None, top=25): + """Most-handled SKUs by done stock-move count — where warehouse labor concentrates. + Carries default_code so the rows open the canonical SKU drawer (cross-module interop).""" + t = t or P.today() + y1 = (t - dt.timedelta(days=365)).isoformat() + g = O.read_group('stock.move', [('state', '=', 'done'), ('date', '>=', y1)], + ['product_uom_qty:sum'], ['product_id'], lazy=False) + rows = [] + total = 0 + for r in g: + n = r.get('__count') or 0 + total += n + if r.get('product_id'): + rows.append({'pid': O.m2o_id(r['product_id']), 'product': O.m2o_name(r['product_id']), + 'moves': n, 'units': r.get('product_uom_qty') or 0.0}) + rows.sort(key=lambda x: -x['moves']) + rows = rows[:top] + pids = [r['pid'] for r in rows] + codes = {p['id']: (p.get('default_code') or '').strip() + for p in O.search_read('product.product', + [('id', 'in', pids), ('active', 'in', [True, False])], + ['default_code'])} + for r in rows: + r['share_pct'] = r['moves'] / total * 100 if total else None + r['code'] = codes.get(r['pid'], '') + return {'top': rows, 'total_moves': total} + + +def _sop(diagnose, fix, verify, cadence): + """Embedded runbook per flag: versioned IN CODE (like metric definitions — an SOP that drifts + in a UI editor breaks the explainability contract). Rendered as a drill-down under the flag.""" + return {'diagnose': diagnose, 'fix': fix, 'verify': verify, 'cadence': cadence} + + +def flags(b, adj, ph=None): + """The recommendation engine: explainable rules from the warehouse research brief + (wiki/research/warehouse.md), each terminating in an action AND carrying its SOP drill-down. + Three Wheeler rules on the weekly on-time XmR, stage bottleneck/stalled-subset, clean-order + breach, backorder jump, backlog breach, capacity-ahead-of-season, adjustment worklists, + counting-program health.""" + out = [] + + if ph and ph.get('ratio_pct') is not None and ph['ratio_pct'] > 1: + out.append({'sev': 'high', + 'title': f"Corrections churn = {ph['ratio_pct']:.0f}% of inventory value per year " + "(the accepted band is 1%; world-class runs under 0.5%)", + 'detail': f"Gross adjustments {ph['gross_12m']:,.0f} against ~{ph['inv_value_approx']:,.0f} " + "of stock (ratio uses current-standard-cost inventory value - an approximation " + "for this ratio only).", + 'action': 'This is a counting-PROGRAM problem, not per-SKU noise. Until it heals, ' + 'quantity-based recommendations for flagged SKUs carry count-trust ' + 'warnings on the Procurement, Inventory and Assortment pages.', + 'sop': _sop( + ['Confirm the churn is process, not shrink: the gains-vs-losses chart ' + 'below - roughly offsetting bars = counting/UoM/receiving errors.', + 'The Pareto names where the value concentrates; the churn worklist ' + 'names the process suspects.'], + ['Stand up the weekly cycle-count ritual (worklist below, 10-20 SKUs/wk, ' + 'counter =/= approver, root-cause tag on every correction).', + 'Fix the top-5 churn SKUs\' unit-of-measure/receiving practice first - ' + 'one UoM fix can kill hundreds of corrections.', + 'Track this ratio monthly; it should fall as counts come clean.'], + ['Ratio trending toward 1%, then 0.5%; the cycle-count worklist shrinking ' + 'month over month.'], + 'Monthly program review; weekly count ritual.')}) + + sigs = [p for p in b['xmr']['points'][-8:] if p['signal']] + if sigs: + out.append({'sev': 'high', 'title': 'On-time shipping shows a real signal, not noise', + 'detail': '; '.join(f"{p['label']}: {p['value']:.0f}% ({p['signal']})" for p in sigs[-3:]), + 'action': 'Investigate those weeks specifically - carrier, holiday, stockout or ' + 'staffing. XmR limits say this is beyond routine variation.', + 'sop': _sop( + ['Open the signal week(s) in Odoo: Inventory > Transfers, filter Delivery ' + 'Orders, Done, date_done in that week, then sort by scheduled date.', + 'Classify the late ones: carrier pickup missed / short-staffed day / ' + 'stock not available (backorder created) / holiday cluster.', + 'One cause should cover most of the week - a signal week is a COMMON ' + 'cause, not many little ones.'], + ['Carrier: escalate with the carrier rep, add a second pickup on peak days.', + 'Staffing: move the pack-day roster; see the Weekly-volume chart for which ' + 'weekdays run hot.', + 'Stock: the backorder share in the Clean-orders chart confirms it - work ' + 'the Procurement buy list, not the floor.'], + ['The next 2 weekly points fall back inside the band.', + 'If 8+ points stay above/below center, re-baseline the chart (the process ' + 'genuinely changed).'], + 'On signal only - do NOT investigate routine (in-band) weeks.')}) + + if b.get('clean_pct_90d') is not None and b['clean_pct_90d'] < 90: + out.append({'sev': 'high', 'title': f"Clean-order rate {b['clean_pct_90d']:.0f}% (90d) - below the 90% bar", + 'detail': 'Clean = shipped on time AND complete (no backorder). Top-quartile B2B is 95%+.', + 'action': 'Decompose in the clean-order chart: a late-driven gap is a warehouse/carrier ' + 'issue; a backorder-driven gap is availability - route to Procurement.', + 'sop': _sop( + ['Read the Clean-orders chart: which slice eats the gap - Late only (gold), ' + 'Backordered only (gray), or both (red)?', + 'Late-driven: pull the expedite list + the signal-week SOP above.', + 'Backorder-driven: these SKUs stocked out at ship time - cross-reference ' + 'the Procurement buy list and the open-PO late chase list.'], + ['Late-driven: fix the floor/carrier cause (see the on-time SOP).', + 'Backorder-driven: expedite the late POs for the affected SKUs (Procurement ' + '> Late deliveries names the supplier and days late); raise reorder points ' + 'on repeat offenders.', + 'Both: work availability first - a complete late order beats an on-time ' + 'partial for most wholesale customers.'], + ['Trailing-90d clean rate back over 90% (top-quartile bar: 95%).'], + 'Weekly review until >90%, then monthly.')}) + + pr = b.get('promise') or {} + if pr.get('on_promise_pct') is not None and pr['on_promise_pct'] < 85: + out.append({'sev': 'high' if pr['on_promise_pct'] < 70 else 'medium', + 'title': f"Promise-date OTIF {pr['on_promise_pct']:.0f}% - customers get " + f"{len(pr['missed']):,} of {pr['n_measured']:,} promised orders late", + 'detail': 'Delivered vs the commitment_date QUOTED to the customer (+1d ' + f"grace); p90 lateness +{pr['late_p90']}d. The internal on-time " + 'XmR can look healthy while promises are still missed - the ' + 'promise is set at order entry, not by the warehouse.', + 'action': 'Fix the PROMISE first: quote commitment dates from the real ' + 'schedule (order-to-ship p50 + a buffer), then work the missed ' + 'list below.', + 'sop': _sop( + ['Open the Promise vs delivered section: compare the promise-OTIF trend ' + 'to the internal on-time XmR - a wide gap = promises are quoted ' + 'tighter than the process can ship.', + 'Sample 10 missed orders: was the commitment_date realistic at entry ' + '(vs the median order-to-ship time), or did the floor slip?'], + ['Unrealistic promises -> set commitment dates at entry from the real ' + 'p50 ship time + 1 day buffer (sales SOP, not a warehouse fix).', + 'Floor slippage -> the expedite list + backlog SOPs above own it.'], + ['Promise-OTIF trend back over 90% and converging with the internal ' + 'on-time line.'], + 'Weekly with the on-time review.')}) + + om = b['ontime'] + if len(om) >= 7: + last_bo = om[-1]['backorder_pct'] or 0 + base_bo = sum((r['backorder_pct'] or 0) for r in om[-7:-1]) / 6 + if last_bo - base_bo > 5: + out.append({'sev': 'medium', 'title': f"Backorder rate jumped to {last_bo:.0f}% (was ~{base_bo:.0f}%)", + 'detail': 'First-pass fill is slipping - an availability problem, not a floor problem.', + 'action': 'Cross-check the Procurement buy list and the open-PO late chase list.', + 'sop': _sop( + ['List last month\'s backordered deliveries in Odoo (Delivery Orders ' + 'with a backorder) and tally the missing SKUs.', + 'For each: is it on the Procurement BUY list (never ordered) or on the ' + 'open-PO LATE list (ordered, supplier late)?'], + ['Never ordered: place the PO - the buy list has qty and supplier.', + 'Supplier late: expedite via the late chase list; consider the ' + 'alternative vendor shown on the Procurement page.', + 'Chronic repeat SKUs: raise the reorder baseline (lead-time cover).'], + ['Backorder share back within 5pts of its 6-month norm.'], + 'Check at the biweekly procurement ritual.')}) + + sm = b['stage_monthly'] + for stage in ('Pick', 'Pack', 'Ship', 'Putaway'): + hist = [r for r in sm if r['stage'] == stage] + if len(hist) >= 7: + last = hist[-1] + base = sorted(r['p90_days'] for r in hist[-7:-1])[3] # median of prior 6 + if base > 0 and last['p90_days'] > 2 * base: + out.append({'sev': 'medium', 'title': f"{stage} p90 cycle {last['p90_days']:.1f}d - over 2x its 6-month norm ({base:.1f}d)", + 'detail': f"Month {last['month']}, n={last['n']}.", + 'action': f'Bottleneck forming at {stage} - rebalance labor or clear holds there.', + 'sop': _sop( + [f'Open the {stage} queue in Odoo, sort oldest first: is the tail a ' + 'few ancient transfers (holds) or is the WHOLE queue slower (capacity)?', + 'Compare the stage trend chart: did volume also jump that month?'], + ['Holds: clear or cancel the stuck transfers (supervisor sign-off).', + 'Capacity: shift labor from the fastest stage for 1-2 weeks; the ' + 'stage chart shows which stage has slack.'], + [f'{stage} p90 back under 2x its 6-month norm next month.'], + 'Monthly, at the stage-trend review.')}) + for s in b['stages']: + if s['median_days'] > 0 and s['p90_days'] / max(s['median_days'], 0.01) > 4 and s['p90_days'] > 1: + out.append({'sev': 'info', 'title': f"{s['stage']}: a subset of transfers stalls (p90 {s['p90_days']:.1f}d vs median {s['median_days']:.1f}d)", + 'detail': 'A long tail this wide is usually holds or stockouts, not capacity.', + 'action': 'Audit the stalled tail - the expedite list below names the current ones.', + 'sop': _sop( + [f'The median {s["stage"]} transfer takes {s["median_days"]:.1f}d - the slowest ' + '10% take 4x+ longer. Sample 10 of the slowest done transfers this month ' + '(Odoo: sort by date_done minus create date).', + 'Tag each: waiting on stock / waiting on a person or approval / lost paperwork.'], + ['Stock waits: those belong to Procurement (see the stock-blocked flag).', + 'Approval waits: name the approval and set a same-day rule for it.', + 'Lost paperwork: cancel-and-recreate is usually cheaper than archaeology.'], + ['p90/median ratio trending toward 2-3x within two months.'], + 'Monthly sample of 10.')}) + + k = b['kpi'] + if k['open_now'] and k['backlog_overdue'] / k['open_now'] > 0.10: + out.append({'sev': 'high', 'title': f"{k['backlog_overdue']:,} of {k['open_now']:,} open transfers are past schedule " + f"({k['backlog_overdue'] / k['open_now'] * 100:.0f}% - the healthy bar is under 10%)", + 'detail': 'Old queues (see Backlog) almost always contain dead paperwork as well as real work.', + 'action': 'Clear or cancel the ancient queues (Putaway/Move Back to Stock first); ' + 'then work the expedite list daily until past-due is under 5%.', + 'sop': _sop( + ['Backlog table below, oldest queue first: anything older than ~60 days is ' + 'almost certainly dead paperwork, not real work.', + 'For each ancient queue, sample 10 transfers in Odoo: does the stock ' + 'movement it describes still need to happen?'], + ['Dead paperwork: CANCEL in Odoo (ops supervisor does it - this app is ' + 'read-only by design). One focused afternoon usually clears years.', + 'Real work: schedule a catch-up block per queue (Putaway first - it ' + 'poisons on-hand accuracy, which poisons the buy list).', + 'Then: the expedite list is the daily ritual until past-due < 5%.'], + ['Past-due share under 10% within a month, 5% steady-state; oldest-days ' + 'column under 30 everywhere.'], + 'One-time cleanup, then daily expedite ritual + weekly backlog glance.')}) + + vol = b['weekly_vol'] + if len(vol) >= 4: + now4 = sum(r['shipped'] for r in vol[-4:]) + ly4 = sum(r['shipped_ly'] for r in vol[-4:]) + p90_rising = False + ship_hist = [r for r in sm if r['stage'] == 'Ship'] + if len(ship_hist) >= 7: + p90_rising = ship_hist[-1]['p90_days'] > sorted(r['p90_days'] for r in ship_hist[-7:-1])[3] + if ly4 and now4 / ly4 > 1.15 and p90_rising: + out.append({'sev': 'medium', 'title': f'Volume running {now4 / ly4 * 100 - 100:.0f}% above last year AND ship times rising', + 'detail': f'Last 4 weeks: {now4:,} shipped vs {ly4:,} same weeks last year.', + 'action': 'Capacity signal - open the staffing conversation now (practice: hire ' + '8-12 weeks before peak).', + 'sop': _sop( + ['Weekly-volume chart: how many weeks until the seasonal peak (last ' + 'year\'s shape shows it)?', + 'Estimate the gap: volume % above LY ~= extra hands needed % (floor ' + 'labor scales near-linearly at this size).'], + ['Under 12 weeks to peak: start temp hiring NOW (8-12 week practice; ' + '12-14 in a tight market).', + 'Over 12 weeks: schedule the decision, prep onboarding docs.', + 'Short-term relief: shift labor to the bottleneck stage (stage trend ' + 'chart names it).'], + ['Ship p90 flat while volume grows; on-time XmR stays in band through ' + 'the peak.'], + 'Re-check weekly during the run-up.')}) + + if adj['n_cycle_count']: + out.append({'sev': 'medium', 'title': f"{adj['n_cycle_count']} SKUs adjusted 3+ times in 90 days", + 'detail': 'Repeat corrections = the system count for these is not trusted.', + 'action': 'Work the cycle-count list below weekly until each SKU has two clean counts.', + 'sop': _sop( + ['Print the cycle-count worklist (top of the list first - it is ranked by ' + 'repeat count then value).', + 'For each SKU note WHERE the count went wrong last time (the SKU drawer ' + 'shows its movement; the churn list says if it is a process error).'], + ['Count 10-20 SKUs per week from the top of the list.', + 'Counter and approver are DIFFERENT people (segregation - the practice ' + 'that keeps counts honest).', + 'Tag every correction with a root cause (miscount / damage / receiving ' + 'error / UoM) - the tags are what fix the process.', + 'A SKU exits the list after TWO consecutive clean counts.'], + ['Worklist shrinking month over month; adjustment gross $ trending down ' + 'on the monthly chart.'], + 'Weekly, 10-20 SKUs; the list re-ranks itself as data updates.')}) + if adj['n_churn']: + out.append({'sev': 'medium', 'title': f"{adj['n_churn']} SKUs churn both ways (gross corrections > 2x net)", + 'detail': 'Offsetting gains and losses = a counting/receiving/UoM process error, not shrink.', + 'action': 'Check unit-of-measure, receiving and put-away practice for the churn list below.', + 'sop': _sop( + ['Take the top 5 churn SKUs: open each in the SKU drawer and note the pack ' + 'size (e.g. "288-Piece per Pack" - candles are classic UoM churn).', + 'Ask receiving: are these counted in PIECES or PACKS on arrival? Ask the ' + 'floor: which unit does picking use?'], + ['UoM mismatch found: fix the unit on the product / retrain the count ' + 'habit - one setting usually kills hundreds of corrections.', + 'Put-away confusion (same SKU in several spots): consolidate locations.', + 'Receiving misses: count at the dock against the PO line, not from the ' + 'packing slip.'], + ['The SKU stops appearing in the churn list within two cycles; gross ' + 'corrections fall while net stays flat.'], + 'Top-5 review at the weekly count session.')}) + + real_exp = [e for e in b['expedite'] if not e['blocked_on_stock']] + stock_exp = [e for e in b['expedite'] if e['blocked_on_stock']] + if real_exp: + out.append({'sev': 'medium', 'title': f'{len(real_exp)} shipments to expedite today', + 'detail': 'Past schedule and older than the normal ship cycle, stock available.', + 'action': 'Expedite list below, worst first.', + 'sop': _sop( + ['Expedite list below, top rows first (worst days-late).', + 'For each: confirm in Odoo the stock is actually reserved (assigned ' + 'state) and the order is not on customer hold.'], + ['Pick-pack-ship the top of the list TODAY; anything a customer has ' + 'called about jumps the queue.', + 'On hold / no longer wanted: cancel the transfer so it stops polluting ' + 'the backlog stats.'], + ['Expedite list under 10 rows and oldest under 7 days late.'], + 'Daily, first thing - it is a 15-minute ritual once the backlog is clean.')}) + if stock_exp: + out.append({'sev': 'info', 'title': f'{len(stock_exp)} late shipments are blocked on STOCK, not the floor', + 'detail': 'Waiting-state transfers cannot be picked - expediting them is pointless.', + 'action': 'Routed to Procurement: cross-check the buy list / open-PO ETAs for these.', + 'sop': _sop( + ['Filter the expedite list to "Blocked on stock = YES" - these SKUs are ' + 'sold but not on the shelf.', + 'Procurement page: is each SKU on an open PO (check the inbound-matched ' + 'table for its ETA) or not ordered at all?'], + ['On a PO: tell the customer the ETA; expedite the PO if the customer ' + 'matters (late chase list has the supplier contact context).', + 'Not ordered: it should be flashing on the buy list - order it or offer ' + 'the customer a substitute.'], + ['Stock-blocked count trending to near zero outside deep season.'], + 'Reviewed at the biweekly procurement ritual.')}) + + sev_rank = {'high': 0, 'medium': 1, 'info': 2} + out.sort(key=lambda f: sev_rank.get(f['sev'], 9)) + return out + + +def untrusted_counts(t=None): + """The cross-module COUNT-TRUST set: default_code -> adjustment count for SKUs adjusted 3+ + times in the last 90 days. For these, the system quantity is unverified until two clean + cycle counts — any recommendation built on their on-hand (buy list cover, dead-stock DSI, + season readiness) should say so. Deliberately lightweight (grouped reads over 90 days) so + any page can join it without running the full adjustments().""" + t = t or P.today() + d90 = (t - dt.timedelta(days=90)).isoformat() + per = {} + for loc in ('location_id', 'location_dest_id'): + for g in O.read_group('stock.move', + [('state', '=', 'done'), ('date', '>=', d90), + (loc + '.usage', '=', 'inventory')], + ['id'], ['product_id'], lazy=False): + if g.get('product_id'): + pid = O.m2o_id(g['product_id']) + per[pid] = per.get(pid, 0) + (g.get('__count') or 0) + pids = [p for p, n in per.items() if n >= 3] + codes = {} + for i in range(0, len(pids), 5000): + for p in O.search_read('product.product', + [('id', 'in', pids[i:i + 5000]), ('active', 'in', [True, False])], + ['default_code']): + c = (p.get('default_code') or '').strip() + if c: + codes[c] = per[p['id']] + return codes + + +def program_health(adj): + """Counting-program health: gross corrections (12m, layer-valued) as a share of inventory + value. Inventory value here is qty x CURRENT standard cost — an approximation used only for + this RATIO (the GL-tied book value lives in Overstock). WERC practice band: <=0.5% target, + ~1% acceptable, beyond that = formal program review.""" + prods = O.search_read('product.product', + [('default_code', '!=', False), ('active', 'in', [True, False])], + ['default_code', 'name', 'qty_available', 'standard_price']) + valued = [{'code': (p.get('default_code') or '').strip(), 'product': p.get('name') or '', + 'qty': p.get('qty_available') or 0.0, 'std_cost': p.get('standard_price') or 0.0, + 'value': (p.get('qty_available') or 0.0) * (p.get('standard_price') or 0.0)} + for p in prods if (p.get('qty_available') or 0.0) > 0] + inv_val = sum(v['value'] for v in valued) + gross = adj['total_losses'] + adj['total_gains'] + # denominator composition, so the ratio is verifiable in-app down to SKUs (owner rule); + # the numerator's rows are the adjustment worklists/Pareto on the same page + valued.sort(key=lambda v: -v['value']) + return {'inv_value_approx': inv_val, 'gross_12m': gross, + 'ratio_pct': (gross / inv_val * 100) if inv_val else None, + 'n_skus_valued': len(valued), 'value_rows': valued} + + +def freight(t=None): + """Carrier mix + delivery-charge economics: who we ship with, what delivery revenue we + charge, and who rides the free-delivery carrier (the coverage question: free delivery for a + customer we then pay Uber/local fees to serve).""" + t = t or P.today() + y1 = (t - dt.timedelta(days=365)).isoformat() + mix = [] + for g in O.read_group('stock.picking', + [('state', '=', 'done'), ('date_done', '>=', y1), + ('carrier_id', '!=', False)], + ['id'], ['carrier_id'], lazy=False): + if g.get('carrier_id'): + mix.append({'carrier': O.m2o_name(g['carrier_id']), 'pickings': g.get('__count') or 0}) + mix.sort(key=lambda x: -x['pickings']) + tot_pk = sum(m['pickings'] for m in mix) or 1 + for m in mix: + m['share_pct'] = m['pickings'] / tot_pk * 100 + + ex = O.excluded_partner_ids() + dom = [('is_delivery', '=', True), ('order_id.state', 'in', ['sale', 'done']), + ('order_id.date_order', '>=', y1)] + if ex: + dom.append(('order_partner_id', 'not in', list(ex))) + lines = O.search_read('sale.order.line', dom, + ['price_subtotal', 'order_partner_id', 'create_date']) + monthly = {} + for l in lines: + m = str(l.get('create_date') or '')[:7] + e = monthly.setdefault(m, {'month': m, 'revenue': 0.0, 'lines': 0}) + e['revenue'] += l.get('price_subtotal') or 0.0 + e['lines'] += 1 + rev_total = sum(l.get('price_subtotal') or 0.0 for l in lines) + + free = [] + for g in O.read_group('stock.picking', + [('state', '=', 'done'), ('date_done', '>=', y1), + ('carrier_id.name', 'ilike', 'free')], + ['id'], ['partner_id'], lazy=False): + if g.get('partner_id'): + free.append({'pid': O.m2o_id(g['partner_id']), 'customer': O.m2o_name(g['partner_id']), + 'free_deliveries': g.get('__count') or 0}) + free.sort(key=lambda x: -x['free_deliveries']) + + return {'mix': mix, 'monthly': sorted(monthly.values(), key=lambda x: x['month']), + 'revenue_12m': rev_total, 'n_charge_lines': len(lines), + 'free_top': free[:20], 'n_free_customers': len(free), + 'n_free_deliveries': sum(f['free_deliveries'] for f in free), + '_rowsum_rev': rev_total, '_dom_is_delivery': dom} + + +def promise_otif(t=None): + """Delivered vs PROMISED — the CUSTOMER's on-time, distinct from the internal schedule the + on-time XmR measures. 94% of confirmed orders carry a commitment_date; an order is + on-promise when its LAST completed outgoing shipment finished by that date +1 day (the same + grace the internal measure uses). Measured set = promised orders that HAVE shipped.""" + t = t or P.today() + d12 = (t - dt.timedelta(days=365)).isoformat() + orders = O.search_read('sale.order', + [('state', '=', 'sale'), ('date_order', '>=', d12), + ('commitment_date', '!=', False)], + ['name', 'partner_id', 'commitment_date', 'date_order']) + picks = O.search_read('stock.picking', + [('picking_type_code', '=', 'outgoing'), ('state', '=', 'done'), + ('date_done', '>=', d12)], + ['origin', 'date_done']) + last_done = {} + for p_ in picks: + o_, d = p_.get('origin'), str(p_.get('date_done') or '')[:10] + if o_ and d and (o_ not in last_done or d > last_done[o_]): + last_done[o_] = d + rows = [] + for so in orders: + dd = last_done.get(so['name']) + if not dd: + continue + promised = str(so['commitment_date'])[:10] + rows.append({'order': so['name'], 'pid': O.m2o_id(so.get('partner_id')), + 'customer': O.m2o_name(so.get('partner_id')), + 'promised': promised, 'delivered': dd, + 'days_late': (dt.date.fromisoformat(dd) + - dt.date.fromisoformat(promised)).days}) + n = len(rows) + ontime = sum(1 for r in rows if r['days_late'] <= 1) + lates = sorted(r['days_late'] for r in rows) + per_m = {} + for r in rows: + e = per_m.setdefault(r['promised'][:7], [0, 0]) + e[0] += 1 + e[1] += 1 if r['days_late'] <= 1 else 0 + return {'n_measured': n, 'n_promised': len(orders), 'n_unshipped': len(orders) - n, + 'n_on_promise': ontime, + 'on_promise_pct': (ontime / n * 100) if n else None, + 'late_p90': lates[int(len(lates) * .9)] if lates else None, + 'missed': sorted((r for r in rows if r['days_late'] > 1), + key=lambda r: -r['days_late']), + # trend only for months with a real sample — orders PROMISED before the pull window + # can ship inside it and leave a 1-2 order month that drags the line to 0/100% + 'trend': [{'month': k, 'measured': v[0], + 'on_promise_pct': v[1] / v[0] * 100} + for k, v in sorted(per_m.items()) if v[0] >= 10], + '_d12': d12} + + +def summary(t=None): + t = t or P.today() + b = build(t) + b['adjustments'] = adjustments(t) + b['touches'] = touches(t) + b['freight'] = freight(t) + b['program_health'] = program_health(b['adjustments']) + b['promise'] = promise_otif(t) + b['flags'] = flags(b, b['adjustments'], b['program_health']) + b['pulled_at'] = dt.datetime.now().strftime('%Y-%m-%d %H:%M') + return b + + +def validate(t=None, team_id=None, pre=None, adj=None, fr=None): + """pre/adj/fr: pass the bundle's precomputed build()/adjustments()/freight() to avoid + re-running the heavy pulls (the ar.py lesson); validate.py calls bare and computes fresh.""" + t = t or P.today() + y1 = (t - dt.timedelta(days=365)).isoformat() + o = O.get_odoo() + b = pre or build(t) + checks = [] + + n_out_direct = o.search_count('stock.picking', + [('state', '=', 'done'), ('date_done', '>=', y1), + ('picking_type_code', '=', 'outgoing')]) + checks.append({'check': 'Warehouse: row-built outgoing done == search_count (12m)', + 'a': b['_n_out_rowpath'], 'b': n_out_direct, + 'gap': b['_n_out_rowpath'] - n_out_direct, + 'ok': abs(b['_n_out_rowpath'] - n_out_direct) <= max(2, n_out_direct * 0.001)}) + + pr = b.get('promise') or promise_otif(t) + if pr: + n_srv = o.search_count('sale.order', + [('state', '=', 'sale'), ('date_order', '>=', pr['_d12']), + ('commitment_date', '!=', False)]) + checks.append({'check': 'Promise-OTIF: promised-order universe == server count', + 'a': pr['n_promised'], 'b': n_srv, 'gap': pr['n_promised'] - n_srv, + 'ok': abs(pr['n_promised'] - n_srv) <= max(2, n_srv * 0.001)}) + part = pr['n_on_promise'] + len(pr['missed']) + checks.append({'check': 'Promise-OTIF: on-promise + missed == measured (partition)', + 'a': part, 'b': pr['n_measured'], 'gap': part - pr['n_measured'], + 'ok': part == pr['n_measured']}) + + mm = sum(sum(v for k, v in m.items() if k != 'month') for m in b['monthly']) + n_all = o.search_count('stock.picking', [('state', '=', 'done'), ('date_done', '>=', y1)]) + checks.append({'check': 'Warehouse: Σ(monthly throughput) == all done pickings (12m)', + 'a': mm, 'b': n_all, 'gap': mm - n_all, + 'ok': abs(mm - n_all) <= max(2, n_all * 0.001)}) + + # Clean-order decomposition is exhaustive: the four categories partition every shipped picking. + clean_sum = sum(sum(v for kk, v in m.items() if kk != 'month') for m in b['clean']) + ship_sum = sum(r['shipped'] for r in b['ontime']) + checks.append({'check': 'Warehouse: Σ(clean-order categories) == Σ shipped (partition test)', + 'a': clean_sum, 'b': ship_sum, 'gap': clean_sum - ship_sum, + 'ok': clean_sum == ship_sum}) + + adj = adj or adjustments(t) + agg_loss_qty = O.sum_field('stock.move', + [('state', '=', 'done'), ('date', '>=', adj['_y1']), + ('location_dest_id.usage', '=', 'inventory')], + 'product_uom_qty') + checks.append({'check': 'Warehouse: adjustment loss qty row-sum == read_group Σ (live-drift tol)', + 'a': round(adj['_rowsum_loss_qty'], 2), 'b': round(agg_loss_qty, 2), + 'gap': round(adj['_rowsum_loss_qty'] - agg_loss_qty, 2), + 'ok': abs(adj['_rowsum_loss_qty'] - agg_loss_qty) <= max(1.0, abs(agg_loss_qty) * 0.001)}) + + # Valuation layers: our per-move Σ|value| vs the signed read_group sums over the same layer + # domains (gains positive, losses negative → |gains| + |losses| should tie). + lay_g = O.sum_field('stock.valuation.layer', + [('stock_move_id.state', '=', 'done'), ('stock_move_id.date', '>=', adj['_y1']), + ('stock_move_id.location_id.usage', '=', 'inventory')], 'value') + lay_l = O.sum_field('stock.valuation.layer', + [('stock_move_id.state', '=', 'done'), ('stock_move_id.date', '>=', adj['_y1']), + ('stock_move_id.location_dest_id.usage', '=', 'inventory')], 'value') + lay_agg = abs(lay_g) + abs(lay_l) + checks.append({'check': 'Warehouse: Σ|layer value| row-path == |read_group gains|+|losses| (tol)', + 'a': round(adj['_rowsum_layer_abs'], 2), 'b': round(lay_agg, 2), + 'gap': round(adj['_rowsum_layer_abs'] - lay_agg, 2), + 'ok': abs(adj['_rowsum_layer_abs'] - lay_agg) <= max(5.0, lay_agg * 0.005)}) + + # Freight: delivery-charge revenue row-sum vs an independent read_group sum, same domain. + fr = fr or freight(t) + agg_rev = O.sum_field('sale.order.line', fr['_dom_is_delivery'], 'price_subtotal') + checks.append({'check': 'Warehouse: Σ delivery-charge revenue row-sum == read_group Σ (tol)', + 'a': round(fr['_rowsum_rev'], 2), 'b': round(agg_rev, 2), + 'gap': round(fr['_rowsum_rev'] - agg_rev, 2), + 'ok': abs(fr['_rowsum_rev'] - agg_rev) <= max(1.0, abs(agg_rev) * 0.001)}) + return checks diff --git a/royalimports_os/odoo_client.py b/royalimports_os/odoo_client.py new file mode 100644 index 0000000000000000000000000000000000000000..94094548d59a8cb6ddd0d7cf0b7ce2d8bef57ec2 --- /dev/null +++ b/royalimports_os/odoo_client.py @@ -0,0 +1,127 @@ +"""Read-only Odoo XML-RPC client for the FFS/RI analytics dashboard. + +Credentials come from environment variables (local .env OR Hugging Face Secrets): + ODOO_URL, ODOO_DB, ODOO_USER, ODOO_API_KEY + +READ_ONLY is hard-enforced: any write/create/delete/action method is blocked. +This is a defense-in-depth measure on top of using a read-only Odoo user. +""" +import os +import http.client +import xmlrpc.client + +try: + from dotenv import load_dotenv + from pathlib import Path + load_dotenv(Path(__file__).parent / '.env') +except Exception: + pass # On HF, env comes from Secrets — no .env file needed. + +WRITE_METHODS = { + 'write', 'create', 'unlink', 'copy', 'load', 'import_data', 'execute', +} +WRITE_PREFIXES = ('write_', 'create_', 'unlink_', 'delete_', 'remove_', 'update_', + 'set_', 'do_', 'action_', 'button_', 'process_', 'send_', + 'cancel_', 'confirm_', 'approve_', 'validate_', 'commit_', 'apply_') + + +class ReadOnlyError(RuntimeError): + pass + + +# stdlib ServerProxy has NO timeout: a stalled socket hangs a worker forever (one cause of the +# transport-break hangs the reconnect logic below papers over). Generous default — this is a +# hang-guard, not a performance knob; big ledger pulls legitimately take a while. +_RPC_TIMEOUT = float(os.environ.get('ODOO_RPC_TIMEOUT', '180')) + + +class _TimeoutSafeTransport(xmlrpc.client.SafeTransport): + def make_connection(self, host): + conn = super().make_connection(host) + conn.timeout = _RPC_TIMEOUT + return conn + + +class _TimeoutTransport(xmlrpc.client.Transport): + def make_connection(self, host): + conn = super().make_connection(host) + conn.timeout = _RPC_TIMEOUT + return conn + + +def _proxy(uri): + tr = _TimeoutSafeTransport() if uri.startswith('https') else _TimeoutTransport() + return xmlrpc.client.ServerProxy(uri, transport=tr) + + +class OdooClient: + READ_ONLY = True + + def __init__(self): + self.url = os.environ.get('ODOO_URL', '').rstrip('/') + self.db = os.environ.get('ODOO_DB', '') + self.user = os.environ.get('ODOO_USER', '') + self.api_key = os.environ.get('ODOO_API_KEY', '') + missing = [k for k, v in [('ODOO_URL', self.url), ('ODOO_DB', self.db), + ('ODOO_USER', self.user), ('ODOO_API_KEY', self.api_key)] if not v] + if missing: + raise RuntimeError(f"Missing Odoo env vars: {', '.join(missing)}") + self._connect() + + def _connect(self, _retried=False): + """(Re)establish the XML-RPC connection. Called on init and to recover a broken socket. + uid is cached for the client's lifetime (authenticate is just a uid lookup — never + re-fire it per request). One retry on a transient connect failure (WinError 10060 etc.) + so a momentary network blip during prewarm doesn't error a whole module.""" + try: + common = _proxy(f'{self.url}/xmlrpc/2/common') + self.uid = common.authenticate(self.db, self.user, self.api_key, {}) + except (http.client.HTTPException, ConnectionError, OSError): + if _retried: + raise + import time as _time + _time.sleep(3) + return self._connect(_retried=True) + if not self.uid: + raise RuntimeError("Odoo authentication failed — check credentials.") + self.models = _proxy(f'{self.url}/xmlrpc/2/object') + + def _guard(self, method): + if self.READ_ONLY and (method in WRITE_METHODS or any(method.startswith(p) for p in WRITE_PREFIXES)): + raise ReadOnlyError(f"BLOCKED write op '{method}' — dashboard is read-only.") + + def execute_kw(self, model, method, args, kwargs=None): + self._guard(method) + try: + return self.models.execute_kw(self.db, self.uid, self.api_key, model, method, args, kwargs or {}) + except (http.client.HTTPException, ConnectionError, OSError) as e: + # transient transport breakage (CannotSendRequest / broken pipe / reset) leaves the + # persistent socket dead — reconnect once and retry. xmlrpc Faults are NOT caught here, + # so genuine Odoo errors still propagate. + try: + self._connect() + except Exception: + raise e + return self.models.execute_kw(self.db, self.uid, self.api_key, model, method, args, kwargs or {}) + + def search_read(self, model, domain=None, fields=None, limit=None, offset=0, order=None): + kwargs = {'fields': fields or [], 'offset': offset} + if limit is not None: + kwargs['limit'] = limit + if order: + kwargs['order'] = order + return self.execute_kw(model, 'search_read', [domain or []], kwargs) + + def search_count(self, model, domain=None): + return self.execute_kw(model, 'search_count', [domain or []]) + + def read_group(self, model, domain=None, fields=None, groupby=None, lazy=False, orderby=None, limit=None): + kwargs = {'fields': fields or [], 'groupby': groupby or [], 'lazy': lazy} + if orderby: + kwargs['orderby'] = orderby + if limit is not None: + kwargs['limit'] = limit + return self.execute_kw(model, 'read_group', [domain or []], kwargs) + + def fields_get(self, model, attributes=('string', 'type')): + return self.execute_kw(model, 'fields_get', [], {'attributes': list(attributes)}) diff --git a/web/.gitignore b/web/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a547bf36d8d11a4f89c59c144f24795749086dd1 --- /dev/null +++ b/web/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/web/.oxlintrc.json b/web/.oxlintrc.json new file mode 100644 index 0000000000000000000000000000000000000000..6fa991dad24b57eb19c9cefba8ea12fd1870ed07 --- /dev/null +++ b/web/.oxlintrc.json @@ -0,0 +1,8 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["react", "typescript", "oxc"], + "rules": { + "react/rules-of-hooks": "error", + "react/only-export-components": ["warn", { "allowConstantExport": true }] + } +} diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d6af7e39ffac6183b891838d6ab1d6eb7ac70dda --- /dev/null +++ b/web/README.md @@ -0,0 +1,32 @@ +# React + TypeScript + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some Oxlint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs) +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) + +## React Compiler + +The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). + +## Expanding the Oxlint configuration + +If you are developing a production application, we recommend enabling type-aware lint rules by installing `oxlint-tsgolint` and editing `.oxlintrc.json`: + +```json +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["react", "typescript", "oxc"], + "options": { + "typeAware": true + }, + "rules": { + "react/rules-of-hooks": "error", + "react/only-export-components": ["warn", { "allowConstantExport": true }] + } +} +``` + +See the [Oxlint rules documentation](https://oxc.rs/docs/guide/usage/linter/rules) for the full list of rules and categories. diff --git a/web/dist-embed/favicon.svg b/web/dist-embed/favicon.svg new file mode 100644 index 0000000000000000000000000000000000000000..a4f82828dbb9622d8d6c781b1243f91d7fee89b7 --- /dev/null +++ b/web/dist-embed/favicon.svg @@ -0,0 +1,8 @@ + + diff --git a/web/dist-embed/index.html b/web/dist-embed/index.html new file mode 100644 index 0000000000000000000000000000000000000000..21b132bb4db19b768d8480e698990eef104b04b6 --- /dev/null +++ b/web/dist-embed/index.html @@ -0,0 +1,225 @@ + + + + + + Loopable + + + + +
+
+ + diff --git a/web/dist-embed/sample_customers.json b/web/dist-embed/sample_customers.json new file mode 100644 index 0000000000000000000000000000000000000000..043835a1491db59996905c377bcca2cf276df4bc --- /dev/null +++ b/web/dist-embed/sample_customers.json @@ -0,0 +1,466 @@ +{ + "fields": [ + { + "key": "customer", + "label": "Customer", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + }, + { + "key": "status", + "label": "Status", + "type": "status", + "source": "odoo", + "default": true, + "description": "Where this customer stands versus last year: New, Growing, Declining, Lost or Dormant." + }, + { + "key": "odoo_status", + "label": "Odoo record", + "type": "status", + "source": "odoo", + "default": false, + "description": "Whether this customer still exists in Odoo. Archived means deleted there." + }, + { + "key": "agent", + "label": "Agent", + "type": "text", + "source": "odoo", + "default": true, + "description": "The sales agent who owns this account." + }, + { + "key": "salesperson", + "label": "Salesperson", + "type": "text", + "source": "odoo", + "default": false, + "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." + }, + { + "key": "city", + "label": "City", + "type": "text", + "source": "odoo", + "default": true, + "description": "City on the customer's Odoo address." + }, + { + "key": "state", + "label": "State", + "type": "text", + "source": "odoo", + "default": true, + "description": "State or province on the customer's Odoo address." + }, + { + "key": "country", + "label": "Country", + "type": "text", + "source": "odoo", + "default": false, + "description": "Country on the customer's Odoo address." + }, + { + "key": "zip", + "label": "ZIP", + "type": "text", + "source": "odoo", + "default": false, + "description": "Postal code on the customer's Odoo address." + }, + { + "key": "customer_since", + "label": "Customer since", + "type": "date", + "source": "odoo", + "default": false, + "description": "When this customer was first set up in Odoo." + }, + { + "key": "tags", + "label": "Tags", + "type": "text", + "source": "odoo", + "default": false, + "description": "Odoo labels on this customer, comma-separated." + }, + { + "key": "pricelist", + "label": "Price list", + "type": "text", + "source": "odoo", + "default": false, + "description": "The price list this customer buys on." + }, + { + "key": "payment_terms", + "label": "Payment terms", + "type": "text", + "source": "odoo", + "default": false, + "description": "Payment terms on this customer's account — Net 30, for example." + }, + { + "key": "last_order", + "label": "Last order", + "type": "date", + "source": "odoo", + "default": true, + "description": "Date of the most recent confirmed order." + }, + { + "key": "overdue_days", + "label": "Overdue days", + "type": "int", + "source": "odoo", + "default": true, + "description": "How many days late this customer is running against their own usual ordering rhythm." + }, + { + "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", + "key": "est_missed", + "label": "Est. missed $", + "type": "currency", + "source": "odoo", + "default": true, + "agg": "sum", + "filterable": false, + "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + }, + { + "key": "ar_open", + "label": "AR open $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money owed, not yet overdue." + }, + { + "key": "ar_overdue", + "label": "AR overdue $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money past due — same basis as the Collections page." + }, + { + "key": "ar_exposure", + "label": "Credit exposure $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + }, + { + "key": "days_to_pay", + "label": "Days to pay", + "type": "int", + "source": "odoo", + "default": false, + "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + }, + { + "key": "top_category", + "label": "Top category", + "type": "text", + "source": "odoo", + "default": false, + "description": "The category this customer spent the most on in the last 12 months." + }, + { + "key": "top_category_pct", + "label": "Top category %", + "type": "pct", + "source": "odoo", + "default": false, + "description": "Share of last-12-months spend that went to the top category." + }, + { + "key": "sku_count", + "label": "SKUs bought", + "type": "int", + "source": "odoo", + "default": false, + "description": "Distinct products bought in the last 12 months." + }, + { + "key": "top_sku", + "label": "Top SKU", + "type": "text", + "source": "odoo", + "default": false, + "description": "The product this customer spent the most on in the last 12 months." + }, + { + "key": "days_since", + "label": "Days since order", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days since the last confirmed order." + }, + { + "key": "typical_gap_days", + "label": "Typical gap days", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days this customer usually goes between orders, from their own history." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "description": "Your notes on this customer. Saved in this app only, visible only to you." + } + ], + "rows": [ + { + "pid": 101, + "customer": "Poppy Flowers", + "status": "New", + "agent": "Naomi Linnell Rivera", + "city": "Charlottesville", + "state": "Virginia (US)", + "last_order": "2026-07-21", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "02720", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Royal", + "customer_since": "2023-01-10", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-01-15 09:10:00", + "lat": 25.7617, + "lon": -80.1918, + "odoo_status": "Archived" + }, + { + "pid": 102, + "customer": "Meadow & Vine Wholesale", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Portland", + "state": "Oregon (US)", + "last_order": "2026-07-19", + "est_missed": 0, + "notes": "Expanding to a second storefront.", + "country": "United States", + "zip": "77041", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-02-11", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-02-15 09:11:00", + "lat": 27.9506, + "lon": -82.4572, + "odoo_status": "Active" + }, + { + "pid": 103, + "customer": "Bluestem Floral Supply", + "status": "Growing", + "agent": "Naomi Linnell Rivera", + "city": "Kansas City", + "state": "Missouri (US)", + "last_order": "2026-07-17", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "11219", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-03-12", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-03-15 09:12:00", + "lat": 28.5384, + "lon": -81.3789, + "odoo_status": "Active" + }, + { + "pid": 104, + "customer": "Camellia Row Florist", + "status": "Declining", + "agent": "Devon Marsh", + "city": "Savannah", + "state": "Georgia (US)", + "last_order": "2026-05-30", + "est_missed": 41200, + "notes": "Switched some volume to a local grower.", + "country": "United States", + "zip": "07649", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "(none)", + "customer_since": "2026-04-13", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-04-15 09:13:00", + "lat": 30.3322, + "lon": -81.6557, + "odoo_status": "Active" + }, + { + "pid": 105, + "customer": "Harborlight Wholesale Blooms", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Seattle", + "state": "Washington (US)", + "last_order": "2026-07-22", + "est_missed": 0, + "notes": "Top-10 account.", + "country": "United States", + "zip": "33125", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal", + "customer_since": "2023-05-14", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-05-15 09:14:00", + "lat": 26.1224, + "lon": -80.1373, + "odoo_status": "Active" + }, + { + "pid": 106, + "customer": "Dogwood & Fern Co.", + "status": "Dormant", + "agent": "Devon Marsh", + "city": "Asheville", + "state": "North Carolina (US)", + "last_order": "2026-02-11", + "est_missed": 52400, + "notes": "No spring order this year.", + "country": "United States", + "zip": "90210", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-06-15", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-06-15 09:15:00", + "lat": 27.3364, + "lon": -82.5307, + "odoo_status": "Active" + }, + { + "pid": 107, + "customer": "Verbena Market Florals", + "status": "Lost", + "agent": "Naomi Linnell Rivera", + "city": "Austin", + "state": "Texas (US)", + "last_order": "2025-11-04", + "est_missed": 78300, + "notes": "Went with a competitor on freight terms.", + "country": "United States", + "zip": "08701", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-07-16", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-07-15 09:16:00", + "lat": null, + "lon": null, + "odoo_status": "Active" + }, + { + "pid": 108, + "customer": "Larkspur Lane Supply", + "status": "New", + "agent": "Carla Jimenez", + "city": "Denver", + "state": "Colorado (US)", + "last_order": "2026-07-14", + "est_missed": 0, + "notes": "First order in April.", + "country": "United States", + "zip": "60614", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "(none)", + "customer_since": "2026-08-17", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-08-15 09:17:00", + "lat": 33.749, + "lon": -84.388, + "odoo_status": "Active" + } + ] +} diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000000000000000000000000000000000000..af3619ad7cada483ab9b0b64caf178ce69a9abdb --- /dev/null +++ b/web/index.html @@ -0,0 +1,14 @@ + + + + + + + Loopable + + +
+
+ + + diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000000000000000000000000000000000000..cf77cbfa1f38f4b06ad96eb36dce691947d35a63 --- /dev/null +++ b/web/package.json @@ -0,0 +1,30 @@ +{ + "name": "web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "build:embed": "vite build --config vite.config.embed.ts", + "lint": "oxlint", + "preview": "vite preview" + }, + "dependencies": { + "@glideapps/glide-data-grid": "^6.0.3", + "lodash": "^4.18.1", + "marked": "^4.3.0", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/node": "^24.13.2", + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^6.0.3", + "oxlint": "^1.71.0", + "typescript": "~6.0.2", + "vite": "^8.1.1", + "vite-plugin-singlefile": "^2.3.3" + } +} diff --git a/web/public/favicon.svg b/web/public/favicon.svg new file mode 100644 index 0000000000000000000000000000000000000000..a4f82828dbb9622d8d6c781b1243f91d7fee89b7 --- /dev/null +++ b/web/public/favicon.svg @@ -0,0 +1,8 @@ + + diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json new file mode 100644 index 0000000000000000000000000000000000000000..043835a1491db59996905c377bcca2cf276df4bc --- /dev/null +++ b/web/public/sample_customers.json @@ -0,0 +1,466 @@ +{ + "fields": [ + { + "key": "customer", + "label": "Customer", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + }, + { + "key": "status", + "label": "Status", + "type": "status", + "source": "odoo", + "default": true, + "description": "Where this customer stands versus last year: New, Growing, Declining, Lost or Dormant." + }, + { + "key": "odoo_status", + "label": "Odoo record", + "type": "status", + "source": "odoo", + "default": false, + "description": "Whether this customer still exists in Odoo. Archived means deleted there." + }, + { + "key": "agent", + "label": "Agent", + "type": "text", + "source": "odoo", + "default": true, + "description": "The sales agent who owns this account." + }, + { + "key": "salesperson", + "label": "Salesperson", + "type": "text", + "source": "odoo", + "default": false, + "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." + }, + { + "key": "city", + "label": "City", + "type": "text", + "source": "odoo", + "default": true, + "description": "City on the customer's Odoo address." + }, + { + "key": "state", + "label": "State", + "type": "text", + "source": "odoo", + "default": true, + "description": "State or province on the customer's Odoo address." + }, + { + "key": "country", + "label": "Country", + "type": "text", + "source": "odoo", + "default": false, + "description": "Country on the customer's Odoo address." + }, + { + "key": "zip", + "label": "ZIP", + "type": "text", + "source": "odoo", + "default": false, + "description": "Postal code on the customer's Odoo address." + }, + { + "key": "customer_since", + "label": "Customer since", + "type": "date", + "source": "odoo", + "default": false, + "description": "When this customer was first set up in Odoo." + }, + { + "key": "tags", + "label": "Tags", + "type": "text", + "source": "odoo", + "default": false, + "description": "Odoo labels on this customer, comma-separated." + }, + { + "key": "pricelist", + "label": "Price list", + "type": "text", + "source": "odoo", + "default": false, + "description": "The price list this customer buys on." + }, + { + "key": "payment_terms", + "label": "Payment terms", + "type": "text", + "source": "odoo", + "default": false, + "description": "Payment terms on this customer's account — Net 30, for example." + }, + { + "key": "last_order", + "label": "Last order", + "type": "date", + "source": "odoo", + "default": true, + "description": "Date of the most recent confirmed order." + }, + { + "key": "overdue_days", + "label": "Overdue days", + "type": "int", + "source": "odoo", + "default": true, + "description": "How many days late this customer is running against their own usual ordering rhythm." + }, + { + "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", + "key": "est_missed", + "label": "Est. missed $", + "type": "currency", + "source": "odoo", + "default": true, + "agg": "sum", + "filterable": false, + "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + }, + { + "key": "ar_open", + "label": "AR open $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money owed, not yet overdue." + }, + { + "key": "ar_overdue", + "label": "AR overdue $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money past due — same basis as the Collections page." + }, + { + "key": "ar_exposure", + "label": "Credit exposure $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + }, + { + "key": "days_to_pay", + "label": "Days to pay", + "type": "int", + "source": "odoo", + "default": false, + "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + }, + { + "key": "top_category", + "label": "Top category", + "type": "text", + "source": "odoo", + "default": false, + "description": "The category this customer spent the most on in the last 12 months." + }, + { + "key": "top_category_pct", + "label": "Top category %", + "type": "pct", + "source": "odoo", + "default": false, + "description": "Share of last-12-months spend that went to the top category." + }, + { + "key": "sku_count", + "label": "SKUs bought", + "type": "int", + "source": "odoo", + "default": false, + "description": "Distinct products bought in the last 12 months." + }, + { + "key": "top_sku", + "label": "Top SKU", + "type": "text", + "source": "odoo", + "default": false, + "description": "The product this customer spent the most on in the last 12 months." + }, + { + "key": "days_since", + "label": "Days since order", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days since the last confirmed order." + }, + { + "key": "typical_gap_days", + "label": "Typical gap days", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days this customer usually goes between orders, from their own history." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "description": "Your notes on this customer. Saved in this app only, visible only to you." + } + ], + "rows": [ + { + "pid": 101, + "customer": "Poppy Flowers", + "status": "New", + "agent": "Naomi Linnell Rivera", + "city": "Charlottesville", + "state": "Virginia (US)", + "last_order": "2026-07-21", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "02720", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Royal", + "customer_since": "2023-01-10", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-01-15 09:10:00", + "lat": 25.7617, + "lon": -80.1918, + "odoo_status": "Archived" + }, + { + "pid": 102, + "customer": "Meadow & Vine Wholesale", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Portland", + "state": "Oregon (US)", + "last_order": "2026-07-19", + "est_missed": 0, + "notes": "Expanding to a second storefront.", + "country": "United States", + "zip": "77041", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-02-11", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-02-15 09:11:00", + "lat": 27.9506, + "lon": -82.4572, + "odoo_status": "Active" + }, + { + "pid": 103, + "customer": "Bluestem Floral Supply", + "status": "Growing", + "agent": "Naomi Linnell Rivera", + "city": "Kansas City", + "state": "Missouri (US)", + "last_order": "2026-07-17", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "11219", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-03-12", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-03-15 09:12:00", + "lat": 28.5384, + "lon": -81.3789, + "odoo_status": "Active" + }, + { + "pid": 104, + "customer": "Camellia Row Florist", + "status": "Declining", + "agent": "Devon Marsh", + "city": "Savannah", + "state": "Georgia (US)", + "last_order": "2026-05-30", + "est_missed": 41200, + "notes": "Switched some volume to a local grower.", + "country": "United States", + "zip": "07649", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "(none)", + "customer_since": "2026-04-13", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-04-15 09:13:00", + "lat": 30.3322, + "lon": -81.6557, + "odoo_status": "Active" + }, + { + "pid": 105, + "customer": "Harborlight Wholesale Blooms", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Seattle", + "state": "Washington (US)", + "last_order": "2026-07-22", + "est_missed": 0, + "notes": "Top-10 account.", + "country": "United States", + "zip": "33125", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal", + "customer_since": "2023-05-14", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-05-15 09:14:00", + "lat": 26.1224, + "lon": -80.1373, + "odoo_status": "Active" + }, + { + "pid": 106, + "customer": "Dogwood & Fern Co.", + "status": "Dormant", + "agent": "Devon Marsh", + "city": "Asheville", + "state": "North Carolina (US)", + "last_order": "2026-02-11", + "est_missed": 52400, + "notes": "No spring order this year.", + "country": "United States", + "zip": "90210", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-06-15", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-06-15 09:15:00", + "lat": 27.3364, + "lon": -82.5307, + "odoo_status": "Active" + }, + { + "pid": 107, + "customer": "Verbena Market Florals", + "status": "Lost", + "agent": "Naomi Linnell Rivera", + "city": "Austin", + "state": "Texas (US)", + "last_order": "2025-11-04", + "est_missed": 78300, + "notes": "Went with a competitor on freight terms.", + "country": "United States", + "zip": "08701", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-07-16", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-07-15 09:16:00", + "lat": null, + "lon": null, + "odoo_status": "Active" + }, + { + "pid": 108, + "customer": "Larkspur Lane Supply", + "status": "New", + "agent": "Carla Jimenez", + "city": "Denver", + "state": "Colorado (US)", + "last_order": "2026-07-14", + "est_missed": 0, + "notes": "First order in April.", + "country": "United States", + "zip": "60614", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "(none)", + "customer_since": "2026-08-17", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-08-15 09:17:00", + "lat": 33.749, + "lon": -84.388, + "odoo_status": "Active" + } + ] +} diff --git a/web/src/App.tsx b/web/src/App.tsx new file mode 100644 index 0000000000000000000000000000000000000000..77fd5fb03fc59405df22b003bfed1a8bb3b539f7 --- /dev/null +++ b/web/src/App.tsx @@ -0,0 +1,30 @@ +import CustomerGrid from "./customer-grid/CustomerGrid"; +import { OverlayProvider } from "./customer-grid/OverlaySurface"; +import { isStreamlitComponent } from "./customer-grid/hostBridge"; +import Shell from "./shell/Shell"; + +// HOSTED = either Streamlit path (the component iframe, or the legacy injected +// payload). There the surrounding app owns all chrome and this tree renders the +// BARE grid, byte-for-byte the behavior the embed always had. Only the +// standalone build gets the shell (wave 4, the strangler frame). Computed once +// at module level, like useCustomerData's own mode constants — the host mode +// cannot change within a page lifetime. +// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to +// install the standalone event sink, and two copies of a mode predicate is how +// the two halves of "standalone-only" end up disagreeing. +export const HOSTED = + isStreamlitComponent() || + typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; + +export default function App() { + if (HOSTED) { + return ( +
+ + + +
+ ); + } + return ; +} diff --git a/web/src/apiContract.ts b/web/src/apiContract.ts new file mode 100644 index 0000000000000000000000000000000000000000..92eae1cedb661eb6d8932a5554e0bdbdf6d814db --- /dev/null +++ b/web/src/apiContract.ts @@ -0,0 +1,62 @@ +// --------------------------------------------------------------------------- +// apiContract.ts — the handful of constants BOTH trees need, and the two +// browser-level signals the data layer raises to the frame. +// +// WHY A ROOT-LEVEL LEAF. `shell/session.ts` and `customer-grid/apiBridge.ts` +// both speak X2, so both need the version prefix and the credentials word. The +// alternatives were worse: a second copy of `"/api/v1"` is a string that drifts +// (this codebase keeps a lock-step GATE for exactly that class of duplication), +// and importing shell code from `customer-grid/**` would point the dependency +// edge the wrong way — the grid is the ASSET, the shell is the disposable +// frame, and the embed bundle must never grow a reason to pull the frame in. +// A leaf both import is the only arrangement with no cycle and no drift. +// +// It imports nothing, by construction. +// --------------------------------------------------------------------------- + +/** X2: every route lives under this prefix. `/api/health` is the one exception + * and is not ours — it is the unauthenticated liveness probe. */ +export const API_V1 = "/api/v1"; + +/** + * ⚠ EXPLICIT, not defaulted. Same-origin is already the browser default, but + * this is the one word that decides whether the X3 session cookie rides the + * request at all — and the cookie is HttpOnly, so getting it wrong produces no + * client-side symptom, just a 401 from a server that never saw a session. An + * implicit default is not something a reader (or a gate) can check. + */ +export const CREDENTIALS: RequestCredentials = "same-origin"; + +/** + * THE SESSION DIED UNDER US. Raised by any authenticated call that comes back + * 401, and handled by the shell (sign out, show the door). + * + * A DOM CustomEvent rather than an import, deliberately: `customer-grid/**` is + * host-neutral and must not know a shell exists — it is the same tree the + * Streamlit embed ships. The window is the one channel both sides already + * share (`hostBridge` uses it for the host-render signal for the same reason). + */ +export const UNAUTHORIZED_EVENT = "aios:unauthorized"; + +/** The authenticated read FAILED for a reason that is not authentication. The + * frame says so plainly; it never substitutes sample data (see apiBridge). */ +export const DATA_ERROR_EVENT = "aios:data-error"; + +/** The server sent a human-readable confirmation with a write (X2's `toast`). + * `detail` is the string. */ +export const TOAST_EVENT = "aios:toast"; + +/** A write CHANGED durable workspace state (a cohort's membership, a list add, a + * folder move) and the client's copy is now stale. + * + * ⛔ WHY THIS EXISTS. `/customers` carries rows; the workspace lives at its own + * URL. So in the embed the Streamlit host reruns and the left panel repaints, + * while standalone had NOTHING — the server's `toast` was the only evidence a + * cohort add had happened, and the panel beside it still showed the old + * membership. A toast is a receipt, not a refresh. */ +export const WORKSPACE_STALE_EVENT = "aios:workspace-stale"; + +export function signal(name: string, detail?: unknown): void { + if (typeof window === "undefined") return; + window.dispatchEvent(new CustomEvent(name, { detail })); +} diff --git a/web/src/customer-grid/CohortSidebar.tsx b/web/src/customer-grid/CohortSidebar.tsx new file mode 100644 index 0000000000000000000000000000000000000000..79fb40056f7a1da1c6f6301916fe47ad148ff9a6 --- /dev/null +++ b/web/src/customer-grid/CohortSidebar.tsx @@ -0,0 +1,585 @@ +// --------------------------------------------------------------------------- +// customer-grid / CohortSidebar.tsx +// The Cohort page's left panel (wave-2 item 2c, 2026-07-27): the COHORT LIST +// replaces the Views rail. A cohort is a fixed, hand-curated subset — the rail +// is a switcher over `workspace.lists` (name · member count) plus each row's +// lifecycle menu: the host page's Manage popover retired with wave 2, so +// RENAME and DELETE live here (the contract's cohort_rename / cohort_delete +// events; capability leaves a UI only with a replacement). Creation stays on +// the Customer page ("Add to cohort"); membership edits happen on the grid +// itself (the selection bar's "Remove from cohort", the toolbar's "+ Add +// customers"). +// +// The member count is the cohort's SERVABLE size — its pids within the +// caller's pool. When members fall outside the 24-month pool the host says so +// via `missing`, and the active row DISCLOSES it: a cohort silently smaller +// than its true membership is a forbidden silent truncation (rule 8b). +// --------------------------------------------------------------------------- + +import { useState } from "react"; +import type { KeyboardEvent as ReactKeyboardEvent } from "react"; +import { AnchoredOverlay } from "./OverlaySurface"; +import { EXPORT_FORMATS, EXPORT_LABELS } from "./export"; +import type { ExportFormat } from "./export"; +import type { GridFolder } from "./types"; +import { groupByFolder } from "./folders"; + +export interface CohortSidebarProps { + lists: { id: string; name: string; pids?: number[]; missing?: number; folderId?: string | null }[]; + /** + * Wave-8 I11c (contract C4) — folders, the cohort rail's half. Same model as the + * Views rail (folders.ts is surface-agnostic); all optional, so a caller that + * wires none of it renders the pre-wave-8 flat list. + * + * The bulk "Add to cohort" here is a plain SET UNION of the contained cohorts' + * member pids — no engine run, and therefore none of the Views rail's two + * refusals: a cohort already IS a fixed pid set, so nothing about it can be + * pending or unanswerable. + */ + folders?: GridFolder[]; + folderIdOf?: (cohortId: string) => string | null; + onFolderCreate?: (name: string) => void; + onFolderRename?: (folderId: string, name: string) => void; + onFolderDelete?: (folderId: string) => void; + onFolderDuplicate?: (folderId: string) => void; + onItemMove?: (cohortId: string, folderId: string | null) => void; + /** Add the UNION of this folder's cohorts to `cohortId` (empty = create `name`). */ + onFolderAddToList?: (folderId: string, cohortId: string, name: string) => void; + /** The tenant's today, seeding a new cohort's name. Never the browser clock. */ + today?: string; + activeCohortId: string | null; + onSelect: (id: string) => void; + /** Emits `cohort_rename` — the host validates ownership and re-renders with the new name. */ + onRename: (id: string, name: string) => void; + /** Emits `cohort_delete`. Destructive, so the menu arms first (same pattern as field + * delete): first click reads "Delete permanently?", the second fires. */ + onDelete: (id: string) => void; + /** Wave-7 item W2 (contract C2) — export THIS cohort's current matched rows + * (its saved view's filters + sorts over the cohort's fixed membership). */ + onExport?: (cohortId: string, format: ExportFormat) => void; +} + +export default function CohortSidebar({ + lists, + activeCohortId, + onSelect, + folders, + folderIdOf, + onFolderCreate, + onFolderRename, + onFolderDelete, + onFolderDuplicate, + onItemMove, + onFolderAddToList, + today, + onRename, + onDelete, + onExport, +}: CohortSidebarProps) { + const [menu, setMenu] = useState<{ + cohortId: string; + anchor: HTMLButtonElement; + } | null>(null); + const [renamingId, setRenamingId] = useState(null); + const [confirmDelete, setConfirmDelete] = useState(false); + /** W2 — the Export format pane, anchored where the "…" menu was. */ + const [exportFor, setExportFor] = useState<{ + cohortId: string; + anchor: HTMLButtonElement; + } | null>(null); + const exportCohort = exportFor + ? lists.find((l) => l.id === exportFor.cohortId) + : undefined; + const menuCohort = menu ? lists.find((l) => l.id === menu.cohortId) : undefined; + // I11c - folder UI state. Collapse is LOCAL (C4), as on the Views rail. + const [collapsed, setCollapsed] = useState>(new Set()); + const [folderMenu, setFolderMenu] = useState<{ id: string; anchor: HTMLElement } | null>(null); + const [renamingFolder, setRenamingFolder] = useState(null); + const [creatingFolder, setCreatingFolder] = useState(false); + const [confirmFolderDelete, setConfirmFolderDelete] = useState(null); + const [addTarget, setAddTarget] = useState(null); + /** The element the bulk-add pane hangs off. Anchoring to document.body + * put it at the bottom of the DOCUMENT rather than beside the folder — + * painted, and effectively unreachable ([[ui-invisible-to-assertions]]). + * Captured from the folder menu before that menu is dismissed. */ + const [addAnchor, setAddAnchor] = useState(null); + const [addName, setAddName] = useState(""); + const [dropTarget, setDropTarget] = useState(null); + const foldersOn = !!folders && !!folderIdOf && !!onItemMove; + const groups = groupByFolder(lists, foldersOn ? (folders as GridFolder[]) : [], (l) => + folderIdOf ? folderIdOf(l.id) : null + ); + const folderMenuF = folderMenu ? folders?.find((f) => f.id === folderMenu.id) : undefined; + /** C4 - a cohort folder's union is a plain SET UNION of member pids. No engine + * run, so none of the Views rail's two refusals apply: a cohort already IS a + * fixed pid set, so nothing about it can be pending or unanswerable. */ + const folderUnion = (folderId: string): number[] => { + const pids = new Set(); + for (const l of lists) { + if (!folderIdOf || folderIdOf(l.id) !== folderId) continue; + for (const pid of l.pids ?? []) pids.add(pid); + } + return [...pids]; + }; + const addUnion = addTarget ? folderUnion(addTarget) : []; + const addCount = addTarget + ? lists.filter((l) => folderIdOf && folderIdOf(l.id) === addTarget).length + : 0; + const dropHandlers = (folderId: string | null) => + foldersOn + ? { + onDragOver: (e: React.DragEvent) => { + e.preventDefault(); + setDropTarget(folderId ?? "__root__"); + }, + onDragLeave: () => setDropTarget(null), + onDrop: (e: React.DragEvent) => { + e.preventDefault(); + setDropTarget(null); + const id = e.dataTransfer.getData("text/plain"); + if (id) onItemMove?.(id, folderId); + }, + } + : {}; + + const closeMenu = () => { + setMenu(null); + setConfirmDelete(false); + }; + + const onMenuKeyDown = (event: ReactKeyboardEvent) => { + if (!["ArrowDown", "ArrowUp", "Home", "End"].includes(event.key)) return; + const items = [ + ...event.currentTarget.querySelectorAll('[role="menuitem"]'), + ]; + if (!items.length) return; + event.preventDefault(); + const current = items.indexOf(document.activeElement as HTMLButtonElement); + const next = + event.key === "Home" + ? 0 + : event.key === "End" + ? items.length - 1 + : event.key === "ArrowDown" + ? (current + 1) % items.length + : (current - 1 + items.length) % items.length; + items[next]?.focus(); + }; + + return ( + + ); +} diff --git a/web/src/customer-grid/ColumnMenu.tsx b/web/src/customer-grid/ColumnMenu.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e6e6ed5dc72f161300a704d80826c0e1af7b7ae0 --- /dev/null +++ b/web/src/customer-grid/ColumnMenu.tsx @@ -0,0 +1,1584 @@ +import { useMemo, useState } from "react"; +import type { ReactNode } from "react"; +import { AnchoredOverlay } from "./OverlaySurface"; +import type { AnchorRect } from "./OverlaySurface"; +import { CREATABLE_TYPES, choiceOptions, directionLabel, parseOptions, ratingMax } + from "./types"; +import type { Field, FieldFormat, FieldScope, FieldType, Measure, Viewer } from "./types"; +import type { WindowSpec } from "./windows"; +import { normalizeWindow, windowLabel } from "./windows"; +import { WindowPicker } from "./Toolbar"; +import { TYPE_LABELS } from "./iconShapes"; +import { validateFormula } from "./formulaEngine"; + +export interface ColumnMenuState { + fieldKey: string; + anchor: AnchorRect; +} + +interface ColumnMenuProps { + state: ColumnMenuState; + field: Field; + /** Every field of the table, for "Change field" (owner item 9): the clicked column can be + * swapped for any other field — including hidden ones like Notes — in place. */ + fields: Field[]; + locked: boolean; + /** Wave-5 item 1 — who is looking. Gates the permissions entry (creator-or-admin). */ + viewer?: Viewer; + /** Wave-5 item 3 — what the CURRENT VIEW does with this field, so the conditional + * "Don't sort/filter/group by this field" entries render exactly when they apply. */ + sortedDir: "asc" | "desc" | null; + isFiltered: boolean; + isGrouped: boolean; + /** types.isGroupableField, computed by the caller (it owns lockedKey). */ + groupable: boolean; + onClose: () => void; + onNote: (note: string) => void; + onHide: () => void; + onCreate: ( + label: string, + type: FieldType, + anchorKey: string | null, + side: "left" | "right" | "end", + options?: string[], + measureSpec?: { key: string; window: WindowSpec }, + extra?: { formula?: string; max?: number; scope?: FieldScope } + ) => void; + /** Swap THIS column for another field, in place (owner item 9). */ + onChangeField: (newKey: string) => void; + /** + * Wave-2 item 5: the Change-field window's "New field" half — create a fresh overlay field + * of the chosen type and swap this column for it in one motion. The old field hides exactly + * as an ordinary change does. Wave-6 item 2 REPLACES this flow on created (`custom_`) + * fields with `onRetype` below. + */ + onCreateAndSwap: ( + label: string, + type: FieldType, + options?: string[], + extra?: { formula?: string; max?: number; scope?: FieldScope } + ) => void; + /** + * Wave-6 item 2 — supplied for CREATED (`custom_`) fields only: the Change-field pane's + * type half RETYPES THIS FIELD IN PLACE (same key, new type/options; values never + * converted — unreadable cells show blank, disclosed in the pane). Absent = the pane keeps + * the create-and-swap flow. + */ + onRetype?: ( + type: FieldType, + options?: string[], + extra?: { formula?: string; max?: number } + ) => void; + /** Wave-6 item 5 — rename this created field (ordinary field_upsert with a new label). + * Supplied for creatable strata only. */ + onRename?: (label: string) => void; + /** Wave-6 item 4 — open straight into the create form at this position (the header "+" + * passes "end"). Cancel then CLOSES rather than falling back to the action list. */ + initialPosition?: CreatePosition; + /** Wave-6 item 9c — true on the cohort page: the create forms offer the Scope control + * ("This cohort table only" = DEFAULT vs "All customer tables"). */ + scopeChoice?: boolean; + /** Wave-6 item 11c — pin state + actions. `onPinTo` absent when pinning would be a no-op + * (identity column, or already the boundary); `onUnpin` absent when nothing is pinned. */ + pinnedTo?: boolean; + onPinTo?: () => void; + onUnpin?: () => void; + /** + * Wave-2 item 8c: change the PERIOD of a measure-carrying column. Supplied for any field with + * `.measure` — user-created `measure_*` columns and the pre-set measure fields alike. The + * caller emits the ordinary `field_upsert` (and auto-renames the header when the user never + * renamed it). + */ + onPeriod?: (window: WindowSpec) => void; + /** + * Delete this USER-CREATED column outright (owner gap closed 2026-07-27). Supplied ONLY for + * the deletable strata (`custom_` overlay fields, `measure_` formula columns) — absent means + * the control is not rendered at all, so a base field cannot even be offered the action. + */ + onDelete?: () => void; + /** Wave-5 item 1 — Duplicate field. Supplied for creatable strata only (base Odoo fields + * offer no Duplicate — cloning the source of truth into an editable copy is out of scope). */ + onDuplicate?: () => void; + /** Wave-5 item 1 — save a permissions change. Supplied iff the VIEWER may change them + * (isAdmin || createdBy === viewer.name, and only on creatable strata). */ + onPermissions?: (edit: "everyone" | "creator" | "admins") => void; + /** Wave-5 item 10 — save a display format. Supplied for number/currency/formula and + * date/created_time fields. */ + onFormat?: (format: FieldFormat) => void; + /** Wave-5 item 1 — Sort by this field (replaces the view's sort), and the conditional + * clears (item 3). All of these mutate the EXISTING view config — no new host surface. */ + onSort: (dir: "asc" | "desc") => void; + onClearSort: () => void; + onFilterBy: () => void; + onClearFilter: () => void; + onGroupByField: () => void; + onClearGroup: () => void; + /** Assignable people, from the host's real user list. Empty = the host did not supply one, + * and "Assignee" is offered without choices rather than with invented ones. */ + userOptions?: string[]; + /** + * CG-8's measures, reused as the vocabulary of a FORMULA-MEASURE column (owner item 7): + * `Sales · the last 90 days` as a column the user creates. Empty = the semantic store is + * unavailable here, and the option is simply not offered (same rule as measure conditions). + */ + measures?: Measure[]; +} + +type CreatePosition = "left" | "right" | "end"; + +// --- wave-7 item W4: menu-action icons (16px, stroke, currentColor — never emoji, never an +// icon font; hand-rolled lucide-style paths inline, matching the toolbar's own icon set). --- +const mi = { width: 16, height: 16, viewBox: "0 0 16 16", fill: "none" } as const; +const miStroke = { + stroke: "currentColor", + strokeWidth: 1.3, + strokeLinecap: "round" as const, + strokeLinejoin: "round" as const, +}; + +const MenuIcons = { + duplicate: ( + + + + + ), + insertLeft: ( + + + + + ), + insertRight: ( + + + + + ), + addEnd: ( + + + + ), + rename: ( + + + + ), + description: ( + + + + ), + permissions: ( + + + + + ), + format: ( + + + + + + ), + swap: ( + + + + ), + sortAsc: ( + + + + + ), + sortDesc: ( + + + + + ), + off: ( + + + + + ), + filter: ( + + + + ), + group: ( + + + + + + ), + pin: ( + + + + + ), + unpin: ( + + + + + + ), + hide: ( + + + + + + ), + trash: ( + + + + ), +} as const; + +/** One action row: icon + label. The icon column is fixed-width so labels align. */ +function MenuLabel({ icon, text }: { icon: ReactNode; text: string }) { + return ( + <> + + {icon} + + {text} + + ); +} + +/** + * The menu's panes (wave-5 item 2). The NOTE editor and CHANGE FIELD used to render inline in + * the menu body; each is now a button opening its OWN dedicated window — as are the wave-5 + * permissions and format editors and wave-6's RENAME. One overlay, one pane at a time; + * "menu" is the action list. + */ +type MenuPane = "menu" | "note" | "swap" | "permissions" | "format" | "rename"; + +/** + * How each creatable type reads in the menu. DERIVED from CREATABLE_TYPES rather than listed + * again: a hand-maintained copy is a third place the vocabulary lives, and the failure mode is + * silent — add a type to the contract, forget this list, and the type is accepted by the host, + * asserted by the drift gate, and simply never offered to anyone. + * + * `Record` is what forces it: adding a member to the union without a label + * here fails `tsc`, so the compiler is the gate and no test has to click a canvas header menu. + * + * Wave-8 I20: the map itself MOVED to icons.tsx, beside the per-type glyphs — a type's label + * and its icon are one vocabulary, and splitting them across two files is how they drift. + * Imported here; the totality guarantee above is unchanged (icons.tsx types it identically). + */ +const FIELD_TYPES = CREATABLE_TYPES.map((value) => ({ value, label: TYPE_LABELS[value] })); + +/** Types whose choices are declared in an options editor at creation time. */ +function needsOptions(t: FieldType | "measure"): boolean { + return t === "select" || t === "multiselect"; +} + +/** + * The create-picker's kind: a creatable overlay type, or the FORMULA-MEASURE pseudo-kind. + * "measure" is deliberately NOT a FieldType — a measure column's real type comes from the + * measure itself (currency/int/pct), and widening the union would let it leak into every + * switch that renders cells. + */ +type CreateKind = FieldType | "measure"; + +/** The Change-field dropdown's "New field" entries. A colon cannot occur in a real field key + * (keys are slugs), so the prefix cannot collide with one. */ +const NEW_PREFIX = "new:"; + +/** Wave-6 item 2 — the dropdown's RETYPE entries (created fields only). Same colon rule. */ +const RETYPE_PREFIX = "retype:"; + +/** + * The types a created field may be retyped TO: the overlay-editable strata only. `formula` + * and `created_time` are a DIFFERENT stratum (source 'odoo', host-computed shape) — turning + * an overlay column into one is not a retype, and the host accepts in-place rebuilds on + * `custom_` overlay defs only. + */ +const RETYPE_TYPES = CREATABLE_TYPES.filter( + (t) => t !== "formula" && t !== "created_time" +); + +const DEFAULT_MEASURE_WINDOW: WindowSpec = { kind: "ltm" }; + +/** The head every pane shares: title line, context line, close X. */ +function PaneHead({ + title, + sub, + onClose, +}: { + title: string; + sub?: string; + onClose: () => void; +}) { + return ( +
+
+
{title}
+ {sub &&
{sub}
} +
+ +
+ ); +} + +/** + * Wave-6 item 9c — the Scope control, rendered by BOTH create surfaces on the cohort page + * (one component so the two doors cannot drift). "This cohort table only" is the DEFAULT — + * the owner's ask: a field made while working a cohort belongs to that work unless the user + * says otherwise. + */ +function ScopeControl({ + value, + onValue, +}: { + value: FieldScope; + onValue: (v: FieldScope) => void; +}) { + return ( + + ); +} + +/** + * Wave-5 items 9/11 — the extra editors a new field's type may need, shared VERBATIM by the + * two create surfaces (insert-field and Change-field's "New field" half) so the two doors + * cannot drift: a FORMULA gets its source editor with live validation and a reference picker; + * a RATING gets its star count (the contract's 2..10). + */ +function ExtraTypeEditor({ + kind, + fields, + formulaText, + onFormulaText, + formulaError, + maxValue, + onMaxValue, + idPrefix, +}: { + kind: FieldType | "measure"; + fields: Field[]; + formulaText: string; + onFormulaText: (v: string) => void; + formulaError: string | null; + maxValue: number; + onMaxValue: (n: number) => void; + idPrefix: string; +}) { + if (kind === "formula") { + const referencable = fields.filter( + (f) => f.type !== "formula" && f.key !== "__proto__" + ); + return ( + <> +