Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- RELEASES.json +7 -1
- VERSION +1 -1
- api/ai_enrich.py +96 -0
- api/ai_review.py +123 -44
- api/automation_engine.py +83 -0
- api/main.py +32 -0
- api/odoo_relational.py +27 -2
- api/providers.py +448 -4
- api/routes_admin.py +68 -7
- api/routes_agent_harness.py +455 -0
- api/routes_automation.py +279 -8
- api/routes_customers.py +471 -13
- api/routes_geo.py +369 -0
- api/routes_nav.py +96 -4
- api/routes_products.py +217 -8
- api/routes_query.py +332 -88
- api/routes_script_views.py +439 -0
- api/routes_shares.py +71 -6
- api/routes_slack.py +28 -12
- api/routes_statements.py +31 -2
- api/routes_tables.py +331 -13
- api/routes_web_agent.py +5 -4
- platform/aios_grid.py +394 -5
- platform/aios_grid_fields.json +14 -0
- platform/core/grid_events.py +31 -1
- platform/core/perm_scope.py +162 -10
- platform/core/script_sandbox.py +519 -0
- platform/core/shares.py +64 -14
- platform/core/store_pg.py +166 -3
- platform/core/table_store.py +189 -10
- platform/core/user_tables.py +46 -0
- platform/core/users.py +7 -0
- platform/harness/datastore.py +95 -4
- platform/harness/semantic.py +528 -0
- platform/model/metrics/sales.yml +42 -1
- platform/model/metrics/stock.yml +57 -0
- platform/model/topics/odoo_agents.yml +32 -0
- platform/model/topics/odoo_products.yml +51 -0
- platform/model/topics/sales_lines.yml +13 -0
- platform/model/topics/stock_moves.yml +72 -0
- platform/modules/agent.py +108 -0
- platform/modules/collections_send.py +7 -3
- platform/modules/product_data.py +411 -0
- platform/modules/products.py +172 -0
- requirements.txt +8 -0
- web/index.html +33 -1
- web/src/App.tsx +13 -2
- web/src/account/SubscriptionPage.tsx +2 -0
- web/src/account/UsagePage.tsx +1 -1
- web/src/account/account.css +1 -1
RELEASES.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
{
|
| 5 |
"version": "v29",
|
| 6 |
"sha": "55ced50",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "v30 (71776ca)",
|
| 3 |
"releases": [
|
| 4 |
+
{
|
| 5 |
+
"version": "v30",
|
| 6 |
+
"sha": "71776ca",
|
| 7 |
+
"date": "2026-08-20",
|
| 8 |
+
"subject": "v30: W39-T29 Member-owned View first-share hotfix"
|
| 9 |
+
},
|
| 10 |
{
|
| 11 |
"version": "v29",
|
| 12 |
"sha": "55ced50",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
v30 (71776ca)
|
api/ai_enrich.py
CHANGED
|
@@ -314,6 +314,102 @@ def _ask(provider, model, prompt, max_tokens, timeout):
|
|
| 314 |
return '', None, f"{provider['name']} failed: {type(exc).__name__}"
|
| 315 |
|
| 316 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
def on_change_fields(defn, changed_keys):
|
| 318 |
"""Which `ai_enrich` columns in this table want a run because one of their inputs moved.
|
| 319 |
|
|
|
|
| 314 |
return '', None, f"{provider['name']} failed: {type(exc).__name__}"
|
| 315 |
|
| 316 |
|
| 317 |
+
# ===========================================================================
|
| 318 |
+
# W37-T37 (R4 / C7) -- THE GEOCODE FIELD's pure half: turning a record into ONE query string.
|
| 319 |
+
#
|
| 320 |
+
# β WHY IT IS HERE AND NOT IN `routes_geo.py`. This module's contract, stated in its own header,
|
| 321 |
+
# is that everything in it is a pure function of data it is handed -- which is what lets a gate
|
| 322 |
+
# exercise it with no network, no store and no server. Composing an address is exactly that shape.
|
| 323 |
+
# `routes_geo.py` owns the part that cannot be pure: the outbound call, the throttle and the cache.
|
| 324 |
+
#
|
| 325 |
+
# β WHY IT COMPOSES AT ALL, rather than sending the one column the field names. The field config
|
| 326 |
+
# is `{addressField, country?}` -- a single column -- but a customers table stores an address across
|
| 327 |
+
# six of them. B MEASURED the fill rates on the real 3,641-row mirror (W37-T11):
|
| 328 |
+
#
|
| 329 |
+
# street 96.2% Β· city 96.0% Β· zip 94.6% Β· country 95.6% Β· state 92.0% Β· street2 5.2%
|
| 330 |
+
# street AND city together: 95.9% <- the pair this feature actually needs
|
| 331 |
+
#
|
| 332 |
+
# Sending `street` alone would ask a global geocoder to place "123 Main Street" with no town, which
|
| 333 |
+
# resolves confidently and often wrongly. So the named column LEADS and the conventional siblings
|
| 334 |
+
# follow, which is also how a person would write the address down.
|
| 335 |
+
# ===========================================================================
|
| 336 |
+
|
| 337 |
+
#: The sibling columns appended after the named one, in the order an address is spoken. β A LIST
|
| 338 |
+
#: rather than a formatted string: a template would have to decide what to do about the ~4% of rows
|
| 339 |
+
#: missing any one part, and the answer is always "leave it out", never "leave a gap".
|
| 340 |
+
GEOCODE_PARTS = ('street2', 'city', 'state', 'zip', 'postal_code', 'country')
|
| 341 |
+
|
| 342 |
+
#: β B's SECOND MEASURED TRAP. Odoo serves `state` as its DISPLAY name, country code included:
|
| 343 |
+
#: `New York (US)`, `British Columbia (CA)`. A naive join yields "SCARSDALE, New York (US)", and the
|
| 344 |
+
#: parenthetical is noise to a geocoder at best. Stripped here, once, rather than at each caller.
|
| 345 |
+
_PAREN_TAIL = re.compile(r'\s*\([^)]*\)\s*$')
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
def geocode_query(row, cfg, seen_parts=None):
|
| 349 |
+
"""One record plus a geocode field's config -> the query string, or '' when there is nothing.
|
| 350 |
+
|
| 351 |
+
β RETURNS '' RATHER THAN A PARTIAL GUESS when the named column is empty. B measured ~150 of
|
| 352 |
+
3,641 customers with no street at all, and the honest UI for those is "no address on file", not
|
| 353 |
+
a request that spends a second of a shared service's budget to place a bare postcode.
|
| 354 |
+
|
| 355 |
+
β `seen_parts=None` RESOLVES `GEOCODE_PARTS` AT CALL TIME, and that is not style. It was
|
| 356 |
+
`seen_parts=GEOCODE_PARTS`, which binds the tuple once at import, so the module constant stopped
|
| 357 |
+
being the live answer the moment anything wanted to vary it: the gate's own negative control set
|
| 358 |
+
`ai_enrich.GEOCODE_PARTS = ()` and the function carried on composing the full address. A default
|
| 359 |
+
argument that captures a module constant is a SECOND COPY of it with an earlier timestamp, and
|
| 360 |
+
the copy is the one that runs ([[a-constant-two-features-share]]). Caught by that control."""
|
| 361 |
+
if not isinstance(row, dict) or not isinstance(cfg, dict):
|
| 362 |
+
return ''
|
| 363 |
+
if seen_parts is None:
|
| 364 |
+
seen_parts = GEOCODE_PARTS
|
| 365 |
+
key = str(cfg.get('addressField') or '').strip()
|
| 366 |
+
if not key:
|
| 367 |
+
return ''
|
| 368 |
+
lead = _clean_part(row.get(key))
|
| 369 |
+
if not lead:
|
| 370 |
+
return ''
|
| 371 |
+
parts = [lead]
|
| 372 |
+
for k in seen_parts:
|
| 373 |
+
if k == key:
|
| 374 |
+
continue
|
| 375 |
+
v = _clean_part(row.get(k))
|
| 376 |
+
# β The dedupe is not tidiness. `country` is often both the field's own value and already
|
| 377 |
+
# present in the street line on an imported record, and "USA, USA" measurably degrades a
|
| 378 |
+
# free-form geocoder's confidence.
|
| 379 |
+
if v and v.lower() not in {p.lower() for p in parts}:
|
| 380 |
+
parts.append(v)
|
| 381 |
+
return ', '.join(parts)
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
def _clean_part(v):
|
| 385 |
+
"""One address component, trimmed and stripped of a trailing country parenthetical."""
|
| 386 |
+
if v is None:
|
| 387 |
+
return ''
|
| 388 |
+
s = re.sub(r'\s+', ' ', str(v)).strip()
|
| 389 |
+
if not s or s.lower() in ('false', 'none', 'null'):
|
| 390 |
+
return ''
|
| 391 |
+
return _PAREN_TAIL.sub('', s).strip()
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def geocode_plan(rows, cfg, coord_key=None, overwrite=False):
|
| 395 |
+
"""Which records a run would actually ask about, and why the rest are being left out.
|
| 396 |
+
|
| 397 |
+
β THIS IS THE R3 GUARD IN ITS ARITHMETIC FORM. It is handed the records a PERSON chose; it
|
| 398 |
+
never reads a table. `skipped` is returned rather than swallowed so the surface can say "142 to
|
| 399 |
+
look up, 9 have no address, 31 already have coordinates" instead of a bare number."""
|
| 400 |
+
run, skipped = [], {}
|
| 401 |
+
for rid, row in (rows or {}).items():
|
| 402 |
+
q = geocode_query(row, cfg)
|
| 403 |
+
if not q:
|
| 404 |
+
skipped['no_address'] = skipped.get('no_address', 0) + 1
|
| 405 |
+
continue
|
| 406 |
+
if coord_key and not overwrite and str((row or {}).get(coord_key) or '').strip():
|
| 407 |
+
skipped['already_located'] = skipped.get('already_located', 0) + 1
|
| 408 |
+
continue
|
| 409 |
+
run.append((str(rid), q))
|
| 410 |
+
return {'run': run, 'skipped': skipped}
|
| 411 |
+
|
| 412 |
+
|
| 413 |
def on_change_fields(defn, changed_keys):
|
| 414 |
"""Which `ai_enrich` columns in this table want a run because one of their inputs moved.
|
| 415 |
|
api/ai_review.py
CHANGED
|
@@ -17,15 +17,23 @@ the work. Nothing here can move a card somewhere the review does not already per
|
|
| 17 |
last rather than absent β it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER`
|
| 18 |
pins one; `AIOS_AI_MODEL` overrides the model.
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and
|
| 31 |
`stop_reason: "refusal"` is checked BEFORE reading `content` β a refusal answers HTTP 200 with an
|
|
@@ -41,9 +49,13 @@ import requests
|
|
| 41 |
|
| 42 |
#: The ladder. Order IS the policy (R14) β cheapest capable first, Anthropic last.
|
| 43 |
PROVIDERS = [
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
{"name": "groq", "env": "GROQ_API_KEY", "shape": "openai",
|
| 45 |
"url": "https://api.groq.com/openai/v1/chat/completions",
|
| 46 |
-
"model": "
|
| 47 |
{"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai",
|
| 48 |
"url": "https://api.cerebras.ai/v1/chat/completions",
|
| 49 |
"model": "gpt-oss-120b"},
|
|
@@ -56,7 +68,9 @@ PROVIDERS = [
|
|
| 56 |
# labels it already has in front of it.
|
| 57 |
{"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
|
| 58 |
"url": "https://api.anthropic.com/v1/messages",
|
| 59 |
-
|
|
|
|
|
|
|
| 60 |
]
|
| 61 |
ANTHROPIC_VERSION = "2023-06-01"
|
| 62 |
TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
|
|
@@ -159,16 +173,35 @@ def _call_openai(p, model, system, user, timeout):
|
|
| 159 |
return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body
|
| 160 |
|
| 161 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
def _call_anthropic(p, model, system, user, timeout):
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
# β stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so
|
| 173 |
# reading content[0] before this check turns a refusal into an IndexError inside a run.
|
| 174 |
# β A refusal IS billed and its body carries `usage`, so the body rides back on this branch too.
|
|
@@ -277,13 +310,23 @@ MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engi
|
|
| 277 |
MAX_PROMPT_CHARS = 2000
|
| 278 |
|
| 279 |
|
| 280 |
-
def flow_providers():
|
| 281 |
-
"""The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
|
| 282 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
by_name = {p["name"]: p for p in PROVIDERS}
|
| 284 |
-
|
| 285 |
-
return [by_name[n] for n in order
|
| 286 |
if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
|
| 288 |
|
| 289 |
def flow_schema(kinds, trigger_keys, table_keys):
|
|
@@ -421,7 +464,7 @@ def _flow_from_tool_call(obj):
|
|
| 421 |
|
| 422 |
|
| 423 |
def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
|
| 424 |
-
st=None, user=""):
|
| 425 |
"""A sentence -> `(draft, refusal_sentence, provider)`. β NOTHING IS SAVED HERE.
|
| 426 |
|
| 427 |
Exactly one of `draft` and `refusal_sentence` is truthy β the same contract
|
|
@@ -432,6 +475,9 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou
|
|
| 432 |
`routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and
|
| 433 |
D asserts it; the two keywords are what D has to pass for the line to be attributable.
|
| 434 |
"""
|
|
|
|
|
|
|
|
|
|
| 435 |
import usage_ledger # noqa: PLC0415
|
| 436 |
text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
|
| 437 |
if not text:
|
|
@@ -456,7 +502,7 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou
|
|
| 456 |
draft, refusal = _flow_from_tool_call(chat(messages, tools))
|
| 457 |
return draft, refusal, "injected"
|
| 458 |
|
| 459 |
-
provs = flow_providers()
|
| 460 |
if not provs:
|
| 461 |
# β SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
|
| 462 |
# never built [[flag-shipped-without-its-writer]].
|
|
@@ -464,30 +510,63 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou
|
|
| 464 |
"be drafted from a description yet"), None
|
| 465 |
tmo = float(timeout or TIMEOUT_SECONDS)
|
| 466 |
problems = []
|
|
|
|
| 467 |
for p in provs:
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
try:
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 480 |
except Exception as e: # noqa: BLE001
|
| 481 |
problems.append(f"{p['name']}: {type(e).__name__}")
|
| 482 |
continue
|
| 483 |
-
if
|
| 484 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 485 |
continue
|
| 486 |
try:
|
| 487 |
-
body =
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 491 |
except Exception as e: # noqa: BLE001
|
| 492 |
problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
|
| 493 |
continue
|
|
|
|
| 17 |
last rather than absent β it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER`
|
| 18 |
pins one; `AIOS_AI_MODEL` overrides the model.
|
| 19 |
|
| 20 |
+
ββ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). β THIS PARAGRAPH
|
| 21 |
+
USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting
|
| 22 |
+
the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it.
|
| 23 |
+
The retired text said raw HTTP was *"a deliberate deviation β¦ booked as a DEBT line so the
|
| 24 |
+
integrator can overturn it deliberately rather than by drift"*. This is that overturn.
|
| 25 |
+
|
| 26 |
+
Its two reasons were real and are both answered rather than waved away:
|
| 27 |
+
1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is
|
| 28 |
+
not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers
|
| 29 |
+
the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no
|
| 30 |
+
Anthropic knowledge β it lost some.
|
| 31 |
+
2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the
|
| 32 |
+
import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the
|
| 33 |
+
same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH
|
| 34 |
+
manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave.
|
| 35 |
+
β The other three legs stay OpenAI-chat-shaped over `requests` β they have no shared SDK, and that
|
| 36 |
+
half of the original reasoning never expired.
|
| 37 |
|
| 38 |
The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and
|
| 39 |
`stop_reason: "refusal"` is checked BEFORE reading `content` β a refusal answers HTTP 200 with an
|
|
|
|
| 49 |
|
| 50 |
#: The ladder. Order IS the policy (R14) β cheapest capable first, Anthropic last.
|
| 51 |
PROVIDERS = [
|
| 52 |
+
# ββ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED
|
| 53 |
+
# it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent
|
| 54 |
+
# from `GET /openai/v1/models`. β THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo
|
| 55 |
+
# for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`.
|
| 56 |
{"name": "groq", "env": "GROQ_API_KEY", "shape": "openai",
|
| 57 |
"url": "https://api.groq.com/openai/v1/chat/completions",
|
| 58 |
+
"model": "openai/gpt-oss-120b"},
|
| 59 |
{"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai",
|
| 60 |
"url": "https://api.cerebras.ai/v1/chat/completions",
|
| 61 |
"model": "gpt-oss-120b"},
|
|
|
|
| 68 |
# labels it already has in front of it.
|
| 69 |
{"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
|
| 70 |
"url": "https://api.anthropic.com/v1/messages",
|
| 71 |
+
# β AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above,
|
| 72 |
+
# and a deployment whose side rungs are out of credit can raise the tier without a release.
|
| 73 |
+
"model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"},
|
| 74 |
]
|
| 75 |
ANTHROPIC_VERSION = "2023-06-01"
|
| 76 |
TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
|
|
|
|
| 173 |
return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body
|
| 174 |
|
| 175 |
|
| 176 |
+
#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
|
| 177 |
+
#: β PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say
|
| 178 |
+
#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit.
|
| 179 |
+
LAST_ANTHROPIC_TRANSPORT = ""
|
| 180 |
+
|
| 181 |
+
|
| 182 |
def _call_anthropic(p, model, system, user, timeout):
|
| 183 |
+
"""One classification turn on the Messages API, through `providers.anthropic_send`.
|
| 184 |
+
|
| 185 |
+
β D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which
|
| 186 |
+
prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is
|
| 187 |
+
absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises
|
| 188 |
+
both transports to `(status, body)`, so every line below this call is untouched.
|
| 189 |
+
"""
|
| 190 |
+
global LAST_ANTHROPIC_TRANSPORT
|
| 191 |
+
import providers as _prov # noqa: PLC0415
|
| 192 |
+
# β NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is
|
| 193 |
+
# nothing to strip here β the rule lives in `anthropic_request`, beside the thing it constrains.
|
| 194 |
+
req = _prov.anthropic_request(
|
| 195 |
+
model=model, key=os.environ[p["env"]].strip(), system=system,
|
| 196 |
+
messages=[{"role": "user", "content": user}], tools=[], max_tokens=300)
|
| 197 |
+
status, body, transport = _prov.anthropic_send(req, timeout=timeout)
|
| 198 |
+
LAST_ANTHROPIC_TRANSPORT = transport
|
| 199 |
+
if status >= 400 or status == 0:
|
| 200 |
+
# β THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the
|
| 201 |
+
# vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402"
|
| 202 |
+
# (which is what this line used to say, and what the owner quoted back at us) cannot recur.
|
| 203 |
+
return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None
|
| 204 |
+
body = body or {}
|
| 205 |
# β stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so
|
| 206 |
# reading content[0] before this check turns a refusal into an IndexError inside a run.
|
| 207 |
# β A refusal IS billed and its body carries `usage`, so the body rides back on this branch too.
|
|
|
|
| 310 |
MAX_PROMPT_CHARS = 2000
|
| 311 |
|
| 312 |
|
| 313 |
+
def flow_providers(pin=None):
|
| 314 |
+
"""The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
|
| 315 |
+
|
| 316 |
+
β `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and
|
| 317 |
+
the draft door used to read `prompt` off the body and nothing else β so the key the client sent
|
| 318 |
+
was accepted and dropped, and the picker was a control over nothing.
|
| 319 |
+
β AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the
|
| 320 |
+
ladder stopped offering must not turn every later draft into an error; the caller is told which
|
| 321 |
+
rung actually answered, which is the honest half.
|
| 322 |
+
"""
|
| 323 |
by_name = {p["name"]: p for p in PROVIDERS}
|
| 324 |
+
live = [n for n in FLOW_PROVIDER_ORDER
|
|
|
|
| 325 |
if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
|
| 326 |
+
wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
|
| 327 |
+
if wanted and wanted in live:
|
| 328 |
+
return [by_name[wanted]]
|
| 329 |
+
return [by_name[n] for n in live]
|
| 330 |
|
| 331 |
|
| 332 |
def flow_schema(kinds, trigger_keys, table_keys):
|
|
|
|
| 464 |
|
| 465 |
|
| 466 |
def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
|
| 467 |
+
st=None, user="", model=None):
|
| 468 |
"""A sentence -> `(draft, refusal_sentence, provider)`. β NOTHING IS SAVED HERE.
|
| 469 |
|
| 470 |
Exactly one of `draft` and `refusal_sentence` is truthy β the same contract
|
|
|
|
| 475 |
`routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and
|
| 476 |
D asserts it; the two keywords are what D has to pass for the line to be attributable.
|
| 477 |
"""
|
| 478 |
+
# β DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop.
|
| 479 |
+
# It parses either way; a reader scanning for the declaration does not look inside a `for`.
|
| 480 |
+
global LAST_ANTHROPIC_TRANSPORT
|
| 481 |
import usage_ledger # noqa: PLC0415
|
| 482 |
text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
|
| 483 |
if not text:
|
|
|
|
| 502 |
draft, refusal = _flow_from_tool_call(chat(messages, tools))
|
| 503 |
return draft, refusal, "injected"
|
| 504 |
|
| 505 |
+
provs = flow_providers(model)
|
| 506 |
if not provs:
|
| 507 |
# β SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
|
| 508 |
# never built [[flag-shipped-without-its-writer]].
|
|
|
|
| 510 |
"be drafted from a description yet"), None
|
| 511 |
tmo = float(timeout or TIMEOUT_SECONDS)
|
| 512 |
problems = []
|
| 513 |
+
import providers as _prov
|
| 514 |
for p in provs:
|
| 515 |
+
# ββ W36-T35 / R4 β THE OWNER QUOTED THIS LINE BACK AT US. It used to read
|
| 516 |
+
# `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`,
|
| 517 |
+
# so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the
|
| 518 |
+
# Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."*
|
| 519 |
+
# It is wired now, through the ONE wire in `providers`, and it matters more than it looks:
|
| 520 |
+
# every side rung on this account is refusing today (cerebras 402, groq 404, openrouter
|
| 521 |
+
# 402), so without this branch the drafter cannot answer at all.
|
| 522 |
+
# β NO `effort` ON THIS DOOR β `output_config.effort` errors on Haiku 4.5, the tier this
|
| 523 |
+
# ladder runs. The parameter is the caller's to send, which is why the wire takes it.
|
| 524 |
+
# β D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch
|
| 525 |
+
# below reads one thing. The anthropic side gets there through `anthropic_send` (official
|
| 526 |
+
# SDK when installed, the identical raw POST when not); the openai side still posts here
|
| 527 |
+
# because there is no shared SDK across three different vendors on that wire.
|
| 528 |
+
global LAST_ANTHROPIC_TRANSPORT
|
| 529 |
try:
|
| 530 |
+
if p["shape"] == "anthropic":
|
| 531 |
+
req = _prov.anthropic_request(
|
| 532 |
+
model=p["model"], key=os.environ[p["env"]].strip(), system=None,
|
| 533 |
+
messages=messages, tools=tools, max_tokens=1500, tool_choice="required")
|
| 534 |
+
status, body, transport = _prov.anthropic_send(req, timeout=tmo)
|
| 535 |
+
LAST_ANTHROPIC_TRANSPORT = transport
|
| 536 |
+
else:
|
| 537 |
+
r = requests.post(p["url"], timeout=tmo,
|
| 538 |
+
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
|
| 539 |
+
"Content-Type": "application/json"},
|
| 540 |
+
json={"model": p["model"], "messages": messages, "tools": tools,
|
| 541 |
+
"tool_choice": "required", "temperature": 0.1,
|
| 542 |
+
"max_tokens": 1500})
|
| 543 |
+
status = r.status_code
|
| 544 |
+
body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text}
|
| 545 |
except Exception as e: # noqa: BLE001
|
| 546 |
problems.append(f"{p['name']}: {type(e).__name__}")
|
| 547 |
continue
|
| 548 |
+
if status != 200:
|
| 549 |
+
# β A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off
|
| 550 |
+
# this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and
|
| 551 |
+
# the memo makes the next turn SKIP the empty account instead of paying for it again.
|
| 552 |
+
# β The body is re-serialised for the substring test because the SDK path never had a
|
| 553 |
+
# `.text` β that is the one thing this normalisation costs, and it costs nothing else.
|
| 554 |
+
detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600]
|
| 555 |
+
if _prov.is_credit_failure(status, detail):
|
| 556 |
+
_prov.mark_no_credit(p["name"])
|
| 557 |
+
problems.append(_prov.refusal_sentence(p["name"], status, detail))
|
| 558 |
continue
|
| 559 |
try:
|
| 560 |
+
body = body or {}
|
| 561 |
+
if p["shape"] == "anthropic":
|
| 562 |
+
_text, args, _refused = _prov.anthropic_read(body)
|
| 563 |
+
if _refused:
|
| 564 |
+
problems.append(f"{p['name']}: {_refused}")
|
| 565 |
+
continue
|
| 566 |
+
else:
|
| 567 |
+
calls = (((body.get("choices") or [{}])[0].get("message") or {})
|
| 568 |
+
.get("tool_calls") or [])
|
| 569 |
+
args = json.loads(calls[0]["function"]["arguments"]) if calls else None
|
| 570 |
except Exception as e: # noqa: BLE001
|
| 571 |
problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
|
| 572 |
continue
|
api/automation_engine.py
CHANGED
|
@@ -9219,6 +9219,36 @@ ACTION_CATALOG = [
|
|
| 9219 |
"ready": True,
|
| 9220 |
"detail": "Assemble this month's statements and park them for review. Nothing is sent until "
|
| 9221 |
"somebody opens the batch and clicks Send"},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9222 |
{"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
|
| 9223 |
"detail": "Needs the Slack connector"},
|
| 9224 |
# ββ 4. ADVANCED LOGIC ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -9306,6 +9336,33 @@ def _tenant_may_use(kind, rt):
|
|
| 9306 |
return True if gate is None else bool(rt is not None and gate(rt))
|
| 9307 |
|
| 9308 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9309 |
def action_catalog(rt=None):
|
| 9310 |
"""The catalog as the wire carries it β a copy, because a caller that mutated the module
|
| 9311 |
constant would change every later reader's answer.
|
|
@@ -10831,6 +10888,32 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 10831 |
"and nothing was sent.")
|
| 10832 |
counts["webBlocked"] += 1
|
| 10833 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10834 |
elif kind == "ai_agent":
|
| 10835 |
# ββ W33-T56 (owner item 7, ruling R3) β THE FUZZY STEP, AT RUN TIME.
|
| 10836 |
#
|
|
|
|
| 9219 |
"ready": True,
|
| 9220 |
"detail": "Assemble this month's statements and park them for review. Nothing is sent until "
|
| 9221 |
"somebody opens the batch and clicks Send"},
|
| 9222 |
+
# ββ WAVE 36 Β· W36-T39 β D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an
|
| 9223 |
+
# `ACTION_CATALOG` row with `menu: false` β one line, reusing the door R18 built this same wave
|
| 9224 |
+
# to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."*
|
| 9225 |
+
#
|
| 9226 |
+
# β THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows`
|
| 9227 |
+
# emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and
|
| 9228 |
+
# that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The
|
| 9229 |
+
# builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls
|
| 9230 |
+
# back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN.
|
| 9231 |
+
#
|
| 9232 |
+
# β `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" β a promise β
|
| 9233 |
+
# and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment
|
| 9234 |
+
# anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker,
|
| 9235 |
+
# still resolvable as a caption, still valid.
|
| 9236 |
+
# β AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation
|
| 9237 |
+
# canvas is not where one is created, which is why `patch_automation` already refuses a
|
| 9238 |
+
# `field:` id with "change its prompt, model or schedule on the column itself".
|
| 9239 |
+
# ββ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row`
|
| 9240 |
+
# emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and
|
| 9241 |
+
# that kind was in no catalog either β the same raw token on the same screen as D-277, one row
|
| 9242 |
+
# down. Two known instances were enough to justify the check; the check then produced a third
|
| 9243 |
+
# nobody had booked, which is the difference between a gate and a regression test.
|
| 9244 |
+
{"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected",
|
| 9245 |
+
"ready": True, "menu": False,
|
| 9246 |
+
"detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not "
|
| 9247 |
+
"here"},
|
| 9248 |
+
{"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected",
|
| 9249 |
+
"ready": True, "menu": False,
|
| 9250 |
+
"detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
|
| 9251 |
+
"here"},
|
| 9252 |
{"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
|
| 9253 |
"detail": "Needs the Slack connector"},
|
| 9254 |
# ββ 4. ADVANCED LOGIC ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 9336 |
return True if gate is None else bool(rt is not None and gate(rt))
|
| 9337 |
|
| 9338 |
|
| 9339 |
+
def catalog_kinds():
|
| 9340 |
+
"""Every action kind this module knows about β the LABEL vocabulary.
|
| 9341 |
+
|
| 9342 |
+
β D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the
|
| 9343 |
+
catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog
|
| 9344 |
+
does not carry is a token on somebody's screen. This is the set that must cover every kind any
|
| 9345 |
+
server path can put into a `flow.actions` entry, whether or not a person may add it.
|
| 9346 |
+
"""
|
| 9347 |
+
return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG)
|
| 9348 |
+
|
| 9349 |
+
|
| 9350 |
+
def configurable_kinds():
|
| 9351 |
+
"""The kinds a PERSON may add from the picker, and must therefore be able to configure.
|
| 9352 |
+
|
| 9353 |
+
ββ W36-T39 β THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is
|
| 9354 |
+
the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is
|
| 9355 |
+
ready and `menu: False` (D-277) β all six are real, runnable, captioned, and unofferable. What
|
| 9356 |
+
a client must be able to CONFIGURE is exactly what a person can ADD.
|
| 9357 |
+
|
| 9358 |
+
β DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list
|
| 9359 |
+
would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile
|
| 9360 |
+
were already one too many.
|
| 9361 |
+
"""
|
| 9362 |
+
return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG
|
| 9363 |
+
if row.get("ready") and row.get("menu", True) is not False)
|
| 9364 |
+
|
| 9365 |
+
|
| 9366 |
def action_catalog(rt=None):
|
| 9367 |
"""The catalog as the wire carries it β a copy, because a caller that mutated the module
|
| 9368 |
constant would change every later reader's answer.
|
|
|
|
| 10888 |
"and nothing was sent.")
|
| 10889 |
counts["webBlocked"] += 1
|
| 10890 |
continue
|
| 10891 |
+
elif kind == "odoo_sync":
|
| 10892 |
+
# Same refusal, same reason as the arm below: a catalog row with no arm is walked,
|
| 10893 |
+
# counted, and reports success having done nothing. The connector owns this sync.
|
| 10894 |
+
if "odoo_sync_not_run_here" not in web_notes:
|
| 10895 |
+
web_notes.append("odoo_sync_not_run_here")
|
| 10896 |
+
log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own "
|
| 10897 |
+
"schedule, not from this canvas. Nothing was done")
|
| 10898 |
+
counts["webBlocked"] += 1
|
| 10899 |
+
continue
|
| 10900 |
+
elif kind == "ai_enrich":
|
| 10901 |
+
# ββ A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE
|
| 10902 |
+
# STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is
|
| 10903 |
+
# walked, COUNTED, reports the run `ok` and writes nothing β a step somebody
|
| 10904 |
+
# configured that succeeds at doing nothing. Adding the row (D-277) without this
|
| 10905 |
+
# arm would have opened exactly that window.
|
| 10906 |
+
# β AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by
|
| 10907 |
+
# `ai_enrich`, driven from the column's own editor; the synthetic agent row this
|
| 10908 |
+
# kind appears in is DERIVED from that column and is never stored, so nothing
|
| 10909 |
+
# reaches here through the ordinary path. If something ever does, it must say so
|
| 10910 |
+
# rather than report success.
|
| 10911 |
+
if "ai_enrich_not_run_here" not in web_notes:
|
| 10912 |
+
web_notes.append("ai_enrich_not_run_here")
|
| 10913 |
+
log("[aios-auto] ai_enrich: an AI column is filled from the column's own "
|
| 10914 |
+
"editor, not from this canvas. Nothing was done")
|
| 10915 |
+
counts["webBlocked"] += 1
|
| 10916 |
+
continue
|
| 10917 |
elif kind == "ai_agent":
|
| 10918 |
# ββ W33-T56 (owner item 7, ruling R3) β THE FUZZY STEP, AT RUN TIME.
|
| 10919 |
#
|
api/main.py
CHANGED
|
@@ -85,6 +85,9 @@ import routes_slack # noqa: E402 (wave 33 R4/C2 β Manage agent + the Slack d
|
|
| 85 |
import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 β the star; E's router, E's line)
|
| 86 |
import routes_usage # noqa: E402 (wave 35 R9/C7 β the AI usage meter; E's router, E's line)
|
| 87 |
import routes_feedback # noqa: E402 (wave 35 R8/C6 β feedback to the operator plane; E's router)
|
|
|
|
|
|
|
|
|
|
| 88 |
from core import grid_events # noqa: E402
|
| 89 |
# D-315 / D-305 (2026-08-18) β the two store REFUSALS get their own app-level handlers below.
|
| 90 |
# β Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
|
@@ -376,6 +379,35 @@ app.include_router(routes_usage.router) # R9 / C7 β GET /usage, the one
|
|
| 376 |
# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does
|
| 377 |
# not widen anything a tenant admin can reach β `verify_api` proves that by having one try.
|
| 378 |
app.include_router(routes_feedback.router) # R8 / C6 β feedback to the operator plane
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 379 |
|
| 380 |
|
| 381 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
|
|
| 85 |
import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 β the star; E's router, E's line)
|
| 86 |
import routes_usage # noqa: E402 (wave 35 R9/C7 β the AI usage meter; E's router, E's line)
|
| 87 |
import routes_feedback # noqa: E402 (wave 35 R8/C6 β feedback to the operator plane; E's router)
|
| 88 |
+
import routes_agent_harness # noqa: E402 (wave 36 R8/C4 β the agent harness store; E's router)
|
| 89 |
+
import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 β script Views; E's router)
|
| 90 |
+
import routes_geo # noqa: E402 (wave 37 R14/A11 β the keyless map provider seam; D's router)
|
| 91 |
from core import grid_events # noqa: E402
|
| 92 |
# D-315 / D-305 (2026-08-18) β the two store REFUSALS get their own app-level handlers below.
|
| 93 |
# β Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
|
|
|
| 379 |
# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does
|
| 380 |
# not widen anything a tenant admin can reach β `verify_api` proves that by having one try.
|
| 381 |
app.include_router(routes_feedback.router) # R8 / C6 β feedback to the operator plane
|
| 382 |
+
# ββ WAVE 36 (R8 / C4) β THE AGENT HARNESS FILE STORE, mounted in the SAME change that created
|
| 383 |
+
# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the
|
| 384 |
+
# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in
|
| 385 |
+
# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107).
|
| 386 |
+
# β Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the
|
| 387 |
+
# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green.
|
| 388 |
+
# β ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES β and that is safe
|
| 389 |
+
# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot
|
| 390 |
+
# match `/agents/x/harness`. The two routers share a prefix and no route.
|
| 391 |
+
app.include_router(routes_agent_harness.router) # R8 / C4 β versioned agent harness files
|
| 392 |
+
# ββ WAVE 36 (R3 / R10 / C3) β THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that
|
| 393 |
+
# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in
|
| 394 |
+
# `app.openapi()["paths"]` AND calls them, and its NC comments this line out.
|
| 395 |
+
# β ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool.
|
| 396 |
+
# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header
|
| 397 |
+
# says why that is not a style choice.
|
| 398 |
+
app.include_router(routes_script_views.router) # R3 / R10 / C3 β code-script database Views
|
| 399 |
+
# ββ WAVE 37 (R14, amendment A11) β THE KEYLESS MAP PROVIDER SEAM, mounted on `ASK D-1` in the
|
| 400 |
+
# same wave that created `routes_geo.py`. NINTH consecutive wave in which this block is the
|
| 401 |
+
# artefact the protocol nearly loses, and the first in which the ask that carries it was itself
|
| 402 |
+
# lost: D raised it inside a ``` fence, so `wave_board.py` blanked it and the bus never delivered
|
| 403 |
+
# it (D-366). The integrator found it reading the peer mailbox by hand.
|
| 404 |
+
# β Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the
|
| 405 |
+
# end of this file, or GET answers 404 and POST answers 405 while every one of its own gates is
|
| 406 |
+
# green. D is adding a `web_map` leg that reads THIS FILE and asserts this line β keep it.
|
| 407 |
+
# β R14 put the provider behind a seam precisely so swapping Google in later is a config change:
|
| 408 |
+
# tiles = OpenStreetMap Β· routing = OSRM Β· geocoding = Nominatim, none of them keyed, all of them
|
| 409 |
+
# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden).
|
| 410 |
+
app.include_router(routes_geo.router) # R14 / A11 β D's router, A's line (map providers)
|
| 411 |
|
| 412 |
|
| 413 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
api/odoo_relational.py
CHANGED
|
@@ -636,8 +636,25 @@ def customer_fields():
|
|
| 636 |
"default": True, "pinned": True},
|
| 637 |
{"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay",
|
| 638 |
"default": False, "description": "The `res.partner` id. Also this row's id."},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 639 |
{"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
|
| 640 |
{"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True},
|
|
|
|
| 641 |
{"key": "country", "label": "Country", "type": "text", "source": "overlay",
|
| 642 |
"default": False},
|
| 643 |
{"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay",
|
|
@@ -976,8 +993,13 @@ def read_customers(cur, excluded=None):
|
|
| 976 |
# the mirror carries. Absent β the leg is dropped WHOLE, exactly like the agent join above.
|
| 977 |
rank_leg = (" OR (p.customer_rank > 0 AND p.active) "
|
| 978 |
if {"customer_rank", "active"} <= have else "")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 979 |
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
|
| 980 |
-
f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} "
|
|
|
|
| 981 |
"FROM res_partner p "
|
| 982 |
f"{join}"
|
| 983 |
"WHERE p.id IN ("
|
|
@@ -988,13 +1010,16 @@ def read_customers(cur, excluded=None):
|
|
| 988 |
f"{rank_leg}")
|
| 989 |
out = []
|
| 990 |
for r in cur.execute(sql).fetchall():
|
| 991 |
-
(pid, name, city, state, country, agent, agent_id) = r
|
| 992 |
out.append({
|
| 993 |
"_id": str(pid),
|
| 994 |
"customer": str(name or ""),
|
| 995 |
JOIN_KEY: int(pid),
|
|
|
|
|
|
|
| 996 |
"city": str(city or ""),
|
| 997 |
"state": str(state or ""),
|
|
|
|
| 998 |
"country": str(country or ""),
|
| 999 |
"agent": str(agent or ""),
|
| 1000 |
AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
|
|
|
|
| 636 |
"default": True, "pinned": True},
|
| 637 |
{"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay",
|
| 638 |
"default": False, "description": "The `res.partner` id. Also this row's id."},
|
| 639 |
+
# ββ W37-T11 / D-165 β THE POSTAL ADDRESS, which this table declared nowhere and served
|
| 640 |
+
# never. `verify_odoo_relational._prove_customer_address` recorded the blocker in its own
|
| 641 |
+
# docstring on 2026-08-12 β *"`street`/`street2`/`zip` are not among [the mirror's 16
|
| 642 |
+
# columns] β¦ blocked on a `sync_all()` backfill no ticket in wave 30 owns"*. That backfill
|
| 643 |
+
# HAS since run (`_sync_state` carries `res_partner.cols.β¦,street,street2,β¦,zip`, done
|
| 644 |
+
# 2026-08-15), so the half that could not be built now can be.
|
| 645 |
+
# β THIS IS ALSO LANE D'S DEPENDENCY. R4 puts addressβcoordinates in an enrichment field,
|
| 646 |
+
# and a geocoder pointed at a column that does not exist enriches nothing.
|
| 647 |
+
# β `default: False` on the postal lines, `True` on city/state β the same split
|
| 648 |
+
# `vendor_fields` made and for the same reason: four address columns on by default push
|
| 649 |
+
# the useful ones off the first screen. A locked database still allows fields, so every
|
| 650 |
+
# one is a click away in the picker.
|
| 651 |
+
{"key": "street", "label": "Street", "type": "text", "source": "overlay",
|
| 652 |
+
"default": False},
|
| 653 |
+
{"key": "street2", "label": "Street 2", "type": "text", "source": "overlay",
|
| 654 |
+
"default": False},
|
| 655 |
{"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
|
| 656 |
{"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True},
|
| 657 |
+
{"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False},
|
| 658 |
{"key": "country", "label": "Country", "type": "text", "source": "overlay",
|
| 659 |
"default": False},
|
| 660 |
{"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay",
|
|
|
|
| 993 |
# the mirror carries. Absent β the leg is dropped WHOLE, exactly like the agent join above.
|
| 994 |
rank_leg = (" OR (p.customer_rank > 0 AND p.active) "
|
| 995 |
if {"customer_rank", "active"} <= have else "")
|
| 996 |
+
# ββ W37-T11 / D-165 β the POSTAL columns join city/state/country here. They ride `_col` like
|
| 997 |
+
# every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL
|
| 998 |
+
# to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the
|
| 999 |
+
# live 500 that rule was written after).
|
| 1000 |
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
|
| 1001 |
+
f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, "
|
| 1002 |
+
f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} "
|
| 1003 |
"FROM res_partner p "
|
| 1004 |
f"{join}"
|
| 1005 |
"WHERE p.id IN ("
|
|
|
|
| 1010 |
f"{rank_leg}")
|
| 1011 |
out = []
|
| 1012 |
for r in cur.execute(sql).fetchall():
|
| 1013 |
+
(pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r
|
| 1014 |
out.append({
|
| 1015 |
"_id": str(pid),
|
| 1016 |
"customer": str(name or ""),
|
| 1017 |
JOIN_KEY: int(pid),
|
| 1018 |
+
"street": str(street or ""),
|
| 1019 |
+
"street2": str(street2 or ""),
|
| 1020 |
"city": str(city or ""),
|
| 1021 |
"state": str(state or ""),
|
| 1022 |
+
"zip": str(zipc or ""),
|
| 1023 |
"country": str(country or ""),
|
| 1024 |
"agent": str(agent or ""),
|
| 1025 |
AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
|
api/providers.py
CHANGED
|
@@ -39,6 +39,12 @@ import time
|
|
| 39 |
from dataclasses import dataclass, field
|
| 40 |
from typing import Callable
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
#: β APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK
|
| 43 |
#: providers and to estimate a run's spend for the operator. They are not billing truth β the
|
| 44 |
#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is
|
|
@@ -124,7 +130,7 @@ PROVIDERS: dict[str, Provider] = {
|
|
| 124 |
"likes + comments + shares + saves, and play_count inline (DECLARED "
|
| 125 |
"no-empties-or-zeros, UNPROVEN until one live pull)"),
|
| 126 |
"tt_comments": Capability(True, _COST_BRIGHTDATA,
|
| 127 |
-
"separate paid dataset, opt-in
|
| 128 |
}),
|
| 129 |
"apify": Provider(
|
| 130 |
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
|
|
@@ -132,7 +138,7 @@ PROVIDERS: dict[str, Provider] = {
|
|
| 132 |
# β MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
|
| 133 |
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
|
| 134 |
"ig_post_views": Capability(True, _COST_APIFY,
|
| 135 |
-
"videoPlayCount
|
| 136 |
"ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
|
| 137 |
"ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
|
| 138 |
}),
|
|
@@ -240,12 +246,12 @@ def run(capability, work, satisfied=None, log=None):
|
|
| 240 |
secs = round(time.time() - started, 2)
|
| 241 |
if note:
|
| 242 |
attempts.append(Attempt(provider.key, False, note, 0, secs))
|
| 243 |
-
log(f" {provider.label}: {note}
|
| 244 |
continue
|
| 245 |
if not ok(result):
|
| 246 |
attempts.append(Attempt(provider.key, False, "answered without the field asked for",
|
| 247 |
0, secs))
|
| 248 |
-
log(f" {provider.label}: answered, but not with what was asked for
|
| 249 |
last = result if last is None else last
|
| 250 |
continue
|
| 251 |
attempts.append(Attempt(provider.key, True, "", _count(result), secs))
|
|
@@ -282,6 +288,444 @@ def wire():
|
|
| 282 |
}
|
| 283 |
|
| 284 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
# =============================================================================================
|
| 286 |
# THE CANONICAL SCHEMA β owner ruling 2026-08-08:
|
| 287 |
# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
|
|
|
|
| 39 |
from dataclasses import dataclass, field
|
| 40 |
from typing import Callable
|
| 41 |
|
| 42 |
+
# β TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in
|
| 43 |
+
# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the
|
| 44 |
+
# line that stops a container booting. That asymmetry is the whole reason the two are imported
|
| 45 |
+
# differently β see `anthropic_sdk()`.
|
| 46 |
+
import requests
|
| 47 |
+
|
| 48 |
#: β APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK
|
| 49 |
#: providers and to estimate a run's spend for the operator. They are not billing truth β the
|
| 50 |
#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is
|
|
|
|
| 130 |
"likes + comments + shares + saves, and play_count inline (DECLARED "
|
| 131 |
"no-empties-or-zeros, UNPROVEN until one live pull)"),
|
| 132 |
"tt_comments": Capability(True, _COST_BRIGHTDATA,
|
| 133 |
+
"separate paid dataset, opt-in: 17 fields"),
|
| 134 |
}),
|
| 135 |
"apify": Provider(
|
| 136 |
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
|
|
|
|
| 138 |
# β MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
|
| 139 |
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
|
| 140 |
"ig_post_views": Capability(True, _COST_APIFY,
|
| 141 |
+
"videoPlayCount, matches Instagram's displayed views"),
|
| 142 |
"ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
|
| 143 |
"ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
|
| 144 |
}),
|
|
|
|
| 246 |
secs = round(time.time() - started, 2)
|
| 247 |
if note:
|
| 248 |
attempts.append(Attempt(provider.key, False, note, 0, secs))
|
| 249 |
+
log(f" {provider.label}: {note}, falling through")
|
| 250 |
continue
|
| 251 |
if not ok(result):
|
| 252 |
attempts.append(Attempt(provider.key, False, "answered without the field asked for",
|
| 253 |
0, secs))
|
| 254 |
+
log(f" {provider.label}: answered, but not with what was asked for, falling through")
|
| 255 |
last = result if last is None else last
|
| 256 |
continue
|
| 257 |
attempts.append(Attempt(provider.key, True, "", _count(result), secs))
|
|
|
|
| 288 |
}
|
| 289 |
|
| 290 |
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
# βββββββββββββββββββ WAVE 36 Β· W36-T35 (ruling R4, contract C5) β THE LLM LADDER ββββββββββββββββ
|
| 294 |
+
#
|
| 295 |
+
# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the
|
| 296 |
+
# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras:
|
| 297 |
+
# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this
|
| 298 |
+
# shape'."*
|
| 299 |
+
#
|
| 300 |
+
# ββ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the
|
| 301 |
+
# tool-calling path that always works β a WIRE, in `routes_query`. (2) A provider with no credit is
|
| 302 |
+
# SKIPPED rather than tried β a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a
|
| 303 |
+
# screen β a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read.
|
| 304 |
+
#
|
| 305 |
+
# β WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is
|
| 306 |
+
# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is
|
| 307 |
+
# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half
|
| 308 |
+
# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the
|
| 309 |
+
# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both
|
| 310 |
+
# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does.
|
| 311 |
+
#
|
| 312 |
+
# β THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that
|
| 313 |
+
# recovers a tool call out of a provider's 400 β the evidence that guessing at capability failed.
|
| 314 |
+
# A row that says `llm_tool_calling: Capability(False, β¦)` is never offered for a tool-calling
|
| 315 |
+
# turn at all, so the guess never has to be made.
|
| 316 |
+
|
| 317 |
+
#: How long a provider stays skipped after it tells us it is out of credit. β A MEMO, NOT A FACT:
|
| 318 |
+
#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen
|
| 319 |
+
#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short
|
| 320 |
+
#: enough that a top-up is picked up without a restart.
|
| 321 |
+
CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900)
|
| 322 |
+
|
| 323 |
+
#: `{provider name: unix ts when the memo expires}`. β PROCESS-LOCAL AND DELIBERATELY SO β it is a
|
| 324 |
+
#: latency optimisation, not a billing record. A second container learns the same thing from its
|
| 325 |
+
#: own first 402, and neither one can be wrong for longer than the cooldown.
|
| 326 |
+
_NO_CREDIT: dict[str, float] = {}
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
@dataclass(frozen=True)
|
| 330 |
+
class LlmProvider:
|
| 331 |
+
"""One chat-completions endpoint, and what it is DECLARED able to do."""
|
| 332 |
+
name: str
|
| 333 |
+
label: str
|
| 334 |
+
env: str
|
| 335 |
+
url: str
|
| 336 |
+
model: str
|
| 337 |
+
#: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API,
|
| 338 |
+
#: which is a different body, a different auth header and a different result shape.
|
| 339 |
+
wire: str
|
| 340 |
+
caps: dict = field(default_factory=dict)
|
| 341 |
+
|
| 342 |
+
def configured(self) -> bool:
|
| 343 |
+
return bool((os.environ.get(self.env) or "").strip())
|
| 344 |
+
|
| 345 |
+
def can(self, capability: str) -> bool:
|
| 346 |
+
cap = self.caps.get(capability)
|
| 347 |
+
return bool(cap and cap.capable and self.configured())
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
#: β ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put
|
| 351 |
+
#: cerebras first *"because this path needs tool calling and cerebras carries this account's
|
| 352 |
+
#: tool-capable model"* β R4 replaces that premise: Anthropic is the tool-calling path that always
|
| 353 |
+
#: works, and the others are the cheap seats it falls through to.
|
| 354 |
+
LLM_PROVIDERS: dict[str, LlmProvider] = {
|
| 355 |
+
"anthropic": LlmProvider(
|
| 356 |
+
name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY",
|
| 357 |
+
url="https://api.anthropic.com/v1/messages",
|
| 358 |
+
# β Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with
|
| 359 |
+
# `AIOS_ANTHROPIC_MODEL` β the id is read at call time, so a cheaper tier
|
| 360 |
+
# (`claude-sonnet-5`, $3 / $15) is an environment change, not a release.
|
| 361 |
+
# β W37-T39 re-verified this default against the vendor rather than the docs: a real
|
| 362 |
+
# Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`.
|
| 363 |
+
model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5",
|
| 364 |
+
wire="anthropic",
|
| 365 |
+
caps={
|
| 366 |
+
# β MEASURED, and it is the whole of R4's first clause: the Messages API answers with a
|
| 367 |
+
# typed `tool_use` content block carrying parsed `input`. There is nothing to recover
|
| 368 |
+
# out of a 400 and no regex in the path β which is exactly what `_FAILED_GEN` exists to
|
| 369 |
+
# apologise for on the other wire.
|
| 370 |
+
"llm_tool_calling": Capability(True, 0.0,
|
| 371 |
+
"typed tool_use content block; no text recovery path"),
|
| 372 |
+
"llm_chat": Capability(True, 0.0, "Messages API"),
|
| 373 |
+
"llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"),
|
| 374 |
+
}),
|
| 375 |
+
"cerebras": LlmProvider(
|
| 376 |
+
name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY",
|
| 377 |
+
url="https://api.cerebras.ai/v1/chat/completions",
|
| 378 |
+
# β STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the
|
| 379 |
+
# two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on
|
| 380 |
+
# this account β a real POST answers `HTTP 402 payment_required`, which `mark_no_credit`
|
| 381 |
+
# already knows how to survive (R4: skipped, not tried).
|
| 382 |
+
model="gpt-oss-120b", wire="openai",
|
| 383 |
+
caps={
|
| 384 |
+
# β STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not
|
| 385 |
+
# measure this rung: a 402 comes back before any tool is considered, so there is no
|
| 386 |
+
# observation to record. Leaving the old wording is the honest answer β upgrading it
|
| 387 |
+
# to MEASURED beside its two neighbours would be claiming an account balance as
|
| 388 |
+
# evidence about a capability.
|
| 389 |
+
"llm_tool_calling": Capability(True, 0.0,
|
| 390 |
+
"DECLARED, not measured: this account's tool-capable "
|
| 391 |
+
"model per the wave-32 ladder note. β 2026-08-19: the "
|
| 392 |
+
"account answers 402, so this stays unmeasured"),
|
| 393 |
+
"llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
|
| 394 |
+
"llm_json_mode": Capability(True, 0.0, "response_format json_object"),
|
| 395 |
+
}),
|
| 396 |
+
"groq": LlmProvider(
|
| 397 |
+
name="groq", label="Groq", env="GROQ_API_KEY",
|
| 398 |
+
# ββ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq
|
| 399 |
+
# RETIRED it. Measured, not inferred β a real POST with the live key answered
|
| 400 |
+
# `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with
|
| 401 |
+
# that one absent. β THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A
|
| 402 |
+
# RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the
|
| 403 |
+
# account and says nothing about its model id (`gpt-oss-120b` is in its /models list).
|
| 404 |
+
# "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe.
|
| 405 |
+
# β THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while
|
| 406 |
+
# Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY
|
| 407 |
+
# spelled differently and a sweep that "normalises" them breaks one of them.
|
| 408 |
+
url="https://api.groq.com/openai/v1/chat/completions",
|
| 409 |
+
model="openai/gpt-oss-120b", wire="openai",
|
| 410 |
+
caps={
|
| 411 |
+
# β MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old
|
| 412 |
+
# text read *"DECLARED, not measured β¦ `routes_query._FAILED_GEN` exists because SOME
|
| 413 |
+
# provider on this wire answers 400 with the call in `failed_generation`; the repo
|
| 414 |
+
# never recorded which. Flip this to False the day it is"*. It is recorded now: on
|
| 415 |
+
# this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed
|
| 416 |
+
# `arguments`, so it is not the provider `_FAILED_GEN` apologises for.
|
| 417 |
+
"llm_tool_calling": Capability(True, 0.0,
|
| 418 |
+
"MEASURED 2026-08-19: HTTP 200, typed tool_calls block "
|
| 419 |
+
"with parsed arguments; no failed_generation recovery"),
|
| 420 |
+
"llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
|
| 421 |
+
"llm_json_mode": Capability(True, 0.0, "response_format json_object"),
|
| 422 |
+
}),
|
| 423 |
+
"openrouter": LlmProvider(
|
| 424 |
+
name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY",
|
| 425 |
+
url="https://openrouter.ai/api/v1/chat/completions",
|
| 426 |
+
model="openai/gpt-4o-mini", wire="openai",
|
| 427 |
+
caps={
|
| 428 |
+
# β MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed
|
| 429 |
+
# `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and
|
| 430 |
+
# `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use.
|
| 431 |
+
"llm_tool_calling": Capability(True, 0.0,
|
| 432 |
+
"MEASURED 2026-08-19: HTTP 200, typed tool_calls block"),
|
| 433 |
+
"llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
|
| 434 |
+
"llm_json_mode": Capability(True, 0.0, "response_format json_object"),
|
| 435 |
+
}),
|
| 436 |
+
}
|
| 437 |
+
|
| 438 |
+
LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter")
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
def mark_no_credit(name, seconds=None):
|
| 442 |
+
"""Remember that `name` said it is out of credit, so the next turn SKIPS it (R4).
|
| 443 |
+
|
| 444 |
+
β THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for
|
| 445 |
+
that to live: a stateless ladder re-tries the empty account on every single turn, pays its
|
| 446 |
+
round trip, and shows the reader a longer error each time. This is that memo.
|
| 447 |
+
"""
|
| 448 |
+
_NO_CREDIT[str(name)] = time.time() + float(
|
| 449 |
+
CREDIT_COOLDOWN_S if seconds is None else seconds)
|
| 450 |
+
return _NO_CREDIT[str(name)]
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
def no_credit(name):
|
| 454 |
+
"""Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed."""
|
| 455 |
+
until = _NO_CREDIT.get(str(name))
|
| 456 |
+
if not until:
|
| 457 |
+
return False
|
| 458 |
+
if time.time() >= until:
|
| 459 |
+
_NO_CREDIT.pop(str(name), None)
|
| 460 |
+
return False
|
| 461 |
+
return True
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
def clear_credit_memo(name=None):
|
| 465 |
+
"""Forget one memo, or all of them. For a gate, and for an operator after a top-up."""
|
| 466 |
+
if name is None:
|
| 467 |
+
_NO_CREDIT.clear()
|
| 468 |
+
else:
|
| 469 |
+
_NO_CREDIT.pop(str(name), None)
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
def llm_chain(capability="llm_tool_calling"):
|
| 473 |
+
"""The provider order for `capability` β declaration first, credit memo second.
|
| 474 |
+
|
| 475 |
+
Three filters, in this order, and each removes a DIFFERENT kind of row:
|
| 476 |
+
1. `can()` β declared capable AND configured. An incapable row is never offered, so a
|
| 477 |
+
turn cannot be spent discovering it (the `chain()` rule, one layer up).
|
| 478 |
+
2. `no_credit()` β R4's skip. A provider that told us its balance is empty is passed over
|
| 479 |
+
until the memo expires.
|
| 480 |
+
3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment
|
| 481 |
+
can be moved off a vendor without a release β the same clause `AIOS_PROVIDER_ORDER`
|
| 482 |
+
carries for the scraping side.
|
| 483 |
+
"""
|
| 484 |
+
raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip()
|
| 485 |
+
names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER)
|
| 486 |
+
return [LLM_PROVIDERS[n] for n in names
|
| 487 |
+
if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)]
|
| 488 |
+
|
| 489 |
+
|
| 490 |
+
#: What an HTTP status MEANS, in words a person can act on. ββ R4's THIRD CLAUSE LIVES HERE AND
|
| 491 |
+
#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner
|
| 492 |
+
#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION.
|
| 493 |
+
_STATUS_WORDS = {
|
| 494 |
+
401: "{label} would not accept our key",
|
| 495 |
+
403: "{label} would not accept our key",
|
| 496 |
+
402: "{label} is out of credit",
|
| 497 |
+
404: "{label} does not offer the model we asked it for",
|
| 498 |
+
408: "{label} took too long",
|
| 499 |
+
413: "the question was too long for {label}",
|
| 500 |
+
429: "{label} is rate limiting us right now",
|
| 501 |
+
}
|
| 502 |
+
|
| 503 |
+
#: Substrings that mean "no money" on a wire that does not use 402. β Anthropic answers a spent
|
| 504 |
+
#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the
|
| 505 |
+
#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and
|
| 506 |
+
#: it only ever decides whether to SKIP a provider, never whether to trust one.
|
| 507 |
+
_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit",
|
| 508 |
+
"quota exceeded", "billing", "payment required", "add credits")
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def is_credit_failure(status, body=""):
|
| 512 |
+
"""Did this response mean "the account is empty"? Status first, then the vendor's own words."""
|
| 513 |
+
if int(status or 0) == 402:
|
| 514 |
+
return True
|
| 515 |
+
if int(status or 0) not in (400, 403, 429):
|
| 516 |
+
return False
|
| 517 |
+
return any(word in str(body or "").lower() for word in _CREDIT_WORDS)
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
def refusal_sentence(name, status, body=""):
|
| 521 |
+
"""One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace.
|
| 522 |
+
|
| 523 |
+
β THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key
|
| 524 |
+
prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a
|
| 525 |
+
credit problem", and what reaches the caller is this module's own wording.
|
| 526 |
+
"""
|
| 527 |
+
label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label
|
| 528 |
+
if is_credit_failure(status, body):
|
| 529 |
+
return f"{label} is out of credit"
|
| 530 |
+
code = int(status or 0)
|
| 531 |
+
if code in _STATUS_WORDS:
|
| 532 |
+
return _STATUS_WORDS[code].format(label=label)
|
| 533 |
+
if 500 <= code <= 599:
|
| 534 |
+
return f"{label} is having trouble at their end"
|
| 535 |
+
return f"{label} did not answer"
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
# βββββββββββ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) βββββββββββββββββββββββββββ
|
| 539 |
+
#
|
| 540 |
+
# ββ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API.
|
| 541 |
+
# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation
|
| 542 |
+
# drafter) both need it, and the owner quoted an error from EACH of them in one breath:
|
| 543 |
+
# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool
|
| 544 |
+
# calls are not wired for this shape"*. Two implementations of one wire is
|
| 545 |
+
# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the
|
| 546 |
+
# ladder that declares the rung.
|
| 547 |
+
#
|
| 548 |
+
# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own:
|
| 549 |
+
# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message
|
| 550 |
+
# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function`
|
| 551 |
+
# 3. `temperature` and friends are REMOVED on the current model family
|
| 552 |
+
# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string
|
| 553 |
+
#
|
| 554 |
+
# β AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on
|
| 555 |
+
# Haiku 4.5, so it is a PARAMETER here and the caller decides β the drafter runs on haiku and omits
|
| 556 |
+
# it, the assistant runs on the Opus tier and sends it.
|
| 557 |
+
|
| 558 |
+
#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model.
|
| 559 |
+
ANTHROPIC_VERSION = "2023-06-01"
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
def anthropic_request(*, model, key, system, messages, tools, max_tokens,
|
| 563 |
+
tool_choice="auto", effort=None):
|
| 564 |
+
"""`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing.
|
| 565 |
+
|
| 566 |
+
`messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted
|
| 567 |
+
out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list,
|
| 568 |
+
re-addressed rather than re-derived, so a schema change happens in one place.
|
| 569 |
+
"""
|
| 570 |
+
system_text = "\n\n".join(str(m.get("content") or "") for m in messages
|
| 571 |
+
if m.get("role") == "system")
|
| 572 |
+
if system:
|
| 573 |
+
system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system)
|
| 574 |
+
turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"),
|
| 575 |
+
"content": str(m.get("content") or "")}
|
| 576 |
+
for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()]
|
| 577 |
+
wire_tools = [{"name": t["function"]["name"],
|
| 578 |
+
"description": t["function"]["description"],
|
| 579 |
+
"input_schema": t["function"]["parameters"]} for t in (tools or [])]
|
| 580 |
+
body = {
|
| 581 |
+
"model": str(model),
|
| 582 |
+
"max_tokens": int(max_tokens),
|
| 583 |
+
"system": system_text,
|
| 584 |
+
"messages": turns,
|
| 585 |
+
"tools": wire_tools,
|
| 586 |
+
}
|
| 587 |
+
# β W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that
|
| 588 |
+
# rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list,
|
| 589 |
+
# so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no
|
| 590 |
+
# tools β a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]:
|
| 591 |
+
# the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem.
|
| 592 |
+
# β Safe for all three callers today, checked rather than assumed: `routes_query` and
|
| 593 |
+
# `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before.
|
| 594 |
+
if wire_tools:
|
| 595 |
+
body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"}
|
| 596 |
+
if effort:
|
| 597 |
+
body["output_config"] = {"effort": str(effort)}
|
| 598 |
+
return {"url": LLM_PROVIDERS["anthropic"].url,
|
| 599 |
+
"headers": {"x-api-key": str(key),
|
| 600 |
+
"anthropic-version": ANTHROPIC_VERSION,
|
| 601 |
+
"content-type": "application/json"},
|
| 602 |
+
"json": body}
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent.
|
| 606 |
+
#: β `None` is NOT the "absent" sentinel β a plain falsy check would re-attempt the import on every
|
| 607 |
+
#: call, and a failed import is not cheap. `False` says "asked and answered".
|
| 608 |
+
_SDK: object = None
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
def anthropic_sdk():
|
| 612 |
+
"""The official `anthropic` SDK, or `None` if this deployment does not carry it.
|
| 613 |
+
|
| 614 |
+
ββ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK.
|
| 615 |
+
`aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's
|
| 616 |
+
fence this wave, so the pin is another session's edit. A hard `import anthropic` at module
|
| 617 |
+
scope would therefore turn a missing line in a manifest into a container that cannot boot at
|
| 618 |
+
all: the API imports this module on every request path.
|
| 619 |
+
|
| 620 |
+
So the import is LAZY and its absence is REPORTED rather than fatal β `anthropic_send` falls
|
| 621 |
+
back to the raw `requests` POST that has always worked, and says which transport ran. When the
|
| 622 |
+
pin lands the SDK path takes over with no further change.
|
| 623 |
+
β The fallback is a BRIDGE, not a second implementation: both transports send the SAME body
|
| 624 |
+
`anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`,
|
| 625 |
+
`is_credit_failure` and `refusal_sentence` each keep exactly one normalizer
|
| 626 |
+
([[one-question-two-normalizers]]).
|
| 627 |
+
"""
|
| 628 |
+
global _SDK
|
| 629 |
+
if _SDK is None:
|
| 630 |
+
try:
|
| 631 |
+
import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring
|
| 632 |
+
_SDK = _mod
|
| 633 |
+
except Exception:
|
| 634 |
+
_SDK = False
|
| 635 |
+
return _SDK or None
|
| 636 |
+
|
| 637 |
+
|
| 638 |
+
def anthropic_send(req, timeout=None, use_sdk=None):
|
| 639 |
+
"""POST one `anthropic_request()` dict. Returns `(status, body, transport)`.
|
| 640 |
+
|
| 641 |
+
β `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader
|
| 642 |
+
who cannot tell which path ran cannot tell whether the requirements pin reached the container
|
| 643 |
+
[[report-the-cause-before-you-fix-it]]. β A gate must assert on THIS RETURN VALUE, never on
|
| 644 |
+
`ai_review.LAST_ANTHROPIC_TRANSPORT` β that global is a convenience for `GET /meta`, and a
|
| 645 |
+
check keyed to it passes on a stale value written by an earlier check in the same process.
|
| 646 |
+
|
| 647 |
+
`status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error β
|
| 648 |
+
the SDK raises where `requests` returns, and normalising that difference here is the only
|
| 649 |
+
reason this function exists rather than the caller branching on transport.
|
| 650 |
+
|
| 651 |
+
β `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports
|
| 652 |
+
without assigning to `_SDK`: a test that mutates a module global and then throws leaves every
|
| 653 |
+
later check in that process running on the wrong path and passing for the wrong reason.
|
| 654 |
+
"""
|
| 655 |
+
sdk = None if use_sdk is False else anthropic_sdk()
|
| 656 |
+
payload = dict(req.get("json") or {})
|
| 657 |
+
if sdk is None:
|
| 658 |
+
r = requests.post(req["url"], headers=req["headers"], json=payload,
|
| 659 |
+
timeout=(timeout or 60))
|
| 660 |
+
try:
|
| 661 |
+
return r.status_code, (r.json() if r.content else {}), "http"
|
| 662 |
+
except Exception:
|
| 663 |
+
return r.status_code, {"_text": (r.text or "")[:2000]}, "http"
|
| 664 |
+
|
| 665 |
+
key = (req.get("headers") or {}).get("x-api-key") or ""
|
| 666 |
+
try:
|
| 667 |
+
client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60))
|
| 668 |
+
msg = client.messages.create(**payload)
|
| 669 |
+
# β `.model_dump()` is what makes ONE reader serve both transports: it hands back the same
|
| 670 |
+
# wire-shaped dict the raw POST parses out of the response body.
|
| 671 |
+
return 200, msg.model_dump(), "sdk"
|
| 672 |
+
except Exception as exc:
|
| 673 |
+
# β THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by
|
| 674 |
+
# `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out
|
| 675 |
+
# of credit" into "Anthropic did not answer" β the exact regression R4 was written to end.
|
| 676 |
+
status = int(getattr(exc, "status_code", 0) or 0)
|
| 677 |
+
body = getattr(exc, "body", None)
|
| 678 |
+
if not isinstance(body, dict):
|
| 679 |
+
body = {"_text": str(exc)[:2000]}
|
| 680 |
+
if not status:
|
| 681 |
+
# No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one
|
| 682 |
+
# code `_STATUS_WORDS` already words as a reachability problem rather than a refusal.
|
| 683 |
+
status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0
|
| 684 |
+
return status, body, "sdk"
|
| 685 |
+
|
| 686 |
+
|
| 687 |
+
def anthropic_read(body):
|
| 688 |
+
"""`(text, tool_input, refusal)` out of a Messages API answer.
|
| 689 |
+
|
| 690 |
+
β `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with
|
| 691 |
+
`stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]`
|
| 692 |
+
unconditionally breaks on exactly the turn a person most needs explained.
|
| 693 |
+
β AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict β no `json.loads`, and
|
| 694 |
+
no regex recovering a call out of a 400, which is what the other wire needs.
|
| 695 |
+
"""
|
| 696 |
+
body = body if isinstance(body, dict) else {}
|
| 697 |
+
if str(body.get("stop_reason") or "") == "refusal":
|
| 698 |
+
return "", None, "the assistant declined to answer that one"
|
| 699 |
+
blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)]
|
| 700 |
+
text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip()
|
| 701 |
+
calls = [b for b in blocks if b.get("type") == "tool_use"]
|
| 702 |
+
got = calls[0].get("input") if calls else None
|
| 703 |
+
return text, (got if isinstance(got, dict) else None), None
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
def llm_status(capability="llm_tool_calling"):
|
| 707 |
+
"""Per provider: configured, declared-capable, in cooldown, and WHY β contract C5's payload.
|
| 708 |
+
|
| 709 |
+
β ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read
|
| 710 |
+
THIS, so a model offered in one place cannot be missing from the other, and neither can offer a
|
| 711 |
+
provider the ladder would refuse to call [[permitted-is-not-answerable]].
|
| 712 |
+
"""
|
| 713 |
+
rows = []
|
| 714 |
+
for name in LLM_DEFAULT_ORDER:
|
| 715 |
+
p = LLM_PROVIDERS[name]
|
| 716 |
+
cap = p.caps.get(capability)
|
| 717 |
+
rows.append({
|
| 718 |
+
"provider": p.name, "label": p.label, "model": p.model, "wire": p.wire,
|
| 719 |
+
"configured": p.configured(),
|
| 720 |
+
"toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable),
|
| 721 |
+
"jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable),
|
| 722 |
+
"capable": bool(cap and cap.capable),
|
| 723 |
+
"outOfCredit": no_credit(name),
|
| 724 |
+
"note": (cap.note if cap else ""),
|
| 725 |
+
})
|
| 726 |
+
return rows
|
| 727 |
+
|
| 728 |
+
|
| 729 |
# =============================================================================================
|
| 730 |
# THE CANONICAL SCHEMA β owner ruling 2026-08-08:
|
| 731 |
# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
|
api/routes_admin.py
CHANGED
|
@@ -46,6 +46,18 @@ router = APIRouter(prefix="/api/v1")
|
|
| 46 |
#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no
|
| 47 |
#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename.
|
| 48 |
_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog
|
| 50 |
#: enforces nothing; this does, and the two are allowed to differ because only one of them is
|
| 51 |
#: reachable from the internet.
|
|
@@ -168,11 +180,19 @@ def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
|
|
| 168 |
|
| 169 |
# ββ request validation: every rule below is fail-closed ββββββββββββββββββββββββββββββββββββββββββ
|
| 170 |
def _clean_username(v):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
uname = str(v or "").strip().lower()
|
| 172 |
-
if not _UNAME_RE.match(uname):
|
| 173 |
raise err(400, "bad_username",
|
| 174 |
-
"a username is 2
|
| 175 |
-
"underscore,
|
| 176 |
return uname
|
| 177 |
|
| 178 |
|
|
@@ -297,8 +317,35 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi
|
|
| 297 |
agent = str(body.get("agent") or "").strip() or None
|
| 298 |
email = str(body.get("email") or "").strip() or None
|
| 299 |
|
| 300 |
-
|
|
|
|
| 301 |
raise err(409, "user_exists", f"{uname} already exists β PATCH it to change it")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
# Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's β
|
| 303 |
# the tenant is stamped from the session, not taken from the body.
|
| 304 |
users.create_user(uname, pw, name, role=role, bus=bus, modules=modules,
|
|
@@ -708,9 +755,15 @@ def _clean_perms(v, governed_keys=None, session=None):
|
|
| 708 |
f"saved with the bad conditions silently removed, which would store a "
|
| 709 |
f"weaker wall than the one on screen.")
|
| 710 |
clean_tree = {"conj": conj, "nodes": cleaned}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 711 |
out[key] = {"access": bool(raw.get("access", True)),
|
| 712 |
"filter": clean_tree,
|
| 713 |
-
"hiddenFields": sorted({str(h) for h in hidden})
|
|
|
|
| 714 |
_refuse_unshapeable_bu(out)
|
| 715 |
return out
|
| 716 |
|
|
@@ -1031,8 +1084,16 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 1031 |
_default = (perm_scope.nav_may_open(users._public(uname, rec), k)
|
| 1032 |
if any(m.get("surface") and m["key"] == k for m in modules) else
|
| 1033 |
perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
|
| 1034 |
-
|
| 1035 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1036 |
return {"username": uname,
|
| 1037 |
"perms": perms_out,
|
| 1038 |
# S3 ask 2 β the marker rides the GET. An absent module entry means DENY on a
|
|
|
|
| 46 |
#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no
|
| 47 |
#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename.
|
| 48 |
_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$")
|
| 49 |
+
#: ββ WAVE 37 Β· T03 (owner ruling R8) β AN EMAIL ADDRESS IS ALSO A USERNAME. Owner: the username
|
| 50 |
+
#: IS the email, one identity. β THIS IS NOT A NEW CAPABILITY, IT IS THE DOOR CATCHING UP WITH THE
|
| 51 |
+
#: DATA: measured on deployed staging before the change, FIVE of tenant #0's eight accounts already
|
| 52 |
+
#: carry an `@` in the `username` field (`david@fischfloralsupply.com`, `farhan@`, `florencia@`,
|
| 53 |
+
#: `karen@`, `naomi@`) β every one of them illegal under the slug pattern above and longer than its
|
| 54 |
+
#: 32-character cap, so this route could not have created any of them.
|
| 55 |
+
#: β DELIBERATELY PERMISSIVE, because the store key is the thing being constrained and not the
|
| 56 |
+
#: deliverability of the address: one `@`, a dot-bearing domain, and the same character class the
|
| 57 |
+
#: slug already trusts. A stricter grammar here would refuse addresses that exist.
|
| 58 |
+
_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$")
|
| 59 |
+
#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`.
|
| 60 |
+
_UNAME_MAX = 254
|
| 61 |
#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog
|
| 62 |
#: enforces nothing; this does, and the two are allowed to differ because only one of them is
|
| 63 |
#: reachable from the internet.
|
|
|
|
| 180 |
|
| 181 |
# ββ request validation: every rule below is fail-closed ββββββββββββββββββββββββββββββββββββββββββ
|
| 182 |
def _clean_username(v):
|
| 183 |
+
"""The ONE validator for a new account's key. R8 widened it to admit a full email address.
|
| 184 |
+
|
| 185 |
+
β IT HAS EXACTLY ONE CALL SITE AND THAT IS LOAD-BEARING, not an oversight to tidy up.
|
| 186 |
+
`PATCH /admin/users/{username}`, the password route and both `/perms` routes take the username
|
| 187 |
+
as a PATH parameter and never re-validate it, which is precisely why the five email-keyed
|
| 188 |
+
accounts above are administrable today despite being unconstructable. Arming this validator on
|
| 189 |
+
those routes would look like hardening and would break five live accounts.
|
| 190 |
+
"""
|
| 191 |
uname = str(v or "").strip().lower()
|
| 192 |
+
if len(uname) > _UNAME_MAX or not (_UNAME_RE.match(uname) or _UNAME_EMAIL_RE.match(uname)):
|
| 193 |
raise err(400, "bad_username",
|
| 194 |
+
"a username is either 2 to 32 characters of lowercase letters, digits, dot, "
|
| 195 |
+
"dash or underscore, or a full email address of up to 254 characters")
|
| 196 |
return uname
|
| 197 |
|
| 198 |
|
|
|
|
| 317 |
agent = str(body.get("agent") or "").strip() or None
|
| 318 |
email = str(body.get("email") or "").strip() or None
|
| 319 |
|
| 320 |
+
_reg_before = _registry()
|
| 321 |
+
if uname in _reg_before:
|
| 322 |
raise err(409, "user_exists", f"{uname} already exists β PATCH it to change it")
|
| 323 |
+
# ββ WAVE 37 Β· T03 (R8) β THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME.
|
| 324 |
+
# `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only
|
| 325 |
+
# scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was
|
| 326 |
+
# incidental while a username could not contain `@`; now it decides which of two accounts a
|
| 327 |
+
# person reaches, so it is stated in both files rather than left to be re-derived.
|
| 328 |
+
# β BOTH DIRECTIONS, because each one silently takes a login away from an account that has it:
|
| 329 |
+
# (a) a new USERNAME equal to somebody's EMAIL wins the key lookup and shadows their address;
|
| 330 |
+
# (b) a new EMAIL equal to somebody's USERNAME can never be reached, because their key wins.
|
| 331 |
+
# Refused with a named error rather than allowed and explained afterwards: the failure is a
|
| 332 |
+
# person signing in and landing in the wrong account, which nothing downstream can detect.
|
| 333 |
+
if "@" in uname:
|
| 334 |
+
_shadowed = next((k for k, r in _reg_before.items()
|
| 335 |
+
if isinstance(r, dict)
|
| 336 |
+
and str(r.get("email") or "").strip().lower() == uname), None)
|
| 337 |
+
if _shadowed:
|
| 338 |
+
raise err(409, "username_shadows_email",
|
| 339 |
+
f"{uname} is already the email address on the account {_shadowed}, which "
|
| 340 |
+
f"signs in with it today. This account would take that login over. Choose a "
|
| 341 |
+
f"different username, or clear the email address on {_shadowed} first.")
|
| 342 |
+
if email:
|
| 343 |
+
_e = email.strip().lower()
|
| 344 |
+
if _e != uname and _e in _reg_before:
|
| 345 |
+
raise err(409, "email_shadows_username",
|
| 346 |
+
f"{_e} is already a username on this deployment, so anyone signing in with "
|
| 347 |
+
f"it reaches that account and never this one. Choose a different email "
|
| 348 |
+
f"address for this account.")
|
| 349 |
# Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's β
|
| 350 |
# the tenant is stamped from the session, not taken from the body.
|
| 351 |
users.create_user(uname, pw, name, role=role, bus=bus, modules=modules,
|
|
|
|
| 755 |
f"saved with the bad conditions silently removed, which would store a "
|
| 756 |
f"weaker wall than the one on screen.")
|
| 757 |
clean_tree = {"conj": conj, "nodes": cleaned}
|
| 758 |
+
# ββ W38-T19 β `metrics` IS THE FOURTH FIELD OF AN ENTRY, AND ITS DEFAULT IS GRANT.
|
| 759 |
+
# `raw.get("metrics", True)` rather than a required key: a PUT composed by an older
|
| 760 |
+
# client, or a copy of a record written before this ticket, must not read as a
|
| 761 |
+
# revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so
|
| 762 |
+
# the validator and the wall cannot disagree about what an absent key means.
|
| 763 |
out[key] = {"access": bool(raw.get("access", True)),
|
| 764 |
"filter": clean_tree,
|
| 765 |
+
"hiddenFields": sorted({str(h) for h in hidden}),
|
| 766 |
+
"metrics": bool(raw.get("metrics", True))}
|
| 767 |
_refuse_unshapeable_bu(out)
|
| 768 |
return out
|
| 769 |
|
|
|
|
| 1084 |
_default = (perm_scope.nav_may_open(users._public(uname, rec), k)
|
| 1085 |
if any(m.get("surface") and m["key"] == k for m in modules) else
|
| 1086 |
perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
|
| 1087 |
+
# ββ W38-T19 β `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
|
| 1088 |
+
# BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already
|
| 1089 |
+
# reads an absent key as granted, so this line changes no rendering β what it changes is
|
| 1090 |
+
# what a HUMAN reads off the payload while debugging, and what the PUT round-trips. The
|
| 1091 |
+
# stored dict is copied rather than mutated: `rec` is the live registry record and
|
| 1092 |
+
# stamping a key onto it here would write a permission nobody saved.
|
| 1093 |
+
perms_out[k] = (dict(e, metrics=bool(e.get("metrics", True)))
|
| 1094 |
+
if isinstance(e, dict) else
|
| 1095 |
+
{"access": bool(_default), "filter": None, "hiddenFields": [],
|
| 1096 |
+
"metrics": True})
|
| 1097 |
return {"username": uname,
|
| 1098 |
"perms": perms_out,
|
| 1099 |
# S3 ask 2 β the marker rides the GET. An absent module entry means DENY on a
|
api/routes_agent_harness.py
ADDED
|
@@ -0,0 +1,455 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_agent_harness.py β CONTRACT C4: the agent's HARNESS, kept as versioned files.
|
| 2 |
+
|
| 3 |
+
Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file
|
| 4 |
+
pertaining to the agent skills, router etc. So we build a user can build a custom harness for each
|
| 5 |
+
agent through the chat interface."*
|
| 6 |
+
|
| 7 |
+
GET /api/v1/agents/{id}/harness the file LIST (no bodies)
|
| 8 |
+
GET /api/v1/agents/{id}/harness?path=β¦ one file: current body + its versions
|
| 9 |
+
GET /api/v1/agents/{id}/harness?path=β¦&version=N one older body, verbatim
|
| 10 |
+
PUT /api/v1/agents/{id}/harness write a NEW version of one file
|
| 11 |
+
DELETE /api/v1/agents/{id}/harness?path=β¦ drop a file and its history
|
| 12 |
+
|
| 13 |
+
ββ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write
|
| 14 |
+
appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the
|
| 15 |
+
history is a record of what happened rather than of what somebody last wanted it to look like.
|
| 16 |
+
|
| 17 |
+
β **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8,
|
| 18 |
+
`routes_automation`); a harness file is not. Do not copy this file's posture over there or that
|
| 19 |
+
one's over here β the two rulings differ on purpose and they sit one screen apart in the product.
|
| 20 |
+
|
| 21 |
+
β **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every
|
| 22 |
+
write through this router is made BY a signed-in administrator, so `author` is stamped from the
|
| 23 |
+
session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by
|
| 24 |
+
a person or by the agent in the chat β a claim the client is entitled to make, because both are
|
| 25 |
+
permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side
|
| 26 |
+
door the automation engine uses when the agent writes with no session at all; that one stamps the
|
| 27 |
+
agent's own id as the author, which is the only case where `author` is not a username.
|
| 28 |
+
|
| 29 |
+
β **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of
|
| 30 |
+
one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in
|
| 31 |
+
`trimmed` on every payload that mentions the file, so a reader can see that the history is partial
|
| 32 |
+
rather than infer that the file was only ever saved twice. This is the tenant document, which is
|
| 33 |
+
already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is
|
| 34 |
+
a store-sized leak with a UI in front of it.
|
| 35 |
+
"""
|
| 36 |
+
from datetime import datetime, timezone
|
| 37 |
+
|
| 38 |
+
from fastapi import APIRouter, Body, Depends
|
| 39 |
+
|
| 40 |
+
from deps import Session, err, require_session
|
| 41 |
+
|
| 42 |
+
router = APIRouter(prefix="/api/v1")
|
| 43 |
+
|
| 44 |
+
#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides
|
| 45 |
+
#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace β the same rule
|
| 46 |
+
#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off.
|
| 47 |
+
HARNESS_KEY = "agent_harness"
|
| 48 |
+
|
| 49 |
+
#: One body. Generous for a skill or a router file and far under the point where a single write
|
| 50 |
+
#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a
|
| 51 |
+
#: truncation: a silently truncated skill file is a harness that does not do what its text says.
|
| 52 |
+
MAX_BODY_BYTES = 128 * 1024
|
| 53 |
+
|
| 54 |
+
#: Files per agent. A refusal, not a trim β creating the 65th file is a different act from saving
|
| 55 |
+
#: the 101st version of one, and only the second can be a routine consequence of ordinary editing.
|
| 56 |
+
MAX_FILES = 64
|
| 57 |
+
|
| 58 |
+
#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header).
|
| 59 |
+
MAX_VERSIONS = 100
|
| 60 |
+
|
| 61 |
+
MAX_PATH = 200
|
| 62 |
+
|
| 63 |
+
#: What a path may contain. β THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK β
|
| 64 |
+
#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is
|
| 65 |
+
#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a
|
| 66 |
+
#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a
|
| 67 |
+
#: list of forbidden sequences: an allow-list cannot be walked around by a spelling.
|
| 68 |
+
_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/")
|
| 69 |
+
|
| 70 |
+
AUTHOR_KINDS = ("user", "agent")
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _now():
|
| 74 |
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _all(rt):
|
| 78 |
+
"""`{agent_id: {path: record}}` for one tenant. `{}` on any failure β an unreadable bucket must
|
| 79 |
+
degrade to "this agent has no harness files", never to a 500 on the pane that lists them."""
|
| 80 |
+
try:
|
| 81 |
+
found = rt.get(HARNESS_KEY) or {}
|
| 82 |
+
except Exception: # noqa: BLE001
|
| 83 |
+
return {}
|
| 84 |
+
return found if isinstance(found, dict) else {}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _files(rt, agent_id):
|
| 88 |
+
found = _all(rt).get(str(agent_id))
|
| 89 |
+
return found if isinstance(found, dict) else {}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
#: What kind of principal an id names. `None` = this tenant has no agent with that id.
|
| 93 |
+
#:
|
| 94 |
+
#: ββ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY
|
| 95 |
+
#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under
|
| 96 |
+
#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface β
|
| 97 |
+
#: `Shell.tsx` mounts `<Lazily surface="Agents"><AutomationSurface /></Lazily>`, and
|
| 98 |
+
#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds
|
| 99 |
+
#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that
|
| 100 |
+
#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival
|
| 101 |
+
#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on
|
| 102 |
+
#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO
|
| 103 |
+
#: occurrences of Canvas, Properties, Run history or panelTabs.
|
| 104 |
+
#:
|
| 105 |
+
#: β ONE STORE, BOTH PRINCIPALS β never a second bucket keyed by surface. The owner's "agent
|
| 106 |
+
#: skills, router" files in two places is the parallel code path item 13 exists to refuse.
|
| 107 |
+
AGENT_SLACK = "slack"
|
| 108 |
+
AGENT_AUTOMATION = "automation"
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def agent_kind(rt, agent_id):
|
| 112 |
+
"""Which registry holds this id β `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`.
|
| 113 |
+
|
| 114 |
+
β A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at
|
| 115 |
+
read time from a column definition or a connector schedule; they have no stored home, so a
|
| 116 |
+
harness file hung off one is an orphan the moment the column changes. `patch_automation`
|
| 117 |
+
already refuses those ids for the neighbouring reason, and this refuses them by simply not
|
| 118 |
+
finding them β `all_definitions` holds stored automations only.
|
| 119 |
+
"""
|
| 120 |
+
import routes_slack
|
| 121 |
+
aid = str(agent_id or "")
|
| 122 |
+
if isinstance(routes_slack._agents(rt).get(aid), dict):
|
| 123 |
+
return AGENT_SLACK
|
| 124 |
+
import automation_engine as engine
|
| 125 |
+
try:
|
| 126 |
+
known = engine.all_definitions(rt) or {}
|
| 127 |
+
except Exception: # noqa: BLE001
|
| 128 |
+
return None
|
| 129 |
+
return AGENT_AUTOMATION if aid in known else None
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _known_agent(rt, agent_id):
|
| 133 |
+
"""Does this tenant have an agent with this id, in EITHER registry?"""
|
| 134 |
+
return agent_kind(rt, agent_id) is not None
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def agent_wall(session, agent_id):
|
| 138 |
+
"""404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies.
|
| 139 |
+
|
| 140 |
+
ββ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH β it is the refusal to invent a
|
| 141 |
+
THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door
|
| 142 |
+
in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in
|
| 143 |
+
`routes_automation` is `module_gate("automation")`. A harness file is configuration OF the
|
| 144 |
+
agent it hangs off, so it is reached by whoever may already configure that agent. Picking one
|
| 145 |
+
of the two walls for both would either lock the Agents module's own users out of a tab the
|
| 146 |
+
owner asked for, or hand the Slack permission wall to anyone with an automation grant.
|
| 147 |
+
"""
|
| 148 |
+
kind = agent_kind(session.runtime, agent_id)
|
| 149 |
+
if kind is None:
|
| 150 |
+
raise err(404, "no_agent", "there is no agent with that id in this workspace")
|
| 151 |
+
if kind == AGENT_SLACK:
|
| 152 |
+
import core.perms as perms
|
| 153 |
+
if not perms.is_admin(session.user):
|
| 154 |
+
raise err(403, "forbidden", "administrators only")
|
| 155 |
+
else:
|
| 156 |
+
session.require("automation")
|
| 157 |
+
return kind
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def normalize_path(raw):
|
| 161 |
+
"""THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable.
|
| 162 |
+
|
| 163 |
+
ββ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There
|
| 164 |
+
are two ways into this store β the HTTP route (which must answer 400) and `record_version()`
|
| 165 |
+
(which must raise `ValueError`, having no response to put a status into). Written twice, the
|
| 166 |
+
two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one
|
| 167 |
+
copy is invisible because the other still refuses, so nothing goes red and the wall is now
|
| 168 |
+
half there. Written once, a change to the rule is felt at both doors and by the gate.
|
| 169 |
+
"""
|
| 170 |
+
path = str(raw or "").strip().strip("/")
|
| 171 |
+
if not path or len(path) > MAX_PATH:
|
| 172 |
+
return None
|
| 173 |
+
if set(path) - _PATH_OK or ".." in path or "//" in path:
|
| 174 |
+
return None
|
| 175 |
+
return path
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _clean_path(raw):
|
| 179 |
+
"""`normalize_path` at the HTTP door, where a refusal is a 400 with a reason."""
|
| 180 |
+
path = normalize_path(raw)
|
| 181 |
+
if path is None:
|
| 182 |
+
if not str(raw or "").strip().strip("/"):
|
| 183 |
+
raise err(400, "no_path", "a harness file needs a path, for example skills/router.md")
|
| 184 |
+
if len(str(raw)) > MAX_PATH:
|
| 185 |
+
raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters")
|
| 186 |
+
raise err(400, "bad_path",
|
| 187 |
+
"a harness path may use letters, digits, dot, dash, underscore and / only")
|
| 188 |
+
return path
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _blank(path, author, author_kind):
|
| 192 |
+
return {"path": path, "versions": [], "trimmed": 0,
|
| 193 |
+
"created": _now(), "createdBy": author, "createdKind": author_kind}
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _append(record, body, author, author_kind, restored_from=None):
|
| 197 |
+
"""Append ONE version to a file record, in place, and report what was trimmed.
|
| 198 |
+
|
| 199 |
+
The version NUMBER is monotonic and survives trimming β it counts writes, not stored entries.
|
| 200 |
+
A version list whose numbers restart at 1 after a trim would make two different bodies share a
|
| 201 |
+
name, and `restoredFrom` would then point at whichever one happened to be in the window.
|
| 202 |
+
"""
|
| 203 |
+
# β A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback
|
| 204 |
+
# the live document and may run it more than once; appending in place would then stack two
|
| 205 |
+
# copies of the same version into the history on a retry.
|
| 206 |
+
prior = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 207 |
+
last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0)
|
| 208 |
+
entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind,
|
| 209 |
+
"created": _now(), "bytes": len(body.encode("utf-8"))}
|
| 210 |
+
if restored_from:
|
| 211 |
+
entry["restoredFrom"] = int(restored_from)
|
| 212 |
+
versions = [*prior, entry]
|
| 213 |
+
dropped = max(0, len(versions) - MAX_VERSIONS)
|
| 214 |
+
if dropped:
|
| 215 |
+
versions = versions[dropped:]
|
| 216 |
+
record["versions"] = versions
|
| 217 |
+
record["trimmed"] = int(record.get("trimmed") or 0) + dropped
|
| 218 |
+
return entry
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _head(record):
|
| 222 |
+
"""The newest version of a file record, or `None` for a record with no versions at all."""
|
| 223 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 224 |
+
return versions[-1] if versions else None
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _row(record):
|
| 228 |
+
"""One file, as the LIST door reports it: everything except the bodies.
|
| 229 |
+
|
| 230 |
+
β NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would
|
| 231 |
+
ship the whole harness on every pane render; the Harness tab lists first and opens one file
|
| 232 |
+
second, which is exactly the shape this answers.
|
| 233 |
+
"""
|
| 234 |
+
head = _head(record) or {}
|
| 235 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 236 |
+
return {"path": record.get("path") or "",
|
| 237 |
+
"version": int(head.get("version") or 0),
|
| 238 |
+
"bytes": int(head.get("bytes") or 0),
|
| 239 |
+
"author": head.get("author") or record.get("createdBy") or "",
|
| 240 |
+
"authorKind": head.get("authorKind") or record.get("createdKind") or "user",
|
| 241 |
+
"updated": head.get("created") or record.get("created") or "",
|
| 242 |
+
"created": record.get("created") or "",
|
| 243 |
+
"versions": len(versions),
|
| 244 |
+
"trimmed": int(record.get("trimmed") or 0)}
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _version_rows(record):
|
| 248 |
+
"""The history of one file, newest first, WITHOUT the bodies.
|
| 249 |
+
|
| 250 |
+
A body per version is what makes a diff possible, and it is also what makes this payload big:
|
| 251 |
+
100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing
|
| 252 |
+
(`?path=β¦&version=N`), which is two round trips for a diff and none for a history list.
|
| 253 |
+
"""
|
| 254 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 255 |
+
out = []
|
| 256 |
+
for entry in reversed(versions):
|
| 257 |
+
if not isinstance(entry, dict):
|
| 258 |
+
continue
|
| 259 |
+
row = {"version": int(entry.get("version") or 0),
|
| 260 |
+
"author": entry.get("author") or "",
|
| 261 |
+
"authorKind": entry.get("authorKind") or "user",
|
| 262 |
+
"created": entry.get("created") or "",
|
| 263 |
+
"bytes": int(entry.get("bytes") or 0)}
|
| 264 |
+
if entry.get("restoredFrom"):
|
| 265 |
+
row["restoredFrom"] = int(entry["restoredFrom"])
|
| 266 |
+
out.append(row)
|
| 267 |
+
return out
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def _limits():
|
| 271 |
+
"""The caps, IN the payload, so a client can say "this file is full" before a write fails.
|
| 272 |
+
|
| 273 |
+
β A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the
|
| 274 |
+
one cap that acts without refusing; these report the three that refuse.
|
| 275 |
+
"""
|
| 276 |
+
return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES,
|
| 277 |
+
"maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH}
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
# ββ the write door the SERVER uses (no session) ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 281 |
+
def record_version(runtime, agent_id, path, body, author="", author_kind="agent",
|
| 282 |
+
restored_from=None):
|
| 283 |
+
"""Write one version from INSIDE the server β the agent's own half of R8.
|
| 284 |
+
|
| 285 |
+
β THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent"
|
| 286 |
+
true. An agent acting inside an automation run holds no session and no cookie; if its only way
|
| 287 |
+
to write were the HTTP door it would have to borrow a person's identity, and the authorship
|
| 288 |
+
column would then be a record of who was logged in rather than of what wrote the file.
|
| 289 |
+
|
| 290 |
+
Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a
|
| 291 |
+
`ValueError`, because a server-side caller has no HTTP response to put a 404 into.
|
| 292 |
+
"""
|
| 293 |
+
agent_id = str(agent_id or "")
|
| 294 |
+
if not _known_agent(runtime, agent_id):
|
| 295 |
+
raise ValueError(f"no agent {agent_id!r} in this workspace")
|
| 296 |
+
path = normalize_path(path)
|
| 297 |
+
if path is None:
|
| 298 |
+
raise ValueError("that is not an acceptable harness path")
|
| 299 |
+
body = str(body or "")
|
| 300 |
+
if len(body.encode("utf-8")) > MAX_BODY_BYTES:
|
| 301 |
+
raise ValueError("harness body is over the size limit")
|
| 302 |
+
kind = author_kind if author_kind in AUTHOR_KINDS else "agent"
|
| 303 |
+
author = str(author or agent_id)
|
| 304 |
+
# β THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a
|
| 305 |
+
# store `update` callback propagates out of a half-run read-modify-write, and the one thing a
|
| 306 |
+
# refusal must never do is leave the caller unsure whether the write happened.
|
| 307 |
+
existing = _files(runtime, agent_id)
|
| 308 |
+
if path not in existing and len(existing) >= MAX_FILES:
|
| 309 |
+
raise ValueError(f"this agent already has {MAX_FILES} harness files")
|
| 310 |
+
appended = {}
|
| 311 |
+
|
| 312 |
+
def _set(cur):
|
| 313 |
+
cur = dict(cur or {})
|
| 314 |
+
files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
|
| 315 |
+
record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind)
|
| 316 |
+
appended.clear()
|
| 317 |
+
appended.update(_append(record, body, author, kind, restored_from))
|
| 318 |
+
files[path] = record
|
| 319 |
+
cur[agent_id] = files
|
| 320 |
+
return cur
|
| 321 |
+
|
| 322 |
+
runtime.update(HARNESS_KEY, _set, flush="sync")
|
| 323 |
+
return appended
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
# ββ the routes ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 327 |
+
@router.get("/agents/{agent_id}/harness")
|
| 328 |
+
def get_harness(agent_id: str, path: str = "", version: int = 0,
|
| 329 |
+
session: Session = Depends(require_session)):
|
| 330 |
+
"""The list, one file, or one older body β decided by the query string (C4).
|
| 331 |
+
|
| 332 |
+
β ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than
|
| 333 |
+
consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to
|
| 334 |
+
editing a permission than to editing a document.
|
| 335 |
+
"""
|
| 336 |
+
agent_id = str(agent_id or "")
|
| 337 |
+
agent_wall(session, agent_id)
|
| 338 |
+
files = _files(session.runtime, agent_id)
|
| 339 |
+
if not path:
|
| 340 |
+
rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)]
|
| 341 |
+
return {"agent": agent_id, "files": rows, "limits": _limits()}
|
| 342 |
+
|
| 343 |
+
wanted = _clean_path(path)
|
| 344 |
+
record = files.get(wanted)
|
| 345 |
+
if not isinstance(record, dict):
|
| 346 |
+
raise err(404, "no_file", f"this agent has no harness file at {wanted}")
|
| 347 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 348 |
+
if version:
|
| 349 |
+
for entry in versions:
|
| 350 |
+
if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version):
|
| 351 |
+
return {"agent": agent_id, **_row(record), "body": entry.get("body") or "",
|
| 352 |
+
"atVersion": int(version), "history": _version_rows(record),
|
| 353 |
+
"limits": _limits()}
|
| 354 |
+
# β A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked
|
| 355 |
+
# for something that existed and no longer does, and telling it apart from a bad number is
|
| 356 |
+
# the difference between "roll back to v3" failing loudly and failing as if v3 never was.
|
| 357 |
+
trimmed = int(record.get("trimmed") or 0)
|
| 358 |
+
if trimmed and int(version) <= trimmed:
|
| 359 |
+
raise err(410, "version_trimmed",
|
| 360 |
+
f"version {int(version)} is older than the {MAX_VERSIONS} versions kept "
|
| 361 |
+
f"for this file, and its body is gone")
|
| 362 |
+
raise err(404, "no_version", f"this file has no version {int(version)}")
|
| 363 |
+
head = _head(record) or {}
|
| 364 |
+
return {"agent": agent_id, **_row(record), "body": head.get("body") or "",
|
| 365 |
+
"atVersion": int(head.get("version") or 0), "history": _version_rows(record),
|
| 366 |
+
"limits": _limits()}
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
@router.put("/agents/{agent_id}/harness")
|
| 370 |
+
def put_harness(agent_id: str, body: dict = Body(default=None),
|
| 371 |
+
session: Session = Depends(require_session)):
|
| 372 |
+
"""Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4).
|
| 373 |
+
|
| 374 |
+
β THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a
|
| 375 |
+
UI affordance; it is this function refusing to have a code path that replaces a body. A
|
| 376 |
+
roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that
|
| 377 |
+
somebody went back rather than pretending the intervening versions never happened.
|
| 378 |
+
"""
|
| 379 |
+
agent_id = str(agent_id or "")
|
| 380 |
+
agent_wall(session, agent_id)
|
| 381 |
+
body = body if isinstance(body, dict) else {}
|
| 382 |
+
path = _clean_path(body.get("path"))
|
| 383 |
+
text = body.get("body")
|
| 384 |
+
if not isinstance(text, str):
|
| 385 |
+
raise err(400, "no_body", "a harness file needs a body, even an empty one")
|
| 386 |
+
if len(text.encode("utf-8")) > MAX_BODY_BYTES:
|
| 387 |
+
raise err(413, "body_too_long",
|
| 388 |
+
f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger")
|
| 389 |
+
|
| 390 |
+
# β THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a
|
| 391 |
+
# claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the
|
| 392 |
+
# session and is never read off the request. Nothing is bought by forging the first β both
|
| 393 |
+
# principals may write (R8) β and everything would be bought by forging the second.
|
| 394 |
+
kind = str(body.get("authorKind") or "user").strip().lower()
|
| 395 |
+
if kind not in AUTHOR_KINDS:
|
| 396 |
+
raise err(400, "bad_author_kind", "authorKind is either user or agent")
|
| 397 |
+
restored = body.get("restoredFrom")
|
| 398 |
+
try:
|
| 399 |
+
restored = int(restored) if restored else None
|
| 400 |
+
except (TypeError, ValueError):
|
| 401 |
+
raise err(400, "bad_version", "restoredFrom must be a version number")
|
| 402 |
+
|
| 403 |
+
files = _files(session.runtime, agent_id)
|
| 404 |
+
if path not in files and len(files) >= MAX_FILES:
|
| 405 |
+
raise err(409, "too_many_files",
|
| 406 |
+
f"this agent already has {MAX_FILES} harness files; delete one to add another")
|
| 407 |
+
try:
|
| 408 |
+
record_version(session.runtime, agent_id, path, text,
|
| 409 |
+
author=session.uname, author_kind=kind, restored_from=restored)
|
| 410 |
+
except ValueError as exc:
|
| 411 |
+
raise err(400, "refused", str(exc))
|
| 412 |
+
|
| 413 |
+
fresh = _files(session.runtime, agent_id).get(path)
|
| 414 |
+
if not isinstance(fresh, dict) or not _head(fresh):
|
| 415 |
+
# The store took the write and did not record it. A 200 here would tell an administrator
|
| 416 |
+
# their skill file was saved when it was not β the shape `routes_slack` refuses too.
|
| 417 |
+
raise err(503, "store_unavailable", "the harness file was NOT saved")
|
| 418 |
+
head = _head(fresh)
|
| 419 |
+
return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "",
|
| 420 |
+
"atVersion": int(head.get("version") or 0), "history": _version_rows(fresh),
|
| 421 |
+
"limits": _limits()}
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
@router.delete("/agents/{agent_id}/harness")
|
| 425 |
+
def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)):
|
| 426 |
+
"""Drop one harness file AND its history.
|
| 427 |
+
|
| 428 |
+
β THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete β losing
|
| 429 |
+
versions as a side effect of an edit. Deleting a file is a person deciding the file should not
|
| 430 |
+
exist, which is a different act with a different button, and a store with no way to remove a
|
| 431 |
+
file is one where a typo'd path is permanent.
|
| 432 |
+
"""
|
| 433 |
+
agent_id = str(agent_id or "")
|
| 434 |
+
agent_wall(session, agent_id)
|
| 435 |
+
wanted = _clean_path(path)
|
| 436 |
+
if wanted not in _files(session.runtime, agent_id):
|
| 437 |
+
raise err(404, "no_file", f"this agent has no harness file at {wanted}")
|
| 438 |
+
|
| 439 |
+
def _set(cur):
|
| 440 |
+
cur = dict(cur or {})
|
| 441 |
+
files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
|
| 442 |
+
files.pop(wanted, None)
|
| 443 |
+
# An agent with no harness files leaves NO key behind. An empty dict per agent id is how a
|
| 444 |
+
# bucket accumulates a row for every agent anybody ever opened the tab on.
|
| 445 |
+
if files:
|
| 446 |
+
cur[agent_id] = files
|
| 447 |
+
else:
|
| 448 |
+
cur.pop(agent_id, None)
|
| 449 |
+
return cur
|
| 450 |
+
|
| 451 |
+
session.runtime.update(HARNESS_KEY, _set, flush="sync")
|
| 452 |
+
return {"agent": agent_id, "deleted": wanted,
|
| 453 |
+
"files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items())
|
| 454 |
+
if isinstance(rec, dict)],
|
| 455 |
+
"limits": _limits()}
|
api/routes_automation.py
CHANGED
|
@@ -14,6 +14,7 @@ trigger becomes a public one β the same class of mistake as an empty-200 permi
|
|
| 14 |
import os
|
| 15 |
import re
|
| 16 |
import time
|
|
|
|
| 17 |
|
| 18 |
from fastapi import APIRouter, Body, Depends, Header, Request
|
| 19 |
|
|
@@ -53,7 +54,7 @@ MODULE = "automation"
|
|
| 53 |
_GATE = module_gate(MODULE)
|
| 54 |
|
| 55 |
|
| 56 |
-
def _wire(defn, tenant):
|
| 57 |
"""One automation, as the client reads it. `running` is PROCESS state, never store state β
|
| 58 |
see the engine header on why a persisted 'running' is a permanent lock."""
|
| 59 |
live = engine.running(tenant, defn.get("id"))
|
|
@@ -141,6 +142,16 @@ def _wire(defn, tenant):
|
|
| 141 |
# back. What needs the target database's schema (an enrich binding resolved by the profile
|
| 142 |
# FLAG) stays a run-time refusal β see `ACTION_REQUIRED`'s note.
|
| 143 |
"unconfigured": engine.unconfigured_actions(defn),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
}
|
| 145 |
|
| 146 |
|
|
@@ -243,7 +254,7 @@ def _field_agent_rows(session):
|
|
| 243 |
# β the exact way `awaitingResults` shipped inert for a whole wave. Passing the
|
| 244 |
# synthetic DEFINITION through the same function makes them identical by
|
| 245 |
# construction; only `system` is stamped afterwards, because no stored row has it.
|
| 246 |
-
row = _wire(defn_syn, session.tenant)
|
| 247 |
row["system"] = SYSTEM_FIELD_AGENT
|
| 248 |
out.append(row)
|
| 249 |
return out
|
|
@@ -334,7 +345,7 @@ def _odoo_sync_row(session, detail=False):
|
|
| 334 |
"config": {"connector": "odoo", "every": every,
|
| 335 |
"frozen": frozen}}]},
|
| 336 |
}
|
| 337 |
-
row = _wire(defn, session.tenant)
|
| 338 |
row["system"] = SYSTEM_ODOO_SYNC
|
| 339 |
# β THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence`
|
| 340 |
# speaks the automation vocabulary β it builds "When <cron phrase>, run N actions" out of
|
|
@@ -745,7 +756,7 @@ def list_automations(session: Session = Depends(_GATE)):
|
|
| 745 |
pass
|
| 746 |
_STATEMENTS_SEEDED.add(session.tenant)
|
| 747 |
defs = engine.all_definitions(session.runtime)
|
| 748 |
-
items = [_wire(d, session.tenant) for _, d in
|
| 749 |
sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())]
|
| 750 |
# ββ WAVE 34 Β· CONTRACT C3 (W34-T48) β field agents join the list as SYNTHETIC rows.
|
| 751 |
# β MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be
|
|
@@ -1133,6 +1144,17 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
|
|
| 1133 |
prompt = str((body or {}).get("prompt") or "").strip()
|
| 1134 |
if not prompt:
|
| 1135 |
raise err(400, "no_prompt", "type what you want the automation to do")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1136 |
# β THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open`
|
| 1137 |
# per table, so the model is shown exactly the databases this caller may already see and cannot
|
| 1138 |
# name one they were not granted β the permission wall re-used, never a second one built beside
|
|
@@ -1170,6 +1192,7 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
|
|
| 1170 |
# sites, this one has a real person behind it: somebody typed the sentence, so `user` is
|
| 1171 |
# the caller rather than the automation's owner.
|
| 1172 |
st=session.runtime, user=getattr(session, "uname", "") or "",
|
|
|
|
| 1173 |
chat=_DRAFT_CHAT[0])
|
| 1174 |
if refusal or not draft:
|
| 1175 |
raise err(400, "draft_refused", refusal or "no automation could be drafted from that")
|
|
@@ -1263,6 +1286,244 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
|
|
| 1263 |
"provider": provider, "dropped": dropped, "notes": _notes, "saved": False}
|
| 1264 |
|
| 1265 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1266 |
@router.get("/automations/{auto_id}")
|
| 1267 |
def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
| 1268 |
"""One automation, by id.
|
|
@@ -1295,7 +1556,7 @@ def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
|
| 1295 |
defn = engine.all_definitions(session.runtime).get(str(auto_id))
|
| 1296 |
if defn is None:
|
| 1297 |
raise err(404, "unknown_automation", "no automation with that id")
|
| 1298 |
-
return {"automation": _wire(defn, session.tenant)}
|
| 1299 |
|
| 1300 |
|
| 1301 |
@router.post("/automations")
|
|
@@ -1317,7 +1578,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend
|
|
| 1317 |
notes=_notes)
|
| 1318 |
if error:
|
| 1319 |
raise err(400, "invalid_automation", error)
|
| 1320 |
-
return {"automation": _wire(defn, session.tenant), "notes": _notes}
|
| 1321 |
|
| 1322 |
|
| 1323 |
@router.patch("/automations/{auto_id}")
|
|
@@ -1339,6 +1600,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
|
|
| 1339 |
return _patch_odoo_sync(session, body or {})
|
| 1340 |
# D-75, same channel as `create` above β an EDIT is where this matters most, because the
|
| 1341 |
# person has just typed the thing that gets dropped.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1342 |
_notes = []
|
| 1343 |
defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
|
| 1344 |
notes=_notes)
|
|
@@ -1346,7 +1611,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
|
|
| 1346 |
raise err(400 if error != "no such automation" else 404,
|
| 1347 |
"invalid_automation" if error != "no such automation" else "unknown_automation",
|
| 1348 |
error)
|
| 1349 |
-
|
|
|
|
|
|
|
|
|
|
| 1350 |
|
| 1351 |
|
| 1352 |
@router.delete("/automations/{auto_id}")
|
|
@@ -1384,6 +1652,9 @@ def delete_automation(auto_id: str, session: Session = Depends(_GATE)):
|
|
| 1384 |
if _sys:
|
| 1385 |
raise err(409, "system_agent", engine.system_agent_refusal(_sys))
|
| 1386 |
engine.remove(session.runtime, auto_id)
|
|
|
|
|
|
|
|
|
|
| 1387 |
return {"deleted": str(auto_id)}
|
| 1388 |
|
| 1389 |
|
|
@@ -1439,7 +1710,7 @@ def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depend
|
|
| 1439 |
raise err(404 if error == "no such automation" else 400,
|
| 1440 |
"unknown_automation" if error == "no such automation" else "node_not_toggleable",
|
| 1441 |
error)
|
| 1442 |
-
return {"automation": _wire(defn, session.tenant)}
|
| 1443 |
|
| 1444 |
|
| 1445 |
@router.post("/automations/{auto_id}/hook/{token}")
|
|
|
|
| 14 |
import os
|
| 15 |
import re
|
| 16 |
import time
|
| 17 |
+
from datetime import datetime, timezone
|
| 18 |
|
| 19 |
from fastapi import APIRouter, Body, Depends, Header, Request
|
| 20 |
|
|
|
|
| 54 |
_GATE = module_gate(MODULE)
|
| 55 |
|
| 56 |
|
| 57 |
+
def _wire(defn, tenant, rt=None):
|
| 58 |
"""One automation, as the client reads it. `running` is PROCESS state, never store state β
|
| 59 |
see the engine header on why a persisted 'running' is a permanent lock."""
|
| 60 |
live = engine.running(tenant, defn.get("id"))
|
|
|
|
| 142 |
# back. What needs the target database's schema (an enrich binding resolved by the profile
|
| 143 |
# FLAG) stays a run-time refusal β see `ACTION_REQUIRED`'s note.
|
| 144 |
"unconfigured": engine.unconfigured_actions(defn),
|
| 145 |
+
# ββ WAVE 36 Β· W36-T38 (owner item 8 / R9) β WHICH ACTIONS AN AGENT BUILT.
|
| 146 |
+
# `{action_id: {agent, agentName, created, by}}`, empty for an ordinary automation.
|
| 147 |
+
# β THE CONFIG IS NOT HIDDEN AND MUST NOT BE. R9 is a WRITE rule: the client uses this to
|
| 148 |
+
# draw a lock and explain who owns the step, never to withhold what the step does β an
|
| 149 |
+
# automation nobody can audit is worse than one nobody can edit.
|
| 150 |
+
# β `rt=None` YIELDS `{}` RATHER THAN OMITTING THE KEY, so a caller that forgets the
|
| 151 |
+
# runtime produces "nothing is agent-owned" (a visible, wrong-but-safe answer) instead of
|
| 152 |
+
# an absent prop the client silently reads as undefined
|
| 153 |
+
# [[flag-shipped-without-its-writer]].
|
| 154 |
+
"agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {},
|
| 155 |
}
|
| 156 |
|
| 157 |
|
|
|
|
| 254 |
# β the exact way `awaitingResults` shipped inert for a whole wave. Passing the
|
| 255 |
# synthetic DEFINITION through the same function makes them identical by
|
| 256 |
# construction; only `system` is stamped afterwards, because no stored row has it.
|
| 257 |
+
row = _wire(defn_syn, session.tenant, rt=session.runtime)
|
| 258 |
row["system"] = SYSTEM_FIELD_AGENT
|
| 259 |
out.append(row)
|
| 260 |
return out
|
|
|
|
| 345 |
"config": {"connector": "odoo", "every": every,
|
| 346 |
"frozen": frozen}}]},
|
| 347 |
}
|
| 348 |
+
row = _wire(defn, session.tenant, rt=session.runtime)
|
| 349 |
row["system"] = SYSTEM_ODOO_SYNC
|
| 350 |
# β THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence`
|
| 351 |
# speaks the automation vocabulary β it builds "When <cron phrase>, run N actions" out of
|
|
|
|
| 756 |
pass
|
| 757 |
_STATEMENTS_SEEDED.add(session.tenant)
|
| 758 |
defs = engine.all_definitions(session.runtime)
|
| 759 |
+
items = [_wire(d, session.tenant, rt=session.runtime) for _, d in
|
| 760 |
sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())]
|
| 761 |
# ββ WAVE 34 Β· CONTRACT C3 (W34-T48) β field agents join the list as SYNTHETIC rows.
|
| 762 |
# β MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be
|
|
|
|
| 1144 |
prompt = str((body or {}).get("prompt") or "").strip()
|
| 1145 |
if not prompt:
|
| 1146 |
raise err(400, "no_prompt", "type what you want the automation to do")
|
| 1147 |
+
# ββ ASK D-18 (2026-08-18) β THE KEY THE CLIENT SENDS IS NOW READ. This door took `prompt` off
|
| 1148 |
+
# the body and nothing else, so the Agent chat's model toggle (W36-T34) was a control that
|
| 1149 |
+
# configured nothing: the value was accepted and dropped, which is the shape three tickets in
|
| 1150 |
+
# this wave already tripped over. D held T34 rather than mark a picker BUILT while its value
|
| 1151 |
+
# stopped at the browser, and was right to.
|
| 1152 |
+
# β UNKNOWN OR UNCONFIGURED FALLS BACK TO THE LADDER, never refuses (D-18's asked-for posture,
|
| 1153 |
+
# and C5's everywhere else): a model the ladder stops offering must not turn every later draft
|
| 1154 |
+
# into an error. The response already names the rung that ANSWERED, which is the honest half.
|
| 1155 |
+
_model = str((body or {}).get("model") or "").strip().lower()
|
| 1156 |
+
if _model in ("", "auto"):
|
| 1157 |
+
_model = None
|
| 1158 |
# β THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open`
|
| 1159 |
# per table, so the model is shown exactly the databases this caller may already see and cannot
|
| 1160 |
# name one they were not granted β the permission wall re-used, never a second one built beside
|
|
|
|
| 1192 |
# sites, this one has a real person behind it: somebody typed the sentence, so `user` is
|
| 1193 |
# the caller rather than the automation's owner.
|
| 1194 |
st=session.runtime, user=getattr(session, "uname", "") or "",
|
| 1195 |
+
model=_model,
|
| 1196 |
chat=_DRAFT_CHAT[0])
|
| 1197 |
if refusal or not draft:
|
| 1198 |
raise err(400, "draft_refused", refusal or "no automation could be drafted from that")
|
|
|
|
| 1286 |
"provider": provider, "dropped": dropped, "notes": _notes, "saved": False}
|
| 1287 |
|
| 1288 |
|
| 1289 |
+
# ββββββββββββββββββ WAVE 36 Β· W36-T38 (owner item 8, ruling R9) β AGENT-AUTHORED ACTIONS ββββββ
|
| 1290 |
+
#
|
| 1291 |
+
# Owner, verbatim (2026-08-18): *"Create the ability for an automation agent to create any
|
| 1292 |
+
# 'Action' under the Canvas, so its configuration can only be touched by the agent. Be it a tool a
|
| 1293 |
+
# script etc. We need to really guardrail the reach of this script."*
|
| 1294 |
+
#
|
| 1295 |
+
# ββ R9 IS A **WRITE** RULE, NOT A VISIBILITY RULE, and the ticket says so in as many words: the
|
| 1296 |
+
# user MAY read an agent-authored action's configuration and MAY delete it; what they may not do
|
| 1297 |
+
# is hand-edit it. Hiding a config from the person whose workspace it runs in is a different
|
| 1298 |
+
# product, and a worse one β nobody can audit what they cannot see.
|
| 1299 |
+
#
|
| 1300 |
+
# β AND IT IS ENFORCED AT THE DOOR, NEVER IN A `disabled` ATTRIBUTE. `routes_automation` already
|
| 1301 |
+
# carries that lesson for a different control; a client-side lock is a suggestion, and the whole
|
| 1302 |
+
# point of R9 is that the agent's configuration stays coherent with what the agent believes it
|
| 1303 |
+
# built.
|
| 1304 |
+
#
|
| 1305 |
+
# β WHY THE MARK IS A SEPARATE BUCKET RATHER THAN A KEY ON THE ACTION. `clean_actions` builds every
|
| 1306 |
+
# action KEY BY KEY from an allowlist and drops anything it does not recognise (D-75) β so a
|
| 1307 |
+
# marker stored inside the action would be silently erased on the next save, and the wall would
|
| 1308 |
+
# quietly stop existing with every gate still green. This is an ANNOTATION layer keyed by
|
| 1309 |
+
# `(automation id, action id)`; the configuration itself stays where it always was, in the
|
| 1310 |
+
# automation, with exactly one writer.
|
| 1311 |
+
AGENT_ACTIONS_KEY = "automation_agent_actions"
|
| 1312 |
+
|
| 1313 |
+
#: Ids this door mints. β It must match `clean_actions`' `act_[a-z0-9_]{1,32}` or the engine
|
| 1314 |
+
#: re-mints it and the annotation points at an action that no longer carries that id.
|
| 1315 |
+
_AGENT_ACTION_PREFIX = "act_ag"
|
| 1316 |
+
|
| 1317 |
+
|
| 1318 |
+
def _agent_marks(rt):
|
| 1319 |
+
"""`{auto_id: {action_id: mark}}` for one tenant. `{}` on any failure β an unreadable
|
| 1320 |
+
annotation must degrade to "nothing is agent-owned", never to a 500 on the automations rail.
|
| 1321 |
+
|
| 1322 |
+
β AND "DEGRADE TO NOTHING IS OWNED" IS THE SAFE DIRECTION HERE, which is worth stating because
|
| 1323 |
+
it usually is not. The wall this feeds protects the AGENT's coherence, not the tenant's data: a
|
| 1324 |
+
lost mark lets a person edit a config they own anyway, in their own workspace, on a step they
|
| 1325 |
+
could always have deleted outright. A wall that failed CLOSED would instead make an automation
|
| 1326 |
+
permanently unsavable because one annotation read timed out.
|
| 1327 |
+
"""
|
| 1328 |
+
try:
|
| 1329 |
+
found = rt.get(AGENT_ACTIONS_KEY) or {}
|
| 1330 |
+
except Exception: # noqa: BLE001
|
| 1331 |
+
return {}
|
| 1332 |
+
return found if isinstance(found, dict) else {}
|
| 1333 |
+
|
| 1334 |
+
|
| 1335 |
+
def _flat_actions(actions, out=None, depth=0):
|
| 1336 |
+
"""Every action in a flow, INCLUDING the ones nested inside a group's arms.
|
| 1337 |
+
|
| 1338 |
+
β NESTED ACTIONS ARE THE ONES THIS MUST NOT MISS. `unconfigured_actions` walks them for the
|
| 1339 |
+
same reason: a step inside an If/then branch is exactly the one a person cannot see, and a
|
| 1340 |
+
wall that only looked at the top level would leave the agent's own nested step editable.
|
| 1341 |
+
"""
|
| 1342 |
+
out = [] if out is None else out
|
| 1343 |
+
if depth > 6 or not isinstance(actions, list):
|
| 1344 |
+
return out
|
| 1345 |
+
for action in actions:
|
| 1346 |
+
if not isinstance(action, dict):
|
| 1347 |
+
continue
|
| 1348 |
+
out.append(action)
|
| 1349 |
+
for key in ("then", "else", "actions"):
|
| 1350 |
+
_flat_actions(action.get(key), out, depth + 1)
|
| 1351 |
+
return out
|
| 1352 |
+
|
| 1353 |
+
|
| 1354 |
+
def _flow_actions(defn):
|
| 1355 |
+
return _flat_actions(((defn or {}).get("flow") or {}).get("actions") or [])
|
| 1356 |
+
|
| 1357 |
+
|
| 1358 |
+
def _same_action(left, right, rt):
|
| 1359 |
+
"""Do these two actions carry the SAME kind and configuration?
|
| 1360 |
+
|
| 1361 |
+
β COMPARED AFTER `clean_actions`, ON BOTH SIDES, and that is the difference between a wall and
|
| 1362 |
+
a nuisance. The stored action has already been through the cleaner; a client round-trip has
|
| 1363 |
+
not, so it may carry a key order, a blank string or a dropped condition that means nothing.
|
| 1364 |
+
Comparing raw shapes would refuse an edit that changes nothing, and a wall that fires on a
|
| 1365 |
+
no-op is one an operator learns to route around [[one-question-two-normalizers]].
|
| 1366 |
+
"""
|
| 1367 |
+
def _clean(action):
|
| 1368 |
+
cleaned, error = engine.clean_actions([dict(action or {}, id="act_1")], rt=rt)
|
| 1369 |
+
if error or not cleaned:
|
| 1370 |
+
return None
|
| 1371 |
+
one = dict(cleaned[0])
|
| 1372 |
+
one.pop("id", None)
|
| 1373 |
+
return one
|
| 1374 |
+
|
| 1375 |
+
a, b = _clean(left), _clean(right)
|
| 1376 |
+
return a is not None and a == b
|
| 1377 |
+
|
| 1378 |
+
|
| 1379 |
+
def _agent_owned_guard(session, auto_id, body):
|
| 1380 |
+
"""R9 AT THE DOOR: a user may not change an agent-authored action's kind or configuration.
|
| 1381 |
+
|
| 1382 |
+
Three outcomes, and the middle one is the ruling:
|
| 1383 |
+
* the action is ABSENT from the incoming flow -> a DELETE, and R9 allows it
|
| 1384 |
+
* the action is present and CHANGED -> **409**, naming the action and the agent
|
| 1385 |
+
* the action is present and identical -> nothing happens, so an ordinary save of a
|
| 1386 |
+
flow that merely CONTAINS an agent action
|
| 1387 |
+
is not refused
|
| 1388 |
+
"""
|
| 1389 |
+
marks = _agent_marks(session.runtime).get(str(auto_id)) or {}
|
| 1390 |
+
if not isinstance(marks, dict) or not marks:
|
| 1391 |
+
return
|
| 1392 |
+
incoming = (body or {}).get("flow")
|
| 1393 |
+
if not isinstance(incoming, dict) or "actions" not in incoming:
|
| 1394 |
+
return
|
| 1395 |
+
stored = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {}
|
| 1396 |
+
was = {str(a.get("id") or ""): a for a in _flow_actions(stored)}
|
| 1397 |
+
now = {str(a.get("id") or ""): a
|
| 1398 |
+
for a in _flat_actions(incoming.get("actions") or [])}
|
| 1399 |
+
for action_id, mark in marks.items():
|
| 1400 |
+
action_id = str(action_id)
|
| 1401 |
+
if action_id not in now or action_id not in was:
|
| 1402 |
+
continue # removed, or never stored: not an EDIT
|
| 1403 |
+
if _same_action(was[action_id], now[action_id], session.runtime):
|
| 1404 |
+
continue
|
| 1405 |
+
who = str((mark or {}).get("agentName") or (mark or {}).get("agent") or "an agent")
|
| 1406 |
+
raise err(409, "agent_owned",
|
| 1407 |
+
f"this step was built by {who} and only {who} can change how it is set up. "
|
| 1408 |
+
f"You can read it, and you can delete it, but it cannot be edited by hand")
|
| 1409 |
+
|
| 1410 |
+
|
| 1411 |
+
def _prune_marks(session, auto_id):
|
| 1412 |
+
"""Drop annotations whose action is gone, and the whole entry when the automation is.
|
| 1413 |
+
|
| 1414 |
+
β CALLED AFTER EVERY WRITE, because the alternative is an annotation bucket that only ever
|
| 1415 |
+
grows: a mark on a deleted action would keep refusing an id nobody can see, and a mark on a
|
| 1416 |
+
deleted automation would sit in the tenant document forever.
|
| 1417 |
+
"""
|
| 1418 |
+
auto_id = str(auto_id)
|
| 1419 |
+
|
| 1420 |
+
def _set(cur):
|
| 1421 |
+
cur = dict(cur or {})
|
| 1422 |
+
marks = cur.get(auto_id)
|
| 1423 |
+
if not isinstance(marks, dict):
|
| 1424 |
+
cur.pop(auto_id, None)
|
| 1425 |
+
return cur
|
| 1426 |
+
defn = (engine.all_definitions(session.runtime) or {}).get(auto_id)
|
| 1427 |
+
if defn is None:
|
| 1428 |
+
cur.pop(auto_id, None)
|
| 1429 |
+
return cur
|
| 1430 |
+
alive = {str(a.get("id") or "") for a in _flow_actions(defn)}
|
| 1431 |
+
kept = {k: v for k, v in marks.items() if str(k) in alive}
|
| 1432 |
+
if kept:
|
| 1433 |
+
cur[auto_id] = kept
|
| 1434 |
+
else:
|
| 1435 |
+
cur.pop(auto_id, None)
|
| 1436 |
+
return cur
|
| 1437 |
+
|
| 1438 |
+
session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync")
|
| 1439 |
+
|
| 1440 |
+
|
| 1441 |
+
@router.post("/automations/{auto_id}/agent-actions")
|
| 1442 |
+
def agent_author_action(auto_id: str, body: dict = Body(default=None),
|
| 1443 |
+
session: Session = Depends(_GATE)):
|
| 1444 |
+
"""THE AGENT'S DOOR: add or replace one Action under the Canvas, marked agent-owned (R9).
|
| 1445 |
+
|
| 1446 |
+
{agent: "<agent id>", action: {kind, config, when?, id?}}
|
| 1447 |
+
-> {automation, actionId, agent}
|
| 1448 |
+
|
| 1449 |
+
β THE ACTION GOES THROUGH `clean_actions` LIKE EVERY OTHER ONE, and that is what "guardrail
|
| 1450 |
+
the reach of this script" means in code: an agent cannot invent a config key, cannot name a
|
| 1451 |
+
kind that is not in the catalog, and cannot reach a kind this tenant is not entitled to. The
|
| 1452 |
+
agent gets a different WALL on editing, never a wider vocabulary.
|
| 1453 |
+
|
| 1454 |
+
β AND IT CANNOT AUTHOR A KIND THAT IS NOT BUILT. `run_script` is `ready: False` in the
|
| 1455 |
+
catalog, so `clean_actions` refuses it here exactly as it refuses it for a person β see this
|
| 1456 |
+
lane's mailbox for why the script ARM is booked rather than half-built: a per-row script
|
| 1457 |
+
contract and a client card are both missing, and a step that reports success and does nothing
|
| 1458 |
+
is the failure this repo has already paid for.
|
| 1459 |
+
|
| 1460 |
+
β THE AGENT ID IS CHECKED AGAINST THE TENANT'S OWN AGENTS. A mark naming an agent that does
|
| 1461 |
+
not exist would refuse every future edit with a sentence naming nobody.
|
| 1462 |
+
"""
|
| 1463 |
+
import routes_slack # noqa: PLC0415
|
| 1464 |
+
|
| 1465 |
+
body = body if isinstance(body, dict) else {}
|
| 1466 |
+
agent_id = str(body.get("agent") or "").strip()
|
| 1467 |
+
agent = routes_slack._agents(session.runtime).get(agent_id)
|
| 1468 |
+
if not isinstance(agent, dict):
|
| 1469 |
+
raise err(404, "no_agent", "there is no agent with that id in this workspace")
|
| 1470 |
+
action = body.get("action")
|
| 1471 |
+
if not isinstance(action, dict) or not str(action.get("kind") or "").strip():
|
| 1472 |
+
raise err(400, "no_action", "an action needs a kind")
|
| 1473 |
+
|
| 1474 |
+
defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id))
|
| 1475 |
+
if defn is None:
|
| 1476 |
+
raise err(404, "unknown_automation", "no automation with that id")
|
| 1477 |
+
if str(defn.get("system") or "").strip():
|
| 1478 |
+
raise err(409, "system_agent", engine.system_agent_refusal(str(defn["system"])))
|
| 1479 |
+
|
| 1480 |
+
existing = list(((defn.get("flow") or {}).get("actions") or []))
|
| 1481 |
+
taken = {str(a.get("id") or "") for a in _flat_actions(existing)}
|
| 1482 |
+
action_id = str(action.get("id") or "").strip()
|
| 1483 |
+
if not action_id or action_id not in taken:
|
| 1484 |
+
n = 1
|
| 1485 |
+
while f"{_AGENT_ACTION_PREFIX}{n}" in taken:
|
| 1486 |
+
n += 1
|
| 1487 |
+
action_id = f"{_AGENT_ACTION_PREFIX}{n}"
|
| 1488 |
+
fresh = dict(action, id=action_id)
|
| 1489 |
+
|
| 1490 |
+
# β VALIDATED BEFORE ANYTHING IS WRITTEN, so a refusal leaves the automation exactly as it was.
|
| 1491 |
+
checked, error = engine.clean_actions([fresh], rt=session.runtime)
|
| 1492 |
+
if error or not checked:
|
| 1493 |
+
raise err(400, "invalid_action", error or "that action could not be built")
|
| 1494 |
+
|
| 1495 |
+
replaced = False
|
| 1496 |
+
for i, one in enumerate(existing):
|
| 1497 |
+
if isinstance(one, dict) and str(one.get("id") or "") == action_id:
|
| 1498 |
+
existing[i], replaced = fresh, True
|
| 1499 |
+
break
|
| 1500 |
+
if not replaced:
|
| 1501 |
+
existing.append(fresh)
|
| 1502 |
+
|
| 1503 |
+
notes = []
|
| 1504 |
+
updated, error = engine.patch(session.runtime, str(auto_id),
|
| 1505 |
+
{"flow": {**(defn.get("flow") or {}), "actions": existing}},
|
| 1506 |
+
username=session.uname, notes=notes)
|
| 1507 |
+
if error:
|
| 1508 |
+
raise err(400, "invalid_automation", error)
|
| 1509 |
+
|
| 1510 |
+
def _set(cur):
|
| 1511 |
+
cur = dict(cur or {})
|
| 1512 |
+
marks = dict(cur.get(str(auto_id)) or {})
|
| 1513 |
+
marks[action_id] = {"agent": agent_id,
|
| 1514 |
+
"agentName": str(agent.get("label") or agent.get("channelName")
|
| 1515 |
+
or agent_id),
|
| 1516 |
+
"created": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
| 1517 |
+
"by": session.uname}
|
| 1518 |
+
cur[str(auto_id)] = marks
|
| 1519 |
+
return cur
|
| 1520 |
+
|
| 1521 |
+
session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync")
|
| 1522 |
+
_prune_marks(session, auto_id)
|
| 1523 |
+
return {"automation": _wire(updated, session.tenant, rt=session.runtime),
|
| 1524 |
+
"actionId": action_id, "agent": agent_id, "notes": notes}
|
| 1525 |
+
|
| 1526 |
+
|
| 1527 |
@router.get("/automations/{auto_id}")
|
| 1528 |
def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
| 1529 |
"""One automation, by id.
|
|
|
|
| 1556 |
defn = engine.all_definitions(session.runtime).get(str(auto_id))
|
| 1557 |
if defn is None:
|
| 1558 |
raise err(404, "unknown_automation", "no automation with that id")
|
| 1559 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime)}
|
| 1560 |
|
| 1561 |
|
| 1562 |
@router.post("/automations")
|
|
|
|
| 1578 |
notes=_notes)
|
| 1579 |
if error:
|
| 1580 |
raise err(400, "invalid_automation", error)
|
| 1581 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes}
|
| 1582 |
|
| 1583 |
|
| 1584 |
@router.patch("/automations/{auto_id}")
|
|
|
|
| 1600 |
return _patch_odoo_sync(session, body or {})
|
| 1601 |
# D-75, same channel as `create` above β an EDIT is where this matters most, because the
|
| 1602 |
# person has just typed the thing that gets dropped.
|
| 1603 |
+
# ββ W36-T38 / R9 β THE AGENT-OWNED WALL, AT THE DOOR, BEFORE ANY WRITE. A `disabled`
|
| 1604 |
+
# attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
|
| 1605 |
+
# step and permits an ordinary save of a flow that merely contains one.
|
| 1606 |
+
_agent_owned_guard(session, auto_id, body or {})
|
| 1607 |
_notes = []
|
| 1608 |
defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
|
| 1609 |
notes=_notes)
|
|
|
|
| 1611 |
raise err(400 if error != "no such automation" else 404,
|
| 1612 |
"invalid_automation" if error != "no such automation" else "unknown_automation",
|
| 1613 |
error)
|
| 1614 |
+
# An action the user removed takes its annotation with it, or the mark outlives its subject
|
| 1615 |
+
# and refuses an id nobody can see.
|
| 1616 |
+
_prune_marks(session, auto_id)
|
| 1617 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes}
|
| 1618 |
|
| 1619 |
|
| 1620 |
@router.delete("/automations/{auto_id}")
|
|
|
|
| 1652 |
if _sys:
|
| 1653 |
raise err(409, "system_agent", engine.system_agent_refusal(_sys))
|
| 1654 |
engine.remove(session.runtime, auto_id)
|
| 1655 |
+
# R9's other half: the user MAY delete an agent-authored step, and deleting the whole
|
| 1656 |
+
# automation takes its annotations with it rather than stranding them in the document.
|
| 1657 |
+
_prune_marks(session, auto_id)
|
| 1658 |
return {"deleted": str(auto_id)}
|
| 1659 |
|
| 1660 |
|
|
|
|
| 1710 |
raise err(404 if error == "no such automation" else 400,
|
| 1711 |
"unknown_automation" if error == "no such automation" else "node_not_toggleable",
|
| 1712 |
error)
|
| 1713 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime)}
|
| 1714 |
|
| 1715 |
|
| 1716 |
@router.post("/automations/{auto_id}/hook/{token}")
|
api/routes_customers.py
CHANGED
|
@@ -141,6 +141,79 @@ def _measure_err(tag, e):
|
|
| 141 |
pass
|
| 142 |
|
| 143 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 145 |
consume_corrections: bool = True):
|
| 146 |
"""ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
|
|
@@ -183,29 +256,87 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 183 |
# exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer.
|
| 184 |
rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS)
|
| 185 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
ws = grid_events.table_workspace(
|
| 187 |
-
_ctx_for(session, pids), allowed_pids=pids,
|
| 188 |
consume_corrections=consume_corrections)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 190 |
ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
# THE FIELD WALL β a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides
|
| 192 |
# every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so
|
| 193 |
# shipping a dependent formula while withholding its input either leaks the input through
|
| 194 |
# the formula's value or silently computes a wrong one; only removing both is coherent.
|
| 195 |
# Applied AFTER workspace_wire because custom + measure columns are what it must cover.
|
| 196 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
if hidden:
|
| 198 |
fields = [f for f in fields if f.get("key") not in hidden]
|
| 199 |
# The field LIST and the ROW payload are two different wires. Narrowing only the first
|
| 200 |
# would leave the value sitting in the second, where anything can read it.
|
| 201 |
rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
|
| 203 |
today = time.strftime("%Y-%m-%d")
|
| 204 |
stamp = _pool_stamp(rt, team_id, agent)
|
| 205 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
measure_sets = measure_resolve.condition_sets(
|
| 207 |
[v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp,
|
| 208 |
-
rt.mset_memo, on_error=_measure_err)
|
| 209 |
# The derived channel: cohort membership cells + measure column values, ONE dict β the
|
| 210 |
# same read-only channel the embed host hands to rows_from_pool.
|
| 211 |
derived = aios_grid.cohort_cells(lists)
|
|
@@ -220,9 +351,17 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 220 |
# this β a value the host knows per render and the pool has no business storing.
|
| 221 |
for pid in pids:
|
| 222 |
derived.setdefault(pid, {})["partner_id"] = pid
|
| 223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
fields, team_id, pids, today, stamp, rt.measure_memo,
|
| 225 |
-
on_error=_measure_err).items():
|
| 226 |
derived.setdefault(pid, {}).update(cells)
|
| 227 |
|
| 228 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
|
@@ -271,30 +410,48 @@ def _payload(session: Session):
|
|
| 271 |
"pulled_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 272 |
|
| 273 |
|
| 274 |
-
def _ctx_for(session: Session, pids):
|
| 275 |
"""An EventCtx for the READ path β no fallback workspace, so a store outage is a 503 rather
|
| 276 |
-
than a phantom in-memory workspace an API request cannot persist.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
from core import grid_events
|
| 278 |
return grid_events.EventCtx(
|
| 279 |
uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[],
|
| 280 |
-
# C-PERM: the write wall's field half. Computed from the CANONICAL contract
|
| 281 |
-
# `fields=[]` here β the closure only needs the schema, not
|
| 282 |
-
|
|
|
|
| 283 |
admin=session.admin, fallback_ws=None, seen_ids={})
|
| 284 |
|
| 285 |
|
| 286 |
|
| 287 |
-
def _hidden_for(session: Session):
|
| 288 |
"""The fields this session's permissions hide β the write wall's half of C-PERM.
|
| 289 |
|
| 290 |
Read paths strip these from both wires so they cannot be SEEN; this is what stops them
|
| 291 |
being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the
|
| 292 |
same schema `routes_admin._clean_perms` validates a hiddenFields entry against.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 293 |
"""
|
| 294 |
import aios_grid
|
| 295 |
import core.perm_scope as perm_scope
|
| 296 |
|
| 297 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
|
| 299 |
def allowed_pids(session: Session):
|
| 300 |
"""The pids this session may touch β the POOL's own ids, so the write wall and the read scope
|
|
@@ -374,3 +531,304 @@ def patch_customer(pid: int, body: dict = Body(default=None),
|
|
| 374 |
if refused:
|
| 375 |
out["refused"] = refused
|
| 376 |
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
pass
|
| 142 |
|
| 143 |
|
| 144 |
+
# ββββββββββββββββββββββββββ THE TENANT-WIDE STRATUM, ON THE CUSTOMER TOPIC (W38-T20 / D-425) ββ
|
| 145 |
+
#
|
| 146 |
+
# ββ WHY THIS FILE GREW A SHARED STRATUM AT ALL. `core/shared_overlay.py` has been generic since
|
| 147 |
+
# W29-T62, `routes_products.py` has merged it since W30-T36 and `routes_tables.py` since W38-T16 β
|
| 148 |
+
# and the CUSTOMER topic had neither a write door nor a read merge. Every user-created column here
|
| 149 |
+
# lives in `data[username]`, so two accounts looking at "the same" column are looking at two
|
| 150 |
+
# columns. That is fine for a private note and fatal for a ROUTE ORDER: a visit sequence one rep
|
| 151 |
+
# can see and their colleague cannot is not a plan, it is a rumour.
|
| 152 |
+
#
|
| 153 |
+
# β ONE SPELLING OF THE BUCKET. `modules.customer_data.TABLE_KEY` is the per-user workspace key
|
| 154 |
+
# and `shared_overlay.bucket()` derives `<key>__shared` from it. Resolving it here rather than
|
| 155 |
+
# writing the string means the write door and the read merge cannot disagree about where the
|
| 156 |
+
# values live β which is exactly the failure T16 found on the materialised `ut_*` tables, where
|
| 157 |
+
# `patch_shared_cell` wrote into a bucket no reader ever opened.
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _shared_key():
|
| 161 |
+
"""The store key this topic's per-user AND tenant-wide strata are both named from."""
|
| 162 |
+
import modules.customer_data as cl
|
| 163 |
+
return cl.TABLE_KEY
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def shared_fields(st=None):
|
| 167 |
+
"""`{field_key: Field}` β the columns this topic shares tenant-wide.
|
| 168 |
+
|
| 169 |
+
Unscoped on purpose, exactly as `shared_overlay.fields` is: a shared column's EXISTENCE is
|
| 170 |
+
tenant-wide by definition. WHO MAY SEE IT is a separate question, answered one layer up by
|
| 171 |
+
`perm_scope.hidden_keys` (the per-field grant wall T16 landed), and WHOSE ROWS by `cells`.
|
| 172 |
+
"""
|
| 173 |
+
from core import shared_overlay
|
| 174 |
+
try:
|
| 175 |
+
return shared_overlay.fields(_shared_key(), st=st)
|
| 176 |
+
except Exception: # noqa: BLE001
|
| 177 |
+
# Lenient like every other display read: an unreachable store degrades to "nothing is
|
| 178 |
+
# shared yet", never to a 500 on a grid that would otherwise render. The WALL does not
|
| 179 |
+
# degrade with it β `field_grant_hidden` hides a marked column it cannot resolve.
|
| 180 |
+
return {}
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def shared_cells(pids, st=None):
|
| 184 |
+
"""`{"<pid>": {key: value}}` for the rows named by `pids`, and ONLY those.
|
| 185 |
+
|
| 186 |
+
β `pids` IS THE ROW WALL, PASSED AND NEVER DEFAULTED. `shared_overlay.cells` refuses an
|
| 187 |
+
"everything" read by signature for this reason; the set handed in is the one `grid_assembly`
|
| 188 |
+
has already narrowed with `apply_row_scope`, so a cell belonging to the other BU has nothing
|
| 189 |
+
to attach itself to.
|
| 190 |
+
"""
|
| 191 |
+
from core import shared_overlay
|
| 192 |
+
try:
|
| 193 |
+
return shared_overlay.cells(_shared_key(), list(pids or ()), st=st)
|
| 194 |
+
except Exception: # noqa: BLE001
|
| 195 |
+
return {}
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _merge_shared_fields(fields, defs):
|
| 199 |
+
"""`fields` PLUS the tenant-wide columns this topic declares β `routes_tables._ut_shared_fields`
|
| 200 |
+
on the customer topic.
|
| 201 |
+
|
| 202 |
+
β MERGED BEFORE THE WALL, NEVER AFTER. `hidden_keys` is a TRANSITIVE closure, so it must run
|
| 203 |
+
on the WHOLE contract: a formula over a shared column that reads a hidden one sits outside the
|
| 204 |
+
closure's reach otherwise and carries the hidden value out wearing a second name. It is also
|
| 205 |
+
the only order in which `field_grant_hidden` can ever see the `granted` marker at all β merge
|
| 206 |
+
afterwards and the per-field wall is inert while every test still passes.
|
| 207 |
+
β A key the canonical contract already declares WINS. A shared column is an ADDITION to this
|
| 208 |
+
database's contract, never a redefinition of a column it already has.
|
| 209 |
+
"""
|
| 210 |
+
if not defs:
|
| 211 |
+
return fields
|
| 212 |
+
have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)}
|
| 213 |
+
return list(fields or ()) + [dict(f, source="overlay")
|
| 214 |
+
for k, f in defs.items() if k not in have]
|
| 215 |
+
|
| 216 |
+
|
| 217 |
def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 218 |
consume_corrections: bool = True):
|
| 219 |
"""ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
|
|
|
|
| 256 |
# exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer.
|
| 257 |
rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS)
|
| 258 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
| 259 |
+
# β W38-T20 β THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because
|
| 260 |
+
# `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this
|
| 261 |
+
# dict on one request (the write ctx's wall, the read merge, and the closure), and letting each
|
| 262 |
+
# take its own copy is the shape D-214 spent a whole ticket removing one document over.
|
| 263 |
+
_defs = shared_fields(st=rt)
|
| 264 |
ws = grid_events.table_workspace(
|
| 265 |
+
_ctx_for(session, pids, defs=_defs), allowed_pids=pids,
|
| 266 |
consume_corrections=consume_corrections)
|
| 267 |
+
# ββ W38-T20 / D-425 β THE TENANT-WIDE CELLS, LAYERED OVER THE PER-USER ONES, IN THE
|
| 268 |
+
# ASSEMBLY SO EVERY CONSUMER SEES ONE TRUTH. `routes_grid`'s /workspace route serves
|
| 269 |
+
# `workspace["overlays"] = g["ws"].get("overlays")` verbatim and `_payload` hands the same
|
| 270 |
+
# dict to `rows_from_pool`, so merging HERE reaches both without touching either file.
|
| 271 |
+
#
|
| 272 |
+
# β SAFE TO MUTATE, and checked rather than assumed (the same check `product_assembly`
|
| 273 |
+
# records): `table_workspace` reads through `store.get`, which deep-copies, so `ws` is a
|
| 274 |
+
# detached copy and nothing writes it back. A shared value can never leak INTO the per-user
|
| 275 |
+
# bucket by way of this merge.
|
| 276 |
+
# β SHARED WINS PER KEY. The whole point of the stratum is that every reader sees the same
|
| 277 |
+
# number, so a per-user leftover under the same key is stale by construction. It is also what
|
| 278 |
+
# makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not
|
| 279 |
+
# promoted.
|
| 280 |
+
_shared = shared_cells(pids, st=rt)
|
| 281 |
+
if _shared:
|
| 282 |
+
_ov = dict(ws.get("overlays") or {})
|
| 283 |
+
for _pid, _cells in _shared.items():
|
| 284 |
+
_ov[_pid] = {**(_ov.get(_pid) or {}), **_cells}
|
| 285 |
+
ws["overlays"] = _ov
|
| 286 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 287 |
ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key)
|
| 288 |
+
# ββ W38-T20 β AND THE COLUMN DEFINITIONS, BEFORE THE WALL. See `_merge_shared_fields`: this
|
| 289 |
+
# position is load-bearing twice, once for the transitive closure and once because it is the
|
| 290 |
+
# only order in which the per-field grant marker is ever presented to `hidden_keys`.
|
| 291 |
+
fields = _merge_shared_fields(fields, _defs)
|
| 292 |
# THE FIELD WALL β a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides
|
| 293 |
# every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so
|
| 294 |
# shipping a dependent formula while withholding its input either leaks the input through
|
| 295 |
# the formula's value or silently computes a wrong one; only removing both is coherent.
|
| 296 |
# Applied AFTER workspace_wire because custom + measure columns are what it must cover.
|
| 297 |
+
# β W38-T20 β `st=rt` IS NOT TIDINESS. `perm_scope.field_grant_hidden` resolves a marked
|
| 298 |
+
# column against `object_shares`, and without a tenant handle it reads the module-default
|
| 299 |
+
# bucket: on any tenant but #0 that finds no grant, and no grant on a MARKED column means
|
| 300 |
+
# HIDDEN. So an unthreaded `st` UNDER-shares (a grantee cannot see their own column) rather
|
| 301 |
+
# than over-shares β visible and reportable, but still wrong, and `visible_fields`' own note
|
| 302 |
+
# requires the two calls to agree about it.
|
| 303 |
+
hidden = perm_scope.hidden_keys(session.user, MODULE, fields, st=rt)
|
| 304 |
if hidden:
|
| 305 |
fields = [f for f in fields if f.get("key") not in hidden]
|
| 306 |
# The field LIST and the ROW payload are two different wires. Narrowing only the first
|
| 307 |
# would leave the value sitting in the second, where anything can read it.
|
| 308 |
rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
|
| 309 |
+
# ββ AND THE OVERLAY DICT IS A THIRD WIRE, WHICH IS NEW THIS TICKET AND WAS A HOLE
|
| 310 |
+
# BEFORE IT. `rows_from_pool` iterates `fields`, so a narrowed contract already keeps a
|
| 311 |
+
# hidden key off a ROW β but `/workspace` serves `ws["overlays"]` RAW, and that dict is
|
| 312 |
+
# where both the per-user cells and (as of the merge above) the tenant-wide ones sit.
|
| 313 |
+
# One narrowing cannot speak for a wire it never touches; `routes_tables` writes the same
|
| 314 |
+
# sentence over its own `shared_cells`, and `routes_odoo_tables` over its `overlays`.
|
| 315 |
+
# β `ws` AND NOT `workspace`: `routes_grid.workspace` copies the dict ACROSS
|
| 316 |
+
# (`workspace["overlays"] = g["ws"].get("overlays")`) after this returns, so narrowing the
|
| 317 |
+
# source is what reaches the wire. Narrowing the copy would be narrowing a key that gets
|
| 318 |
+
# overwritten a moment later.
|
| 319 |
+
_ov = ws.get("overlays") or {}
|
| 320 |
+
if _ov:
|
| 321 |
+
ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden}
|
| 322 |
+
for pid, cells in _ov.items()}
|
| 323 |
|
| 324 |
today = time.strftime("%Y-%m-%d")
|
| 325 |
stamp = _pool_stamp(rt, team_id, agent)
|
| 326 |
+
# ββ W38-T19 β THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE
|
| 327 |
+
# OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their
|
| 328 |
+
# condition answers through helpers that short-circuit on an empty `offer`, so emptying the
|
| 329 |
+
# offer there stops the whole feature. `core.measure_resolve` does not take an offer at all:
|
| 330 |
+
# `condition_sets` and `column_values` both re-derive their work from the caller's OWN saved
|
| 331 |
+
# views and field list. So gating only the offer here would take the Metric kind off the
|
| 332 |
+
# picker and refuse new creates while an EXISTING Metric column kept computing and an
|
| 333 |
+
# EXISTING measure condition kept resolving β a revoked capability still answering, on the
|
| 334 |
+
# grain with the most of them. Three calls, one predicate.
|
| 335 |
+
may_metrics = perm_scope.may_metrics(session.user, MODULE)
|
| 336 |
+
measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else []
|
| 337 |
measure_sets = measure_resolve.condition_sets(
|
| 338 |
[v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp,
|
| 339 |
+
rt.mset_memo, on_error=_measure_err) if may_metrics else {}
|
| 340 |
# The derived channel: cohort membership cells + measure column values, ONE dict β the
|
| 341 |
# same read-only channel the embed host hands to rows_from_pool.
|
| 342 |
derived = aios_grid.cohort_cells(lists)
|
|
|
|
| 351 |
# this β a value the host knows per render and the pool has no business storing.
|
| 352 |
for pid in pids:
|
| 353 |
derived.setdefault(pid, {})["partner_id"] = pid
|
| 354 |
+
# ββ W38-T19 β SKIPPED ENTIRELY WHEN THE CAPABILITY IS REVOKED, rather than filtered after.
|
| 355 |
+
# `column_values` selects its own subjects (`isinstance(f.get('measure'), dict)`) off the
|
| 356 |
+
# field list, so there is no argument that could narrow it; not calling it is the narrowing.
|
| 357 |
+
# β THE COLUMN STAYS AND ITS CELLS GO BLANK, which is this channel's OWN documented degrade
|
| 358 |
+
# ("blank is could not compute, 0 is a real zero") and is what the product and user-table
|
| 359 |
+
# grains already do under an empty offer. Deleting the column instead would be a second,
|
| 360 |
+
# louder behaviour for the same fact on one surface out of three, and it would destroy a
|
| 361 |
+
# definition the admin can restore with one tick.
|
| 362 |
+
for pid, cells in (measure_resolve.column_values(
|
| 363 |
fields, team_id, pids, today, stamp, rt.measure_memo,
|
| 364 |
+
on_error=_measure_err) if may_metrics else {}).items():
|
| 365 |
derived.setdefault(pid, {}).update(cells)
|
| 366 |
|
| 367 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
|
|
|
| 410 |
"pulled_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 411 |
|
| 412 |
|
| 413 |
+
def _ctx_for(session: Session, pids, defs=None):
|
| 414 |
"""An EventCtx for the READ path β no fallback workspace, so a store outage is a 503 rather
|
| 415 |
+
than a phantom in-memory workspace an API request cannot persist.
|
| 416 |
+
|
| 417 |
+
`defs` (W38-T20) is this topic's tenant-wide column definitions when the caller already holds
|
| 418 |
+
them; absent, they are read. One read per assembly rather than one per question asked of it.
|
| 419 |
+
"""
|
| 420 |
from core import grid_events
|
| 421 |
return grid_events.EventCtx(
|
| 422 |
uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[],
|
| 423 |
+
# C-PERM: the write wall's field half. Computed from the CANONICAL contract PLUS the
|
| 424 |
+
# tenant-wide columns, because `fields=[]` here β the closure only needs the schema, not
|
| 425 |
+
# this user's column list, and a runtime column is part of that schema now.
|
| 426 |
+
hidden_keys=_hidden_for(session, defs=defs),
|
| 427 |
admin=session.admin, fallback_ws=None, seen_ids={})
|
| 428 |
|
| 429 |
|
| 430 |
|
| 431 |
+
def _hidden_for(session: Session, defs=None):
|
| 432 |
"""The fields this session's permissions hide β the write wall's half of C-PERM.
|
| 433 |
|
| 434 |
Read paths strip these from both wires so they cannot be SEEN; this is what stops them
|
| 435 |
being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the
|
| 436 |
same schema `routes_admin._clean_perms` validates a hiddenFields entry against.
|
| 437 |
+
|
| 438 |
+
ββ W38-T20 β AND THE TENANT-WIDE COLUMNS ARE PART OF THAT CONTRACT NOW, WHICH IS A WALL AND
|
| 439 |
+
NOT A COMPLETENESS TIDY-UP. `patch_customer` builds its ctx with `fields=payload["fields"]`,
|
| 440 |
+
which carries the merged shared columns, so `grid_events.handle_one` would happily accept an
|
| 441 |
+
`overlay_patch` naming one. The canonical list cannot mention them (they are created at
|
| 442 |
+
runtime), so a wall computed from `aios_grid.FIELDS` alone answers "not hidden" for every
|
| 443 |
+
grant-governed column and the write door is open to a reader who was never granted it.
|
| 444 |
+
β Widening the field list can only ever ADD to the hidden set, never remove from it: the
|
| 445 |
+
closure hides what it is told to hide plus whatever depends on it.
|
| 446 |
"""
|
| 447 |
import aios_grid
|
| 448 |
import core.perm_scope as perm_scope
|
| 449 |
|
| 450 |
+
if defs is None:
|
| 451 |
+
defs = shared_fields(st=session.runtime)
|
| 452 |
+
return perm_scope.hidden_keys(
|
| 453 |
+
session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), defs),
|
| 454 |
+
st=session.runtime)
|
| 455 |
|
| 456 |
def allowed_pids(session: Session):
|
| 457 |
"""The pids this session may touch β the POOL's own ids, so the write wall and the read scope
|
|
|
|
| 531 |
if refused:
|
| 532 |
out["refused"] = refused
|
| 533 |
return out
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
# ββββββββββββββββββββββββββββββββββ THE ROUTE-ORDER COLUMN (W38-T20 / ruling R7 / contract C1) ββ
|
| 537 |
+
#
|
| 538 |
+
# ββ WHY THIS IS A DOOR HERE AND NOT A KIND IN THE COLUMN MENU. `ColumnMenu.onCreate` is the only
|
| 539 |
+
# persistence the menu has, and it lands a PER-USER `custom_` column through
|
| 540 |
+
# `aios_grid._field_extras`, which is a strict allowlist: a route-order bag created that way is
|
| 541 |
+
# silently stripped and its values are private to their author. That is the wave-19 `image`
|
| 542 |
+
# failure verbatim ("created, named, configured, gone", recorded in `_clean_geocode`'s own
|
| 543 |
+
# docstring) plus a done-when clause that cannot hold, because a colleague reading a per-user
|
| 544 |
+
# stratum gets a clean 200 with nothing in it. The `geocode` pseudo-kind is the precedent for the
|
| 545 |
+
# PICKER; its persistence half does not transfer.
|
| 546 |
+
#
|
| 547 |
+
# β SO THE COLUMN IS BORN SHARED. `shared_overlay.put_field` stores the definition verbatim (no
|
| 548 |
+
# allowlist), which is what lets the input fingerprint ride the DEFINITION rather than the rows β
|
| 549 |
+
# `shared_overlay._value` RAISES on a dict, so `{order, inputsHash}` could never be one cell, and
|
| 550 |
+
# one solve fingerprints the whole cohort identically anyway, so per-row would be the same string
|
| 551 |
+
# written N times.
|
| 552 |
+
#
|
| 553 |
+
# β AND THE NUMBER ON THE RECORD IS THE INVERSE OF THE PLANNER'S ANSWER. `mapProjection.planRoute`
|
| 554 |
+
# returns `order`, where `order[i]` is WHICH STOP is visited i-th; the cell holds the RANK. The
|
| 555 |
+
# client inverts it with `routeRanks` (gated in `map.test.ts` at the desktop shape, over a fixture
|
| 556 |
+
# chosen so the two differ); this door then refuses anything that is not a clean 1..N, so a
|
| 557 |
+
# truncated or double-posted body cannot land as a half-route.
|
| 558 |
+
|
| 559 |
+
#: The key prefix every route-order column wears. `custom_` and `measure_` are the two existing
|
| 560 |
+
#: created-column namespaces and both are PER USER; this one is tenant-wide, so it takes its own
|
| 561 |
+
#: rather than borrowing a prefix whose readers assume a per-user home.
|
| 562 |
+
ROUTE_KEY_PREFIX = "route_"
|
| 563 |
+
|
| 564 |
+
#: The marker on the stored definition that says WHAT this column is. Read by the GET below and by
|
| 565 |
+
#: the client; never inferred from the key, because a prefix is a naming convention and a
|
| 566 |
+
#: convention is not a declaration.
|
| 567 |
+
ROUTE_KIND = "route_order"
|
| 568 |
+
|
| 569 |
+
#: ββ THE TOPIC A FIELD GRANT IS NAMED UNDER, AND IT IS **NOT** THE STORE BUCKET. Two namespaces
|
| 570 |
+
#: meet on this column and they are spelled differently:
|
| 571 |
+
#:
|
| 572 |
+
#: the STRATUM lives at `shared_overlay.bucket(customer_data.TABLE_KEY)`
|
| 573 |
+
#: = `customer_table_workspace__shared`
|
| 574 |
+
#: the GRANT is named `shares.field_oid(<what hidden_keys was passed>, key)`
|
| 575 |
+
#: = `customer_data:<key>`
|
| 576 |
+
#:
|
| 577 |
+
#: `perm_scope.hidden_keys(user, module, fields)` hands its `module` argument straight through to
|
| 578 |
+
#: `field_grant_hidden`, which builds the oid from it. On a `ut_*` database the module argument IS
|
| 579 |
+
#: the table key, so W38-T16 never had to tell them apart; on a REGISTRY topic they differ, and a
|
| 580 |
+
#: door that claims the grant under the bucket name writes a record the wall will never look for.
|
| 581 |
+
#: MEASURED, not reasoned: the first run of this ticket's gate did exactly that, and user B was
|
| 582 |
+
#: refused a column that had been shared with them through the real share door, with a 200 at
|
| 583 |
+
#: every step ([[one-question-two-normalizers]]).
|
| 584 |
+
SHARE_TOPIC = MODULE
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def _route_slug(label):
|
| 588 |
+
"""A stable store key from a human label. Lower case, non-alphanumerics collapsed to `_`."""
|
| 589 |
+
import re as _re
|
| 590 |
+
slug = _re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_")
|
| 591 |
+
return f"{ROUTE_KEY_PREFIX}{slug[:48]}" if slug else ""
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
def _route_defs(session: Session):
|
| 595 |
+
"""This topic's route-order columns MINUS the ones this session was not granted.
|
| 596 |
+
|
| 597 |
+
β THE WALL IS THE SAME ONE THE GRID USES, NOT A SECOND OPINION. `hidden_keys` is where T16
|
| 598 |
+
put the per-field grant check, so filtering on it here means the listing, the grid contract
|
| 599 |
+
and the row payload agree by construction. A column a reader cannot see must not appear here
|
| 600 |
+
either: the done-when says *does not see the field at all*, and a picker that names a column
|
| 601 |
+
whose values are withheld has already leaked its existence.
|
| 602 |
+
"""
|
| 603 |
+
import aios_grid
|
| 604 |
+
import core.perm_scope as perm_scope
|
| 605 |
+
|
| 606 |
+
all_defs = shared_fields(st=session.runtime) or {}
|
| 607 |
+
defs = {k: v for k, v in all_defs.items()
|
| 608 |
+
if isinstance(v, dict) and v.get("kind") == ROUTE_KIND}
|
| 609 |
+
if not defs:
|
| 610 |
+
return {}
|
| 611 |
+
hide = perm_scope.hidden_keys(
|
| 612 |
+
session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), all_defs),
|
| 613 |
+
st=session.runtime)
|
| 614 |
+
return {k: v for k, v in defs.items() if k not in hide}
|
| 615 |
+
|
| 616 |
+
|
| 617 |
+
@router.get("/customers/route-order")
|
| 618 |
+
def route_order_list(session: Session = Depends(module_gate(MODULE))):
|
| 619 |
+
"""The route-order columns this session may see, with the fingerprint each was solved from.
|
| 620 |
+
|
| 621 |
+
ββ THE FINGERPRINT IS WHY THIS ROUTE EXISTS RATHER THAN THE CLIENT READING `fields`.
|
| 622 |
+
Staleness is DERIVED, never stored: what is written down is the INPUT FINGERPRINT the numbers
|
| 623 |
+
were produced from, so *is this order still current* is a question asked at READ time against
|
| 624 |
+
what is on screen NOW, and can never itself be out of date. A stored `stale: true` is a fact
|
| 625 |
+
about a moment that has already passed.
|
| 626 |
+
"""
|
| 627 |
+
out = []
|
| 628 |
+
for key, defn in sorted(_route_defs(session).items()):
|
| 629 |
+
route = defn.get("route") if isinstance(defn.get("route"), dict) else {}
|
| 630 |
+
out.append({
|
| 631 |
+
"key": key,
|
| 632 |
+
"label": defn.get("label") or key,
|
| 633 |
+
"inputsHash": str(route.get("inputsHash") or ""),
|
| 634 |
+
"roundTrip": bool(route.get("roundTrip")),
|
| 635 |
+
"startPid": route.get("startPid"),
|
| 636 |
+
"stops": route.get("stops"),
|
| 637 |
+
"solvedAt": route.get("solvedAt") or "",
|
| 638 |
+
"solvedBy": defn.get("createdBy") or "",
|
| 639 |
+
# Who may re-solve it. The same creator-or-admin wall the write door enforces, said on
|
| 640 |
+
# the way out so the client can grey the control instead of discovering a 403.
|
| 641 |
+
"mine": bool(session.admin
|
| 642 |
+
or str(defn.get("createdBy") or "") == session.uname),
|
| 643 |
+
})
|
| 644 |
+
return {"fields": out}
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
@router.post("/customers/route-order")
|
| 648 |
+
def route_order_write(body: dict = Body(default=None),
|
| 649 |
+
session: Session = Depends(module_gate(MODULE))):
|
| 650 |
+
"""Create (or re-solve) a tenant-wide route-order column and fill it, in ONE call.
|
| 651 |
+
|
| 652 |
+
`{label, field?, ranks: {"<pid>": <int>}, inputsHash, roundTrip?, startPid?, stops?}`
|
| 653 |
+
|
| 654 |
+
β THE DEFINITION IS WRITTEN FIRST AND THE GRANT CLAIMED SECOND, which is `patch_shared_cell`'s
|
| 655 |
+
order and it is deliberate: the window where a column is MARKED and UNCLAIMED fails CLOSED
|
| 656 |
+
(governed, nobody granted, so only an admin and the creator see it, and an admin can share
|
| 657 |
+
it). The other order would leave a grant record pointing at nothing.
|
| 658 |
+
|
| 659 |
+
β RE-SOLVING IS CREATOR-OR-ADMIN, WHICH CREATING IS NOT. Writing these numbers changes what
|
| 660 |
+
every account in the workspace reads, at once, for the whole cohort. That is the wall
|
| 661 |
+
`routes_tables.delete_shared_field` already applies to the destructive half of this stratum,
|
| 662 |
+
and a new door does not get to inherit the loose half of an asymmetry somebody has flagged.
|
| 663 |
+
A permitted teammate READS the numbers; they do not silently re-plan somebody's day.
|
| 664 |
+
"""
|
| 665 |
+
from core import shared_overlay
|
| 666 |
+
import core.perm_scope as perm_scope
|
| 667 |
+
from routes_grid import MAX_BULK_ROWS
|
| 668 |
+
|
| 669 |
+
body = body if isinstance(body, dict) else {}
|
| 670 |
+
label = " ".join(str(body.get("label") or "").split())[:120]
|
| 671 |
+
key = str(body.get("field") or "").strip() or _route_slug(label)
|
| 672 |
+
if not key:
|
| 673 |
+
raise err(400, "bad_request", "a name is required for the route order column")
|
| 674 |
+
if not key.startswith(ROUTE_KEY_PREFIX):
|
| 675 |
+
raise err(400, "bad_field_key",
|
| 676 |
+
f"a route order column's key starts with '{ROUTE_KEY_PREFIX}', so it cannot "
|
| 677 |
+
f"collide with a column this database already owns")
|
| 678 |
+
import aios_grid
|
| 679 |
+
if key in {f.get("key") for f in aios_grid.FIELDS}:
|
| 680 |
+
raise err(400, "field_key_taken",
|
| 681 |
+
"this database already has a column with that key")
|
| 682 |
+
|
| 683 |
+
defs = shared_fields(st=session.runtime) or {}
|
| 684 |
+
existing = defs.get(key) if isinstance(defs.get(key), dict) else None
|
| 685 |
+
if existing is not None:
|
| 686 |
+
# ββ A REFUSAL MUST NOT DESCRIBE A COLUMN THE CALLER CANNOT SEE. The two refusals below
|
| 687 |
+
# name the column's KIND and its CREATOR, which is exactly the information the field wall
|
| 688 |
+
# exists to withhold: a stranger who guesses the key would otherwise learn that a route
|
| 689 |
+
# order exists on this database and who planned it. `routes_shares._can_see_object` makes
|
| 690 |
+
# the same choice for the same reason (a non-grantee gets the answer a non-existent id
|
| 691 |
+
# gets). The name is already taken either way, so the honest refusal says only that.
|
| 692 |
+
if key in perm_scope.hidden_keys(
|
| 693 |
+
session.user, MODULE,
|
| 694 |
+
_merge_shared_fields(list(aios_grid.FIELDS), defs), st=session.runtime):
|
| 695 |
+
raise err(400, "field_key_taken",
|
| 696 |
+
"that column name is already in use on this database")
|
| 697 |
+
if existing.get("kind") != ROUTE_KIND:
|
| 698 |
+
raise err(400, "not_a_route_column",
|
| 699 |
+
"that column is shared but it is not a route order column, so re-solving "
|
| 700 |
+
"it would overwrite values this door did not write")
|
| 701 |
+
owner = str(existing.get("createdBy") or "")
|
| 702 |
+
if not session.admin and owner != session.uname:
|
| 703 |
+
raise err(403, "forbidden",
|
| 704 |
+
f"a route order can be re-solved by the person who planned it or by an "
|
| 705 |
+
f"administrator. This one was planned by {owner or 'somebody else'}, and "
|
| 706 |
+
f"re-solving it would change the visit numbers for every account at once")
|
| 707 |
+
|
| 708 |
+
raw = body.get("ranks")
|
| 709 |
+
if not isinstance(raw, dict) or not raw:
|
| 710 |
+
raise err(400, "bad_ranks", 'expected {ranks: {"<pid>": <visit number>}}')
|
| 711 |
+
# β REPORTED, NEVER TRUNCATED (standing rule 1's second sentence, and `MAX_BULK_ROWS`' own
|
| 712 |
+
# note). The ceiling is `routes_grid`'s so there is ONE of them, not two that drift.
|
| 713 |
+
if len(raw) > MAX_BULK_ROWS:
|
| 714 |
+
raise err(400, "too_many_rows",
|
| 715 |
+
f"at most {MAX_BULK_ROWS} records per route; this one carried {len(raw)}")
|
| 716 |
+
|
| 717 |
+
pool = allowed_pids(session)
|
| 718 |
+
ranks, not_in_pool, bad_value = {}, [], []
|
| 719 |
+
for raw_pid, value in raw.items():
|
| 720 |
+
try:
|
| 721 |
+
pid = int(raw_pid)
|
| 722 |
+
except (TypeError, ValueError):
|
| 723 |
+
not_in_pool.append(str(raw_pid)[:40])
|
| 724 |
+
continue
|
| 725 |
+
# β THE POOL IS THE WALL, and it is the SAME predicate the read path applies
|
| 726 |
+
# (`apply_row_scope` inside `allowed_pids`), so a caller cannot number a record they
|
| 727 |
+
# could not be shown, including one in the other business unit.
|
| 728 |
+
if pid not in pool:
|
| 729 |
+
not_in_pool.append(str(raw_pid)[:40])
|
| 730 |
+
continue
|
| 731 |
+
# β A BOOL IS AN `int` IN PYTHON and `True` would store as a visit number 1. Excluded by
|
| 732 |
+
# name rather than by hoping nobody sends one.
|
| 733 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
| 734 |
+
bad_value.append(str(raw_pid)[:40])
|
| 735 |
+
continue
|
| 736 |
+
ranks[pid] = value
|
| 737 |
+
|
| 738 |
+
# ββ A PARTLY HONOURED ROUTE IS NOT A ROUTE, AND THIS REFUSES THE WHOLE CALL RATHER THAN
|
| 739 |
+
# WRITING THE PART IT COULD. Caught by this ticket's own gate: a body carrying one record
|
| 740 |
+
# outside the caller's book still produced a clean 1..N over what was left, so the door minted
|
| 741 |
+
# a permanent tenant-wide column and filled it with a SHORTER route than the one that was
|
| 742 |
+
# solved. Every number in it was plausible and the day was wrong.
|
| 743 |
+
# β REPORTED, WITH THE SAMPLE, which is standing rule 1's second sentence: the refusal names
|
| 744 |
+
# what it could not take, so the caller can fix it rather than guess.
|
| 745 |
+
if not_in_pool:
|
| 746 |
+
raise err(400, "rows_not_in_your_book",
|
| 747 |
+
f"{len(not_in_pool)} of those records are not in your book, so the route "
|
| 748 |
+
f"cannot be written as it was solved. First few: "
|
| 749 |
+
f"{', '.join(sorted(not_in_pool)[:5])}")
|
| 750 |
+
if bad_value:
|
| 751 |
+
raise err(400, "rows_not_a_visit_number",
|
| 752 |
+
f"a visit number is a whole number from 1 upwards; {len(bad_value)} records "
|
| 753 |
+
f"carried something else. First few: {', '.join(sorted(bad_value)[:5])}")
|
| 754 |
+
if not ranks:
|
| 755 |
+
raise err(400, "no_rows_in_your_book",
|
| 756 |
+
"none of those records are in your book, so there is nothing to number")
|
| 757 |
+
# β A ROUTE IS A SEQUENCE, SO THE NUMBERS ARE 1..N WITH NO REPEAT AND NO GAP. A body that
|
| 758 |
+
# arrives truncated, doubled or partly applied would otherwise land as a plausible half
|
| 759 |
+
# route: every row carrying a number, and the day in the wrong order.
|
| 760 |
+
seq = sorted(ranks.values())
|
| 761 |
+
if seq != list(range(1, len(seq) + 1)):
|
| 762 |
+
raise err(400, "not_a_sequence",
|
| 763 |
+
f"a route order is the numbers 1 to {len(seq)}, each used once. This one "
|
| 764 |
+
f"carried {len(seq)} records numbered up to {seq[-1]} with "
|
| 765 |
+
f"{len(seq) - len(set(seq))} repeated")
|
| 766 |
+
|
| 767 |
+
stamp = time.strftime("%Y-%m-%d %H:%M")
|
| 768 |
+
defn = {
|
| 769 |
+
"key": key,
|
| 770 |
+
"label": label or (existing or {}).get("label") or key,
|
| 771 |
+
# β `int`, NEVER `text`. A text column sorts 1, 10, 11, 2 β fully populated, entirely
|
| 772 |
+
# plausible, wrong, and named by no gate. `patch_shared_cell` defaults to text; this door
|
| 773 |
+
# cannot.
|
| 774 |
+
"type": "int",
|
| 775 |
+
"source": "overlay",
|
| 776 |
+
"shared": True,
|
| 777 |
+
"kind": ROUTE_KIND,
|
| 778 |
+
# ββ THE PER-FIELD GRANT MARKER (T16). It is an EXPLICIT write-once declaration and it is
|
| 779 |
+
# what makes the wall fail CLOSED: a grant wall has the opposite absence-polarity to a
|
| 780 |
+
# deny wall, so keying visibility on "does a grant record exist" would publish this column
|
| 781 |
+
# to the whole tenant on one unreadable read of `object_shares`. Stamped, never inferred.
|
| 782 |
+
perm_scope.FIELD_GRANT_MARK: True,
|
| 783 |
+
"createdBy": (existing or {}).get("createdBy") or session.uname,
|
| 784 |
+
# β THE FINGERPRINT RIDES THE DEFINITION, ONCE. One `planRoute` run solves the whole
|
| 785 |
+
# cohort, so this string is identical for every record in it; per row it would be the same
|
| 786 |
+
# value written N times, and `shared_overlay._value` raises on a dict anyway.
|
| 787 |
+
"route": {
|
| 788 |
+
"inputsHash": str(body.get("inputsHash") or "")[:64],
|
| 789 |
+
"roundTrip": bool(body.get("roundTrip")),
|
| 790 |
+
"startPid": int(body["startPid"]) if isinstance(body.get("startPid"), int)
|
| 791 |
+
and not isinstance(body.get("startPid"), bool) else None,
|
| 792 |
+
"stops": len(ranks),
|
| 793 |
+
"solvedAt": stamp,
|
| 794 |
+
},
|
| 795 |
+
}
|
| 796 |
+
shared_overlay.put_field(_shared_key(), key, defn, st=session.runtime)
|
| 797 |
+
if existing is None:
|
| 798 |
+
try:
|
| 799 |
+
import core.shares as shares
|
| 800 |
+
# β THE EMPTY ENTRY LIST IS THE POINT. `set_grants` keeps a record with an owner and
|
| 801 |
+
# no entries, so "shared with nobody" is STORED and is a different fact from "never
|
| 802 |
+
# shared". Without the owner the column is unmanageable: `may_administer` fails closed
|
| 803 |
+
# on an ownerless record, so nobody could ever share it.
|
| 804 |
+
shares.set_grants("field", shares.field_oid(SHARE_TOPIC, key), [],
|
| 805 |
+
owner=session.uname, st=session.runtime)
|
| 806 |
+
except Exception: # noqa: BLE001
|
| 807 |
+
# The mark is already written, so a failed claim fails CLOSED: governed, nobody
|
| 808 |
+
# granted, admin-and-creator only. Recoverable. The other order is not.
|
| 809 |
+
pass
|
| 810 |
+
|
| 811 |
+
# β AND THE RECORDS THAT LOST THEIR NUMBER ARE CLEARED. A re-solve over a SMALLER cohort
|
| 812 |
+
# would otherwise leave the previous run's ranks sitting on the records that dropped out β
|
| 813 |
+
# plausible integers, from a route nobody is driving. `""` is the house spelling of an empty
|
| 814 |
+
# overlay cell (`rows_from_pool` defaults an absent one to exactly that), so this needs no new
|
| 815 |
+
# vocabulary and no tombstone nobody else reads.
|
| 816 |
+
stale = {}
|
| 817 |
+
for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items():
|
| 818 |
+
if not isinstance(cells, dict) or cells.get(key) in (None, ""):
|
| 819 |
+
continue
|
| 820 |
+
try:
|
| 821 |
+
gone = int(raw_pid)
|
| 822 |
+
except (TypeError, ValueError):
|
| 823 |
+
continue
|
| 824 |
+
if gone not in ranks:
|
| 825 |
+
stale[gone] = {key: ""}
|
| 826 |
+
written = shared_overlay.put_rows(
|
| 827 |
+
_shared_key(), {**{p: {key: v} for p, v in ranks.items()}, **stale},
|
| 828 |
+
st=session.runtime)
|
| 829 |
+
|
| 830 |
+
out = {"ok": True, "field": key, "label": defn["label"], "type": "int",
|
| 831 |
+
"stops": len(ranks), "cleared": len(stale),
|
| 832 |
+
"rows_written": len(written), "inputsHash": defn["route"]["inputsHash"],
|
| 833 |
+
"solvedAt": stamp}
|
| 834 |
+
return out
|
api/routes_geo.py
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_geo.py β the MAP PROVIDER SEAM (wave 37, R3/R14, tickets T35/T36/T37).
|
| 2 |
+
|
| 3 |
+
β WHY THIS FILE EXISTS AT ALL, and why the map does not simply call a vendor.
|
| 4 |
+
|
| 5 |
+
R14 (owner, 2026-08-19): no Google Maps API key this wave. The map work is not dropped, it is
|
| 6 |
+
retargeted to providers that need no key: OpenStreetMap tiles, OSRM routing, Nominatim geocoding.
|
| 7 |
+
The deferral only holds if swapping a paid vendor back in is a CONFIG change rather than a rewrite,
|
| 8 |
+
so every provider is a (base URL + attribution) pair read from the environment and served from ONE
|
| 9 |
+
door. The renderer receives a URL template and a credit line; it knows nothing about who is behind
|
| 10 |
+
them. That is the seam.
|
| 11 |
+
|
| 12 |
+
tiles AIOS_MAP_TILE_URL default https://tile.openstreetmap.org/{z}/{x}/{y}.png
|
| 13 |
+
geocode AIOS_GEOCODE_URL default https://nominatim.openstreetmap.org/search
|
| 14 |
+
routing AIOS_ROUTE_URL default https://router.project-osrm.org/route/v1/driving/
|
| 15 |
+
|
| 16 |
+
β R3'S ON-DEMAND RULE, AND WHY IT IS STRUCTURAL HERE RATHER THAN A COMMENT.
|
| 17 |
+
R3 forbids geocoding a whole table automatically. Under R14 the reason moves from money to manners
|
| 18 |
+
and gets sharper: Nominatim's usage policy allows roughly one request per second and explicitly
|
| 19 |
+
forbids bulk harvesting, so a loop over 3,636 customers does not produce a bill, it produces a
|
| 20 |
+
BLOCKED tenant. Three mechanisms enforce it, none of them advisory:
|
| 21 |
+
|
| 22 |
+
1. `/geo/geocode` takes an explicit LIST OF ADDRESSES from the caller. It cannot read a table, so
|
| 23 |
+
there is no code path from "a database exists" to "its rows were geocoded". Somebody had to
|
| 24 |
+
choose the records.
|
| 25 |
+
2. `MAX_BATCH` refuses an oversized list with a named error rather than truncating it. The client
|
| 26 |
+
drives the loop and can therefore SHOW the wait, which is the half of the ticket a comment
|
| 27 |
+
cannot satisfy.
|
| 28 |
+
3. `_throttle()` blocks in-process until the minimum interval has elapsed, so even a caller that
|
| 29 |
+
ignores everything above cannot exceed the published rate.
|
| 30 |
+
|
| 31 |
+
β AND A CACHE, because the policy asks for one: an address geocoded once is answered from memory
|
| 32 |
+
for the rest of the container's life. It is the cheapest way to be a good citizen and it makes a
|
| 33 |
+
re-run of the same records free rather than merely legal.
|
| 34 |
+
|
| 35 |
+
β NOTHING HERE PROXIES TILES. Tiles are fetched by the BROWSER, straight from the provider, which
|
| 36 |
+
is what every OSM client does and what the tile policy expects. Proxying them through this app
|
| 37 |
+
would put a free tier in the path of every pan and would breach the same policy it looks like it is
|
| 38 |
+
respecting. This door serves the tile URL, never the tile.
|
| 39 |
+
"""
|
| 40 |
+
import os
|
| 41 |
+
import re
|
| 42 |
+
import threading
|
| 43 |
+
import time
|
| 44 |
+
from collections import OrderedDict
|
| 45 |
+
|
| 46 |
+
import requests
|
| 47 |
+
from fastapi import APIRouter, Body, Depends
|
| 48 |
+
|
| 49 |
+
from deps import Session, err, require_session
|
| 50 |
+
|
| 51 |
+
router = APIRouter(prefix="/api/v1")
|
| 52 |
+
|
| 53 |
+
# --------------------------------------------------------------------- the seam
|
| 54 |
+
#
|
| 55 |
+
# Every value below is an environment override with a keyless default. Pointing the map at a paid
|
| 56 |
+
# vendor is three variables on the Space and no code change, which is what makes R14 a deferral.
|
| 57 |
+
|
| 58 |
+
_TILE_URL_DEFAULT = "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
|
| 59 |
+
_TILE_CREDIT_DEFAULT = "Β© OpenStreetMap contributors"
|
| 60 |
+
_TILE_CREDIT_HREF_DEFAULT = "https://www.openstreetmap.org/copyright"
|
| 61 |
+
_GEOCODE_URL_DEFAULT = "https://nominatim.openstreetmap.org/search"
|
| 62 |
+
_ROUTE_URL_DEFAULT = "https://router.project-osrm.org/route/v1/driving/"
|
| 63 |
+
|
| 64 |
+
#: Both policies require a real, identifying User-Agent. A generic one is how a shared service
|
| 65 |
+
#: decides an application is a scraper, so this carries the product name and a contact URL.
|
| 66 |
+
_UA_DEFAULT = "AIOS-Loopable/1.0 (+https://runloopable.com)"
|
| 67 |
+
|
| 68 |
+
#: Nominatim publishes one request per second. 1100 ms leaves room for clock jitter rather than
|
| 69 |
+
#: sitting exactly on the published edge.
|
| 70 |
+
_MIN_INTERVAL_MS_DEFAULT = 1100
|
| 71 |
+
|
| 72 |
+
#: The most addresses one call may carry. Small on purpose: the client loops and shows progress,
|
| 73 |
+
#: and no single request can sit on the connection for a minute waiting out the throttle.
|
| 74 |
+
MAX_BATCH = 25
|
| 75 |
+
|
| 76 |
+
#: Tile zoom the provider actually serves. OSM stops at 19.
|
| 77 |
+
_TILE_MAX_Z_DEFAULT = 19
|
| 78 |
+
|
| 79 |
+
_CACHE_MAX = 4096
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _env(name, default):
|
| 83 |
+
return (os.environ.get(name) or "").strip() or default
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _env_int(name, default):
|
| 87 |
+
try:
|
| 88 |
+
n = int((os.environ.get(name) or "").strip())
|
| 89 |
+
return n if n > 0 else default
|
| 90 |
+
except (TypeError, ValueError):
|
| 91 |
+
return default
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def provider_config():
|
| 95 |
+
"""The whole seam as one dict. `GET /geo/providers` serves it and the gate reads it."""
|
| 96 |
+
return {
|
| 97 |
+
"tiles": {
|
| 98 |
+
"url": _env("AIOS_MAP_TILE_URL", _TILE_URL_DEFAULT),
|
| 99 |
+
"attribution": _env("AIOS_MAP_TILE_ATTRIBUTION", _TILE_CREDIT_DEFAULT),
|
| 100 |
+
"attributionUrl": _env("AIOS_MAP_TILE_ATTRIBUTION_URL", _TILE_CREDIT_HREF_DEFAULT),
|
| 101 |
+
"maxZoom": _env_int("AIOS_MAP_TILE_MAX_Z", _TILE_MAX_Z_DEFAULT),
|
| 102 |
+
},
|
| 103 |
+
"geocode": {
|
| 104 |
+
"available": True,
|
| 105 |
+
"maxBatch": MAX_BATCH,
|
| 106 |
+
"minIntervalMs": _env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT),
|
| 107 |
+
# ββ THE NUMBER THE SURFACE MUST QUOTE, AND IT IS NOT THE RATE LIMIT.
|
| 108 |
+
# T37 asks for the estimated wait to be shown BEFORE a run. The obvious source is
|
| 109 |
+
# `minIntervalMs`, and it is wrong: 1.1 s is a floor on POLITENESS, not a prediction of
|
| 110 |
+
# LATENCY. One real lookup was MEASURED at 3.28 s end to end on 2026-08-19. Quoting the
|
| 111 |
+
# limit would promise a minute and take three, and a progress estimate that runs out
|
| 112 |
+
# before the work does reads as a hang rather than as a wait.
|
| 113 |
+
"secondsPerAddress": float(_env("AIOS_GEOCODE_SECONDS_EACH", "3.3")),
|
| 114 |
+
"attribution": _env("AIOS_GEOCODE_ATTRIBUTION", _TILE_CREDIT_DEFAULT),
|
| 115 |
+
},
|
| 116 |
+
"route": {
|
| 117 |
+
"available": True,
|
| 118 |
+
"attribution": _env("AIOS_ROUTE_ATTRIBUTION", "Routing by OSRM"),
|
| 119 |
+
},
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# ------------------------------------------------------------------ the throttle
|
| 124 |
+
#
|
| 125 |
+
# β ONE lock and ONE timestamp for the whole process, shared by geocoding and routing, because the
|
| 126 |
+
# rate limit belongs to the SERVICE and not to the endpoint. Two independent throttles would each
|
| 127 |
+
# stay honest and together double the published rate.
|
| 128 |
+
|
| 129 |
+
_gate = threading.Lock()
|
| 130 |
+
_last_call = [0.0]
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _throttle(min_interval_ms):
|
| 134 |
+
"""Block until the minimum interval since the last outbound call has elapsed.
|
| 135 |
+
|
| 136 |
+
Returns the seconds actually waited, so a caller can report the wait rather than hide it."""
|
| 137 |
+
wait = 0.0
|
| 138 |
+
with _gate:
|
| 139 |
+
gap = min_interval_ms / 1000.0
|
| 140 |
+
now = time.monotonic()
|
| 141 |
+
due = _last_call[0] + gap
|
| 142 |
+
if now < due:
|
| 143 |
+
wait = due - now
|
| 144 |
+
time.sleep(wait)
|
| 145 |
+
_last_call[0] = time.monotonic()
|
| 146 |
+
return wait
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
# --------------------------------------------------------------------- the cache
|
| 150 |
+
|
| 151 |
+
_cache = OrderedDict()
|
| 152 |
+
_cache_lock = threading.Lock()
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _cache_key(q, country):
|
| 156 |
+
return (re.sub(r"\s+", " ", str(q or "")).strip().lower(), str(country or "").strip().lower())
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _cache_get(key):
|
| 160 |
+
with _cache_lock:
|
| 161 |
+
if key not in _cache:
|
| 162 |
+
return None
|
| 163 |
+
_cache.move_to_end(key)
|
| 164 |
+
return _cache[key]
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def _cache_put(key, value):
|
| 168 |
+
with _cache_lock:
|
| 169 |
+
_cache[key] = value
|
| 170 |
+
_cache.move_to_end(key)
|
| 171 |
+
while len(_cache) > _CACHE_MAX:
|
| 172 |
+
_cache.popitem(last=False)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def geocode_one(address, country=None, timeout=12):
|
| 176 |
+
"""One address to {lat, lon, label} or None.
|
| 177 |
+
|
| 178 |
+
β THE ONLY PLACE AN OUTBOUND GEOCODE HAPPENS. `ai_enrich.py`'s geocode field calls this rather
|
| 179 |
+
than reaching for `requests` itself, so the throttle and the cache cannot be walked around by a
|
| 180 |
+
second caller. A cache hit costs no request and no wait."""
|
| 181 |
+
q = re.sub(r"\s+", " ", str(address or "")).strip()
|
| 182 |
+
if not q:
|
| 183 |
+
return None
|
| 184 |
+
key = _cache_key(q, country)
|
| 185 |
+
hit = _cache_get(key)
|
| 186 |
+
if hit is not None:
|
| 187 |
+
return dict(hit) if hit else None
|
| 188 |
+
|
| 189 |
+
cfg = provider_config()["geocode"]
|
| 190 |
+
_throttle(cfg["minIntervalMs"])
|
| 191 |
+
params = {"q": q, "format": "jsonv2", "limit": 1}
|
| 192 |
+
if country:
|
| 193 |
+
params["countrycodes"] = str(country).strip().lower()
|
| 194 |
+
try:
|
| 195 |
+
r = requests.get(
|
| 196 |
+
_env("AIOS_GEOCODE_URL", _GEOCODE_URL_DEFAULT),
|
| 197 |
+
params=params,
|
| 198 |
+
headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT),
|
| 199 |
+
"Accept": "application/json"},
|
| 200 |
+
timeout=timeout,
|
| 201 |
+
)
|
| 202 |
+
r.raise_for_status()
|
| 203 |
+
rows = r.json()
|
| 204 |
+
except Exception:
|
| 205 |
+
# β A transport failure is NOT cached. Caching it would turn one bad minute into a
|
| 206 |
+
# permanently empty column that no re-run could ever repair.
|
| 207 |
+
return None
|
| 208 |
+
if not isinstance(rows, list) or not rows:
|
| 209 |
+
# A genuine "no such place" IS cached, as a negative: asking again gets the same answer and
|
| 210 |
+
# spends another second of a shared service's budget to hear it.
|
| 211 |
+
_cache_put(key, {})
|
| 212 |
+
return None
|
| 213 |
+
top = rows[0] or {}
|
| 214 |
+
try:
|
| 215 |
+
lat = float(top.get("lat"))
|
| 216 |
+
lon = float(top.get("lon"))
|
| 217 |
+
except (TypeError, ValueError):
|
| 218 |
+
_cache_put(key, {})
|
| 219 |
+
return None
|
| 220 |
+
if not (-90 <= lat <= 90) or not (-180 <= lon <= 180):
|
| 221 |
+
_cache_put(key, {})
|
| 222 |
+
return None
|
| 223 |
+
out = {"lat": lat, "lon": lon, "label": str(top.get("display_name") or q)}
|
| 224 |
+
_cache_put(key, out)
|
| 225 |
+
return dict(out)
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
# ---------------------------------------------------------------------- the doors
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
@router.get("/geo/providers")
|
| 232 |
+
def geo_providers(session: Session = Depends(require_session)):
|
| 233 |
+
"""What the map should draw with, and who to credit for it.
|
| 234 |
+
|
| 235 |
+
Session gated like every other door here. The values are not secret, but an unauthenticated
|
| 236 |
+
endpoint on this app is a door somebody eventually hangs something else on."""
|
| 237 |
+
return provider_config()
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
@router.post("/geo/geocode")
|
| 241 |
+
def geo_geocode(body: dict = Body(...), session: Session = Depends(require_session)):
|
| 242 |
+
"""Turn a bounded list of addresses into coordinates.
|
| 243 |
+
|
| 244 |
+
β Takes ADDRESSES, never a table key, a view id or a filter. There is deliberately no shape of
|
| 245 |
+
request that means "geocode everything", which is R3 expressed as an API rather than as a
|
| 246 |
+
warning."""
|
| 247 |
+
items = body.get("addresses")
|
| 248 |
+
# β THE SECOND SHAPE, AND IT EXISTS TO KEEP ONE COMPOSER. A caller may send whole RECORDS plus
|
| 249 |
+
# the geocode field's config instead of finished address strings, and the server builds the
|
| 250 |
+
# query with `ai_enrich.geocode_query`. The alternative was a TypeScript twin of that function
|
| 251 |
+
# on the client, and an address composer that exists twice will disagree the first time either
|
| 252 |
+
# copy learns about a new column ([[one-question-two-normalizers]]) -- which is exactly the trap
|
| 253 |
+
# B's measured `New York (US)` suffix would spring, silently, in whichever copy forgot.
|
| 254 |
+
if items is None:
|
| 255 |
+
records = body.get("records")
|
| 256 |
+
cfg = body.get("config") or {}
|
| 257 |
+
if isinstance(records, list) and records:
|
| 258 |
+
import ai_enrich
|
| 259 |
+
items = []
|
| 260 |
+
for rec in records:
|
| 261 |
+
if not isinstance(rec, dict):
|
| 262 |
+
continue
|
| 263 |
+
q = ai_enrich.geocode_query(rec.get("row") or {}, cfg)
|
| 264 |
+
# β A record with no usable address is REPORTED, not dropped: it comes back
|
| 265 |
+
# `found: false` with an empty address, so the caller can name it on screen instead
|
| 266 |
+
# of quietly returning fewer answers than it asked questions.
|
| 267 |
+
items.append({"key": rec.get("key"), "address": q})
|
| 268 |
+
if not body.get("country") and cfg.get("country"):
|
| 269 |
+
body = {**body, "country": cfg.get("country")}
|
| 270 |
+
if not isinstance(items, list) or not items:
|
| 271 |
+
raise err(400, "no_addresses", "Send at least one address to look up.")
|
| 272 |
+
if len(items) > MAX_BATCH:
|
| 273 |
+
raise err(
|
| 274 |
+
400,
|
| 275 |
+
"batch_too_large",
|
| 276 |
+
f"Look up at most {MAX_BATCH} addresses per request. "
|
| 277 |
+
f"The map sends them in batches of {MAX_BATCH} so the wait stays visible.",
|
| 278 |
+
)
|
| 279 |
+
country = body.get("country")
|
| 280 |
+
out = []
|
| 281 |
+
started = time.monotonic()
|
| 282 |
+
for raw in items:
|
| 283 |
+
if isinstance(raw, dict):
|
| 284 |
+
key, addr = raw.get("key"), raw.get("address")
|
| 285 |
+
else:
|
| 286 |
+
key, addr = None, raw
|
| 287 |
+
# β AN EMPTY QUERY COSTS NO REQUEST AND CARRIES ITS OWN REASON. Nominatim's policy is a
|
| 288 |
+
# budget shared with everyone else using it, and asking it to place "" would spend a second
|
| 289 |
+
# of that budget to be told nothing. `reason` is what lets the surface say "no address on
|
| 290 |
+
# file" rather than "not found", which are different facts and want different actions.
|
| 291 |
+
if not str(addr or "").strip():
|
| 292 |
+
out.append({"key": key, "address": "", "found": False, "lat": None, "lon": None,
|
| 293 |
+
"label": None, "reason": "no_address"})
|
| 294 |
+
continue
|
| 295 |
+
hit = geocode_one(addr, country=country)
|
| 296 |
+
out.append({"key": key, "address": str(addr or ""), "found": bool(hit),
|
| 297 |
+
"reason": None if hit else "not_found",
|
| 298 |
+
"lat": hit["lat"] if hit else None,
|
| 299 |
+
"lon": hit["lon"] if hit else None,
|
| 300 |
+
"label": hit["label"] if hit else None})
|
| 301 |
+
elapsed = time.monotonic() - started
|
| 302 |
+
return {"results": out,
|
| 303 |
+
"found": sum(1 for r in out if r["found"]),
|
| 304 |
+
"asked": len(out),
|
| 305 |
+
"secondsElapsed": round(elapsed, 2),
|
| 306 |
+
"attribution": provider_config()["geocode"]["attribution"]}
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
@router.post("/geo/route")
|
| 310 |
+
def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
|
| 311 |
+
"""Road distance, duration and the drawn line, for stops ALREADY put in order.
|
| 312 |
+
|
| 313 |
+
β THE ORDERING IS NOT DONE HERE. `mapProjection.planRoute` sequences the stops on the client,
|
| 314 |
+
for free, and keeps working when this service does not answer. This door adds the half that
|
| 315 |
+
arithmetic cannot produce: what the ROADS actually cost. Splitting it that way is why the Start
|
| 316 |
+
picker and the round-trip toggle keep working with no network at all."""
|
| 317 |
+
stops = body.get("stops")
|
| 318 |
+
if not isinstance(stops, list) or len(stops) < 2:
|
| 319 |
+
raise err(400, "too_few_stops", "A route needs at least two stops with a location.")
|
| 320 |
+
if len(stops) > 25:
|
| 321 |
+
raise err(400, "too_many_stops",
|
| 322 |
+
"Route at most 25 stops at once. Narrow the selection and try again.")
|
| 323 |
+
pairs = []
|
| 324 |
+
for s in stops:
|
| 325 |
+
try:
|
| 326 |
+
lat, lon = float(s["lat"]), float(s["lon"])
|
| 327 |
+
except (TypeError, ValueError, KeyError, IndexError):
|
| 328 |
+
raise err(400, "bad_stop", "Every stop needs a numeric latitude and longitude.")
|
| 329 |
+
if not (-90 <= lat <= 90) or not (-180 <= lon <= 180):
|
| 330 |
+
raise err(400, "bad_stop", "Every stop needs a latitude and longitude on the globe.")
|
| 331 |
+
pairs.append(f"{lon:.6f},{lat:.6f}")
|
| 332 |
+
|
| 333 |
+
base = _env("AIOS_ROUTE_URL", _ROUTE_URL_DEFAULT)
|
| 334 |
+
_throttle(_env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT))
|
| 335 |
+
try:
|
| 336 |
+
r = requests.get(
|
| 337 |
+
base.rstrip("/") + "/" + ";".join(pairs),
|
| 338 |
+
params={"overview": "simplified", "geometries": "geojson", "steps": "false"},
|
| 339 |
+
headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT)},
|
| 340 |
+
timeout=20,
|
| 341 |
+
)
|
| 342 |
+
r.raise_for_status()
|
| 343 |
+
data = r.json()
|
| 344 |
+
except Exception:
|
| 345 |
+
raise err(502, "route_service_unavailable",
|
| 346 |
+
"The routing service did not answer. The stop order and the straight line "
|
| 347 |
+
"distance are still on the map.")
|
| 348 |
+
routes = (data or {}).get("routes") or []
|
| 349 |
+
if not routes:
|
| 350 |
+
raise err(
|
| 351 |
+
502,
|
| 352 |
+
"no_route",
|
| 353 |
+
"No road route connects these stops. They may be on different land masses, or one of "
|
| 354 |
+
"them may be far from any road.",
|
| 355 |
+
)
|
| 356 |
+
top = routes[0]
|
| 357 |
+
geom = ((top.get("geometry") or {}).get("coordinates")) or []
|
| 358 |
+
line = []
|
| 359 |
+
for c in geom:
|
| 360 |
+
try:
|
| 361 |
+
line.append([float(c[0]), float(c[1])])
|
| 362 |
+
except (TypeError, ValueError, IndexError):
|
| 363 |
+
continue
|
| 364 |
+
return {
|
| 365 |
+
"km": round(float(top.get("distance") or 0.0) / 1000.0, 1),
|
| 366 |
+
"minutes": int(round(float(top.get("duration") or 0.0) / 60.0)),
|
| 367 |
+
"line": line,
|
| 368 |
+
"attribution": provider_config()["route"]["attribution"],
|
| 369 |
+
}
|
api/routes_nav.py
CHANGED
|
@@ -494,8 +494,37 @@ def nav(session: Session = Depends(require_session)):
|
|
| 494 |
# Manage user. An administrator unticking a shared database left it on the rail.
|
| 495 |
# `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key.
|
| 496 |
import core.perm_scope as _perm_scope
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 497 |
for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin,
|
| 498 |
st=_lent):
|
|
|
|
|
|
|
| 499 |
if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent):
|
| 500 |
continue
|
| 501 |
pages.append({"key": e["key"], "label": e["label"],
|
|
@@ -838,8 +867,22 @@ def save_nav_prefs(body: dict = Body(default=None),
|
|
| 838 |
@router.get("/nav/schema/{key}")
|
| 839 |
def nav_schema(key: str, session: Session = Depends(require_session)):
|
| 840 |
"""The database's schema drawer payload: its field contract + the semantic measures this
|
| 841 |
-
session may build with.
|
| 842 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 843 |
# Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate
|
| 844 |
# (`may_open`) rather than the module grant machinery β `session.require` would 403 every
|
| 845 |
# ut key because a user table is deliberately not a module.
|
|
@@ -849,6 +892,25 @@ def nav_schema(key: str, session: Session = Depends(require_session)):
|
|
| 849 |
if not defn or not user_tables.may_open(key, session.uname, session.admin,
|
| 850 |
st=session.runtime):
|
| 851 |
raise err(403, "forbidden", "that database belongs to another user")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 852 |
# WAVE 19 (R8) β the drawer wears the RENAMED name. A rail that says one thing and a
|
| 853 |
# schema panel opened from it that says another is the drift a rename is supposed to
|
| 854 |
# remove, not create.
|
|
@@ -859,7 +921,7 @@ def nav_schema(key: str, session: Session = Depends(require_session)):
|
|
| 859 |
"fields": [{"key": f["key"], "label": f["label"], "type": f["type"],
|
| 860 |
"source": f.get("source") or "overlay",
|
| 861 |
"description": str(f.get("description") or "")}
|
| 862 |
-
for f in
|
| 863 |
"measures": []}
|
| 864 |
session.require(key)
|
| 865 |
pages = perms.nav_pages(session.user) or []
|
|
@@ -870,7 +932,26 @@ def nav_schema(key: str, session: Session = Depends(require_session)):
|
|
| 870 |
if key in ("customer_data", "cohort", "customers"):
|
| 871 |
try:
|
| 872 |
import aios_grid
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 873 |
for f in aios_grid.FIELDS:
|
|
|
|
|
|
|
| 874 |
entry = {"key": f["key"], "label": f["label"], "type": f["type"],
|
| 875 |
"source": f["source"],
|
| 876 |
"description": str(f.get("description") or "")}
|
|
@@ -882,8 +963,19 @@ def nav_schema(key: str, session: Session = Depends(require_session)):
|
|
| 882 |
measures = []
|
| 883 |
try:
|
| 884 |
from core import measure_resolve
|
|
|
|
| 885 |
team_id = perms.scope_team_id(session.user)
|
| 886 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 887 |
measures.append({"key": str(m.get("key") or ""),
|
| 888 |
"label": str(m.get("label") or m.get("key") or ""),
|
| 889 |
"type": str(m.get("type") or "")})
|
|
|
|
| 494 |
# Manage user. An administrator unticking a shared database left it on the rail.
|
| 495 |
# `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key.
|
| 496 |
import core.perm_scope as _perm_scope
|
| 497 |
+
# ββ WAVE 37 Β· T07 (D-267, PRD amendment A8) β A RETIRED KEY CANNOT REACH THE NAV, WHATEVER
|
| 498 |
+
# THE TENANT DOCUMENT STILL CARRIES.
|
| 499 |
+
#
|
| 500 |
+
# W33-T44 retired `ut_odoo_customers` / `ut_odoo_products` in favour of `customer_data` /
|
| 501 |
+
# `product_data` β ONE SUBJECT, ONE DATABASE β and `apply_plan` pops them from a tenant's
|
| 502 |
+
# document on the next rebuild. The owner then reported *"Odoo products"* TWICE, and a live
|
| 503 |
+
# label census returned `{'Odoo products': 2}`: the retirement MECHANISM was correct and the
|
| 504 |
+
# PATH DID NOT RUN, for causes booked separately (D-251's `frozen(rt)` early return, and the
|
| 505 |
+
# rebuild being scoped to `is_royal`).
|
| 506 |
+
#
|
| 507 |
+
# β SO THIS IS A READ-SIDE GUARD, NOT THE MIGRATION, AND THE DIFFERENCE IS STATED RATHER THAN
|
| 508 |
+
# BLURRED: the stale ROW may still sit in a tenant's document, and removing it is
|
| 509 |
+
# `apply_plan`'s job in `odoo_relational.py`. What this line guarantees is that a document
|
| 510 |
+
# which still carries one cannot put a SECOND "Odoo products" on somebody's rail while that
|
| 511 |
+
# migration has not reached them. A defect whose fix depends on a migration running is a
|
| 512 |
+
# defect that comes back on the one tenant the migration missed
|
| 513 |
+
# [[a-migration-that-runs-on-the-next-write]].
|
| 514 |
+
#
|
| 515 |
+
# β `RETIRED_KEYS` is IMPORTED, never re-typed here. A hand-copied tuple is exactly the drift
|
| 516 |
+
# this guard exists to prevent, and it would go stale the first time a third key is retired.
|
| 517 |
+
# β Inside the same `try:` as the rest of the user-table read on purpose β if the import
|
| 518 |
+
# fails, the block's own `except` marks the nav DEGRADED, which is the honest outcome. A
|
| 519 |
+
# guard that silently stops guarding is worse than one that says it could not run.
|
| 520 |
+
try:
|
| 521 |
+
from odoo_relational import RETIRED_KEYS as _retired
|
| 522 |
+
except Exception:
|
| 523 |
+
_retired = ()
|
| 524 |
for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin,
|
| 525 |
st=_lent):
|
| 526 |
+
if e["key"] in _retired:
|
| 527 |
+
continue
|
| 528 |
if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent):
|
| 529 |
continue
|
| 530 |
pages.append({"key": e["key"], "label": e["label"],
|
|
|
|
| 867 |
@router.get("/nav/schema/{key}")
|
| 868 |
def nav_schema(key: str, session: Session = Depends(require_session)):
|
| 869 |
"""The database's schema drawer payload: its field contract + the semantic measures this
|
| 870 |
+
session may build with.
|
| 871 |
+
|
| 872 |
+
TWO WALLS, ANSWERING TWO QUESTIONS, AND THEY FAIL DIFFERENTLY ON PURPOSE. The DATABASE wall
|
| 873 |
+
is the same predicate the nav uses β a key this session may not open answers 403, never a
|
| 874 |
+
redacted schema. The FIELD wall runs *after* it, on a database this session may open, and
|
| 875 |
+
its whole job is to narrow: the drawer serves the columns this reader receives everywhere
|
| 876 |
+
else, and no more.
|
| 877 |
+
|
| 878 |
+
ββ W38-T18 β THE FIELD WALL IS NEW HERE AND THIS DOOR IS WHERE IT WAS MISSING. Every grid
|
| 879 |
+
door has stripped hidden columns from both wires since W36-T21; the schema drawer beside them
|
| 880 |
+
served the raw contract to anyone `require`/`may_open` admitted. Owner instruction 10 β
|
| 881 |
+
*"Metrics field lookback and value must abide by the Filter permissioning"* β is satisfied on
|
| 882 |
+
the picker side already (the condition kit takes `fields` as a PROP off a payload that is
|
| 883 |
+
walled upstream), so this route is the one place the same reader could still learn the name,
|
| 884 |
+
the label and the VALUE VOCABULARY of a column their grid refuses them.
|
| 885 |
+
"""
|
| 886 |
# Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate
|
| 887 |
# (`may_open`) rather than the module grant machinery β `session.require` would 403 every
|
| 888 |
# ut key because a user table is deliberately not a module.
|
|
|
|
| 892 |
if not defn or not user_tables.may_open(key, session.uname, session.admin,
|
| 893 |
st=session.runtime):
|
| 894 |
raise err(403, "forbidden", "that database belongs to another user")
|
| 895 |
+
# ββ W38-T18 β THE FIELD WALL ON A `ut_*` DATABASE, WHICH THIS DOOR HAS NEVER RUN.
|
| 896 |
+
# `may_open` answers *IF* you reach the database and says nothing about WHICH COLUMNS, so
|
| 897 |
+
# this drawer handed `defn["fields"]` whole to every account it admitted. The bypass is
|
| 898 |
+
# older than this ticket (it dates to the wall itself); what changed is the CATEGORY it
|
| 899 |
+
# leaks β since W38-T16 a hidden `ut_*` column means an administrator's `hiddenFields`
|
| 900 |
+
# *or* a column granted away, and a share door whose schema drawer still names the column
|
| 901 |
+
# tells the ungranted reader exactly what they were refused.
|
| 902 |
+
# β MERGED BEFORE THE WALL, NEVER AFTER β the same order `routes_tables.ut_assembly`
|
| 903 |
+
# runs, and for the reason `_ut_shared_fields` gives one door over: the closure must see
|
| 904 |
+
# the WHOLE contract, or a definition field whose formula reads a hidden tenant-wide
|
| 905 |
+
# column sits outside its reach and carries that value out wearing a second name.
|
| 906 |
+
# β ONLY THE DEFINITION'S OWN FIELDS ARE RETURNED. This drawer has never published the
|
| 907 |
+
# shared stratum and this ticket is not where it starts; widening the list handed to the
|
| 908 |
+
# closure can only ever ADD to the hidden set, never remove from it.
|
| 909 |
+
import routes_tables as _rt
|
| 910 |
+
base = list(defn.get("fields") or [])
|
| 911 |
+
hidden = _rt._ut_hidden(session, key,
|
| 912 |
+
_rt._ut_shared_fields(session, key, base),
|
| 913 |
+
st=session.runtime)
|
| 914 |
# WAVE 19 (R8) β the drawer wears the RENAMED name. A rail that says one thing and a
|
| 915 |
# schema panel opened from it that says another is the drift a rename is supposed to
|
| 916 |
# remove, not create.
|
|
|
|
| 921 |
"fields": [{"key": f["key"], "label": f["label"], "type": f["type"],
|
| 922 |
"source": f.get("source") or "overlay",
|
| 923 |
"description": str(f.get("description") or "")}
|
| 924 |
+
for f in base if f.get("key") not in hidden],
|
| 925 |
"measures": []}
|
| 926 |
session.require(key)
|
| 927 |
pages = perms.nav_pages(session.user) or []
|
|
|
|
| 932 |
if key in ("customer_data", "cohort", "customers"):
|
| 933 |
try:
|
| 934 |
import aios_grid
|
| 935 |
+
import routes_customers as _rc
|
| 936 |
+
# ββ W38-T18 β THE FIELD WALL ON THE CUSTOMER CONTRACT, AND THE `options` LINE FOUR
|
| 937 |
+
# ROWS DOWN IS WHY IT IS A LEAK RATHER THAN AN UNTIDINESS. A schema drawer that names
|
| 938 |
+
# a hidden column has told the reader it exists; one that ships `options` has handed
|
| 939 |
+
# them the VALUE VOCABULARY of every choice and status column their grid refuses β
|
| 940 |
+
# the set of statuses the business uses, read straight off a door with no wall.
|
| 941 |
+
# β `_hidden_for` IS REUSED, NOT RE-DERIVED, and the difference is not cosmetic.
|
| 942 |
+
# `hidden_keys(user, MODULE, aios_grid.FIELDS)` is the obvious spelling and it is
|
| 943 |
+
# INCOMPLETE: that function's own docstring records that a wall computed from the
|
| 944 |
+
# canonical list alone answers "not hidden" for every runtime column, so the
|
| 945 |
+
# tenant-wide stratum never reaches the closure and a formula over a hidden column
|
| 946 |
+
# comes back. One evaluator, one caller, no second idea of what this session hides.
|
| 947 |
+
# β INSIDE THE `try`, DELIBERATELY. If the wall cannot be computed the route already
|
| 948 |
+
# answers with an empty contract rather than a full one β fail-closed is the only
|
| 949 |
+
# direction a schema door may fail, and the gate's ADMIN control is what stops that
|
| 950 |
+
# silent empty being mistaken for a wall that worked.
|
| 951 |
+
hidden = _rc._hidden_for(session)
|
| 952 |
for f in aios_grid.FIELDS:
|
| 953 |
+
if f["key"] in hidden:
|
| 954 |
+
continue
|
| 955 |
entry = {"key": f["key"], "label": f["label"], "type": f["type"],
|
| 956 |
"source": f["source"],
|
| 957 |
"description": str(f.get("description") or "")}
|
|
|
|
| 963 |
measures = []
|
| 964 |
try:
|
| 965 |
from core import measure_resolve
|
| 966 |
+
import core.perm_scope as _perm_scope
|
| 967 |
team_id = perms.scope_team_id(session.user)
|
| 968 |
+
# ββ W38-T19 β THE METRICS CAPABILITY REACHES THIS DRAWER TOO, AND IT IS THE ONE DOOR
|
| 969 |
+
# THE THREE GRID ASSEMBLIES DO NOT SPEAK FOR. This route calls `measure_resolve.offer`
|
| 970 |
+
# ITSELF rather than reading a `measures` key off an assembly, so a wall applied in
|
| 971 |
+
# `routes_customers` / `routes_products` / `routes_tables` narrows every grid and leaves
|
| 972 |
+
# the panel beside them naming the full lookback catalogue β the same one-door-short
|
| 973 |
+
# shape W38-T18 fixed for FIELDS in this very function, one key over.
|
| 974 |
+
# β THE UT_ BRANCH ABOVE RETURNS BEFORE THIS AND ALREADY SERVES `[]`, so there is
|
| 975 |
+
# nothing to narrow there; if it ever starts publishing an offer, it needs this line.
|
| 976 |
+
_offer = ((measure_resolve.offer(team_id) or [])
|
| 977 |
+
if _perm_scope.may_metrics(session.user, key) else [])
|
| 978 |
+
for m in _offer:
|
| 979 |
measures.append({"key": str(m.get("key") or ""),
|
| 980 |
"label": str(m.get("label") or m.get("key") or ""),
|
| 981 |
"type": str(m.get("type") or "")})
|
api/routes_products.py
CHANGED
|
@@ -115,15 +115,207 @@ def scoped_pool(session: Session):
|
|
| 115 |
return pids, team_id, rows_src, fields_base
|
| 116 |
|
| 117 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 119 |
consume_corrections: bool = True):
|
| 120 |
"""The product topic's mirror of `routes_customers.grid_assembly` β SAME g-dict keys, so
|
| 121 |
the /workspace and events routes consume either interchangeably.
|
| 122 |
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
|
| 128 |
β WAVE 19 / R9 β `lists` IS NO LONGER EMPTY. Wave 16 passed `with_cohorts=False` because
|
| 129 |
there was one customer-keyed cohort bucket and product pids are CRC32 hashes of SKU codes;
|
|
@@ -172,12 +364,29 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str
|
|
| 172 |
fields = [f for f in fields if f.get("key") not in hidden]
|
| 173 |
rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
|
| 174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
| 176 |
"fields": fields, "views": views, "lists": lists,
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
"derived": aios_grid.cohort_cells(lists),
|
| 180 |
-
"measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
|
| 181 |
"team_id": team_id}
|
| 182 |
|
| 183 |
|
|
|
|
| 115 |
return pids, team_id, rows_src, fields_base
|
| 116 |
|
| 117 |
|
| 118 |
+
#: The semantic ENTITY topic this grid IS. `model/topics/odoo_products.yml` names the same store
|
| 119 |
+
#: key in its `grid:` field, so the binding is stated at both ends rather than inferred.
|
| 120 |
+
TOPIC = "product_data"
|
| 121 |
+
SEM_TOPIC = "odoo_products"
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _measure_err(tag, e):
|
| 125 |
+
try:
|
| 126 |
+
import harness.telemetry as _tel
|
| 127 |
+
_tel.error(f"api:{tag}", e)
|
| 128 |
+
except Exception:
|
| 129 |
+
pass
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _measure_stamp(rt, team_id):
|
| 133 |
+
"""The cached pool's build timestamp β the DATA STAMP in every memo key, so a pool refresh
|
| 134 |
+
invalidates memoised measure answers exactly when the underlying rows changed. Same
|
| 135 |
+
discipline as `routes_customers._pool_stamp`, over THIS route's cache key."""
|
| 136 |
+
entry = rt.pool_cache.get(("product_pool", team_id))
|
| 137 |
+
return entry[0] if isinstance(entry, tuple) and entry else 0
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def product_measures(session):
|
| 141 |
+
"""The measure OFFER for this SESSION on this database, or `[]`.
|
| 142 |
+
|
| 143 |
+
ββ W38-T19 β `session` IS REQUIRED, NOT DEFAULTED, AND THAT IS THE POINT OF THE EDIT. This
|
| 144 |
+
function was PRINCIPAL-BLIND: it answered the same catalogue to every caller, so there was
|
| 145 |
+
nothing for a per-user capability to be consulted by. A `session=None` default would have
|
| 146 |
+
left the old call shape working and silently ungated β the fail-open spelling of the same
|
| 147 |
+
bug, and the one `_module_fields` in `routes_admin` was rewritten to refuse. A caller with
|
| 148 |
+
no principal cannot ask this question and must not get an answer.
|
| 149 |
+
|
| 150 |
+
β EMPTYING THE OFFER IS THE WHOLE WALL ON THIS GRAIN, and it is three refusals rather than
|
| 151 |
+
one: `_measure_cells` and `_measure_condition_sets` both return `{}` for a falsy `offer`, and
|
| 152 |
+
`routes_grid` passes this list into `clean_measure_field` as the ADMISSION set, which
|
| 153 |
+
fail-closes `measure_not_offered` on a create. The client's `offerMeasure={measures.length >
|
| 154 |
+
0}` then takes the Metric kind off the picker for the same reason it hides it on a database
|
| 155 |
+
with no binding.
|
| 156 |
+
|
| 157 |
+
ββ W37-T10 / owner item 4 (R1) β this is what turns `measures: []` into a real catalogue.
|
| 158 |
+
Owner: *"Odoo products should have a lookbsck metrics like sales etc."* The list is DERIVED
|
| 159 |
+
from `model/topics/odoo_products.yml`'s `measures:` binding, never written here β a second
|
| 160 |
+
list would be a second definition of the same fact.
|
| 161 |
+
|
| 162 |
+
β `[]` ON FAILURE IS THE CORRECT DEGRADE and it is not silent: `entity_measures` refuses a key
|
| 163 |
+
it cannot prove, the reasons are readable through `semantic.entity_measure_refusals`, and the
|
| 164 |
+
client's own rule (`offerMeasure={measures.length > 0}`) then hides the Metric kind rather
|
| 165 |
+
than offering a column that could only render blank.
|
| 166 |
+
"""
|
| 167 |
+
import core.perm_scope as perm_scope
|
| 168 |
+
if not perm_scope.may_metrics(session.user, MODULE):
|
| 169 |
+
return []
|
| 170 |
+
try:
|
| 171 |
+
from harness import semantic as sem
|
| 172 |
+
return sem.entity_measures(SEM_TOPIC)
|
| 173 |
+
except Exception as e: # noqa: BLE001
|
| 174 |
+
_measure_err("product-measure-offer", e)
|
| 175 |
+
return []
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
|
| 179 |
+
"""`{pid: {field key: value}}` for every measure column on this grid.
|
| 180 |
+
|
| 181 |
+
β THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's
|
| 182 |
+
`offerMeasure` on and lets a user mint a Metric column that would then be blank forever β
|
| 183 |
+
this repo's most repeated defect ([[reachable-is-not-the-same-as-built]]). Nothing else
|
| 184 |
+
computes product-grain measure values: `core.measure_resolve.column_values` is CUSTOMER-grain
|
| 185 |
+
(it resolves through `harness.measure_filter`, whose entity is the partner), which is exactly
|
| 186 |
+
what this route's docstring used to record as an honest absence.
|
| 187 |
+
|
| 188 |
+
β ONE QUERY PER WINDOW, not per column. Two Metric columns over the same 90 days are one
|
| 189 |
+
grouped scan; the fan-out to field keys happens in memory afterwards.
|
| 190 |
+
|
| 191 |
+
β THE JOIN IS BY SKU CODE, per contract C1's trap. `sales_lines.product_code` reproduces this
|
| 192 |
+
pool's identity (`default_code`, or `pid:<odoo id>` for the 33 codeless actives) and `pid` is
|
| 193 |
+
a CRC32 of it β so the map is `row['code'] -> row['pid']`, taken off the rows we already hold.
|
| 194 |
+
Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries.
|
| 195 |
+
"""
|
| 196 |
+
import aios_grid
|
| 197 |
+
|
| 198 |
+
mfields = aios_grid.measure_fields_of(fields)
|
| 199 |
+
if not mfields or not offer:
|
| 200 |
+
return {}
|
| 201 |
+
from harness import semantic as sem
|
| 202 |
+
from harness import windows as _wn
|
| 203 |
+
|
| 204 |
+
by_key = {m["key"]: m for m in offer}
|
| 205 |
+
codes = {r["pid"]: r["code"] for r in rows_src if r.get("pid") is not None}
|
| 206 |
+
|
| 207 |
+
# Group the columns by WINDOW: same window, same scan.
|
| 208 |
+
by_window = {}
|
| 209 |
+
for f in mfields:
|
| 210 |
+
spec = f.get("measure") or {}
|
| 211 |
+
if spec.get("key") not in by_key:
|
| 212 |
+
continue # not offered for this caller -> the column stays blank, honestly
|
| 213 |
+
w = _wn.normalize(spec.get("window"))
|
| 214 |
+
rng = _wn.resolve(spec.get("window"), today) if w is not None else None
|
| 215 |
+
if rng is None:
|
| 216 |
+
# β NEVER WIDEN TO ALL TIME. An unresolvable window is an unanswerable column, and
|
| 217 |
+
# silently answering a different question is worse than a blank one.
|
| 218 |
+
continue
|
| 219 |
+
by_window.setdefault((rng[0], rng[1]), []).append(f)
|
| 220 |
+
|
| 221 |
+
out = {}
|
| 222 |
+
for (dfrom, dto), group in by_window.items():
|
| 223 |
+
keys = tuple(sorted({f["measure"]["key"] for f in group}))
|
| 224 |
+
mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys)
|
| 225 |
+
if mkey not in memo:
|
| 226 |
+
try:
|
| 227 |
+
memo[mkey] = sem.entity_measure_values(
|
| 228 |
+
SEM_TOPIC, list(keys), date_from=dfrom, date_to=dto, team_id=team_id,
|
| 229 |
+
# β SERVICES ARE IN THIS CATALOGUE (37 active), so they must NOT be excluded
|
| 230 |
+
# here. The service filter exists to stop Delivery Charges polluting a SKU
|
| 231 |
+
# RANKING; on a per-row column the row IS the service product, and excluding
|
| 232 |
+
# it would print a FALSE 0 β which under C1's rule asserts "sold nothing".
|
| 233 |
+
exclude_services=False, offer=offer)
|
| 234 |
+
except Exception as e: # noqa: BLE001
|
| 235 |
+
_measure_err("product-measure-column", e)
|
| 236 |
+
# Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure
|
| 237 |
+
# memoises, a TRANSIENT one (store still warming after a restart) must not, or
|
| 238 |
+
# the cells stay blank for the rest of the session long after the data landed.
|
| 239 |
+
transient = False
|
| 240 |
+
try:
|
| 241 |
+
from harness import datastore as _ds
|
| 242 |
+
transient = not _ds.ready()
|
| 243 |
+
except Exception: # noqa: BLE001
|
| 244 |
+
transient = False
|
| 245 |
+
if transient:
|
| 246 |
+
continue
|
| 247 |
+
memo[mkey] = None
|
| 248 |
+
vals = memo[mkey]
|
| 249 |
+
if vals is None:
|
| 250 |
+
continue
|
| 251 |
+
# C1's empty-window rule, applied to EVERY row β not only the ones with a group. 72% of
|
| 252 |
+
# this catalogue has no group in a 90-day window, so this loop IS the common case.
|
| 253 |
+
filled = {pid: sem.entity_zero_fill(vals.get(code), list(keys), offer)
|
| 254 |
+
for pid, code in codes.items()}
|
| 255 |
+
for f in group:
|
| 256 |
+
k, fkey = f["measure"]["key"], f["key"]
|
| 257 |
+
for pid, cell in filled.items():
|
| 258 |
+
if k in cell:
|
| 259 |
+
out.setdefault(pid, {})[fkey] = cell[k]
|
| 260 |
+
if len(memo) > 100: # a lifetime cache, not a leak
|
| 261 |
+
memo.clear()
|
| 262 |
+
return out
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _measure_condition_sets(views, offer, rows_src, team_id, today):
|
| 266 |
+
"""`{rule id: [pid, β¦]}` for every measure CONDITION across this session's saved views.
|
| 267 |
+
|
| 268 |
+
β THE ANSWER IS KEYED BY pid, NOT BY SKU CODE. `entity_measure_sets` answers in the SOURCE
|
| 269 |
+
dim's own values (the code) because it cannot know how a grid hashes identity; the map back
|
| 270 |
+
is this route's job, exactly as it is for the column cells.
|
| 271 |
+
"""
|
| 272 |
+
if not offer:
|
| 273 |
+
return {}
|
| 274 |
+
from harness import semantic as sem
|
| 275 |
+
|
| 276 |
+
admitted = {m["key"] for m in offer}
|
| 277 |
+
by_id = {}
|
| 278 |
+
for v in views or []:
|
| 279 |
+
cfg = (v.get("config") or {})
|
| 280 |
+
for rule in sem.entity_measure_leaves(cfg.get("filters"), admitted):
|
| 281 |
+
rid = str(rule.get("id") or "")
|
| 282 |
+
if rid:
|
| 283 |
+
by_id[rid] = rule # last statement of the question wins
|
| 284 |
+
if not by_id:
|
| 285 |
+
return {}
|
| 286 |
+
pid_by_code = {r["code"]: r["pid"] for r in rows_src if r.get("pid") is not None}
|
| 287 |
+
try:
|
| 288 |
+
sets = sem.entity_measure_sets(
|
| 289 |
+
SEM_TOPIC, list(by_id.values()), today, team_id=team_id,
|
| 290 |
+
keys_by_id=list(pid_by_code), offer=offer, exclude_services=False)
|
| 291 |
+
except Exception as e: # noqa: BLE001
|
| 292 |
+
_measure_err("product-measure-sets", e)
|
| 293 |
+
return {}
|
| 294 |
+
# A code the pool does not carry is dropped rather than keyed on None β the same rule the
|
| 295 |
+
# cell path applies, for the same reason.
|
| 296 |
+
return {rid: sorted(pid_by_code[c] for c in codes if c in pid_by_code)
|
| 297 |
+
for rid, codes in sets.items()}
|
| 298 |
+
|
| 299 |
+
|
| 300 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 301 |
consume_corrections: bool = True):
|
| 302 |
"""The product topic's mirror of `routes_customers.grid_assembly` β SAME g-dict keys, so
|
| 303 |
the /workspace and events routes consume either interchangeably.
|
| 304 |
|
| 305 |
+
ββ W37-T10 / OWNER ITEM 4 (R1) β `measures` IS NO LONGER EMPTY, and the header above it used
|
| 306 |
+
to record the opposite as a topic fact. It was one: `core.measure_resolve` is CUSTOMER-grain,
|
| 307 |
+
so nothing here could resolve a product-grain measure. What changed is that the ENTITY path
|
| 308 |
+
exists now (`semantic.entity_measures` / `entity_measure_values`, bound in
|
| 309 |
+
`model/topics/odoo_products.yml`), so this surface serves a real six-metric catalogue AND the
|
| 310 |
+
cells to go with it β the offer and the resolver landing together, deliberately, because an
|
| 311 |
+
offer whose values nobody computes is a column that renders blank forever.
|
| 312 |
+
|
| 313 |
+
One deliberate absence remains, and it is a real one:
|
| 314 |
+
* `measure_sets` stays EMPTY β that is the measure CONDITION channel (a filter answered
|
| 315 |
+
server-side as a pid set), which still runs through the customer-grain
|
| 316 |
+
`measure_resolve.condition_sets`. So a Metric COLUMN works here; a Metric FILTER does
|
| 317 |
+
not yet. Stated rather than left to be discovered: `clean_filter_tree` drops a measure
|
| 318 |
+
condition whose key is not in `measure_keys`, so the refusal is already fail-closed.
|
| 319 |
|
| 320 |
β WAVE 19 / R9 β `lists` IS NO LONGER EMPTY. Wave 16 passed `with_cohorts=False` because
|
| 321 |
there was one customer-keyed cohort bucket and product pids are CRC32 hashes of SKU codes;
|
|
|
|
| 364 |
fields = [f for f in fields if f.get("key") not in hidden]
|
| 365 |
rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
|
| 366 |
|
| 367 |
+
today = time.strftime("%Y-%m-%d")
|
| 368 |
+
stamp = _measure_stamp(session.runtime, team_id)
|
| 369 |
+
measures = product_measures(session)
|
| 370 |
+
|
| 371 |
+
# R9: the Cohorts column's cells, built from THIS topic's lists. β W37-T10 β the measure
|
| 372 |
+
# half of this same read-only channel is no longer empty; the customer assembly merges its
|
| 373 |
+
# measure columns here for exactly the same reason and through the same door.
|
| 374 |
+
derived = aios_grid.cohort_cells(lists)
|
| 375 |
+
for pid, cells in _measure_cells(fields, rows_src, team_id, today, stamp,
|
| 376 |
+
session.runtime.measure_memo, measures).items():
|
| 377 |
+
derived.setdefault(pid, {}).update(cells)
|
| 378 |
+
|
| 379 |
+
# β THE CONDITION CHANNEL SHIPS WITH THE OFFER, NOT AFTER IT. `routes_grid` derives
|
| 380 |
+
# `measure_keys` from this same list, so the moment `measures` is non-empty the client's
|
| 381 |
+
# filter builder offers measure conditions β and an id missing from `measureSets` renders as
|
| 382 |
+
# PENDING ("Calculatingβ¦") and matches nothing, permanently. Serving the offer without this
|
| 383 |
+
# would trade one honest absence for a spinner that never resolves.
|
| 384 |
+
measure_sets = _measure_condition_sets(views, measures, rows_src, team_id, today)
|
| 385 |
+
|
| 386 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
| 387 |
"fields": fields, "views": views, "lists": lists,
|
| 388 |
+
"derived": derived,
|
| 389 |
+
"measures": measures, "measure_sets": measure_sets, "today": today,
|
|
|
|
|
|
|
| 390 |
"team_id": team_id}
|
| 391 |
|
| 392 |
|
api/routes_query.py
CHANGED
|
@@ -12,6 +12,7 @@ import io
|
|
| 12 |
import json
|
| 13 |
import os
|
| 14 |
import re
|
|
|
|
| 15 |
import uuid
|
| 16 |
|
| 17 |
from fastapi import APIRouter, Body, Depends, Response
|
|
@@ -44,24 +45,37 @@ MAX_HISTORY_CHARS = 6000
|
|
| 44 |
MAX_RATING_REASON = 200
|
| 45 |
QUERY_RATINGS = ("up", "down")
|
| 46 |
MODEL_AUTO = "auto"
|
| 47 |
-
#
|
| 48 |
-
#
|
| 49 |
-
#
|
| 50 |
-
#
|
| 51 |
-
#
|
| 52 |
-
#
|
| 53 |
-
#
|
| 54 |
-
#
|
| 55 |
-
#
|
| 56 |
-
#
|
| 57 |
-
# and
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
_LADDER_DRIFT = []
|
| 66 |
QUERY_KINDS = ("grid", "chart", "calendar", "kanban", "timeseries", "map", "list")
|
| 67 |
QUERY_WORKSPACE_EVENTS = {"view_create", "view_upsert", "view_delete"}
|
|
@@ -124,11 +138,37 @@ def _new_id(prefix):
|
|
| 124 |
|
| 125 |
def model_choices():
|
| 126 |
"""Choices are stable even when one is not configured, so explicit means explicit."""
|
| 127 |
-
return [MODEL_AUTO, *
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
|
| 129 |
|
| 130 |
def model_status():
|
| 131 |
-
"""Per choice: can this deployment actually CALL it, and if not, why not.
|
| 132 |
|
| 133 |
ββ THE SAME DEFECT THE SOURCE CHIPS WERE FIXED FOR IN WAVE 33, ONE CONTROL TO THE LEFT.
|
| 134 |
`permitted` is not `answerable` for a database; OFFERED is not CALLABLE for a model. The picker
|
|
@@ -136,40 +176,59 @@ def model_status():
|
|
| 136 |
spent a click and a turn to be told "the selected model is unavailable" by the server, which
|
| 137 |
knew before the click. The reason travels with the flag, exactly as `sources` does.
|
| 138 |
|
| 139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
cause a tenant user can act on (ask an admin); the variable name is our deployment's business.
|
| 141 |
"""
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
|
| 155 |
|
| 156 |
def _providers(model=MODEL_AUTO):
|
| 157 |
-
"""Auto returns the permitted ladder; an explicit choice returns at most one provider.
|
| 158 |
-
import harness.analyst as analyst
|
| 159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
requested = str(model or MODEL_AUTO).strip().lower()
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
_LADDER_DRIFT[:] = sorted(
|
| 167 |
-
set(by_name) - set(QUERY_PROVIDER_ENV)
|
| 168 |
-
| {name for name, row in by_name.items()
|
| 169 |
-
if name in QUERY_PROVIDER_ENV and row.get("env") != QUERY_PROVIDER_ENV[name]})
|
| 170 |
-
names = list(QUERY_PROVIDER_ORDER) if requested == MODEL_AUTO else [requested]
|
| 171 |
-
return [by_name[name] for name in names
|
| 172 |
-
if name in by_name and os.environ.get(by_name[name]["env"])]
|
| 173 |
|
| 174 |
|
| 175 |
def _history(state, thread_id):
|
|
@@ -266,6 +325,100 @@ def _spec_schema(field_keys):
|
|
| 266 |
}
|
| 267 |
|
| 268 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
def _system_prompt(snapshot):
|
| 270 |
fields = snapshot["fields"]
|
| 271 |
cols = json.dumps([{"key": field["key"], "label": field.get("label", field["key"]),
|
|
@@ -298,7 +451,7 @@ To compare one column against ANOTHER column rather than a typed value, give the
|
|
| 298 |
"priced below what it costs us". Both columns must be in the FIELDS list above.
|
| 299 |
Set important=true when the view is one somebody should be chased about: an error, a mismatch, or
|
| 300 |
money at risk. Leave it out otherwise.
|
| 301 |
-
|
| 302 |
Write with ordinary punctuation. Never use an em dash or an en dash: use a comma, a colon or a full
|
| 303 |
stop instead."""
|
| 304 |
|
|
@@ -363,6 +516,78 @@ def _from_chat(answer, provider):
|
|
| 363 |
return answer, None, provider, None
|
| 364 |
|
| 365 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, session=None):
|
| 367 |
"""Return ``(spec, said, provider, reason)`` without any source-data fallback.
|
| 368 |
|
|
@@ -398,9 +623,23 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s
|
|
| 398 |
|
| 399 |
providers = _providers(requested)
|
| 400 |
if not providers:
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 404 |
|
| 405 |
import requests
|
| 406 |
|
|
@@ -413,9 +652,9 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s
|
|
| 413 |
a failed one has already SPENT tokens at that vendor; booking only the winner would report
|
| 414 |
a week cheaper than it was, which is the cost-surprise R13 cited. Every 200 this loop reads
|
| 415 |
gets a line, including the one whose answer turned out to be empty.
|
| 416 |
-
β `tokens_from`
|
| 417 |
-
|
| 418 |
-
|
| 419 |
β `record` never raises, by its own contract, so this cannot break the assistant.
|
| 420 |
"""
|
| 421 |
ins, outs = usage_ledger.tokens_from(body)
|
|
@@ -424,63 +663,68 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s
|
|
| 424 |
total=usage_ledger.total_from(body),
|
| 425 |
st=getattr(session, "runtime", None), user=getattr(session, "uname", ""))
|
| 426 |
|
| 427 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 428 |
for provider in providers:
|
|
|
|
| 429 |
try:
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
# could not ask one back, and could not decline to build. "The AI chat must TALK
|
| 436 |
-
# with the user, not only build queries, and decide for itself whether to create a
|
| 437 |
-
# query view" is unreachable with the tool forced, whatever the prompt says.
|
| 438 |
-
# β AND IT IS A LIVE VENDOR-BEHAVIOUR CHANGE THAT NO GATE HERE CAN SEE, because
|
| 439 |
-
# `verify_query` injects `chat` and never builds this body. `_spec_from_400` below
|
| 440 |
-
# exists precisely because providers differ in how they emit a forced call; under
|
| 441 |
-
# `auto` a provider may also decline to build for a view-shaped question. That is
|
| 442 |
-
# why this ticket's evidence is a real three-turn run against a real provider and
|
| 443 |
-
# not a green gate.
|
| 444 |
-
json={"model": provider["model"], "messages": messages, "tools": tools,
|
| 445 |
-
"tool_choice": "auto", "temperature": 0.1, "max_tokens": 1200},
|
| 446 |
-
)
|
| 447 |
-
except Exception as exc:
|
| 448 |
-
last = f"{provider['name']}: {type(exc).__name__}"
|
| 449 |
continue
|
| 450 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 451 |
spec = _spec_from_400(response.text)
|
| 452 |
if spec is not None:
|
| 453 |
return spec, None, provider["name"], None
|
| 454 |
-
|
| 455 |
-
continue
|
| 456 |
if response.status_code != 200:
|
| 457 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
continue
|
|
|
|
| 459 |
try:
|
| 460 |
body = response.json()
|
| 461 |
# C7: booked from the RAW body, before anything below can raise on its shape. A
|
| 462 |
# provider that answered 200 and billed for it has spent tokens whether or not this
|
| 463 |
# function can read what it said.
|
| 464 |
_book(provider, body)
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
|
|
|
| 469 |
# The prose rides ALONG with the view when the model wrote both, so the reader gets
|
| 470 |
# a sentence instead of `_explain`'s machine description of its own output.
|
| 471 |
-
spec = json.loads(calls[0]["function"].get("arguments") or "{}")
|
| 472 |
return spec, said, provider["name"], None
|
| 473 |
if said:
|
| 474 |
return None, said, provider["name"], None
|
| 475 |
# β Neither a call nor a word is a FAILED turn, not a silent one: falling through to
|
| 476 |
# the next provider is right, and swallowing it as an empty answer would show the
|
| 477 |
# reader a blank reply [[empty-answer-vs-unfinished-answer]].
|
| 478 |
-
|
| 479 |
-
except Exception as exc:
|
| 480 |
-
|
|
|
|
| 481 |
if requested != MODEL_AUTO:
|
| 482 |
-
return None, f"the selected model ({requested})
|
| 483 |
-
|
|
|
|
|
|
|
| 484 |
None, "provider_unreachable")
|
| 485 |
|
| 486 |
|
|
|
|
| 12 |
import json
|
| 13 |
import os
|
| 14 |
import re
|
| 15 |
+
import sys
|
| 16 |
import uuid
|
| 17 |
|
| 18 |
from fastapi import APIRouter, Body, Depends, Response
|
|
|
|
| 45 |
MAX_RATING_REASON = 200
|
| 46 |
QUERY_RATINGS = ("up", "down")
|
| 47 |
MODEL_AUTO = "auto"
|
| 48 |
+
# ββ WAVE 36 Β· W36-T35 (R4 / contract C5) β THE LADDER IS DECLARED IN ONE PLACE NOW.
|
| 49 |
+
#
|
| 50 |
+
# β THIS BLOCK USED TO BE A SECOND COPY OF THE LADDER, and it said so: *"A SECOND DECLARATION IS A
|
| 51 |
+
# DIVERGENCE WAITING TO HAPPEN."* It existed because reading the platform ladder meant
|
| 52 |
+
# `import harness.analyst`, MEASURED at **5 to 8 seconds** warm (it pulls `harness.tools`, which
|
| 53 |
+
# pulls yaml and the skill recipes) β eight seconds on the first paint of the AI module, to answer
|
| 54 |
+
# "which models can this deployment call?".
|
| 55 |
+
#
|
| 56 |
+
# `providers.LLM_PROVIDERS` is now that answer: a plain declaration in a module this file already
|
| 57 |
+
# imports, with no yaml and no skill recipes behind it. So the copy is DELETED rather than kept in
|
| 58 |
+
# step β one list, one door (C5), and the eight seconds go with it.
|
| 59 |
+
# β `harness.analyst.PROVIDERS` still exists and is still the ANALYST's ladder. `_ladder_drift()`
|
| 60 |
+
# reconciles the two WITHOUT forcing the import, so a divergence is still reported and nobody pays
|
| 61 |
+
# the eight seconds to hear about it.
|
| 62 |
+
import providers as prov # noqa: E402
|
| 63 |
+
|
| 64 |
+
#: The Messages API version header. β A DATE, NOT A MODEL: it pins the wire format, not the model,
|
| 65 |
+
#: and Anthropic requires it on every request.
|
| 66 |
+
ANTHROPIC_VERSION = "2023-06-01"
|
| 67 |
+
#: β THINKING IS ON BY DEFAULT ON THIS MODEL AND `max_tokens` CAPS THINKING PLUS TEXT TOGETHER, so
|
| 68 |
+
#: a budget sized around the answer alone truncates mid-thought. It is deliberately NOT disabled:
|
| 69 |
+
#: with thinking off, the model occasionally writes a tool call into its visible TEXT instead of
|
| 70 |
+
#: emitting a `tool_use` block, and the turn then reports success having called nothing β the exact
|
| 71 |
+
#: silent failure this ticket exists to remove.
|
| 72 |
+
ANTHROPIC_MAX_TOKENS = int(os.environ.get("AIOS_ANTHROPIC_MAX_TOKENS") or 8000)
|
| 73 |
+
#: `low` | `medium` | `high` | `xhigh` | `max`. Medium is the balance for a short, scoped
|
| 74 |
+
#: view-building turn; raise it per deployment if the specs come back shallow.
|
| 75 |
+
ANTHROPIC_EFFORT = os.environ.get("AIOS_ANTHROPIC_EFFORT") or "medium"
|
| 76 |
+
|
| 77 |
+
#: Anything the ANALYST's ladder carries that this one does not, once something has already paid to
|
| 78 |
+
#: import it. Empty means "nothing known", never "clean".
|
| 79 |
_LADDER_DRIFT = []
|
| 80 |
QUERY_KINDS = ("grid", "chart", "calendar", "kanban", "timeseries", "map", "list")
|
| 81 |
QUERY_WORKSPACE_EVENTS = {"view_create", "view_upsert", "view_delete"}
|
|
|
|
| 138 |
|
| 139 |
def model_choices():
|
| 140 |
"""Choices are stable even when one is not configured, so explicit means explicit."""
|
| 141 |
+
return [MODEL_AUTO, *(p.name for p in prov.LLM_PROVIDERS.values())]
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _ladder_drift():
|
| 145 |
+
"""Does the ANALYST's ladder carry a provider this one does not? Answered only if it is free.
|
| 146 |
+
|
| 147 |
+
β IT MUST NEVER FORCE `import harness.analyst`. That import is 5-8 seconds warm and this
|
| 148 |
+
function is reached from `GET /query`, which both AI surfaces call on mount. Reading
|
| 149 |
+
`sys.modules` means the reconciliation happens for whoever already paid, and costs nothing for
|
| 150 |
+
everybody else β a check that is free is a check that stays on [[limit-with-no-enforcer]].
|
| 151 |
+
"""
|
| 152 |
+
analyst = sys.modules.get("harness.analyst")
|
| 153 |
+
if analyst is None:
|
| 154 |
+
return list(_LADDER_DRIFT)
|
| 155 |
+
rows = [row for row in getattr(analyst, "PROVIDERS", []) if isinstance(row, dict)]
|
| 156 |
+
known = prov.LLM_PROVIDERS
|
| 157 |
+
# β TWO KINDS OF DRIFT, AND THE SECOND IS THE QUIET ONE. A provider the analyst gained is
|
| 158 |
+
# ABSENT here, which a reader can at least notice. An env name that changed on one side leaves
|
| 159 |
+
# the provider listed and its availability answered from the wrong variable β present, wrong,
|
| 160 |
+
# and silent. Both were checked when this file held its own env map; both are still checked now
|
| 161 |
+
# that `providers.LLM_PROVIDERS` holds it, because the two ladders can still disagree.
|
| 162 |
+
_LADDER_DRIFT[:] = sorted(
|
| 163 |
+
{str(row.get("name")) for row in rows} - set(known)
|
| 164 |
+
| {str(row.get("name")) for row in rows
|
| 165 |
+
if str(row.get("name")) in known
|
| 166 |
+
and str(row.get("env") or "") != known[str(row.get("name"))].env})
|
| 167 |
+
return list(_LADDER_DRIFT)
|
| 168 |
|
| 169 |
|
| 170 |
def model_status():
|
| 171 |
+
"""Per choice: can this deployment actually CALL it, and if not, why not (C5).
|
| 172 |
|
| 173 |
ββ THE SAME DEFECT THE SOURCE CHIPS WERE FIXED FOR IN WAVE 33, ONE CONTROL TO THE LEFT.
|
| 174 |
`permitted` is not `answerable` for a database; OFFERED is not CALLABLE for a model. The picker
|
|
|
|
| 176 |
spent a click and a turn to be told "the selected model is unavailable" by the server, which
|
| 177 |
knew before the click. The reason travels with the flag, exactly as `sources` does.
|
| 178 |
|
| 179 |
+
β W36-T35 ADDS THREE REASONS THE FIRST VERSION COULD NOT EXPRESS, and each is a distinct fact
|
| 180 |
+
a picker has to be able to say out loud:
|
| 181 |
+
* `outOfCredit` β configured, capable, and SKIPPED because it told us its balance is empty
|
| 182 |
+
(R4). Before this, an empty account was indistinguishable from a working one until the
|
| 183 |
+
turn came back with `HTTP 402` on the screen.
|
| 184 |
+
* `toolCalling` β DECLARED per provider. W36-T34's `done-when` is that a model the ladder
|
| 185 |
+
declares incapable of tool calling is not offered for a tool-calling turn; that sentence
|
| 186 |
+
needs a declaration to read, and this is it.
|
| 187 |
+
* `jsonMode` β the same shape for the other capability staged item 3 named.
|
| 188 |
+
|
| 189 |
+
β IT NAMES NO ENVIRONMENT VARIABLE. "This model is not configured on this deployment" is the
|
| 190 |
cause a tenant user can act on (ask an admin); the variable name is our deployment's business.
|
| 191 |
"""
|
| 192 |
+
rows_by_name = {row["provider"]: row for row in prov.llm_status()}
|
| 193 |
+
live = [name for name, row in rows_by_name.items()
|
| 194 |
+
if row["configured"] and row["capable"] and not row["outOfCredit"]]
|
| 195 |
+
out = [{"model": MODEL_AUTO, "available": bool(live),
|
| 196 |
+
"reason": "" if live else "no model can be reached from this deployment right now",
|
| 197 |
+
"toolCalling": True, "jsonMode": True, "outOfCredit": False}]
|
| 198 |
+
for name, row in rows_by_name.items():
|
| 199 |
+
if not row["configured"]:
|
| 200 |
+
reason = "this model is not configured on this deployment"
|
| 201 |
+
elif row["outOfCredit"]:
|
| 202 |
+
reason = f"{row['label']} is out of credit, so it is being skipped for now"
|
| 203 |
+
elif not row["capable"]:
|
| 204 |
+
reason = f"{row['label']} cannot carry this kind of request"
|
| 205 |
+
else:
|
| 206 |
+
reason = ""
|
| 207 |
+
out.append({"model": name, "available": name in live, "reason": reason,
|
| 208 |
+
"toolCalling": row["toolCalling"], "jsonMode": row["jsonMode"],
|
| 209 |
+
"outOfCredit": row["outOfCredit"]})
|
| 210 |
+
# Anything the ANALYST's ladder carries that Query does not offer, once a chat has taught us.
|
| 211 |
+
out.extend({"model": name, "available": False, "toolCalling": False, "jsonMode": False,
|
| 212 |
+
"outOfCredit": False,
|
| 213 |
+
"reason": "the platform can reach this model but Query does not offer it yet"}
|
| 214 |
+
for name in _ladder_drift())
|
| 215 |
+
return out
|
| 216 |
|
| 217 |
|
| 218 |
def _providers(model=MODEL_AUTO):
|
| 219 |
+
"""Auto returns the permitted ladder; an explicit choice returns at most one provider.
|
|
|
|
| 220 |
|
| 221 |
+
β THE FILTERING IS `providers.llm_chain`'s, NOT THIS FUNCTION'S β declared-capable AND
|
| 222 |
+
configured AND not inside an out-of-credit cooldown, in the ladder's own order (R4). Re-deriving
|
| 223 |
+
any of those three here would be a second opinion about which model to call, and the first
|
| 224 |
+
place it would disagree is the one nobody watches.
|
| 225 |
+
"""
|
| 226 |
requested = str(model or MODEL_AUTO).strip().lower()
|
| 227 |
+
chain = prov.llm_chain("llm_tool_calling")
|
| 228 |
+
if requested != MODEL_AUTO:
|
| 229 |
+
chain = [p for p in chain if p.name == requested]
|
| 230 |
+
return [{"name": p.name, "label": p.label, "env": p.env, "url": p.url,
|
| 231 |
+
"model": p.model, "wire": p.wire} for p in chain]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
|
| 233 |
|
| 234 |
def _history(state, thread_id):
|
|
|
|
| 325 |
}
|
| 326 |
|
| 327 |
|
| 328 |
+
# ββββββββββ WAVE 36 Β· W36-T40 β THE GROUNDING CORPUS THIS PRODUCT ALREADY OWNED ββββββββββββββββ
|
| 329 |
+
#
|
| 330 |
+
# ββ 25 SENTENCES OF THE OWNER'S OWN SEMANTICS WERE SITTING ON DISK, UNREAD BY THE ASSISTANT.
|
| 331 |
+
# `platform/model/metrics/*.yml` carries an `ai_context` on 25 of its 26 registered metrics: what
|
| 332 |
+
# "revenue" means here (untaxed, Amazon excluded), that an AR balance takes no date window, that a
|
| 333 |
+
# returns figure arriving negative means a lost negation rather than no returns. They were written
|
| 334 |
+
# for the AIOS Analyst, which EXIT-6 deleted, and nothing has read them since.
|
| 335 |
+
#
|
| 336 |
+
# β THEY ARE VOCABULARY, NOT FIELDS, AND THE PROMPT HAS TO SAY SO. The topics they describe
|
| 337 |
+
# (`sales_lines`, `receivables`, `gl_lines`, ...) are NOT databases this assistant can read: its
|
| 338 |
+
# sources are the two built-ins plus the `ut_*` tables (`deps.assistant_source_status`). Handing a
|
| 339 |
+
# model 25 confident notes about `team_id` and `price_subtotal` without that fence is an invitation
|
| 340 |
+
# to filter on a column that is not in the snapshot.
|
| 341 |
+
#
|
| 342 |
+
# ββ AND NO FIGURE INSIDE THEM IS EVER AN ANSWER. Five of the 25 carry a measured landmark
|
| 343 |
+
# (`Fisch 2025 = 518`, `$1,447,579.59 vs $1,086,381.35`, `~$192k`, `$3,187,571`, `~2% apart`),
|
| 344 |
+
# because a human analyst reading a metric definition wants the order of magnitude. A model that
|
| 345 |
+
# repeats one has put an aggregate on a screen that no `validate()` ever tied back to Odoo, which
|
| 346 |
+
# is the single thing standing rule 8 forbids. β THE PROMPT SAYING SO IS NOT THE CONTROL: this
|
| 347 |
+
# file already measured a prompt instruction failing on the very next turn (see `_no_dashes`), so
|
| 348 |
+
# `verify_query` asks the live model for one of those numbers and asserts it does not come back.
|
| 349 |
+
#
|
| 350 |
+
# β ONE READER, NOT TWO. `harness.semantic` is the governed registry, already parsed, validated
|
| 351 |
+
# and lru_cached, and `rollup_sql`/`routes_tables` reach it by this same lazy import. Measured
|
| 352 |
+
# after `import main`: the import costs **0.003 s** (its whole dependency set, `core.odoo`, `yaml`
|
| 353 |
+
# and `modules.sales` included, is already resident) and the first `metrics()` call 0.195 s, once.
|
| 354 |
+
# Nothing like the 5 to 8 seconds `harness.analyst` cost, which is why the ladder above stopped
|
| 355 |
+
# reading that one and why this does not need a second YAML parser of its own.
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def _grounding_notes():
|
| 359 |
+
"""`[(topic, key, label, sentence), ...]` β this tenant's metric vocabulary, dash free.
|
| 360 |
+
|
| 361 |
+
β FAIL SOFT, AND THE GATE IS WHAT MAKES THAT SAFE. A missing or malformed corpus returns []
|
| 362 |
+
and the assistant answers without the vocabulary, because turning a YAML typo into a dead chat
|
| 363 |
+
door is worse than answering with less. That is precisely the silent degradation
|
| 364 |
+
`deploy_web.py` warns about in the comment above its own upload of these files, so
|
| 365 |
+
`verify_query` derives the expected count FROM the YAML and asserts the built prompt carries
|
| 366 |
+
exactly that many: the corpus may go missing, but it may not go missing quietly.
|
| 367 |
+
|
| 368 |
+
β `_no_dashes` ON THE WAY IN, and it is load bearing. 11 of the 25 sentences carry an em dash,
|
| 369 |
+
and the prompt they are joining ends with *"Never use an em dash or an en dash"*. Shipping them
|
| 370 |
+
raw would demonstrate the forbidden style eleven times in the same breath as the ban. β AND
|
| 371 |
+
`web_prose` STRUCTURALLY CANNOT SEE IT: that gate reads Python string VALUES off the AST, and
|
| 372 |
+
these strings live in a data file, so prose loaded at runtime is a hole in rule 6's scanner.
|
| 373 |
+
"""
|
| 374 |
+
try:
|
| 375 |
+
from harness import semantic as sem
|
| 376 |
+
registry = sem.metrics()
|
| 377 |
+
except Exception:
|
| 378 |
+
# Named metrics are a data-layer concern; a chat turn is not the place to surface a
|
| 379 |
+
# corpus problem to a person who asked about their own table.
|
| 380 |
+
return []
|
| 381 |
+
notes = []
|
| 382 |
+
for key, metric in (registry or {}).items():
|
| 383 |
+
if not isinstance(metric, dict):
|
| 384 |
+
continue
|
| 385 |
+
sentence = _no_dashes(str(metric.get("ai_context") or "")).strip()
|
| 386 |
+
if sentence:
|
| 387 |
+
# β THE LABEL GETS THE SAME TREATMENT, and it is not belt-and-braces: measured on
|
| 388 |
+
# this corpus, `sales_orders.orders_invoiced` is labelled "Orders β fully invoiced"
|
| 389 |
+
# and TWO em dashes reached the built prompt while every sentence was clean. One
|
| 390 |
+
# field of a row scrubbed and its neighbour not is [[one-question-two-normalizers]]
|
| 391 |
+
# at the smallest possible scale, so the rule applies to the whole row.
|
| 392 |
+
notes.append((str(metric.get("topic") or ""), str(key),
|
| 393 |
+
_no_dashes(str(metric.get("label") or key)).strip(), sentence))
|
| 394 |
+
return notes
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def _grounding_block():
|
| 398 |
+
"""The system-prompt section, or "" when this deployment has no corpus to state."""
|
| 399 |
+
notes = _grounding_notes()
|
| 400 |
+
if not notes:
|
| 401 |
+
return ""
|
| 402 |
+
lines = "\n".join(f"- {topic}.{key} ({label}): {sentence}"
|
| 403 |
+
for topic, key, label, sentence in notes)
|
| 404 |
+
return f"""
|
| 405 |
+
BUSINESS VOCABULARY ({len(notes)} notes this tenant wrote about their own reporting):
|
| 406 |
+
{lines}
|
| 407 |
+
|
| 408 |
+
Read those for what a WORD means here: which rows a number counts, which reading of an ambiguous
|
| 409 |
+
question is the house one, what a figure deliberately leaves out. They describe the finance topics,
|
| 410 |
+
not the database above, so never treat a name in that list as a field and never filter on anything
|
| 411 |
+
absent from FIELDS.
|
| 412 |
+
|
| 413 |
+
Some notes quote a MEASUREMENT taken in an earlier year: a total, a customer count, a gap between
|
| 414 |
+
two figures. Never repeat one of those to the reader, not even labelled as old or approximate. A
|
| 415 |
+
number on screen is read as the answer whatever sits beside it, and nothing checked that one today.
|
| 416 |
+
When this database cannot produce the figure somebody asked for, say only that. Codes and formats
|
| 417 |
+
in the notes are different and you may use them freely: a team_id, a percentage scale, which field
|
| 418 |
+
carries what. Every number you state comes from build_view over the snapshot above.
|
| 419 |
+
"""
|
| 420 |
+
|
| 421 |
+
|
| 422 |
def _system_prompt(snapshot):
|
| 423 |
fields = snapshot["fields"]
|
| 424 |
cols = json.dumps([{"key": field["key"], "label": field.get("label", field["key"]),
|
|
|
|
| 451 |
"priced below what it costs us". Both columns must be in the FIELDS list above.
|
| 452 |
Set important=true when the view is one somebody should be chased about: an error, a mismatch, or
|
| 453 |
money at risk. Leave it out otherwise.
|
| 454 |
+
{_grounding_block()}
|
| 455 |
Write with ordinary punctuation. Never use an em dash or an en dash: use a comma, a colon or a full
|
| 456 |
stop instead."""
|
| 457 |
|
|
|
|
| 516 |
return answer, None, provider, None
|
| 517 |
|
| 518 |
|
| 519 |
+
# ββββββββββββββββββ WAVE 36 Β· W36-T35 β TWO WIRES, ONE LOOP (ruling R4) βββββββββββββββββββββββββ
|
| 520 |
+
#
|
| 521 |
+
# β THE SECOND WIRE IS NOT A SECOND CODE PATH BY CHOICE β Anthropic's Messages API genuinely is a
|
| 522 |
+
# different request: `x-api-key` instead of a bearer, the system prompt as a TOP-LEVEL field rather
|
| 523 |
+
# than a message, `input_schema` instead of `function.parameters`, and typed content BLOCKS instead
|
| 524 |
+
# of `choices[0].message`. What is kept single is everything around it: one ladder, one loop, one
|
| 525 |
+
# meter, one set of sentences. The wire is a pair of pure functions the loop dispatches on.
|
| 526 |
+
#
|
| 527 |
+
# β WHY RAW HTTP AND NOT THE `anthropic` SDK, which is the documented default. The SDK is not in
|
| 528 |
+
# `aios-web/api/requirements.txt`, so it is NOT in the deployed image β adding it is a dependency
|
| 529 |
+
# on a Space this lane cannot rebuild or test, mid-wave. `requests` is already there and is already
|
| 530 |
+
# how the three sibling providers in this same loop are called. Booked for the integrator as a
|
| 531 |
+
# `PENDING:`; the day it lands, `_anthropic_request`/`_anthropic_read` are the two functions that
|
| 532 |
+
# go, and nothing else moves.
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
def _openai_request(provider, messages, tools):
|
| 536 |
+
"""The OpenAI-compatible `/chat/completions` shape β cerebras, groq, openrouter."""
|
| 537 |
+
return {
|
| 538 |
+
"url": provider["url"],
|
| 539 |
+
"headers": {"Authorization": f"Bearer {os.environ[provider['env']]}"},
|
| 540 |
+
"json": {"model": provider["model"], "messages": messages, "tools": tools,
|
| 541 |
+
# ββ `auto`, NOT `required`, AND THAT ONE WORD IS R16. Under `required` the model
|
| 542 |
+
# had to emit a `build_view` call for EVERY turn: it could not answer a question,
|
| 543 |
+
# could not ask one back, and could not decline to build. "The AI chat must TALK
|
| 544 |
+
# with the user, not only build queries, and decide for itself whether to create a
|
| 545 |
+
# query view" is unreachable with the tool forced, whatever the prompt says.
|
| 546 |
+
"tool_choice": "auto", "temperature": 0.1, "max_tokens": 1200},
|
| 547 |
+
}
|
| 548 |
+
|
| 549 |
+
|
| 550 |
+
def _anthropic_request(provider, messages, tools):
|
| 551 |
+
"""The Messages API shape β built by `providers.anthropic_request`, never re-derived here.
|
| 552 |
+
|
| 553 |
+
β ONE WIRE, TWO DOORS. `ai_review.draft_flow` calls the same builder; the owner quoted an
|
| 554 |
+
error from each of these paths in one sentence, and two implementations of one API is the
|
| 555 |
+
defect this product keeps finding under a different name.
|
| 556 |
+
β THIS DOOR SENDS `effort` AND THE DRAFTER DOES NOT β `output_config.effort` errors on Haiku
|
| 557 |
+
4.5, which is the tier the drafter runs on. Model-gated, so it is the caller's call.
|
| 558 |
+
"""
|
| 559 |
+
return prov.anthropic_request(
|
| 560 |
+
model=provider["model"], key=os.environ[provider["env"]],
|
| 561 |
+
system=None, messages=messages, tools=tools,
|
| 562 |
+
max_tokens=ANTHROPIC_MAX_TOKENS, tool_choice="auto", effort=ANTHROPIC_EFFORT)
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
def _openai_read(body):
|
| 566 |
+
"""`(spec, said, refusal)` out of an OpenAI-compatible answer."""
|
| 567 |
+
answer = ((body.get("choices") or [{}])[0] or {}).get("message") or {}
|
| 568 |
+
said = _said(answer.get("content"))
|
| 569 |
+
calls = answer.get("tool_calls") or []
|
| 570 |
+
if calls:
|
| 571 |
+
try:
|
| 572 |
+
spec = json.loads(calls[0]["function"].get("arguments") or "{}")
|
| 573 |
+
except (TypeError, ValueError, KeyError):
|
| 574 |
+
return None, said, None
|
| 575 |
+
return (spec if isinstance(spec, dict) else None), said, None
|
| 576 |
+
return None, said, None
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
def _anthropic_read(body):
|
| 580 |
+
"""`(spec, said, refusal)` β the shared reader, with this door's own dash normaliser applied."""
|
| 581 |
+
text, spec, refused = prov.anthropic_read(body)
|
| 582 |
+
if refused:
|
| 583 |
+
return None, None, refused
|
| 584 |
+
return spec, _said(text), None
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
_WIRES = {"openai": (_openai_request, _openai_read),
|
| 588 |
+
"anthropic": (_anthropic_request, _anthropic_read)}
|
| 589 |
+
|
| 590 |
+
|
| 591 |
def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, session=None):
|
| 592 |
"""Return ``(spec, said, provider, reason)`` without any source-data fallback.
|
| 593 |
|
|
|
|
| 623 |
|
| 624 |
providers = _providers(requested)
|
| 625 |
if not providers:
|
| 626 |
+
# β THREE DIFFERENT EMPTINESSES, THREE DIFFERENT SENTENCES (R4). "Not configured" and
|
| 627 |
+
# "every account is out of credit" are the same empty list and completely different
|
| 628 |
+
# problems: one needs an administrator, the other needs a card. Answering both with one
|
| 629 |
+
# sentence is how the owner ended up reading `HTTP 402` to find out which.
|
| 630 |
+
broke = [row for row in prov.llm_status() if row["outOfCredit"]]
|
| 631 |
+
if requested != MODEL_AUTO:
|
| 632 |
+
row = next((r for r in prov.llm_status() if r["provider"] == requested), None)
|
| 633 |
+
if row and row["outOfCredit"]:
|
| 634 |
+
sentence = f"{row['label']} is out of credit, so it cannot answer right now"
|
| 635 |
+
else:
|
| 636 |
+
sentence = f"the selected model ({requested}) is unavailable"
|
| 637 |
+
return None, sentence, None, "model_unavailable"
|
| 638 |
+
if broke:
|
| 639 |
+
names = ", ".join(row["label"] for row in broke)
|
| 640 |
+
return (None, f"every model this deployment can reach is out of credit ({names}). "
|
| 641 |
+
f"Top one up and it will answer again", None, "no_credit")
|
| 642 |
+
return None, "the assistant is not configured on this deployment", None, "not_configured"
|
| 643 |
|
| 644 |
import requests
|
| 645 |
|
|
|
|
| 652 |
a failed one has already SPENT tokens at that vendor; booking only the winner would report
|
| 653 |
a week cheaper than it was, which is the cost-surprise R13 cited. Every 200 this loop reads
|
| 654 |
gets a line, including the one whose answer turned out to be empty.
|
| 655 |
+
β `tokens_from` reads `input_tokens`/`output_tokens` as well as the OpenAI spellings, so
|
| 656 |
+
the Messages API body is measured by the SAME reader with no branch here. That is not luck
|
| 657 |
+
β it is why that function takes a key LIST.
|
| 658 |
β `record` never raises, by its own contract, so this cannot break the assistant.
|
| 659 |
"""
|
| 660 |
ins, outs = usage_ledger.tokens_from(body)
|
|
|
|
| 663 |
total=usage_ledger.total_from(body),
|
| 664 |
st=getattr(session, "runtime", None), user=getattr(session, "uname", ""))
|
| 665 |
|
| 666 |
+
# β EVERY FAILURE IS COLLECTED, NOT JUST THE LAST ONE. The owner's own complaint was a LIST
|
| 667 |
+
# ("cerebras: HTTP 402; groq: HTTP 404; openrouter: HTTP 402") and the list is the useful part
|
| 668 |
+
# β it is what tells an operator whether one vendor is down or the account is empty everywhere.
|
| 669 |
+
# What changes is that each entry is now a sentence rather than a status code.
|
| 670 |
+
trouble = []
|
| 671 |
for provider in providers:
|
| 672 |
+
build, read = _WIRES.get(provider["wire"], _WIRES["openai"])
|
| 673 |
try:
|
| 674 |
+
request = build(provider, messages, tools)
|
| 675 |
+
response = requests.post(request["url"], timeout=90,
|
| 676 |
+
headers=request["headers"], json=request["json"])
|
| 677 |
+
except Exception as exc: # noqa: BLE001
|
| 678 |
+
trouble.append(f"{provider['label']} could not be reached ({type(exc).__name__})")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 679 |
continue
|
| 680 |
+
|
| 681 |
+
if response.status_code == 400 and provider["wire"] == "openai":
|
| 682 |
+
# `_FAILED_GEN`'s recovery, kept for the wire that needs it and NOT extended to the one
|
| 683 |
+
# that does not: Anthropic answers a tool call as a typed block or not at all, so a 400
|
| 684 |
+
# there is a malformed REQUEST and recovering a spec out of it would be reading tea
|
| 685 |
+
# leaves.
|
| 686 |
spec = _spec_from_400(response.text)
|
| 687 |
if spec is not None:
|
| 688 |
return spec, None, provider["name"], None
|
| 689 |
+
|
|
|
|
| 690 |
if response.status_code != 200:
|
| 691 |
+
# ββ R4's SECOND CLAUSE, AND THE ONLY PLACE IT CAN BE LEARNED: a provider only tells
|
| 692 |
+
# you the balance is empty by refusing. The memo is written here so the NEXT turn skips
|
| 693 |
+
# it instead of paying the round trip again.
|
| 694 |
+
if prov.is_credit_failure(response.status_code, response.text):
|
| 695 |
+
prov.mark_no_credit(provider["name"])
|
| 696 |
+
trouble.append(prov.refusal_sentence(provider["name"], response.status_code,
|
| 697 |
+
response.text))
|
| 698 |
continue
|
| 699 |
+
|
| 700 |
try:
|
| 701 |
body = response.json()
|
| 702 |
# C7: booked from the RAW body, before anything below can raise on its shape. A
|
| 703 |
# provider that answered 200 and billed for it has spent tokens whether or not this
|
| 704 |
# function can read what it said.
|
| 705 |
_book(provider, body)
|
| 706 |
+
spec, said, refused = read(body)
|
| 707 |
+
if refused:
|
| 708 |
+
trouble.append(f"{provider['label']}: {refused}")
|
| 709 |
+
continue
|
| 710 |
+
if spec is not None:
|
| 711 |
# The prose rides ALONG with the view when the model wrote both, so the reader gets
|
| 712 |
# a sentence instead of `_explain`'s machine description of its own output.
|
|
|
|
| 713 |
return spec, said, provider["name"], None
|
| 714 |
if said:
|
| 715 |
return None, said, provider["name"], None
|
| 716 |
# β Neither a call nor a word is a FAILED turn, not a silent one: falling through to
|
| 717 |
# the next provider is right, and swallowing it as an empty answer would show the
|
| 718 |
# reader a blank reply [[empty-answer-vs-unfinished-answer]].
|
| 719 |
+
trouble.append(f"{provider['label']} answered with nothing")
|
| 720 |
+
except Exception as exc: # noqa: BLE001
|
| 721 |
+
trouble.append(f"{provider['label']} sent something unreadable "
|
| 722 |
+
f"({type(exc).__name__})")
|
| 723 |
if requested != MODEL_AUTO:
|
| 724 |
+
return (None, f"the selected model ({requested}) could not answer: "
|
| 725 |
+
+ (trouble[0] if trouble else "it did not respond"), None, "model_unavailable")
|
| 726 |
+
return (None, "the assistant could not be reached just now: "
|
| 727 |
+
+ ("; ".join(trouble) if trouble else "no provider answered"),
|
| 728 |
None, "provider_unreachable")
|
| 729 |
|
| 730 |
|
api/routes_script_views.py
ADDED
|
@@ -0,0 +1,439 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_script_views.py β CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10).
|
| 2 |
+
|
| 3 |
+
Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user
|
| 4 |
+
can build whatever they want through the Agent chat interface. be able to create any dashboard
|
| 5 |
+
they want. User should have the ability to see the code AND the dashboard output of course⦠Limit
|
| 6 |
+
the code script View per database⦠Any agent can add into more AI script, so we can see different
|
| 7 |
+
versions or different things the AI code for us."*
|
| 8 |
+
|
| 9 |
+
GET /api/v1/script-views?database=K the views bound to ONE database
|
| 10 |
+
POST /api/v1/script-views create one {database, name?, source}
|
| 11 |
+
GET /api/v1/script-views/{id} one view, its source and its history
|
| 12 |
+
PUT /api/v1/script-views/{id} a NEW VERSION of the source
|
| 13 |
+
DELETE /api/v1/script-views/{id} drop it
|
| 14 |
+
POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms}
|
| 15 |
+
POST /api/v1/script-views/{id}/revert go back to an earlier version {version}
|
| 16 |
+
|
| 17 |
+
ββ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may
|
| 18 |
+
read ONLY the database it lives in, and a script that names another database is REFUSED with a
|
| 19 |
+
message naming both. *Not capped*: there is **no limit on how many script views a database may
|
| 20 |
+
carry**, because that is how an agent offers three attempts and the owner picks one. So nothing
|
| 21 |
+
below counts views. What IS bounded is what makes them big β one source is capped, and one view's
|
| 22 |
+
edit history is capped and REPORTS what it dropped.
|
| 23 |
+
|
| 24 |
+
β **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed
|
| 25 |
+
`session.user`, so a script written by an administrator and opened by a scoped analyst reads the
|
| 26 |
+
ANALYST's rows. The author decides what the code does; the reader decides what it can see.
|
| 27 |
+
β And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script
|
| 28 |
+
that exfiltrates anything, because the only thing a script can return is a render spec drawn on
|
| 29 |
+
the screen of the person who ran it. There is no network, no file and no second reader.
|
| 30 |
+
|
| 31 |
+
β **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a
|
| 32 |
+
coroutine that would block the event loop for every other request in the container. FastAPI runs a
|
| 33 |
+
plain `def` in the threadpool, which is what makes one slow script one slow REQUEST.
|
| 34 |
+
"""
|
| 35 |
+
import threading
|
| 36 |
+
from datetime import datetime, timezone
|
| 37 |
+
|
| 38 |
+
from fastapi import APIRouter, Body, Depends
|
| 39 |
+
|
| 40 |
+
from deps import Session, err, require_session
|
| 41 |
+
|
| 42 |
+
router = APIRouter(prefix="/api/v1")
|
| 43 |
+
|
| 44 |
+
#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`.
|
| 45 |
+
VIEWS_KEY = "script_views"
|
| 46 |
+
|
| 47 |
+
MAX_NAME = 80
|
| 48 |
+
MAX_SOURCE_BYTES = 128 * 1024
|
| 49 |
+
|
| 50 |
+
#: Edit history per view. β NOT a cap on the NUMBER of views (R3 forbids that) β a cap on how far
|
| 51 |
+
#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader
|
| 52 |
+
#: can see the history is partial instead of concluding the view was only ever saved twice.
|
| 53 |
+
MAX_HISTORY = 40
|
| 54 |
+
|
| 55 |
+
#: β HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal
|
| 56 |
+
#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this,
|
| 57 |
+
#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what
|
| 58 |
+
#: gives up. A 429 that says so is honest; an unbounded fork is not.
|
| 59 |
+
MAX_CONCURRENT_RUNS = 4
|
| 60 |
+
_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _now():
|
| 64 |
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _all(rt):
|
| 68 |
+
"""`{id: record}` for one tenant. `{}` on any failure β an unreadable bucket must degrade to
|
| 69 |
+
"this database has no script views", never to a 500 on the view rail."""
|
| 70 |
+
try:
|
| 71 |
+
found = rt.get(VIEWS_KEY) or {}
|
| 72 |
+
except Exception: # noqa: BLE001
|
| 73 |
+
return {}
|
| 74 |
+
return found if isinstance(found, dict) else {}
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _database_ok(session, database):
|
| 78 |
+
"""Does this database EXIST, and may this caller read it? Answered by C1, never by a list.
|
| 79 |
+
|
| 80 |
+
β `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True
|
| 81 |
+
for an ADMIN on any key at all, including one no database answers to. So a create validated
|
| 82 |
+
with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing
|
| 83 |
+
can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it
|
| 84 |
+
RAISES `UnknownTable`, which is exactly the question being asked.
|
| 85 |
+
"""
|
| 86 |
+
import core.perm_scope as perm_scope
|
| 87 |
+
try:
|
| 88 |
+
perm_scope.scoped_fields(session.user, database, st=session.runtime)
|
| 89 |
+
except perm_scope.UnknownTable:
|
| 90 |
+
raise err(404, "no_database", f"there is no database '{database}' in this workspace")
|
| 91 |
+
except perm_scope.Denied:
|
| 92 |
+
raise err(403, "forbidden", f"your account may not read '{database}'")
|
| 93 |
+
except perm_scope.Unresolvable as exc:
|
| 94 |
+
# The database is real and cannot be served under this call's constraints. Standing rule
|
| 95 |
+
# 1's second sentence: report the cause and the recommendation, never a bare refusal.
|
| 96 |
+
raise err(409, "unresolvable", str(exc)) from None
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _clean_source(raw, *, allow_empty=False):
|
| 100 |
+
"""The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point.
|
| 101 |
+
|
| 102 |
+
ββ W37-T41 β WHY CREATE MAY BE EMPTY AND SAVE MAY NOT.
|
| 103 |
+
Picking "Custom View" in the mode picker mints the view immediately, before a line of code
|
| 104 |
+
exists, so that `mode === 'script'` always implies a real id and no reader has to carry a
|
| 105 |
+
"the id might be missing" branch (the create semantics handed to lane C in mailbox E-7).
|
| 106 |
+
A view that has been created and not yet written is therefore a REAL, legible state: the editor
|
| 107 |
+
shows its starter placeholder and the Run control is right there.
|
| 108 |
+
A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a
|
| 109 |
+
working script by saving nothing over it is not an edit anybody means to make. So the guard
|
| 110 |
+
stays exactly where it was on that door, and a person who wants the view gone deletes it.
|
| 111 |
+
|
| 112 |
+
β THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns
|
| 113 |
+
None, so "an empty script is storable" looked true and was written into a mailbox answer another
|
| 114 |
+
lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox
|
| 115 |
+
knows nothing about. Two validators for one question, disagreeing
|
| 116 |
+
[[one-question-two-normalizers]] β and the one that would have bitten a person is the one no
|
| 117 |
+
unit of this feature was asserting.
|
| 118 |
+
"""
|
| 119 |
+
source = str(raw or "")
|
| 120 |
+
if not source.strip() and not allow_empty:
|
| 121 |
+
raise err(400, "no_source", "a script view needs some code")
|
| 122 |
+
if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
|
| 123 |
+
raise err(413, "source_too_long",
|
| 124 |
+
f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code")
|
| 125 |
+
return source
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _row(rec, *, source=False):
|
| 129 |
+
"""One view as the list door reports it. β NO SOURCE unless asked: the rail lists names."""
|
| 130 |
+
out = {"id": rec.get("id") or "", "database": rec.get("database") or "",
|
| 131 |
+
"name": rec.get("name") or "", "author": rec.get("author") or "",
|
| 132 |
+
"version": int(rec.get("version") or 1),
|
| 133 |
+
"created": rec.get("created") or "", "updated": rec.get("updated") or "",
|
| 134 |
+
"versions": len(rec.get("history") or []) + 1,
|
| 135 |
+
"trimmed": int(rec.get("trimmed") or 0),
|
| 136 |
+
# β W37-T46: which version this one was RESTORED from, when it was. Present on the
|
| 137 |
+
# row (not only the history) because the editor's header is where a person reads "what
|
| 138 |
+
# am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back
|
| 139 |
+
# legible as an event rather than as a coincidence of matching code.
|
| 140 |
+
"restoredFrom": rec.get("restoredFrom")}
|
| 141 |
+
if source:
|
| 142 |
+
out["source"] = rec.get("source") or ""
|
| 143 |
+
out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "",
|
| 144 |
+
"created": h.get("created") or "",
|
| 145 |
+
"bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))}
|
| 146 |
+
for h in reversed(rec.get("history") or []) if isinstance(h, dict)]
|
| 147 |
+
return out
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _limits():
|
| 151 |
+
return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME,
|
| 152 |
+
"maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS,
|
| 153 |
+
# β SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong.
|
| 154 |
+
"maxViewsPerDatabase": None}
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _put(session, view_id, mutate):
|
| 158 |
+
"""Read-modify-write ONE view, synchronously β the client re-reads the rail immediately."""
|
| 159 |
+
def _set(cur):
|
| 160 |
+
cur = dict(cur or {})
|
| 161 |
+
nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None)
|
| 162 |
+
if nxt is None:
|
| 163 |
+
cur.pop(view_id, None)
|
| 164 |
+
else:
|
| 165 |
+
cur[view_id] = nxt
|
| 166 |
+
return cur
|
| 167 |
+
|
| 168 |
+
session.runtime.update(VIEWS_KEY, _set, flush="sync")
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _mine_or_admin(session, rec):
|
| 172 |
+
"""Who may EDIT or DELETE a view: its author, or an administrator.
|
| 173 |
+
|
| 174 |
+
β RUNNING IS A DIFFERENT QUESTION and deliberately wider β anybody who may read the database
|
| 175 |
+
may run any view on it, under their OWN scope. That is the whole of "so we can see different
|
| 176 |
+
versions or different things the AI code for us": a colleague's attempt is worth nothing if
|
| 177 |
+
only its author can open it.
|
| 178 |
+
"""
|
| 179 |
+
if session.admin or str(rec.get("author") or "") == session.uname:
|
| 180 |
+
return
|
| 181 |
+
raise err(403, "forbidden", "only the author or an administrator can change this script view")
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
# ββ the routes ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 185 |
+
@router.get("/script-views")
|
| 186 |
+
def list_script_views(database: str = "", session: Session = Depends(require_session)):
|
| 187 |
+
"""Every script view bound to ONE database, newest first. `database` is required."""
|
| 188 |
+
key = str(database or "").strip()
|
| 189 |
+
if not key:
|
| 190 |
+
raise err(400, "no_database", "name the database whose script views you want")
|
| 191 |
+
_database_ok(session, key)
|
| 192 |
+
rows = [_row(rec) for rec in _all(session.runtime).values()
|
| 193 |
+
if isinstance(rec, dict) and str(rec.get("database") or "") == key]
|
| 194 |
+
rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True)
|
| 195 |
+
return {"database": key, "views": rows, "limits": _limits()}
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
@router.post("/script-views")
|
| 199 |
+
def create_script_view(body: dict = Body(default=None),
|
| 200 |
+
session: Session = Depends(require_session)):
|
| 201 |
+
"""Create one. β THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time.
|
| 202 |
+
|
| 203 |
+
A script view that cannot be run is a broken feature the reader discovers by pressing a button,
|
| 204 |
+
and the agent that wrote it is long gone by then. `check_source` is pure and costs no process,
|
| 205 |
+
so the refusal arrives while the author still has the code in front of them.
|
| 206 |
+
"""
|
| 207 |
+
import secrets # noqa: PLC0415
|
| 208 |
+
import core.script_sandbox as sandbox # noqa: PLC0415
|
| 209 |
+
|
| 210 |
+
body = body if isinstance(body, dict) else {}
|
| 211 |
+
database = str(body.get("database") or "").strip()
|
| 212 |
+
if not database:
|
| 213 |
+
raise err(400, "no_database", "a script view is bound to one database")
|
| 214 |
+
_database_ok(session, database)
|
| 215 |
+
# β `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the
|
| 216 |
+
# moment a person chooses the mode, so the empty source is the normal first state, not a slip.
|
| 217 |
+
source = _clean_source(body.get("source"), allow_empty=True)
|
| 218 |
+
refusal = sandbox.check_source(source)
|
| 219 |
+
if refusal is not None:
|
| 220 |
+
raise err(400, refusal.code, refusal.message)
|
| 221 |
+
|
| 222 |
+
view_id = "sv_" + secrets.token_urlsafe(9)
|
| 223 |
+
rec = {"id": view_id, "database": database,
|
| 224 |
+
"name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME],
|
| 225 |
+
"source": source, "author": session.uname, "version": 1,
|
| 226 |
+
"created": _now(), "updated": _now(), "history": [], "trimmed": 0}
|
| 227 |
+
_put(session, view_id, lambda _prior: rec)
|
| 228 |
+
fresh = _all(session.runtime).get(view_id)
|
| 229 |
+
if not isinstance(fresh, dict):
|
| 230 |
+
# The store took the write and did not record it. A 200 here would tell the author their
|
| 231 |
+
# script was saved when it was not.
|
| 232 |
+
raise err(503, "store_unavailable", "the script view was NOT created")
|
| 233 |
+
return {"view": _row(fresh, source=True), "limits": _limits()}
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
@router.get("/script-views/{view_id}")
|
| 237 |
+
def get_script_view(view_id: str, session: Session = Depends(require_session)):
|
| 238 |
+
"""One view WITH its source and its edit history. Owner item 6's *"see the code"* half."""
|
| 239 |
+
rec = _all(session.runtime).get(str(view_id))
|
| 240 |
+
if not isinstance(rec, dict):
|
| 241 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 242 |
+
_database_ok(session, str(rec.get("database") or ""))
|
| 243 |
+
return {"view": _row(rec, source=True), "limits": _limits()}
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
@router.put("/script-views/{view_id}")
|
| 247 |
+
def update_script_view(view_id: str, body: dict = Body(default=None),
|
| 248 |
+
session: Session = Depends(require_session)):
|
| 249 |
+
"""A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place."""
|
| 250 |
+
import core.script_sandbox as sandbox # noqa: PLC0415
|
| 251 |
+
|
| 252 |
+
view_id = str(view_id)
|
| 253 |
+
rec = _all(session.runtime).get(view_id)
|
| 254 |
+
if not isinstance(rec, dict):
|
| 255 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 256 |
+
_mine_or_admin(session, rec)
|
| 257 |
+
body = body if isinstance(body, dict) else {}
|
| 258 |
+
source = _clean_source(body.get("source"))
|
| 259 |
+
refusal = sandbox.check_source(source)
|
| 260 |
+
if refusal is not None:
|
| 261 |
+
raise err(400, refusal.code, refusal.message)
|
| 262 |
+
|
| 263 |
+
def _mutate(prior):
|
| 264 |
+
prior = dict(prior or rec)
|
| 265 |
+
history = list(prior.get("history") or [])
|
| 266 |
+
history.append({"version": int(prior.get("version") or 1),
|
| 267 |
+
"source": prior.get("source") or "",
|
| 268 |
+
"author": prior.get("author") or "", "created": prior.get("updated") or ""})
|
| 269 |
+
dropped = max(0, len(history) - MAX_HISTORY)
|
| 270 |
+
prior["history"] = history[dropped:] if dropped else history
|
| 271 |
+
prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
|
| 272 |
+
prior["source"] = source
|
| 273 |
+
prior["version"] = int(prior.get("version") or 1) + 1
|
| 274 |
+
prior["updated"] = _now()
|
| 275 |
+
if body.get("name"):
|
| 276 |
+
prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME]
|
| 277 |
+
return prior
|
| 278 |
+
|
| 279 |
+
_put(session, view_id, _mutate)
|
| 280 |
+
fresh = _all(session.runtime).get(view_id)
|
| 281 |
+
if not isinstance(fresh, dict):
|
| 282 |
+
raise err(503, "store_unavailable", "the new version was NOT saved")
|
| 283 |
+
return {"view": _row(fresh, source=True), "limits": _limits()}
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
@router.post("/script-views/{view_id}/revert")
|
| 287 |
+
def revert_script_view(view_id: str, body: dict = Body(default=None),
|
| 288 |
+
session: Session = Depends(require_session)):
|
| 289 |
+
"""Put one view back to an earlier version. D-338 / W37-T46.
|
| 290 |
+
|
| 291 |
+
ββ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole
|
| 292 |
+
point of the ticket rather than an implementation detail. The history is what lets a person
|
| 293 |
+
trust an agent with their code: if reverting destroyed the versions it stepped over, one
|
| 294 |
+
mistaken revert would cost exactly what the history existed to protect, and there would be no
|
| 295 |
+
way back from the way back. So this appends, and `restoredFrom` records where the text came
|
| 296 |
+
from β the same posture the agent-harness roll-back already takes, so a reader who has seen one
|
| 297 |
+
is not surprised by the other.
|
| 298 |
+
|
| 299 |
+
β IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history
|
| 300 |
+
entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old
|
| 301 |
+
version's text to re-PUT it. Handing the source out just so the client could send it straight
|
| 302 |
+
back would widen a payload for a round trip that does not need to exist, and would make the
|
| 303 |
+
revert non-atomic: two calls, and a failure between them leaves the person looking at code that
|
| 304 |
+
is not what is stored.
|
| 305 |
+
β `_clean_source` is NOT re-run. The text being restored was already validated when it was
|
| 306 |
+
first saved, and a source that a later, stricter rule would now reject is exactly the source a
|
| 307 |
+
person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is
|
| 308 |
+
about what the code would DO and that must never be bypassed by a route.
|
| 309 |
+
"""
|
| 310 |
+
import core.script_sandbox as sandbox # noqa: PLC0415
|
| 311 |
+
|
| 312 |
+
view_id = str(view_id)
|
| 313 |
+
rec = _all(session.runtime).get(view_id)
|
| 314 |
+
if not isinstance(rec, dict):
|
| 315 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 316 |
+
_mine_or_admin(session, rec)
|
| 317 |
+
body = body if isinstance(body, dict) else {}
|
| 318 |
+
try:
|
| 319 |
+
want = int(body.get("version"))
|
| 320 |
+
except (TypeError, ValueError):
|
| 321 |
+
raise err(400, "no_version", "say which version to go back to") from None
|
| 322 |
+
|
| 323 |
+
history = [h for h in (rec.get("history") or []) if isinstance(h, dict)]
|
| 324 |
+
match = next((h for h in history if int(h.get("version") or 0) == want), None)
|
| 325 |
+
if match is None:
|
| 326 |
+
# β TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never
|
| 327 |
+
# written; the other was TRIMMED off the 40-deep history and the record even counts how
|
| 328 |
+
# many. Saying "that version is gone" for the first would be a lie a person could act on.
|
| 329 |
+
trimmed = int(rec.get("trimmed") or 0)
|
| 330 |
+
if trimmed and want <= trimmed:
|
| 331 |
+
raise err(410, "version_trimmed",
|
| 332 |
+
f"version {want} is older than this view's history keeps "
|
| 333 |
+
f"({trimmed} earlier versions have been dropped)")
|
| 334 |
+
raise err(404, "no_version", f"this view has no version {want}")
|
| 335 |
+
|
| 336 |
+
source = str(match.get("source") or "")
|
| 337 |
+
refusal = sandbox.check_source(source)
|
| 338 |
+
if refusal is not None:
|
| 339 |
+
# An older version that today's sandbox refuses. The person is told which version and why,
|
| 340 |
+
# rather than being handed a 400 about code they did not just type.
|
| 341 |
+
raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}")
|
| 342 |
+
|
| 343 |
+
def _mutate(prior):
|
| 344 |
+
prior = dict(prior or rec)
|
| 345 |
+
history_now = list(prior.get("history") or [])
|
| 346 |
+
history_now.append({"version": int(prior.get("version") or 1),
|
| 347 |
+
"source": prior.get("source") or "",
|
| 348 |
+
"author": prior.get("author") or "",
|
| 349 |
+
"created": prior.get("updated") or ""})
|
| 350 |
+
dropped = max(0, len(history_now) - MAX_HISTORY)
|
| 351 |
+
prior["history"] = history_now[dropped:] if dropped else history_now
|
| 352 |
+
prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
|
| 353 |
+
prior["source"] = source
|
| 354 |
+
prior["version"] = int(prior.get("version") or 1) + 1
|
| 355 |
+
prior["updated"] = _now()
|
| 356 |
+
# β ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that
|
| 357 |
+
# mysteriously matches an older one. Without it a reader sees v5 and v2 with identical
|
| 358 |
+
# code and no way to tell a revert from a coincidence.
|
| 359 |
+
prior["restoredFrom"] = want
|
| 360 |
+
return prior
|
| 361 |
+
|
| 362 |
+
_put(session, view_id, _mutate)
|
| 363 |
+
fresh = _all(session.runtime).get(view_id)
|
| 364 |
+
if not isinstance(fresh, dict):
|
| 365 |
+
raise err(503, "store_unavailable", "the roll-back was NOT saved")
|
| 366 |
+
return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()}
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
@router.delete("/script-views/{view_id}")
|
| 370 |
+
def delete_script_view(view_id: str, session: Session = Depends(require_session)):
|
| 371 |
+
view_id = str(view_id)
|
| 372 |
+
rec = _all(session.runtime).get(view_id)
|
| 373 |
+
if not isinstance(rec, dict):
|
| 374 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 375 |
+
_mine_or_admin(session, rec)
|
| 376 |
+
_put(session, view_id, lambda _prior: None)
|
| 377 |
+
return {"deleted": view_id}
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
@router.post("/script-views/{view_id}/run")
|
| 381 |
+
def run_script_view(view_id: str, body: dict = Body(default=None),
|
| 382 |
+
session: Session = Depends(require_session)):
|
| 383 |
+
"""CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`.
|
| 384 |
+
|
| 385 |
+
β `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser
|
| 386 |
+
executes β the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key
|
| 387 |
+
before this function ever sees it, so a renderer cannot be talked into running something by a
|
| 388 |
+
script that was itself perfectly well behaved.
|
| 389 |
+
|
| 390 |
+
β AND IT IS PLAIN `def`, NOT `async def` β see this module's header. A ten-second subprocess
|
| 391 |
+
wait on the event loop is a ten-second outage for the whole container.
|
| 392 |
+
"""
|
| 393 |
+
import core.script_sandbox as sandbox # noqa: PLC0415
|
| 394 |
+
|
| 395 |
+
rec = _all(session.runtime).get(str(view_id))
|
| 396 |
+
if not isinstance(rec, dict):
|
| 397 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 398 |
+
database = str(rec.get("database") or "")
|
| 399 |
+
# A DRAFT run: the editor sends unsaved code so the author can try it before committing to it.
|
| 400 |
+
# It is checked exactly as a stored one is, because "unsaved" is not a permission.
|
| 401 |
+
draft = (body or {}).get("source") if isinstance(body, dict) else None
|
| 402 |
+
source = _clean_source(draft) if draft else str(rec.get("source") or "")
|
| 403 |
+
# β NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and
|
| 404 |
+
# asking twice builds a registry topic's pool twice. The three refusals are translated below
|
| 405 |
+
# instead, which is the same wall reached through the same door.
|
| 406 |
+
|
| 407 |
+
if not _RUN_SLOTS.acquire(blocking=False):
|
| 408 |
+
raise err(429, "busy",
|
| 409 |
+
f"{MAX_CONCURRENT_RUNS} script views are already running on this server. "
|
| 410 |
+
f"Try again in a moment")
|
| 411 |
+
try:
|
| 412 |
+
out = sandbox.run_view(session.user, database, source, st=session.runtime)
|
| 413 |
+
finally:
|
| 414 |
+
_RUN_SLOTS.release()
|
| 415 |
+
|
| 416 |
+
# β C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database"
|
| 417 |
+
# answered 200 would be a permission decision the client has to go looking for, and every
|
| 418 |
+
# other door in this app answers 403 for it. Everything BELOW this line is a well-formed
|
| 419 |
+
# request whose ANSWER is that the script did not produce a view β that is a 200 with
|
| 420 |
+
# `ok:false`, the shape `routes_web_agent.test` already uses for the same reason.
|
| 421 |
+
if out.get("code") == "unknown_table":
|
| 422 |
+
raise err(404, "no_database", out.get("message") or f"there is no database '{database}'")
|
| 423 |
+
if out.get("code") == "denied":
|
| 424 |
+
raise err(403, "forbidden", out.get("message") or "your account may not read that database")
|
| 425 |
+
if out.get("code") == "unresolvable":
|
| 426 |
+
refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served")
|
| 427 |
+
refusal.detail["error"]["limit"] = out.get("limit") or {}
|
| 428 |
+
raise refusal
|
| 429 |
+
|
| 430 |
+
answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"),
|
| 431 |
+
"stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")),
|
| 432 |
+
"ms": int(out.get("ms") or 0), "code": out.get("code") or "",
|
| 433 |
+
# β `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits
|
| 434 |
+
# were applied; on a Windows host the memory and CPU ones were not, and a screen
|
| 435 |
+
# that claims an enforcement which did not happen is the failure the rule is about.
|
| 436 |
+
"caps": out.get("caps") or {}}
|
| 437 |
+
if not answer["ok"]:
|
| 438 |
+
answer["error"] = out.get("message") or "the script view did not produce a view"
|
| 439 |
+
return answer
|
api/routes_shares.py
CHANGED
|
@@ -4,9 +4,19 @@
|
|
| 4 |
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
| 6 |
|
| 7 |
-
`kind` β view | folder | database. Roles are `view` | `edit` β the same two words the
|
| 8 |
-
already speaks, now extended to folders and
|
| 9 |
-
(R10: "the same picker views use").
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
β **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
|
| 12 |
`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
|
|
@@ -102,6 +112,24 @@ def _owns_object(session, kind, oid):
|
|
| 102 |
"""
|
| 103 |
if session.admin:
|
| 104 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
if kind == "database":
|
| 106 |
# β `may_open` is THE resolver for a user table (its own docstring says so) and already
|
| 107 |
# admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
|
|
@@ -157,6 +185,10 @@ def _can_see_object(session, kind, oid):
|
|
| 157 |
if kind != "view":
|
| 158 |
# A folder carries no per-object visibility flag of its own, and a database's `may_open`
|
| 159 |
# (inside `_owns_object`) already admits grantees. Nothing wider to ask.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
return False
|
| 161 |
try:
|
| 162 |
import core.table_store as table_store
|
|
@@ -195,7 +227,7 @@ def _entries_or_400(session, entries):
|
|
| 195 |
raise err(400, "unknown_people",
|
| 196 |
"no active account in this workspace is named "
|
| 197 |
+ ", ".join(sorted(set(unknown)))
|
| 198 |
-
+ "
|
| 199 |
|
| 200 |
|
| 201 |
@router.get("/share/mine")
|
|
@@ -216,6 +248,12 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
|
| 216 |
role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
|
| 217 |
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 218 |
st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
# β W32-T26 (audit S-3) β A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 220 |
# who owns it, everyone it is granted to, and the tenant's whole usernameβname directory β
|
| 221 |
# to any signed-in session, about objects it cannot open. Now a caller with no role on an
|
|
@@ -286,7 +324,7 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
|
| 286 |
entries = body.get("entries")
|
| 287 |
if not isinstance(entries, list):
|
| 288 |
raise err(400, "bad_entries",
|
| 289 |
-
"entries must be a list of {user, role}
|
| 290 |
"revoking is expressed")
|
| 291 |
_entries_or_400(session, entries)
|
| 292 |
out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
|
|
@@ -367,6 +405,32 @@ def _object_ref(session, kind, oid):
|
|
| 367 |
a row that opens nowhere.
|
| 368 |
"""
|
| 369 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 370 |
if kind == "database":
|
| 371 |
import core.user_tables as ut
|
| 372 |
defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
|
|
@@ -390,4 +454,5 @@ def _object_ref(session, kind, oid):
|
|
| 390 |
route, str(oid) if kind == "view" else "")
|
| 391 |
except Exception: # noqa: BLE001
|
| 392 |
pass
|
| 393 |
-
return ({"view": "A view", "folder": "A folder"
|
|
|
|
|
|
| 4 |
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
| 6 |
|
| 7 |
+
`kind` β view | folder | database | field. Roles are `view` | `edit` β the same two words the
|
| 8 |
+
view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE
|
| 9 |
+
vocabulary in the UI (R10: "the same picker views use").
|
| 10 |
+
|
| 11 |
+
ββ **W38-T16 β `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `"<table_key>:<field_key>"`**
|
| 12 |
+
(`shares.field_oid`). A bare column key repeats across databases β `notes` exists on a dozen β
|
| 13 |
+
so a grant stored under one would admit the grantee to every `notes` column in the tenant at
|
| 14 |
+
once. β THREE functions in this file branch on kind and ALL THREE need the new one, which is not
|
| 15 |
+
obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator
|
| 16 |
+
is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None,
|
| 17 |
+
`_notify_new_grantees` returns early, and the grantee is **granted and never told** β owner item
|
| 18 |
+
18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for
|
| 19 |
+
anything that is not a view.
|
| 20 |
|
| 21 |
β **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
|
| 22 |
`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
|
|
|
|
| 112 |
"""
|
| 113 |
if session.admin:
|
| 114 |
return True
|
| 115 |
+
if kind == "field":
|
| 116 |
+
# ββ W38-T16 β A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS
|
| 117 |
+
# (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 /
|
| 118 |
+
# D-172: creator-or-admin). Read from the same place by all three, so a column cannot be
|
| 119 |
+
# deletable by one person and shareable by another.
|
| 120 |
+
# β THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION:
|
| 121 |
+
# a brand-new column has no grant record, so `put_share` falls to this predicate β and
|
| 122 |
+
# without it the column's own creator is answered `404 no_object` on the first attempt to
|
| 123 |
+
# share the thing they just made.
|
| 124 |
+
table_key, field_key = shares.split_field_oid(oid)
|
| 125 |
+
if not table_key:
|
| 126 |
+
return False
|
| 127 |
+
try:
|
| 128 |
+
import core.shared_overlay as shared_overlay
|
| 129 |
+
defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key)
|
| 130 |
+
except Exception: # noqa: BLE001
|
| 131 |
+
return False
|
| 132 |
+
return bool(defn) and str(defn.get("createdBy") or "") == str(session.uname)
|
| 133 |
if kind == "database":
|
| 134 |
# β `may_open` is THE resolver for a user table (its own docstring says so) and already
|
| 135 |
# admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
|
|
|
|
| 185 |
if kind != "view":
|
| 186 |
# A folder carries no per-object visibility flag of its own, and a database's `may_open`
|
| 187 |
# (inside `_owns_object`) already admits grantees. Nothing wider to ask.
|
| 188 |
+
# β W38-T16 β AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here:
|
| 189 |
+
# `get_share` tests `role is None` first and a grant answers a role, so the only caller
|
| 190 |
+
# left is an account with no relationship to the column at all. Widening it would let any
|
| 191 |
+
# signed-in session enumerate who holds which column on a database they cannot open.
|
| 192 |
return False
|
| 193 |
try:
|
| 194 |
import core.table_store as table_store
|
|
|
|
| 227 |
raise err(400, "unknown_people",
|
| 228 |
"no active account in this workspace is named "
|
| 229 |
+ ", ".join(sorted(set(unknown)))
|
| 230 |
+
+ ". Nothing was shared. Pick people from the list rather than typing a name.")
|
| 231 |
|
| 232 |
|
| 233 |
@router.get("/share/mine")
|
|
|
|
| 248 |
role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
|
| 249 |
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 250 |
st=session.runtime)
|
| 251 |
+
# W39-T29 β the dialog opens with GET before its first PUT. Until that PUT exists the grant
|
| 252 |
+
# registry has no owner to return from `may_administer`, even though the same caller may safely
|
| 253 |
+
# claim their own object through the PUT path below. Reflect that exact claim predicate here:
|
| 254 |
+
# a Member who owns an unshared View receives the people picker; a collaborator still does not.
|
| 255 |
+
if not rec["owner"] and not may_admin and _owns_object(session, kind, oid):
|
| 256 |
+
may_admin = True
|
| 257 |
# β W32-T26 (audit S-3) β A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 258 |
# who owns it, everyone it is granted to, and the tenant's whole usernameβname directory β
|
| 259 |
# to any signed-in session, about objects it cannot open. Now a caller with no role on an
|
|
|
|
| 324 |
entries = body.get("entries")
|
| 325 |
if not isinstance(entries, list):
|
| 326 |
raise err(400, "bad_entries",
|
| 327 |
+
"entries must be a list of {user, role}. Send [] to un-share, which is how "
|
| 328 |
"revoking is expressed")
|
| 329 |
_entries_or_400(session, entries)
|
| 330 |
out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
|
|
|
|
| 405 |
a row that opens nowhere.
|
| 406 |
"""
|
| 407 |
try:
|
| 408 |
+
if kind == "field":
|
| 409 |
+
# ββ W38-T16 β A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it
|
| 410 |
+
# is a column INSIDE a database, so the target that travels is the DATABASE. Without
|
| 411 |
+
# this branch the function falls through to the view/folder loop, finds nothing,
|
| 412 |
+
# answers `route=None` β and `_notify_new_grantees` returns EARLY. The grant lands and
|
| 413 |
+
# the receiver is never told, which is the silent half of owner item 18 reopened one
|
| 414 |
+
# kind over.
|
| 415 |
+
from routes_alerts import route_for_topic
|
| 416 |
+
table_key, field_key = shares.split_field_oid(oid)
|
| 417 |
+
if not table_key:
|
| 418 |
+
return ("A column", None, "")
|
| 419 |
+
try:
|
| 420 |
+
import core.shared_overlay as shared_overlay
|
| 421 |
+
defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key)
|
| 422 |
+
except Exception: # noqa: BLE001
|
| 423 |
+
defn = None
|
| 424 |
+
label = str((defn or {}).get("label") or "").strip() or field_key
|
| 425 |
+
# β TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed
|
| 426 |
+
# by whatever the calling door already held: a `ut_*` database uses its bare key,
|
| 427 |
+
# while a registry topic uses `<topic>_table_workspace` (`product_data.TABLE_KEY`).
|
| 428 |
+
# `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not
|
| 429 |
+
# `customer_data`), so the suffix comes off before it is asked β rather than a second
|
| 430 |
+
# route table being written here, which is how the two come apart.
|
| 431 |
+
_WS = "_table_workspace"
|
| 432 |
+
scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key
|
| 433 |
+
return (label, route_for_topic(scope) or None, "")
|
| 434 |
if kind == "database":
|
| 435 |
import core.user_tables as ut
|
| 436 |
defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
|
|
|
|
| 454 |
route, str(oid) if kind == "view" else "")
|
| 455 |
except Exception: # noqa: BLE001
|
| 456 |
pass
|
| 457 |
+
return ({"view": "A view", "folder": "A folder",
|
| 458 |
+
"field": "A column"}.get(kind, "An item"), None, "")
|
api/routes_slack.py
CHANGED
|
@@ -229,7 +229,10 @@ def slack_creds(rt):
|
|
| 229 |
def _summary(agent, modules):
|
| 230 |
"""One sentence per agent for the list row β computed here, because the alternative is one
|
| 231 |
round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
|
| 232 |
-
|
|
|
|
|
|
|
|
|
|
| 233 |
perms = agent.get("perms") or {}
|
| 234 |
open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
|
| 235 |
if not governed:
|
|
@@ -304,16 +307,26 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
|
|
| 304 |
if not isinstance(a, dict):
|
| 305 |
raise err(404, "no_such_agent", "no agent with that id")
|
| 306 |
modules = routes_admin._perm_modules(session)
|
| 307 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
stored = a.get("perms") or {}
|
| 309 |
principal = agent_principal(a)
|
| 310 |
import core.perm_scope as perm_scope
|
| 311 |
perms_out = {}
|
| 312 |
-
for k in
|
| 313 |
e = stored.get(k)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
perms_out[k] = e if isinstance(e, dict) else {
|
| 315 |
-
"access": bool(perm_scope.
|
| 316 |
-
"hiddenFields": []}
|
| 317 |
return {"id": str(agent_id), "channel": a.get("channel") or "",
|
| 318 |
"channelName": a.get("channelName") or "", "label": a.get("label") or "",
|
| 319 |
"active": a.get("active", True) is not False,
|
|
@@ -326,7 +339,8 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
|
|
| 326 |
# the "Everything (admin)" state for a principal that cannot have it.
|
| 327 |
"is_admin": False,
|
| 328 |
"modules": modules,
|
| 329 |
-
"fields_by_module": {k: routes_admin._module_fields(k
|
|
|
|
| 330 |
|
| 331 |
|
| 332 |
@router.put("/agents/{agent_id}/perms")
|
|
@@ -341,14 +355,16 @@ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None),
|
|
| 341 |
raise err(400, "empty_patch", "no perms to save")
|
| 342 |
mods = routes_admin._perm_modules(session)
|
| 343 |
# β THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
|
| 344 |
-
# evaluate, refuses a hiddenFields key that names nothing, refuses
|
| 345 |
-
#
|
| 346 |
-
#
|
| 347 |
-
#
|
|
|
|
|
|
|
|
|
|
| 348 |
cleaned = routes_admin._clean_perms(
|
| 349 |
body.get("perms"),
|
| 350 |
-
|
| 351 |
-
listed_keys={m["key"] for m in mods}) or {}
|
| 352 |
|
| 353 |
import core.perm_scope as perm_scope
|
| 354 |
|
|
|
|
| 229 |
def _summary(agent, modules):
|
| 230 |
"""One sentence per agent for the list row β computed here, because the alternative is one
|
| 231 |
round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
|
| 232 |
+
# β W36-T22 / C2 β every listed database is governed now; the `enforced` flag it used to
|
| 233 |
+
# filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and
|
| 234 |
+
# summarised every agent as "" (no databases at all).
|
| 235 |
+
governed = list(modules)
|
| 236 |
perms = agent.get("perms") or {}
|
| 237 |
open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
|
| 238 |
if not governed:
|
|
|
|
| 307 |
if not isinstance(a, dict):
|
| 308 |
raise err(404, "no_such_agent", "no agent with that id")
|
| 309 |
modules = routes_admin._perm_modules(session)
|
| 310 |
+
# ββ W36-T22 / C2 β EVERY listed database, `ut_*` included. β THIS IS WHY THE FLAG'S
|
| 311 |
+
# DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the
|
| 312 |
+
# key is None, so this list would have been EMPTY and the channel perms editor would have
|
| 313 |
+
# governed ZERO databases while looking entirely correct. A Slack channel is a principal of
|
| 314 |
+
# the SAME wall (`verify_perm_scope` section H) and gets the same catalogue.
|
| 315 |
+
governed = [m["key"] for m in modules]
|
| 316 |
stored = a.get("perms") or {}
|
| 317 |
principal = agent_principal(a)
|
| 318 |
import core.perm_scope as perm_scope
|
| 319 |
perms_out = {}
|
| 320 |
+
for k in governed:
|
| 321 |
e = stored.get(k)
|
| 322 |
+
# β W36-T22 β `may_read`, the same evaluator the read door uses, for the reason spelled
|
| 323 |
+
# out at `routes_admin.get_perms`. β It answers DIFFERENTLY here and correctly so: a
|
| 324 |
+
# channel agent is not a person with a share, so a `ut_*` key it has not been granted
|
| 325 |
+
# defaults CLOSED β which is the fail-closed direction for a bot, and the same answer the
|
| 326 |
+
# table routes would give it.
|
| 327 |
perms_out[k] = e if isinstance(e, dict) else {
|
| 328 |
+
"access": bool(perm_scope.may_read(principal, k, st=session.runtime)),
|
| 329 |
+
"filter": None, "hiddenFields": []}
|
| 330 |
return {"id": str(agent_id), "channel": a.get("channel") or "",
|
| 331 |
"channelName": a.get("channelName") or "", "label": a.get("label") or "",
|
| 332 |
"active": a.get("active", True) is not False,
|
|
|
|
| 339 |
# the "Everything (admin)" state for a principal that cannot have it.
|
| 340 |
"is_admin": False,
|
| 341 |
"modules": modules,
|
| 342 |
+
"fields_by_module": {k: routes_admin._module_fields(k, session=session)
|
| 343 |
+
for k in governed}}
|
| 344 |
|
| 345 |
|
| 346 |
@router.put("/agents/{agent_id}/perms")
|
|
|
|
| 355 |
raise err(400, "empty_patch", "no perms to save")
|
| 356 |
mods = routes_admin._perm_modules(session)
|
| 357 |
# β THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
|
| 358 |
+
# evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the
|
| 359 |
+
# pushdown cannot read. Every one of those is exactly as true for a channel as for a person,
|
| 360 |
+
# and a second validator here is a second place for one of them to go missing.
|
| 361 |
+
# β CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal
|
| 362 |
+
# is DELETED β W36-T21 armed the wall over every database, so there is no unenforced key left
|
| 363 |
+
# to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives
|
| 364 |
+
# it ([[two-gates-can-assert-opposite-things]]).
|
| 365 |
cleaned = routes_admin._clean_perms(
|
| 366 |
body.get("perms"),
|
| 367 |
+
governed_keys={m["key"] for m in mods}, session=session) or {}
|
|
|
|
| 368 |
|
| 369 |
import core.perm_scope as perm_scope
|
| 370 |
|
api/routes_statements.py
CHANGED
|
@@ -129,9 +129,14 @@ def preview(body: dict = Body(default=None), session: Session = Depends(_gate)):
|
|
| 129 |
subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
|
| 130 |
try:
|
| 131 |
subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
|
| 132 |
-
except
|
| 133 |
# An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
|
| 134 |
# can see what they typed rather than getting an opaque error.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
pass
|
| 136 |
return {
|
| 137 |
"html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
|
|
@@ -153,7 +158,31 @@ def send(body: dict = Body(default=None), session: Session = Depends(_gate)):
|
|
| 153 |
names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
|
| 154 |
if not names:
|
| 155 |
raise err(400, "bad_request", "name at least one customer")
|
| 156 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
if override and len(names) != 1:
|
| 158 |
raise err(400, "bad_request", "a test send takes exactly one customer")
|
| 159 |
t = body.get("templates") or {}
|
|
|
|
| 129 |
subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
|
| 130 |
try:
|
| 131 |
subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
|
| 132 |
+
except Exception: # noqa: BLE001
|
| 133 |
# An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
|
| 134 |
# can see what they typed rather than getting an opaque error.
|
| 135 |
+
# β W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE.
|
| 136 |
+
# `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a
|
| 137 |
+
# TypeError, and every one of them is the same user mistake this branch was written for.
|
| 138 |
+
# Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the
|
| 139 |
+
# one screen somebody opens to avoid mailing 200 people the wrong thing.
|
| 140 |
pass
|
| 141 |
return {
|
| 142 |
"html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
|
|
|
|
| 158 |
names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
|
| 159 |
if not names:
|
| 160 |
raise err(400, "bad_request", "name at least one customer")
|
| 161 |
+
# ββ W36-T42 / D-299 β A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A
|
| 162 |
+
# REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent
|
| 163 |
+
# `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space)
|
| 164 |
+
# got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On
|
| 165 |
+
# the one route in this product that is allowed to write to Odoo, the difference between a
|
| 166 |
+
# rehearsal and mailing a live customer was one empty string.
|
| 167 |
+
#
|
| 168 |
+
# β THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo`
|
| 169 |
+
# key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature
|
| 170 |
+
# D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key
|
| 171 |
+
# being present, or `test: true`, is that statement, and an empty address beside it is the
|
| 172 |
+
# mistake. So both spellings of "absent" are covered: the key present and blank, and `test`
|
| 173 |
+
# asserted with no key at all.
|
| 174 |
+
#
|
| 175 |
+
# β THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND.
|
| 176 |
+
# `automationApi.testSendStatement` refuses an empty address before the request is built, and
|
| 177 |
+
# SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client
|
| 178 |
+
# wall is bypassable by construction (anything can POST); this one is not.
|
| 179 |
+
declared_test = ("overrideTo" in body) or bool(body.get("test"))
|
| 180 |
+
override = str(body.get("overrideTo") or "").strip()
|
| 181 |
+
if declared_test and not override:
|
| 182 |
+
raise err(400, "no_override_address",
|
| 183 |
+
"a test send needs the address to send the test to. Without one this would "
|
| 184 |
+
"mail the customer's own address, which is the opposite of a test")
|
| 185 |
+
override = override or None
|
| 186 |
if override and len(names) != 1:
|
| 187 |
raise err(400, "bad_request", "a test send takes exactly one customer")
|
| 188 |
t = body.get("templates") or {}
|
api/routes_tables.py
CHANGED
|
@@ -429,11 +429,36 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 429 |
raise err(400, "bad_request", "a field key is required")
|
| 430 |
_defn_or_refuse(session, table_key)
|
| 431 |
from core import shared_overlay
|
|
|
|
| 432 |
if not shared_overlay.is_shared(table_key, key, st=session.runtime):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 433 |
shared_overlay.put_field(table_key, key, {
|
| 434 |
"key": key, "label": str(body.get("label") or key), "source": "overlay",
|
| 435 |
"type": str(body.get("type") or "text"), "shared": True,
|
|
|
|
| 436 |
"createdBy": session.uname}, st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 437 |
try:
|
| 438 |
# β `put_cell`, not `put_cells` β this door writes exactly ONE cell, and the singular is
|
| 439 |
# the API that says so. It delegates to the plural, so both stay reachable through the one
|
|
@@ -477,6 +502,18 @@ def delete_shared_field(table_key: str, field_key: str,
|
|
| 477 |
f"was added by {owner or 'somebody else'}, and dropping it would delete the "
|
| 478 |
f"value for every account")
|
| 479 |
dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 480 |
return {"ok": True, "dropped": bool(dropped),
|
| 481 |
"fields": list(shared_overlay.fields(table_key, st=session.runtime))}
|
| 482 |
|
|
@@ -617,7 +654,7 @@ def ut_write_ctx(session: Session, table_key: str):
|
|
| 617 |
# `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully
|
| 618 |
# writable on the EVENTS transport β a wall on one wire and not the other is the shape
|
| 619 |
# `strip_row`'s own note warns about, with the sign flipped.
|
| 620 |
-
hidden = _ut_hidden(session, table_key, fields_base)
|
| 621 |
ctx = grid_events.EventCtx(
|
| 622 |
uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden,
|
| 623 |
admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime,
|
|
@@ -626,28 +663,99 @@ def ut_write_ctx(session: Session, table_key: str):
|
|
| 626 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 627 |
ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="",
|
| 628 |
fields_base=fields_base)
|
| 629 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 630 |
return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace,
|
| 631 |
"fields": fields, "views": views, "lists": lists, "hidden": hidden,
|
| 632 |
"derived": aios_grid.cohort_cells(lists),
|
| 633 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 634 |
# β W31-T20 β the write door reads this to refuse a PID-BEARING event loudly rather
|
| 635 |
# than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch.
|
| 636 |
"limits": limits, "defn": defn}
|
| 637 |
|
| 638 |
|
| 639 |
-
def _ut_hidden(session, table_key, fields):
|
| 640 |
"""The hidden-field closure for THIS session on THIS database β C1's field half, once.
|
| 641 |
|
| 642 |
β Named rather than inlined at its four call sites for the reason `may_open`'s own note gives:
|
| 643 |
four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE
|
| 644 |
evaluator; this is just the `ut_*` caller's shorthand for it.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 645 |
"""
|
| 646 |
import core.perm_scope as perm_scope
|
| 647 |
-
return perm_scope.hidden_keys(session.user, table_key, fields)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 648 |
|
|
|
|
|
|
|
| 649 |
|
| 650 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 651 |
"""`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires.
|
| 652 |
|
| 653 |
ββ W36-T21 β the same three lines `routes_customers.grid_assembly` runs for `customer_data`,
|
|
@@ -666,7 +774,7 @@ def _ut_field_wall(session, table_key, fields, rows_src):
|
|
| 666 |
path with nothing to say ([[one-question-two-normalizers]]).
|
| 667 |
"""
|
| 668 |
import core.perm_scope as perm_scope
|
| 669 |
-
hide = perm_scope.hidden_keys(session.user, table_key, fields)
|
| 670 |
if not hide:
|
| 671 |
return fields, rows_src, frozenset()
|
| 672 |
fields = [f for f in fields if f.get("key") not in hide]
|
|
@@ -724,7 +832,7 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 724 |
ops = _ops(session, table_key, st=st)
|
| 725 |
# ββ W36-T21 β the WRITE half of the field wall. `frozenset()` here meant a column an
|
| 726 |
# administrator had hidden was still writable through the events transport.
|
| 727 |
-
hidden = _ut_hidden(session, table_key, fields_base)
|
| 728 |
ctx = grid_events.EventCtx(
|
| 729 |
uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden,
|
| 730 |
admin=session.admin, fallback_ws=None, seen_ids={},
|
|
@@ -737,20 +845,190 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 737 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 738 |
ws, session.uname, set(pids), defs={}, scope_key=table_key,
|
| 739 |
storage_key=storage_key, fields_base=fields_base)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 740 |
# ββ W36-T21 / R6 β the READ half, in `grid_assembly`'s own position: after `workspace_wire`,
|
| 741 |
# so the closure covers this user's `custom_` and `measure_` columns too.
|
| 742 |
-
fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 743 |
|
| 744 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
| 745 |
"fields": fields, "views": views, "lists": lists, "hidden": hidden,
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 749 |
# β ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY β a key a consumer has to test
|
| 750 |
# for is a key a consumer forgets to test for, and this one carries a refusal.
|
| 751 |
"limits": limits, "defn": defn}
|
| 752 |
|
| 753 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 754 |
def ut_label(defn, key, meta=None):
|
| 755 |
"""THE name of a user table, resolved ONCE (wave 20, item 6a).
|
| 756 |
|
|
@@ -857,11 +1135,38 @@ def _rollup_source_offer():
|
|
| 857 |
"description": m.get("description") or ""})
|
| 858 |
|
| 859 |
out = []
|
|
|
|
| 860 |
for tkey, t in sorted(topics.items()):
|
| 861 |
dims = ((t.get("store") or {}).get("dims") or {})
|
| 862 |
measures = by_topic.get(tkey) or []
|
| 863 |
if not dims or not measures:
|
| 864 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 865 |
out.append({
|
| 866 |
"key": tkey,
|
| 867 |
"label": t.get("label") or tkey,
|
|
@@ -883,7 +1188,10 @@ def _rollup_source_offer():
|
|
| 883 |
windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")}
|
| 884 |
for k in ut.ROLLUP_SOURCE_WINDOWS]
|
| 885 |
offer = {"topics": out, "windows": windows}
|
| 886 |
-
|
|
|
|
|
|
|
|
|
|
| 887 |
return offer
|
| 888 |
|
| 889 |
|
|
@@ -1162,6 +1470,16 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
|
|
| 1162 |
if isinstance(_ov, dict):
|
| 1163 |
_cells.update(_ov)
|
| 1164 |
merged[_pid] = _cells
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1165 |
# ββ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) β THE
|
| 1166 |
# JSON DOCUMENTS DO NOT RIDE THE LIST.
|
| 1167 |
#
|
|
|
|
| 429 |
raise err(400, "bad_request", "a field key is required")
|
| 430 |
_defn_or_refuse(session, table_key)
|
| 431 |
from core import shared_overlay
|
| 432 |
+
import core.perm_scope as perm_scope
|
| 433 |
if not shared_overlay.is_shared(table_key, key, st=session.runtime):
|
| 434 |
+
# ββ W38-T16 / R7 β A COLUMN IS BORN OWNED AND PRIVATE, AND THIS IS THE PRODUCT CHANGE.
|
| 435 |
+
# Until now this door minted a column every account in the tenant could read, with no way
|
| 436 |
+
# to say who. `granted` marks it as governed by `core.shares` (see
|
| 437 |
+
# `perm_scope.FIELD_GRANT_MARK` for why an explicit marker and not the absence of a grant
|
| 438 |
+
# record), and the grant record below claims it for the creator.
|
| 439 |
+
#
|
| 440 |
+
# β THE EMPTY ENTRY LIST IS THE POINT, NOT A PLACEHOLDER. `set_grants` keeps a record that
|
| 441 |
+
# has an owner and no entries, so "shared with nobody" is STORED and is a different fact
|
| 442 |
+
# from "never shared" β the same explicit-resolution shape `perms_v` uses. Without the
|
| 443 |
+
# owner the column would be unmanageable: `may_administer` fails closed on an ownerless
|
| 444 |
+
# record, so nobody could ever share or re-share it.
|
| 445 |
+
# β COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading
|
| 446 |
+
# their absence as "granted to nobody" would blank every existing shared column at once.
|
| 447 |
shared_overlay.put_field(table_key, key, {
|
| 448 |
"key": key, "label": str(body.get("label") or key), "source": "overlay",
|
| 449 |
"type": str(body.get("type") or "text"), "shared": True,
|
| 450 |
+
perm_scope.FIELD_GRANT_MARK: True,
|
| 451 |
"createdBy": session.uname}, st=session.runtime)
|
| 452 |
+
try:
|
| 453 |
+
import core.shares as shares
|
| 454 |
+
shares.set_grants("field", shares.field_oid(table_key, key), [],
|
| 455 |
+
owner=session.uname, st=session.runtime)
|
| 456 |
+
except Exception: # noqa: BLE001
|
| 457 |
+
# β THE MARK IS ALREADY WRITTEN, SO A FAILED CLAIM FAILS **CLOSED**: the column is
|
| 458 |
+
# governed and nobody holds a grant, i.e. only an admin sees it. That is recoverable
|
| 459 |
+
# (an admin can share it) and the other order is not β a marked column with a claim
|
| 460 |
+
# that landed first and a definition that did not would be a grant on nothing.
|
| 461 |
+
pass
|
| 462 |
try:
|
| 463 |
# β `put_cell`, not `put_cells` β this door writes exactly ONE cell, and the singular is
|
| 464 |
# the API that says so. It delegates to the plural, so both stay reachable through the one
|
|
|
|
| 502 |
f"was added by {owner or 'somebody else'}, and dropping it would delete the "
|
| 503 |
f"value for every account")
|
| 504 |
dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime)
|
| 505 |
+
# ββ W38-T16 β THE GRANTS DIE WITH THE COLUMN, which is `shares.drop_objects`' whole reason
|
| 506 |
+
# for existing (wave 21, C3): a deleted object's grants would otherwise serve a ghost id into
|
| 507 |
+
# every receiver's "Shared with me" forever, and the ghost 404s on open. It matters twice as
|
| 508 |
+
# much for a field, because `drop_field` scrubs the CELLS so the key can be re-used β and a
|
| 509 |
+
# surviving grant record would silently re-arm on the next column that took the name.
|
| 510 |
+
# β R8 / D-172 IS UNCHANGED ABOVE: delete is creator-or-admin. This ticket did not widen it.
|
| 511 |
+
try:
|
| 512 |
+
import core.shares as shares
|
| 513 |
+
shares.drop_objects([("field", shares.field_oid(table_key, str(field_key)))],
|
| 514 |
+
st=session.runtime)
|
| 515 |
+
except Exception: # noqa: BLE001
|
| 516 |
+
pass
|
| 517 |
return {"ok": True, "dropped": bool(dropped),
|
| 518 |
"fields": list(shared_overlay.fields(table_key, st=session.runtime))}
|
| 519 |
|
|
|
|
| 654 |
# `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully
|
| 655 |
# writable on the EVENTS transport β a wall on one wire and not the other is the shape
|
| 656 |
# `strip_row`'s own note warns about, with the sign flipped.
|
| 657 |
+
hidden = _ut_hidden(session, table_key, fields_base, st=lent)
|
| 658 |
ctx = grid_events.EventCtx(
|
| 659 |
uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden,
|
| 660 |
admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime,
|
|
|
|
| 663 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 664 |
ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="",
|
| 665 |
fields_base=fields_base)
|
| 666 |
+
# ββ W38-T16 β THE SHARED COLUMNS JOIN THE **WRITE** CTX'S CONTRACT TOO, and for the reason
|
| 667 |
+
# W36-T21 gave one wall over: `grid_events` refuses a key by asking `ctx.hidden_keys`, so a
|
| 668 |
+
# column missing from this list is a column the events transport does not know it must
|
| 669 |
+
# refuse. The read door narrowing a name the write door has never heard of is the same
|
| 670 |
+
# one-wire wall with the sign flipped.
|
| 671 |
+
fields = _ut_shared_fields(session, table_key, fields)
|
| 672 |
+
fields, _rows, hidden = _ut_field_wall(session, table_key, fields, [], st=lent)
|
| 673 |
return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace,
|
| 674 |
"fields": fields, "views": views, "lists": lists, "hidden": hidden,
|
| 675 |
"derived": aios_grid.cohort_cells(lists),
|
| 676 |
+
# β W37-T12 β the OFFER, because `routes_grid` reads `g["measures"]` as the ADMISSION
|
| 677 |
+
# list for `clean_measure_field`: without it a Metric create on this surface is
|
| 678 |
+
# refused `measure_not_offered` even though the read door offers the kind.
|
| 679 |
+
# β THE CELLS ARE DELIBERATELY ABSENT HERE. This ctx validates a write and
|
| 680 |
+
# materialises no rows at all (W30-T30 removed a full table build from the write
|
| 681 |
+
# path); computing measure values would put that work straight back. `measure_sets`
|
| 682 |
+
# stays empty for the same reason β a write is not a render.
|
| 683 |
+
# β AND THE ASYMMETRY IS SAFE, WHICH IS NOT OBVIOUS AND WAS CHECKED RATHER THAN
|
| 684 |
+
# ASSUMED. `routes_grid._ctx` derives TWO things from this dict and they have
|
| 685 |
+
# different jobs: `measure_keys` (from `measures`) is the ADMISSION list
|
| 686 |
+
# `clean_filter_tree` uses β non-empty here, so a saved measure CONDITION survives the
|
| 687 |
+
# write, which is the failure that comment records for the customer path. Empty
|
| 688 |
+
# `resolved_ids` (from `measure_sets`) only makes `view_upsert` answer "rerender",
|
| 689 |
+
# so the next `/workspace` β which DOES compute the sets, in `ut_assembly` β resolves
|
| 690 |
+
# it. Conservative in the right direction: one extra repaint, never a dropped filter.
|
| 691 |
+
# ββ W38-T19 β THE CAPABILITY IS APPLIED HERE TOO, and this is the site that makes
|
| 692 |
+
# the refusal REAL rather than cosmetic: `routes_grid` reads this list as
|
| 693 |
+
# `measure_offer`, and `clean_measure_field` fail-closes `measure_not_offered` over
|
| 694 |
+
# an empty one. A picker hidden on the read door with this list still full would be
|
| 695 |
+
# a control that only stops the honest.
|
| 696 |
+
"measures": ut_measures(table_key, session),
|
| 697 |
+
"measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
|
| 698 |
# β W31-T20 β the write door reads this to refuse a PID-BEARING event loudly rather
|
| 699 |
# than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch.
|
| 700 |
"limits": limits, "defn": defn}
|
| 701 |
|
| 702 |
|
| 703 |
+
def _ut_hidden(session, table_key, fields, st=None):
|
| 704 |
"""The hidden-field closure for THIS session on THIS database β C1's field half, once.
|
| 705 |
|
| 706 |
β Named rather than inlined at its four call sites for the reason `may_open`'s own note gives:
|
| 707 |
four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE
|
| 708 |
evaluator; this is just the `ut_*` caller's shorthand for it.
|
| 709 |
+
|
| 710 |
+
β W38-T16 β `st` IS THE LEND THE CALLER ALREADY HOLDS, and passing it is free rather than
|
| 711 |
+
merely tidy: the closure's field-grant leg resolves against `object_shares`, which is exactly
|
| 712 |
+
the bucket `ut_assembly`'s own lend note says the pass is already serving.
|
| 713 |
"""
|
| 714 |
import core.perm_scope as perm_scope
|
| 715 |
+
return perm_scope.hidden_keys(session.user, table_key, fields, st=st)
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
def _ut_shared_fields(session, table_key, fields):
|
| 719 |
+
"""`fields` PLUS this database's TENANT-WIDE columns β the read-side merge, W38-T16.
|
| 720 |
+
|
| 721 |
+
ββ WHY THIS DID NOT EXIST AND WHY THAT WAS A HOLE. `patch_shared_cell` has written into
|
| 722 |
+
`<key>__shared` for any `ut_*` database since W30-T28, and only the READ-THROUGH door
|
| 723 |
+
(`routes_odoo_tables`) ever merged that stratum back. So a shared column written on a
|
| 724 |
+
MATERIALISED user table went into the store and was readable by nobody, through any door β
|
| 725 |
+
complete, correct, and invisible ([[reachable-is-not-the-same-as-built]]). This is the other
|
| 726 |
+
half of that door.
|
| 727 |
+
|
| 728 |
+
β MERGED BEFORE THE WALL, NEVER AFTER, and `routes_odoo_tables` learned this the same way:
|
| 729 |
+
the hidden closure must run on the WHOLE contract, or a formula in a shared column that reads
|
| 730 |
+
a hidden one sits outside its reach and carries the hidden value out wearing a second name.
|
| 731 |
+
β A key the definition already declares WINS. A shared column is an addition to a database's
|
| 732 |
+
contract, never a redefinition of a column that database already has.
|
| 733 |
+
"""
|
| 734 |
+
from core import shared_overlay
|
| 735 |
+
defs = shared_overlay.fields(table_key, st=session.runtime) or {}
|
| 736 |
+
if not defs:
|
| 737 |
+
return fields
|
| 738 |
+
have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)}
|
| 739 |
+
return list(fields or ()) + [dict(f, source="overlay")
|
| 740 |
+
for k, f in defs.items() if k not in have]
|
| 741 |
+
|
| 742 |
|
| 743 |
+
def _ut_shared_cells(session, table_key, pids):
|
| 744 |
+
"""`{"<pid>": {key: value}}` for the TENANT-WIDE stratum, over THIS session's row set.
|
| 745 |
|
| 746 |
+
β `pids` IS THE ROW WALL AND IT IS PASSED, NOT DEFAULTED β `shared_overlay.cells` refuses an
|
| 747 |
+
"everything" read by signature for exactly this reason, and the set handed in is the one
|
| 748 |
+
`scoped_pool`/`scoped_pids` already narrowed. A cell for a row this session may not see
|
| 749 |
+
therefore has nothing to attach to.
|
| 750 |
+
"""
|
| 751 |
+
from core import shared_overlay
|
| 752 |
+
try:
|
| 753 |
+
return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime)
|
| 754 |
+
except Exception: # noqa: BLE001
|
| 755 |
+
return {}
|
| 756 |
+
|
| 757 |
+
|
| 758 |
+
def _ut_field_wall(session, table_key, fields, rows_src, st=None):
|
| 759 |
"""`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires.
|
| 760 |
|
| 761 |
ββ W36-T21 β the same three lines `routes_customers.grid_assembly` runs for `customer_data`,
|
|
|
|
| 774 |
path with nothing to say ([[one-question-two-normalizers]]).
|
| 775 |
"""
|
| 776 |
import core.perm_scope as perm_scope
|
| 777 |
+
hide = perm_scope.hidden_keys(session.user, table_key, fields, st=st)
|
| 778 |
if not hide:
|
| 779 |
return fields, rows_src, frozenset()
|
| 780 |
fields = [f for f in fields if f.get("key") not in hide]
|
|
|
|
| 832 |
ops = _ops(session, table_key, st=st)
|
| 833 |
# ββ W36-T21 β the WRITE half of the field wall. `frozenset()` here meant a column an
|
| 834 |
# administrator had hidden was still writable through the events transport.
|
| 835 |
+
hidden = _ut_hidden(session, table_key, fields_base, st=st)
|
| 836 |
ctx = grid_events.EventCtx(
|
| 837 |
uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden,
|
| 838 |
admin=session.admin, fallback_ws=None, seen_ids={},
|
|
|
|
| 845 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 846 |
ws, session.uname, set(pids), defs={}, scope_key=table_key,
|
| 847 |
storage_key=storage_key, fields_base=fields_base)
|
| 848 |
+
# ββ W38-T16 β THE TENANT-WIDE STRATUM, ON A MATERIALISED GRID. Merged HERE, between
|
| 849 |
+
# `workspace_wire` and the wall, which is `routes_odoo_tables`' own position for the same two
|
| 850 |
+
# lines and for the same reason: the closure must be recomputed on the MERGED contract or a
|
| 851 |
+
# formula in a shared column reaches past it.
|
| 852 |
+
fields = _ut_shared_fields(session, table_key, fields)
|
| 853 |
+
# β THE CELLS COST A READ, SO THE ENVELOPE ARM DOES NOT PAY IT. `with_rows=False` renders no
|
| 854 |
+
# row at all (D-174), and `cells()` over the whole pid set would be a scoped read whose
|
| 855 |
+
# result nothing consumes. The COLUMNS still merge above: `/workspace` needs the contract.
|
| 856 |
+
shared_cells = _ut_shared_cells(session, table_key, pids) if with_rows else {}
|
| 857 |
# ββ W36-T21 / R6 β the READ half, in `grid_assembly`'s own position: after `workspace_wire`,
|
| 858 |
# so the closure covers this user's `custom_` and `measure_` columns too.
|
| 859 |
+
fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st)
|
| 860 |
+
if hidden and shared_cells:
|
| 861 |
+
# β BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the
|
| 862 |
+
# DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak
|
| 863 |
+
# for both β the same sentence `routes_odoo_tables` writes over its own `overlays` dict.
|
| 864 |
+
shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden}
|
| 865 |
+
for pid, cells in shared_cells.items()}
|
| 866 |
+
|
| 867 |
+
# ββ W37-T12 / owner item 4 (R1) β `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database
|
| 868 |
+
# whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real
|
| 869 |
+
# lookback catalogue AND the cells to go with it; every other user table still serves `[]`,
|
| 870 |
+
# which is the honest answer for a hand-typed one rather than a descope.
|
| 871 |
+
today = time.strftime("%Y-%m-%d")
|
| 872 |
+
measures = ut_measures(table_key, session)
|
| 873 |
+
derived = aios_grid.cohort_cells(lists) # R9: this table's own lists
|
| 874 |
+
for _pid, _cells in _ut_measure_cells(table_key, fields, pids, today, measures,
|
| 875 |
+
session.runtime.measure_memo).items():
|
| 876 |
+
derived.setdefault(_pid, {}).update(_cells)
|
| 877 |
|
| 878 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
| 879 |
"fields": fields, "views": views, "lists": lists, "hidden": hidden,
|
| 880 |
+
"derived": derived,
|
| 881 |
+
# β W38-T16 β ALWAYS PRESENT, `{}` when this database shares nothing. A key a
|
| 882 |
+
# consumer has to test for is a key a consumer forgets to test for, and the consumer
|
| 883 |
+
# here (`table_rows`) is layering strata in a fixed order.
|
| 884 |
+
"shared_cells": shared_cells,
|
| 885 |
+
"measures": measures,
|
| 886 |
+
"measure_sets": _ut_measure_sets(table_key, views, measures, pids, today),
|
| 887 |
+
"today": today,
|
| 888 |
# β ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY β a key a consumer has to test
|
| 889 |
# for is a key a consumer forgets to test for, and this one carries a refusal.
|
| 890 |
"limits": limits, "defn": defn}
|
| 891 |
|
| 892 |
|
| 893 |
+
def _ut_measure_err(tag, e):
|
| 894 |
+
try:
|
| 895 |
+
import harness.telemetry as _tel
|
| 896 |
+
_tel.error(f"api:{tag}", e)
|
| 897 |
+
except Exception: # noqa: BLE001
|
| 898 |
+
pass
|
| 899 |
+
|
| 900 |
+
|
| 901 |
+
def ut_measures(table_key, session):
|
| 902 |
+
"""The lookback-measure OFFER for a user database FOR THIS SESSION, or `[]` β W37-T12 /
|
| 903 |
+
owner item 4 (R1), narrowed by the metrics capability in W38-T19.
|
| 904 |
+
|
| 905 |
+
ββ W38-T19 β `session` IS REQUIRED, NOT DEFAULTED. The function was PRINCIPAL-BLIND, so a
|
| 906 |
+
per-user capability had nothing to be consulted by. Defaulting it would leave both existing
|
| 907 |
+
call sites compiling and silently ungated, which is the fail-open spelling of the bug rather
|
| 908 |
+
than a fix for it β the argument `_module_fields` in `routes_admin` records for its own
|
| 909 |
+
`session`. β AND BOTH CALL SITES MATTER, WHICH IS EASY TO GET HALF RIGHT: `ut_assembly` is
|
| 910 |
+
the READ and `ut_write_ctx` is the door `routes_grid` turns into `clean_measure_field`'s
|
| 911 |
+
admission set. Gating only the read would take the kind off the picker and leave the CREATE
|
| 912 |
+
working for anyone who posts the event.
|
| 913 |
+
|
| 914 |
+
β THE SAME ENGINE THE PRODUCT GRID USES (`semantic.entity_measures`), reached through the
|
| 915 |
+
topic that DESCRIBES this database. A table with no entity topic, or a topic with no
|
| 916 |
+
`measures:` binding, gets `[]` β which is the honest answer for a hand-typed database and is
|
| 917 |
+
what keeps the Metric kind hidden there (`ColumnMenu`'s `offerMeasure={measures.length > 0}`).
|
| 918 |
+
|
| 919 |
+
β NOT A `{table_key: [keys]}` MAP IN THIS FILE. The binding is declared in the model
|
| 920 |
+
(`model/topics/odoo_agents.yml`'s `measures:` block) and the topic already names its own grid,
|
| 921 |
+
so a list here would be a second definition of one fact β the argument `_rollup_source_offer`
|
| 922 |
+
makes about itself one screen down.
|
| 923 |
+
"""
|
| 924 |
+
import core.perm_scope as perm_scope
|
| 925 |
+
if not perm_scope.may_metrics(session.user, table_key):
|
| 926 |
+
return []
|
| 927 |
+
try:
|
| 928 |
+
from harness import semantic as sem
|
| 929 |
+
topic = sem.topic_for_grid(table_key)
|
| 930 |
+
return sem.entity_measures(topic) if topic else []
|
| 931 |
+
except Exception as e: # noqa: BLE001
|
| 932 |
+
_ut_measure_err("ut-measure-offer", e)
|
| 933 |
+
return []
|
| 934 |
+
|
| 935 |
+
|
| 936 |
+
def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None):
|
| 937 |
+
"""`{pid: {field key: value}}` for this database's Metric columns.
|
| 938 |
+
|
| 939 |
+
β THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need
|
| 940 |
+
one. `sales_lines.agent` is `rp.agent_id` β a `res.partner` id β and a `ut_odoo_agents` row's
|
| 941 |
+
`pid` IS its `res.partner` id (`scoped_pool` sets `pid = int(rid)` and `read_agents` keys
|
| 942 |
+
`_id` on the partner id). So the group key and the pid are the same integer. The product grid
|
| 943 |
+
keys on `default_code` instead, which is exactly why contract C1 calls the join key a trap:
|
| 944 |
+
the right answer differs per database and must be read off the binding, never assumed.
|
| 945 |
+
"""
|
| 946 |
+
import aios_grid
|
| 947 |
+
|
| 948 |
+
mfields = aios_grid.measure_fields_of(fields)
|
| 949 |
+
if not mfields or not offer:
|
| 950 |
+
return {}
|
| 951 |
+
from harness import semantic as sem
|
| 952 |
+
from harness import windows as _wn
|
| 953 |
+
|
| 954 |
+
topic = sem.topic_for_grid(table_key)
|
| 955 |
+
by_key = {m["key"]: m for m in offer}
|
| 956 |
+
by_window = {}
|
| 957 |
+
for f in mfields:
|
| 958 |
+
spec = f.get("measure") or {}
|
| 959 |
+
if spec.get("key") not in by_key:
|
| 960 |
+
continue
|
| 961 |
+
rng = _wn.resolve(spec.get("window"), today) if _wn.normalize(spec.get("window")) else None
|
| 962 |
+
if rng is None:
|
| 963 |
+
continue # an unresolvable window is unanswerable β never widen to all time
|
| 964 |
+
by_window.setdefault((rng[0], rng[1]), []).append(f)
|
| 965 |
+
|
| 966 |
+
out = {}
|
| 967 |
+
for (dfrom, dto), group in by_window.items():
|
| 968 |
+
keys = tuple(sorted({f["measure"]["key"] for f in group}))
|
| 969 |
+
mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys)
|
| 970 |
+
if mkey not in memo:
|
| 971 |
+
try:
|
| 972 |
+
memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom,
|
| 973 |
+
date_to=dto, team_id=team_id, offer=offer)
|
| 974 |
+
except Exception as e: # noqa: BLE001
|
| 975 |
+
_ut_measure_err("ut-measure-column", e)
|
| 976 |
+
transient = False
|
| 977 |
+
try:
|
| 978 |
+
from harness import datastore as _ds
|
| 979 |
+
transient = not _ds.ready()
|
| 980 |
+
except Exception: # noqa: BLE001
|
| 981 |
+
transient = False
|
| 982 |
+
if transient:
|
| 983 |
+
continue # a warming store must not memoise as a permanent failure
|
| 984 |
+
memo[mkey] = None
|
| 985 |
+
vals = memo[mkey]
|
| 986 |
+
if vals is None:
|
| 987 |
+
continue
|
| 988 |
+
# C1's empty-window rule for EVERY row, not only the grouped ones.
|
| 989 |
+
filled = {pid: sem.entity_zero_fill(vals.get(pid), list(keys), offer) for pid in pids}
|
| 990 |
+
for f in group:
|
| 991 |
+
k, fkey = f["measure"]["key"], f["key"]
|
| 992 |
+
for pid, cell in filled.items():
|
| 993 |
+
if k in cell:
|
| 994 |
+
out.setdefault(pid, {})[fkey] = cell[k]
|
| 995 |
+
if len(memo) > 100:
|
| 996 |
+
memo.clear()
|
| 997 |
+
return out
|
| 998 |
+
|
| 999 |
+
|
| 1000 |
+
def _ut_measure_sets(table_key, views, offer, pids, today, team_id=None):
|
| 1001 |
+
"""`{rule id: [pid, β¦]}` for the measure CONDITIONS in this database's saved views.
|
| 1002 |
+
|
| 1003 |
+
β SHIPS WITH THE OFFER, NEVER AFTER IT. `routes_grid` derives `measure_keys` from the same
|
| 1004 |
+
list, so a non-empty `measures` also opens the measure condition in the filter builder β and a
|
| 1005 |
+
rule id missing from `measureSets` renders PENDING ("Calculatingβ¦") and matches nothing,
|
| 1006 |
+
permanently. Offer and sets are one feature.
|
| 1007 |
+
"""
|
| 1008 |
+
if not offer:
|
| 1009 |
+
return {}
|
| 1010 |
+
from harness import semantic as sem
|
| 1011 |
+
|
| 1012 |
+
topic = sem.topic_for_grid(table_key)
|
| 1013 |
+
admitted = {m["key"] for m in offer}
|
| 1014 |
+
by_id = {}
|
| 1015 |
+
for v in views or []:
|
| 1016 |
+
for rule in sem.entity_measure_leaves((v.get("config") or {}).get("filters"), admitted):
|
| 1017 |
+
rid = str(rule.get("id") or "")
|
| 1018 |
+
if rid:
|
| 1019 |
+
by_id[rid] = rule
|
| 1020 |
+
if not by_id:
|
| 1021 |
+
return {}
|
| 1022 |
+
try:
|
| 1023 |
+
sets = sem.entity_measure_sets(topic, list(by_id.values()), today, team_id=team_id,
|
| 1024 |
+
keys_by_id=list(pids), offer=offer)
|
| 1025 |
+
except Exception as e: # noqa: BLE001
|
| 1026 |
+
_ut_measure_err("ut-measure-sets", e)
|
| 1027 |
+
return {}
|
| 1028 |
+
return {rid: sorted(int(k) for k in ks if str(k).lstrip("-").isdigit())
|
| 1029 |
+
for rid, ks in sets.items()}
|
| 1030 |
+
|
| 1031 |
+
|
| 1032 |
def ut_label(defn, key, meta=None):
|
| 1033 |
"""THE name of a user table, resolved ONCE (wave 20, item 6a).
|
| 1034 |
|
|
|
|
| 1135 |
"description": m.get("description") or ""})
|
| 1136 |
|
| 1137 |
out = []
|
| 1138 |
+
indeterminate = False
|
| 1139 |
for tkey, t in sorted(topics.items()):
|
| 1140 |
dims = ((t.get("store") or {}).get("dims") or {})
|
| 1141 |
measures = by_topic.get(tkey) or []
|
| 1142 |
if not dims or not measures:
|
| 1143 |
continue
|
| 1144 |
+
# ββ W37-T13 / /validate-wave 2026-08-19 β THE SOURCE TABLE MUST BE LIVE, AND THIS DOOR
|
| 1145 |
+
# DID NOT ASK. The docstring above already promises *"ONLY COMBINATIONS THAT CAN RESOLVE
|
| 1146 |
+
# ARE OFFERED"*; adding `stock_move` to `datastore.ENTITIES` broke that promise on the one
|
| 1147 |
+
# door nobody re-read. `semantic._one_binding_offer` β the MEASURE-CATALOG door β refuses
|
| 1148 |
+
# the same bindings with a stated cause, so the wave shipped **two doors onto one
|
| 1149 |
+
# vocabulary with only one of them guarded**, and the unguarded one is the picker.
|
| 1150 |
+
# MEASURED: `_rollup_source_offer()` listed `stock_moves` with both measures and all four
|
| 1151 |
+
# dims while `royal.duckdb` has no `stock_move` table at all, so choosing it raised
|
| 1152 |
+
# `CatalogException: Table with name stock_move does not exist!` β five of the fifty
|
| 1153 |
+
# (topic, measure, dim) triples in `verify_odoo_relational`. A user meets a raw catalog
|
| 1154 |
+
# error, or a column that looks configured and is blank forever
|
| 1155 |
+
# ([[permitted-is-not-answerable]]).
|
| 1156 |
+
# β REUSES `_source_ready` rather than re-deriving readiness: one question, one
|
| 1157 |
+
# normalizer. `None` (the store could not be asked) refuses TOO β fail closed, exactly as
|
| 1158 |
+
# the measure door does.
|
| 1159 |
+
_tbl = (t.get("store") or {}).get("table")
|
| 1160 |
+
if _tbl:
|
| 1161 |
+
_rdy = sem._source_ready(_tbl)
|
| 1162 |
+
if _rdy is not True:
|
| 1163 |
+
# β AND AN INDETERMINATE ANSWER MUST NOT BE CACHED. `_source_ready`'s own comment
|
| 1164 |
+
# records the measured version of this: a cold container with no mirror yet
|
| 1165 |
+
# refused every binding, the refusal was memoised, and the grid served no measures
|
| 1166 |
+
# long after the mirror arrived. Here the same shape would freeze a truncated
|
| 1167 |
+
# PICKER for the life of the process.
|
| 1168 |
+
indeterminate = indeterminate or (_rdy is None)
|
| 1169 |
+
continue
|
| 1170 |
out.append({
|
| 1171 |
"key": tkey,
|
| 1172 |
"label": t.get("label") or tkey,
|
|
|
|
| 1188 |
windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")}
|
| 1189 |
for k in ut.ROLLUP_SOURCE_WINDOWS]
|
| 1190 |
offer = {"topics": out, "windows": windows}
|
| 1191 |
+
# β Cache only a DEFINITE answer (see the readiness block above): if any topic was dropped
|
| 1192 |
+
# because the store could not be asked, this offer is a snapshot of an unknown, not a fact.
|
| 1193 |
+
if not indeterminate:
|
| 1194 |
+
_ROLLUP_CACHE["offer"] = offer
|
| 1195 |
return offer
|
| 1196 |
|
| 1197 |
|
|
|
|
| 1470 |
if isinstance(_ov, dict):
|
| 1471 |
_cells.update(_ov)
|
| 1472 |
merged[_pid] = _cells
|
| 1473 |
+
# ββ W38-T16 β THE TENANT-WIDE STRATUM GOES ON TOP, and the order is the contract rather
|
| 1474 |
+
# than a preference. A shared column is one the whole permitted audience must read the SAME
|
| 1475 |
+
# value in β that is the entire reason `shared_overlay` exists β so a stale per-user value
|
| 1476 |
+
# left under the same key must not win. `routes_odoo_tables` layers the two in this exact
|
| 1477 |
+
# order (`overlays.setdefault(pid, {}).update(cells)`) and the two doors must agree.
|
| 1478 |
+
# β NEVER `setdefault` ON `merged`: these cells are already narrowed to this session's pids,
|
| 1479 |
+
# but inventing a row id here would put a row in the payload that `rows_src` never admitted.
|
| 1480 |
+
for _pid, _cells in (g.get("shared_cells") or {}).items():
|
| 1481 |
+
if str(_pid) in merged and isinstance(_cells, dict):
|
| 1482 |
+
merged[str(_pid)].update(_cells)
|
| 1483 |
# ββ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) β THE
|
| 1484 |
# JSON DOCUMENTS DO NOT RIDE THE LIST.
|
| 1485 |
#
|
api/routes_web_agent.py
CHANGED
|
@@ -22,10 +22,11 @@ admin-gated, because it spends money and because D-51 Β§5's authorisation postur
|
|
| 22 |
the tenant is authorised to use, at their instruction") is not something an ordinary member
|
| 23 |
should be able to commit the tenant to.
|
| 24 |
|
| 25 |
-
|
| 26 |
-
`
|
| 27 |
-
|
| 28 |
-
|
|
|
|
| 29 |
"""
|
| 30 |
from fastapi import APIRouter, Body, Depends
|
| 31 |
|
|
|
|
| 22 |
the tenant is authorised to use, at their instruction") is not something an ordinary member
|
| 23 |
should be able to commit the tenant to.
|
| 24 |
|
| 25 |
+
β MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and
|
| 26 |
+
`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting
|
| 27 |
+
either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which
|
| 28 |
+
is the same class of stale claim as a green gate on a router nobody wired: three finished routers
|
| 29 |
+
once shipped 404-dead behind entirely green gates, and prose is not the control that stops it.
|
| 30 |
"""
|
| 31 |
from fastapi import APIRouter, Body, Depends
|
| 32 |
|
platform/aios_grid.py
CHANGED
|
@@ -24,6 +24,7 @@ Design notes:
|
|
| 24 |
injected-HTML path remains a read/local-write fallback when component assets are absent.
|
| 25 |
"""
|
| 26 |
import json
|
|
|
|
| 27 |
import re
|
| 28 |
from pathlib import Path
|
| 29 |
|
|
@@ -89,6 +90,22 @@ def _round(v):
|
|
| 89 |
return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
|
| 90 |
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors
|
| 93 |
# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step.
|
| 94 |
# select β a single-select with its own `options` (the "Status" a user wants to add; distinct
|
|
@@ -415,6 +432,52 @@ def _clean_formula(raw, valid_keys=None):
|
|
| 415 |
return s
|
| 416 |
|
| 417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
def _field_extras(saved, ftype):
|
| 419 |
"""createdBy / permissions / format / scope β the validated passthrough the created strata
|
| 420 |
share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it
|
|
@@ -434,6 +497,27 @@ def _field_extras(saved, ftype):
|
|
| 434 |
out["format"] = fmt
|
| 435 |
if saved.get("scope") == "cohort":
|
| 436 |
out["scope"] = "cohort"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 437 |
corrected_from = saved.get("labelCorrectedFrom")
|
| 438 |
correction_id = saved.get("labelCorrectionId")
|
| 439 |
if (isinstance(corrected_from, str) and corrected_from.strip()
|
|
@@ -752,7 +836,10 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
|
|
| 752 |
if field.get("derived"):
|
| 753 |
continue # not on the pool row β filled from `derived` below
|
| 754 |
v = r.get(k)
|
| 755 |
-
|
|
|
|
|
|
|
|
|
|
| 756 |
saved = overlays.get(str(pid), {}) or {}
|
| 757 |
for field in overlay_fields:
|
| 758 |
row[field["key"]] = saved.get(field["key"], "")
|
|
@@ -789,6 +876,19 @@ COHORT_FIELD = '__cohort__'
|
|
| 789 |
#: a set op reaching a column leaf would fall through the client engine's switch to "no
|
| 790 |
#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work.
|
| 791 |
COHORT_OPS = {'anyOf', 'allOf', 'noneOf'}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 792 |
#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here:
|
| 793 |
#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the
|
| 794 |
#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES.
|
|
@@ -912,8 +1012,32 @@ def _clean_rhs(raw, valid_keys):
|
|
| 912 |
#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by
|
| 913 |
#: `verify_icons.py::mode_parity` β offering an unmounted mode is red, and mounting an unoffered
|
| 914 |
#: one is red too, so the hold cannot outlive its reason the way wave 27's did).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 915 |
DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart',
|
| 916 |
-
'timeseries', 'catalog', 'swipe', 'form'}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 917 |
|
| 918 |
#: ββ WAVE-29 R7 (owner item 10, contracts C3/C4) β THE FORM INTERFACE's stored spec, at
|
| 919 |
#: `views[<id>].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read
|
|
@@ -926,6 +1050,166 @@ DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'char
|
|
| 926 |
#: so one tenant setting its token to another tenant's value would silently receive that tenant's
|
| 927 |
#: submissions. The token is minted server-side and lives in a bucket no client write can reach;
|
| 928 |
#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 929 |
FORM_ACCESS = ('public', 'emails')
|
| 930 |
MAX_FORM_FIELDS = 60
|
| 931 |
MAX_FORM_EMAILS = 200
|
|
@@ -1373,7 +1657,11 @@ def _clean_display(raw, valid_keys):
|
|
| 1373 |
# saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
|
| 1374 |
mode = _LEGACY_MODES.get(mode, mode)
|
| 1375 |
out = {'mode': mode}
|
| 1376 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1377 |
if raw.get(ref) in valid_keys:
|
| 1378 |
out[ref] = raw[ref]
|
| 1379 |
# ββ C-DISP (wave 2026-08-02) βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -1508,6 +1796,12 @@ def _clean_display(raw, valid_keys):
|
|
| 1508 |
catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys)
|
| 1509 |
if catalogs:
|
| 1510 |
out['catalogs'] = catalogs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1511 |
# ββ β WAVE-27 C3 (item 8 / R2): the swipe binding ββββββββββββββββββββββββββββββββββββ
|
| 1512 |
# `{fieldKey, leftOption, rightOption}` β WHOLE-KEY drop, never a partial one, and that
|
| 1513 |
# asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight.
|
|
@@ -1536,6 +1830,26 @@ def _clean_display(raw, valid_keys):
|
|
| 1536 |
# `kanbanClamp` law) wearing a control that promises a choice.
|
| 1537 |
if left and right and left.casefold() != right.casefold():
|
| 1538 |
out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1539 |
# ββ ββ WAVE-29 R7 (item 10): the FORM spec β see `_clean_form` for why the token is not here.
|
| 1540 |
form = _clean_form(raw.get('form'), valid_keys)
|
| 1541 |
if form:
|
|
@@ -1743,7 +2057,59 @@ def clean_item_folders(raw, folders, valid_ids):
|
|
| 1743 |
return out
|
| 1744 |
|
| 1745 |
|
| 1746 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1747 |
"""Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
|
| 1748 |
|
| 1749 |
Returns a clean tree of leaf conditions ({colId, op, value, value2}) and
|
|
@@ -1762,6 +2128,14 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None):
|
|
| 1762 |
condition that can only match nothing, so `List is not [deleted]` would show an empty table
|
| 1763 |
forever with no way to tell why. `None` means this host has no cohorts, and then every
|
| 1764 |
cohort leaf is dropped: fail-closed, like every other unknown key.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1765 |
"""
|
| 1766 |
if budget is None:
|
| 1767 |
budget = [MAX_FILTER_NODES]
|
|
@@ -1776,7 +2150,7 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None):
|
|
| 1776 |
continue # too deep -> drop
|
| 1777 |
budget[0] -= 1
|
| 1778 |
children = clean_filter_tree(node['children'], valid_keys,
|
| 1779 |
-
depth + 1, budget, cohort_ids)
|
| 1780 |
if children:
|
| 1781 |
out.append({'conj': 'or' if node.get('conj') == 'or' else 'and',
|
| 1782 |
'children': children})
|
|
@@ -1793,6 +2167,21 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None):
|
|
| 1793 |
out.append({'colId': COHORT_FIELD, 'op': op,
|
| 1794 |
'value': ','.join(named), 'value2': ''})
|
| 1795 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1796 |
if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS:
|
| 1797 |
budget[0] -= 1
|
| 1798 |
# `or ''` would be wrong here: it maps every FALSY value to '', and '' is the
|
|
|
|
| 24 |
injected-HTML path remains a read/local-write fallback when component assets are absent.
|
| 25 |
"""
|
| 26 |
import json
|
| 27 |
+
import math
|
| 28 |
import re
|
| 29 |
from pathlib import Path
|
| 30 |
|
|
|
|
| 90 |
return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
|
| 91 |
|
| 92 |
|
| 93 |
+
def _round2(v):
|
| 94 |
+
"""D-153 (W37, asked by lane B) β MONEY AND PERCENT KEEP THEIR DECIMALS.
|
| 95 |
+
|
| 96 |
+
`_round`'s whole-number convention is right for counts and wrong for currency: `first_cost`
|
| 97 |
+
3.47 shipped as `3`, so every money cell on every Odoo grid was a whole dollar.
|
| 98 |
+
|
| 99 |
+
β `pct` IS AFFECTED TOO, and that is the half most likely to be dropped. `cells.ts` renders a
|
| 100 |
+
percent as `num(v).toFixed(1) + "%"`, so a `yoy_pct` of 12.34 was `round()`ed to 12 and painted
|
| 101 |
+
"12.0%" β one decimal of a number that HAS one, which reads as precision rather than as loss.
|
| 102 |
+
β NOT written as a `_round(v, nd=0)` default argument, deliberately: `round(v, 0)` returns a
|
| 103 |
+
FLOAT where `round(v)` returns an INT, so every integer column's wire shape would change under
|
| 104 |
+
an edit that looks purely cosmetic.
|
| 105 |
+
"""
|
| 106 |
+
return round(v, 2) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
|
| 107 |
+
|
| 108 |
+
|
| 109 |
# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors
|
| 110 |
# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step.
|
| 111 |
# select β a single-select with its own `options` (the "Status" a user wants to add; distinct
|
|
|
|
| 432 |
return s
|
| 433 |
|
| 434 |
|
| 435 |
+
def _clean_geocode(raw):
|
| 436 |
+
"""One `geocode` bag β the stored shape, or None (dropped).
|
| 437 |
+
|
| 438 |
+
ββ W37-T22 (owner item 15 / ruling R4, contract C7). THE ONE validator for the geocode
|
| 439 |
+
pseudo-kind, called by BOTH field doors: `_field_extras` here (the Odoo grids) and
|
| 440 |
+
`core/user_tables._clean_field` via `_ag_geocode` (every `ut_*` user database).
|
| 441 |
+
|
| 442 |
+
β IT WAS INLINE IN `_field_extras` UNTIL 2026-08-19 AND THAT IS WHY IT SHIPPED HALF-DONE.
|
| 443 |
+
`user_tables._clean_field` is a strict allowlist that returns only the bags it NAMES, so a
|
| 444 |
+
geocode column created on a user database was stored WITHOUT its `{addressField}` β the
|
| 445 |
+
*"created, named, configured, gone"* failure the `image` type shipped as in wave 19. Extracting
|
| 446 |
+
it rather than copying the rules is [[one-question-two-normalizers]]: a second copy is how the
|
| 447 |
+
two doors come to disagree about what a valid bag is.
|
| 448 |
+
|
| 449 |
+
β WHOLE-KEY DROP (the `swipe` construction): a geocode column with no address column is not
|
| 450 |
+
a degraded column, it is a column no run can ever fill, and the honest state for that is the
|
| 451 |
+
unconfigured one the client refuses at `canCreate`.
|
| 452 |
+
|
| 453 |
+
β THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND
|
| 454 |
+
in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`,
|
| 455 |
+
`CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched β three
|
| 456 |
+
parity gates chain over those sets and none permits a partial landing. This is also why the
|
| 457 |
+
bag takes `format`'s posture and NOT `formula`'s: there is no type to pair it with, so an
|
| 458 |
+
absent bag is an ordinary text column rather than a refusal.
|
| 459 |
+
|
| 460 |
+
β `addressField` is NOT checked against the table's keys: neither door is given `valid_keys`,
|
| 461 |
+
and the split this module already draws puts that loss on the surface that RENDERS. The
|
| 462 |
+
enricher must SAY the address column is gone rather than geocode an empty string.
|
| 463 |
+
"""
|
| 464 |
+
if not isinstance(raw, dict):
|
| 465 |
+
return None
|
| 466 |
+
addr = raw.get("addressField")
|
| 467 |
+
if not isinstance(addr, str) or not addr.strip():
|
| 468 |
+
return None
|
| 469 |
+
out = {"addressField": addr.strip()[:80]}
|
| 470 |
+
# D-3 β the optional ISO 3166-1 alpha-2 hint, mapped straight onto the geocoder's
|
| 471 |
+
# `countrycodes` parameter. NORMALISED to upper case and stored only when it is exactly two
|
| 472 |
+
# letters: absent means "anywhere", and `""` would be that state's second spelling. A
|
| 473 |
+
# three-letter or numeric code is REFUSED rather than truncated, because a truncated code is
|
| 474 |
+
# a valid code for a different country.
|
| 475 |
+
cc = raw.get("country")
|
| 476 |
+
if isinstance(cc, str) and re.fullmatch(r"[A-Za-z]{2}", cc.strip() or ""):
|
| 477 |
+
out["country"] = cc.strip().upper()
|
| 478 |
+
return out
|
| 479 |
+
|
| 480 |
+
|
| 481 |
def _field_extras(saved, ftype):
|
| 482 |
"""createdBy / permissions / format / scope β the validated passthrough the created strata
|
| 483 |
share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it
|
|
|
|
| 497 |
out["format"] = fmt
|
| 498 |
if saved.get("scope") == "cohort":
|
| 499 |
out["scope"] = "cohort"
|
| 500 |
+
# ββ ββ W37-T22 (owner item 15 / ruling R4, contract C7): THE GEOCODE COLUMN'S CONFIG ββ
|
| 501 |
+
#
|
| 502 |
+
# `{addressField}` β which column this one reads. WHOLE-KEY drop (the `swipe` construction):
|
| 503 |
+
# a geocode column with no address column is not a degraded one, it is a column no run can
|
| 504 |
+
# ever fill, and the honest state for that is the unconfigured one the client refuses at
|
| 505 |
+
# `canCreate`.
|
| 506 |
+
#
|
| 507 |
+
# β THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND
|
| 508 |
+
# in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`,
|
| 509 |
+
# `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched β three
|
| 510 |
+
# parity gates chain over those sets and none permits a partial landing.
|
| 511 |
+
# β `addressField` is NOT checked against this table's keys: `_field_extras` is not given
|
| 512 |
+
# `valid_keys`, and the split this module already draws puts that loss on the surface that
|
| 513 |
+
# RENDERS ("what each mode MEANS ... is the client's business"). The enricher must SAY the
|
| 514 |
+
# address column is gone rather than geocode an empty string.
|
| 515 |
+
# β EXTRACTED to `_clean_geocode` 2026-08-19 (/validate-wave, W37-T22 clause 1). The rules
|
| 516 |
+
# did not change; what changed is that `core/user_tables.py` now reaches THIS function through
|
| 517 |
+
# `_ag_geocode` instead of carrying a second copy. See `_clean_geocode`'s own note.
|
| 518 |
+
geo = _clean_geocode(saved.get("geocode"))
|
| 519 |
+
if geo:
|
| 520 |
+
out["geocode"] = geo
|
| 521 |
corrected_from = saved.get("labelCorrectedFrom")
|
| 522 |
correction_id = saved.get("labelCorrectionId")
|
| 523 |
if (isinstance(corrected_from, str) and corrected_from.strip()
|
|
|
|
| 836 |
if field.get("derived"):
|
| 837 |
continue # not on the pool row β filled from `derived` below
|
| 838 |
v = r.get(k)
|
| 839 |
+
# D-153 β three roundings, not two: text/status/date pass through verbatim, currency
|
| 840 |
+
# and pct keep two decimals (`_round2`), everything else stays a whole number.
|
| 841 |
+
row[k] = v if field["type"] in {"text", "status", "date"} else (
|
| 842 |
+
_round2(v) if field["type"] in {"currency", "pct"} else _round(v))
|
| 843 |
saved = overlays.get(str(pid), {}) or {}
|
| 844 |
for field in overlay_fields:
|
| 845 |
row[field["key"]] = saved.get(field["key"], "")
|
|
|
|
| 876 |
#: a set op reaching a column leaf would fall through the client engine's switch to "no
|
| 877 |
#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work.
|
| 878 |
COHORT_OPS = {'anyOf', 'allOf', 'noneOf'}
|
| 879 |
+
|
| 880 |
+
#: ββ WAVE-37 T26 (owner item 14, contract C4) β THE VIEW-MEMBERSHIP LEAF's reserved
|
| 881 |
+
#: pseudo-column and its operators. Mirrors `customer-grid/types.ts` VIEW_FIELD / VIEW_OPS.
|
| 882 |
+
#:
|
| 883 |
+
#: β DISJOINT FROM `FILTER_OPS`, for the reason COHORT_OPS is: `matchFilter` answers an operator
|
| 884 |
+
#: it does not know by NOT NARROWING, so an op that reached a column leaf would widen the result
|
| 885 |
+
#: under an authoritative count. Keeping the two vocabularies apart means a column leaf carrying
|
| 886 |
+
#: `inView` is dropped by the same fail-closed path that drops `dropTable`.
|
| 887 |
+
#: β The leaf names exactly ONE view (singular where a cohort leaf names a set), so there is no
|
| 888 |
+
#: all-or-nothing rule to get wrong here β the id either resolves or the leaf goes.
|
| 889 |
+
VIEW_FIELD = '__view__'
|
| 890 |
+
VIEW_OPS = {'inView', 'notInView'}
|
| 891 |
+
MAX_VIEW_ID = 64
|
| 892 |
#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here:
|
| 893 |
#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the
|
| 894 |
#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES.
|
|
|
|
| 1012 |
#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by
|
| 1013 |
#: `verify_icons.py::mode_parity` β offering an unmounted mode is red, and mounting an unoffered
|
| 1014 |
#: one is red too, so the hold cannot outlive its reason the way wave 27's did).
|
| 1015 |
+
#: ββ WAVE-37 T20 (owner item 10, ruling R2, contract C2) β 'script', and the reason it lands
|
| 1016 |
+
#: here is NOT the one the PRD gives. The PRD says the Custom View is invisible because this set
|
| 1017 |
+
#: omits the name so `_clean_display` drops the mode on every write. MEASURED, that write never
|
| 1018 |
+
#: happens: a script view is a rail PROJECTION built client-side by
|
| 1019 |
+
#: `CustomerGrid.tsx::scriptProjectionView`, its panel is gated on `activeScriptId !== null`
|
| 1020 |
+
#: (a row in E's per-database store), there is no `displayMode === "script"` branch at all, and the
|
| 1021 |
+
#: autosave effect reads `views.find(...)` while the projections are appended to `railViews` ALONE.
|
| 1022 |
+
#: Nothing was ever dropped, because nothing was ever sent.
|
| 1023 |
+
#: β WHAT THE NAME ACTUALLY BUYS is the PRECONDITION for the create door the owner asked for: a
|
| 1024 |
+
#: script view can only be offered through the display-mode picker once a view STORED as
|
| 1025 |
+
#: `mode: 'script'` survives a round trip, and until this line it could not. The picker entry
|
| 1026 |
+
#: itself is W37-T41 (lane E) plus `CREATABLE_MODES`, and `_test/icons.test.ts::HELD_MODES` keeps
|
| 1027 |
+
#: the hold with its release condition rewritten to say so.
|
| 1028 |
+
#: β CONTRACT C2 MAKES THIS SET AND `customer-grid/types.ts::DISPLAY_MODES` IDENTICAL NAME FOR
|
| 1029 |
+
#: NAME, and `verify_fields_contract.py` now asserts it in BOTH directions. That retires the staged
|
| 1030 |
+
#: hold every mode from `timeseries` to `form` was landed with β a client-first name is now RED
|
| 1031 |
+
#: rather than merely risky. The protection the hold gave is structural instead: both halves are in
|
| 1032 |
+
#: one lane's fence, so they ship in one change, and whether a mode may be CREATED is a separate
|
| 1033 |
+
#: question answered by `CREATABLE_MODES` / `HELD_MODES`.
|
| 1034 |
DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart',
|
| 1035 |
+
'timeseries', 'catalog', 'swipe', 'form', 'script'}
|
| 1036 |
+
|
| 1037 |
+
#: W37-T20 (C2) β the cap on `display.script.id`. Script-view ids are minted server-side as
|
| 1038 |
+
#: `'sv_' + secrets.token_urlsafe(9)` (`routes_script_views.py::create_script_view`), so ~15 chars;
|
| 1039 |
+
#: 64 is a bound on a hostile payload, not an opinion about the format.
|
| 1040 |
+
MAX_SCRIPT_ID = 64
|
| 1041 |
|
| 1042 |
#: ββ WAVE-29 R7 (owner item 10, contracts C3/C4) β THE FORM INTERFACE's stored spec, at
|
| 1043 |
#: `views[<id>].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read
|
|
|
|
| 1050 |
#: so one tenant setting its token to another tenant's value would silently receive that tenant's
|
| 1051 |
#: submissions. The token is minted server-side and lives in a bucket no client write can reach;
|
| 1052 |
#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape.
|
| 1053 |
+
#: ββ WAVE-37 T21 (owner items 1/12, rulings R5+R6, contract C3) β THE PER-VIEW CATALOG SPEC,
|
| 1054 |
+
#: stored at `views[<id>].config.display.catalog`.
|
| 1055 |
+
#:
|
| 1056 |
+
#: ββ IT IS **`catalog`**, SINGULAR, AND IT IS NOT `catalogs`. Both keys live on `display`, they
|
| 1057 |
+
#: are one letter apart, and that is the single most likely mistake anybody touching this file will
|
| 1058 |
+
#: make. They are different things and BOTH are live:
|
| 1059 |
+
#: Β· `catalogs` (wave-18 C6) β a LIST of up to 12 print artifacts, each with its own paper,
|
| 1060 |
+
#: brand and pages of kind cover/intro/section/gallery. Authored CONTENT. UNTOUCHED by T21.
|
| 1061 |
+
#: Β· `catalog` (wave-37 C3) β THIS view's catalog SETTINGS: which column is the picture, which
|
| 1062 |
+
#: is the title (R5: per view, so two catalog views on one database may differ), plus the page
|
| 1063 |
+
#: order and the free-placed items R6 asks for.
|
| 1064 |
+
#: Neither replaces the other, and neither validator reads the other's key.
|
| 1065 |
+
#:
|
| 1066 |
+
#: β THE MIGRATION IS A RULE ABOUT WHAT THIS FUNCTION MUST **NOT** DO (R6, C3). An item authored
|
| 1067 |
+
#: before free placement has no `x`/`y`, and an ABSENT coordinate must read back ABSENT β never
|
| 1068 |
+
#: normalised to 0. Defaulting to zero would pile every pre-existing item at the origin, which is
|
| 1069 |
+
#: "a catalog that silently loses its layout is a FAIL, not a migration" stated as code. The
|
| 1070 |
+
#: RENDERER turns an absent coordinate into flow position N; the host's whole job is to not destroy
|
| 1071 |
+
#: the distinction it reads.
|
| 1072 |
+
#: β `w`/`h` carry the OPPOSITE asymmetry, deliberately: a zero WIDTH is an invisible item, so 0
|
| 1073 |
+
#: is refused and absent means "the renderer's default size". A zero X is the top-left corner and is
|
| 1074 |
+
#: a real, storable position. Two keys of one shape with two rules β the `kanbanClamp` family β
|
| 1075 |
+
#: which is why each is written out rather than looped.
|
| 1076 |
+
MAX_CATALOG_SECTIONS = 20
|
| 1077 |
+
#: A bound on a hostile payload, not an opinion about the renderer's coordinate space: D owns what
|
| 1078 |
+
#: the numbers MEAN. Stored to 2 decimals, finer than any print surface resolves.
|
| 1079 |
+
MAX_CATALOG_COORD = 10000
|
| 1080 |
+
#: The page `order` bound. ABSENT => that page's ARRAY INDEX, and `order` WINS where present; ties
|
| 1081 |
+
#: break by array index. Decided HERE rather than left to the renderer, because two readers guessing
|
| 1082 |
+
#: differently about page order is exactly the divergence contract C3 exists to close.
|
| 1083 |
+
MAX_CATALOG_ORDER = 9999
|
| 1084 |
+
|
| 1085 |
+
|
| 1086 |
+
def _clean_catalog_coord(raw, allow_zero):
|
| 1087 |
+
"""One free-placement number, or None. `allow_zero` False is the w/h leg (see the header).
|
| 1088 |
+
|
| 1089 |
+
β THE ROUNDING IS `floor(v * 100 + 0.5) / 100` AND NOT `round(v, 2)`, AND THAT IS THE WHOLE
|
| 1090 |
+
POINT OF THIS FUNCTION EXISTING SEPARATELY. Python's `round` is BANKER'S rounding and
|
| 1091 |
+
JavaScript's `Math.round` is half-up-toward-+Infinity, so the two engines disagree on every
|
| 1092 |
+
exact half: `round(0.125, 2)` is 0.12 here and 0.13 in the browser. One dragged item landing two
|
| 1093 |
+
hundredths apart on the two engines is a `display` object that never compares equal β the
|
| 1094 |
+
silent mirror drift contract C2 exists to prevent, arriving through arithmetic instead of
|
| 1095 |
+
through a missing key. The expression below is `Math.floor(v * 100 + 0.5) / 100` written in
|
| 1096 |
+
Python, evaluated in the same IEEE doubles, so the two agree bit for bit.
|
| 1097 |
+
β AND AN INTEGRAL RESULT IS EMITTED AS AN `int`: `12.0` serialises as `12.0` in Python and
|
| 1098 |
+
`12` in JSON.stringify, which is one stored value with two spellings.
|
| 1099 |
+
"""
|
| 1100 |
+
if isinstance(raw, bool) or not isinstance(raw, (int, float)):
|
| 1101 |
+
return None
|
| 1102 |
+
if raw != raw or raw in (float("inf"), float("-inf")): # NaN / +-inf
|
| 1103 |
+
return None
|
| 1104 |
+
val = math.floor(float(raw) * 100 + 0.5) / 100
|
| 1105 |
+
if val > MAX_CATALOG_COORD or val < -MAX_CATALOG_COORD:
|
| 1106 |
+
return None
|
| 1107 |
+
if not allow_zero and val <= 0:
|
| 1108 |
+
return None
|
| 1109 |
+
return int(val) if val == int(val) else val
|
| 1110 |
+
|
| 1111 |
+
|
| 1112 |
+
def _clean_catalog_item(raw):
|
| 1113 |
+
"""One free-placed item: `{code, x?, y?, w?, h?}`. `code` REQUIRED; every coordinate optional.
|
| 1114 |
+
|
| 1115 |
+
Key EMISSION ORDER is fixed and mirrored byte-for-byte by `types.ts::cleanCatalogSpec` β an
|
| 1116 |
+
accepted save has to read back identically on both engines, and this pair is where that is
|
| 1117 |
+
decided.
|
| 1118 |
+
"""
|
| 1119 |
+
if not isinstance(raw, dict):
|
| 1120 |
+
return None
|
| 1121 |
+
code = raw.get("code")
|
| 1122 |
+
if not isinstance(code, str) or not code.strip():
|
| 1123 |
+
return None
|
| 1124 |
+
out = {"code": code.strip()[:CATALOG_CODE_MAX]}
|
| 1125 |
+
for key, allow_zero in (("x", True), ("y", True), ("w", False), ("h", False)):
|
| 1126 |
+
val = _clean_catalog_coord(raw.get(key), allow_zero)
|
| 1127 |
+
if val is not None:
|
| 1128 |
+
out[key] = val
|
| 1129 |
+
return out
|
| 1130 |
+
|
| 1131 |
+
|
| 1132 |
+
def _clean_catalog_spec(raw, valid_keys):
|
| 1133 |
+
"""C3 β `display.catalog`, fail-closed. See the header block above for what it is NOT.
|
| 1134 |
+
|
| 1135 |
+
Field refs follow the `dateField`/`stackField` rule this module already runs: a ref naming a
|
| 1136 |
+
deleted column is dropped INDIVIDUALLY and the renderer falls back to its per-mode default. It
|
| 1137 |
+
never costs the user their pages β the `charts` per-entry-drop precedent, applied to the one
|
| 1138 |
+
payload here that is AUTHORED rather than derived from the rows.
|
| 1139 |
+
"""
|
| 1140 |
+
if not isinstance(raw, dict):
|
| 1141 |
+
return None
|
| 1142 |
+
out = {}
|
| 1143 |
+
for ref in ("imageField", "titleField"):
|
| 1144 |
+
if raw.get(ref) in valid_keys:
|
| 1145 |
+
out[ref] = raw[ref]
|
| 1146 |
+
pages_raw = raw.get("pages")
|
| 1147 |
+
pages = []
|
| 1148 |
+
if isinstance(pages_raw, list):
|
| 1149 |
+
# ONE cumulative item budget across the whole spec, spent in page order then section order
|
| 1150 |
+
# β the `_clean_catalogs` construction, so a 40-page catalog cannot smuggle 20,000 items
|
| 1151 |
+
# past a per-page cap.
|
| 1152 |
+
budget = MAX_CATALOG_CODES
|
| 1153 |
+
seen_pages = set()
|
| 1154 |
+
for raw_page in pages_raw:
|
| 1155 |
+
if len(pages) >= MAX_CATALOG_PAGES:
|
| 1156 |
+
break
|
| 1157 |
+
if not isinstance(raw_page, dict):
|
| 1158 |
+
continue
|
| 1159 |
+
pid = str(raw_page.get("id") or "").strip()[:CATALOG_ID_MAX]
|
| 1160 |
+
if not pid or pid in seen_pages:
|
| 1161 |
+
continue
|
| 1162 |
+
seen_pages.add(pid)
|
| 1163 |
+
page = {"id": pid}
|
| 1164 |
+
order = raw_page.get("order")
|
| 1165 |
+
if (isinstance(order, int) and not isinstance(order, bool)
|
| 1166 |
+
and 0 <= order <= MAX_CATALOG_ORDER):
|
| 1167 |
+
page["order"] = order
|
| 1168 |
+
sections_raw = raw_page.get("sections")
|
| 1169 |
+
sections = []
|
| 1170 |
+
if isinstance(sections_raw, list):
|
| 1171 |
+
seen_sections = set()
|
| 1172 |
+
for raw_section in sections_raw:
|
| 1173 |
+
if len(sections) >= MAX_CATALOG_SECTIONS:
|
| 1174 |
+
break
|
| 1175 |
+
if not isinstance(raw_section, dict):
|
| 1176 |
+
continue
|
| 1177 |
+
sid = str(raw_section.get("id") or "").strip()[:CATALOG_ID_MAX]
|
| 1178 |
+
if not sid or sid in seen_sections:
|
| 1179 |
+
continue
|
| 1180 |
+
seen_sections.add(sid)
|
| 1181 |
+
section = {"id": sid}
|
| 1182 |
+
items_raw = raw_section.get("items")
|
| 1183 |
+
items = []
|
| 1184 |
+
if isinstance(items_raw, list) and budget > 0:
|
| 1185 |
+
for raw_item in items_raw:
|
| 1186 |
+
item = _clean_catalog_item(raw_item)
|
| 1187 |
+
if item is None:
|
| 1188 |
+
continue
|
| 1189 |
+
items.append(item)
|
| 1190 |
+
budget -= 1
|
| 1191 |
+
if budget <= 0:
|
| 1192 |
+
break
|
| 1193 |
+
# β NOT de-duplicated by `code`. The same product legitimately appears twice in
|
| 1194 |
+
# one section at two positions β that is what free placement IS β and wave-18's
|
| 1195 |
+
# "deduped WITHIN a page" rule was written for a LISTING, where a repeat is a
|
| 1196 |
+
# mistake. Here it is the feature.
|
| 1197 |
+
if items:
|
| 1198 |
+
section["items"] = items
|
| 1199 |
+
sections.append(section)
|
| 1200 |
+
# An EMPTY section is KEPT: a person adds a New Section and then drags into it, and
|
| 1201 |
+
# dropping it would delete the thing they just made, between two autosaves.
|
| 1202 |
+
if sections:
|
| 1203 |
+
page["sections"] = sections
|
| 1204 |
+
pages.append(page)
|
| 1205 |
+
if pages:
|
| 1206 |
+
out["pages"] = pages
|
| 1207 |
+
# A spec that reduces to nothing is DROPPED, never stored as `{}`. "No image field, no title
|
| 1208 |
+
# field, no pages" IS the unconfigured state, and `{}` would be its second spelling οΏ½οΏ½οΏ½ the
|
| 1209 |
+
# no-churn law every literal-only key in `_clean_display` follows.
|
| 1210 |
+
return out or None
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
FORM_ACCESS = ('public', 'emails')
|
| 1214 |
MAX_FORM_FIELDS = 60
|
| 1215 |
MAX_FORM_EMAILS = 200
|
|
|
|
| 1657 |
# saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
|
| 1658 |
mode = _LEGACY_MODES.get(mode, mode)
|
| 1659 |
out = {'mode': mode}
|
| 1660 |
+
# β W37-T28 (owner item 15) β `coordField`: ONE column holding `"lat,lon"`, so a map is no
|
| 1661 |
+
# longer a customers-only feature. It joins this loop rather than getting a clause of its own
|
| 1662 |
+
# because it IS a field ref with the same drop-if-deleted rule; `customer-grid/types.ts`
|
| 1663 |
+
# gains it in the SAME change, which is contract C2's whole point.
|
| 1664 |
+
for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'):
|
| 1665 |
if raw.get(ref) in valid_keys:
|
| 1666 |
out[ref] = raw[ref]
|
| 1667 |
# ββ C-DISP (wave 2026-08-02) βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 1796 |
catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys)
|
| 1797 |
if catalogs:
|
| 1798 |
out['catalogs'] = catalogs
|
| 1799 |
+
# ββ ββ WAVE-37 C3 (T21) β `catalog`, SINGULAR, and it is NOT the line above it. See the
|
| 1800 |
+
# header on `_clean_catalog_spec` for the two-keys-one-letter-apart warning; this call site is
|
| 1801 |
+
# where a reader is most likely to "fix" one into the other.
|
| 1802 |
+
catalog = _clean_catalog_spec(raw.get('catalog'), valid_keys)
|
| 1803 |
+
if catalog:
|
| 1804 |
+
out['catalog'] = catalog
|
| 1805 |
# ββ β WAVE-27 C3 (item 8 / R2): the swipe binding ββββββββββββββββββββββββββββββββββββ
|
| 1806 |
# `{fieldKey, leftOption, rightOption}` β WHOLE-KEY drop, never a partial one, and that
|
| 1807 |
# asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight.
|
|
|
|
| 1830 |
# `kanbanClamp` law) wearing a control that promises a choice.
|
| 1831 |
if left and right and left.casefold() != right.casefold():
|
| 1832 |
out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right}
|
| 1833 |
+
# ββ ββ WAVE-37 T20 (owner item 10 / R2, contract C2): WHICH SCRIPT THIS VIEW RENDERS ββ
|
| 1834 |
+
#
|
| 1835 |
+
# `{id}` β the row in E's per-database script store (`/api/v1/script-views`) whose source this
|
| 1836 |
+
# view draws. WHOLE-KEY drop, the `swipe` construction rather than the `charts` one: a script
|
| 1837 |
+
# view with no id is not a degraded script view, it is a panel with nothing to run, and the
|
| 1838 |
+
# honest state for that is the unconfigured one the client already has a surface for.
|
| 1839 |
+
#
|
| 1840 |
+
# β THE ID IS DELIBERATELY NOT CHECKED AGAINST `valid_keys`, AND THAT IS NOT AN OVERSIGHT.
|
| 1841 |
+
# `valid_keys` is this table's FIELD keys; a script id names a record in a DIFFERENT store that
|
| 1842 |
+
# this function cannot reach. So the host bounds the shape and nothing else, and the renderer
|
| 1843 |
+
# owns the one loss this is blind to β the script row was deleted β which it must SHOW rather
|
| 1844 |
+
# than fall back to another script (the `viewModes.tsx` house rule `_clean_swipe`'s note cites).
|
| 1845 |
+
# β Mirrored key-for-key by `customer-grid/types.ts::cleanDisplay`. That file rebuilds the
|
| 1846 |
+
# config key by key on every autosave, so a key accepted HERE and unknown THERE is erased on the
|
| 1847 |
+
# next column resize β which is the half of the round trip this docstring's W33 note warns about.
|
| 1848 |
+
script = raw.get('script')
|
| 1849 |
+
if isinstance(script, dict):
|
| 1850 |
+
s_id = script.get('id')
|
| 1851 |
+
if isinstance(s_id, str) and s_id.strip():
|
| 1852 |
+
out['script'] = {'id': s_id.strip()[:MAX_SCRIPT_ID]}
|
| 1853 |
# ββ ββ WAVE-29 R7 (item 10): the FORM spec β see `_clean_form` for why the token is not here.
|
| 1854 |
form = _clean_form(raw.get('form'), valid_keys)
|
| 1855 |
if form:
|
|
|
|
| 2057 |
return out
|
| 2058 |
|
| 2059 |
|
| 2060 |
+
def view_refs_of(nodes):
|
| 2061 |
+
"""C4 β every view id a tree REFERS to, deduped, in order. Mirrors `types.ts::viewRefsOf`.
|
| 2062 |
+
|
| 2063 |
+
β An INACTIVE leaf (no view chosen yet) is not a reference, on both engines: counting one
|
| 2064 |
+
would refuse a save the moment somebody opened the condition and had not picked anything.
|
| 2065 |
+
"""
|
| 2066 |
+
out = []
|
| 2067 |
+
for node in nodes or []:
|
| 2068 |
+
if not isinstance(node, dict):
|
| 2069 |
+
continue
|
| 2070 |
+
if 'children' in node:
|
| 2071 |
+
for vid in view_refs_of(node.get('children')):
|
| 2072 |
+
if vid not in out:
|
| 2073 |
+
out.append(vid)
|
| 2074 |
+
continue
|
| 2075 |
+
if node.get('colId') != VIEW_FIELD:
|
| 2076 |
+
continue
|
| 2077 |
+
vid = str(node.get('value') or '').strip()[:MAX_VIEW_ID]
|
| 2078 |
+
if vid and vid not in out:
|
| 2079 |
+
out.append(vid)
|
| 2080 |
+
return out
|
| 2081 |
+
|
| 2082 |
+
|
| 2083 |
+
def view_filter_cycle(view_id, nodes, filters_of):
|
| 2084 |
+
"""ββ W37-T26 / C4 β THE CYCLE REFUSAL's host half. Mirrors `types.ts::viewFilterCycle`.
|
| 2085 |
+
|
| 2086 |
+
View X filtered on "is in View Y" where Y is filtered on X cannot be resolved by anything:
|
| 2087 |
+
X needs Y needs X. Returns the CYCLE PATH (ids, starting and ending at `view_id`) or None.
|
| 2088 |
+
|
| 2089 |
+
β THE CLIENT REFUSES THE SAME SHAPE AND THAT IS NOT DUPLICATION. The browser's guard is what
|
| 2090 |
+
a PERSON sees and is the only one that can explain itself; this one is what makes the rule true
|
| 2091 |
+
for a caller that POSTs a view directly, which is every guard-in-the-browser's blind spot.
|
| 2092 |
+
β A view referring to ITSELF is a cycle of length one and is caught by the same walk β also
|
| 2093 |
+
the case a person reaches most easily, by duplicating a view.
|
| 2094 |
+
"""
|
| 2095 |
+
def walk(current, path, seen):
|
| 2096 |
+
refs = view_refs_of(nodes if current == view_id else filters_of(current))
|
| 2097 |
+
for nxt in refs:
|
| 2098 |
+
if nxt == view_id:
|
| 2099 |
+
return path + [nxt]
|
| 2100 |
+
if nxt in seen:
|
| 2101 |
+
continue
|
| 2102 |
+
seen.add(nxt)
|
| 2103 |
+
found = walk(nxt, path + [nxt], seen)
|
| 2104 |
+
if found:
|
| 2105 |
+
return found
|
| 2106 |
+
return None
|
| 2107 |
+
|
| 2108 |
+
return walk(view_id, [view_id], {view_id})
|
| 2109 |
+
|
| 2110 |
+
|
| 2111 |
+
def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None,
|
| 2112 |
+
visible_view_ids=None):
|
| 2113 |
"""Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
|
| 2114 |
|
| 2115 |
Returns a clean tree of leaf conditions ({colId, op, value, value2}) and
|
|
|
|
| 2128 |
condition that can only match nothing, so `List is not [deleted]` would show an empty table
|
| 2129 |
forever with no way to tell why. `None` means this host has no cohorts, and then every
|
| 2130 |
cohort leaf is dropped: fail-closed, like every other unknown key.
|
| 2131 |
+
|
| 2132 |
+
ββ W37-T26 (C4) β `visible_view_ids` is the same parameter one leaf over: the views this
|
| 2133 |
+
caller may name in a `__view__` membership leaf. A leaf naming anything else is DROPPED here
|
| 2134 |
+
for the identical reason, and `None` (this host has no views to offer) drops every one.
|
| 2135 |
+
β DEFAULTS TO `None` SO EVERY EXISTING CALLER IS BYTE-IDENTICAL. Three callers pass nothing
|
| 2136 |
+
(`routes_admin`, `routes_query` twice) and each of them is a surface with no saved-view
|
| 2137 |
+
vocabulary of its own; under the fail-closed rule they drop the leaf, which is correct rather
|
| 2138 |
+
than merely safe.
|
| 2139 |
"""
|
| 2140 |
if budget is None:
|
| 2141 |
budget = [MAX_FILTER_NODES]
|
|
|
|
| 2150 |
continue # too deep -> drop
|
| 2151 |
budget[0] -= 1
|
| 2152 |
children = clean_filter_tree(node['children'], valid_keys,
|
| 2153 |
+
depth + 1, budget, cohort_ids, visible_view_ids)
|
| 2154 |
if children:
|
| 2155 |
out.append({'conj': 'or' if node.get('conj') == 'or' else 'and',
|
| 2156 |
'children': children})
|
|
|
|
| 2167 |
out.append({'colId': COHORT_FIELD, 'op': op,
|
| 2168 |
'value': ','.join(named), 'value2': ''})
|
| 2169 |
continue
|
| 2170 |
+
if node.get('colId') == VIEW_FIELD: # C4: a view-membership leaf
|
| 2171 |
+
# β THE SAME ALL-OR-NOTHING THE COHORT ARM ABOVE APPLIES, AND FOR THE SAME REASON.
|
| 2172 |
+
# A leaf naming a view this caller cannot see is DROPPED rather than kept, because a
|
| 2173 |
+
# kept one can only ever match nothing: `View is not [a view you cannot see]` would
|
| 2174 |
+
# show an empty table forever with no way to tell why. Dropping it puts the view back
|
| 2175 |
+
# in its honest unfiltered state, which is what `clean_filter_tree`'s own docstring
|
| 2176 |
+
# already promises for cohorts.
|
| 2177 |
+
# β `visible_views` is what the caller passes; `None` means this host does not know,
|
| 2178 |
+
# and then EVERY view leaf is dropped β fail-closed, like every other unknown key.
|
| 2179 |
+
v_op = node.get('op')
|
| 2180 |
+
v_id = str(node.get('value') or '').strip()[:MAX_VIEW_ID]
|
| 2181 |
+
if v_op in VIEW_OPS and v_id and v_id in (visible_view_ids or ()):
|
| 2182 |
+
budget[0] -= 1
|
| 2183 |
+
out.append({'colId': VIEW_FIELD, 'op': v_op, 'value': v_id, 'value2': ''})
|
| 2184 |
+
continue
|
| 2185 |
if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS:
|
| 2186 |
budget[0] -= 1
|
| 2187 |
# `or ''` would be wrong here: it maps every FALSY value to '', and '' is the
|
platform/aios_grid_fields.json
CHANGED
|
@@ -386,6 +386,20 @@
|
|
| 386 |
"source": "odoo",
|
| 387 |
"description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 388 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 389 |
{
|
| 390 |
"key": "rev_ytd",
|
| 391 |
"label": "Revenue YTD",
|
|
|
|
| 386 |
"source": "odoo",
|
| 387 |
"description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 388 |
},
|
| 389 |
+
{
|
| 390 |
+
"key": "tier_prices",
|
| 391 |
+
"label": "Tier prices",
|
| 392 |
+
"type": "json",
|
| 393 |
+
"source": "odoo",
|
| 394 |
+
"description": "Every live pricelist that prices this SKU today, as [{pricelist, unit_price}] at qty 1. Blank when no list prices it. This is the honest SET; the three Fisch/Royal columns beside it are the DECLARED subset and cannot show a price on a list the contract does not name."
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"key": "units",
|
| 398 |
+
"label": "Units",
|
| 399 |
+
"type": "json",
|
| 400 |
+
"source": "odoo",
|
| 401 |
+
"description": "The units of measure this SKU is really sold in, as [{name, qty}] where qty is in the product's own unit. Blank means it is sold in ONE unit, not that data is missing - only 1,150 of 5,873 active SKUs (19.6%) carry a unit tier."
|
| 402 |
+
},
|
| 403 |
{
|
| 404 |
"key": "rev_ytd",
|
| 405 |
"label": "Revenue YTD",
|
platform/core/grid_events.py
CHANGED
|
@@ -801,9 +801,39 @@ def handle_one(event, ctx):
|
|
| 801 |
# else is dropped here. Leaving it in would persist a condition the engine can only
|
| 802 |
# answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table
|
| 803 |
# forever with nothing on screen explaining why.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 804 |
filters = _ag.clean_filter_tree(cfg.get('filters'),
|
| 805 |
valid_keys | set(measure_keys),
|
| 806 |
-
cohort_ids=set(cohort_ids)
|
|
|
|
| 807 |
sorts = []
|
| 808 |
for rule in list(cfg.get('sorts') or [])[:20]:
|
| 809 |
if (isinstance(rule, dict) and rule.get('colId') in valid_keys
|
|
|
|
| 801 |
# else is dropped here. Leaving it in would persist a condition the engine can only
|
| 802 |
# answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table
|
| 803 |
# forever with nothing on screen explaining why.
|
| 804 |
+
# ββ W37-T26 / CONTRACT C4 β `visible_view_ids` is the same PERMISSION list one leaf over:
|
| 805 |
+
# a `__view__` membership leaf naming a view this caller cannot see is dropped here, for
|
| 806 |
+
# the identical reason the cohort line above gives. Without it the arm is fail-closed and
|
| 807 |
+
# every view leaf is dropped on save, so the condition would vanish on the next read β
|
| 808 |
+
# D-90's shape exactly.
|
| 809 |
+
# β `ctx.visible_views` is already populated (`routes_grid.py` passes it, `:714` reads it),
|
| 810 |
+
# so this needs no new plumbing; it is the value nobody had handed to the validator.
|
| 811 |
+
# β Edited by lane C, not this file's owner: `platform/core/grid_events.py` is in NO fence
|
| 812 |
+
# and lane C was the last session open. See `mailbox/C.md` NOTE C-24.
|
| 813 |
+
_view_ids = {str(v.get('id')) for v in (visible_views or ())
|
| 814 |
+
if isinstance(v, dict) and v.get('id')}
|
| 815 |
+
# ββ THE CYCLE REFUSAL'S HOST HALF (C4), and it is the ticket's `done-when` rather than a
|
| 816 |
+
# guard bolted on afterwards. View X filtered on "is in View Y" where Y is filtered on X
|
| 817 |
+
# cannot be resolved by anything: X needs Y needs X. The browser refuses it too, with a
|
| 818 |
+
# sentence a person reads; this one is what makes the rule true for a caller that POSTs a
|
| 819 |
+
# view directly, which is every guard-in-the-browser's blind spot.
|
| 820 |
+
# β REFUSED, NOT REPAIRED. Dropping the offending leaf would save a view the person did not
|
| 821 |
+
# write and leave the panel showing a condition the config does not carry.
|
| 822 |
+
_cycle = _ag.view_filter_cycle(
|
| 823 |
+
view_id, cfg.get('filters'),
|
| 824 |
+
lambda vid: next((v.get('config', {}).get('filters')
|
| 825 |
+
for v in (visible_views or ())
|
| 826 |
+
if isinstance(v, dict) and str(v.get('id')) == vid), None))
|
| 827 |
+
if _cycle:
|
| 828 |
+
_refuse(ctx, 'view_cycle',
|
| 829 |
+
'that would make a loop: ' + ' refers to '.join(_cycle) +
|
| 830 |
+
'. A view cannot filter on a view that filters on it, because neither can '
|
| 831 |
+
'be worked out without the other.', view_id)
|
| 832 |
+
return False
|
| 833 |
filters = _ag.clean_filter_tree(cfg.get('filters'),
|
| 834 |
valid_keys | set(measure_keys),
|
| 835 |
+
cohort_ids=set(cohort_ids),
|
| 836 |
+
visible_view_ids=_view_ids)
|
| 837 |
sorts = []
|
| 838 |
for rule in list(cfg.get('sorts') or [])[:20]:
|
| 839 |
if (isinstance(rule, dict) and rule.get('colId') in valid_keys
|
platform/core/perm_scope.py
CHANGED
|
@@ -87,6 +87,36 @@ def may_access(user, module):
|
|
| 87 |
return perms.may_open(user, module) # legacy record: the old grant wall
|
| 88 |
|
| 89 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
def nav_may_open(user, key, st=None):
|
| 91 |
"""May this account SEE `key` β in the nav, and at the route gate? The ADMISSION question.
|
| 92 |
|
|
@@ -148,7 +178,103 @@ def assistant_entry(user, module):
|
|
| 148 |
|
| 149 |
|
| 150 |
# ββ FIELDS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 151 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
"""The TRANSITIVE closure of hidden field keys (C-PERM amendment 5).
|
| 153 |
|
| 154 |
β WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER
|
|
@@ -180,10 +306,26 @@ def hidden_keys(user, module, fields):
|
|
| 180 |
and `rollup.field` is deliberately not treated as one β it names another database's column,
|
| 181 |
which has its own wall.
|
| 182 |
"""
|
| 183 |
-
|
| 184 |
-
if perms.is_admin(user) or not e:
|
| 185 |
return frozenset()
|
| 186 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
if not hidden:
|
| 188 |
return frozenset()
|
| 189 |
|
|
@@ -210,9 +352,16 @@ def hidden_keys(user, module, fields):
|
|
| 210 |
return frozenset(hidden)
|
| 211 |
|
| 212 |
|
| 213 |
-
def visible_fields(fields, user, module):
|
| 214 |
-
"""`fields` minus the hidden closure. Order preserved β the column order is the user's.
|
| 215 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
if not hide:
|
| 217 |
return list(fields or ())
|
| 218 |
return [f for f in (fields or ())
|
|
@@ -767,16 +916,19 @@ def _scoped(user, table_key, st=None, ctx=None, want_rows=True):
|
|
| 767 |
# THE FIELD WALL β a TRANSITIVE closure, so hiding a column also hides every formula computed
|
| 768 |
# FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference
|
| 769 |
# is the wrong shape here.
|
| 770 |
-
|
|
|
|
|
|
|
|
|
|
| 771 |
if not want_rows:
|
| 772 |
-
return (visible_fields(fields, user, key) if hide else fields), []
|
| 773 |
# THE ROW WALL β `permits()`, so a permanent filter this evaluator cannot answer DENIES
|
| 774 |
# rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a
|
| 775 |
# permanent filter may name a column the reader is not allowed to SEE, and dropping the
|
| 776 |
# predicate would widen the read rather than narrow it.
|
| 777 |
rows = apply_row_scope(rows, user, key, fields, ctx)
|
| 778 |
if hide:
|
| 779 |
-
fields = visible_fields(fields, user, key)
|
| 780 |
rows = [strip_row(r, hide) for r in rows]
|
| 781 |
return fields, rows
|
| 782 |
|
|
|
|
| 87 |
return perms.may_open(user, module) # legacy record: the old grant wall
|
| 88 |
|
| 89 |
|
| 90 |
+
def may_metrics(user, module):
|
| 91 |
+
"""May this account build and receive METRIC columns β lookback measures β on `module`?
|
| 92 |
+
|
| 93 |
+
ββ W38-T19 β A CAPABILITY, NOT A SECOND SPELLING OF ACCESS, and the distinction is the
|
| 94 |
+
ticket. A rollup aggregates the CHILDREN a row is linked to; a Metric answers *"this number,
|
| 95 |
+
over this window"* against a governed topic with no relation at all (CLAUDE.md standing rule
|
| 96 |
+
9). So it reads the book behind the rows rather than the rows: an account can be exactly the
|
| 97 |
+
right person to see a customer list and the wrong one to mint 12-month revenue over it. Two
|
| 98 |
+
decisions, two controls.
|
| 99 |
+
|
| 100 |
+
β ABSENCE GRANTS, AND THE ASYMMETRY WITH `may_access` IS DELIBERATE RATHER THAN AN
|
| 101 |
+
OVERSIGHT TO TIDY. `may_access` reads migrated-and-undeclared as DENY, which is right there
|
| 102 |
+
because every save writes an entry for every governed database β absence means an
|
| 103 |
+
administrator decided. No `metrics` KEY was STORABLE before this ticket, so every migrated
|
| 104 |
+
record in every tenant carries none, and reading that absence as DENY would revoke Metrics
|
| 105 |
+
for everybody on the day this shipped with nobody having decided anything. That is the exact
|
| 106 |
+
failure `routes_admin.get_perms` records twice already (the `ut_*` default and the surface
|
| 107 |
+
default), one field further down the same entry. **Only an explicit `metrics: false`
|
| 108 |
+
refuses.**
|
| 109 |
+
|
| 110 |
+
β IT DOES NOT RE-ASK ACCESS. Every caller is already behind the access wall (`module_gate`,
|
| 111 |
+
`may_read`, `session.require`), and folding admission in here would give a denial two
|
| 112 |
+
possible causes with one answer β the shape that makes a permission bug take an afternoon.
|
| 113 |
+
"""
|
| 114 |
+
if perms.is_admin(user):
|
| 115 |
+
return True
|
| 116 |
+
e = entry(user, module)
|
| 117 |
+
return not (isinstance(e, dict) and e.get('metrics') is False)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
def nav_may_open(user, key, st=None):
|
| 121 |
"""May this account SEE `key` β in the nav, and at the route gate? The ADMISSION question.
|
| 122 |
|
|
|
|
| 178 |
|
| 179 |
|
| 180 |
# ββ FIELDS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 181 |
+
#: ββ W38-T16 β THE MARKER THAT SAYS "THIS COLUMN IS GOVERNED BY A GRANT", stamped once by the
|
| 182 |
+
#: door that creates a shared column (`routes_tables.patch_shared_cell`) and never flipped by any
|
| 183 |
+
#: door afterwards. It is the `perms_v` idiom one object down, and it is here for the reason that
|
| 184 |
+
#: marker exists: **absence must not be read as a decision.**
|
| 185 |
+
#:
|
| 186 |
+
#: ββ WHY A MARKER AND NOT "DOES A GRANT RECORD EXIST". Every tenant-wide column shipped before
|
| 187 |
+
#: this ticket carries no `field` grant record, because none was STORABLE β `shares.KINDS` had
|
| 188 |
+
#: three members. Reading that absence as "granted to nobody" would blank every existing shared
|
| 189 |
+
#: column for every account the moment this arms, which is not a permission fix, it is an outage
|
| 190 |
+
#: (`may_read` leg 3 carries the same argument for `ut_*` keys, in the same words).
|
| 191 |
+
#:
|
| 192 |
+
#: ββ AND IT IS WHAT MAKES THE WALL FAIL **CLOSED**. The polarity here is the opposite of every
|
| 193 |
+
#: other grant check in this repo: a field share is a GRANT, so "no grant" has to mean HIDDEN or
|
| 194 |
+
#: the wall does nothing β but "no grant record" also describes a legacy column and a store that
|
| 195 |
+
#: could not be read. The marker separates the three: `granted` on the column means an explicit
|
| 196 |
+
#: decision was taken, so an unreadable registry hides it; no marker means legacy, so nothing
|
| 197 |
+
#: changes. Without it, one unreadable read of `object_shares` would publish every governed
|
| 198 |
+
#: column to the whole tenant, silently ([[a-guard-for-the-dangerous-case]]).
|
| 199 |
+
#:
|
| 200 |
+
#: β IT IS NOT `shared_overlay`'s `"shared": True` AND MUST NOT BE CONFUSED WITH IT. That flag is
|
| 201 |
+
#: written and never read (D-414) β `is_shared` is a dict-membership test β so it is not evidence
|
| 202 |
+
#: of anything. This one is read HERE, on every assembly, and the only writer is the create door.
|
| 203 |
+
FIELD_GRANT_MARK = 'granted'
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def granted_field_keys(fields):
|
| 207 |
+
"""Every column in `fields` that declares itself GOVERNED by a `shares` grant.
|
| 208 |
+
|
| 209 |
+
β THE CHEAP HALF OF THE WALL, AND IT IS WHY THE WALL COSTS NOTHING FOR ALMOST EVERYBODY. It
|
| 210 |
+
is a scan of dicts already in memory, so a database with no governed column reaches no store
|
| 211 |
+
at all and `hidden_keys` behaves exactly as it did before this ticket. The registry is only
|
| 212 |
+
opened once this answers non-empty.
|
| 213 |
+
"""
|
| 214 |
+
return {str(f['key']) for f in (fields or ())
|
| 215 |
+
if isinstance(f, dict) and f.get('key')
|
| 216 |
+
and f.get(FIELD_GRANT_MARK) is True}
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def field_grant_hidden(user, table_key, fields, st=None):
|
| 220 |
+
"""The governed columns of `table_key` this principal holds NO grant on β C1's per-FIELD wall.
|
| 221 |
+
|
| 222 |
+
ββ W38-T16 / R7 / R8 β THE THIRD WALL, AND IT COMPOSES ALONGSIDE THE OTHER TWO RATHER THAN
|
| 223 |
+
INSIDE THEM. `may_open` answers *IF* you reach a database; `may_read`'s stored `access: false`
|
| 224 |
+
overlay may REVOKE one; this answers *WHICH COLUMNS* of it you receive. It is deliberately not
|
| 225 |
+
threaded through that overlay: the overlay is **deny-only** by its own docstring (*"it may
|
| 226 |
+
revoke a database the wall below would admit, and it may never grant one that wall refuses"*)
|
| 227 |
+
and a field share is a GRANT β the widening direction. Merging them would give the codebase a
|
| 228 |
+
second idea of who grants what, which is the failure `may_open`'s own note is the record of.
|
| 229 |
+
It composes the way `may_open`'s grant leg does: additively, last, fail-closed.
|
| 230 |
+
|
| 231 |
+
β AN ADMIN IS NOT WALLED (break-glass, as everywhere else) and neither is the column's OWNER β
|
| 232 |
+
`shares.role_for` answers `'owner'` for the creator, so a person cannot lose their own column
|
| 233 |
+
by forgetting to share it with themselves.
|
| 234 |
+
|
| 235 |
+
β `st` IS THE TENANT HANDLE AND ITS ABSENCE IS SAFE HERE, unlike everywhere else. A caller
|
| 236 |
+
that cannot lend one reads the module-default bucket; on any tenant but #0 that finds no
|
| 237 |
+
grant, and no grant on a MARKED column means HIDDEN. So a door that has not learned to thread
|
| 238 |
+
`st` under-shares rather than over-shares, and the symptom is a grantee who cannot see their
|
| 239 |
+
column β visible, reportable, and the opposite of a leak.
|
| 240 |
+
"""
|
| 241 |
+
if perms.is_admin(user):
|
| 242 |
+
return set()
|
| 243 |
+
marked = granted_field_keys(fields)
|
| 244 |
+
if not marked:
|
| 245 |
+
return set()
|
| 246 |
+
uname = str((user or {}).get('username') or '').strip().lower()
|
| 247 |
+
try:
|
| 248 |
+
import core.shares as shares
|
| 249 |
+
except Exception: # noqa: BLE001
|
| 250 |
+
return set(marked)
|
| 251 |
+
if not uname:
|
| 252 |
+
# No principal, and a marked column is an explicit decision: nobody is not somebody.
|
| 253 |
+
return set(marked)
|
| 254 |
+
# β THE CREATOR IS READ OFF THE COLUMN, NOT OUT OF THE REGISTRY, AND THAT IS NOT A SECOND
|
| 255 |
+
# AUTHORITY. `createdBy` is ALREADY what decides who may DELETE a shared column (R8 / D-172,
|
| 256 |
+
# `routes_tables.delete_shared_field`) and who may CLAIM it (`routes_shares._owns_object`);
|
| 257 |
+
# asking the same field here keeps one answer to "whose column is this" across all three.
|
| 258 |
+
# β IT IS ALSO THE ONLY THING THAT SURVIVES A CLAIM THAT NEVER LANDED. The create door writes
|
| 259 |
+
# the definition first and the grant record second, on purpose β so the window where a column
|
| 260 |
+
# is marked and unclaimed exists, and without this line its own author would be walled out of
|
| 261 |
+
# the column they just made, permanently and with no way to fix it but an admin.
|
| 262 |
+
mine = {str(f['key']) for f in (fields or ())
|
| 263 |
+
if isinstance(f, dict) and f.get('key') and str(f['key']) in marked
|
| 264 |
+
and str(f.get('createdBy') or '').strip().lower() == uname}
|
| 265 |
+
hide = set()
|
| 266 |
+
for key in marked - mine:
|
| 267 |
+
try:
|
| 268 |
+
oid = shares.field_oid(table_key, key)
|
| 269 |
+
role = shares.role_for('field', oid, uname, is_admin=False, st=st)
|
| 270 |
+
except Exception: # noqa: BLE001
|
| 271 |
+
role = None
|
| 272 |
+
if role is None:
|
| 273 |
+
hide.add(key)
|
| 274 |
+
return hide
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def hidden_keys(user, module, fields, st=None):
|
| 278 |
"""The TRANSITIVE closure of hidden field keys (C-PERM amendment 5).
|
| 279 |
|
| 280 |
β WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER
|
|
|
|
| 306 |
and `rollup.field` is deliberately not treated as one β it names another database's column,
|
| 307 |
which has its own wall.
|
| 308 |
"""
|
| 309 |
+
if perms.is_admin(user):
|
|
|
|
| 310 |
return frozenset()
|
| 311 |
+
# ββ W38-T16 β TWO SOURCES OF HIDING, ONE CLOSURE, AND THAT UNION IS THE WHOLE INTEGRATION.
|
| 312 |
+
#
|
| 313 |
+
# The administrator's `hiddenFields` and a field's own grant answer different questions and
|
| 314 |
+
# both end in the same place: a key this reader may not receive. Seeded together HERE, before
|
| 315 |
+
# the fixpoint, so the transitive argument above covers the new source unchanged β a formula
|
| 316 |
+
# (or a rollup) over a column this reader was not granted comes out with it, or the value
|
| 317 |
+
# leaks wearing the derived column's name.
|
| 318 |
+
#
|
| 319 |
+
# β AND THIS FUNCTION IS THE INSERTION POINT RATHER THAN `_scoped`, WHICH IS WHAT THE TICKET
|
| 320 |
+
# ASSUMED. `_scoped` is C1's evaluator and reaches C1's two doors; **the product reads through
|
| 321 |
+
# neither of them.** Every grid door calls THIS: `routes_customers:196`, `routes_products:345`,
|
| 322 |
+
# `routes_odoo_tables:920/1119`, `routes_tables._ut_hidden/_ut_field_wall`, `routes_slack:190`,
|
| 323 |
+
# `routes_grid:57/69/787` β and `grid_events` walls WRITES off the same set through
|
| 324 |
+
# `EventCtx.hidden_keys`. One evaluator, every door, both wires, read and write.
|
| 325 |
+
hidden = set(field_grant_hidden(user, module, fields, st=st))
|
| 326 |
+
e = entry(user, module)
|
| 327 |
+
if e:
|
| 328 |
+
hidden |= {str(k) for k in (e.get('hiddenFields') or ()) if k}
|
| 329 |
if not hidden:
|
| 330 |
return frozenset()
|
| 331 |
|
|
|
|
| 352 |
return frozenset(hidden)
|
| 353 |
|
| 354 |
|
| 355 |
+
def visible_fields(fields, user, module, st=None):
|
| 356 |
+
"""`fields` minus the hidden closure. Order preserved β the column order is the user's.
|
| 357 |
+
|
| 358 |
+
β W38-T16 β `st` MUST MATCH WHAT ITS CALLER PASSED TO `hidden_keys`, and that is not a style
|
| 359 |
+
note. This recomputes the closure, and since the field-grant leg under-hides without a tenant
|
| 360 |
+
handle, a caller that lends one to `hidden_keys` and not to this would narrow the FIELD LIST
|
| 361 |
+
by MORE than it stripped from the ROWS β the value left sitting in the payload under a column
|
| 362 |
+
nobody can see, which is exactly the half-wall `strip_row`'s note exists to forbid.
|
| 363 |
+
"""
|
| 364 |
+
hide = hidden_keys(user, module, fields, st=st)
|
| 365 |
if not hide:
|
| 366 |
return list(fields or ())
|
| 367 |
return [f for f in (fields or ())
|
|
|
|
| 916 |
# THE FIELD WALL β a TRANSITIVE closure, so hiding a column also hides every formula computed
|
| 917 |
# FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference
|
| 918 |
# is the wrong shape here.
|
| 919 |
+
# β W38-T16 β `st` RIDES INTO THE WALL, not just into the read. The field-grant leg resolves
|
| 920 |
+
# against `object_shares` in THIS tenant's store; without the handle it would answer from the
|
| 921 |
+
# module default (tenant #0) and hide a grantee's own column on every other tenant.
|
| 922 |
+
hide = hidden_keys(user, key, fields, st=st)
|
| 923 |
if not want_rows:
|
| 924 |
+
return (visible_fields(fields, user, key, st=st) if hide else fields), []
|
| 925 |
# THE ROW WALL β `permits()`, so a permanent filter this evaluator cannot answer DENIES
|
| 926 |
# rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a
|
| 927 |
# permanent filter may name a column the reader is not allowed to SEE, and dropping the
|
| 928 |
# predicate would widen the read rather than narrow it.
|
| 929 |
rows = apply_row_scope(rows, user, key, fields, ctx)
|
| 930 |
if hide:
|
| 931 |
+
fields = visible_fields(fields, user, key, st=st)
|
| 932 |
rows = [strip_row(r, hide) for r in rows]
|
| 933 |
return fields, rows
|
| 934 |
|
platform/core/script_sandbox.py
ADDED
|
@@ -0,0 +1,519 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/script_sandbox.py β WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python.
|
| 2 |
+
|
| 3 |
+
Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they
|
| 4 |
+
want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this
|
| 5 |
+
script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was
|
| 6 |
+
stated, so this file is the guardrail, and one engine serves both items.
|
| 7 |
+
|
| 8 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 9 |
+
ββ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE.
|
| 10 |
+
|
| 11 |
+
In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated
|
| 12 |
+
namespace stops import, file, network and environment access β but it **cannot cap memory and
|
| 13 |
+
cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow
|
| 14 |
+
request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**:
|
| 15 |
+
`resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the
|
| 16 |
+
allow-list inside. Neither half is sufficient; both are load-bearing.
|
| 17 |
+
|
| 18 |
+
β AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under
|
| 19 |
+
the CALLING user's record and serialises the result; the child imports nothing from this repo and
|
| 20 |
+
holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store
|
| 21 |
+
path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child
|
| 22 |
+
reports its own `sys.modules`, and no `core.*` name may appear in it.
|
| 23 |
+
|
| 24 |
+
β NEVER A BLACKLIST. Every rule below is an ALLOW-LIST β a set of node types, a set of attribute
|
| 25 |
+
names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and
|
| 26 |
+
the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute
|
| 27 |
+
lookup inside `format`, so there is no `ast.Attribute` node to refuse).
|
| 28 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 29 |
+
|
| 30 |
+
The two layers, and they refuse DIFFERENT things on purpose:
|
| 31 |
+
|
| 32 |
+
1. `check_source()` β a pure function over source text. Refuses a construct the language offers
|
| 33 |
+
and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every
|
| 34 |
+
attribute name outside `ALLOWED_ATTRS`.
|
| 35 |
+
2. `SANDBOX_BUILTINS` β the names that resolve at all. `__import__`, `open`, `eval`, `exec`,
|
| 36 |
+
`compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets
|
| 37 |
+
past layer 1 still finds nothing to call.
|
| 38 |
+
|
| 39 |
+
β THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is
|
| 40 |
+
refused twice, so a negative control that drops ONE layer sees the other refuse and reports
|
| 41 |
+
green β the shape that already cost this wave one missed control in `routes_agent_harness`. So
|
| 42 |
+
each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings,
|
| 43 |
+
and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching
|
| 44 |
+
boundary goes red.
|
| 45 |
+
"""
|
| 46 |
+
import ast
|
| 47 |
+
import json
|
| 48 |
+
import os
|
| 49 |
+
import subprocess
|
| 50 |
+
import sys
|
| 51 |
+
import tempfile
|
| 52 |
+
import time
|
| 53 |
+
from pathlib import Path
|
| 54 |
+
|
| 55 |
+
#: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform.
|
| 56 |
+
DEFAULT_TIMEOUT_S = 10.0
|
| 57 |
+
|
| 58 |
+
#: Address space for the child (`RLIMIT_AS`). POSIX only β see `run()`'s `caps` report.
|
| 59 |
+
DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024
|
| 60 |
+
|
| 61 |
+
#: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the
|
| 62 |
+
#: wall-clock kill is the primary control and this is the backstop for a child that stops being
|
| 63 |
+
#: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal.
|
| 64 |
+
DEFAULT_CPU_SECONDS = 15
|
| 65 |
+
|
| 66 |
+
#: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer,
|
| 67 |
+
#: and the child's real stdout goes to DEVNULL β so a script cannot fill a pipe, and anything
|
| 68 |
+
#: that escaped far enough to write to fd 1 has nowhere for it to land.
|
| 69 |
+
MAX_STDOUT_BYTES = 64 * 1024
|
| 70 |
+
|
| 71 |
+
#: The serialised ROW payload handed to the child. β A REFUSAL, NEVER A TRUNCATION (standing rule
|
| 72 |
+
#: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()`
|
| 73 |
+
#: returns a named limit carrying its cause and a recommendation.
|
| 74 |
+
MAX_PAYLOAD_BYTES = 32 * 1024 * 1024
|
| 75 |
+
|
| 76 |
+
#: The emitted spec. A render spec is a description of a picture; one larger than this is data
|
| 77 |
+
#: pretending to be a description.
|
| 78 |
+
MAX_SPEC_BYTES = 2 * 1024 * 1024
|
| 79 |
+
|
| 80 |
+
MAX_SOURCE_BYTES = 128 * 1024
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
# βββββββοΏ½οΏ½οΏ½ββββββββββββββββββββββββββββββββββββββββββββββ LAYER 1 β the AST allow-list βββββββββββ
|
| 84 |
+
#: Every `ast` node class a script may contain. β THE ABSENCES ARE THE POLICY: `Import` /
|
| 85 |
+
#: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its
|
| 86 |
+
#: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__`
|
| 87 |
+
#: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every
|
| 88 |
+
#: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never
|
| 89 |
+
#: awaited is a silent no-op that looks like a working script).
|
| 90 |
+
ALLOWED_NODES = frozenset("""
|
| 91 |
+
Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise
|
| 92 |
+
If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword
|
| 93 |
+
BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name
|
| 94 |
+
Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension
|
| 95 |
+
Load Store Del
|
| 96 |
+
And Or Not Invert UAdd USub
|
| 97 |
+
Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult
|
| 98 |
+
Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn
|
| 99 |
+
""".split())
|
| 100 |
+
|
| 101 |
+
#: Every attribute name a script may READ or CALL. ββ THIS IS THE LOAD-BEARING SET, and it is
|
| 102 |
+
#: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are
|
| 103 |
+
#: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace,
|
| 104 |
+
#: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__`
|
| 105 |
+
#: reaches `object.__subclasses__`. None of those names is here, and neither is any name this
|
| 106 |
+
#: sandbox has not been asked for.
|
| 107 |
+
#: β `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup
|
| 108 |
+
#: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are
|
| 109 |
+
#: fine β `f"{x.__class__}"` compiles to a real `Attribute` node and is refused.
|
| 110 |
+
ALLOWED_ATTRS = frozenset("""
|
| 111 |
+
append extend insert pop remove clear sort reverse copy count index
|
| 112 |
+
keys values items get setdefault update
|
| 113 |
+
add discard union intersection difference issubset issuperset
|
| 114 |
+
join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold
|
| 115 |
+
replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix
|
| 116 |
+
isdigit isalpha isalnum isspace isupper islower isnumeric
|
| 117 |
+
real imag numerator denominator
|
| 118 |
+
""".split())
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
class Refused(Exception):
|
| 122 |
+
"""A named refusal: `code` for a caller to branch on, `message` for a person to read."""
|
| 123 |
+
|
| 124 |
+
def __init__(self, code, message):
|
| 125 |
+
self.code, self.message = code, message
|
| 126 |
+
super().__init__(f"{code}: {message}")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _attr_ok(name):
|
| 130 |
+
"""An attribute name passes only if it is on the list AND is not private.
|
| 131 |
+
|
| 132 |
+
β THE SECOND TEST IS NOT A BLACKLIST β it narrows an allow-list that already excludes every
|
| 133 |
+
private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by
|
| 134 |
+
accident, which is the one edit a future reader is most likely to make in a hurry.
|
| 135 |
+
"""
|
| 136 |
+
return name in ALLOWED_ATTRS and not name.startswith("_")
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def check_source(source):
|
| 140 |
+
"""LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`.
|
| 141 |
+
|
| 142 |
+
β PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no
|
| 143 |
+
process and touches no store, so a gate can hand it a hundred hostile strings for free and an
|
| 144 |
+
NC can drop one entry from one frozenset and watch exactly this function change its answer.
|
| 145 |
+
"""
|
| 146 |
+
text = str(source or "")
|
| 147 |
+
if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
|
| 148 |
+
return Refused("source_too_long",
|
| 149 |
+
f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source")
|
| 150 |
+
try:
|
| 151 |
+
tree = ast.parse(text)
|
| 152 |
+
except SyntaxError as exc:
|
| 153 |
+
return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}")
|
| 154 |
+
|
| 155 |
+
for node in ast.walk(tree):
|
| 156 |
+
kind = type(node).__name__
|
| 157 |
+
if kind not in ALLOWED_NODES:
|
| 158 |
+
return Refused("refused_construct",
|
| 159 |
+
f"line {getattr(node, 'lineno', 0)}: this sandbox does not run "
|
| 160 |
+
f"{_english(kind)}")
|
| 161 |
+
if isinstance(node, ast.Attribute) and not _attr_ok(node.attr):
|
| 162 |
+
return Refused("refused_attribute",
|
| 163 |
+
f"line {getattr(node, 'lineno', 0)}: the attribute "
|
| 164 |
+
f"'{node.attr}' is not available inside a script view")
|
| 165 |
+
# β A NAME may not be private either. `_` prefixed names are the runner's own, and a
|
| 166 |
+
# script that could bind one could shadow the machinery it runs on top of.
|
| 167 |
+
if isinstance(node, ast.Name) and node.id.startswith("_"):
|
| 168 |
+
return Refused("reserved_name",
|
| 169 |
+
f"line {getattr(node, 'lineno', 0)}: names starting with an "
|
| 170 |
+
f"underscore are reserved by the sandbox")
|
| 171 |
+
if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str(
|
| 172 |
+
getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"):
|
| 173 |
+
return Refused("reserved_name",
|
| 174 |
+
f"line {getattr(node, 'lineno', 0)}: names starting with an "
|
| 175 |
+
f"underscore are reserved by the sandbox")
|
| 176 |
+
if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"):
|
| 177 |
+
return Refused("reserved_name",
|
| 178 |
+
f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with "
|
| 179 |
+
f"an underscore are reserved by the sandbox")
|
| 180 |
+
return None
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
_ENGLISH = {
|
| 184 |
+
"Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition",
|
| 185 |
+
"With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop",
|
| 186 |
+
"AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield",
|
| 187 |
+
"YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement",
|
| 188 |
+
"Match": "a match statement",
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _english(kind):
|
| 193 |
+
return _ENGLISH.get(kind, f"a {kind} expression")
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
# ββββββββββββββββββββββββββββββββββββββββββ LAYER 2 β the namespace, and the child program βββββ
|
| 197 |
+
#: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child.
|
| 198 |
+
#: β THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input`
|
| 199 |
+
#: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help`
|
| 200 |
+
#: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step
|
| 201 |
+
#: on a published escape, and none has ever been asked for by a script that shapes rows.
|
| 202 |
+
#: β THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause
|
| 203 |
+
#: cannot name what it catches is a construct that reads as supported and is not. They are safe
|
| 204 |
+
#: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this
|
| 205 |
+
#: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway.
|
| 206 |
+
SANDBOX_BUILTIN_NAMES = (
|
| 207 |
+
"abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex "
|
| 208 |
+
"int isinstance issubclass iter len list map max min next oct ord pow range repr reversed "
|
| 209 |
+
"round set slice sorted str sum tuple zip True False None "
|
| 210 |
+
"Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError "
|
| 211 |
+
"AttributeError StopIteration OverflowError"
|
| 212 |
+
).split()
|
| 213 |
+
|
| 214 |
+
#: The literal program the child runs. It is TEXT rather than a module because the child must
|
| 215 |
+
#: import nothing from this repo: a module would be found on `sys.path` and would drag `core`
|
| 216 |
+
#: with it, which is exactly the second store path W1 forbids.
|
| 217 |
+
#: β Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so
|
| 218 |
+
#: the runner's own machinery cannot be shadowed by the source it executes.
|
| 219 |
+
_RUNNER = r'''
|
| 220 |
+
import json as _json, os as _os, sys as _sys
|
| 221 |
+
|
| 222 |
+
_pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read())
|
| 223 |
+
_out = {"ok": False, "code": "not_run", "message": "the script did not run",
|
| 224 |
+
"stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}}
|
| 225 |
+
|
| 226 |
+
# ββ the caps this platform can actually apply, reported either way (standing rule 1) ββββββββββ
|
| 227 |
+
try:
|
| 228 |
+
import resource as _res
|
| 229 |
+
_mem = int(_pay["memoryBytes"])
|
| 230 |
+
_res.setrlimit(_res.RLIMIT_AS, (_mem, _mem))
|
| 231 |
+
_out["caps"]["memory"] = True
|
| 232 |
+
_cpu = int(_pay["cpuSeconds"])
|
| 233 |
+
_res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu))
|
| 234 |
+
_out["caps"]["cpu"] = True
|
| 235 |
+
except Exception:
|
| 236 |
+
# `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says
|
| 237 |
+
# which of the three held, never a silent partial.
|
| 238 |
+
pass
|
| 239 |
+
|
| 240 |
+
_printed = []
|
| 241 |
+
_spent = [0]
|
| 242 |
+
_LIMIT = int(_pay["maxStdout"])
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def _print(*_a, **_k):
|
| 246 |
+
_text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n")
|
| 247 |
+
_room = _LIMIT - _spent[0]
|
| 248 |
+
if _room > 0:
|
| 249 |
+
_printed.append(_text[:_room])
|
| 250 |
+
_spent[0] += len(_text)
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
class _Refusal(Exception):
|
| 254 |
+
"""The SANDBOX refusing, as distinct from the SCRIPT failing.
|
| 255 |
+
|
| 256 |
+
Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the
|
| 257 |
+
script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's
|
| 258 |
+
own code. Two different facts, two different codes.
|
| 259 |
+
"""
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
_emitted = []
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _emit(_spec):
|
| 266 |
+
if not isinstance(_spec, dict):
|
| 267 |
+
raise _Refusal("emit() takes a view spec, which is a dictionary")
|
| 268 |
+
if _emitted:
|
| 269 |
+
raise _Refusal("emit() was already called; a script view emits exactly one view")
|
| 270 |
+
_emitted.append(_spec)
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
_rows = _pay["rows"]
|
| 274 |
+
_fields = _pay["fields"]
|
| 275 |
+
_bound = _pay["table"]
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _scoped_table(_table=None):
|
| 279 |
+
if _table is not None and str(_table) != _bound:
|
| 280 |
+
raise _Refusal(
|
| 281 |
+
"this script view is bound to the database '" + _bound + "' and asked for '"
|
| 282 |
+
+ str(_table) + "'. A script view reads its own database only")
|
| 283 |
+
return [dict(_r) for _r in _rows]
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _scoped_fields():
|
| 287 |
+
return [dict(_f) for _f in _fields]
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
_ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict)
|
| 291 |
+
else getattr(__builtins__, _n)
|
| 292 |
+
for _n in _pay["builtins"]}}
|
| 293 |
+
_ns["__builtins__"]["print"] = _print
|
| 294 |
+
_ns["print"] = _print
|
| 295 |
+
_ns["emit"] = _emit
|
| 296 |
+
_ns["scoped_table"] = _scoped_table
|
| 297 |
+
_ns["scoped_fields"] = _scoped_fields
|
| 298 |
+
_ns["table"] = _bound
|
| 299 |
+
|
| 300 |
+
try:
|
| 301 |
+
exec(compile(_pay["source"], "<script view>", "exec"), _ns)
|
| 302 |
+
if not _emitted:
|
| 303 |
+
_out.update(ok=False, code="no_view",
|
| 304 |
+
message="the script finished without calling emit(spec)")
|
| 305 |
+
else:
|
| 306 |
+
_out.update(ok=True, code="", message="", spec=_emitted[0])
|
| 307 |
+
except _Refusal as _e:
|
| 308 |
+
_out.update(ok=False, code="refused", message=str(_e)[:400])
|
| 309 |
+
except MemoryError:
|
| 310 |
+
_out.update(ok=False, code="memory",
|
| 311 |
+
message="the script used more memory than a script view is allowed")
|
| 312 |
+
except NameError as _e:
|
| 313 |
+
_out.update(ok=False, code="refused_name",
|
| 314 |
+
message=str(_e)[:200] + ". A script view may use only the names the sandbox "
|
| 315 |
+
"provides")
|
| 316 |
+
except BaseException as _e:
|
| 317 |
+
_out.update(ok=False, code="error",
|
| 318 |
+
message=type(_e).__name__ + ": " + str(_e)[:400])
|
| 319 |
+
|
| 320 |
+
_out["stdout"] = "".join(_printed)
|
| 321 |
+
_out["truncated"] = _spent[0] > _LIMIT
|
| 322 |
+
# β THE PROBE: what this child actually had. The PARENT strips it unless it was asked for, so a
|
| 323 |
+
# production run never carries it and a gate can still prove that no `core.*` module and no
|
| 324 |
+
# secret-shaped environment key was ever inside this process.
|
| 325 |
+
# β SNAPSHOTTED AFTER `exec`, and `os` is imported at the TOP so this list does not depend on
|
| 326 |
+
# dict-literal evaluation order. The first draft called `__import__("os")` inside this very
|
| 327 |
+
# expression, so whether `os` appeared depended on which value Python built first: a probe whose
|
| 328 |
+
# contents move with an unrelated edit is a probe a gate cannot assert against.
|
| 329 |
+
_out["probe"] = {"modules": sorted(_sys.modules), "env": sorted(_os.environ)}
|
| 330 |
+
open(_sys.argv[2], "w", encoding="utf-8").write(_json.dumps(_out, default=str))
|
| 331 |
+
'''
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def _child_env():
|
| 335 |
+
"""The child's WHOLE environment. An allow-list of two keys, and neither is a credential.
|
| 336 |
+
|
| 337 |
+
β NOT `os.environ.copy()` MINUS SOMETHING. A subtractive environment ships every key nobody
|
| 338 |
+
thought to name: `HF_TOKEN`, `ODOO_PASSWORD`, `ANTHROPIC_API_KEY` and whatever the next
|
| 339 |
+
connector adds. The three names below are here because Python will not start on Windows
|
| 340 |
+
without them; on Linux this returns `{}` and the child runs with no environment at all.
|
| 341 |
+
"""
|
| 342 |
+
env = {}
|
| 343 |
+
for name in ("SystemRoot", "SYSTEMROOT", "WINDIR"):
|
| 344 |
+
if os.environ.get(name):
|
| 345 |
+
env[name] = os.environ[name]
|
| 346 |
+
return env
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def run(source, rows, fields, table_key, *, timeout_s=DEFAULT_TIMEOUT_S,
|
| 350 |
+
memory_bytes=DEFAULT_MEMORY_BYTES, cpu_seconds=DEFAULT_CPU_SECONDS, probe=False):
|
| 351 |
+
"""Run ONE script over rows that are ALREADY scoped. Returns C3's envelope plus `caps`.
|
| 352 |
+
|
| 353 |
+
{ok, code, message, spec, stdout, truncated, ms, caps: {wallClock, memory, cpu}}
|
| 354 |
+
|
| 355 |
+
β THIS FUNCTION NEVER TOUCHES A STORE, AND THAT IS THE POINT: it takes rows. `run_view()`
|
| 356 |
+
below is the door that fetches them through C1; keeping the two apart is what lets a gate
|
| 357 |
+
drive the sandbox with no tenant, no runtime and no credential anywhere in the process.
|
| 358 |
+
|
| 359 |
+
β `caps` IS PART OF THE ANSWER, NOT DEBUG OUTPUT. On Windows `resource` does not exist, so
|
| 360 |
+
the memory and CPU limits are NOT applied and this says so. A caller that reports `ok:true`
|
| 361 |
+
without reading `caps` is claiming an enforcement that did not happen (standing rule 1).
|
| 362 |
+
"""
|
| 363 |
+
started = time.monotonic()
|
| 364 |
+
refusal = check_source(source)
|
| 365 |
+
if refusal is not None:
|
| 366 |
+
return _refusal(refusal.code, refusal.message, started)
|
| 367 |
+
|
| 368 |
+
payload = {"source": str(source or ""), "rows": rows, "fields": fields,
|
| 369 |
+
"table": str(table_key or ""), "builtins": SANDBOX_BUILTIN_NAMES,
|
| 370 |
+
"maxStdout": MAX_STDOUT_BYTES, "memoryBytes": int(memory_bytes),
|
| 371 |
+
"cpuSeconds": int(cpu_seconds)}
|
| 372 |
+
try:
|
| 373 |
+
blob = json.dumps(payload, default=str)
|
| 374 |
+
except (TypeError, ValueError) as exc:
|
| 375 |
+
return _refusal("bad_rows", f"these rows cannot be handed to a script ({exc})", started)
|
| 376 |
+
if len(blob.encode("utf-8", "replace")) > MAX_PAYLOAD_BYTES:
|
| 377 |
+
# β REPORTED, NOT TRUNCATED (standing rule 1's second sentence): cause and recommendation,
|
| 378 |
+
# in the words the owner asked for, rather than a quietly short answer.
|
| 379 |
+
return _refusal(
|
| 380 |
+
"payload_too_large",
|
| 381 |
+
f"this database's rows are larger than the {MAX_PAYLOAD_BYTES // (1024 * 1024)} MB a "
|
| 382 |
+
f"script view can be handed at once. Narrow the view with a filter, or raise the "
|
| 383 |
+
f"sandbox payload limit for this deployment", started)
|
| 384 |
+
|
| 385 |
+
with tempfile.TemporaryDirectory(prefix="aios-script-") as work:
|
| 386 |
+
pay_path = Path(work) / "payload.json"
|
| 387 |
+
res_path = Path(work) / "result.json"
|
| 388 |
+
pay_path.write_text(blob, encoding="utf-8")
|
| 389 |
+
# `-I` isolates the interpreter (no PYTHON* env, no user site), `-S` skips site-packages,
|
| 390 |
+
# and the program arrives on STDIN so there is no file for anything to import it as.
|
| 391 |
+
# β `-X utf8` AND AN EXPLICIT `encoding` ARE NOT TIDINESS. Without them this pipe is
|
| 392 |
+
# encoded with the parent's locale codec, which on this Windows box is cp1252: the runner
|
| 393 |
+
# text below cannot be represented in it and `subprocess.run` died with
|
| 394 |
+
# `UnicodeEncodeError` before the child ever started. A sandbox whose behaviour depends on
|
| 395 |
+
# the operator's locale is a sandbox with two behaviours. `-I` implies `-E`, so
|
| 396 |
+
# `PYTHONUTF8` in the environment could not have carried this, it has to be a flag.
|
| 397 |
+
argv = [sys.executable, "-I", "-S", "-X", "utf8", "-", str(pay_path), str(res_path)]
|
| 398 |
+
try:
|
| 399 |
+
done = subprocess.run(
|
| 400 |
+
argv, input=_RUNNER, text=True, encoding="utf-8", errors="replace",
|
| 401 |
+
cwd=work, env=_child_env(),
|
| 402 |
+
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout_s)
|
| 403 |
+
except subprocess.TimeoutExpired:
|
| 404 |
+
return _refusal("timeout",
|
| 405 |
+
f"the script ran longer than {timeout_s:g} seconds and was stopped",
|
| 406 |
+
started)
|
| 407 |
+
except OSError as exc:
|
| 408 |
+
return _refusal("no_sandbox",
|
| 409 |
+
f"a script view could not be started on this deployment ({exc})",
|
| 410 |
+
started)
|
| 411 |
+
|
| 412 |
+
if not res_path.is_file():
|
| 413 |
+
# The child died without writing an answer: an rlimit signal, an OOM kill, or a crash.
|
| 414 |
+
# β NAMED BY ITS RETURN CODE rather than reported as a generic failure β a memory kill
|
| 415 |
+
# and a bug in this file must not read identically to an operator.
|
| 416 |
+
return _refusal(*_died(done), started)
|
| 417 |
+
try:
|
| 418 |
+
out = json.loads(res_path.read_text(encoding="utf-8"))
|
| 419 |
+
except (OSError, ValueError) as exc:
|
| 420 |
+
return _refusal("unreadable", f"the script's answer could not be read ({exc})",
|
| 421 |
+
started)
|
| 422 |
+
|
| 423 |
+
out["ms"] = int((time.monotonic() - started) * 1000)
|
| 424 |
+
if out.get("ok"):
|
| 425 |
+
spec_error = _check_spec(out.get("spec"))
|
| 426 |
+
if spec_error:
|
| 427 |
+
out.update(ok=False, code="bad_spec", message=spec_error, spec=None)
|
| 428 |
+
if not probe:
|
| 429 |
+
out.pop("probe", None)
|
| 430 |
+
return out
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
def _died(done):
|
| 434 |
+
"""`(code, message)` for a child that produced no answer."""
|
| 435 |
+
rc = done.returncode
|
| 436 |
+
tail = " ".join((done.stderr or "").split())[-300:]
|
| 437 |
+
if rc in (-9, 137):
|
| 438 |
+
return "memory", "the script was stopped for using too much memory"
|
| 439 |
+
if rc in (-24, 152):
|
| 440 |
+
return "timeout", "the script used more processor time than a script view is allowed"
|
| 441 |
+
return "crashed", f"the script view engine stopped without an answer{': ' + tail if tail else ''}"
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
def _refusal(code, message, started):
|
| 445 |
+
return {"ok": False, "code": code, "message": message, "spec": None, "stdout": "",
|
| 446 |
+
"truncated": False, "ms": int((time.monotonic() - started) * 1000),
|
| 447 |
+
"caps": {"wallClock": True, "memory": False, "cpu": False}}
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _check_spec(spec):
|
| 451 |
+
"""C3: a spec is a DESCRIPTION the client draws. Never HTML, never a script, never a URL.
|
| 452 |
+
|
| 453 |
+
β THE CHECK IS ON THE KEYS, NOT ON THE STRING CONTENTS. Scanning values for `<script>` is a
|
| 454 |
+
blacklist and would pass `<SCR` + `IPT>`; refusing a spec that carries an `html`, `script`,
|
| 455 |
+
`src` or `onclick` key refuses the SHAPE that would let a renderer be talked into executing
|
| 456 |
+
something. The vocabulary of legal `kind`s is the ROUTE's business (W36-T37) β this is the
|
| 457 |
+
floor every caller gets whether or not the route above it remembers.
|
| 458 |
+
"""
|
| 459 |
+
if not isinstance(spec, dict):
|
| 460 |
+
return "the script emitted something that is not a view spec"
|
| 461 |
+
try:
|
| 462 |
+
blob = json.dumps(spec)
|
| 463 |
+
except (TypeError, ValueError):
|
| 464 |
+
return "the emitted view spec is not something the client can be sent"
|
| 465 |
+
if len(blob.encode("utf-8", "replace")) > MAX_SPEC_BYTES:
|
| 466 |
+
return (f"the emitted view spec is over {MAX_SPEC_BYTES // (1024 * 1024)} MB. A view spec "
|
| 467 |
+
f"describes a picture; it is not where the rows go")
|
| 468 |
+
banned = {"html", "innerhtml", "script", "src", "srcdoc", "href", "style", "onclick", "onload"}
|
| 469 |
+
found = sorted(k for k in _keys_of(spec) if str(k).lower() in banned)
|
| 470 |
+
if found:
|
| 471 |
+
return (f"a view spec may not carry {', '.join(found)}. The client DRAWS a spec, so a "
|
| 472 |
+
f"markup or URL key would be a script by another name")
|
| 473 |
+
return None
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
def _keys_of(value, depth=0):
|
| 477 |
+
"""Every key anywhere in a nested spec. Bounded, so a deep structure cannot spin this."""
|
| 478 |
+
if depth > 12:
|
| 479 |
+
return
|
| 480 |
+
if isinstance(value, dict):
|
| 481 |
+
for key, sub in value.items():
|
| 482 |
+
yield key
|
| 483 |
+
yield from _keys_of(sub, depth + 1)
|
| 484 |
+
elif isinstance(value, (list, tuple)):
|
| 485 |
+
for sub in value:
|
| 486 |
+
yield from _keys_of(sub, depth + 1)
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββ THE DOOR β C1 is the ONLY way to a row βββββββββ
|
| 490 |
+
def run_view(user, table_key, source, st=None, **kw):
|
| 491 |
+
"""Fetch through C1 under `user`'s scope, then run the script over what came back (R5).
|
| 492 |
+
|
| 493 |
+
ββ THE FETCH HAPPENS IN THE PARENT AND ONLY ROWS CROSS INTO THE CHILD. That is wiring W1
|
| 494 |
+
made structural: the child has no runtime to ask, no store handle to open and no credential
|
| 495 |
+
to use, so "a script cannot read what its caller cannot read" is not a rule anybody has to
|
| 496 |
+
keep β there is no second path for it to be broken through.
|
| 497 |
+
|
| 498 |
+
β C1'S THREE EXCEPTIONS ARE ANSWERED, NEVER SWALLOWED. `UnknownTable`, `Denied` and
|
| 499 |
+
`Unresolvable` mean three different things to a person; collapsing them into "no rows" is the
|
| 500 |
+
silent-empty answer C1 was written to make impossible. `Unresolvable.as_limit()` is handed
|
| 501 |
+
through in the words it was raised with β standing rule 1's second sentence, verbatim.
|
| 502 |
+
"""
|
| 503 |
+
import core.perm_scope as perm_scope
|
| 504 |
+
|
| 505 |
+
key = str(table_key or "")
|
| 506 |
+
try:
|
| 507 |
+
rows = perm_scope.scoped_table(user, key, st=st)
|
| 508 |
+
fields = perm_scope.scoped_fields(user, key, st=st)
|
| 509 |
+
except perm_scope.UnknownTable as exc:
|
| 510 |
+
return _refusal("unknown_table", str(exc) or f"there is no database '{key}'",
|
| 511 |
+
time.monotonic())
|
| 512 |
+
except perm_scope.Denied as exc:
|
| 513 |
+
return _refusal("denied", str(exc) or "this account may not read that database",
|
| 514 |
+
time.monotonic())
|
| 515 |
+
except perm_scope.Unresolvable as exc:
|
| 516 |
+
out = _refusal("unresolvable", str(exc), time.monotonic())
|
| 517 |
+
out["limit"] = exc.as_limit()
|
| 518 |
+
return out
|
| 519 |
+
return run(source, rows, fields, key, **kw)
|
platform/core/shares.py
CHANGED
|
@@ -41,7 +41,14 @@ SHARES_KEY = 'object_shares'
|
|
| 41 |
|
| 42 |
#: The shareable kinds. A CLOSED vocabulary: an unknown kind raises rather than creating a new
|
| 43 |
#: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
|
| 44 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
|
| 47 |
#: Spelled as a single character so it can never collide with a username (usernames are lower-case
|
|
@@ -57,22 +64,65 @@ KINDS = ('view', 'folder', 'database')
|
|
| 57 |
#: the sentence holds. `require_session` plus the topic's own gate run first, and the
|
| 58 |
#: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
|
| 59 |
#: merged, so a grant can only narrow-or-equal what that account could already reach.
|
| 60 |
-
#: * `kind='database'` on a `ut_*` table β **THE SENTENCE
|
| 61 |
-
#:
|
| 62 |
-
#: `_clean_perms`
|
| 63 |
-
#:
|
| 64 |
-
#:
|
| 65 |
-
#:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
#:
|
| 67 |
-
#: β
|
| 68 |
-
#:
|
| 69 |
-
#:
|
| 70 |
-
#:
|
| 71 |
-
#: gets the confident half [[one-question-two-normalizers]].
|
| 72 |
EVERYONE = '*'
|
| 73 |
|
| 74 |
ROLES = ('view', 'edit')
|
| 75 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
def _st(st):
|
| 78 |
return st if st is not None else store
|
|
@@ -81,8 +131,8 @@ def _st(st):
|
|
| 81 |
def _check_kind(kind):
|
| 82 |
k = str(kind or '').strip().lower()
|
| 83 |
if k not in KINDS:
|
| 84 |
-
raise ValueError(f'{kind!r} is not a shareable kind. Use one of {", ".join(KINDS)}
|
| 85 |
-
f'
|
| 86 |
return k
|
| 87 |
|
| 88 |
|
|
|
|
| 41 |
|
| 42 |
#: The shareable kinds. A CLOSED vocabulary: an unknown kind raises rather than creating a new
|
| 43 |
#: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
|
| 44 |
+
#:
|
| 45 |
+
#: ββ W38-T16 β `field` IS THE FOURTH, AND NOTHING IN THIS FILE BRANCHES ON IT. Every function
|
| 46 |
+
#: below treats `kind` as an opaque bucket key (`_check_kind / grants / set_grants / role_for /
|
| 47 |
+
#: may_see / may_edit / may_administer / shared_with / drop_objects`), so the kind's whole cost
|
| 48 |
+
#: here is this tuple member. That is the point of one registry: the new object's WALL is written
|
| 49 |
+
#: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` β never a fourth
|
| 50 |
+
#: permission decision in a fourth file ([[one-evaluator-per-question]]).
|
| 51 |
+
KINDS = ('view', 'folder', 'database', 'field')
|
| 52 |
|
| 53 |
#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
|
| 54 |
#: Spelled as a single character so it can never collide with a username (usernames are lower-case
|
|
|
|
| 64 |
#: the sentence holds. `require_session` plus the topic's own gate run first, and the
|
| 65 |
#: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
|
| 66 |
#: merged, so a grant can only narrow-or-equal what that account could already reach.
|
| 67 |
+
#: * `kind='database'` on a `ut_*` table β ββ **THE SENTENCE HOLDS HERE TOO NOW, AND THAT IS
|
| 68 |
+
#: W36-T21 / OWNER RULING R6 (audit S-8, CLOSED).** It did not until wave 36, and the reason
|
| 69 |
+
#: is worth keeping: `routes_admin._clean_perms` 400'd any key outside
|
| 70 |
+
#: `("customer_data", "product_data")`, so no row filter and no hidden field could even be
|
| 71 |
+
#: DECLARED for a user table, `routes_tables.py` made zero `perm_scope` calls, and it passed
|
| 72 |
+
#: `hidden_keys=frozenset()`. A `database` grant was therefore ALL-OR-NOTHING β every row,
|
| 73 |
+
#: every column β and this registry was the only wall behind it.
|
| 74 |
+
#:
|
| 75 |
+
#: β WHAT CHANGED: `perm_scope.scoped_table` (contract C1) is the ONE door to any database's
|
| 76 |
+
#: rows. `routes_tables` applies the permanent filter before `pids` is taken and the transitive
|
| 77 |
+
#: hidden-field closure after `workspace_wire`, on EVERY `ut_*` read β the same code that walls
|
| 78 |
+
#: `customer_data` β and a door that cannot apply them REFUSES rather than serving the lot. So a
|
| 79 |
+
#: `database` grant is once again bounded by a second wall: it decides WHETHER an account reaches
|
| 80 |
+
#: the database, and C1 decides WHICH rows and columns it then sees. An `*` grant still admits
|
| 81 |
+
#: every account in the tenant, and each of them still only sees what their own wall allows.
|
| 82 |
#:
|
| 83 |
+
#: β `routes_shares.py`'s module docstring carries the OLD sentence at the other door and is in
|
| 84 |
+
#: no wave-36 fence β the audit's own fix was "say so at both", so one of the two is now stale.
|
| 85 |
+
#: Booked in `mailbox/C.md` (C-14) rather than edited across a fence
|
| 86 |
+
#: [[two-gates-can-assert-opposite-things]].
|
|
|
|
| 87 |
EVERYONE = '*'
|
| 88 |
|
| 89 |
ROLES = ('view', 'edit')
|
| 90 |
|
| 91 |
+
#: ββ A FIELD's OBJECT ID IS TOPIC-QUALIFIED, AND THE SEPARATOR IS DECLARED HERE SO THERE IS ONE
|
| 92 |
+
#: SPELLING OF IT. A bare field key is NOT unique: `notes` exists on a dozen databases, and a
|
| 93 |
+
#: grant stored under it would admit a grantee to every `notes` column in the tenant at once β
|
| 94 |
+
#: the widening direction, silently, forever. `table_key` is the qualifier because it is the same
|
| 95 |
+
#: identifier `shared_overlay.bucket()` already keys the values by, so the grant and the data it
|
| 96 |
+
#: governs are named by the same string ([[one-question-two-normalizers]]).
|
| 97 |
+
#:
|
| 98 |
+
#: β A `ut_*` key and a registry topic key both match `[a-z0-9_]+` and neither can contain `:`,
|
| 99 |
+
#: so the split below is unambiguous in both directions.
|
| 100 |
+
FIELD_OID_SEP = ':'
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def field_oid(table_key, field_key):
|
| 104 |
+
"""`"<table_key>:<field_key>"` β the share id of ONE column on ONE database."""
|
| 105 |
+
table = str(table_key or '').strip()
|
| 106 |
+
field = str(field_key or '').strip()
|
| 107 |
+
if not table or not field:
|
| 108 |
+
raise ValueError('shares.field_oid: a field share names BOTH a database and a column. '
|
| 109 |
+
'A bare field key repeats across tables and would grant all of them')
|
| 110 |
+
return f'{table}{FIELD_OID_SEP}{field}'
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def split_field_oid(oid):
|
| 114 |
+
"""`(table_key, field_key)` or `(None, None)` for anything that is not a field oid.
|
| 115 |
+
|
| 116 |
+
β FAIL-CLOSED ON JUNK, like every other read here: a caller that cannot learn WHICH database
|
| 117 |
+
an id names must not fall back to "the one I happen to be looking at", which is how a grant
|
| 118 |
+
on somebody else's column would be read as a grant on this one.
|
| 119 |
+
"""
|
| 120 |
+
raw = str(oid or '')
|
| 121 |
+
table, sep, field = raw.partition(FIELD_OID_SEP)
|
| 122 |
+
if not sep or not table.strip() or not field.strip() or FIELD_OID_SEP in field:
|
| 123 |
+
return (None, None)
|
| 124 |
+
return (table.strip(), field.strip())
|
| 125 |
+
|
| 126 |
|
| 127 |
def _st(st):
|
| 128 |
return st if st is not None else store
|
|
|
|
| 131 |
def _check_kind(kind):
|
| 132 |
k = str(kind or '').strip().lower()
|
| 133 |
if k not in KINDS:
|
| 134 |
+
raise ValueError(f'{kind!r} is not a shareable kind. Use one of {", ".join(KINDS)}. '
|
| 135 |
+
f'This door refuses to invent a namespace from a typo.')
|
| 136 |
return k
|
| 137 |
|
| 138 |
|
platform/core/store_pg.py
CHANGED
|
@@ -40,6 +40,8 @@ import json
|
|
| 40 |
import os
|
| 41 |
import threading
|
| 42 |
|
|
|
|
|
|
|
| 43 |
_URL_ENV = 'DATABASE_URL'
|
| 44 |
_LOCK = threading.RLock()
|
| 45 |
_POOL = {'pool': None, 'url': None, 'schema': None}
|
|
@@ -49,6 +51,19 @@ def _url():
|
|
| 49 |
return os.environ.get(_URL_ENV) or None
|
| 50 |
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
def _schema_for(tenant_slug=None):
|
| 53 |
"""`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
|
| 54 |
|
|
@@ -57,8 +72,133 @@ def _schema_for(tenant_slug=None):
|
|
| 57 |
to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
|
| 58 |
follows for the DuckDB filename.
|
| 59 |
"""
|
| 60 |
-
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
|
| 63 |
|
| 64 |
def _pool():
|
|
@@ -181,6 +321,12 @@ def exists(name, tenant_slug=None):
|
|
| 181 |
|
| 182 |
|
| 183 |
def put(name, data, tenant_slug=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
with _pool().connection() as con:
|
| 185 |
con.execute(
|
| 186 |
_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
|
|
@@ -202,6 +348,7 @@ def update(name, fn, flush='sync', tenant_slug=None):
|
|
| 202 |
`flush` is accepted and ignored (see the module docstring): there is no commit budget to
|
| 203 |
coalesce against, so 'async' has nothing to defer.
|
| 204 |
"""
|
|
|
|
| 205 |
with _pool().connection() as con:
|
| 206 |
with con.transaction():
|
| 207 |
row = con.execute(
|
|
@@ -220,6 +367,7 @@ def update(name, fn, flush='sync', tenant_slug=None):
|
|
| 220 |
|
| 221 |
|
| 222 |
def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
|
|
|
|
| 223 |
with _pool().connection() as con:
|
| 224 |
con.execute(
|
| 225 |
_q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
|
|
@@ -239,7 +387,14 @@ def download_bytes(path_in_repo, tenant_slug=None):
|
|
| 239 |
|
| 240 |
|
| 241 |
def delete_path(path_in_repo, tenant_slug=None):
|
| 242 |
-
"""Absent is SUCCESS β deleting what is already gone is the goal (HF backend's contract).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
with _pool().connection() as con:
|
| 244 |
con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 245 |
(str(path_in_repo),))
|
|
@@ -317,6 +472,14 @@ class PgStore:
|
|
| 317 |
behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, `fresh`
|
| 318 |
ignored because a SELECT has no cache to bypass) is stated once, up there, and cannot drift
|
| 319 |
between the two entry points.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 320 |
"""
|
| 321 |
|
| 322 |
def __init__(self, tenant_slug):
|
|
|
|
| 40 |
import os
|
| 41 |
import threading
|
| 42 |
|
| 43 |
+
import core.data_binding as data_binding
|
| 44 |
+
|
| 45 |
_URL_ENV = 'DATABASE_URL'
|
| 46 |
_LOCK = threading.RLock()
|
| 47 |
_POOL = {'pool': None, 'url': None, 'schema': None}
|
|
|
|
| 51 |
return os.environ.get(_URL_ENV) or None
|
| 52 |
|
| 53 |
|
| 54 |
+
def _effective_slug(tenant_slug=None):
|
| 55 |
+
"""The tenant this call ACTUALLY addresses, after every default has been applied.
|
| 56 |
+
|
| 57 |
+
ββ IT IS A SEPARATE FUNCTION BECAUSE THE DEFAULT IS THE HAZARD, AND THE GUARD HAS TO SEE THE
|
| 58 |
+
SAME ONE THE SQL DOES. `_schema_for` used to inline this chain, so the fallback to tenant #0
|
| 59 |
+
happened *inside* the address builder where nothing else could observe it: a process that was
|
| 60 |
+
handed no slug at all silently addressed `t_royal_imports` β Royal Imports' real business
|
| 61 |
+
data β and the only code that knew had already turned it into an identifier. Resolving once,
|
| 62 |
+
here, is what lets `write_refusal` below refuse the DEFAULT rather than only an explicit slug.
|
| 63 |
+
"""
|
| 64 |
+
return (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower()
|
| 65 |
+
|
| 66 |
+
|
| 67 |
def _schema_for(tenant_slug=None):
|
| 68 |
"""`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
|
| 69 |
|
|
|
|
| 72 |
to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
|
| 73 |
follows for the DuckDB filename.
|
| 74 |
"""
|
| 75 |
+
return 't_' + _effective_slug(tenant_slug).replace('-', '_')
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
# =============================================================================================
|
| 79 |
+
# ββ THE WRITE GUARD β D-353. WHY IT KEYS ON A TENANT SLUG AND NOT ON A REPO ID.
|
| 80 |
+
#
|
| 81 |
+
# `core/data_binding.py` is *"the one enforcement door"* and `core/store.py` calls it from six
|
| 82 |
+
# places. This file called it from ZERO, and the settings pane said so out loud: staging reported
|
| 83 |
+
# `writable: false, refusal: "fsanyoto/loopable was not given pg:t_royal_imports"` **while product
|
| 84 |
+
# writes on that same build succeeded** (wave-36 QA, F12). The banner was comparing a pg address
|
| 85 |
+
# against HF-repo-shaped literals β a comparison that can never match β and nothing on the write
|
| 86 |
+
# path was consulting anything at all.
|
| 87 |
+
#
|
| 88 |
+
# β SO THE OBVIOUS PORT IS THE WRONG ONE, AND IT IS WRONG IN THE DANGEROUS DIRECTION. Matching
|
| 89 |
+
# `PgStore.repo` (`pg:t_royal_imports`) against `data_binding.PRODUCTION_STORES` (HF dataset ids)
|
| 90 |
+
# never hits rule 2, never hits rule 3's allowlist either, and lands on rule 4 β so it refuses
|
| 91 |
+
# EVERY write from every non-production deployment, including the legitimate ones, while the one
|
| 92 |
+
# case it was written for (a staging build on tenant #0's rows) is refused for the wrong reason
|
| 93 |
+
# and would be "fixed" by adding the pg address to an allowlist. A guard that fires on everything
|
| 94 |
+
# gets its allowlist widened until it fires on nothing.
|
| 95 |
+
#
|
| 96 |
+
# ββ AND THE REPO ID CANNOT DISCRIMINATE HERE EVEN IN PRINCIPLE. Under `hf`, isolation rides on
|
| 97 |
+
# the address: two deployments write two different dataset repos, so comparing repo strings is
|
| 98 |
+
# comparing destinations. Under `pg` there is no per-deployment address to compare β `deploy_web`
|
| 99 |
+
# pushes ONE `DATABASE_URL`, `store.handle()` routes by tenant SLUG, staging's `tenants.json` is
|
| 100 |
+
# byte-identical to production's, and `_schema_for` defaults the slug to `royal-imports`. Every
|
| 101 |
+
# deployment therefore resolves the SAME schema for the same tenant. The only two facts that
|
| 102 |
+
# differ between a legitimate write and a catastrophic one are WHICH TENANT is being written and
|
| 103 |
+
# WHICH DEPLOYMENT is writing it β so those are the two inputs, and there are no others.
|
| 104 |
+
#
|
| 105 |
+
# β READS ARE NOT REFUSED, exactly as on the HF side and for the reason stated there: a refused
|
| 106 |
+
# read hands back an empty grid that reads as "no data" rather than as "refused". `get`,
|
| 107 |
+
# `get_projection`, `exists`, `download_bytes` and `revision` are deliberately unguarded.
|
| 108 |
+
#
|
| 109 |
+
# β WHAT THIS DOES NOT COVER, said here rather than discovered later: raw SQL through `_pool()`.
|
| 110 |
+
# `ops/seed_pg_from_hf.py` opens a connection directly to apply the DDL, and any future tool can.
|
| 111 |
+
# The guard is on the STORE INTERFACE β the four functions every product write crosses β not on
|
| 112 |
+
# the database handle. A caller that reaches past the interface reaches past the guard.
|
| 113 |
+
# =============================================================================================
|
| 114 |
+
|
| 115 |
+
#: The tenant slugs whose `t_<slug>` schema holds REAL customer data.
|
| 116 |
+
#:
|
| 117 |
+
#: β THIS IS RULE 2, NOT THE WHOLE GUARD β the same standing as `data_binding.PRODUCTION_STORES`,
|
| 118 |
+
#: which it mirrors one-for-one (`verify_store_binding.py::section_constants` pins the mapping, so
|
| 119 |
+
#: a fifth tenant added to one list and not the other goes red). It exists to make a DELIBERATE
|
| 120 |
+
#: mis-grant loud: `AIOS_SANDBOX_TENANTS=royal-imports` typed by hand must not open tenant #0.
|
| 121 |
+
#: Rule 4 below is what covers a tenant provisioned after this line was written.
|
| 122 |
+
PRODUCTION_TENANTS = frozenset({
|
| 123 |
+
'royal-imports', # tenant #0 β Royal Imports (royal-imports/cfo-os-data)
|
| 124 |
+
'nurilab', # Nurilab (royal-imports/aios-nurilab-data)
|
| 125 |
+
'gtmlab', # GTM Lab (royal-imports/aios-gtmlab-data)
|
| 126 |
+
'loopable', # Loopable (royal-imports/aios-loopable-data)
|
| 127 |
+
})
|
| 128 |
+
|
| 129 |
+
#: The env a NON-production deployment uses to name the tenant schemas it may write.
|
| 130 |
+
#: Comma-separated slugs. The pg twin of `AIOS_SANDBOX_STORES`, and the polarity is the same one
|
| 131 |
+
#: that keeps the HF guard honest: forgetting it makes a deployment write LESS, never more.
|
| 132 |
+
#:
|
| 133 |
+
#: β `AIOS_TENANT` IS DELIBERATELY NOT A GRANT. It is a DEFAULT β unset, it resolves to tenant #0 β
|
| 134 |
+
#: so treating it as a hand-off would mean "a process that named no tenant is authorised for the
|
| 135 |
+
#: one it did not name". That is `--data-repo`'s exact failure, which is the failure this file's
|
| 136 |
+
#: existence is a response to.
|
| 137 |
+
_SANDBOX_TENANTS_VAR = 'AIOS_SANDBOX_TENANTS'
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def sandbox_tenants():
|
| 141 |
+
"""The tenant slugs a non-production deployment was explicitly handed."""
|
| 142 |
+
raw = str(os.environ.get(_SANDBOX_TENANTS_VAR) or '')
|
| 143 |
+
return frozenset(s.strip().lower() for s in raw.split(',') if s.strip())
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def write_refusal(tenant_slug=None):
|
| 147 |
+
"""`None` if this deployment may write this tenant's schema, else the sentence explaining why.
|
| 148 |
+
|
| 149 |
+
The pg twin of `data_binding.write_refusal`, and the four rules are the same four β only the
|
| 150 |
+
thing being classified changes, from a dataset repo to a tenant slug:
|
| 151 |
+
|
| 152 |
+
1. the PRODUCTION deployment -> may write any tenant
|
| 153 |
+
2. any other deployment writing a PRODUCTION tenant -> REFUSED (a laptop may opt in)
|
| 154 |
+
3. any other deployment writing a tenant it was HANDED -> allowed
|
| 155 |
+
4. anything else -> REFUSED
|
| 156 |
+
|
| 157 |
+
β RULE 2 BEATS RULE 3, deliberately, which is why the order above is the order below: an
|
| 158 |
+
allowlist must not be able to re-open tenant #0. β And the Space/laptop asymmetry is carried
|
| 159 |
+
over unchanged β a Space that is not production gets NO override, because that is the case the
|
| 160 |
+
owner said "EVER" about.
|
| 161 |
+
"""
|
| 162 |
+
slug = _effective_slug(tenant_slug)
|
| 163 |
+
if data_binding.is_production_deployment():
|
| 164 |
+
return None # rule 1
|
| 165 |
+
if slug in PRODUCTION_TENANTS: # rule 2
|
| 166 |
+
if data_binding.local_override():
|
| 167 |
+
return None
|
| 168 |
+
who = data_binding.deployment_id() or 'this machine (no Space identity)'
|
| 169 |
+
where = f'{slug!r} ({_schema_for(slug)})'
|
| 170 |
+
if data_binding.is_space():
|
| 171 |
+
return (f'{who} is not the production deployment, so it may not write tenant {where}, '
|
| 172 |
+
f'which holds real customer data. Point this deployment at its own database, '
|
| 173 |
+
f'or name the sandbox tenants it may write in {_SANDBOX_TENANTS_VAR}. There is '
|
| 174 |
+
f'no override for a Space: a build that is not live never writes live data.')
|
| 175 |
+
return (f'{who} may not write tenant {where}, which holds real customer data. Set '
|
| 176 |
+
f'AIOS_TENANT to a sandbox tenant, or set '
|
| 177 |
+
f'{data_binding._LOCAL_OPT_IN_VAR}=1 to opt in deliberately.')
|
| 178 |
+
allow = sandbox_tenants() # rule 3
|
| 179 |
+
if slug in allow:
|
| 180 |
+
return None
|
| 181 |
+
who = data_binding.deployment_id() or 'this machine (no Space identity)'
|
| 182 |
+
listed = ', '.join(sorted(allow)) or '(nothing)'
|
| 183 |
+
return (f'{who} was not given tenant {slug!r}. A non-production deployment may write only the '
|
| 184 |
+
f'tenants it was handed: {listed}. Add it to {_SANDBOX_TENANTS_VAR} if that is '
|
| 185 |
+
f'intended.')
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def check_write(tenant_slug=None, operation='write'):
|
| 189 |
+
"""Raise `StoreWriteRefused` unless this deployment may write this tenant. The pg door.
|
| 190 |
+
|
| 191 |
+
β IT RAISES `data_binding.StoreWriteRefused` β the SAME type the HF backend raises, not a pg
|
| 192 |
+
twin β and that is load-bearing rather than tidy. `api/main.py:185` carries one exception
|
| 193 |
+
handler for that type which answers 503 with the reason, and the type is deliberately NOT a
|
| 194 |
+
subclass of `StoreUnavailable` so it falls through every route that would otherwise DEGRADE to
|
| 195 |
+
a session-scoped fallback and tell the user their change was saved. A new type would have had
|
| 196 |
+
to re-earn both properties, and would have failed the second one silently.
|
| 197 |
+
"""
|
| 198 |
+
reason = write_refusal(tenant_slug)
|
| 199 |
+
if reason:
|
| 200 |
+
raise data_binding.StoreWriteRefused(
|
| 201 |
+
f'{operation} to tenant {_effective_slug(tenant_slug)!r} refused: {reason}')
|
| 202 |
|
| 203 |
|
| 204 |
def _pool():
|
|
|
|
| 321 |
|
| 322 |
|
| 323 |
def put(name, data, tenant_slug=None):
|
| 324 |
+
# β BEFORE `_pool()`, NOT AFTER. A refusal that fires once a connection is open has already
|
| 325 |
+
# told the database this deployment is here, and on a pooled driver it has already borrowed a
|
| 326 |
+
# slot. More to the point, ordering is the only thing an offline gate can observe: with the
|
| 327 |
+
# pool stubbed to raise, "refused" and "reached the database" are two distinguishable answers,
|
| 328 |
+
# which is what makes the negative control in `verify_store_pg.py` bite without a server.
|
| 329 |
+
check_write(tenant_slug, f'put {name!r}')
|
| 330 |
with _pool().connection() as con:
|
| 331 |
con.execute(
|
| 332 |
_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
|
|
|
|
| 348 |
`flush` is accepted and ignored (see the module docstring): there is no commit budget to
|
| 349 |
coalesce against, so 'async' has nothing to defer.
|
| 350 |
"""
|
| 351 |
+
check_write(tenant_slug, f'update {name!r}') # D-353 β see `put`, and before `_pool()`
|
| 352 |
with _pool().connection() as con:
|
| 353 |
with con.transaction():
|
| 354 |
row = con.execute(
|
|
|
|
| 367 |
|
| 368 |
|
| 369 |
def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
|
| 370 |
+
check_write(tenant_slug, f'write bytes {path_in_repo!r}') # D-353 β before `_pool()`
|
| 371 |
with _pool().connection() as con:
|
| 372 |
con.execute(
|
| 373 |
_q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
|
|
|
|
| 387 |
|
| 388 |
|
| 389 |
def delete_path(path_in_repo, tenant_slug=None):
|
| 390 |
+
"""Absent is SUCCESS β deleting what is already gone is the goal (HF backend's contract).
|
| 391 |
+
|
| 392 |
+
β ABSENT-IS-SUCCESS DOES NOT EXTEND TO REFUSED. A delete this deployment may not make RAISES;
|
| 393 |
+
it does not answer True on the grounds that nothing was removed. The lenient return exists so a
|
| 394 |
+
caller need not care whether a blob was already gone β not so a caller can be told a delete it
|
| 395 |
+
was forbidden went fine.
|
| 396 |
+
"""
|
| 397 |
+
check_write(tenant_slug, f'delete {path_in_repo!r}') # D-353 β before `_pool()`
|
| 398 |
with _pool().connection() as con:
|
| 399 |
con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 400 |
(str(path_in_repo),))
|
|
|
|
| 472 |
behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, `fresh`
|
| 473 |
ignored because a SELECT has no cache to bypass) is stated once, up there, and cannot drift
|
| 474 |
between the two entry points.
|
| 475 |
+
|
| 476 |
+
ββ AND THAT IS WHY THE D-353 GUARD IS NOT REPEATED HERE. `put`, `update`, `upload_bytes` and
|
| 477 |
+
`delete_path` below pass `tenant_slug=self.tenant_slug` down to the module functions, which
|
| 478 |
+
call `check_write` before touching the pool β so this class is guarded by delegation rather
|
| 479 |
+
than by four more copies that could drift out of step with the four above. It matters because
|
| 480 |
+
THIS is the object the app actually holds: `store.handle()` returns it under `STORE_BACKEND=pg`
|
| 481 |
+
and `harness.runtime` binds one per tenant, so every product write on the pg path arrives here
|
| 482 |
+
first and leaves through a guarded module function.
|
| 483 |
"""
|
| 484 |
|
| 485 |
def __init__(self, tenant_slug):
|
platform/core/table_store.py
CHANGED
|
@@ -25,6 +25,30 @@ import core.store as store
|
|
| 25 |
#: divergent copies the moment either was edited.
|
| 26 |
SHARED_KEY = '__shared__'
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
def _may_see(view, viewer, is_admin=False):
|
| 30 |
"""Visibility for ONE shared view, fail-closed.
|
|
@@ -72,6 +96,37 @@ def is_shared(view):
|
|
| 72 |
return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
|
| 73 |
|
| 74 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
def source_override_is_empty(payload):
|
| 76 |
"""Does this stored definition of a SOURCE (non-custom) column carry any user state?
|
| 77 |
|
|
@@ -239,12 +294,25 @@ class TableStore:
|
|
| 239 |
data = self._st.update(self.table_key, _take, flush='async')
|
| 240 |
ws = (data or {}).get(username, {}) or {}
|
| 241 |
except Exception:
|
|
|
|
| 242 |
ws = {}
|
| 243 |
corrections = {}
|
| 244 |
fields = {
|
| 245 |
key: dict(value) if isinstance(value, dict) else value
|
| 246 |
for key, value in (ws.get('fields') or {}).items()
|
| 247 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
for key, ack in corrections.items():
|
| 249 |
field = fields.get(key)
|
| 250 |
accepted_label = str(ack.get('label') or '')[:120]
|
|
@@ -282,7 +350,16 @@ class TableStore:
|
|
| 282 |
return out
|
| 283 |
|
| 284 |
# ---------------------------------------------------------------- write
|
| 285 |
-
def _update(self, username, change):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
def _up(data):
|
| 287 |
ws = data.setdefault(username, {})
|
| 288 |
ws.setdefault('views', {})
|
|
@@ -291,6 +368,8 @@ class TableStore:
|
|
| 291 |
ws.setdefault('folders', {})
|
| 292 |
ws.setdefault('itemFolders', {})
|
| 293 |
change(ws)
|
|
|
|
|
|
|
| 294 |
return data
|
| 295 |
# flush='async' (wave-7 W3): this is THE hot path β every autosaved filter tweak,
|
| 296 |
# column note and typed overlay cell lands here inside the component round-trip, and
|
|
@@ -492,7 +571,14 @@ class TableStore:
|
|
| 492 |
self._st.update(self.table_key, _up, flush='async')
|
| 493 |
|
| 494 |
def save_field(self, username, field, reserved_names=(), correction_id=None):
|
| 495 |
-
"""Persist a column note or a user-created (custom_/measure_) field definition.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
key = str((field or {}).get('key') or '').strip()
|
| 497 |
if not key:
|
| 498 |
raise ValueError('field key is required')
|
|
@@ -524,12 +610,32 @@ class TableStore:
|
|
| 524 |
ws.pop('fieldCorrections', None)
|
| 525 |
accepted.clear()
|
| 526 |
accepted.update(payload)
|
| 527 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 528 |
ws['fields'].pop(key, None)
|
| 529 |
else:
|
| 530 |
-
ws['fields'][key] =
|
| 531 |
|
| 532 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 533 |
return dict(accepted)
|
| 534 |
|
| 535 |
def duplicate_field(self, username, source_key, new_key, field,
|
|
@@ -572,13 +678,28 @@ class TableStore:
|
|
| 572 |
ws.pop('fieldCorrections', None)
|
| 573 |
accepted.clear()
|
| 574 |
accepted.update(payload)
|
| 575 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 576 |
if source_key.startswith('custom_'):
|
| 577 |
for row in ws['overlays'].values():
|
| 578 |
if isinstance(row, dict) and source_key in row:
|
| 579 |
row[new_key] = row[source_key]
|
| 580 |
|
| 581 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 582 |
return dict(accepted)
|
| 583 |
|
| 584 |
def delete_field(self, username, key):
|
|
@@ -589,6 +710,13 @@ class TableStore:
|
|
| 589 |
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 590 |
silently resurface if the key were ever reused. Views referencing the key self-heal on
|
| 591 |
their next autosave (an unknown colId is dropped) β the rule every stale key rides.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 592 |
"""
|
| 593 |
key = str(key or '').strip()
|
| 594 |
if not key:
|
|
@@ -600,16 +728,67 @@ class TableStore:
|
|
| 600 |
if isinstance(row, dict):
|
| 601 |
row.pop(key, None)
|
| 602 |
|
| 603 |
-
self._update(username, _drop)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 604 |
|
| 605 |
def patch_overlay(self, username, pid, updates):
|
| 606 |
-
"""Patch only the external editable stratum; never writes to the source system.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 607 |
clean = dict(updates or {})
|
| 608 |
if not clean:
|
| 609 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 610 |
|
| 611 |
def _patch(ws):
|
| 612 |
-
ws['overlays'].setdefault(str(int(pid)), {}).update(
|
| 613 |
|
| 614 |
self._update(username, _patch)
|
| 615 |
|
|
|
|
| 25 |
#: divergent copies the moment either was edited.
|
| 26 |
SHARED_KEY = '__shared__'
|
| 27 |
|
| 28 |
+
#: ββ W36-T25 β THE COLUMN SUMMARY IS THE DATABASE'S, NOT ONE ACCOUNT'S.
|
| 29 |
+
#:
|
| 30 |
+
#: Owner item 1, second half, verbatim: *"When i filter Avi on my computer sum works for the Sales
|
| 31 |
+
#: last 365 days column but not when my colleague filter it β The sum of the field is not showing
|
| 32 |
+
#: at the bottom even when i zoom out. It only works on my screen??"* Measured while scouting it:
|
| 33 |
+
#: `save_field` writes the WHOLE field payload into `data[username]`, so a column summary β which
|
| 34 |
+
#: is a statement about the COLUMN, identical for every reader by construction β was stored once
|
| 35 |
+
#: per account. Set it, and the totals row exists for you and for nobody else. `computeAggs` then
|
| 36 |
+
#: paints no totals row at all for the colleague (`showTotals` is false when no field carries an
|
| 37 |
+
#: `agg`), which is exactly "not showing at the bottom".
|
| 38 |
+
#:
|
| 39 |
+
#: β ONLY `agg` MOVES, AND THE LINE IS NOT ARBITRARY. Width, column order, the note, the display
|
| 40 |
+
#: format and every `custom_`/`measure_` definition stay per user, because each of those is a
|
| 41 |
+
#: statement about how ONE PERSON reads the column. "Sum this column" is a statement about what
|
| 42 |
+
#: the column MEANS, and two accounts disagreeing about it is the defect, not a preference.
|
| 43 |
+
#:
|
| 44 |
+
#: β IT LIVES IN THE `__shared__` MEMBER OF THIS SAME BUCKET rather than in
|
| 45 |
+
#: `core/shared_overlay.py`'s `<key>__shared` document, and the reason is transactional: a summary
|
| 46 |
+
#: is written by the same `save_field` call that writes the note beside it, and the two must land
|
| 47 |
+
#: or fail together. `__shared__` is already this store's tenant-wide member (shared VIEWS live
|
| 48 |
+
#: there) and is guarded against colliding with a username by `is_shared` and by `core/users.py`'s
|
| 49 |
+
#: never-dunder rule, so there is no second bucket, no second flush and no second failure mode.
|
| 50 |
+
SHARED_FIELD_KEYS = ('agg',)
|
| 51 |
+
|
| 52 |
|
| 53 |
def _may_see(view, viewer, is_admin=False):
|
| 54 |
"""Visibility for ONE shared view, fail-closed.
|
|
|
|
| 96 |
return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
|
| 97 |
|
| 98 |
|
| 99 |
+
def _shared_fields(data):
|
| 100 |
+
"""The tenant-wide field stratum of one workspace document β `{field_key: {'agg': β¦}}`.
|
| 101 |
+
|
| 102 |
+
β TOTAL AND FAIL-SOFT: a document written before W36-T25 has no `__shared__` member at all,
|
| 103 |
+
and one written by a future version may have something else in it. Either way this answers an
|
| 104 |
+
empty mapping rather than raising, because the caller is a READ that must still serve the
|
| 105 |
+
workspace β a column with no summary is the state every column was in yesterday.
|
| 106 |
+
"""
|
| 107 |
+
shared = (data or {}).get(SHARED_KEY)
|
| 108 |
+
fields = (shared or {}).get('fields') if isinstance(shared, dict) else None
|
| 109 |
+
return fields if isinstance(fields, dict) else {}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def split_shared(payload):
|
| 113 |
+
"""`(per_user, shared)` β one field payload divided at the strata boundary.
|
| 114 |
+
|
| 115 |
+
β W36-T25. `shared` carries only `SHARED_FIELD_KEYS` that are actually SET; `per_user` is the
|
| 116 |
+
payload without them. Split HERE rather than at each write door so the three doors that store
|
| 117 |
+
a field (`save_field`, `duplicate_field`, and the delete that must clear it) cannot come apart
|
| 118 |
+
about where a summary lives β which is the whole reason this module exists rather than being
|
| 119 |
+
copied per topic.
|
| 120 |
+
"""
|
| 121 |
+
payload = dict(payload or {})
|
| 122 |
+
shared = {}
|
| 123 |
+
for key in SHARED_FIELD_KEYS:
|
| 124 |
+
value = str(payload.pop(key, '') or '').strip()
|
| 125 |
+
if value:
|
| 126 |
+
shared[key] = value
|
| 127 |
+
return payload, shared
|
| 128 |
+
|
| 129 |
+
|
| 130 |
def source_override_is_empty(payload):
|
| 131 |
"""Does this stored definition of a SOURCE (non-custom) column carry any user state?
|
| 132 |
|
|
|
|
| 294 |
data = self._st.update(self.table_key, _take, flush='async')
|
| 295 |
ws = (data or {}).get(username, {}) or {}
|
| 296 |
except Exception:
|
| 297 |
+
data = {}
|
| 298 |
ws = {}
|
| 299 |
corrections = {}
|
| 300 |
fields = {
|
| 301 |
key: dict(value) if isinstance(value, dict) else value
|
| 302 |
for key, value in (ws.get('fields') or {}).items()
|
| 303 |
}
|
| 304 |
+
# ββ W36-T25 β THE TENANT-WIDE COLUMN SUMMARY, MERGED OVER THIS USER'S STRATUM.
|
| 305 |
+
# β IT MUST BE ABLE TO CREATE AN ENTRY, not only decorate one, and that is the whole
|
| 306 |
+
# reason this is a merge rather than a lookup: the colleague who never touched the column
|
| 307 |
+
# has NO per-user record for it, so a decorate-only pass would have left them with exactly
|
| 308 |
+
# the blank totals row the owner reported. `aios_grid.workspace_wire` reads `meta['agg']`
|
| 309 |
+
# off whatever is here, base column or custom one alike.
|
| 310 |
+
for key, shared in _shared_fields(data).items():
|
| 311 |
+
agg = str((shared or {}).get('agg') or '').strip()
|
| 312 |
+
if not agg:
|
| 313 |
+
continue
|
| 314 |
+
entry = fields.get(key)
|
| 315 |
+
fields[key] = {**entry, 'agg': agg} if isinstance(entry, dict) else {'agg': agg}
|
| 316 |
for key, ack in corrections.items():
|
| 317 |
field = fields.get(key)
|
| 318 |
accepted_label = str(ack.get('label') or '')[:120]
|
|
|
|
| 350 |
return out
|
| 351 |
|
| 352 |
# ---------------------------------------------------------------- write
|
| 353 |
+
def _update(self, username, change, shared=None):
|
| 354 |
+
"""Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the
|
| 355 |
+
tenant-wide one, in ONE transaction.
|
| 356 |
+
|
| 357 |
+
β W36-T25 β `shared` IS A SECOND CALLBACK RATHER THAN A SECOND `update`, and that is the
|
| 358 |
+
whole reason it exists here instead of at the caller. `save_field` writes a note (per
|
| 359 |
+
user) and a column summary (tenant-wide) from ONE payload; two transactions would let the
|
| 360 |
+
summary land while the note did not, and the store's own commit is asynchronous, so the
|
| 361 |
+
window is real rather than theoretical. One `update`, one flush, one failure mode.
|
| 362 |
+
"""
|
| 363 |
def _up(data):
|
| 364 |
ws = data.setdefault(username, {})
|
| 365 |
ws.setdefault('views', {})
|
|
|
|
| 368 |
ws.setdefault('folders', {})
|
| 369 |
ws.setdefault('itemFolders', {})
|
| 370 |
change(ws)
|
| 371 |
+
if shared is not None:
|
| 372 |
+
shared(data.setdefault(SHARED_KEY, {}).setdefault('fields', {}))
|
| 373 |
return data
|
| 374 |
# flush='async' (wave-7 W3): this is THE hot path β every autosaved filter tweak,
|
| 375 |
# column note and typed overlay cell lands here inside the component round-trip, and
|
|
|
|
| 571 |
self._st.update(self.table_key, _up, flush='async')
|
| 572 |
|
| 573 |
def save_field(self, username, field, reserved_names=(), correction_id=None):
|
| 574 |
+
"""Persist a column note or a user-created (custom_/measure_) field definition.
|
| 575 |
+
|
| 576 |
+
ββ W36-T25 β THE COLUMN SUMMARY GOES TO THE TENANT-WIDE STRATUM, EVERYTHING ELSE STAYS
|
| 577 |
+
PER USER, in ONE transaction. See `SHARED_FIELD_KEYS` above for the owner's report and for
|
| 578 |
+
why the line falls where it does. The returned `accepted` still carries the summary: the
|
| 579 |
+
caller is echoing back the field it just stored, and dropping a key from that echo would
|
| 580 |
+
tell the client its write was refused ([[read-path-cannot-witness-write-path]]).
|
| 581 |
+
"""
|
| 582 |
key = str((field or {}).get('key') or '').strip()
|
| 583 |
if not key:
|
| 584 |
raise ValueError('field key is required')
|
|
|
|
| 610 |
ws.pop('fieldCorrections', None)
|
| 611 |
accepted.clear()
|
| 612 |
accepted.update(payload)
|
| 613 |
+
# ββ W36-T25 β SPLIT AFTER the label allocation and the correction bookkeeping, so
|
| 614 |
+
# both still see the whole payload, and BEFORE the per-user write.
|
| 615 |
+
mine, shared_now = split_shared(payload)
|
| 616 |
+
shared_write.clear()
|
| 617 |
+
shared_write.update(shared_now)
|
| 618 |
+
# β EMPTINESS IS JUDGED ON THE PER-USER HALF. A source column whose ONLY state was a
|
| 619 |
+
# summary now has no per-user state at all, and leaving an `{}` override behind is the
|
| 620 |
+
# meaningless row `source_override_is_empty` exists to prevent.
|
| 621 |
+
if source_override_is_empty(mine):
|
| 622 |
ws['fields'].pop(key, None)
|
| 623 |
else:
|
| 624 |
+
ws['fields'][key] = mine
|
| 625 |
|
| 626 |
+
shared_write = {}
|
| 627 |
+
|
| 628 |
+
def _share(shared_fields):
|
| 629 |
+
# β A CLEARED SUMMARY MUST REMOVE THE ROW, not leave an empty one: `workspace` treats
|
| 630 |
+
# any entry it finds as a live tenant-wide summary, so an `{'agg': ''}` husk would be
|
| 631 |
+
# skipped today and become a resurrection hazard the moment the read grows a second
|
| 632 |
+
# shared key. Absence is the only honest spelling of "nobody set one".
|
| 633 |
+
if shared_write:
|
| 634 |
+
shared_fields[key] = {**(shared_fields.get(key) or {}), **shared_write}
|
| 635 |
+
else:
|
| 636 |
+
shared_fields.pop(key, None)
|
| 637 |
+
|
| 638 |
+
self._update(username, _save, shared=_share)
|
| 639 |
return dict(accepted)
|
| 640 |
|
| 641 |
def duplicate_field(self, username, source_key, new_key, field,
|
|
|
|
| 678 |
ws.pop('fieldCorrections', None)
|
| 679 |
accepted.clear()
|
| 680 |
accepted.update(payload)
|
| 681 |
+
# β W36-T25: a CLONE carries the original's summary, and a summary is the database's
|
| 682 |
+
# (see `SHARED_FIELD_KEYS`). Storing it per user here would give the clone a different
|
| 683 |
+
# residency from every other column β one door out of three disagreeing about where a
|
| 684 |
+
# thing lives is how `save_field` and this function drift.
|
| 685 |
+
mine, shared_now = split_shared(payload)
|
| 686 |
+
shared_write.clear()
|
| 687 |
+
shared_write.update(shared_now)
|
| 688 |
+
ws['fields'][new_key] = mine
|
| 689 |
if source_key.startswith('custom_'):
|
| 690 |
for row in ws['overlays'].values():
|
| 691 |
if isinstance(row, dict) and source_key in row:
|
| 692 |
row[new_key] = row[source_key]
|
| 693 |
|
| 694 |
+
shared_write = {}
|
| 695 |
+
|
| 696 |
+
def _share(shared_fields):
|
| 697 |
+
if shared_write:
|
| 698 |
+
shared_fields[new_key] = {**(shared_fields.get(new_key) or {}), **shared_write}
|
| 699 |
+
else:
|
| 700 |
+
shared_fields.pop(new_key, None)
|
| 701 |
+
|
| 702 |
+
self._update(username, _dup, shared=_share)
|
| 703 |
return dict(accepted)
|
| 704 |
|
| 705 |
def delete_field(self, username, key):
|
|
|
|
| 710 |
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 711 |
silently resurface if the key were ever reused. Views referencing the key self-heal on
|
| 712 |
their next autosave (an unknown colId is dropped) β the rule every stale key rides.
|
| 713 |
+
|
| 714 |
+
ββ W36-T25 β AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the
|
| 715 |
+
paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can
|
| 716 |
+
see and nobody can clear, and it would attach itself to the next column that happens to
|
| 717 |
+
take the key back. β This is the ONE stratum a per-user delete may reach across accounts,
|
| 718 |
+
and it is safe because the summary was never this user's to begin with β deleting the
|
| 719 |
+
COLUMN is a tenant-wide act already.
|
| 720 |
"""
|
| 721 |
key = str(key or '').strip()
|
| 722 |
if not key:
|
|
|
|
| 728 |
if isinstance(row, dict):
|
| 729 |
row.pop(key, None)
|
| 730 |
|
| 731 |
+
self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None))
|
| 732 |
+
|
| 733 |
+
def _tenant_wide_keys(self):
|
| 734 |
+
"""The columns of THIS table whose values live in the tenant-wide stratum.
|
| 735 |
+
|
| 736 |
+
β THE CHEAP HALF. A table that shares nothing reads one small per-key file (cache-first
|
| 737 |
+
per process) and answers the empty set, so `patch_overlay` behaves exactly as it did
|
| 738 |
+
before W38-T20 for every database that has no shared column. It is deliberately not
|
| 739 |
+
memoised on the instance: `modules/customer_data.py` and `modules/product_data.py` both
|
| 740 |
+
hold a MODULE-LEVEL `TABLE_OPS`, so a per-instance cache would serve one request's answer
|
| 741 |
+
to the next, and this one decides WHERE a value is written.
|
| 742 |
+
β LENIENT LIKE EVERY OTHER STRATUM READ, and the failure direction is the safe one: an
|
| 743 |
+
unreachable shared bucket routes the write to the PER-USER stratum, which is the
|
| 744 |
+
pre-ticket behaviour, rather than dropping it.
|
| 745 |
+
"""
|
| 746 |
+
try:
|
| 747 |
+
import core.shared_overlay as shared_overlay
|
| 748 |
+
return set(shared_overlay.fields(self.table_key, st=self._st) or ())
|
| 749 |
+
except Exception: # noqa: BLE001
|
| 750 |
+
return set()
|
| 751 |
|
| 752 |
def patch_overlay(self, username, pid, updates):
|
| 753 |
+
"""Patch only the external editable stratum; never writes to the source system.
|
| 754 |
+
|
| 755 |
+
ββ W38-T20 / D-423 β A CELL IN A TENANT-WIDE COLUMN GOES TO THE TENANT-WIDE STRATUM,
|
| 756 |
+
AND WITHOUT THIS SPLIT THE EDIT SILENTLY DISAPPEARS. The read path layers the shared
|
| 757 |
+
stratum OVER the per-user one (it has to: that is what makes every reader see the same
|
| 758 |
+
number). This method wrote PER USER. So the sequence was: type a new value, see it accept,
|
| 759 |
+
come back, and read the shared value again β the owner's *"I went back and it all got
|
| 760 |
+
reseted"*, with a successful 200 at every step and nothing in any log.
|
| 761 |
+
|
| 762 |
+
β IT IS DECIDED BY WHERE THE COLUMN LIVES, NOT BY WHO IS WRITING OR THROUGH WHICH ROUTE.
|
| 763 |
+
`modules/product_data._ProductTableStore` has done exactly this since W30-T36 against its
|
| 764 |
+
CANONICAL list; the only reason it needed a subclass is that its shared columns are
|
| 765 |
+
declared in a contract file. Columns created at runtime cannot be, so the general form
|
| 766 |
+
asks the stratum itself. Both doors (`PATCH /customers/{pid}` and `POST /grid/events`)
|
| 767 |
+
arrive here through `grid_events._tops(ctx)`, which is why the split belongs at the STORE
|
| 768 |
+
and not at either route: intercepting at one leaves the other writing into the shadow.
|
| 769 |
+
|
| 770 |
+
β NO PERMISSION IS ANSWERED HERE. `shared_overlay`'s header is explicit that it is not a
|
| 771 |
+
wall, and neither is this: whether this session may write this key is settled upstream by
|
| 772 |
+
`EventCtx.hidden_keys`, which `routes_customers._hidden_for` now computes over the MERGED
|
| 773 |
+
contract precisely so a grant-governed column is refused before it reaches this line.
|
| 774 |
+
"""
|
| 775 |
clean = dict(updates or {})
|
| 776 |
if not clean:
|
| 777 |
return
|
| 778 |
+
wide = self._tenant_wide_keys()
|
| 779 |
+
shared = {k: v for k, v in clean.items() if k in wide}
|
| 780 |
+
personal = {k: v for k, v in clean.items() if k not in wide}
|
| 781 |
+
if shared:
|
| 782 |
+
import core.shared_overlay as shared_overlay
|
| 783 |
+
# β `st=self._st`, NEVER the module default. The two strata must resolve to the SAME
|
| 784 |
+
# tenant handle, or a value written by one is invisible to the other and the user's
|
| 785 |
+
# edit vanishes the moment they save it (`_ProductTableStore` records the same rule).
|
| 786 |
+
shared_overlay.put_cells(self.table_key, pid, shared, st=self._st)
|
| 787 |
+
if not personal:
|
| 788 |
+
return
|
| 789 |
|
| 790 |
def _patch(ws):
|
| 791 |
+
ws['overlays'].setdefault(str(int(pid)), {}).update(personal)
|
| 792 |
|
| 793 |
self._update(username, _patch)
|
| 794 |
|
platform/core/user_tables.py
CHANGED
|
@@ -732,6 +732,24 @@ def _ag_formula(raw):
|
|
| 732 |
return _agf._clean_formula(raw)
|
| 733 |
|
| 734 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 735 |
#: Display formats a field may declare. β DISPLAY ONLY β none of these touches the stored value,
|
| 736 |
#: which stays the scalar the fold or the mapper wrote. That separation is the whole safety of the
|
| 737 |
#: feature: a `thousands` setting can never make a number wrong, only easier to read.
|
|
@@ -934,6 +952,16 @@ def clean_fields(raw):
|
|
| 934 |
entry['aiEnrich'] = ae
|
| 935 |
# C3: derived, never typed beside the config. Same call as the other door.
|
| 936 |
entry['automation'] = _field_agent_binding(entry)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 937 |
if entry is None:
|
| 938 |
seen.discard(key)
|
| 939 |
continue
|
|
@@ -2255,6 +2283,24 @@ def _clean_field(raw, previous=None):
|
|
| 2255 |
# a `field_agent` bag that disagrees with the column. Every OTHER automation bag is untouched.
|
| 2256 |
if ftype == 'ai_enrich':
|
| 2257 |
out['automation'] = _field_agent_binding(out)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2258 |
return out
|
| 2259 |
|
| 2260 |
|
|
|
|
| 732 |
return _agf._clean_formula(raw)
|
| 733 |
|
| 734 |
|
| 735 |
+
def _ag_geocode(raw):
|
| 736 |
+
"""`aios_grid._clean_geocode`, reached exactly the way `_ag_formula` reaches its validator.
|
| 737 |
+
|
| 738 |
+
ββ W37-T22 clause 1, landed by /validate-wave 2026-08-19. The geocode PSEUDO-KIND was
|
| 739 |
+
offered in `ColumnMenu` and cleaned in `aios_grid._field_extras`, but THIS module β the
|
| 740 |
+
allowlist every `ut_*` create and patch goes through β named no `geocode` arm, so the bag was
|
| 741 |
+
stripped on every user database and the kind had to be gated OFF there. That gap is what made
|
| 742 |
+
the ticket's *"the kind is offered on ANY database that has a text field"* clause false.
|
| 743 |
+
|
| 744 |
+
β ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and
|
| 745 |
+
`_clean_field`), for the reason `_ag_formula` states: `core` must not import `aios_grid` at
|
| 746 |
+
module level, and a third and fourth copy of `import aios_grid as _agX` is how one of them
|
| 747 |
+
ends up calling a different validator.
|
| 748 |
+
"""
|
| 749 |
+
import aios_grid as _agg
|
| 750 |
+
return _agg._clean_geocode(raw)
|
| 751 |
+
|
| 752 |
+
|
| 753 |
#: Display formats a field may declare. β DISPLAY ONLY β none of these touches the stored value,
|
| 754 |
#: which stays the scalar the fold or the mapper wrote. That separation is the whole safety of the
|
| 755 |
#: feature: a `thousands` setting can never make a number wrong, only easier to read.
|
|
|
|
| 952 |
entry['aiEnrich'] = ae
|
| 953 |
# C3: derived, never typed beside the config. Same call as the other door.
|
| 954 |
entry['automation'] = _field_agent_binding(entry)
|
| 955 |
+
# ββ W37-T22 (R4 / C7) β THE GEOCODE BAG RIDES THIS DOOR TOO, written in the SAME change
|
| 956 |
+
# as `_clean_field`'s arm rather than a wave later. That is the only difference between
|
| 957 |
+
# this entry and the six above it (`pinned`, `code`, `link`/`rollup`, `formula`,
|
| 958 |
+
# `aiEnrich`), every one of which was a bag this validator dropped until somebody found a
|
| 959 |
+
# column that rendered and computed nothing forever.
|
| 960 |
+
# β NO type/bag pairing: geocode is a pseudo-kind on a `text` column (see `_clean_geocode`).
|
| 961 |
+
if entry is not None:
|
| 962 |
+
geo = (_ag_geocode(f.get('geocode')) if f.get('geocode') is not None else None)
|
| 963 |
+
if geo and ftype == 'text':
|
| 964 |
+
entry['geocode'] = geo
|
| 965 |
if entry is None:
|
| 966 |
seen.discard(key)
|
| 967 |
continue
|
|
|
|
| 2283 |
# a `field_agent` bag that disagrees with the column. Every OTHER automation bag is untouched.
|
| 2284 |
if ftype == 'ai_enrich':
|
| 2285 |
out['automation'] = _field_agent_binding(out)
|
| 2286 |
+
# ββ W37-T22 (R4 / C7) β THE GEOCODE PSEUDO-KIND'S BAG, added 2026-08-19 by /validate-wave.
|
| 2287 |
+
# Posture is `format`'s and NOT `formula`'s, and that is the whole design: geocode has no
|
| 2288 |
+
# FieldType to pair with (the column's `type` is `text`), so an absent bag is an ordinary text
|
| 2289 |
+
# column rather than a refusal. Enforcing a pairing here would refuse every text column in
|
| 2290 |
+
# every user database.
|
| 2291 |
+
# β `'geocode' in raw` rather than `raw.get('geocode')`, exactly like `format` and
|
| 2292 |
+
# `description`: an explicit `{}`/None CLEARS the address column while OMITTING the key keeps
|
| 2293 |
+
# what is stored. A PATCH that sends only a label must not silently unconfigure the column β
|
| 2294 |
+
# that is the same rule `agg` and `optionColors` are written for.
|
| 2295 |
+
# β Inheritance is guarded on `ftype == 'text'` for the reason `prev_type == ftype` guards
|
| 2296 |
+
# `link`/`rollup`/`formula`: retyping a geocode column to a number must not resurrect an
|
| 2297 |
+
# address binding through `prev` that nothing will ever read again.
|
| 2298 |
+
geocode_raw = (raw.get('geocode') if 'geocode' in raw
|
| 2299 |
+
else (prev.get('geocode') if ftype == 'text' else None))
|
| 2300 |
+
if geocode_raw is not None:
|
| 2301 |
+
geo = _ag_geocode(geocode_raw)
|
| 2302 |
+
if geo and ftype == 'text':
|
| 2303 |
+
out['geocode'] = geo
|
| 2304 |
return out
|
| 2305 |
|
| 2306 |
|
platform/core/users.py
CHANGED
|
@@ -165,6 +165,13 @@ def verify(username, pw):
|
|
| 165 |
except Exception:
|
| 166 |
reg = {}
|
| 167 |
u = reg.get(username)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
if u is None and '@' in username:
|
| 169 |
# Wave 18 (R1): the login box takes a username OR an email β admin@nurilab.id signs in
|
| 170 |
# without knowing the slug an admin chose. First case-insensitive email match wins;
|
|
|
|
| 165 |
except Exception:
|
| 166 |
reg = {}
|
| 167 |
u = reg.get(username)
|
| 168 |
+
# β KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
|
| 169 |
+
# R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
|
| 170 |
+
# different account's `email` field at the same time. The key wins, here, by construction: this
|
| 171 |
+
# branch is only reached when `reg.get(username)` missed. `routes_admin.py::create_user` refuses
|
| 172 |
+
# to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the
|
| 173 |
+
# ambiguity cannot be introduced through the product; this line is what decides it for any
|
| 174 |
+
# record that arrived some other way.
|
| 175 |
if u is None and '@' in username:
|
| 176 |
# Wave 18 (R1): the login box takes a username OR an email β admin@nurilab.id signs in
|
| 177 |
# without knowing the slug an admin chose. First case-insensitive email match wins;
|
platform/harness/datastore.py
CHANGED
|
@@ -288,6 +288,45 @@ ENTITIES = {
|
|
| 288 |
"model": "account.account", "archivable": False,
|
| 289 |
"fields": ["code", "name", "account_type", "write_date"],
|
| 290 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
}
|
| 292 |
|
| 293 |
BATCH = 2000
|
|
@@ -377,12 +416,45 @@ def ready():
|
|
| 377 |
cur.close()
|
| 378 |
except Exception:
|
| 379 |
return False
|
| 380 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 381 |
_READY["ok"] = True
|
| 382 |
return True
|
| 383 |
return False
|
| 384 |
|
| 385 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 386 |
def ro_con():
|
| 387 |
"""Per-thread cached read CURSOR on the shared instance for page/module store reads (the
|
| 388 |
open cost dominates repeated small queries β the Expenses retrofit measured 7.7sβ1.0s). No
|
|
@@ -611,6 +683,27 @@ def _flatten(rec, fields, m2m=()):
|
|
| 611 |
return out
|
| 612 |
|
| 613 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 614 |
def _cols(spec):
|
| 615 |
cols = ["id BIGINT PRIMARY KEY"]
|
| 616 |
for f in spec["fields"]:
|
|
@@ -621,9 +714,7 @@ def _cols(spec):
|
|
| 621 |
cols.append(f"{f} BOOLEAN")
|
| 622 |
elif f in INT_FIELDS:
|
| 623 |
cols.append(f"{f} BIGINT")
|
| 624 |
-
elif f in
|
| 625 |
-
"price_subtotal", "margin", "purchase_price", "product_uom_qty",
|
| 626 |
-
"standard_price", "debit", "credit", "balance", "amount"):
|
| 627 |
cols.append(f"{f} DOUBLE")
|
| 628 |
else:
|
| 629 |
cols.append(f"{f} VARCHAR")
|
|
|
|
| 288 |
"model": "account.account", "archivable": False,
|
| 289 |
"fields": ["code", "name", "account_type", "write_date"],
|
| 290 |
},
|
| 291 |
+
# ββ W37-T13 (owner: stock moved IN and OUT per SKU) β MEASURED FIRST in
|
| 292 |
+
# `proto/P1-stock-moves.md`, and every clause below is a finding rather than a guess.
|
| 293 |
+
#
|
| 294 |
+
# β THE TWO LOCATION COLUMNS ARE BOTH LOAD-BEARING AND THIS IS THE TICKET'S HEADLINE TRAP.
|
| 295 |
+
# A move is an IN when it ARRIVES somewhere internal from somewhere that is not, and an OUT
|
| 296 |
+
# when it LEAVES somewhere internal for somewhere that is not. `internal -> internal` is
|
| 297 |
+
# **58% of all moves** and is NEITHER β a one-sided domain counts every one of them as BOTH,
|
| 298 |
+
# which roughly doubles both columns while looking entirely plausible.
|
| 299 |
+
#
|
| 300 |
+
# β `picking_code` IS NOT THE DISCRIMINATOR, and the reason is worth the line: it is
|
| 301 |
+
# `store=False`, so a `read_group` on it fails hard, AND `('picking_code','=',False)` returns
|
| 302 |
+
# 0 while 5,800 pickingless done moves exist β **the count that would have warned you also
|
| 303 |
+
# reads 0**. `picking_id` is stored and is carried instead, for provenance only.
|
| 304 |
+
#
|
| 305 |
+
# β `quantity_done` IS THE AGGREGATE, not `product_uom_qty`: it ties to
|
| 306 |
+
# `stock.move.line.qty_done` to the cent over the full window, and it is what actually moved.
|
| 307 |
+
# β NOT ARCHIVABLE β `stock.move` has no `active` field; a move is history and is never
|
| 308 |
+
# archived. Stated so the next reader does not "fix" it by adding `active in [True, False]`.
|
| 309 |
+
# β SIZE: 236k rows over 1,095 days (P1), about a quarter of `account_move_line`'s 963,783.
|
| 310 |
+
"stock_move": {
|
| 311 |
+
# β `optional` β see `ready()`. A NEW entity must never gate the readiness of surfaces
|
| 312 |
+
# that do not read it, or adding one takes the product down until it has synced.
|
| 313 |
+
"model": "stock.move", "archivable": False, "optional": True,
|
| 314 |
+
"fields": ["product_id", "quantity_done", "date", "state",
|
| 315 |
+
"location_id", "location_dest_id", "picking_id", "write_date"],
|
| 316 |
+
},
|
| 317 |
+
# β THE USAGE LOOKUP, WITHOUT WHICH THE ROW ABOVE CANNOT BE READ. `location_id` syncs as
|
| 318 |
+
# `location_id` + `location_id_name` (the m2o convention) β a NAME, never a usage β so the
|
| 319 |
+
# two-sided domain has nothing to test until this table exists. It is tiny (a few hundred
|
| 320 |
+
# rows) and it is the whole difference between a metric and a column of doubled numbers.
|
| 321 |
+
# β `usage` is a selection: `internal` | `supplier` | `customer` | `inventory` | `production`
|
| 322 |
+
# | `transit` | `view`. β Adjustments (`inventory`) and scrap are 30% of IN / 20% of OUT and
|
| 323 |
+
# the two obvious exclusions are NOT equivalent β the 5,320-unit gap is all
|
| 324 |
+
# `Virtual Locations/Scrap` β so the metric that consumes this CHOOSES deliberately and says
|
| 325 |
+
# which; that choice lives in `model/metrics/stock.yml`, not here.
|
| 326 |
+
"stock_location": {
|
| 327 |
+
"model": "stock.location", "archivable": True, "optional": True,
|
| 328 |
+
"fields": ["name", "complete_name", "usage", "active", "write_date"],
|
| 329 |
+
},
|
| 330 |
}
|
| 331 |
|
| 332 |
BATCH = 2000
|
|
|
|
| 416 |
cur.close()
|
| 417 |
except Exception:
|
| 418 |
return False
|
| 419 |
+
# ββ OPTIONAL ENTITIES DO NOT GATE READINESS, AND WITHOUT THIS CLAUSE ADDING ONE IS AN
|
| 420 |
+
# OUTAGE. `ready()` gates EVERY store read β the Analyst, every measure column, every
|
| 421 |
+
# read-through grid β so a new entity in `ENTITIES` makes a mirror that has never synced it
|
| 422 |
+
# report NOT READY and refuses all of them with "the data cache is still warming up". A Space
|
| 423 |
+
# hydrates from a SNAPSHOT taken before the entity existed, so this would fire on every
|
| 424 |
+
# deploy of a wave that adds one: a whole-product outage from a schema line, with the app
|
| 425 |
+
# RUNNING and every gate green. MEASURED on this box the moment `stock_move` was declared
|
| 426 |
+
# (W37-T13): `ready()` flipped to False against an otherwise complete mirror.
|
| 427 |
+
# β THE GUARANTEE IS NOT WEAKENED, IT IS SCOPED. `ready()` still means "every entity the
|
| 428 |
+
# EXISTING surfaces read has finished its backfill". An optional entity's own consumers must
|
| 429 |
+
# check ITS phase before promising a column from it β which is what
|
| 430 |
+
# `semantic._source_ready` does, refusing the offer with a cause rather than serving blanks.
|
| 431 |
+
_required = [k for k, v in ENTITIES.items() if not v.get("optional")]
|
| 432 |
+
if all(phases.get(k) == "live" for k in _required):
|
| 433 |
_READY["ok"] = True
|
| 434 |
return True
|
| 435 |
return False
|
| 436 |
|
| 437 |
|
| 438 |
+
def entity_live(key):
|
| 439 |
+
"""Has THIS entity finished its backfill? The per-entity half of `ready()`.
|
| 440 |
+
|
| 441 |
+
Exists because `ready()` is deliberately blind to optional entities (see above), so a caller
|
| 442 |
+
that depends on one has to ask about it specifically instead of inferring it from a global
|
| 443 |
+
flag that was never about them.
|
| 444 |
+
"""
|
| 445 |
+
if not DB_PATH.exists():
|
| 446 |
+
return False
|
| 447 |
+
try:
|
| 448 |
+
cur = ro_cursor()
|
| 449 |
+
try:
|
| 450 |
+
row = cur.execute("SELECT phase FROM _sync_state WHERE entity = ?", [key]).fetchone()
|
| 451 |
+
finally:
|
| 452 |
+
cur.close()
|
| 453 |
+
except Exception: # noqa: BLE001
|
| 454 |
+
return False
|
| 455 |
+
return bool(row) and row[0] == "live"
|
| 456 |
+
|
| 457 |
+
|
| 458 |
def ro_con():
|
| 459 |
"""Per-thread cached read CURSOR on the shared instance for page/module store reads (the
|
| 460 |
open cost dominates repeated small queries β the Expenses retrofit measured 7.7sβ1.0s). No
|
|
|
|
| 683 |
return out
|
| 684 |
|
| 685 |
|
| 686 |
+
#: ββ A HARD-CODED NAME LIST DECIDES WHETHER A COLUMN IS A NUMBER, and a field missing from it
|
| 687 |
+
#: lands as VARCHAR β which does not fail, it fails LATER and somewhere else. MEASURED (W37-T13):
|
| 688 |
+
#: `stock.move.quantity_done` was absent, so 778,385 rows stored '4.0', '1.0' β¦ as TEXT, every one
|
| 689 |
+
#: of them cleanly castable, and the only symptom was a DuckDB
|
| 690 |
+
#: *"Cannot mix values of type INTEGER_LITERAL and VARCHAR in CASE expression"* thrown by a
|
| 691 |
+
#: FILTERED MEASURE three layers up in `semantic._measure_sql`. Nothing in the sync, the schema or
|
| 692 |
+
#: the row count looked wrong.
|
| 693 |
+
#: β SO ADDING A NUMERIC FIELD TO AN `ENTITIES` SPEC MEANS ADDING IT HERE TOO. There is no
|
| 694 |
+
#: inference: `_row` writes whatever Odoo returned, and Odoo returns `False` for an unset numeric,
|
| 695 |
+
#: so a value-sniffing default would be wrong in the other direction.
|
| 696 |
+
#: β AND THE TYPE IS FIXED AT CREATE. `_ensure_columns` only ADDs missing columns β it never ALTERs
|
| 697 |
+
#: an existing one β so a mirror that already synced the field as VARCHAR keeps it until the table
|
| 698 |
+
#: is dropped and re-synced. Correcting the list fixes every FRESH mirror, which is the Space's
|
| 699 |
+
#: case (it has never synced this entity) and was not mine (I had to ALTER my scratch copy).
|
| 700 |
+
NUMERIC_FIELDS = ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed",
|
| 701 |
+
"price_subtotal", "margin", "purchase_price", "product_uom_qty",
|
| 702 |
+
"standard_price", "debit", "credit", "balance", "amount",
|
| 703 |
+
# W37-T13 β the stock-movement quantity. See the note above.
|
| 704 |
+
"quantity_done")
|
| 705 |
+
|
| 706 |
+
|
| 707 |
def _cols(spec):
|
| 708 |
cols = ["id BIGINT PRIMARY KEY"]
|
| 709 |
for f in spec["fields"]:
|
|
|
|
| 714 |
cols.append(f"{f} BOOLEAN")
|
| 715 |
elif f in INT_FIELDS:
|
| 716 |
cols.append(f"{f} BIGINT")
|
| 717 |
+
elif f in NUMERIC_FIELDS:
|
|
|
|
|
|
|
| 718 |
cols.append(f"{f} DOUBLE")
|
| 719 |
else:
|
| 720 |
cols.append(f"{f} VARCHAR")
|
platform/harness/semantic.py
CHANGED
|
@@ -137,6 +137,9 @@ def _model():
|
|
| 137 |
def reload_model():
|
| 138 |
_model.cache_clear()
|
| 139 |
_tenant.cache_clear()
|
|
|
|
|
|
|
|
|
|
| 140 |
return _model()
|
| 141 |
|
| 142 |
|
|
@@ -926,6 +929,531 @@ def store_field_values(topic, dim, search=None, limit=25):
|
|
| 926 |
con.close()
|
| 927 |
|
| 928 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 929 |
def store_parity(date_from=None, date_to=None):
|
| 930 |
"""THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope."""
|
| 931 |
import core.periods as P
|
|
|
|
| 137 |
def reload_model():
|
| 138 |
_model.cache_clear()
|
| 139 |
_tenant.cache_clear()
|
| 140 |
+
# β The entity-measure offer is derived from the same files and would otherwise survive a
|
| 141 |
+
# reload β a stale offer is a column list that disagrees with the model it claims to come from.
|
| 142 |
+
_ENTITY_OFFER_CACHE.clear()
|
| 143 |
return _model()
|
| 144 |
|
| 145 |
|
|
|
|
| 929 |
con.close()
|
| 930 |
|
| 931 |
|
| 932 |
+
# --- ENTITY LOOKBACK MEASURES (W37-T10 / owner item 4, ruling R1) -------------------------
|
| 933 |
+
#
|
| 934 |
+
# ββ THE FEATURE, STATED ONCE: a database that is a REGISTRY (`odoo_products`, `odoo_agents` β
|
| 935 |
+
# a catalogue, no date column of its own) gets columns that measure a FACT topic over a window
|
| 936 |
+
# and key the answer back to its own rows. "Revenue, last 90 days" as a column on the SKU grid.
|
| 937 |
+
#
|
| 938 |
+
# β WHY THIS IS NOT `core.measure_resolve`. That family is CUSTOMER-GRAIN by construction β it
|
| 939 |
+
# resolves against `allowed_pids` that ARE partner ids, through `harness.measure_filter`, whose
|
| 940 |
+
# `_entity_sql` names the customer entity. Neither file is in this wave's fences, so generalising
|
| 941 |
+
# them was not available; this is the entity-neutral path beside it, and the duplication is
|
| 942 |
+
# DELIBERATE and BOOKED (see the wave-37 lane-B mailbox, ownership-gap learning). The day one
|
| 943 |
+
# lane owns both, `measure_resolve.column_values` should become a thin caller of this.
|
| 944 |
+
#
|
| 945 |
+
# β AN OFFER IS A PROMISE. `_rollup_source_offer`'s docstring makes this argument for rollups and
|
| 946 |
+
# it is the same argument here: a measure this returns but cannot answer mints a column that sits
|
| 947 |
+
# BLANK forever looking configured β no error, nothing to notice. So `entity_measures()` refuses
|
| 948 |
+
# a key it cannot prove, rather than passing the model's list through.
|
| 949 |
+
|
| 950 |
+
|
| 951 |
+
class _EntityMeasureError(ModelError):
|
| 952 |
+
"""A binding problem, not a data problem β raised at OFFER time so it never reaches a cell."""
|
| 953 |
+
|
| 954 |
+
|
| 955 |
+
def topic_for_grid(grid_key):
|
| 956 |
+
"""The ENTITY topic that describes the database `grid_key`, or None.
|
| 957 |
+
|
| 958 |
+
β THE BINDING IS ALREADY DECLARED AT BOTH ENDS and this reads it rather than adding a third
|
| 959 |
+
place to state it: `odoo_products.yml` carries `grid: product_data`, `odoo_agents.yml` carries
|
| 960 |
+
`grid: ut_odoo_agents` (W33-T46's own convention β *"the same store key the nav opens"*). A
|
| 961 |
+
hand-written `{table_key: topic}` map in a route would be a second definition of one fact and
|
| 962 |
+
would drift the day a topic is renamed.
|
| 963 |
+
"""
|
| 964 |
+
key = str(grid_key or "").strip()
|
| 965 |
+
if not key:
|
| 966 |
+
return None
|
| 967 |
+
for tkey, t in _model()["topics"].items():
|
| 968 |
+
if str(t.get("grid") or "") == key:
|
| 969 |
+
return tkey
|
| 970 |
+
return None
|
| 971 |
+
|
| 972 |
+
|
| 973 |
+
def entity_measure_bindings(topic):
|
| 974 |
+
"""`[{source, dim, keys, not_yet}, β¦]` β every FACT topic this entity draws columns from.
|
| 975 |
+
|
| 976 |
+
ββ A LIST, NOT ONE BINDING, since W37-T13. An entity's columns legitimately come from more
|
| 977 |
+
than one fact topic: the product grid takes revenue/units/margin from `sales_lines` and stock
|
| 978 |
+
in/out from `stock_moves`, and those are different tables joined by different dims. Collapsing
|
| 979 |
+
them into one source would have forced stock movement into the sales topic, where it has no
|
| 980 |
+
row β which is how a metric ends up defined twice.
|
| 981 |
+
|
| 982 |
+
β BOTH SHAPES PARSE. A topic may declare `measures:` as a single mapping (the wave's original
|
| 983 |
+
form, still used by `odoo_agents`) or as a LIST of them. Accepting only the list would have
|
| 984 |
+
been a silent break of every binding written before this change.
|
| 985 |
+
|
| 986 |
+
Structural read only: this says what the model CLAIMS. `entity_measures()` is what proves it.
|
| 987 |
+
"""
|
| 988 |
+
t = _model()["topics"].get(topic)
|
| 989 |
+
if not t:
|
| 990 |
+
raise ModelError(f"unknown topic {topic!r}")
|
| 991 |
+
raw = t.get("measures")
|
| 992 |
+
if isinstance(raw, dict):
|
| 993 |
+
raw = [raw]
|
| 994 |
+
if not isinstance(raw, list):
|
| 995 |
+
return []
|
| 996 |
+
out = []
|
| 997 |
+
for b in raw:
|
| 998 |
+
if not isinstance(b, dict) or not b.get("source") or not b.get("dim"):
|
| 999 |
+
continue
|
| 1000 |
+
out.append({"source": str(b["source"]), "dim": str(b["dim"]),
|
| 1001 |
+
"keys": list(b.get("keys") or []), "not_yet": list(b.get("not_yet") or [])})
|
| 1002 |
+
return out
|
| 1003 |
+
|
| 1004 |
+
|
| 1005 |
+
def entity_measure_binding(topic):
|
| 1006 |
+
"""The FIRST binding, or None β kept for callers that predate the multi-source shape."""
|
| 1007 |
+
bs = entity_measure_bindings(topic)
|
| 1008 |
+
return bs[0] if bs else None
|
| 1009 |
+
|
| 1010 |
+
|
| 1011 |
+
def entity_measures(topic, strict=False):
|
| 1012 |
+
"""The measure CATALOGUE an entity topic may offer -> `[{key,label,type,format,empty,β¦}]`.
|
| 1013 |
+
|
| 1014 |
+
The shape is the workspace `measures` contract (`{key, label, type}` β `customer-grid/types.ts`
|
| 1015 |
+
`Measure`), with the extra keys a resolver needs carried alongside; the client reads the three
|
| 1016 |
+
it knows and ignores the rest.
|
| 1017 |
+
|
| 1018 |
+
β FOUR REFUSALS, each one a column that would otherwise render blank forever:
|
| 1019 |
+
1. the metric does not exist, or does not live on the SOURCE topic β a foreign base measure
|
| 1020 |
+
is refused by `store_query` the moment a `group_by` is present, so offering it mints a
|
| 1021 |
+
column that can only ever error (this is exactly what excludes `aov` and `orders`);
|
| 1022 |
+
2. the dim is not a dim of the source topic β nothing to group by;
|
| 1023 |
+
3. the metric declares no `empty:` family β C1: a metric that cannot say what a row with no
|
| 1024 |
+
activity renders as does not ship, because the default IS a claim about truth;
|
| 1025 |
+
4. the metric's format has no grid type β `MEASURE_FIELD_TYPES` is {currency,int,pct}, so a
|
| 1026 |
+
date- or text-valued measure cannot render as a column whatever we promise.
|
| 1027 |
+
|
| 1028 |
+
`strict=True` raises on the first refusal (the gate's mode). Otherwise a refused key is simply
|
| 1029 |
+
absent from the offer and its reason is available through `entity_measure_refusals`.
|
| 1030 |
+
"""
|
| 1031 |
+
return _entity_offer(topic, strict=strict)[0]
|
| 1032 |
+
|
| 1033 |
+
|
| 1034 |
+
def entity_measure_refusals(topic):
|
| 1035 |
+
"""`[{key, reason}]` β every declared key `entity_measures` would NOT offer, and why.
|
| 1036 |
+
|
| 1037 |
+
β The reporting half of standing rule 1 applied to a catalogue: a key that drops out has a
|
| 1038 |
+
stated cause rather than simply being missing from a list nobody diffs.
|
| 1039 |
+
"""
|
| 1040 |
+
return _entity_offer(topic)[1]
|
| 1041 |
+
|
| 1042 |
+
|
| 1043 |
+
#: The offer is a pure function of the MODEL FILES, and `_model()` is already `lru_cache`d β so
|
| 1044 |
+
#: this caches the derivation, not the read. It matters because `routes_grid::events` rebuilds the
|
| 1045 |
+
#: product assembly on EVERY write event (hide a field, save a view, patch a cell), and W30-T30
|
| 1046 |
+
#: records what putting real work on that path costs. Cleared by `reload_model()` like every other
|
| 1047 |
+
#: model-derived cache, so a yml edit is not stale until a restart.
|
| 1048 |
+
_ENTITY_OFFER_CACHE = {}
|
| 1049 |
+
|
| 1050 |
+
|
| 1051 |
+
def _entity_offer(topic, strict=False):
|
| 1052 |
+
if not strict and topic in _ENTITY_OFFER_CACHE:
|
| 1053 |
+
return _ENTITY_OFFER_CACHE[topic]
|
| 1054 |
+
out = _entity_offer_uncached(topic, strict=strict)
|
| 1055 |
+
# ββ AN INDETERMINATE ANSWER IS NEVER CACHED, and this is the more dangerous half of the
|
| 1056 |
+
# guard above. The cache lives for the PROCESS, so a cold start that could not reach the store
|
| 1057 |
+
# would freeze its "I don't know" into the offer every later request reads β long after the
|
| 1058 |
+
# mirror came up. Refusing is safe; refusing FOREVER because of one early request is not.
|
| 1059 |
+
# β The symmetric error is just as bad in the other direction: cache an over-broad offer taken
|
| 1060 |
+
# during warmup and the column is promised for the process lifetime.
|
| 1061 |
+
if not strict and not any(r.get("indeterminate") for r in out[1]):
|
| 1062 |
+
_ENTITY_OFFER_CACHE[topic] = out
|
| 1063 |
+
return out
|
| 1064 |
+
|
| 1065 |
+
|
| 1066 |
+
def _entity_offer_uncached(topic, strict=False):
|
| 1067 |
+
out, refused = [], []
|
| 1068 |
+
seen = set()
|
| 1069 |
+
for b in entity_measure_bindings(topic):
|
| 1070 |
+
o, r = _one_binding_offer(topic, b, strict=strict)
|
| 1071 |
+
# β FIRST BINDING WINS A DUPLICATE KEY, and the collision is REPORTED rather than
|
| 1072 |
+
# silently resolved: two fact topics offering the same metric key would otherwise give a
|
| 1073 |
+
# column whose source depends on dict order.
|
| 1074 |
+
for m in o:
|
| 1075 |
+
if m["key"] in seen:
|
| 1076 |
+
refused.append({"key": m["key"],
|
| 1077 |
+
"reason": f"already offered by another source topic; "
|
| 1078 |
+
f"{b['source']}'s copy is not used"})
|
| 1079 |
+
continue
|
| 1080 |
+
seen.add(m["key"])
|
| 1081 |
+
out.append(m)
|
| 1082 |
+
refused.extend(r)
|
| 1083 |
+
return out, refused
|
| 1084 |
+
|
| 1085 |
+
|
| 1086 |
+
def _source_ready(name):
|
| 1087 |
+
"""Is this mirror table present AND finished backfilling? `None` when it cannot be asked.
|
| 1088 |
+
|
| 1089 |
+
β AN OFFER IS A PROMISE, AND A MODEL BINDING IS NOT EVIDENCE THE DATA IS THERE. A metric can
|
| 1090 |
+
resolve perfectly in the model and still have no rows to group, because the mirror is synced
|
| 1091 |
+
per ENTITY and a Space hydrates from a snapshot taken before the entity existed β which is
|
| 1092 |
+
exactly the window W37-T13 opens by adding `stock_move`. Offering a column then would mint a
|
| 1093 |
+
Metric field that renders BLANK forever with no error anywhere, this repo's most repeated
|
| 1094 |
+
defect shape.
|
| 1095 |
+
"""
|
| 1096 |
+
# β TWO CONDITIONS, NOT ONE. A table can EXIST and be half-backfilled β `_ensure_table` runs
|
| 1097 |
+
# before the first batch β and a grouped query over a partial table returns totals that are
|
| 1098 |
+
# wrong in the one direction nobody checks: too small, per row, with no error. `entity_live`
|
| 1099 |
+
# is the per-entity phase, which `ready()` deliberately no longer covers for an optional
|
| 1100 |
+
# entity (see its comment: covering it would make adding one an outage).
|
| 1101 |
+
try:
|
| 1102 |
+
import harness.datastore as _ds
|
| 1103 |
+
# ββ "THERE IS NO STORE" IS NOT "THIS ENTITY IS NOT SYNCED", AND CONFLATING THEM CACHES
|
| 1104 |
+
# AN EMPTY OFFER FOR THE PROCESS. `entity_live` answers False for both, so without this
|
| 1105 |
+
# line a cold container with no mirror yet refuses EVERY binding, that refusal is
|
| 1106 |
+
# definite rather than indeterminate, and it is memoised β so the grid serves no measures
|
| 1107 |
+
# at all long after the mirror arrives. Measured: with the store path pointed at a file
|
| 1108 |
+
# that does not exist, the offer came back `[]` AND was cached.
|
| 1109 |
+
if not _ds.DB_PATH.exists() or not _ds.ready():
|
| 1110 |
+
return None # cannot ask -> INDETERMINATE, and never cached
|
| 1111 |
+
if not _ds.entity_live(name):
|
| 1112 |
+
return False # asked, and this entity really is not live yet
|
| 1113 |
+
con = _store_con()
|
| 1114 |
+
except Exception: # noqa: BLE001
|
| 1115 |
+
return None # cannot ask -> INDETERMINATE
|
| 1116 |
+
try:
|
| 1117 |
+
con.execute(f"SELECT 1 FROM {name} LIMIT 0")
|
| 1118 |
+
return True
|
| 1119 |
+
except Exception: # noqa: BLE001
|
| 1120 |
+
return False
|
| 1121 |
+
finally:
|
| 1122 |
+
try:
|
| 1123 |
+
con.close()
|
| 1124 |
+
except Exception: # noqa: BLE001
|
| 1125 |
+
pass
|
| 1126 |
+
|
| 1127 |
+
|
| 1128 |
+
def _one_binding_offer(topic, b, strict=False):
|
| 1129 |
+
src = _model()["topics"].get(b["source"])
|
| 1130 |
+
if not src or "store" not in src:
|
| 1131 |
+
raise _EntityMeasureError(
|
| 1132 |
+
f"{topic}: measure source {b['source']!r} has no store binding")
|
| 1133 |
+
dims = (src["store"].get("dims") or {})
|
| 1134 |
+
if b["dim"] not in dims:
|
| 1135 |
+
raise _EntityMeasureError(
|
| 1136 |
+
f"{topic}: measure dim {b['dim']!r} is not a dim of {b['source']!r} "
|
| 1137 |
+
f"(allowed: {sorted(dims)})")
|
| 1138 |
+
# β THE TABLE CHECK, before any key is offered from this binding.
|
| 1139 |
+
_tbl = (src["store"] or {}).get("table")
|
| 1140 |
+
_rdy = _source_ready(_tbl) if _tbl else True
|
| 1141 |
+
if _rdy is not True:
|
| 1142 |
+
# β `None` REFUSES TOO, and that is the opposite of the obvious reading. `None` means "the
|
| 1143 |
+
# store could not be asked" β a warming or absent mirror β and a store that cannot answer
|
| 1144 |
+
# THIS question cannot answer the grouped query either, so offering the column would mint
|
| 1145 |
+
# exactly the permanently-blank field the check exists to prevent. Fail closed.
|
| 1146 |
+
why = ((f"the mirror has not finished syncing `{_tbl}`, so every key from {b['source']!r} "
|
| 1147 |
+
f"would render blank or β worse β as a too-small number from a half-filled table; "
|
| 1148 |
+
f"run `datastore.sync_entity('{_tbl}')` until its phase is `live`")
|
| 1149 |
+
if _rdy is False else
|
| 1150 |
+
(f"the tenant store could not be read, so whether `{_tbl}` can answer is UNKNOWN; "
|
| 1151 |
+
f"refusing rather than promising a column that may render blank"))
|
| 1152 |
+
if strict:
|
| 1153 |
+
raise _EntityMeasureError(f"{topic}: {why}")
|
| 1154 |
+
return [], [{"key": k, "reason": why, "indeterminate": _rdy is None}
|
| 1155 |
+
for k in b["keys"]]
|
| 1156 |
+
|
| 1157 |
+
mts = _model()["metrics"]
|
| 1158 |
+
out, refused = [], []
|
| 1159 |
+
|
| 1160 |
+
def refuse(key, reason):
|
| 1161 |
+
if strict:
|
| 1162 |
+
raise _EntityMeasureError(f"{topic}: measure {key!r} refused β {reason}")
|
| 1163 |
+
refused.append({"key": key, "reason": reason})
|
| 1164 |
+
|
| 1165 |
+
for key in b["keys"]:
|
| 1166 |
+
m = mts.get(key)
|
| 1167 |
+
if not m:
|
| 1168 |
+
refuse(key, "no such metric in the model")
|
| 1169 |
+
continue
|
| 1170 |
+
# β THE CROSS-TOPIC TEST IS ON THE EXPANDED BASE, NOT THE METRIC'S OWN `topic`. `aov`
|
| 1171 |
+
# declares no topic of its own and would pass a naive check; its DENOMINATOR `orders`
|
| 1172 |
+
# lives on `sales_orders`, and that is what `store_query` refuses under a group_by.
|
| 1173 |
+
try:
|
| 1174 |
+
base, _ = _expand_measures([key])
|
| 1175 |
+
except ModelError as e:
|
| 1176 |
+
refuse(key, str(e))
|
| 1177 |
+
continue
|
| 1178 |
+
foreign = sorted({k for k in base if mts[k].get("topic", b["source"]) != b["source"]})
|
| 1179 |
+
if foreign:
|
| 1180 |
+
refuse(key, f"base measure(s) {foreign} live on another topic β cross-topic measures "
|
| 1181 |
+
f"are scalar-only and are refused under a group_by")
|
| 1182 |
+
continue
|
| 1183 |
+
empty = m.get("empty")
|
| 1184 |
+
if empty not in ("zero", "blank"):
|
| 1185 |
+
refuse(key, "declares no `empty:` family (C1: zero for additive, blank for a ratio)")
|
| 1186 |
+
continue
|
| 1187 |
+
ftype = _FORMAT_TYPE.get(m.get("format"))
|
| 1188 |
+
if ftype not in ("currency", "int", "pct"):
|
| 1189 |
+
refuse(key, f"format {m.get('format')!r} has no grid measure type "
|
| 1190 |
+
f"(aios_grid.MEASURE_FIELD_TYPES is currency|int|pct)")
|
| 1191 |
+
continue
|
| 1192 |
+
out.append({
|
| 1193 |
+
"key": key,
|
| 1194 |
+
"label": m.get("label") or key,
|
| 1195 |
+
"type": ftype,
|
| 1196 |
+
"format": m.get("format"),
|
| 1197 |
+
"empty": empty,
|
| 1198 |
+
# The denominator whose zero blanks a ratio. None for an additive metric.
|
| 1199 |
+
"guard": m.get("denominator") if m.get("agg") == "ratio" else None,
|
| 1200 |
+
# ββ TWO NORMALIZERS OF ONE WORD, and this is the seam between them
|
| 1201 |
+
# ([[one-question-two-normalizers]]). A `pct` METRIC is a FRACTION internally
|
| 1202 |
+
# (`margin_pct` = 0.5836); a `pct` GRID COLUMN is PERCENTAGE POINTS β the client
|
| 1203 |
+
# renders it `num(v).toFixed(1) + "%"` (`customer-grid/cells.ts`), and the pool's own
|
| 1204 |
+
# `yoy_pct` has always been on that scale (`core/periods.yoy_pct` multiplies by 100).
|
| 1205 |
+
# Ship the fraction and a 58.4% margin prints as "0.6%": plausible, wrong, and nothing
|
| 1206 |
+
# errors. `measure_filter.resolve_values` states the same rule for the customer path
|
| 1207 |
+
# ("`input_scale` applied OUTWARD⦠the CELL shows points too") and this is that rule,
|
| 1208 |
+
# derived from the metric's own `format` rather than a second hand-written list.
|
| 1209 |
+
# β APPLIED ONCE, in `entity_measure_values`, so cells AND conditions are both in
|
| 1210 |
+
# display units and the filter needs no inward scaling of the typed value.
|
| 1211 |
+
"scale": 100.0 if m.get("format") == "pct" else 1.0,
|
| 1212 |
+
# Cents for money (D-153: whole-dollar money cells are a shipped defect), one decimal
|
| 1213 |
+
# for a percent β the convention `resolve_values` documents, minus its money rounding.
|
| 1214 |
+
"round": 1 if m.get("format") == "pct" else (2 if ftype == "currency" else None),
|
| 1215 |
+
"description": m.get("description") or "",
|
| 1216 |
+
"topic": b["source"],
|
| 1217 |
+
"dim": b["dim"],
|
| 1218 |
+
})
|
| 1219 |
+
return out, refused
|
| 1220 |
+
|
| 1221 |
+
|
| 1222 |
+
def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None,
|
| 1223 |
+
exclude_services=False, offer=None):
|
| 1224 |
+
"""`{group key: {measure key: value}}` for an entity topic's lookback columns.
|
| 1225 |
+
|
| 1226 |
+
ONE grouped query for all `keys` together β they share a topic, a scope and a window, so
|
| 1227 |
+
asking them separately would be N scans of the same rows (measured: the six ship-first
|
| 1228 |
+
product metrics cost 0.22 s together against the mirror).
|
| 1229 |
+
|
| 1230 |
+
β THE GROUP KEY IS THE SOURCE DIM'S VALUE, NOT A pid. This layer does not know how a grid
|
| 1231 |
+
hashes its identity (`product_data.sku_pid` is a CRC32; an agent row's id is the partner id),
|
| 1232 |
+
so the CALLER maps. Keeping the mapping at the caller is what lets one engine serve both.
|
| 1233 |
+
|
| 1234 |
+
ββ `exclude_services` DEFAULTS TO **FALSE** HERE, AND THAT IS THE OPPOSITE OF A RANKING
|
| 1235 |
+
QUERY β measured, after the agent reconciliation went red on it. The service filter exists so
|
| 1236 |
+
Delivery Charges do not top a "best SKUs" list; on an ENTITY LOOKBACK COLUMN the row IS the
|
| 1237 |
+
entity, and dropping part of its activity makes the cell a different number from the one every
|
| 1238 |
+
other surface shows for the same subject. Concretely: with services excluded, per-agent revenue
|
| 1239 |
+
came in **2.87% under** an independent Odoo order-header aggregate and FOUR of nine agents
|
| 1240 |
+
missed their own figure by 4-11%; with them included it ties. A service product on the product
|
| 1241 |
+
grid has the same problem in reverse β its own row would read $0 while it genuinely sold.
|
| 1242 |
+
β So a caller that wants a RANKING must pass `exclude_services=True` deliberately. The default
|
| 1243 |
+
is the one that makes a per-row cell true.
|
| 1244 |
+
|
| 1245 |
+
β THE EMPTY-WINDOW RULE IS APPLIED HERE, not left to a renderer, because it is the difference
|
| 1246 |
+
between a true and a false cell (contract C1):
|
| 1247 |
+
* a group that is ABSENT gets nothing back β the caller fills `zero`-family keys with 0 and
|
| 1248 |
+
leaves `blank`-family keys out, which is what `_entity_zero_fill` does;
|
| 1249 |
+
* a group that is PRESENT but whose ratio denominator is 0 has its ratio DROPPED here.
|
| 1250 |
+
`_post_compute` answers 0.0 for `num/0` and on a grid that prints "0.0%" β a margin
|
| 1251 |
+
nobody measured, under a row the user has no reason to doubt.
|
| 1252 |
+
"""
|
| 1253 |
+
offer = offer if offer is not None else entity_measures(topic)
|
| 1254 |
+
by_key = {m["key"]: m for m in offer}
|
| 1255 |
+
want = [k for k in keys if k in by_key]
|
| 1256 |
+
if not want:
|
| 1257 |
+
return {}
|
| 1258 |
+
# β ONE QUERY PER SOURCE TOPIC (W37-T13). Keys from `sales_lines` and keys from `stock_moves`
|
| 1259 |
+
# are different tables joined by different dims, so they cannot ride one scan β but every key
|
| 1260 |
+
# WITHIN a source still does, which is the whole reason the six product metrics cost one query.
|
| 1261 |
+
groups = {}
|
| 1262 |
+
for k in want:
|
| 1263 |
+
groups.setdefault((by_key[k]["topic"], by_key[k]["dim"]), []).append(k)
|
| 1264 |
+
if len(groups) > 1:
|
| 1265 |
+
merged = {}
|
| 1266 |
+
for (src_, dim_), ks in groups.items():
|
| 1267 |
+
for gk, cell in entity_measure_values(
|
| 1268 |
+
topic, ks, date_from=date_from, date_to=date_to, team_id=team_id,
|
| 1269 |
+
exclude_services=exclude_services, offer=offer).items():
|
| 1270 |
+
merged.setdefault(gk, {}).update(cell)
|
| 1271 |
+
return merged
|
| 1272 |
+
(src, dim), want = next(iter(groups.items()))
|
| 1273 |
+
res = store_query(src, want, group_by=[dim], date_from=date_from, date_to=date_to,
|
| 1274 |
+
team_id=team_id, exclude_services=exclude_services, limit=MAX_GROUPS)
|
| 1275 |
+
# β A TRUNCATED GROUP SET IS A WRONG ANSWER PER ROW, not a short list β the same refusal
|
| 1276 |
+
# `rollup_sql.group_values` makes, for the same reason. Never write cells from one.
|
| 1277 |
+
if res.get("truncated"):
|
| 1278 |
+
raise _EntityMeasureError(
|
| 1279 |
+
f"{topic}: {src} grouped by {dim} exceeded {MAX_GROUPS} groups β refusing to write "
|
| 1280 |
+
f"cells from a truncated result")
|
| 1281 |
+
# `store_query` emits the bare dim name when the dim has no `name_col`, `<dim>_id` when it
|
| 1282 |
+
# does. Both shapes are live in the model, so read whichever arrived.
|
| 1283 |
+
d = (_model()["topics"][src]["store"]["dims"] or {})[dim]
|
| 1284 |
+
kcol = f"{dim}_id" if d.get("name_col") else dim
|
| 1285 |
+
out = {}
|
| 1286 |
+
for row in res["rows"]:
|
| 1287 |
+
gk = row.get(kcol)
|
| 1288 |
+
if gk is None:
|
| 1289 |
+
continue # an unattributed group keys nothing; never key on None
|
| 1290 |
+
cell = {}
|
| 1291 |
+
for k in want:
|
| 1292 |
+
m = by_key[k]
|
| 1293 |
+
if m["guard"] is not None and not row.get(m["guard"]):
|
| 1294 |
+
continue # ratio over a zero denominator -> blank, never 0.0
|
| 1295 |
+
v = row.get(k)
|
| 1296 |
+
if v is None:
|
| 1297 |
+
continue
|
| 1298 |
+
# β THE SCALE BOUNDARY, crossed exactly once β see `scale` in `_entity_offer`.
|
| 1299 |
+
v = v * m["scale"] if m["scale"] != 1.0 else v
|
| 1300 |
+
if m["round"] is not None and isinstance(v, float):
|
| 1301 |
+
v = round(v, m["round"])
|
| 1302 |
+
cell[k] = v
|
| 1303 |
+
out[gk] = cell
|
| 1304 |
+
return out
|
| 1305 |
+
|
| 1306 |
+
|
| 1307 |
+
def entity_zero_fill(cell, keys, offer):
|
| 1308 |
+
"""Complete one row's cells under C1's empty-window rule -> the dict to hand `derived`.
|
| 1309 |
+
|
| 1310 |
+
Additive keys land as a real `0` (it sold nothing, and that is a measurement); ratio keys stay
|
| 1311 |
+
ABSENT so the grid paints an empty cell. Called for EVERY row, including the ones with no
|
| 1312 |
+
group at all β which is 72% of the product catalogue in a 90-day window, and the reason this
|
| 1313 |
+
rule is a contract rather than a default.
|
| 1314 |
+
"""
|
| 1315 |
+
by_key = {m["key"]: m for m in offer}
|
| 1316 |
+
out = dict(cell or {})
|
| 1317 |
+
for k in keys:
|
| 1318 |
+
m = by_key.get(k)
|
| 1319 |
+
if not m or k in out:
|
| 1320 |
+
continue
|
| 1321 |
+
if m["empty"] == "zero":
|
| 1322 |
+
out[k] = 0
|
| 1323 |
+
return out
|
| 1324 |
+
|
| 1325 |
+
|
| 1326 |
+
def entity_measure_leaves(nodes, admitted):
|
| 1327 |
+
"""Every MEASURE leaf in a filter tree whose `colId` is in `admitted` β the entity twin of
|
| 1328 |
+
`measure_filter.collect`, which is bound to the CUSTOMER vocabulary by its own `ADMITTED`."""
|
| 1329 |
+
found = []
|
| 1330 |
+
for n in nodes or []:
|
| 1331 |
+
if not isinstance(n, dict):
|
| 1332 |
+
continue
|
| 1333 |
+
if isinstance(n.get("children"), list):
|
| 1334 |
+
found.extend(entity_measure_leaves(n["children"], admitted))
|
| 1335 |
+
elif n.get("colId") in admitted:
|
| 1336 |
+
found.append(n)
|
| 1337 |
+
return found
|
| 1338 |
+
|
| 1339 |
+
|
| 1340 |
+
def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offer=None,
|
| 1341 |
+
exclude_services=False):
|
| 1342 |
+
"""`{rule id: {group key, β¦}}` β a measure CONDITION answered as a set, per entity row.
|
| 1343 |
+
|
| 1344 |
+
ββ WHY THIS EXISTS AT ALL, because it is not obvious from T10's ticket: serving a non-empty
|
| 1345 |
+
`measures` list does not only offer the Metric COLUMN, it offers the measure CONDITION too β
|
| 1346 |
+
the client feeds the same array to its filter builder. A condition whose id never comes back
|
| 1347 |
+
in `measureSets` is rendered PENDING ("Calculatingβ¦") and matches NOTHING, forever. So the
|
| 1348 |
+
offer and this resolver are one feature; shipping the first alone converts a working filter
|
| 1349 |
+
panel into a permanent spinner.
|
| 1350 |
+
|
| 1351 |
+
β THE POPULATION IS THE WHOLE POOL FOR AN ADDITIVE MEASURE, zero-filled β which is the point.
|
| 1352 |
+
71.5% of the product catalogue has no group in a 90-day window, so `Revenue < 100` is a
|
| 1353 |
+
question ABOUT those rows. Dropping them (the shape a naive GROUP BY gives you) would return
|
| 1354 |
+
exactly the opposite set from the one the user asked for.
|
| 1355 |
+
β A `blank`-family measure (a ratio) is NOT zero-filled and a row without one is simply not
|
| 1356 |
+
comparable β "GM % below 40%" must not match a SKU that sold nothing, because it has no
|
| 1357 |
+
margin percentage at all.
|
| 1358 |
+
|
| 1359 |
+
β A rule that is INCOMPLETE or unanswerable is left OUT of the answer rather than resolved to
|
| 1360 |
+
something β `measure_filter.rule_complete`'s reasoning, unchanged: `to_num(None)` is 0, so a
|
| 1361 |
+
half-typed `Revenue > β¦` would silently become `Revenue > 0` under a confident count.
|
| 1362 |
+
"""
|
| 1363 |
+
from harness import measure_filter as _mf # pure helpers only: percentile / _CMP / STATS
|
| 1364 |
+
from harness import windows as _wn
|
| 1365 |
+
|
| 1366 |
+
offer = offer if offer is not None else entity_measures(topic)
|
| 1367 |
+
by_key = {m["key"]: m for m in offer}
|
| 1368 |
+
if not by_key:
|
| 1369 |
+
return {}
|
| 1370 |
+
cache = {}
|
| 1371 |
+
|
| 1372 |
+
def values_for(mkey, window):
|
| 1373 |
+
"""`{group key: value}` for one (measure, window), memoised within this call."""
|
| 1374 |
+
w = _wn.normalize(window)
|
| 1375 |
+
sig = (mkey, None if w is None else tuple(sorted(w.items())))
|
| 1376 |
+
if sig not in cache:
|
| 1377 |
+
rng = _wn.resolve(window, today)
|
| 1378 |
+
if rng is None:
|
| 1379 |
+
cache[sig] = None # never widen to all time β see resolve()
|
| 1380 |
+
else:
|
| 1381 |
+
vals = entity_measure_values(topic, [mkey], date_from=rng[0], date_to=rng[1],
|
| 1382 |
+
team_id=team_id, exclude_services=exclude_services,
|
| 1383 |
+
offer=offer)
|
| 1384 |
+
cache[sig] = {gk: c[mkey] for gk, c in vals.items() if mkey in c}
|
| 1385 |
+
return cache[sig]
|
| 1386 |
+
|
| 1387 |
+
out = {}
|
| 1388 |
+
for rule in rules or []:
|
| 1389 |
+
rid = str(rule.get("id") or "")
|
| 1390 |
+
if not rid or not _mf.rule_complete(rule):
|
| 1391 |
+
continue
|
| 1392 |
+
mkey = rule.get("colId")
|
| 1393 |
+
m = by_key.get(mkey)
|
| 1394 |
+
op = rule.get("op")
|
| 1395 |
+
if not m or op not in _mf._CMP:
|
| 1396 |
+
continue
|
| 1397 |
+
left = values_for(mkey, rule.get("window"))
|
| 1398 |
+
if left is None:
|
| 1399 |
+
continue
|
| 1400 |
+
pool = list(keys_by_id or left)
|
| 1401 |
+
zero = m["empty"] == "zero"
|
| 1402 |
+
rhs = rule.get("rhs")
|
| 1403 |
+
try:
|
| 1404 |
+
if isinstance(rhs, dict) and rhs.get("kind") == "measure":
|
| 1405 |
+
rk = rhs.get("colId")
|
| 1406 |
+
if rk not in by_key:
|
| 1407 |
+
continue
|
| 1408 |
+
right = values_for(rk, rhs.get("window"))
|
| 1409 |
+
if right is None:
|
| 1410 |
+
continue
|
| 1411 |
+
rzero = by_key[rk]["empty"] == "zero"
|
| 1412 |
+
hit = set()
|
| 1413 |
+
for gk in pool:
|
| 1414 |
+
a, b = left.get(gk), right.get(gk)
|
| 1415 |
+
if a is None:
|
| 1416 |
+
if not zero:
|
| 1417 |
+
continue # a ratio nobody has is not comparable
|
| 1418 |
+
a = 0
|
| 1419 |
+
if b is None:
|
| 1420 |
+
if not rzero:
|
| 1421 |
+
continue
|
| 1422 |
+
b = 0
|
| 1423 |
+
if _mf._CMP[op](a, b):
|
| 1424 |
+
hit.add(gk)
|
| 1425 |
+
elif isinstance(rhs, dict) and rhs.get("kind") == "stat":
|
| 1426 |
+
stat = rhs.get("stat")
|
| 1427 |
+
if stat not in _mf.STATS or op not in _mf.PAIR_OPS:
|
| 1428 |
+
continue
|
| 1429 |
+
# β THE POPULATION IS THE ROWS THAT HAVE THIS MEASURE AT ALL β the owner's ruling
|
| 1430 |
+
# of 2026-07-27, carried over verbatim in effect: with 71.5% of this catalogue at
|
| 1431 |
+
# zero, including them would drag the 25th percentile to exactly 0.0 and make
|
| 1432 |
+
# "below the bottom quartile" match nobody.
|
| 1433 |
+
population = {gk: v for gk, v in left.items()
|
| 1434 |
+
if (keys_by_id is None or gk in set(pool)) and v}
|
| 1435 |
+
cut = _mf.percentile(list(population.values()), stat)
|
| 1436 |
+
if cut is None:
|
| 1437 |
+
hit = set()
|
| 1438 |
+
else:
|
| 1439 |
+
hit = {gk for gk, v in population.items() if _mf._CMP[op](v, cut)}
|
| 1440 |
+
else:
|
| 1441 |
+
target = float(str(rule.get("value")).strip())
|
| 1442 |
+
hit = set()
|
| 1443 |
+
for gk in pool:
|
| 1444 |
+
v = left.get(gk)
|
| 1445 |
+
if v is None:
|
| 1446 |
+
if not zero:
|
| 1447 |
+
continue
|
| 1448 |
+
v = 0
|
| 1449 |
+
if _mf._CMP[op](v, target):
|
| 1450 |
+
hit.add(gk)
|
| 1451 |
+
except (TypeError, ValueError):
|
| 1452 |
+
continue # a non-numeric value is not a question
|
| 1453 |
+
out[rid] = hit
|
| 1454 |
+
return out
|
| 1455 |
+
|
| 1456 |
+
|
| 1457 |
def store_parity(date_from=None, date_to=None):
|
| 1458 |
"""THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope."""
|
| 1459 |
import core.periods as P
|
platform/model/metrics/sales.yml
CHANGED
|
@@ -9,6 +9,24 @@
|
|
| 9 |
# ai_context β what a small model must know to use the metric correctly
|
| 10 |
# validate β the INDEPENDENT Odoo cross-check contract (named method implemented in
|
| 11 |
# harness/semantic.py _VALIDATORS; a metric that can't tie out says so)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
topic: sales_lines
|
| 13 |
metrics:
|
| 14 |
- key: revenue
|
|
@@ -16,6 +34,7 @@ metrics:
|
|
| 16 |
agg: sum
|
| 17 |
field: price_subtotal
|
| 18 |
format: usd
|
|
|
|
| 19 |
description: "Untaxed revenue of confirmed wholesale order lines (the owner's 'sales' number)."
|
| 20 |
ai_context: "Always untaxed; excludes Amazon (GIFTWARE DEALS) and unconfirmed orders. Filter one BU via team_id (Fisch=5, Royal=6)."
|
| 21 |
validate:
|
|
@@ -27,6 +46,7 @@ metrics:
|
|
| 27 |
agg: sum
|
| 28 |
field: product_uom_qty
|
| 29 |
format: int
|
|
|
|
| 30 |
description: "Total quantity across confirmed wholesale order lines."
|
| 31 |
ai_context: "Mixed UoMs are summed as ordered quantity; for weight/case analysis convert per product UoM first."
|
| 32 |
|
|
@@ -35,6 +55,7 @@ metrics:
|
|
| 35 |
agg: sum
|
| 36 |
field: margin
|
| 37 |
format: usd
|
|
|
|
| 38 |
description: "Line revenue minus line cost (Odoo Margin module), summed."
|
| 39 |
ai_context: "margin is read_group-aggregatable (Margin module installed). COGS = revenue - margin. purchase_price is per-UNIT cost β never sum it as a total."
|
| 40 |
|
|
@@ -43,6 +64,7 @@ metrics:
|
|
| 43 |
agg: derived
|
| 44 |
expr: "revenue - margin"
|
| 45 |
format: usd
|
|
|
|
| 46 |
description: "Cost of goods sold, derived: revenue minus gross margin."
|
| 47 |
ai_context: "Derived, not pulled β Odoo carries cost on lines as margin; COGS is the difference."
|
| 48 |
|
|
@@ -52,8 +74,26 @@ metrics:
|
|
| 52 |
numerator: margin
|
| 53 |
denominator: revenue
|
| 54 |
format: pct
|
|
|
|
| 55 |
description: "Gross margin as a share of revenue."
|
| 56 |
-
ai_context: "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
- key: orders
|
| 59 |
label: Orders
|
|
@@ -71,6 +111,7 @@ metrics:
|
|
| 71 |
agg: count_distinct
|
| 72 |
field: order_partner_id
|
| 73 |
format: int
|
|
|
|
| 74 |
description: "Distinct customers with at least one confirmed wholesale order line in the window."
|
| 75 |
ai_context: "Customer = the order's partner. Agent attribution uses res.partner.agent_ids, NOT the order user_id."
|
| 76 |
|
|
|
|
| 9 |
# ai_context β what a small model must know to use the metric correctly
|
| 10 |
# validate β the INDEPENDENT Odoo cross-check contract (named method implemented in
|
| 11 |
# harness/semantic.py _VALIDATORS; a metric that can't tie out says so)
|
| 12 |
+
# empty β W37 C1's EMPTY-WINDOW FAMILY. See below; a metric an ENTITY topic
|
| 13 |
+
# offers as a lookback column MUST declare one.
|
| 14 |
+
#
|
| 15 |
+
# ββ `empty:` β WHAT A ROW WITH NO ACTIVITY IN THE WINDOW RENDERS AS (wave 37, contract C1).
|
| 16 |
+
# Measured and load-bearing: only 1,646 of 5,836 active products sold in the last 90 days, so a
|
| 17 |
+
# per-SKU lookback metric has NO GROUP for 72% of the catalogue. Get the default wrong and every
|
| 18 |
+
# product grid reads as broken. Two values, and the difference is whether the blank cell would be
|
| 19 |
+
# a TRUE STATEMENT:
|
| 20 |
+
# empty: zero ADDITIVE β units, revenue, margin $, COGS. "It sold nothing" is a real
|
| 21 |
+
# measurement, so 0 is the honest cell and a blank would hide a fact.
|
| 22 |
+
# empty: blank RATIO / DERIVED-FROM-A-RATIO β GM %, ASP. A 0% margin on zero sales is a
|
| 23 |
+
# FALSE statement, not a missing one. β AND THE GUARD IS THE DENOMINATOR, NOT
|
| 24 |
+
# THE MISSING GROUP: `semantic._post_compute` returns 0.0 for `num/0`, so a SKU
|
| 25 |
+
# that DID sell at $0 would print "0.0%" with a group behind it. The resolver
|
| 26 |
+
# blanks on a zero denominator, which is the only reading that is never a lie.
|
| 27 |
+
# β A metric with no `empty:` is REFUSED by `entity_measures()` rather than defaulted β a
|
| 28 |
+
# defaulted family is a guess about truth, and C1 says a metric that cannot say which family it
|
| 29 |
+
# is in does not ship.
|
| 30 |
topic: sales_lines
|
| 31 |
metrics:
|
| 32 |
- key: revenue
|
|
|
|
| 34 |
agg: sum
|
| 35 |
field: price_subtotal
|
| 36 |
format: usd
|
| 37 |
+
empty: zero
|
| 38 |
description: "Untaxed revenue of confirmed wholesale order lines (the owner's 'sales' number)."
|
| 39 |
ai_context: "Always untaxed; excludes Amazon (GIFTWARE DEALS) and unconfirmed orders. Filter one BU via team_id (Fisch=5, Royal=6)."
|
| 40 |
validate:
|
|
|
|
| 46 |
agg: sum
|
| 47 |
field: product_uom_qty
|
| 48 |
format: int
|
| 49 |
+
empty: zero
|
| 50 |
description: "Total quantity across confirmed wholesale order lines."
|
| 51 |
ai_context: "Mixed UoMs are summed as ordered quantity; for weight/case analysis convert per product UoM first."
|
| 52 |
|
|
|
|
| 55 |
agg: sum
|
| 56 |
field: margin
|
| 57 |
format: usd
|
| 58 |
+
empty: zero
|
| 59 |
description: "Line revenue minus line cost (Odoo Margin module), summed."
|
| 60 |
ai_context: "margin is read_group-aggregatable (Margin module installed). COGS = revenue - margin. purchase_price is per-UNIT cost β never sum it as a total."
|
| 61 |
|
|
|
|
| 64 |
agg: derived
|
| 65 |
expr: "revenue - margin"
|
| 66 |
format: usd
|
| 67 |
+
empty: zero
|
| 68 |
description: "Cost of goods sold, derived: revenue minus gross margin."
|
| 69 |
ai_context: "Derived, not pulled β Odoo carries cost on lines as margin; COGS is the difference."
|
| 70 |
|
|
|
|
| 74 |
numerator: margin
|
| 75 |
denominator: revenue
|
| 76 |
format: pct
|
| 77 |
+
empty: blank
|
| 78 |
description: "Gross margin as a share of revenue."
|
| 79 |
+
ai_context: "Compare across BUs/categories at the same scope only. β TWO ZERO-REVENUE BEHAVIOURS, deliberately: the scalar/analyst path returns 0 (semantic._post_compute's guard), while an ENTITY LOOKBACK COLUMN renders BLANK (empty: blank) β on a grid, a printed 0.0% beside 5,836 products would assert a margin nobody measured."
|
| 80 |
+
|
| 81 |
+
# β W37-T10 β ASP, the fifth SHIP-FIRST per-SKU metric (proto/P3). Blended $18.76 over 1,646
|
| 82 |
+
# SKUs in the 90 days to 2026-08-19. A RATIO of two same-topic base measures, so it survives a
|
| 83 |
+
# GROUPED store_query (only a CROSS-TOPIC component is refused β that is what stops `aov`,
|
| 84 |
+
# whose denominator `orders` lives on sales_orders).
|
| 85 |
+
# β UNITS ARE AS-ORDERED, mixed UoM. `units` says so and this inherits it: a SKU sold in cases
|
| 86 |
+
# and in singles has an ASP blended across both, which is the true average selling price of a
|
| 87 |
+
# line unit and NOT a per-piece price.
|
| 88 |
+
- key: asp
|
| 89 |
+
label: Average selling price
|
| 90 |
+
agg: ratio
|
| 91 |
+
numerator: revenue
|
| 92 |
+
denominator: units
|
| 93 |
+
format: usd
|
| 94 |
+
empty: blank
|
| 95 |
+
description: "Revenue per unit sold β the blended average selling price."
|
| 96 |
+
ai_context: "revenue Γ· units at identical scope. Mixed units of measure are summed as ordered quantity, so this is per ordered unit, not per piece. Blank when nothing sold: an ASP of $0 on zero units is a false statement, not a missing one."
|
| 97 |
|
| 98 |
- key: orders
|
| 99 |
label: Orders
|
|
|
|
| 111 |
agg: count_distinct
|
| 112 |
field: order_partner_id
|
| 113 |
format: int
|
| 114 |
+
empty: zero
|
| 115 |
description: "Distinct customers with at least one confirmed wholesale order line in the window."
|
| 116 |
ai_context: "Customer = the order's partner. Agent attribution uses res.partner.agent_ids, NOT the order user_id."
|
| 117 |
|
platform/model/metrics/stock.yml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Metrics: stock β physical movement (W37-T13, owner item 4 / ruling R1).
|
| 2 |
+
#
|
| 3 |
+
# ββ THE DIRECTION LIVES IN `store_filter_sql`, WHICH IS WHY THESE ARE TWO METRICS AND NOT ONE
|
| 4 |
+
# METRIC WITH A PARAMETER. The model's filtered-measure support compiles `store_filter_sql` into
|
| 5 |
+
# `sum(CASE WHEN <predicate> THEN <field> ELSE 0 END)`, so IN and OUT are the SAME sum over the
|
| 6 |
+
# SAME rows under two different predicates β one scan answers both, and neither can drift from the
|
| 7 |
+
# other's scope because they share every other clause.
|
| 8 |
+
#
|
| 9 |
+
# β EACH PREDICATE IS TWO-SIDED, and that is the ticket's headline trap (`proto/P1-stock-moves.md`):
|
| 10 |
+
# `internal -> internal` is **58% of all moves**. A one-sided test (`destination is internal`) counts
|
| 11 |
+
# every internal transfer as an arrival AND a one-sided `source is internal` counts it as a
|
| 12 |
+
# departure β both columns roughly double, nothing errors, and the totals stay internally
|
| 13 |
+
# consistent with each other, which is what makes it survive review.
|
| 14 |
+
#
|
| 15 |
+
# β `usage` COMES FROM `stock_location`, NOT from the move. `location_id` mirrors as an id plus a
|
| 16 |
+
# NAME; there is no usage on the move row, so `stock_location` is joined in the topic precisely to
|
| 17 |
+
# make this predicate expressible. If that join is ever dropped, these two metrics do not go red β
|
| 18 |
+
# they go NULL-predicate, which SQL treats as false, and both columns silently read ZERO.
|
| 19 |
+
#
|
| 20 |
+
# β ADJUSTMENTS AND SCRAP ARE INCLUDED, DELIBERATELY, AND THE ALTERNATIVE WAS MEASURED. They are
|
| 21 |
+
# ~30% of IN and ~20% of OUT, and the two obvious exclusions are NOT equivalent β the 5,320-unit
|
| 22 |
+
# gap between them is entirely `Virtual Locations/Scrap`. A physical arrival is an arrival however
|
| 23 |
+
# it was booked, so the honest default is to count it and let the `source`/`destination` dims
|
| 24 |
+
# separate it. Excluding it silently would be a different number under the same label.
|
| 25 |
+
topic: stock_moves
|
| 26 |
+
metrics:
|
| 27 |
+
- key: stock_in
|
| 28 |
+
label: Stock moved in
|
| 29 |
+
agg: sum
|
| 30 |
+
field: quantity_done
|
| 31 |
+
store_filter_sql: "sd.usage = 'internal' AND COALESCE(sl.usage, '') <> 'internal'"
|
| 32 |
+
format: int
|
| 33 |
+
empty: zero
|
| 34 |
+
description: "Units that ARRIVED at an internal location from outside it, over the window."
|
| 35 |
+
ai_context: "Receipts, returns inward and inventory adjustments upward. Two-sided by construction: an internal-to-internal transfer is NOT an arrival. Quantities are in each product's own unit, so never sum this across SKUs."
|
| 36 |
+
|
| 37 |
+
- key: stock_out
|
| 38 |
+
label: Stock moved out
|
| 39 |
+
agg: sum
|
| 40 |
+
field: quantity_done
|
| 41 |
+
store_filter_sql: "sl.usage = 'internal' AND COALESCE(sd.usage, '') <> 'internal'"
|
| 42 |
+
format: int
|
| 43 |
+
empty: zero
|
| 44 |
+
description: "Units that LEFT an internal location for outside it, over the window."
|
| 45 |
+
ai_context: "Deliveries, scrap and adjustments downward. Two-sided by construction: an internal-to-internal transfer is NOT a departure. Quantities are in each product's own unit, so never sum this across SKUs."
|
| 46 |
+
|
| 47 |
+
# β NET IS DERIVED, never a third sum. Deriving it guarantees `net = in - out` exactly, where a
|
| 48 |
+
# separately-summed net could disagree with its own two components under a filter and nothing
|
| 49 |
+
# would say which was right.
|
| 50 |
+
- key: stock_net
|
| 51 |
+
label: Stock moved net
|
| 52 |
+
agg: derived
|
| 53 |
+
expr: "stock_in - stock_out"
|
| 54 |
+
format: int
|
| 55 |
+
empty: zero
|
| 56 |
+
description: "Arrivals minus departures over the window β the period change in units held."
|
| 57 |
+
ai_context: "Derived from stock_in and stock_out, so it always reconciles to them. It is a MOVEMENT figure, not the on-hand balance: the balance is `product_data.on_hand` from stock.quant."
|
platform/model/topics/odoo_agents.yml
CHANGED
|
@@ -21,6 +21,38 @@ store:
|
|
| 21 |
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 22 |
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 25 |
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 26 |
# from another topic; a `link` points at another database.
|
|
|
|
| 21 |
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 22 |
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 23 |
|
| 24 |
+
# ββ W37-T12 / owner item 4 (R1) β THE LOOKBACK-MEASURE BINDING, the agent half of
|
| 25 |
+
# *"It should apply to Odoo agents database as well."* Same contract as
|
| 26 |
+
# `odoo_products.yml`: a FACT topic, and the dim of that topic which carries THIS
|
| 27 |
+
# entity's identity.
|
| 28 |
+
#
|
| 29 |
+
# ββ WHICH AGENT SOURCE, STATED β because there are THREE in this Odoo and they name
|
| 30 |
+
# DIFFERENT PEOPLE (`proto/P3-metric-catalog.md`: 9 agents carry route-1 revenue, 12
|
| 31 |
+
# carry commission lines, 17 carry the flag). This binds ROUTE 1, the CUSTOMER-MASTER
|
| 32 |
+
# BOOK: `sales_lines.agent` is `rp.agent_id`, the customer's assigned agent, so every
|
| 33 |
+
# order of that customer counts toward their agent. That is "whose book is this" and it
|
| 34 |
+
# is the right question for a column on the AGENT REGISTRY.
|
| 35 |
+
# β It is NOT commission. `account.invoice.line.agent` (topic `commission_lines`) is the
|
| 36 |
+
# per-INVOICE-LINE credited agent and answers a different question; `sales_lines.yml`'s
|
| 37 |
+
# own `agent` dim comment carries the full disagreement and says never to "fix" one by
|
| 38 |
+
# reading the other.
|
| 39 |
+
#
|
| 40 |
+
# β THE JOIN NEEDS NO NEW DIM, unlike the product side. `sales_lines.agent` declares a
|
| 41 |
+
# `name_col`, so `store_query` emits `agent_id` β which IS `res.partner.id`, which IS
|
| 42 |
+
# this grid's `odoo_id` identity. Product needed `product_code` because its grid keys on
|
| 43 |
+
# `default_code`; this one already keys on the same integer the fact topic groups by.
|
| 44 |
+
measures:
|
| 45 |
+
source: sales_lines
|
| 46 |
+
dim: agent
|
| 47 |
+
keys: [revenue, units, margin, cogs, margin_pct, asp, customers]
|
| 48 |
+
not_yet:
|
| 49 |
+
- key: orders / aov
|
| 50 |
+
cause: "`orders` lives on topic `sales_orders`, so under a `group_by` it is a CROSS-TOPIC measure and `semantic.store_query` refuses it (scalar-only). `aov` inherits the refusal through its denominator"
|
| 51 |
+
fix: "add an ORDER-COUNT metric to `sales_lines` itself β `count_distinct` over `l.order_id` β which answers the same question at this grain in one pass"
|
| 52 |
+
- key: commission_amount
|
| 53 |
+
cause: "the commission basis lives on `account.invoice.line.agent` (topic `commission_lines`), a different topic AND a different grain; `invoice_lines` deduplicates to at most one agent per line"
|
| 54 |
+
fix: "bind a SECOND measures block per source once the contract allows more than one, and label the columns so the two routes can never be read as the same number"
|
| 55 |
+
|
| 56 |
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 57 |
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 58 |
# from another topic; a `link` points at another database.
|
platform/model/topics/odoo_products.yml
CHANGED
|
@@ -26,6 +26,57 @@ store:
|
|
| 26 |
category: {label: "Category"}
|
| 27 |
supplier: {label: "Supplier"}
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 30 |
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 31 |
# from another topic; a `link` points at another database.
|
|
|
|
| 26 |
category: {label: "Category"}
|
| 27 |
supplier: {label: "Supplier"}
|
| 28 |
|
| 29 |
+
# ββ W37-T10 / owner item 4 (ruling R1) β THE LOOKBACK-MEASURE BINDING. Owner: *"Odoo products
|
| 30 |
+
# should have a lookbsck metrics like sales etc."* This block is what turns `measures: []` on the
|
| 31 |
+
# product workspace into a real catalogue, and it is DECLARED HERE rather than hand-written in a
|
| 32 |
+
# route for the reason `_rollup_source_offer` gives about itself: a second list is a second
|
| 33 |
+
# definition of the same fact, and the two drift the day somebody adds a metric.
|
| 34 |
+
#
|
| 35 |
+
# β THE JOIN KEY IS THE TRAP CONTRACT C1 NAMES, and this is where it is answered. `source` is a
|
| 36 |
+
# FACT topic; `dim` is the dim of THAT topic which carries THIS entity's identity. `sales_lines`
|
| 37 |
+
# has two candidates and only one is right: `product` groups by Odoo's `product_id`, while this
|
| 38 |
+
# database's identity (see `scope.identity` above) is `default_code`. Binding to `product` would
|
| 39 |
+
# key every cell on a number no grid row carries β a column of nulls that reads as "never sold".
|
| 40 |
+
#
|
| 41 |
+
# β A SINGLE-TOPIC GROUPED QUERY, which is why `semantic.store_query`'s cross-topic refusal does
|
| 42 |
+
# not bite: `revenue` lives ON `sales_lines` and we group `sales_lines` by its OWN dim. The same
|
| 43 |
+
# refusal is real and DOES bite `aov`/`orders` (denominator `orders` lives on `sales_orders`), so
|
| 44 |
+
# those two are absent from `keys` below and `entity_measures()` re-proves that rather than
|
| 45 |
+
# trusting this list.
|
| 46 |
+
# β A LIST SINCE W37-T13: an entity's columns legitimately come from more than ONE fact topic.
|
| 47 |
+
# Sales movement is `sales_lines`; physical movement is `stock_moves`. Same grid, same join key
|
| 48 |
+
# (`product_code`), different tables β forcing stock into the sales topic would have defined a
|
| 49 |
+
# metric against rows it does not have.
|
| 50 |
+
measures:
|
| 51 |
+
- source: sales_lines
|
| 52 |
+
dim: product_code
|
| 53 |
+
# β The SHIP-FIRST six of `proto/P3-metric-catalog.md`, and they cost ONE grouped query
|
| 54 |
+
# together (measured 0.22 s over 1,670 groups against the mirror; 2.55 s live).
|
| 55 |
+
keys: [units, revenue, margin, cogs, margin_pct, asp]
|
| 56 |
+
# β REPORTED, NOT SILENTLY DROPPED (standing rule 1's second sentence, applied to a catalogue).
|
| 57 |
+
# Each of these is a real metric `proto/P3-metric-catalog.md` measured and this wave does not
|
| 58 |
+
# ship, with the CAUSE and the fix β so the next session extends the list instead of
|
| 59 |
+
# re-measuring, and nobody reads the six as "all Odoo can answer".
|
| 60 |
+
not_yet:
|
| 61 |
+
- key: days_since_last_sale
|
| 62 |
+
cause: "needs `agg: max` over a DATE, which `semantic._measure_sql` does not implement (sum | count | count_distinct only), and a date-typed measure could not render anyway: `aios_grid.MEASURE_FIELD_TYPES` is {currency, int, pct}"
|
| 63 |
+
fix: "add `agg: max` to `_measure_sql` and express the metric as an INT β `date_diff('day', max(date), today)` β so it renders as a number of days rather than a date. Measured live at 4.61 s over 3,348 SKUs"
|
| 64 |
+
- key: distinct_orders
|
| 65 |
+
cause: "`__count` on the live path is LINE count, not ORDER count (12,707 lines vs 12,645 distinct SKU-order pairs); the honest figure needs a 2-level groupby measured at 10.87 s live"
|
| 66 |
+
fix: "on the STORE path this is `count(DISTINCT l.order_id)` in one pass β add it as a `count_distinct` metric on `sales_lines` with field `order_id`"
|
| 67 |
+
|
| 68 |
+
# ββ W37-T13 β PHYSICAL MOVEMENT. `stock.move` and `stock_location` are mirrored now (the
|
| 69 |
+
# `not_yet` entry that used to sit here said "not mirrored, so there is no store binding to
|
| 70 |
+
# group"; that is what changed). The direction rule is TWO-SIDED and lives in the metrics'
|
| 71 |
+
# `store_filter_sql` β see `model/metrics/stock.yml`, which carries the 58% trap in full.
|
| 72 |
+
- source: stock_moves
|
| 73 |
+
dim: product_code
|
| 74 |
+
keys: [stock_in, stock_out, stock_net]
|
| 75 |
+
not_yet:
|
| 76 |
+
- key: stock_in_excl_adjustments
|
| 77 |
+
cause: "adjustments and scrap are ~30% of IN / ~20% of OUT and the two obvious exclusions are NOT equivalent - the 5,320-unit gap between them is entirely `Virtual Locations/Scrap`, so the choice is a business ruling rather than a filter"
|
| 78 |
+
fix: "ask the owner which exclusion they mean, then add it as a SEPARATE metric with `store_filter_sql` narrowed on `sl.usage`/`sd.usage` - never by changing what `stock_in` means"
|
| 79 |
+
|
| 80 |
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 81 |
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 82 |
# from another topic; a `link` points at another database.
|
platform/model/topics/sales_lines.yml
CHANGED
|
@@ -42,6 +42,19 @@ store:
|
|
| 42 |
dims:
|
| 43 |
order_partner: {col: "l.order_partner_id", name_col: "l.order_partner_name", label: "Customer"}
|
| 44 |
product: {col: "l.product_id", name_col: "l.product_name", label: "Product"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
# team names via CASE (crm_team is not synced; ids are baked tenant scope like the team filter)
|
| 46 |
team: {col: "o.team_id",
|
| 47 |
name_col: "CASE o.team_id WHEN 5 THEN 'Fisch' WHEN 6 THEN 'Royal' ELSE CAST(o.team_id AS VARCHAR) END",
|
|
|
|
| 42 |
dims:
|
| 43 |
order_partner: {col: "l.order_partner_id", name_col: "l.order_partner_name", label: "Customer"}
|
| 44 |
product: {col: "l.product_id", name_col: "l.product_name", label: "Product"}
|
| 45 |
+
# ββ W37-T10 β THE SKU-CODE DIM, added because a per-SKU metric has to group by the key the
|
| 46 |
+
# PRODUCT GRID actually carries. `product` above groups by Odoo's `product_id`; the
|
| 47 |
+
# `odoo_products` database's identity is `default_code` (its own topic file says so), and a
|
| 48 |
+
# metric keyed on the wrong one is a column of nulls that reads as "this never sold".
|
| 49 |
+
# β `COALESCE(NULLIF(...))` REPRODUCES THE GRID'S IDENTITY EXACTLY, including the codeless
|
| 50 |
+
# fallback: `modules/product_data.py` keys an uncoded active product as `pid:<odoo id>`, so
|
| 51 |
+
# this expression has to as well or those rows silently lose their metrics.
|
| 52 |
+
# β It also MERGES a re-SKUed pair: an archived record and the active one sharing a code are
|
| 53 |
+
# one product to the grid, and grouping by `product_id` would split them into two.
|
| 54 |
+
# β NO `name_col` β the code IS its own label, so `store_query` emits the bare `product_code`
|
| 55 |
+
# (the value-keyed shape `rollup_sql.group_values` handles beside `payment_state`).
|
| 56 |
+
product_code: {col: "COALESCE(NULLIF(pp.default_code, ''), 'pid:' || CAST(pp.id AS VARCHAR))",
|
| 57 |
+
label: "SKU code"}
|
| 58 |
# team names via CASE (crm_team is not synced; ids are baked tenant scope like the team filter)
|
| 59 |
team: {col: "o.team_id",
|
| 60 |
name_col: "CASE o.team_id WHEN 5 THEN 'Fisch' WHEN 6 THEN 'Royal' ELSE CAST(o.team_id AS VARCHAR) END",
|
platform/model/topics/stock_moves.yml
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ββ W37-T13 (owner: stock moved IN and OUT, per SKU, over a lookback window).
|
| 2 |
+
#
|
| 3 |
+
# A DATED EVENT STREAM at move grain β the counterpart to `odoo_products`, which is a
|
| 4 |
+
# catalogue and says so ("a catalogue has no date dimension; ask sales_lines for
|
| 5 |
+
# movement"). This is the topic that answers movement for stock, as `sales_lines` does
|
| 6 |
+
# for revenue.
|
| 7 |
+
#
|
| 8 |
+
# β EVERY CLAUSE HERE WAS MEASURED, in `proto/P1-stock-moves.md`, against live Odoo.
|
| 9 |
+
# Do not "simplify" one without re-reading it β three of them are traps that produce a
|
| 10 |
+
# plausible wrong number rather than an error.
|
| 11 |
+
key: stock_moves
|
| 12 |
+
label: Stock movement
|
| 13 |
+
entity: stock.move
|
| 14 |
+
domain_builder: ~
|
| 15 |
+
scope:
|
| 16 |
+
state: "done moves ONLY β a draft/waiting/assigned move is an intention, not a movement"
|
| 17 |
+
direction: >-
|
| 18 |
+
DIRECTION IS A PROPERTY OF THE PAIR OF LOCATIONS, never of one. A move is IN when it
|
| 19 |
+
arrives at an internal location from a non-internal one, and OUT when it leaves an
|
| 20 |
+
internal location for a non-internal one. internal->internal is 58% of all moves and
|
| 21 |
+
is NEITHER; counting it as both is the defect a one-sided domain produces.
|
| 22 |
+
not_picking_code: >-
|
| 23 |
+
`picking_code` is NOT the discriminator. It is store=False, so a groupby faults, and
|
| 24 |
+
('picking_code','=',False) returns 0 while 5,800 pickingless done moves exist - the
|
| 25 |
+
count that would warn you also reads 0.
|
| 26 |
+
scale: "236k rows over 1,095 days; cost is FLAT in window length (30d 2.1s .. 1095d 4.0s live)"
|
| 27 |
+
no_bu: >-
|
| 28 |
+
`stock.move` has no team_id. Stock is HQ-consolidated, like AR and Inventory - stated
|
| 29 |
+
as a fact rather than left to read as a missing filter.
|
| 30 |
+
|
| 31 |
+
grain: "one row per done stock move; time-filterable by move date; NOT BU-filterable"
|
| 32 |
+
|
| 33 |
+
store:
|
| 34 |
+
table: stock_move
|
| 35 |
+
alias: sm
|
| 36 |
+
# β BOTH LOCATION JOINS, and they are the whole of the direction rule. `sl` is where it
|
| 37 |
+
# came FROM, `sd` is where it went TO. Dropping either makes every direction metric
|
| 38 |
+
# silently one-sided.
|
| 39 |
+
join: >-
|
| 40 |
+
LEFT JOIN stock_location sl ON sl.id = sm.location_id
|
| 41 |
+
LEFT JOIN stock_location sd ON sd.id = sm.location_dest_id
|
| 42 |
+
LEFT JOIN product_product pp ON pp.id = sm.product_id
|
| 43 |
+
date_col: "sm.date"
|
| 44 |
+
# NO team_col β see scope.no_bu. A BU-scoped caller is REFUSED by `store_query` with a
|
| 45 |
+
# sentence rather than being served a company-wide number wearing a unit's label.
|
| 46 |
+
scope_sql: "sm.state = 'done'"
|
| 47 |
+
dims:
|
| 48 |
+
# β THE SAME CODE-KEYED SHAPE `sales_lines.product_code` uses, and for the same reason:
|
| 49 |
+
# the product grid's identity is `default_code`, with `pid:<odoo id>` for the codeless
|
| 50 |
+
# actives. A dim keyed on Odoo's `product_id` would key every cell on a value no grid
|
| 51 |
+
# row carries (contract C1's join-key trap).
|
| 52 |
+
product_code: {col: "COALESCE(NULLIF(pp.default_code, ''), 'pid:' || CAST(pp.id AS VARCHAR))",
|
| 53 |
+
label: "SKU code"}
|
| 54 |
+
product: {col: "sm.product_id", name_col: "pp.name", label: "Product"}
|
| 55 |
+
source: {col: "sm.location_id", name_col: "sl.complete_name", label: "From location"}
|
| 56 |
+
destination: {col: "sm.location_dest_id", name_col: "sd.complete_name", label: "To location"}
|
| 57 |
+
|
| 58 |
+
ai_context: >
|
| 59 |
+
Physical stock movement at move grain, from Odoo `stock.move`, done moves only.
|
| 60 |
+
Use it for "what came in", "what went out", "what moved" over a window, per SKU or per location.
|
| 61 |
+
β DIRECTION IS A PAIR TEST, never a single location: `stock_in` counts moves ARRIVING at an
|
| 62 |
+
internal location from a non-internal one, `stock_out` counts moves LEAVING an internal location
|
| 63 |
+
for a non-internal one, and internal-to-internal transfers (58% of all moves) are in NEITHER.
|
| 64 |
+
Quantities are `quantity_done` β what actually moved β in the product's own unit of measure, so
|
| 65 |
+
units are NOT comparable across SKUs and must never be summed into one total across products.
|
| 66 |
+
β THIS TOPIC HAS NO BUSINESS UNIT. `stock.move` carries no team, so a question scoped to Fisch
|
| 67 |
+
or Royal cannot be answered here and must be refused rather than answered company-wide.
|
| 68 |
+
β Adjustments (`inventory` locations) and scrap are ~30% of IN and ~20% of OUT. `stock_in` and
|
| 69 |
+
`stock_out` INCLUDE them, because a physical arrival is an arrival however it was booked; use
|
| 70 |
+
the `source`/`destination` dims to separate them, and say which you did.
|
| 71 |
+
For SALES movement use sales_lines (revenue and units ordered); this topic is the warehouse,
|
| 72 |
+
and the two legitimately disagree because an order is not a shipment.
|
platform/modules/agent.py
CHANGED
|
@@ -214,6 +214,114 @@ def validate(t=None, team_id=None):
|
|
| 214 |
hl = headline(sample, yf, yt, team_id)['revenue']
|
| 215 |
checks.append({'check': f'Agent "{sample}": scorecard YTD == headline YTD', 'a': round(sc_ytd, 2),
|
| 216 |
'b': round(hl, 2), 'gap': round(sc_ytd - hl, 2), 'ok': abs(sc_ytd - hl) <= 1.0})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
return checks
|
| 218 |
|
| 219 |
|
|
|
|
| 214 |
hl = headline(sample, yf, yt, team_id)['revenue']
|
| 215 |
checks.append({'check': f'Agent "{sample}": scorecard YTD == headline YTD', 'a': round(sc_ytd, 2),
|
| 216 |
'b': round(hl, 2), 'gap': round(sc_ytd - hl, 2), 'ok': abs(sc_ytd - hl) <= 1.0})
|
| 217 |
+
checks.extend(validate_measures(t=t, team_id=team_id))
|
| 218 |
+
return checks
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def validate_measures(t=None, team_id=None, days=90):
|
| 222 |
+
"""ββ W37-T12 β the minted PER-AGENT lookback columns, against a DIRECT Odoo aggregate.
|
| 223 |
+
|
| 224 |
+
β WHICH AGENT SOURCE, AND THE TICKET REQUIRES IT SAID OUT LOUD: this reconciles ROUTE 1, the
|
| 225 |
+
CUSTOMER-MASTER BOOK (`res.partner.agent_ids` -> the mirror's `res_partner.agent_id`, which is
|
| 226 |
+
`sales_lines`' `agent` dim). It is NOT the OCA commission route below β measured 2026-08-19,
|
| 227 |
+
9 agents carry route-1 revenue against 12 on commission lines and 17 carrying the flag, so the
|
| 228 |
+
two produce materially different rankings and a check that mixed them would be comparing two
|
| 229 |
+
different questions and calling the gap an error.
|
| 230 |
+
|
| 231 |
+
β THE ORACLE IS THE ORDER HEADER, not the mirror the columns are served from. `sale.order`
|
| 232 |
+
grouped by `partner_id`, mapped to each customer's agent CLIENT-SIDE β a different model, a
|
| 233 |
+
different grain and a different code path from `store_query`'s line-level sum, so agreement
|
| 234 |
+
between them is evidence rather than tautology.
|
| 235 |
+
β Windowed to the MIRROR'S newest order, like `product_data.validate_measures`, so the
|
| 236 |
+
residual is about EDITS to a shared period and not about orders the mirror has never seen.
|
| 237 |
+
"""
|
| 238 |
+
from harness import datastore as DS
|
| 239 |
+
from harness import semantic as sem
|
| 240 |
+
|
| 241 |
+
t = t or P.today()
|
| 242 |
+
checks = []
|
| 243 |
+
try:
|
| 244 |
+
if not DS.ready():
|
| 245 |
+
return [{'check': 'agent lookback measures reconcile to Odoo', 'a': 'no mirror',
|
| 246 |
+
'b': '-', 'ok': False,
|
| 247 |
+
'detail': 'the tenant store is not readable, so this is UNPROVEN, which standing rule '
|
| 248 |
+
'8 does not accept as green'}]
|
| 249 |
+
except Exception as e: # noqa: BLE001
|
| 250 |
+
return [{'check': 'agent lookback measures reconcile to Odoo', 'a': type(e).__name__,
|
| 251 |
+
'b': '-', 'ok': False, 'detail': str(e)[:200]}]
|
| 252 |
+
|
| 253 |
+
offer = sem.entity_measures('odoo_agents')
|
| 254 |
+
checks.append({'check': 'the agent measure OFFER is non-empty and every key resolves '
|
| 255 |
+
'(owner item 4 / R1)',
|
| 256 |
+
'a': len(offer), 'b': '>0', 'ok': bool(offer),
|
| 257 |
+
'detail': {'keys': [m['key'] for m in offer],
|
| 258 |
+
'refused': sem.entity_measure_refusals('odoo_agents')}})
|
| 259 |
+
if not offer:
|
| 260 |
+
return checks
|
| 261 |
+
|
| 262 |
+
con = DS.ro_cursor()
|
| 263 |
+
try:
|
| 264 |
+
newest = con.execute('SELECT max(date_order) FROM sale_order').fetchone()
|
| 265 |
+
finally:
|
| 266 |
+
con.close()
|
| 267 |
+
d_to = t - dt.timedelta(days=2)
|
| 268 |
+
if newest and newest[0]:
|
| 269 |
+
try:
|
| 270 |
+
d_to = min(d_to, dt.date.fromisoformat(str(newest[0])[:10]) - dt.timedelta(days=1))
|
| 271 |
+
except ValueError:
|
| 272 |
+
pass
|
| 273 |
+
d_from = d_to - dt.timedelta(days=days)
|
| 274 |
+
DF, DT = d_from.isoformat(), d_to.isoformat()
|
| 275 |
+
|
| 276 |
+
ours = sem.entity_measure_values('odoo_agents', ['revenue'], date_from=DF, date_to=DT,
|
| 277 |
+
team_id=team_id, offer=offer)
|
| 278 |
+
# THE ORACLE β order headers, grouped by customer, mapped to that customer's agent here.
|
| 279 |
+
o = O.get_odoo()
|
| 280 |
+
grp = o.read_group('sale.order', sales_mod.order_domain(DF, DT, team_id),
|
| 281 |
+
['partner_id', 'amount_untaxed:sum'], ['partner_id'], lazy=False)
|
| 282 |
+
pids = sorted({r['partner_id'][0] for r in grp if r.get('partner_id')})
|
| 283 |
+
agent_of = {}
|
| 284 |
+
for i in range(0, len(pids), 500):
|
| 285 |
+
for p in o.search_read('res.partner', [('id', 'in', pids[i:i + 500])],
|
| 286 |
+
['id', 'agent_ids']):
|
| 287 |
+
ag = (p.get('agent_ids') or [])
|
| 288 |
+
if ag:
|
| 289 |
+
agent_of[p['id']] = ag[0] # the Customers-module convention: agent_ids[0]
|
| 290 |
+
theirs = {}
|
| 291 |
+
for r in grp:
|
| 292 |
+
if not r.get('partner_id'):
|
| 293 |
+
continue
|
| 294 |
+
a = agent_of.get(r['partner_id'][0])
|
| 295 |
+
if a is not None:
|
| 296 |
+
theirs[a] = theirs.get(a, 0.0) + r['amount_untaxed']
|
| 297 |
+
|
| 298 |
+
ours_tot = round(sum(c.get('revenue', 0) for c in ours.values()), 2)
|
| 299 |
+
theirs_tot = round(sum(theirs.values()), 2)
|
| 300 |
+
# β ORDER-HEADER vs LINE-SUM is a REAL basis difference (an order's untaxed total includes
|
| 301 |
+
# lines this topic's service filter drops), so the tolerance is a stated 3% rather than a
|
| 302 |
+
# cent β and the FIGURE is reported so a drift is readable instead of absorbed.
|
| 303 |
+
gap = ours_tot - theirs_tot
|
| 304 |
+
checks.append({
|
| 305 |
+
'check': 'per-agent revenue (BOOK route) vs an INDEPENDENT Odoo order-header aggregate '
|
| 306 |
+
'mapped through res.partner.agent_ids',
|
| 307 |
+
'a': ours_tot, 'b': theirs_tot, 'gap': round(gap, 2),
|
| 308 |
+
'ok': bool(theirs_tot) and abs(gap) <= 0.03 * theirs_tot,
|
| 309 |
+
'detail': {'window': [DF, DT], 'agents_ours': len(ours), 'agents_theirs': len(theirs),
|
| 310 |
+
'gap_pct': round(gap / theirs_tot * 100, 3) if theirs_tot else None,
|
| 311 |
+
'route': 'customer-master book (res.partner.agent_ids), NOT the OCA '
|
| 312 |
+
'commission table; the two name different people'}})
|
| 313 |
+
# β AND PER AGENT, because a total can agree while every row is keyed wrong β the join-key
|
| 314 |
+
# trap contract C1 names. Here the key is the `res.partner` id at both ends.
|
| 315 |
+
off_by = sorted(((abs(ours.get(a, {}).get('revenue', 0.0) - v), a)
|
| 316 |
+
for a, v in theirs.items() if v), reverse=True)
|
| 317 |
+
bad = [(a, round(ours.get(a, {}).get('revenue', 0.0), 2), round(theirs[a], 2))
|
| 318 |
+
for d, a in off_by if d > 0.03 * theirs[a]]
|
| 319 |
+
checks.append({
|
| 320 |
+
'check': 'each agent\'s own figure ties (the C1 join-key test: a wrong key agrees in '
|
| 321 |
+
'total and disagrees on every row)',
|
| 322 |
+
'a': len(bad), 'b': 0, 'ok': not bad,
|
| 323 |
+
'detail': {'worst': [(a, ours_v, th_v) for a, ours_v, th_v in bad[:5]],
|
| 324 |
+
'agents_compared': len(theirs)}})
|
| 325 |
return checks
|
| 326 |
|
| 327 |
|
platform/modules/collections_send.py
CHANGED
|
@@ -180,16 +180,20 @@ def load_collection_list(odoo):
|
|
| 180 |
|
| 181 |
|
| 182 |
# --------------------------------------------------------------- statement email
|
| 183 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
DEFAULT_INTRO = (
|
| 185 |
'Dear {customer},<br><br>'
|
| 186 |
'Please find below your current statement of account with {company}. '
|
| 187 |
'According to our records, the following invoices remain open:'
|
| 188 |
)
|
| 189 |
DEFAULT_FOOTER = (
|
| 190 |
-
'If you have already sent payment, please disregard this notice
|
| 191 |
'For any questions about an invoice, simply reply to this email.<br><br>'
|
| 192 |
-
'Thank you for your business,<br>{company}
|
| 193 |
)
|
| 194 |
|
| 195 |
|
|
|
|
| 180 |
|
| 181 |
|
| 182 |
# --------------------------------------------------------------- statement email
|
| 183 |
+
# β STANDING RULE 2, AND THIS IS THE ONE STRING IN THE PRODUCT THAT LEAVES THE BUILDING.
|
| 184 |
+
# Every other finding `web_prose` reports is copy on a screen somebody here opens; this is the
|
| 185 |
+
# SUBJECT LINE of mail queued to a real debtor, over the single sanctioned Odoo write. It read
|
| 186 |
+
# 'Statement of Account (em dash) {company} (em dash) {month}' until W36-T42.
|
| 187 |
+
DEFAULT_SUBJECT = 'Statement of Account from {company}, {month}'
|
| 188 |
DEFAULT_INTRO = (
|
| 189 |
'Dear {customer},<br><br>'
|
| 190 |
'Please find below your current statement of account with {company}. '
|
| 191 |
'According to our records, the following invoices remain open:'
|
| 192 |
)
|
| 193 |
DEFAULT_FOOTER = (
|
| 194 |
+
'If you have already sent payment, please disregard this notice, and thank you. '
|
| 195 |
'For any questions about an invoice, simply reply to this email.<br><br>'
|
| 196 |
+
'Thank you for your business,<br>{company}<br>Accounts Receivable'
|
| 197 |
)
|
| 198 |
|
| 199 |
|
platform/modules/product_data.py
CHANGED
|
@@ -51,6 +51,7 @@ template and this deliberately mirrors its signature and its row shape.
|
|
| 51 |
carries no team, so a catalogue cannot be BU-shaped at all β which is exactly why the two
|
| 52 |
sources are JOINED rather than merged.
|
| 53 |
"""
|
|
|
|
| 54 |
import json
|
| 55 |
import math as _math
|
| 56 |
import zlib
|
|
@@ -426,6 +427,17 @@ def pool(team_id=None, t=None):
|
|
| 426 |
bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
|
| 427 |
sup = supplier_master()
|
| 428 |
prices, _price_report = _pricelist_by_code()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 429 |
cat = _catalogue_by_code()
|
| 430 |
# β THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the
|
| 431 |
# same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH
|
|
@@ -486,6 +498,19 @@ def pool(team_id=None, t=None):
|
|
| 486 |
p = prices.get(code) or {}
|
| 487 |
for _col, _name in products.PRICELIST_COLUMNS:
|
| 488 |
row[_col] = p.get(_col)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
# Wave 17 R3 β the supplier master, on every pull (it is catalogue data, not stock).
|
| 490 |
s = sup.get(code) or {}
|
| 491 |
row.update({
|
|
@@ -1052,4 +1077,390 @@ def validate(team_id=None, t=None):
|
|
| 1052 |
"note": "shipped = seasonal rows + LTM-fallback rows scaled to the "
|
| 1053 |
"horizon, so it brackets rather than equals the window total."},
|
| 1054 |
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1055 |
return checks
|
|
|
|
| 51 |
carries no team, so a catalogue cannot be BU-shaped at all β which is exactly why the two
|
| 52 |
sources are JOINED rather than merged.
|
| 53 |
"""
|
| 54 |
+
import datetime as _dt
|
| 55 |
import json
|
| 56 |
import math as _math
|
| 57 |
import zlib
|
|
|
|
| 427 |
bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
|
| 428 |
sup = supplier_master()
|
| 429 |
prices, _price_report = _pricelist_by_code()
|
| 430 |
+
# β W37-T14 / T15. β MEASURED COST, stated because it lands on a scope's FIRST build:
|
| 431 |
+
# tier prices 15.1 s + packagings 8.1 s on top of the ~44 s consolidated build. Only the
|
| 432 |
+
# first build for a scope blocks (`routes_products._pool_for`'s stale-while-refresh), and
|
| 433 |
+
# both degrade to `{}` on a read failure rather than taking the grid down β the same
|
| 434 |
+
# asymmetry `pricelist_by_code` documents: a column is not the ROW SET.
|
| 435 |
+
try:
|
| 436 |
+
_prods = products.active_products() # ONE read, shared by both (see its docstring)
|
| 437 |
+
except Exception: # noqa: BLE001
|
| 438 |
+
_prods = None
|
| 439 |
+
tiers, _tier_report = products.tier_prices_by_code(_prods)
|
| 440 |
+
packs, _pack_report = products.packagings_by_code(_prods)
|
| 441 |
cat = _catalogue_by_code()
|
| 442 |
# β THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the
|
| 443 |
# same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH
|
|
|
|
| 498 |
p = prices.get(code) or {}
|
| 499 |
for _col, _name in products.PRICELIST_COLUMNS:
|
| 500 |
row[_col] = p.get(_col)
|
| 501 |
+
# ββ W37-T14 / T15 β THE HONEST SETS, beside the three declared columns rather than
|
| 502 |
+
# instead of them. The columns above answer "what does Fisch charge"; these answer "how
|
| 503 |
+
# many prices/units does this SKU actually have", which the columns structurally cannot:
|
| 504 |
+
# they are named after THIS tenant's lists, so a price on any other list is invisible.
|
| 505 |
+
# β A `json` cell is a STRING on the wire (the type's own contract), so these are dumped
|
| 506 |
+
# here rather than handed over as lists β a bare list would round-trip through the overlay
|
| 507 |
+
# as something `grid_events` refuses to re-parse.
|
| 508 |
+
# β BLANK, NOT "[]" β an empty cell reads as "sold in one unit / not priced on any list",
|
| 509 |
+
# and an empty JSON array on screen reads as a bug.
|
| 510 |
+
_tiers = tiers.get(code)
|
| 511 |
+
row["tier_prices"] = json.dumps(_tiers, ensure_ascii=False) if _tiers else None
|
| 512 |
+
_units = packs.get(code)
|
| 513 |
+
row["units"] = json.dumps(_units, ensure_ascii=False) if _units else None
|
| 514 |
# Wave 17 R3 β the supplier master, on every pull (it is catalogue data, not stock).
|
| 515 |
s = sup.get(code) or {}
|
| 516 |
row.update({
|
|
|
|
| 1077 |
"note": "shipped = seasonal rows + LTM-fallback rows scaled to the "
|
| 1078 |
"horizon, so it brackets rather than equals the window total."},
|
| 1079 |
})
|
| 1080 |
+
checks.extend(validate_measures(t=t, team_id=team_id,
|
| 1081 |
+
pool_codes={r["code"] for r in rows}))
|
| 1082 |
+
checks.extend(validate_price_and_unit_cells(rows))
|
| 1083 |
+
checks.extend(validate_stock_measures(t=t))
|
| 1084 |
+
return checks
|
| 1085 |
+
|
| 1086 |
+
|
| 1087 |
+
def validate_stock_measures(t=None, days=90, sample=4):
|
| 1088 |
+
"""ββ W37-T13 β `stock_in` / `stock_out` per SKU, against a DIRECT live Odoo `read_group`.
|
| 1089 |
+
|
| 1090 |
+
β THE TWO-SIDED DOMAIN IS REPRODUCED ON THE LIVE SIDE, and getting it wrong there would hide
|
| 1091 |
+
exactly the defect this validates. `location_id.usage` is a DOT-PATH FILTER, which Odoo
|
| 1092 |
+
supports; a dot-path GROUPBY faults, which is why the direction is expressed as two separate
|
| 1093 |
+
filtered reads rather than one grouped-by-usage read (`proto/P1-stock-moves.md`, gotcha 1).
|
| 1094 |
+
|
| 1095 |
+
β AND THE DIRECTION SPLIT IS ASSERTED, NOT ASSUMED. A one-sided domain produces IN == OUT for
|
| 1096 |
+
every internal transfer, so a run where the two columns agree everywhere is the signature of
|
| 1097 |
+
the bug rather than of a quiet warehouse. The last leg requires a SKU where they genuinely
|
| 1098 |
+
differ β the ticket's own `done-when` clause, and the only one a total cannot fake.
|
| 1099 |
+
"""
|
| 1100 |
+
from harness import datastore as DS
|
| 1101 |
+
from harness import semantic as sem
|
| 1102 |
+
|
| 1103 |
+
t = t or P.today()
|
| 1104 |
+
checks = []
|
| 1105 |
+
offer = sem.entity_measures("odoo_products")
|
| 1106 |
+
keys = [m["key"] for m in offer if m["key"].startswith("stock_")]
|
| 1107 |
+
if not keys:
|
| 1108 |
+
why = [r for r in sem.entity_measure_refusals("odoo_products")
|
| 1109 |
+
if str(r.get("key", "")).startswith("stock_")]
|
| 1110 |
+
return [{"check": "the product catalogue offers Stock moved in / out (W37-T13)",
|
| 1111 |
+
"ours": 0, "theirs": 3, "ok": False,
|
| 1112 |
+
# β The refusal carries its own cause β reported, not inferred from an absence.
|
| 1113 |
+
"detail": {"refusals": why or "no stock binding declared"}}]
|
| 1114 |
+
checks.append({"check": "the product measure catalogue offers the stock-movement keys "
|
| 1115 |
+
"(W37-T13)", "ours": sorted(keys), "theirs": 3,
|
| 1116 |
+
"ok": {"stock_in", "stock_out"} <= set(keys)})
|
| 1117 |
+
|
| 1118 |
+
con = DS.ro_cursor()
|
| 1119 |
+
try:
|
| 1120 |
+
newest = con.execute("SELECT max(date) FROM stock_move").fetchone()
|
| 1121 |
+
finally:
|
| 1122 |
+
con.close()
|
| 1123 |
+
d_to = t - _dt.timedelta(days=2)
|
| 1124 |
+
if newest and newest[0]:
|
| 1125 |
+
try:
|
| 1126 |
+
d_to = min(d_to, _dt.date.fromisoformat(str(newest[0])[:10]) - _dt.timedelta(days=1))
|
| 1127 |
+
except ValueError:
|
| 1128 |
+
pass
|
| 1129 |
+
d_from = d_to - _dt.timedelta(days=days)
|
| 1130 |
+
DF, DT = d_from.isoformat(), d_to.isoformat()
|
| 1131 |
+
ours = sem.entity_measure_values("odoo_products", ["stock_in", "stock_out"],
|
| 1132 |
+
date_from=DF, date_to=DT, offer=offer)
|
| 1133 |
+
|
| 1134 |
+
o = O.get_odoo()
|
| 1135 |
+
base = [("state", "=", "done"),
|
| 1136 |
+
("date", ">=", f"{DF} 00:00:00"), ("date", "<=", f"{DT} 23:59:59")]
|
| 1137 |
+
IN = base + [("location_dest_id.usage", "=", "internal"),
|
| 1138 |
+
("location_id.usage", "!=", "internal")]
|
| 1139 |
+
OUT = base + [("location_id.usage", "=", "internal"),
|
| 1140 |
+
("location_dest_id.usage", "!=", "internal")]
|
| 1141 |
+
live = {}
|
| 1142 |
+
for dom, side in ((IN, "stock_in"), (OUT, "stock_out")):
|
| 1143 |
+
for r in o.read_group("stock.move", dom, ["product_id", "quantity_done:sum"],
|
| 1144 |
+
["product_id"], lazy=False):
|
| 1145 |
+
if not r.get("product_id"):
|
| 1146 |
+
continue
|
| 1147 |
+
live.setdefault(r["product_id"][0], {})[side] = r["quantity_done"]
|
| 1148 |
+
code_of = _codes_of_odoo_products(o, list(live))
|
| 1149 |
+
by_code = {}
|
| 1150 |
+
for pid_, v in live.items():
|
| 1151 |
+
c = code_of.get(pid_, f"pid:{pid_}")
|
| 1152 |
+
d = by_code.setdefault(c, {"stock_in": 0.0, "stock_out": 0.0})
|
| 1153 |
+
for k in ("stock_in", "stock_out"):
|
| 1154 |
+
d[k] += v.get(k, 0.0)
|
| 1155 |
+
|
| 1156 |
+
for side in ("stock_in", "stock_out"):
|
| 1157 |
+
a = round(sum(c.get(side, 0) for c in ours.values()), 2)
|
| 1158 |
+
b = round(sum(c.get(side, 0) for c in by_code.values()), 2)
|
| 1159 |
+
checks.append({
|
| 1160 |
+
"check": f"{side}: the mirror's per-SKU total vs a DIRECT Odoo read_group under the "
|
| 1161 |
+
f"SAME two-sided location domain",
|
| 1162 |
+
"ours": a, "theirs": b, "gap": round(a - b, 2),
|
| 1163 |
+
"ok": bool(b) and abs(a - b) <= 0.02 * b,
|
| 1164 |
+
"detail": {"window": [DF, DT], "skus_ours": len(ours), "skus_odoo": len(by_code)}})
|
| 1165 |
+
|
| 1166 |
+
# β THE NAMED SKU, and it is chosen for DIFFERING β see the docstring.
|
| 1167 |
+
diff = sorted(((abs((c.get("stock_in") or 0) - (c.get("stock_out") or 0)), k)
|
| 1168 |
+
for k, c in ours.items()), reverse=True)[:sample]
|
| 1169 |
+
named = []
|
| 1170 |
+
for _d, k in diff:
|
| 1171 |
+
named.append({"sku": k,
|
| 1172 |
+
"ours": {s: round(ours[k].get(s, 0), 2) for s in ("stock_in", "stock_out")},
|
| 1173 |
+
"odoo": {s: round((by_code.get(k) or {}).get(s, 0), 2)
|
| 1174 |
+
for s in ("stock_in", "stock_out")}})
|
| 1175 |
+
off = [n for n in named
|
| 1176 |
+
if any(abs(n["ours"][s] - n["odoo"][s]) > max(0.01, 0.02 * (n["odoo"][s] or 1))
|
| 1177 |
+
for s in ("stock_in", "stock_out"))]
|
| 1178 |
+
checks.append({
|
| 1179 |
+
"check": f"each NAMED SKU's in/out ties to Odoo ({len(named)} SKUs, picked for the "
|
| 1180 |
+
f"largest in-vs-out difference)",
|
| 1181 |
+
"ours": len(off), "theirs": 0, "ok": not off,
|
| 1182 |
+
"detail": {"named": named[:3], "mismatched": off[:2]}})
|
| 1183 |
+
genuinely_split = [n for n in named if n["ours"]["stock_in"] != n["ours"]["stock_out"]]
|
| 1184 |
+
checks.append({
|
| 1185 |
+
"check": "β the DIRECTION SPLIT is real: at least one SKU where IN and OUT genuinely "
|
| 1186 |
+
"differ. A one-sided domain makes them equal for every internal transfer, so "
|
| 1187 |
+
"all-equal is the SIGNATURE OF THE BUG, not a quiet warehouse",
|
| 1188 |
+
"ours": len(genuinely_split), "theirs": ">=1", "ok": bool(genuinely_split),
|
| 1189 |
+
"detail": {"example": genuinely_split[0] if genuinely_split else None}})
|
| 1190 |
+
return checks
|
| 1191 |
+
|
| 1192 |
+
|
| 1193 |
+
def validate_price_and_unit_cells(rows, sample=6):
|
| 1194 |
+
"""ββ W37-T14 / T15 β the `Tier prices` and `Units` cells, against a FRESH Odoo read.
|
| 1195 |
+
|
| 1196 |
+
β THE ORACLE IS ASKED PER SKU, not in bulk, and deliberately so: the builders group a
|
| 1197 |
+
bulk read in Python, so re-running the same bulk read would re-run the same grouping and
|
| 1198 |
+
could only ever agree with itself. Asking Odoo for ONE SKU's price rules is a different
|
| 1199 |
+
question shape and can actually disagree ([[no-unverifiable-aggregates]]).
|
| 1200 |
+
|
| 1201 |
+
β WHAT IS NOT PROVEN HERE, said rather than implied: that a PERSON sees the cell. These are
|
| 1202 |
+
`json` columns on the product grid and the render is the client's; the data half is what a
|
| 1203 |
+
module `validate()` can reach.
|
| 1204 |
+
"""
|
| 1205 |
+
checks = []
|
| 1206 |
+
priced = [r for r in rows if r.get("tier_prices")]
|
| 1207 |
+
united = [r for r in rows if r.get("units")]
|
| 1208 |
+
checks.append({
|
| 1209 |
+
"check": "the product grid serves a Tier-prices cell (W37-T14) and a Units cell (T15); "
|
| 1210 |
+
"a DECLARED column that is never filled is the defect these replace",
|
| 1211 |
+
"ours": {"with_tier_prices": len(priced), "with_units": len(united), "rows": len(rows)},
|
| 1212 |
+
"theirs": ">0 each",
|
| 1213 |
+
# β Units are legitimately sparse (19.6% measured), so the floor is existence, not a rate.
|
| 1214 |
+
"ok": bool(priced) and bool(united),
|
| 1215 |
+
"detail": {"multi_price_skus": sum(1 for r in priced
|
| 1216 |
+
if len(json.loads(r["tier_prices"])) > 1)},
|
| 1217 |
+
})
|
| 1218 |
+
if not priced:
|
| 1219 |
+
return checks
|
| 1220 |
+
o = O.get_odoo()
|
| 1221 |
+
pls = {p["id"]: str(p.get("name") or "").strip()
|
| 1222 |
+
for p in O.search_read("product.pricelist", [], ["id", "name"])}
|
| 1223 |
+
today = P.today().isoformat()
|
| 1224 |
+
# Prefer SKUs that carry MORE THAN ONE price β the ticket's own subject.
|
| 1225 |
+
cand = sorted(priced, key=lambda r: -len(json.loads(r["tier_prices"])))[:sample]
|
| 1226 |
+
bad = []
|
| 1227 |
+
for r in cand:
|
| 1228 |
+
mine = sorted((t["pricelist"], round(float(t["unit_price"]), 2))
|
| 1229 |
+
for t in json.loads(r["tier_prices"]))
|
| 1230 |
+
pid = r.get("product_id")
|
| 1231 |
+
tmpl = None
|
| 1232 |
+
if pid:
|
| 1233 |
+
rec = o.search_read("product.product", [("id", "=", pid)], ["product_tmpl_id"])
|
| 1234 |
+
tmpl = O.m2o_id(rec[0].get("product_tmpl_id")) if rec else None
|
| 1235 |
+
dom = [("compute_price", "=", "fixed"),
|
| 1236 |
+
"|", ("date_start", "=", False), ("date_start", "<=", today),
|
| 1237 |
+
"|", ("date_end", "=", False), ("date_end", ">=", today),
|
| 1238 |
+
("fixed_price", ">", 0),
|
| 1239 |
+
"|", ("product_id", "=", pid), ("product_tmpl_id", "=", tmpl)]
|
| 1240 |
+
live = o.search_read("product.pricelist.item", dom,
|
| 1241 |
+
["pricelist_id", "fixed_price", "min_quantity", "applied_on"])
|
| 1242 |
+
best = {}
|
| 1243 |
+
for it in live:
|
| 1244 |
+
nm = pls.get(O.m2o_id(it.get("pricelist_id")), "?")
|
| 1245 |
+
q = it.get("min_quantity") or 0.0
|
| 1246 |
+
if nm not in best or q < best[nm][0]:
|
| 1247 |
+
best[nm] = (q, round(it.get("fixed_price") or 0.0, 2))
|
| 1248 |
+
theirs = sorted((nm, v) for nm, (q, v) in best.items())
|
| 1249 |
+
# β A code carried by TWO active products legitimately holds MORE entries than a single
|
| 1250 |
+
# product's rules (D-309 / `2112-12`), so ours is a SUPERSET, never an equality.
|
| 1251 |
+
if not set(theirs) <= set(mine):
|
| 1252 |
+
bad.append({"sku": r.get("code"), "ours": mine, "odoo": theirs})
|
| 1253 |
+
checks.append({
|
| 1254 |
+
"check": f"each sampled SKU's Tier prices contain every live Odoo price for it "
|
| 1255 |
+
f"({len(cand)} SKUs, chosen for having the MOST prices)",
|
| 1256 |
+
"ours": len(bad), "theirs": 0, "ok": not bad,
|
| 1257 |
+
"detail": {"mismatches": bad[:3],
|
| 1258 |
+
"sampled": [r.get("code") for r in cand]},
|
| 1259 |
+
})
|
| 1260 |
+
return checks
|
| 1261 |
+
|
| 1262 |
+
|
| 1263 |
+
def _codes_of_odoo_products(o, ids):
|
| 1264 |
+
"""`{odoo product id: the identity key the GRID uses}` β `default_code`, or `pid:<id>`.
|
| 1265 |
+
|
| 1266 |
+
β BATCHED, 500 at a time. One call per id is ~1,700 XML-RPC round trips on this window and
|
| 1267 |
+
turns a 5-second reconciliation into a coffee break.
|
| 1268 |
+
β `active in [True, False]`: a re-SKUed line points at the ARCHIVED record and the grid
|
| 1269 |
+
merges it under the surviving code, so an active-only read would key it `pid:<id>` and
|
| 1270 |
+
manufacture a mismatch this check would then report as a defect.
|
| 1271 |
+
"""
|
| 1272 |
+
ids = sorted({i for i in ids if i})
|
| 1273 |
+
out = {}
|
| 1274 |
+
for i in range(0, len(ids), 500):
|
| 1275 |
+
for p in o.search_read('product.product',
|
| 1276 |
+
[('id', 'in', ids[i:i + 500]), ('active', 'in', [True, False])],
|
| 1277 |
+
['default_code']):
|
| 1278 |
+
out[p['id']] = p.get('default_code') or f"pid:{p['id']}"
|
| 1279 |
+
return {i: out.get(i, f"pid:{i}") for i in ids}
|
| 1280 |
+
|
| 1281 |
+
|
| 1282 |
+
def validate_measures(t=None, team_id=None, days=90, pool_codes=None):
|
| 1283 |
+
"""ββ W37-T10 β THE MINTED LOOKBACK MEASURES, against a DIRECT Odoo `read_group`.
|
| 1284 |
+
|
| 1285 |
+
Standing rule 8: a number that does not tie to Odoo does not ship. These columns are minted
|
| 1286 |
+
from the tenant MIRROR (`semantic.entity_measure_values`), so the oracle has to be the live
|
| 1287 |
+
ERP and nothing derived from the mirror β otherwise both sides come from the same place and
|
| 1288 |
+
the check cannot fail, which is the self-sealing shape `validate()`'s own header records
|
| 1289 |
+
costing a wave.
|
| 1290 |
+
|
| 1291 |
+
ββ THE MIRROR IS BEHIND LIVE, ALWAYS, AND THAT IS NOT A DEFECT β so a bare equality here
|
| 1292 |
+
would be RED every day and would teach everyone to ignore it. The reconciliation is therefore
|
| 1293 |
+
two-legged, and the second leg is the one that carries the meaning:
|
| 1294 |
+
|
| 1295 |
+
leg 1 totals agree within the lag, and the lag is REPORTED as a number, not a tolerance;
|
| 1296 |
+
leg 2 β EVERY line-level difference traces to a line ODOO WROTE AFTER THE MIRROR'S OWN
|
| 1297 |
+
WATERMARK. This is what makes the check falsifiable: a join bug produces differences
|
| 1298 |
+
on lines the mirror holds perfectly, and leg 2 goes red on the first one.
|
| 1299 |
+
|
| 1300 |
+
β Do NOT "fix" leg 2 by filtering the live side on `write_date <= watermark` and comparing
|
| 1301 |
+
totals β MEASURED 2026-08-19, that is a far worse instrument: confirming an order touches its
|
| 1302 |
+
lines' `write_date` without changing a value, so the filter drops thousands of lines the
|
| 1303 |
+
mirror holds correctly and the gap grows from $425 to $140,030.
|
| 1304 |
+
"""
|
| 1305 |
+
from harness import datastore as DS
|
| 1306 |
+
from harness import semantic as sem
|
| 1307 |
+
|
| 1308 |
+
t = t or P.today()
|
| 1309 |
+
checks = []
|
| 1310 |
+
try:
|
| 1311 |
+
if not DS.ready():
|
| 1312 |
+
return [{"check": "product lookback measures reconcile to Odoo",
|
| 1313 |
+
"ours": "no mirror", "theirs": "-", "ok": False,
|
| 1314 |
+
"detail": "the tenant store is not readable, so the measures are UNPROVEN, "
|
| 1315 |
+
"and an unproven aggregate is exactly what standing rule 8 bars"}]
|
| 1316 |
+
except Exception as e: # noqa: BLE001
|
| 1317 |
+
return [{"check": "product lookback measures reconcile to Odoo",
|
| 1318 |
+
"ours": f"{type(e).__name__}", "theirs": "-", "ok": False, "detail": str(e)[:200]}]
|
| 1319 |
+
|
| 1320 |
+
# β THE WINDOW END COMES FROM THE MIRROR, NOT FROM `today`, and this was measured the wrong
|
| 1321 |
+
# way round first. A window ending today reaches past what the mirror has ever seen: orders
|
| 1322 |
+
# placed since the last sync exist live and NOWHERE in the store, so the totals leg reported
|
| 1323 |
+
# a 2.69% "lag" that was really "the last three days do not exist here yet". Anchoring on the
|
| 1324 |
+
# mirror's own newest order makes the comparison one about EDITS to a shared period β which
|
| 1325 |
+
# is the only difference that could indicate a join bug.
|
| 1326 |
+
con = DS.ro_cursor()
|
| 1327 |
+
try:
|
| 1328 |
+
_newest = con.execute("SELECT max(date_order) FROM sale_order").fetchone()
|
| 1329 |
+
finally:
|
| 1330 |
+
con.close()
|
| 1331 |
+
d_to = t - _dt.timedelta(days=2)
|
| 1332 |
+
if _newest and _newest[0]:
|
| 1333 |
+
_n = str(_newest[0])[:10]
|
| 1334 |
+
try:
|
| 1335 |
+
# one day INSIDE the mirror's newest order: the final day may be half-synced.
|
| 1336 |
+
d_to = min(d_to, _dt.date.fromisoformat(_n) - _dt.timedelta(days=1))
|
| 1337 |
+
except ValueError:
|
| 1338 |
+
pass
|
| 1339 |
+
d_from = d_to - _dt.timedelta(days=days)
|
| 1340 |
+
DF, DT = d_from.isoformat(), d_to.isoformat()
|
| 1341 |
+
|
| 1342 |
+
offer = sem.entity_measures("odoo_products")
|
| 1343 |
+
checks.append({
|
| 1344 |
+
"check": "the product measure OFFER is non-empty and every key it names resolves in the "
|
| 1345 |
+
"semantic model (owner item 4 / R1)",
|
| 1346 |
+
"ours": len(offer), "theirs": ">0", "ok": len(offer) > 0,
|
| 1347 |
+
"detail": {"keys": [m["key"] for m in offer],
|
| 1348 |
+
# β The REPORTING half of standing rule 1: a declared key that dropped out
|
| 1349 |
+
# says why, rather than being quietly absent from a list nobody diffs.
|
| 1350 |
+
"refused": sem.entity_measure_refusals("odoo_products")},
|
| 1351 |
+
})
|
| 1352 |
+
if not offer:
|
| 1353 |
+
return checks
|
| 1354 |
+
missing_family = [m["key"] for m in offer if m.get("empty") not in ("zero", "blank")]
|
| 1355 |
+
checks.append({
|
| 1356 |
+
"check": "every offered measure declares an EMPTY-WINDOW family (C1: additive->0, "
|
| 1357 |
+
"ratio->blank), because 72% of this catalogue has no group in a 90-day window",
|
| 1358 |
+
"ours": len(missing_family), "theirs": 0, "ok": not missing_family,
|
| 1359 |
+
"detail": {"undeclared": missing_family},
|
| 1360 |
+
})
|
| 1361 |
+
|
| 1362 |
+
store = sem.entity_measure_values("odoo_products", ["revenue", "units", "margin"],
|
| 1363 |
+
date_from=DF, date_to=DT, exclude_services=False,
|
| 1364 |
+
offer=offer)
|
| 1365 |
+
o = O.get_odoo()
|
| 1366 |
+
# β ASKED OF ODOO DIRECTLY, grouped by Odoo's OWN product id β deliberately NOT by the SKU
|
| 1367 |
+
# code the mirror joins on, so the oracle cannot inherit our join key.
|
| 1368 |
+
g = o.read_group('sale.order.line', O.sale_line_domain(DF, DT),
|
| 1369 |
+
['product_id', 'price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'],
|
| 1370 |
+
['product_id'], lazy=False)
|
| 1371 |
+
live_tot = {"revenue": round(sum(r['price_subtotal'] for r in g), 2),
|
| 1372 |
+
"units": round(sum(r['product_uom_qty'] for r in g), 2),
|
| 1373 |
+
"margin": round(sum(r['margin'] for r in g), 2)}
|
| 1374 |
+
ours_tot = {k: round(sum(v.get(k, 0) for v in store.values()), 2)
|
| 1375 |
+
for k in ("revenue", "units", "margin")}
|
| 1376 |
+
|
| 1377 |
+
# leg 2 β the falsifiable one. Every discrepant LINE must post-date the mirror's watermark.
|
| 1378 |
+
con = DS.ro_cursor()
|
| 1379 |
+
try:
|
| 1380 |
+
wm = con.execute("SELECT cursor_wd FROM _sync_state WHERE entity = 'sale_order_line'"
|
| 1381 |
+
).fetchone()
|
| 1382 |
+
wm = wm[0] if wm else None
|
| 1383 |
+
rows = con.execute(
|
| 1384 |
+
"SELECT l.id, l.price_subtotal FROM sale_order_line l "
|
| 1385 |
+
"JOIN sale_order o ON o.id = l.order_id "
|
| 1386 |
+
"WHERE o.state IN ('sale','done') AND o.team_id IN (5,6) "
|
| 1387 |
+
" AND l.product_id IS NOT NULL "
|
| 1388 |
+
" AND l.order_partner_id NOT IN "
|
| 1389 |
+
" (SELECT id FROM res_partner WHERE name LIKE 'GIFTWARE%') "
|
| 1390 |
+
" AND CAST(o.date_order AS TIMESTAMP) >= ? AND CAST(o.date_order AS TIMESTAMP) <= ?",
|
| 1391 |
+
[f"{DF} 00:00:00", f"{DT} 23:59:59"]).fetchall()
|
| 1392 |
+
finally:
|
| 1393 |
+
con.close()
|
| 1394 |
+
mine = {r[0]: (r[1] or 0.0) for r in rows}
|
| 1395 |
+
theirs = {r['id']: (r['price_subtotal'] or 0.0) for r in
|
| 1396 |
+
o.search_read('sale.order.line', O.sale_line_domain(DF, DT),
|
| 1397 |
+
['id', 'price_subtotal', 'write_date'])}
|
| 1398 |
+
wd = {r['id']: str(r['write_date']) for r in
|
| 1399 |
+
o.search_read('sale.order.line', O.sale_line_domain(DF, DT), ['id', 'write_date'])}
|
| 1400 |
+
discrepant = [i for i, v in theirs.items()
|
| 1401 |
+
if i not in mine or abs(mine[i] - v) >= 0.005]
|
| 1402 |
+
unexplained = [i for i in discrepant if not wm or wd.get(i, '') <= str(wm)]
|
| 1403 |
+
checks.append({
|
| 1404 |
+
"check": "every per-SKU measure difference vs live Odoo traces to a line Odoo wrote "
|
| 1405 |
+
"AFTER the mirror's watermark, because a join bug would differ on a mirrored line",
|
| 1406 |
+
"ours": len(unexplained), "theirs": 0, "ok": not unexplained,
|
| 1407 |
+
"detail": {"lines_compared": len(theirs), "discrepant": len(discrepant),
|
| 1408 |
+
"explained_by_mirror_lag": len(discrepant) - len(unexplained),
|
| 1409 |
+
"watermark": str(wm), "window": [DF, DT],
|
| 1410 |
+
"unexplained_line_ids": unexplained[:10]},
|
| 1411 |
+
})
|
| 1412 |
+
# ββ THE LEG THAT CATCHES A WRONG JOIN KEY, and neither leg above can. Both of those sum the
|
| 1413 |
+
# same LINES whatever dim they were grouped by, so swapping `dim: product_code` for
|
| 1414 |
+
# `dim: product` (contract C1's trap, an Odoo id where the grid carries a SKU code) leaves
|
| 1415 |
+
# them both green while every cell on the screen goes blank. This one asks: does the answer
|
| 1416 |
+
# arrive under a key a GRID ROW ACTUALLY HAS, and is the value right FOR THAT SKU?
|
| 1417 |
+
code_of = _codes_of_odoo_products(o, [r['product_id'][0] for r in g if r.get('product_id')])
|
| 1418 |
+
by_code = {}
|
| 1419 |
+
for r in g:
|
| 1420 |
+
if not r.get('product_id'):
|
| 1421 |
+
continue
|
| 1422 |
+
c = code_of.get(r['product_id'][0], f"pid:{r['product_id'][0]}")
|
| 1423 |
+
by_code[c] = by_code.get(c, 0.0) + r['price_subtotal']
|
| 1424 |
+
# β The pool is the CALLER'S when it has one (it has already paid for it), and read fresh
|
| 1425 |
+
# otherwise β `validate_measures` is runnable on its own, and a leg that silently skips when
|
| 1426 |
+
# called directly is a leg nobody runs ([[gate-can-report-green-on-nothing]]).
|
| 1427 |
+
pool_codes = set(pool_codes) if pool_codes is not None else {
|
| 1428 |
+
r["code"] for r in pool(team_id=team_id, t=t)}
|
| 1429 |
+
keyed_to_a_row = [c for c in store if c in pool_codes]
|
| 1430 |
+
top = sorted(store.items(), key=lambda kv: -(kv[1].get("revenue") or 0))[:10]
|
| 1431 |
+
spot = [{"sku": c,
|
| 1432 |
+
"ours": round(v.get("revenue") or 0.0, 2),
|
| 1433 |
+
"odoo": round(by_code.get(c, 0.0), 2)} for c, v in top]
|
| 1434 |
+
# A named SKU may legitimately differ by a line Odoo edited after the watermark; the
|
| 1435 |
+
# assertion is that MOST of the top ten tie exactly and NONE is off by an order of magnitude,
|
| 1436 |
+
# which is what a mis-keyed join looks like (0.00 against a five-figure number).
|
| 1437 |
+
exact = sum(1 for s in spot if abs(s["ours"] - s["odoo"]) < 0.005)
|
| 1438 |
+
checks.append({
|
| 1439 |
+
"check": "the measure answer is KEYED TO THE GRID'S OWN IDENTITY (C1's join-key trap): "
|
| 1440 |
+
"every group key is a SKU code a pool row carries, and the top-10 SKUs' revenue "
|
| 1441 |
+
"ties to Odoo for THAT SKU",
|
| 1442 |
+
"ours": {"keys_matching_a_pool_row": len(keyed_to_a_row), "of": len(store),
|
| 1443 |
+
"top10_exact": exact},
|
| 1444 |
+
"theirs": {"keys_matching_a_pool_row": len(store), "of": len(store), "top10_exact": 10},
|
| 1445 |
+
# β Not `== len(store)`: a code whose ONLY product record is archived legitimately has
|
| 1446 |
+
# revenue and no grid row (R12 keeps archived out), which `validate()`'s own
|
| 1447 |
+
# "outside the grid" leg already reconciles. A mis-keyed join lands at ~0, not at 99%.
|
| 1448 |
+
"ok": len(store) > 0 and len(keyed_to_a_row) / len(store) > 0.95 and exact >= 8,
|
| 1449 |
+
"detail": {"spot_checks": spot,
|
| 1450 |
+
"keys_with_no_pool_row": sorted(set(store) - pool_codes)[:10]},
|
| 1451 |
+
})
|
| 1452 |
+
checks.append({
|
| 1453 |
+
"check": "product lookback totals vs a DIRECT Odoo read_group, where the residual is mirror "
|
| 1454 |
+
"lag and is REPORTED as a figure, never absorbed into a tolerance",
|
| 1455 |
+
"ours": ours_tot, "theirs": live_tot,
|
| 1456 |
+
# The assertion is on leg 2; this leg is red only if the lag is implausibly large, which
|
| 1457 |
+
# is the shape that means "the mirror stopped" rather than "the mirror is a day behind".
|
| 1458 |
+
"ok": abs(ours_tot["revenue"] - live_tot["revenue"]) <= max(
|
| 1459 |
+
0.01, live_tot["revenue"] * 0.02),
|
| 1460 |
+
"detail": {"revenue_lag": round(ours_tot["revenue"] - live_tot["revenue"], 2),
|
| 1461 |
+
"revenue_lag_pct": round(
|
| 1462 |
+
(ours_tot["revenue"] - live_tot["revenue"]) / live_tot["revenue"] * 100, 4)
|
| 1463 |
+
if live_tot["revenue"] else None,
|
| 1464 |
+
"skus_in_grid_answer": len(store)},
|
| 1465 |
+
})
|
| 1466 |
return checks
|
platform/modules/products.py
CHANGED
|
@@ -561,6 +561,178 @@ def pricelist_by_code():
|
|
| 561 |
return out, report
|
| 562 |
|
| 563 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 564 |
def catalogue_count():
|
| 565 |
"""The INDEPENDENT population oracle: Odoo's own count of active products.
|
| 566 |
|
|
|
|
| 561 |
return out, report
|
| 562 |
|
| 563 |
|
| 564 |
+
def active_products(limit=50000):
|
| 565 |
+
"""`[{id, default_code, product_tmpl_id}]` for every ACTIVE product β read ONCE and shared.
|
| 566 |
+
|
| 567 |
+
β RAISES ON A SHORT PULL, like `catalogue()` and for the same reason: a truncated product read
|
| 568 |
+
renders as a plausible smaller set of priced SKUs with nothing reporting it.
|
| 569 |
+
β It exists so `tier_prices_by_code` and `packagings_by_code` can be called from ONE pool
|
| 570 |
+
build without each paying for its own copy of the same 5,873-row read β measured at ~4 s each
|
| 571 |
+
on this connection, on a path that is somebody's first page load.
|
| 572 |
+
"""
|
| 573 |
+
dom = [('active', '=', True)]
|
| 574 |
+
prods = O.search_read('product.product', dom, ['id', 'default_code', 'product_tmpl_id'],
|
| 575 |
+
limit=limit)
|
| 576 |
+
n = O.get_odoo().search_count('product.product', dom)
|
| 577 |
+
if len(prods) != n:
|
| 578 |
+
raise ValueError(f"products.active_products: the product pull is TRUNCATED. Read "
|
| 579 |
+
f"{len(prods)} rows against a search_count of {n}.")
|
| 580 |
+
return prods
|
| 581 |
+
|
| 582 |
+
|
| 583 |
+
def tier_prices_by_code(prods=None):
|
| 584 |
+
"""`({code: [{pricelist, unit_price}]}, report)` β EVERY live price a SKU really has.
|
| 585 |
+
|
| 586 |
+
ββ W37-T14 (owner: multiple prices per SKU). `pricelist_by_code` above answers a DIFFERENT
|
| 587 |
+
question and both are needed: it fills three DECLARED columns (`price_fisch`, `price_royal_1`,
|
| 588 |
+
`price_royal_2`) and therefore cannot show a price on a list the contract does not name. This
|
| 589 |
+
one is the honest set β one entry per pricelist that actually prices this SKU today.
|
| 590 |
+
|
| 591 |
+
β IT READS EVERY LIVE PRICELIST, not `PRICELIST_COLUMNS`. Hardcoding this tenant's three list
|
| 592 |
+
names into "how many prices does a SKU have" is exactly what the ticket forbids, and it is what
|
| 593 |
+
would make the answer wrong for tenant #1 on the day they are onboarded.
|
| 594 |
+
|
| 595 |
+
β ROWS ARE TEMPLATE-SCOPED (`proto/P2-pricing-uom.md`): joining on `product_id` drops 99.9% of
|
| 596 |
+
price rows, because `applied_on` is `1_product` on 10,454 of 10,468 items and `1_product` means
|
| 597 |
+
the TEMPLATE. A variant rule (`0_product_variant`, 13 rows) is the more specific statement and
|
| 598 |
+
wins, matching `pricecomp._tier_for` and `pricelist_by_code`.
|
| 599 |
+
|
| 600 |
+
β FILTER BY `pricelist_id`, NEVER BY `active`: the default `product.pricelist.item` count hides
|
| 601 |
+
2,580 archived items, so an `active` filter reads as a smaller, plausible, wrong set.
|
| 602 |
+
|
| 603 |
+
β THE BASE TIER, at qty 1, for the same reason `pricelist_by_code` gives: a catalogue cell has
|
| 604 |
+
no quantity in hand. Measured: only 10 of 10,468 items carry a quantity break at all, so this
|
| 605 |
+
is very nearly the whole story rather than a simplification.
|
| 606 |
+
β CARDINALITY 1..3 TODAY, mode 3 β and `Royal 2` carries 2,605 price rows against **0 customers
|
| 607 |
+
and 0 orders**, so a SKU reading "3 tiers" is catalogue-true and commercially misleading. The
|
| 608 |
+
entry keeps the list NAME so a reader can see which tier it is rather than a bare count.
|
| 609 |
+
"""
|
| 610 |
+
report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0,
|
| 611 |
+
"rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0}
|
| 612 |
+
try:
|
| 613 |
+
pls = {p['id']: str(p.get('name') or '').strip()
|
| 614 |
+
for p in O.search_read('product.pricelist', [], ['id', 'name'])}
|
| 615 |
+
report["lists"] = sorted(pls.values())
|
| 616 |
+
today = P.today().isoformat()
|
| 617 |
+
dom = [('pricelist_id', 'in', sorted(pls)), ('compute_price', '=', 'fixed'),
|
| 618 |
+
('applied_on', 'in', ['0_product_variant', '1_product']),
|
| 619 |
+
'|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 620 |
+
'|', ('date_end', '=', False), ('date_end', '>=', today),
|
| 621 |
+
('fixed_price', '>', 0)]
|
| 622 |
+
rules = O.search_read('product.pricelist.item', dom,
|
| 623 |
+
['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
|
| 624 |
+
'fixed_price', 'min_quantity'])
|
| 625 |
+
|
| 626 |
+
# R6's second sentence: what this reader cannot see is COUNTED, never dropped.
|
| 627 |
+
def _n(extra):
|
| 628 |
+
try:
|
| 629 |
+
return O.get_odoo().search_count('product.pricelist.item', extra)
|
| 630 |
+
except Exception: # noqa: BLE001
|
| 631 |
+
return -1 # -1 reads as "not measured", never as zero
|
| 632 |
+
report["rules_total"] = _n([])
|
| 633 |
+
report["rules_not_fixed"] = _n([('compute_price', '!=', 'fixed')])
|
| 634 |
+
report["rules_global"] = _n([('applied_on', '=', '3_global')])
|
| 635 |
+
report["rules_out_of_date"] = _n(['|', ('date_end', '!=', False),
|
| 636 |
+
('date_start', '!=', False)])
|
| 637 |
+
|
| 638 |
+
by_var, by_tmpl = {}, {}
|
| 639 |
+
for r in rules:
|
| 640 |
+
plid = O.m2o_id(r.get('pricelist_id'))
|
| 641 |
+
if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
|
| 642 |
+
by_var.setdefault((plid, O.m2o_id(r['product_id'])), []).append(r)
|
| 643 |
+
elif r.get('product_tmpl_id'):
|
| 644 |
+
by_tmpl.setdefault((plid, O.m2o_id(r['product_tmpl_id'])), []).append(r)
|
| 645 |
+
|
| 646 |
+
prods = active_products() if prods is None else prods
|
| 647 |
+
except Exception as e: # noqa: BLE001
|
| 648 |
+
report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 649 |
+
return {}, report
|
| 650 |
+
|
| 651 |
+
out = {}
|
| 652 |
+
for p in prods:
|
| 653 |
+
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 654 |
+
tmpl = O.m2o_id(p.get('product_tmpl_id'))
|
| 655 |
+
tiers = []
|
| 656 |
+
for plid, name in sorted(pls.items(), key=lambda kv: kv[1].lower()):
|
| 657 |
+
cands = by_var.get((plid, p['id'])) or by_tmpl.get((plid, tmpl))
|
| 658 |
+
if not cands:
|
| 659 |
+
continue
|
| 660 |
+
base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
|
| 661 |
+
tiers.append({"pricelist": name, "unit_price": round(base.get('fixed_price') or 0.0, 2)})
|
| 662 |
+
if tiers:
|
| 663 |
+
# β A code carried by TWO active products (D-309: `2112-12`) MERGES here, because the
|
| 664 |
+
# grid is keyed by code and one code is one row. The prices are UNIONED rather than
|
| 665 |
+
# one silently winning β a SKU that really does have two different Fisch prices should
|
| 666 |
+
# show both, and hiding one is how a $60 price disappeared behind a $24 one.
|
| 667 |
+
prior = out.get(code)
|
| 668 |
+
if prior is None:
|
| 669 |
+
out[code] = tiers
|
| 670 |
+
else:
|
| 671 |
+
seen = {(t["pricelist"], t["unit_price"]) for t in prior}
|
| 672 |
+
for t in tiers:
|
| 673 |
+
if (t["pricelist"], t["unit_price"]) not in seen:
|
| 674 |
+
prior.append(t)
|
| 675 |
+
report["identity_breaks"] += 1
|
| 676 |
+
else:
|
| 677 |
+
report["skus_with_no_price"] += 1
|
| 678 |
+
return out, report
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
def packagings_by_code(prods=None):
|
| 682 |
+
"""`({code: [{name, qty}]}, report)` β the UNITS a SKU is really sold in (W37-T15).
|
| 683 |
+
|
| 684 |
+
β IT COMES FROM `product.packaging` ALONE, and `proto/P2-pricing-uom.md` measured why the
|
| 685 |
+
obvious alternative is a dead end: the `uom.uom` Config category holds 26 conversion-bearing
|
| 686 |
+
units (`Case-Packed 12` β¦ `Pallet-Packed 4,800`) referenced by **0 products, 0 sale lines and
|
| 687 |
+
0 stock moves**, and every unit actually in use has `factor_inv = 1`. Building on `uom.uom`
|
| 688 |
+
conversions yields a column of 1s.
|
| 689 |
+
|
| 690 |
+
β `qty > 1` IS A FILTER, NOT A TIDY-UP. 6,283 of 7,471 packaging rows are `qty = 1.0` β a
|
| 691 |
+
packaging that packs one of something is not a unit tier β plus real junk (one-character
|
| 692 |
+
names). Without it "units per SKU" reads **5,859** SKUs instead of **1,150**.
|
| 693 |
+
|
| 694 |
+
β 133 ROWS ARE ORPHANS (`product_id = False`) and would collapse under a `groupby(product_id)`
|
| 695 |
+
into ONE fake product carrying 133 packagings. Dropped, and counted.
|
| 696 |
+
|
| 697 |
+
β HONEST EMPTY: only **1,150 of 5,873 SKUs (19.6%)** have any unit tier, so this cell is
|
| 698 |
+
legitimately blank for 80% of the catalogue β "this SKU is sold in one unit", never "missing".
|
| 699 |
+
The 19.6% is not decoration: units are transacted on 77.9% of confirmed sale lines.
|
| 700 |
+
β `qty` is denominated in the product's OWN `uom_id` (matched on 7,338/7,338).
|
| 701 |
+
"""
|
| 702 |
+
report = {"rows_total": 0, "rows_orphan": 0, "rows_qty_le_1": 0, "skus_with_units": 0}
|
| 703 |
+
try:
|
| 704 |
+
rows = O.search_read('product.packaging', [], ['id', 'name', 'qty', 'product_id'])
|
| 705 |
+
except Exception as e: # noqa: BLE001
|
| 706 |
+
report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 707 |
+
return {}, report
|
| 708 |
+
report["rows_total"] = len(rows)
|
| 709 |
+
by_pid = {}
|
| 710 |
+
for r in rows:
|
| 711 |
+
pid = O.m2o_id(r.get('product_id'))
|
| 712 |
+
if not pid:
|
| 713 |
+
report["rows_orphan"] += 1
|
| 714 |
+
continue
|
| 715 |
+
if (r.get('qty') or 0) <= 1:
|
| 716 |
+
report["rows_qty_le_1"] += 1
|
| 717 |
+
continue
|
| 718 |
+
by_pid.setdefault(pid, []).append(
|
| 719 |
+
{"name": str(r.get('name') or '').strip(), "qty": float(r.get('qty') or 0)})
|
| 720 |
+
try:
|
| 721 |
+
prods = active_products() if prods is None else prods
|
| 722 |
+
except Exception as e: # noqa: BLE001
|
| 723 |
+
report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 724 |
+
return {}, report
|
| 725 |
+
out = {}
|
| 726 |
+
for p in prods:
|
| 727 |
+
got = by_pid.get(p['id'])
|
| 728 |
+
if not got:
|
| 729 |
+
continue
|
| 730 |
+
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 731 |
+
out.setdefault(code, []).extend(sorted(got, key=lambda u: u["qty"]))
|
| 732 |
+
report["skus_with_units"] = len(out)
|
| 733 |
+
return out, report
|
| 734 |
+
|
| 735 |
+
|
| 736 |
def catalogue_count():
|
| 737 |
"""The INDEPENDENT population oracle: Odoo's own count of active products.
|
| 738 |
|
requirements.txt
CHANGED
|
@@ -45,3 +45,11 @@ openpyxl>=3.1
|
|
| 45 |
# was green because the dev box happens to have it for unrelated reasons β the container
|
| 46 |
# installs exactly this file. The environment-parity twin of the streamlit lesson above.
|
| 47 |
python-multipart>=0.0.20
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
# was green because the dev box happens to have it for unrelated reasons β the container
|
| 46 |
# installs exactly this file. The environment-parity twin of the streamlit lesson above.
|
| 47 |
python-multipart>=0.0.20
|
| 48 |
+
# β WAVE 37 (D-346, answering `ASK E-1`) β THE OFFICIAL ANTHROPIC SDK. `ai_review.py` called the
|
| 49 |
+
# Messages API over raw `requests`, which meant re-implementing retries, streaming and error
|
| 50 |
+
# taxonomy by hand and drifting from them silently. E ships the SDK behind a LAZY import with the
|
| 51 |
+
# raw-HTTP call still there as the fallback, so the feature works with or without this line β what
|
| 52 |
+
# the line does is turn the fallback into dead weight instead of the live path.
|
| 53 |
+
# β BOTH MANIFESTS, NOT ONE. This file's own streamlit story at the top is exactly what pinning
|
| 54 |
+
# one of them looks like a year later.
|
| 55 |
+
anthropic>=0.96
|
web/index.html
CHANGED
|
@@ -7,7 +7,39 @@
|
|
| 7 |
<title>Loopable</title>
|
| 8 |
</head>
|
| 9 |
<body>
|
| 10 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
<div id="portal"></div>
|
| 12 |
<script type="module" src="/src/main.tsx"></script>
|
| 13 |
</body>
|
|
|
|
| 7 |
<title>Loopable</title>
|
| 8 |
</head>
|
| 9 |
<body>
|
| 10 |
+
<!-- ββ WAVE 37 Β· T48 β THE BOOT SKELETON, AND EVERY CHOICE HERE IS FORCED BY WHEN IT HAS TO
|
| 11 |
+
PAINT. Measured on the deployed build (proto/P4-first-load.md Β§A.2): `#root` was EMPTY and
|
| 12 |
+
the screen white for 6.0β8.0 s of an 11.6β16.0 s first open, while one 580 KB entry chunk
|
| 13 |
+
downloaded and parsed. The `<body>` was two empty divs, so there was nothing to show.
|
| 14 |
+
|
| 15 |
+
β INLINE STYLES, NOT A CLASS. `index.css` is its own ~324 KB render-blocking chunk that
|
| 16 |
+
took 3.3 s in that same trace β a skeleton styled from it would wait for the exact asset
|
| 17 |
+
whose arrival it exists to cover, and paint white until then. Inline is the only thing that
|
| 18 |
+
can render from the HTML alone.
|
| 19 |
+
|
| 20 |
+
β INSIDE `#root`, NOT BESIDE IT. `createRoot(...).render()` REPLACES the container's
|
| 21 |
+
children, so this is removed by the mount itself. A skeleton outside `#root` needs JS to
|
| 22 |
+
take it away, which is a second thing that can fail and a flash-of-two-uis if it does.
|
| 23 |
+
|
| 24 |
+
β NO ANIMATION. A shimmer would be a keyframe the CSS chunk has not delivered yet, and a
|
| 25 |
+
static block that never moves is honest about a page that is still downloading; a frozen
|
| 26 |
+
shimmer would read as a hung app. `aria-hidden` + `role="presentation"`: a screen reader
|
| 27 |
+
should hear the app, not its scaffolding. -->
|
| 28 |
+
<div id="root"><div aria-hidden="true" role="presentation" style="height:100vh;display:flex;flex-direction:column;background:#ffffff">
|
| 29 |
+
<div style="height:44px;flex:none;border-bottom:1px solid #e8e8ea;display:flex;align-items:center;padding:0 14px">
|
| 30 |
+
<div style="width:22px;height:22px;border-radius:11px;background:#efedf9"></div>
|
| 31 |
+
<div style="width:96px;height:12px;margin-left:10px;border-radius:3px;background:#efedf9"></div>
|
| 32 |
+
</div>
|
| 33 |
+
<div style="flex:1;display:flex;min-height:0">
|
| 34 |
+
<div style="width:288px;flex:none;border-right:1px solid #e8e8ea;padding:14px 12px">
|
| 35 |
+
<div style="height:11px;width:64%;border-radius:3px;background:#f1f0f4;margin-bottom:14px"></div>
|
| 36 |
+
<div style="height:11px;width:78%;border-radius:3px;background:#f4f3f7;margin-bottom:10px"></div>
|
| 37 |
+
<div style="height:11px;width:54%;border-radius:3px;background:#f4f3f7;margin-bottom:10px"></div>
|
| 38 |
+
<div style="height:11px;width:70%;border-radius:3px;background:#f4f3f7"></div>
|
| 39 |
+
</div>
|
| 40 |
+
<div style="flex:1;background:#fafafb"></div>
|
| 41 |
+
</div>
|
| 42 |
+
</div></div>
|
| 43 |
<div id="portal"></div>
|
| 44 |
<script type="module" src="/src/main.tsx"></script>
|
| 45 |
</body>
|
web/src/App.tsx
CHANGED
|
@@ -1,4 +1,13 @@
|
|
| 1 |
-
import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
import { OverlayProvider } from "./customer-grid/OverlaySurface";
|
| 3 |
import { isStreamlitComponent } from "./customer-grid/hostBridge";
|
| 4 |
import Shell from "./shell/Shell";
|
|
@@ -21,7 +30,9 @@ export default function App() {
|
|
| 21 |
return (
|
| 22 |
<div style={{ width: "100%", height: "100%", overflow: "hidden" }}>
|
| 23 |
<OverlayProvider>
|
| 24 |
-
<
|
|
|
|
|
|
|
| 25 |
</OverlayProvider>
|
| 26 |
</div>
|
| 27 |
);
|
|
|
|
| 1 |
+
import { Suspense, lazy } from "react";
|
| 2 |
+
// ββ WAVE 37 Β· T48 β LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715
|
| 3 |
+
// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the
|
| 4 |
+
// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build
|
| 5 |
+
// (`proto/P4-first-load.md` Β§A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s,
|
| 6 |
+
// during which `#root` was EMPTY and the screen was white β 6.0-8.0 s of the 11.6-16.0 s first open.
|
| 7 |
+
// β THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is
|
| 8 |
+
// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is
|
| 9 |
+
// indistinguishable from a broken one.
|
| 10 |
+
const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid"));
|
| 11 |
import { OverlayProvider } from "./customer-grid/OverlaySurface";
|
| 12 |
import { isStreamlitComponent } from "./customer-grid/hostBridge";
|
| 13 |
import Shell from "./shell/Shell";
|
|
|
|
| 30 |
return (
|
| 31 |
<div style={{ width: "100%", height: "100%", overflow: "hidden" }}>
|
| 32 |
<OverlayProvider>
|
| 33 |
+
<Suspense fallback={<div className="lp-boot" aria-hidden="true" />}>
|
| 34 |
+
<CustomerGrid />
|
| 35 |
+
</Suspense>
|
| 36 |
</OverlayProvider>
|
| 37 |
</div>
|
| 38 |
);
|
web/src/account/SubscriptionPage.tsx
CHANGED
|
@@ -4,6 +4,8 @@
|
|
| 4 |
// Owner item 13, verbatim: *"add below settings the following button: Feedback β¦,
|
| 5 |
// Usage credits β¦, and a Subscription module β¦ Specialized for the Royal Imports
|
| 6 |
// tenant, do not show it."*
|
|
|
|
|
|
|
| 7 |
//
|
| 8 |
// This is the third of those three, and the only one with nothing behind it yet.
|
| 9 |
// So it says so, in the words this app already uses for a boarded door
|
|
|
|
| 4 |
// Owner item 13, verbatim: *"add below settings the following button: Feedback β¦,
|
| 5 |
// Usage credits β¦, and a Subscription module β¦ Specialized for the Royal Imports
|
| 6 |
// tenant, do not show it."*
|
| 7 |
+
// β "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the
|
| 8 |
+
// owner's own wording. A quote edited to match a later rename is no longer evidence of anything.
|
| 9 |
//
|
| 10 |
// This is the third of those three, and the only one with nothing behind it yet.
|
| 11 |
// So it says so, in the words this app already uses for a boarded door
|
web/src/account/UsagePage.tsx
CHANGED
|
@@ -142,7 +142,7 @@ export function UsageView({
|
|
| 142 |
}) {
|
| 143 |
return (
|
| 144 |
<div className="acct-page">
|
| 145 |
-
<h1 className="acct-title">Usage
|
| 146 |
<div className="acct-col">
|
| 147 |
{canSeeTenant ? (
|
| 148 |
<div className="acct-scope" role="group" aria-label="Whose usage">
|
|
|
|
| 142 |
}) {
|
| 143 |
return (
|
| 144 |
<div className="acct-page">
|
| 145 |
+
<h1 className="acct-title">Usage</h1>
|
| 146 |
<div className="acct-col">
|
| 147 |
{canSeeTenant ? (
|
| 148 |
<div className="acct-scope" role="group" aria-label="Whose usage">
|
web/src/account/account.css
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
/* βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 2 |
account/account.css β WAVE 35, owner item 13: the three surfaces that hang off the
|
| 3 |
-
account button (Feedback, Usage
|
| 4 |
|
| 5 |
β ITS OWN STYLESHEET, NOT index.css. That file belongs to the INTEGRATOR for the whole
|
| 6 |
wave, and the wave's own rule is that a new surface brings its own sheet. Every selector
|
|
|
|
| 1 |
/* βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 2 |
account/account.css β WAVE 35, owner item 13: the three surfaces that hang off the
|
| 3 |
+
account button (Feedback, Usage, Subscription).
|
| 4 |
|
| 5 |
β ITS OWN STYLESHEET, NOT index.css. That file belongs to the INTEGRATOR for the whole
|
| 6 |
wave, and the wave's own rule is that a new surface brings its own sheet. Every selector
|