Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +13 -1
- VERSION +1 -1
- api/ai_review.py +0 -255
- api/automation_engine.py +0 -0
- api/connectors_tt.py +568 -568
- api/deps.py +1 -8
- api/main.py +0 -0
- api/odoo_relational.py +0 -0
- api/routes_admin.py +0 -0
- api/routes_alerts.py +537 -664
- api/routes_automation.py +0 -0
- api/routes_changes.py +1 -85
- api/routes_connectors.py +13 -42
- api/routes_customers.py +0 -11
- api/routes_keychain.py +0 -0
- api/routes_oauth.py +114 -114
- api/routes_odoo_tables.py +0 -0
- api/routes_query.py +693 -713
- api/routes_records.py +164 -211
- api/routes_shares.py +385 -393
- api/routes_tables.py +0 -0
- jobs/web_agent_job.py +2 -117
- platform/aios_grid.py +0 -0
- platform/aios_grid_fields.json +464 -474
- platform/core/alerts.py +2 -21
- platform/core/grid_events.py +0 -0
- platform/core/perms.py +0 -110
- platform/core/registry.py +2 -115
- platform/core/shares.py +4 -26
- platform/core/store.py +817 -838
- platform/core/table_store.py +618 -618
- platform/core/user_tables.py +0 -0
- platform/core/users.py +25 -45
- platform/harness/datastore.py +2 -23
- platform/harness/meta_store.py +521 -521
- platform/harness/tools.py +9 -38
- platform/modules/product_data.py +797 -804
- platform/modules/products.py +728 -737
- web/public/sample_customers.json +545 -558
- web/src/alerts/alertsModel.ts +365 -418
- web/src/automation/AutomationBuilder.tsx +0 -0
- web/src/automation/AutomationDetail.tsx +1 -26
- web/src/automation/AutomationSurface.tsx +782 -806
- web/src/automation/automationApi.ts +0 -11
- web/src/connectors/ConnectorsPage.tsx +0 -14
- web/src/connectors/brandMarks.tsx +338 -411
- web/src/customer-grid/ColumnMenu.tsx +0 -0
- web/src/customer-grid/CustomerGrid.tsx +0 -0
- web/src/customer-grid/ImportDialog.css +154 -154
- web/src/customer-grid/ImportDialog.tsx +299 -299
RELEASES.json
CHANGED
|
@@ -1,6 +1,18 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
{
|
| 5 |
"version": "v24",
|
| 6 |
"sha": "514d829",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "v25 (bc24829)",
|
| 3 |
"releases": [
|
| 4 |
+
{
|
| 5 |
+
"version": "v25",
|
| 6 |
+
"sha": "bc24829",
|
| 7 |
+
"date": "2026-08-14",
|
| 8 |
+
"subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)"
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"version": "v26",
|
| 12 |
+
"sha": "6a642e1",
|
| 13 |
+
"date": "2026-08-14",
|
| 14 |
+
"subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)"
|
| 15 |
+
},
|
| 16 |
{
|
| 17 |
"version": "v24",
|
| 18 |
"sha": "514d829",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
v25 (bc24829)
|
api/ai_review.py
CHANGED
|
@@ -205,258 +205,3 @@ def decide(*, prompt, options, row, fields=(), label="Review", timeout=None):
|
|
| 205 |
"problem": reason or "the model did not choose one of the stages"}
|
| 206 |
return choice, {"provider": p["name"], "model": model, "reason": reason}
|
| 207 |
return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
# ══════════════════════════════════════════════════════════════════ the FLOW WRITER
|
| 211 |
-
# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION.
|
| 212 |
-
#
|
| 213 |
-
# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the
|
| 214 |
-
# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary:
|
| 215 |
-
# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the
|
| 216 |
-
# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object
|
| 217 |
-
# in that vocabulary.
|
| 218 |
-
#
|
| 219 |
-
# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This
|
| 220 |
-
# is the one module in the API package that already holds an LLM ladder and its credentials, so a
|
| 221 |
-
# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs
|
| 222 |
-
# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23
|
| 223 |
-
# shipped three finished routers 404-dead behind green gates. The DOOR is a route on
|
| 224 |
-
# `routes_automation.py`, which is already mounted.
|
| 225 |
-
#
|
| 226 |
-
# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds
|
| 227 |
-
# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are
|
| 228 |
-
# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema
|
| 229 |
-
# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters
|
| 230 |
-
# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise
|
| 231 |
-
# and answers 200, so a draft accepted without the diff would show a person a flow that is not the
|
| 232 |
-
# flow the model described, with nothing anywhere saying so.
|
| 233 |
-
#
|
| 234 |
-
# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door
|
| 235 |
-
# can be proven end to end with no API key and no spend. A gate that can only run where a
|
| 236 |
-
# credential exists is a gate that never runs.
|
| 237 |
-
|
| 238 |
-
#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it:
|
| 239 |
-
#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you
|
| 240 |
-
#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow,
|
| 241 |
-
#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`.
|
| 242 |
-
FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic")
|
| 243 |
-
MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25
|
| 244 |
-
MAX_PROMPT_CHARS = 2000
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
def flow_providers():
|
| 248 |
-
"""The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off."""
|
| 249 |
-
pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
|
| 250 |
-
by_name = {p["name"]: p for p in PROVIDERS}
|
| 251 |
-
order = [pin] if pin else list(FLOW_PROVIDER_ORDER)
|
| 252 |
-
return [by_name[n] for n in order
|
| 253 |
-
if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
def flow_schema(kinds, trigger_keys, table_keys):
|
| 257 |
-
"""The tool schema the model fills — the catalog's OWN key lists as enums.
|
| 258 |
-
|
| 259 |
-
⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the
|
| 260 |
-
transport rather than by a check afterwards. A model asked for "any action name" invents
|
| 261 |
-
`send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a
|
| 262 |
-
400 from `clean_actions` carrying a sentence about a word the person never typed.
|
| 263 |
-
⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a
|
| 264 |
-
schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a
|
| 265 |
-
transport failure (`routes_query._spec_schema` carries the same note, for the same measured
|
| 266 |
-
reason).
|
| 267 |
-
"""
|
| 268 |
-
return {
|
| 269 |
-
"type": "object",
|
| 270 |
-
"properties": {
|
| 271 |
-
"kind": {"type": "string", "enum": ["flow", "refused"],
|
| 272 |
-
"description": "refused = this cannot be built from the steps available"},
|
| 273 |
-
"refusal": {"type": "string",
|
| 274 |
-
"description": "when kind=refused: ONE plain sentence naming what is "
|
| 275 |
-
"missing, in the words a non-technical person would use"},
|
| 276 |
-
"name": {"type": "string", "description": "a short title for the automation"},
|
| 277 |
-
"trigger": {"type": "string", "enum": sorted(trigger_keys),
|
| 278 |
-
"description": "what starts this automation"},
|
| 279 |
-
"table": {"type": "string", "enum": sorted(table_keys),
|
| 280 |
-
"description": "the database whose records this flow walks, if any"},
|
| 281 |
-
"actions": {
|
| 282 |
-
"type": "array",
|
| 283 |
-
"maxItems": MAX_DRAFT_ACTIONS,
|
| 284 |
-
"items": {
|
| 285 |
-
"type": "object",
|
| 286 |
-
"properties": {
|
| 287 |
-
"kind": {"type": "string", "enum": sorted(kinds)},
|
| 288 |
-
"why": {"type": "string",
|
| 289 |
-
"description": "one short sentence: why this step is here"},
|
| 290 |
-
"config": {"type": "object", "additionalProperties": True,
|
| 291 |
-
"description": "the step's settings, using ONLY the keys named "
|
| 292 |
-
"for that kind in the system message"},
|
| 293 |
-
},
|
| 294 |
-
"required": ["kind"],
|
| 295 |
-
},
|
| 296 |
-
},
|
| 297 |
-
},
|
| 298 |
-
"required": ["kind"],
|
| 299 |
-
}
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
def flow_system_prompt(catalog, required, triggers, tables):
|
| 303 |
-
"""What the model is told it may build with — DERIVED, never written down twice.
|
| 304 |
-
|
| 305 |
-
Every list here is the server's own: the catalog rows the menu paints, the required-key table
|
| 306 |
-
the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases
|
| 307 |
-
with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to
|
| 308 |
-
the catalog is offered here on the same deploy and a kind removed stops being offered.
|
| 309 |
-
"""
|
| 310 |
-
lines = ["You build small automations for a business tool. You are given the EXACT set of "
|
| 311 |
-
"steps this tool can perform. You may use nothing else.",
|
| 312 |
-
"", "THE STEPS YOU MAY USE:"]
|
| 313 |
-
# ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the
|
| 314 |
-
# PHRASE is the only human wording of that requirement anywhere — the one the runner's own
|
| 315 |
-
# refusal sentence is built from. Keys alone leave the model guessing what `field` means on a
|
| 316 |
-
# `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing
|
| 317 |
-
# what to call the setting. Handing it one and hoping is how a draft comes back configured
|
| 318 |
-
# against a key the validator drops.
|
| 319 |
-
req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()}
|
| 320 |
-
for row in catalog or []:
|
| 321 |
-
if not row.get("ready"):
|
| 322 |
-
continue
|
| 323 |
-
kind = str(row.get("kind") or "")
|
| 324 |
-
need = req.get(kind) or []
|
| 325 |
-
lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}"
|
| 326 |
-
+ (f" REQUIRED settings: {', '.join(need)}" if need else ""))
|
| 327 |
-
lines += ["", "WHAT CAN START AN AUTOMATION:"]
|
| 328 |
-
for t in triggers or []:
|
| 329 |
-
if t.get("planned") or not t.get("ready", True):
|
| 330 |
-
continue
|
| 331 |
-
lines.append(f"- {t.get('key')}: {t.get('label')}")
|
| 332 |
-
lines += ["", "THE DATABASES THIS PERSON HAS:"]
|
| 333 |
-
for t in (tables or [])[:40]:
|
| 334 |
-
cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25])
|
| 335 |
-
lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}")
|
| 336 |
-
if not tables:
|
| 337 |
-
lines.append("- (none - do not name a database)")
|
| 338 |
-
lines += [
|
| 339 |
-
"",
|
| 340 |
-
"RULES:",
|
| 341 |
-
"1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, "
|
| 342 |
-
"answer kind=refused and say plainly which capability is missing.",
|
| 343 |
-
"2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank "
|
| 344 |
-
"rather than inventing a web address, a CSS selector or a column name.",
|
| 345 |
-
"3. A web address must start with http:// or https://.",
|
| 346 |
-
"4. To use a value from the record the flow is walking, write it as {{Column name}}.",
|
| 347 |
-
"5. Name a database only from the list above, by its key.",
|
| 348 |
-
"6. Keep it short. Fewer steps that work beat more steps that guess.",
|
| 349 |
-
]
|
| 350 |
-
return "\n".join(lines)
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
def _flow_from_tool_call(obj):
|
| 354 |
-
"""The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash."""
|
| 355 |
-
if not isinstance(obj, dict):
|
| 356 |
-
return None, "the assistant's answer could not be read"
|
| 357 |
-
if str(obj.get("kind") or "") == "refused":
|
| 358 |
-
return None, (str(obj.get("refusal") or "").strip()
|
| 359 |
-
or "the assistant could not build this from the steps available")
|
| 360 |
-
acts = obj.get("actions")
|
| 361 |
-
if not isinstance(acts, list) or not acts:
|
| 362 |
-
return None, ("the assistant did not produce any steps - try describing what should "
|
| 363 |
-
"happen, one action at a time")
|
| 364 |
-
out = []
|
| 365 |
-
for a in acts[:MAX_DRAFT_ACTIONS]:
|
| 366 |
-
if not isinstance(a, dict) or not str(a.get("kind") or "").strip():
|
| 367 |
-
continue
|
| 368 |
-
cfg = a.get("config")
|
| 369 |
-
out.append({"kind": str(a["kind"]).strip(),
|
| 370 |
-
"config": cfg if isinstance(cfg, dict) else {},
|
| 371 |
-
"why": str(a.get("why") or "").strip()[:200]})
|
| 372 |
-
if not out:
|
| 373 |
-
return None, "the assistant's steps could not be read"
|
| 374 |
-
return {"name": str(obj.get("name") or "").strip()[:80] or "New automation",
|
| 375 |
-
"trigger": str(obj.get("trigger") or "").strip(),
|
| 376 |
-
"table": str(obj.get("table") or "").strip(),
|
| 377 |
-
# ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner,
|
| 378 |
-
# 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need
|
| 379 |
-
# to explicitly tell me why and recommend a fix"* — **a silent truncation IS the
|
| 380 |
-
# violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past
|
| 381 |
-
# the twelfth and said nothing, so a model that answered with a twenty-step journey had
|
| 382 |
-
# eight steps deleted between the answer and the screen, with no key anywhere in the
|
| 383 |
-
# response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this
|
| 384 |
-
# ceiling is the draft door's own, which makes reporting it the whole obligation.
|
| 385 |
-
# ⚠ Found by the verifier that checked this ticket, not by a gate.
|
| 386 |
-
"asked": len([a for a in acts if isinstance(a, dict)]),
|
| 387 |
-
"actions": out}, ""
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None):
|
| 391 |
-
"""A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE.
|
| 392 |
-
|
| 393 |
-
Exactly one of `draft` and `refusal_sentence` is truthy — the same contract
|
| 394 |
-
`web_agent.run_step` keeps, so a caller has no third case to get wrong.
|
| 395 |
-
"""
|
| 396 |
-
text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
|
| 397 |
-
if not text:
|
| 398 |
-
return None, "type what you want the automation to do", None
|
| 399 |
-
kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")})
|
| 400 |
-
if not kinds:
|
| 401 |
-
return None, "this deployment offers no automation steps to build with", None
|
| 402 |
-
trigger_keys = sorted({str(t.get("key")) for t in (triggers or [])
|
| 403 |
-
if t.get("key") and not t.get("planned")}) or ["manual"]
|
| 404 |
-
table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""]
|
| 405 |
-
tools = [{"type": "function", "function": {
|
| 406 |
-
"name": "build_automation",
|
| 407 |
-
"description": "Emit the automation, or refuse.",
|
| 408 |
-
"parameters": flow_schema(kinds, trigger_keys, table_keys)}}]
|
| 409 |
-
messages = [{"role": "system",
|
| 410 |
-
"content": flow_system_prompt(catalog, required, triggers, tables)},
|
| 411 |
-
{"role": "user", "content": text}]
|
| 412 |
-
|
| 413 |
-
if chat is not None:
|
| 414 |
-
# ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal
|
| 415 |
-
# branches as a live call; only the transport is replaced.
|
| 416 |
-
draft, refusal = _flow_from_tool_call(chat(messages, tools))
|
| 417 |
-
return draft, refusal, "injected"
|
| 418 |
-
|
| 419 |
-
provs = flow_providers()
|
| 420 |
-
if not provs:
|
| 421 |
-
# ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
|
| 422 |
-
# never built [[flag-shipped-without-its-writer]].
|
| 423 |
-
return None, ("the assistant is not configured on this deployment, so an automation cannot "
|
| 424 |
-
"be drafted from a description yet"), None
|
| 425 |
-
tmo = float(timeout or TIMEOUT_SECONDS)
|
| 426 |
-
problems = []
|
| 427 |
-
for p in provs:
|
| 428 |
-
if p["shape"] != "openai":
|
| 429 |
-
# Anthropic's tool transport differs; it stays on the ladder for `decide()` and is
|
| 430 |
-
# SKIPPED here rather than half-supported. Named in `problems` so it is not invisible.
|
| 431 |
-
problems.append(f"{p['name']}: tool calls are not wired for this shape")
|
| 432 |
-
continue
|
| 433 |
-
try:
|
| 434 |
-
r = requests.post(p["url"], timeout=tmo,
|
| 435 |
-
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
|
| 436 |
-
"Content-Type": "application/json"},
|
| 437 |
-
json={"model": p["model"], "messages": messages, "tools": tools,
|
| 438 |
-
"tool_choice": "required", "temperature": 0.1,
|
| 439 |
-
"max_tokens": 1500})
|
| 440 |
-
except Exception as e: # noqa: BLE001
|
| 441 |
-
problems.append(f"{p['name']}: {type(e).__name__}")
|
| 442 |
-
continue
|
| 443 |
-
if r.status_code != 200:
|
| 444 |
-
problems.append(f"{p['name']}: HTTP {r.status_code}")
|
| 445 |
-
continue
|
| 446 |
-
try:
|
| 447 |
-
calls = (((r.json().get("choices") or [{}])[0].get("message") or {})
|
| 448 |
-
.get("tool_calls") or [])
|
| 449 |
-
args = json.loads(calls[0]["function"]["arguments"]) if calls else None
|
| 450 |
-
except Exception as e: # noqa: BLE001
|
| 451 |
-
problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
|
| 452 |
-
continue
|
| 453 |
-
draft, refusal = _flow_from_tool_call(args)
|
| 454 |
-
if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused":
|
| 455 |
-
# ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more
|
| 456 |
-
# expensive rung — the model has just said this cannot be built. `decide()` takes the
|
| 457 |
-
# same posture for the same reason.
|
| 458 |
-
return None, refusal, p["name"]
|
| 459 |
-
if draft is not None:
|
| 460 |
-
return draft, "", p["name"]
|
| 461 |
-
problems.append(f"{p['name']}: {refusal}")
|
| 462 |
-
return None, ("; ".join(problems)[:300] or "no provider answered"), None
|
|
|
|
| 205 |
"problem": reason or "the model did not choose one of the stages"}
|
| 206 |
return choice, {"provider": p["name"], "model": model, "reason": reason}
|
| 207 |
return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/automation_engine.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/connectors_tt.py
CHANGED
|
@@ -1,568 +1,568 @@
|
|
| 1 |
-
"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2).
|
| 2 |
-
|
| 3 |
-
Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
|
| 4 |
-
automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
|
| 5 |
-
exists at all rather than another thousand lines inside the engine.
|
| 6 |
-
|
| 7 |
-
⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43
|
| 8 |
-
post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read
|
| 9 |
-
live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
|
| 10 |
-
`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
|
| 11 |
-
close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
|
| 12 |
-
name below reads through a candidate list it is because the vendor has two names for one fact
|
| 13 |
-
(`biography`/`signature`, `region`/`country`), never because the name is uncertain.
|
| 14 |
-
|
| 15 |
-
⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned —
|
| 16 |
-
public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
|
| 17 |
-
vendor key is a key to a SUPPLIER.
|
| 18 |
-
|
| 19 |
-
⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
|
| 20 |
-
OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and
|
| 21 |
-
`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not
|
| 22 |
-
Instagram's — and re-implementing them here would be a second copy of the deferral handling, the
|
| 23 |
-
truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
|
| 24 |
-
Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
|
| 25 |
-
`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
|
| 26 |
-
⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
|
| 27 |
-
the sentence above is the kind that quietly stops being true.
|
| 28 |
-
|
| 29 |
-
⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
|
| 30 |
-
1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared
|
| 31 |
-
envelope keys (`timestamp`, `input`) that no metadata call mentions;
|
| 32 |
-
2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number`
|
| 33 |
-
and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one
|
| 34 |
-
TikTok claim that matters most (`play_count`) is exactly a declaration.
|
| 35 |
-
"""
|
| 36 |
-
from __future__ import annotations
|
| 37 |
-
|
| 38 |
-
import automation_engine as engine
|
| 39 |
-
# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
|
| 40 |
-
# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
|
| 41 |
-
# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
|
| 42 |
-
# read them off this module (the engine imports them from the transport itself), so they were four
|
| 43 |
-
# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
|
| 44 |
-
from connectors_bd import (
|
| 45 |
-
_bd_first_url,
|
| 46 |
-
_bd_flag,
|
| 47 |
-
_bd_list,
|
| 48 |
-
_bd_source_payload,
|
| 49 |
-
_first,
|
| 50 |
-
_ig_int,
|
| 51 |
-
bd_scrape,
|
| 52 |
-
)
|
| 53 |
-
|
| 54 |
-
# ---------------------------------------------------------------------------------------------
|
| 55 |
-
# THE DATASETS
|
| 56 |
-
# ---------------------------------------------------------------------------------------------
|
| 57 |
-
# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET
|
| 58 |
-
# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
|
| 59 |
-
# full field list, and the two DISCOVERY halves answer **404 for our key**:
|
| 60 |
-
# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
|
| 61 |
-
# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
|
| 62 |
-
# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
|
| 63 |
-
TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
|
| 64 |
-
TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
|
| 65 |
-
TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
|
| 66 |
-
|
| 67 |
-
#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
|
| 68 |
-
#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
|
| 69 |
-
TT_POST_TYPE_VIDEO = "video"
|
| 70 |
-
TT_POST_TYPE_CONTENT = "content"
|
| 71 |
-
|
| 72 |
-
#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
|
| 73 |
-
#: beside the dataset ids because it is the same class of vendor fact.
|
| 74 |
-
#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
|
| 75 |
-
#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own
|
| 76 |
-
#: batch calls, and the gate asserts the two agree rather than trusting that they do.
|
| 77 |
-
TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
def tt_profile_url(handle):
|
| 81 |
-
"""`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
|
| 82 |
-
h = str(handle or "").strip().lstrip("@")
|
| 83 |
-
return TT_PROFILE_URL.format(handle=h) if h else ""
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
# ---------------------------------------------------------------------------------------------
|
| 87 |
-
# THE FIELD MAPS
|
| 88 |
-
# ---------------------------------------------------------------------------------------------
|
| 89 |
-
# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
|
| 90 |
-
# in `automation_engine`. The rules they all obey, stated once:
|
| 91 |
-
#
|
| 92 |
-
# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
|
| 93 |
-
# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
|
| 94 |
-
# returns `None` (never 0) when nothing matches, which is what makes that possible.
|
| 95 |
-
# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
|
| 96 |
-
# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
|
| 97 |
-
# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
|
| 98 |
-
# vendor's side is preserved rather than silently discarded while our column model catches up.
|
| 99 |
-
# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
|
| 100 |
-
# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
|
| 101 |
-
# ride in `source_payload` where they make no claim.
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
def _tt_str(node, *names):
|
| 105 |
-
"""The first non-empty string among `names`, or None when the vendor sent nothing.
|
| 106 |
-
|
| 107 |
-
⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest —
|
| 108 |
-
an empty string written into a cell claims "we looked and it is empty".
|
| 109 |
-
"""
|
| 110 |
-
v = _first(node, *names)
|
| 111 |
-
if v is None:
|
| 112 |
-
return None
|
| 113 |
-
s = str(v).strip()
|
| 114 |
-
return s or None
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
def _tt_pct(node, *names):
|
| 118 |
-
"""A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None.
|
| 119 |
-
|
| 120 |
-
⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
|
| 121 |
-
the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` —
|
| 122 |
-
measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
|
| 123 |
-
"""
|
| 124 |
-
v = _first(node, *names)
|
| 125 |
-
if v is None:
|
| 126 |
-
return None
|
| 127 |
-
out = engine._pct100(v)
|
| 128 |
-
return out or None
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
def _tt_day(node, *names):
|
| 132 |
-
"""A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day."""
|
| 133 |
-
v = _first(node, *names)
|
| 134 |
-
if v is None:
|
| 135 |
-
return None
|
| 136 |
-
return engine._day(v) or None
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
def _drop_blanks(row):
|
| 140 |
-
"""The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above."""
|
| 141 |
-
return {k: v for k, v in row.items() if v is not None and v != ""}
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
def normalize_profile(node, handle=""):
|
| 145 |
-
"""A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
|
| 146 |
-
|
| 147 |
-
⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
|
| 148 |
-
* `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature`
|
| 149 |
-
populated on 85% of rows and they carry the same text;
|
| 150 |
-
* `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented
|
| 151 |
-
two-letter-ISO description, `country` has no description at all.
|
| 152 |
-
⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
|
| 153 |
-
`ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
|
| 154 |
-
only count of its kind the dataset offers.
|
| 155 |
-
"""
|
| 156 |
-
node = node if isinstance(node, dict) else {}
|
| 157 |
-
account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
|
| 158 |
-
return _drop_blanks({
|
| 159 |
-
"platform": engine.PLATFORM_TIKTOK,
|
| 160 |
-
"handle": account,
|
| 161 |
-
"full_name": _tt_str(node, "nickname"),
|
| 162 |
-
"tt_id": _tt_str(node, "id"),
|
| 163 |
-
"profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
|
| 164 |
-
"bio": _tt_str(node, "biography", "signature"),
|
| 165 |
-
"external_url": _bd_first_url(_first(node, "bio_link")),
|
| 166 |
-
"verified": _bd_flag(node, "is_verified"),
|
| 167 |
-
"is_private": _bd_flag(node, "is_private"),
|
| 168 |
-
# ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
|
| 169 |
-
# on their own description. It is the closest thing TikTok has to Instagram's
|
| 170 |
-
# `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so
|
| 171 |
-
# the approximation only ever fills a cell the vendor actually answered.
|
| 172 |
-
"is_business": _bd_flag(node, "is_commerce_user"),
|
| 173 |
-
"followers": _ig_int(_first(node, "followers")),
|
| 174 |
-
"following": _ig_int(_first(node, "following")),
|
| 175 |
-
"posts_count": _ig_int(_first(node, "videos_count")),
|
| 176 |
-
# ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
|
| 177 |
-
# nulls). It is not a post-level number and it is not our `likes` column, which is why it
|
| 178 |
-
# is stored under a different name.
|
| 179 |
-
"likes_received": _ig_int(_first(node, "likes")),
|
| 180 |
-
"avg_engagement": _tt_pct(node, "awg_engagement_rate"),
|
| 181 |
-
"like_engagement": _tt_pct(node, "like_engagement_rate"),
|
| 182 |
-
"comment_engagement": _tt_pct(node, "comment_engagement_rate"),
|
| 183 |
-
"country_code": _tt_str(node, "region", "country"),
|
| 184 |
-
"region": _tt_str(node, "region"),
|
| 185 |
-
"predicted_lang": _tt_str(node, "predicted_lang"),
|
| 186 |
-
# ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT
|
| 187 |
-
# was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
|
| 188 |
-
# which is why the append law dates a snapshot by when WE read it.
|
| 189 |
-
"account_created_at": _tt_day(node, "create_time"),
|
| 190 |
-
"source_payload": _bd_source_payload(node),
|
| 191 |
-
})
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
def tt_post_type(node):
|
| 195 |
-
"""A TikTok post row → one of OUR three type options, or None.
|
| 196 |
-
|
| 197 |
-
⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
|
| 198 |
-
`"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
|
| 199 |
-
vendor's token would fail `_clean_field`'s option check on the way in and would put an
|
| 200 |
-
untranslated API word in front of a user on the way out.
|
| 201 |
-
|
| 202 |
-
So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the
|
| 203 |
-
row carries more than one `carousel_images` entry, which is what a carousel IS on either
|
| 204 |
-
network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
|
| 205 |
-
token cannot express it, so inferring `carousel` from the word would be inventing a fact.
|
| 206 |
-
⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
|
| 207 |
-
that is exactly what would make the wrong default invisible).
|
| 208 |
-
"""
|
| 209 |
-
raw = str((node or {}).get("post_type") or "").strip().lower()
|
| 210 |
-
if raw == TT_POST_TYPE_VIDEO:
|
| 211 |
-
return "video"
|
| 212 |
-
if raw != TT_POST_TYPE_CONTENT:
|
| 213 |
-
return None
|
| 214 |
-
images = (node or {}).get("carousel_images")
|
| 215 |
-
return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
def normalize_post(node):
|
| 219 |
-
"""A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity.
|
| 220 |
-
|
| 221 |
-
⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
|
| 222 |
-
Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
|
| 223 |
-
fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and
|
| 224 |
-
`ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
|
| 225 |
-
manufacture a second measurement that a rollup could average or double-count, which is a worse
|
| 226 |
-
outcome than the missing column it would paper over.
|
| 227 |
-
|
| 228 |
-
⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
|
| 229 |
-
the probe measured them as the same shape. That equality is what lets the comments→posts link
|
| 230 |
-
join with NO normaliser, which the Instagram side never had.
|
| 231 |
-
⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
|
| 232 |
-
⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and
|
| 233 |
-
countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
|
| 234 |
-
no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
|
| 235 |
-
"""
|
| 236 |
-
node = node if isinstance(node, dict) else {}
|
| 237 |
-
shortcode = _tt_str(node, "shortcode", "post_id")
|
| 238 |
-
if not shortcode:
|
| 239 |
-
return None
|
| 240 |
-
return _drop_blanks({
|
| 241 |
-
"platform": engine.PLATFORM_TIKTOK,
|
| 242 |
-
"shortcode": shortcode,
|
| 243 |
-
# ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12.
|
| 244 |
-
# The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
|
| 245 |
-
# @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so
|
| 246 |
-
# one name means one thing across both corpora. Reading the display name silently broke the
|
| 247 |
-
# only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
|
| 248 |
-
# NOTHING, so a person could not filter posts by creator and a rollup would count zero.
|
| 249 |
-
# ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
|
| 250 |
-
# different fact, and filling a join key with it is worse than leaving it blank — a blank
|
| 251 |
-
# is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
|
| 252 |
-
# The URL is the honest second source: it carries the handle by construction.
|
| 253 |
-
"influencer_key": (_tt_str(node, "account_id")
|
| 254 |
-
or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
|
| 255 |
-
"posted_at": _tt_day(node, "create_time"),
|
| 256 |
-
"type": tt_post_type(node),
|
| 257 |
-
"caption": _tt_str(node, "description"),
|
| 258 |
-
"url": _tt_str(node, "url"),
|
| 259 |
-
"hashtags": _bd_list(node, "hashtags"),
|
| 260 |
-
"tagged_location": _tt_str(node, "commerce_info"),
|
| 261 |
-
"views": _ig_int(_first(node, "play_count")),
|
| 262 |
-
"likes": _ig_int(_first(node, "digg_count")),
|
| 263 |
-
"comments": _ig_int(_first(node, "comment_count")),
|
| 264 |
-
"shares": _ig_int(_first(node, "num_share_count")),
|
| 265 |
-
"saves": _ig_int(_first(node, "collect_count")),
|
| 266 |
-
"video_duration": _ig_int(_first(node, "video_duration")),
|
| 267 |
-
"source_payload": _bd_source_payload(node),
|
| 268 |
-
})
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
def normalize_comment(node):
|
| 272 |
-
"""A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id.
|
| 273 |
-
|
| 274 |
-
⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
|
| 275 |
-
`replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
|
| 276 |
-
`source_payload`. Reading the array's length instead would be a second, disagreeing answer to
|
| 277 |
-
a question the vendor already answers — and it would disagree, because a page of replies is not
|
| 278 |
-
all of them.
|
| 279 |
-
⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
|
| 280 |
-
text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor
|
| 281 |
-
that changes its mind cannot change ours.
|
| 282 |
-
⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
|
| 283 |
-
PII) stay in `source_payload` and are promoted to no column, which is the same posture the
|
| 284 |
-
Instagram comment schema takes for the same D-24 reason.
|
| 285 |
-
"""
|
| 286 |
-
node = node if isinstance(node, dict) else {}
|
| 287 |
-
comment_key = _tt_str(node, "comment_id")
|
| 288 |
-
if not comment_key:
|
| 289 |
-
return None
|
| 290 |
-
return _drop_blanks({
|
| 291 |
-
"platform": engine.PLATFORM_TIKTOK,
|
| 292 |
-
"comment_key": comment_key,
|
| 293 |
-
"shortcode": _tt_str(node, "post_id"),
|
| 294 |
-
# ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
|
| 295 |
-
# vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
|
| 296 |
-
# primary first, so a row carrying the rich form is not silently served the plain one.
|
| 297 |
-
# ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`.
|
| 298 |
-
"text": _tt_str(node, "comment_text", "comment_text_only"),
|
| 299 |
-
"commented_at": _tt_day(node, "date_created"),
|
| 300 |
-
"likes": _ig_int(_first(node, "num_likes")),
|
| 301 |
-
"replies": _ig_int(_first(node, "num_replies")),
|
| 302 |
-
"source_payload": _bd_source_payload(node),
|
| 303 |
-
})
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them
|
| 307 |
-
#: rather than naming three functions, and so adding a fourth dataset is one entry.
|
| 308 |
-
TT_NORMALIZERS = {
|
| 309 |
-
"tt_profile": normalize_profile,
|
| 310 |
-
"tt_post_metrics": normalize_post,
|
| 311 |
-
"tt_comments": normalize_comment,
|
| 312 |
-
}
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
# ---------------------------------------------------------------------------------------------
|
| 316 |
-
# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05)
|
| 317 |
-
# ---------------------------------------------------------------------------------------------
|
| 318 |
-
|
| 319 |
-
def tt_handle(url):
|
| 320 |
-
"""A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither.
|
| 321 |
-
|
| 322 |
-
Deliberately permissive about the input and strict about the output, because the two callers
|
| 323 |
-
hand it different things: an automation stores whatever a person typed in the profile column
|
| 324 |
-
(`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
|
| 325 |
-
`account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
|
| 326 |
-
resolve to two different handles.
|
| 327 |
-
"""
|
| 328 |
-
s = str(url or "").strip()
|
| 329 |
-
if not s:
|
| 330 |
-
return ""
|
| 331 |
-
if "tiktok.com" in s.lower():
|
| 332 |
-
# Everything after the first `@`, up to the next path segment or query.
|
| 333 |
-
tail = s.split("@", 1)[1] if "@" in s else ""
|
| 334 |
-
s = tail.split("/")[0].split("?")[0].split("#")[0]
|
| 335 |
-
s = s.strip().lstrip("@").strip()
|
| 336 |
-
# A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
|
| 337 |
-
return s if s and all(c.isalnum() or c in "._" for c in s) else ""
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
def tt_post_urls(node, limit=0):
|
| 341 |
-
"""⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
|
| 342 |
-
|
| 343 |
-
⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
|
| 344 |
-
we have already bought, so the posts read is a scrape of links we hold, never a search for them.
|
| 345 |
-
The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
|
| 346 |
-
so a design that reached for either would not merely be dearer, it would not work.
|
| 347 |
-
|
| 348 |
-
⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
|
| 349 |
-
array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
|
| 350 |
-
read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's
|
| 351 |
-
`ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
|
| 352 |
-
(measured below), so the reader built against the quoted sentence found nothing, forever, in
|
| 353 |
-
silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
|
| 354 |
-
of X"* are different claims, and prose cannot be told apart by a reader downstream — which is
|
| 355 |
-
why the correction names the method, not just the value.
|
| 356 |
-
|
| 357 |
-
⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
|
| 358 |
-
and preferring whichever happened to be longer is how one creator's window silently differs
|
| 359 |
-
from another's.
|
| 360 |
-
⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`,
|
| 361 |
-
validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is
|
| 362 |
-
never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
|
| 363 |
-
not a number this function may invent.
|
| 364 |
-
"""
|
| 365 |
-
raw = (node or {}).get("top_videos")
|
| 366 |
-
out = []
|
| 367 |
-
for item in raw if isinstance(raw, list) else []:
|
| 368 |
-
# ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
|
| 369 |
-
# `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
|
| 370 |
-
# live handle returns **19 DICTS**, keyed
|
| 371 |
-
# `video_url · video_id · playcount · diggcount · commentcount · share_count ·
|
| 372 |
-
# favorites_count · create_date · cover_image`.
|
| 373 |
-
# The docstring above cites the $0 probe as having "measured" permalinks — it had not, and
|
| 374 |
-
# could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
|
| 375 |
-
# so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
|
| 376 |
-
# POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
|
| 377 |
-
# declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
|
| 378 |
-
# the first. [[reachable-is-not-the-same-as-built]]
|
| 379 |
-
# ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
|
| 380 |
-
# list on every real profile — post capture could not have worked for anybody, and the
|
| 381 |
-
# T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
|
| 382 |
-
# written from a schema tests the schema.
|
| 383 |
-
# ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
|
| 384 |
-
# because it costs nothing and is what a future corpus revision would most likely use. The
|
| 385 |
-
# bare-string branch stays for the same reason — this widens what is ACCEPTED and invents
|
| 386 |
-
# nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
|
| 387 |
-
# before, rather than to a URL built out of a guess.
|
| 388 |
-
# ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
|
| 389 |
-
# whichever array happened to be longer is how one creator's window silently differs from
|
| 390 |
-
# another's, and that reasoning is unchanged by this correction.
|
| 391 |
-
if isinstance(item, dict):
|
| 392 |
-
u = str(item.get("video_url") or item.get("url") or "").strip()
|
| 393 |
-
else:
|
| 394 |
-
u = str(item or "").strip()
|
| 395 |
-
if u.lower().startswith("http") and u not in out:
|
| 396 |
-
out.append(u)
|
| 397 |
-
return out[:limit] if limit and limit > 0 else out
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
def pull_posts_tt(post_urls, log=print, deferred=None):
|
| 401 |
-
"""The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised.
|
| 402 |
-
|
| 403 |
-
⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
|
| 404 |
-
measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
|
| 405 |
-
running at 67 minutes. Nothing here loops per URL.
|
| 406 |
-
"""
|
| 407 |
-
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 408 |
-
if not urls:
|
| 409 |
-
return [], ""
|
| 410 |
-
rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
|
| 411 |
-
if note:
|
| 412 |
-
log(f"[aios-tt] posts: {note}")
|
| 413 |
-
return [], note
|
| 414 |
-
out = [r for r in (normalize_post(n) for n in rows) if r]
|
| 415 |
-
return out, ""
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
def pull_comments_tt(post_urls, log=print, deferred=None):
|
| 419 |
-
"""The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised.
|
| 420 |
-
|
| 421 |
-
⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
|
| 422 |
-
both networks: a comments scrape ingests identifiable third parties who never entered anybody's
|
| 423 |
-
list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
|
| 424 |
-
none of them to a column; this function adds no new exposure, it just has to be asked for.
|
| 425 |
-
"""
|
| 426 |
-
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 427 |
-
if not urls:
|
| 428 |
-
return [], ""
|
| 429 |
-
rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
|
| 430 |
-
if note:
|
| 431 |
-
log(f"[aios-tt] comments: {note}")
|
| 432 |
-
return [], note
|
| 433 |
-
out = [r for r in (normalize_comment(n) for n in rows) if r]
|
| 434 |
-
return out, ""
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
|
| 438 |
-
#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
|
| 439 |
-
#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
|
| 440 |
-
#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
|
| 441 |
-
#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
|
| 442 |
-
TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
def _media_deferrals(deferred):
|
| 446 |
-
"""The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's.
|
| 447 |
-
|
| 448 |
-
⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
|
| 449 |
-
handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
|
| 450 |
-
kind here, so the id already carries everything a mapper choice depends on — and importing
|
| 451 |
-
Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
|
| 452 |
-
"""
|
| 453 |
-
out = []
|
| 454 |
-
for d in deferred or []:
|
| 455 |
-
if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
|
| 456 |
-
out.append(dict(d))
|
| 457 |
-
return out
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
| 461 |
-
max_posts=0, post_metrics=False, comment_metrics=False):
|
| 462 |
-
"""ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
|
| 463 |
-
|
| 464 |
-
`{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
|
| 465 |
-
so the engine's enrich branch treats every network identically and no caller learns a new
|
| 466 |
-
shape.
|
| 467 |
-
|
| 468 |
-
⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
|
| 469 |
-
do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` —
|
| 470 |
-
no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
|
| 471 |
-
then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
|
| 472 |
-
rather than by a rule: their input IS a post permalink, so asking for comments with post capture
|
| 473 |
-
off is a request with no subject, and it returns none instead of quietly buying posts nobody
|
| 474 |
-
asked for.
|
| 475 |
-
|
| 476 |
-
⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
|
| 477 |
-
pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
|
| 478 |
-
(`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
|
| 479 |
-
count and no posts must not paint green over a posts table that did not grow"*). So: posts not
|
| 480 |
-
ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came →
|
| 481 |
-
`partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
|
| 482 |
-
never "did the function finish".
|
| 483 |
-
|
| 484 |
-
⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
|
| 485 |
-
paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
|
| 486 |
-
with its own note explaining that a multi-provider chain is a promise something walks it and
|
| 487 |
-
that nothing walks Instagram's second name today either. So a refusal here is final, and it
|
| 488 |
-
says so instead of implying a retry somewhere.
|
| 489 |
-
"""
|
| 490 |
-
handle = tt_handle(url)
|
| 491 |
-
if not handle:
|
| 492 |
-
return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
|
| 493 |
-
"note": f"{url!r} is not a TikTok profile URL or handle"}
|
| 494 |
-
|
| 495 |
-
# ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
|
| 496 |
-
# one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
|
| 497 |
-
# happen; a miss falls through to the single-URL call below.
|
| 498 |
-
cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
|
| 499 |
-
_deferred = []
|
| 500 |
-
if isinstance(cached, dict) and cached:
|
| 501 |
-
rows, note = [cached], ""
|
| 502 |
-
else:
|
| 503 |
-
rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
|
| 504 |
-
|
| 505 |
-
node = rows[0] if rows else {}
|
| 506 |
-
profile = normalize_profile(node, handle) if node else {}
|
| 507 |
-
# ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
|
| 508 |
-
# drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and
|
| 509 |
-
# carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
|
| 510 |
-
# this reason, and answering "0 followers" instead is the failure it exists to prevent.
|
| 511 |
-
unreadable = profile.get("followers") is None and profile.get("following") is None
|
| 512 |
-
if note or unreadable:
|
| 513 |
-
# ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
|
| 514 |
-
# building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
|
| 515 |
-
# same record. That was live on the Instagram profile path until 2026-08-09 — measured on
|
| 516 |
-
# nurilab as two runs, two fresh snapshots, both abandoned — and it is not being
|
| 517 |
-
# reintroduced here by omission.
|
| 518 |
-
if isinstance(pending_profile, list):
|
| 519 |
-
for d in _deferred:
|
| 520 |
-
pending_profile.append({**d, "kind": "profile", "influencer": handle})
|
| 521 |
-
why = note or ("the scrape answered, but no follower/following counts were readable in it "
|
| 522 |
-
"(the field names may have moved - see tiktok-capture.md)")
|
| 523 |
-
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 524 |
-
"deferredProfile": [d.get("snapshotId") for d in _deferred],
|
| 525 |
-
"note": why}
|
| 526 |
-
|
| 527 |
-
# --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
|
| 528 |
-
if not post_metrics:
|
| 529 |
-
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 530 |
-
"via": "brightdata",
|
| 531 |
-
"note": note or "profile read; post capture is off for this step"}
|
| 532 |
-
urls = tt_post_urls(node, limit=max_posts)
|
| 533 |
-
if not urls:
|
| 534 |
-
# ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
|
| 535 |
-
# saying so is what stops the next run paying to be told the same thing.
|
| 536 |
-
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 537 |
-
"via": "brightdata",
|
| 538 |
-
"note": note or "profile read; this account's row carried no post links"}
|
| 539 |
-
posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
|
| 540 |
-
comments, c_note = ([], "")
|
| 541 |
-
if comment_metrics and posts:
|
| 542 |
-
# The comments dataset is keyed on a POST permalink, so it reads the posts we just bought —
|
| 543 |
-
# `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
|
| 544 |
-
# is not silently asked about again one rung later.
|
| 545 |
-
comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
|
| 546 |
-
log=log, deferred=_deferred)
|
| 547 |
-
# ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
|
| 548 |
-
# the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
|
| 549 |
-
# `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
|
| 550 |
-
# deferral is impossible — the profile branch above returns `blocked` on any note — so **every
|
| 551 |
-
# id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
|
| 552 |
-
# collector writes preset profile cells onto somebody's record from post rows. The engine keeps
|
| 553 |
-
# the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
|
| 554 |
-
# ⇒ So this returns them under their OWN key, filtered by dataset identity
|
| 555 |
-
# (`_media_deferrals`), and the engine files them in the metric queue with the handle it
|
| 556 |
-
# already holds. Returning rather than appending also keeps the queue's vocabulary out of a
|
| 557 |
-
# connector: this module knows which CORPUS deferred, never what the engine calls it.
|
| 558 |
-
# ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
|
| 559 |
-
# matters most — that is exactly the run where the vendor took too long, so a caller reading
|
| 560 |
-
# the ids only from the success path would lose every batch it actually paid for.
|
| 561 |
-
deferred_media = _media_deferrals(_deferred)
|
| 562 |
-
if not posts:
|
| 563 |
-
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 564 |
-
"via": "brightdata", "deferredMedia": deferred_media,
|
| 565 |
-
"note": p_note or note or "profile read; the post source returned nothing"}
|
| 566 |
-
return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
|
| 567 |
-
"via": "brightdata", "deferredMedia": deferred_media,
|
| 568 |
-
"note": c_note or note or ""}
|
|
|
|
| 1 |
+
"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2).
|
| 2 |
+
|
| 3 |
+
Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
|
| 4 |
+
automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
|
| 5 |
+
exists at all rather than another thousand lines inside the engine.
|
| 6 |
+
|
| 7 |
+
⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43
|
| 8 |
+
post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read
|
| 9 |
+
live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
|
| 10 |
+
`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
|
| 11 |
+
close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
|
| 12 |
+
name below reads through a candidate list it is because the vendor has two names for one fact
|
| 13 |
+
(`biography`/`signature`, `region`/`country`), never because the name is uncertain.
|
| 14 |
+
|
| 15 |
+
⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned —
|
| 16 |
+
public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
|
| 17 |
+
vendor key is a key to a SUPPLIER.
|
| 18 |
+
|
| 19 |
+
⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
|
| 20 |
+
OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and
|
| 21 |
+
`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not
|
| 22 |
+
Instagram's — and re-implementing them here would be a second copy of the deferral handling, the
|
| 23 |
+
truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
|
| 24 |
+
Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
|
| 25 |
+
`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
|
| 26 |
+
⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
|
| 27 |
+
the sentence above is the kind that quietly stops being true.
|
| 28 |
+
|
| 29 |
+
⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
|
| 30 |
+
1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared
|
| 31 |
+
envelope keys (`timestamp`, `input`) that no metadata call mentions;
|
| 32 |
+
2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number`
|
| 33 |
+
and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one
|
| 34 |
+
TikTok claim that matters most (`play_count`) is exactly a declaration.
|
| 35 |
+
"""
|
| 36 |
+
from __future__ import annotations
|
| 37 |
+
|
| 38 |
+
import automation_engine as engine
|
| 39 |
+
# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
|
| 40 |
+
# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
|
| 41 |
+
# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
|
| 42 |
+
# read them off this module (the engine imports them from the transport itself), so they were four
|
| 43 |
+
# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
|
| 44 |
+
from connectors_bd import (
|
| 45 |
+
_bd_first_url,
|
| 46 |
+
_bd_flag,
|
| 47 |
+
_bd_list,
|
| 48 |
+
_bd_source_payload,
|
| 49 |
+
_first,
|
| 50 |
+
_ig_int,
|
| 51 |
+
bd_scrape,
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
# ---------------------------------------------------------------------------------------------
|
| 55 |
+
# THE DATASETS
|
| 56 |
+
# ---------------------------------------------------------------------------------------------
|
| 57 |
+
# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET
|
| 58 |
+
# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
|
| 59 |
+
# full field list, and the two DISCOVERY halves answer **404 for our key**:
|
| 60 |
+
# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
|
| 61 |
+
# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
|
| 62 |
+
# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
|
| 63 |
+
TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
|
| 64 |
+
TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
|
| 65 |
+
TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
|
| 66 |
+
|
| 67 |
+
#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
|
| 68 |
+
#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
|
| 69 |
+
TT_POST_TYPE_VIDEO = "video"
|
| 70 |
+
TT_POST_TYPE_CONTENT = "content"
|
| 71 |
+
|
| 72 |
+
#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
|
| 73 |
+
#: beside the dataset ids because it is the same class of vendor fact.
|
| 74 |
+
#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
|
| 75 |
+
#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own
|
| 76 |
+
#: batch calls, and the gate asserts the two agree rather than trusting that they do.
|
| 77 |
+
TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def tt_profile_url(handle):
|
| 81 |
+
"""`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
|
| 82 |
+
h = str(handle or "").strip().lstrip("@")
|
| 83 |
+
return TT_PROFILE_URL.format(handle=h) if h else ""
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# ---------------------------------------------------------------------------------------------
|
| 87 |
+
# THE FIELD MAPS
|
| 88 |
+
# ---------------------------------------------------------------------------------------------
|
| 89 |
+
# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
|
| 90 |
+
# in `automation_engine`. The rules they all obey, stated once:
|
| 91 |
+
#
|
| 92 |
+
# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
|
| 93 |
+
# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
|
| 94 |
+
# returns `None` (never 0) when nothing matches, which is what makes that possible.
|
| 95 |
+
# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
|
| 96 |
+
# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
|
| 97 |
+
# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
|
| 98 |
+
# vendor's side is preserved rather than silently discarded while our column model catches up.
|
| 99 |
+
# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
|
| 100 |
+
# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
|
| 101 |
+
# ride in `source_payload` where they make no claim.
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _tt_str(node, *names):
|
| 105 |
+
"""The first non-empty string among `names`, or None when the vendor sent nothing.
|
| 106 |
+
|
| 107 |
+
⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest —
|
| 108 |
+
an empty string written into a cell claims "we looked and it is empty".
|
| 109 |
+
"""
|
| 110 |
+
v = _first(node, *names)
|
| 111 |
+
if v is None:
|
| 112 |
+
return None
|
| 113 |
+
s = str(v).strip()
|
| 114 |
+
return s or None
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _tt_pct(node, *names):
|
| 118 |
+
"""A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None.
|
| 119 |
+
|
| 120 |
+
⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
|
| 121 |
+
the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` —
|
| 122 |
+
measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
|
| 123 |
+
"""
|
| 124 |
+
v = _first(node, *names)
|
| 125 |
+
if v is None:
|
| 126 |
+
return None
|
| 127 |
+
out = engine._pct100(v)
|
| 128 |
+
return out or None
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _tt_day(node, *names):
|
| 132 |
+
"""A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day."""
|
| 133 |
+
v = _first(node, *names)
|
| 134 |
+
if v is None:
|
| 135 |
+
return None
|
| 136 |
+
return engine._day(v) or None
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _drop_blanks(row):
|
| 140 |
+
"""The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above."""
|
| 141 |
+
return {k: v for k, v in row.items() if v is not None and v != ""}
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def normalize_profile(node, handle=""):
|
| 145 |
+
"""A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
|
| 146 |
+
|
| 147 |
+
⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
|
| 148 |
+
* `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature`
|
| 149 |
+
populated on 85% of rows and they carry the same text;
|
| 150 |
+
* `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented
|
| 151 |
+
two-letter-ISO description, `country` has no description at all.
|
| 152 |
+
⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
|
| 153 |
+
`ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
|
| 154 |
+
only count of its kind the dataset offers.
|
| 155 |
+
"""
|
| 156 |
+
node = node if isinstance(node, dict) else {}
|
| 157 |
+
account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
|
| 158 |
+
return _drop_blanks({
|
| 159 |
+
"platform": engine.PLATFORM_TIKTOK,
|
| 160 |
+
"handle": account,
|
| 161 |
+
"full_name": _tt_str(node, "nickname"),
|
| 162 |
+
"tt_id": _tt_str(node, "id"),
|
| 163 |
+
"profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
|
| 164 |
+
"bio": _tt_str(node, "biography", "signature"),
|
| 165 |
+
"external_url": _bd_first_url(_first(node, "bio_link")),
|
| 166 |
+
"verified": _bd_flag(node, "is_verified"),
|
| 167 |
+
"is_private": _bd_flag(node, "is_private"),
|
| 168 |
+
# ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
|
| 169 |
+
# on their own description. It is the closest thing TikTok has to Instagram's
|
| 170 |
+
# `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so
|
| 171 |
+
# the approximation only ever fills a cell the vendor actually answered.
|
| 172 |
+
"is_business": _bd_flag(node, "is_commerce_user"),
|
| 173 |
+
"followers": _ig_int(_first(node, "followers")),
|
| 174 |
+
"following": _ig_int(_first(node, "following")),
|
| 175 |
+
"posts_count": _ig_int(_first(node, "videos_count")),
|
| 176 |
+
# ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
|
| 177 |
+
# nulls). It is not a post-level number and it is not our `likes` column, which is why it
|
| 178 |
+
# is stored under a different name.
|
| 179 |
+
"likes_received": _ig_int(_first(node, "likes")),
|
| 180 |
+
"avg_engagement": _tt_pct(node, "awg_engagement_rate"),
|
| 181 |
+
"like_engagement": _tt_pct(node, "like_engagement_rate"),
|
| 182 |
+
"comment_engagement": _tt_pct(node, "comment_engagement_rate"),
|
| 183 |
+
"country_code": _tt_str(node, "region", "country"),
|
| 184 |
+
"region": _tt_str(node, "region"),
|
| 185 |
+
"predicted_lang": _tt_str(node, "predicted_lang"),
|
| 186 |
+
# ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT
|
| 187 |
+
# was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
|
| 188 |
+
# which is why the append law dates a snapshot by when WE read it.
|
| 189 |
+
"account_created_at": _tt_day(node, "create_time"),
|
| 190 |
+
"source_payload": _bd_source_payload(node),
|
| 191 |
+
})
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def tt_post_type(node):
|
| 195 |
+
"""A TikTok post row → one of OUR three type options, or None.
|
| 196 |
+
|
| 197 |
+
⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
|
| 198 |
+
`"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
|
| 199 |
+
vendor's token would fail `_clean_field`'s option check on the way in and would put an
|
| 200 |
+
untranslated API word in front of a user on the way out.
|
| 201 |
+
|
| 202 |
+
So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the
|
| 203 |
+
row carries more than one `carousel_images` entry, which is what a carousel IS on either
|
| 204 |
+
network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
|
| 205 |
+
token cannot express it, so inferring `carousel` from the word would be inventing a fact.
|
| 206 |
+
⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
|
| 207 |
+
that is exactly what would make the wrong default invisible).
|
| 208 |
+
"""
|
| 209 |
+
raw = str((node or {}).get("post_type") or "").strip().lower()
|
| 210 |
+
if raw == TT_POST_TYPE_VIDEO:
|
| 211 |
+
return "video"
|
| 212 |
+
if raw != TT_POST_TYPE_CONTENT:
|
| 213 |
+
return None
|
| 214 |
+
images = (node or {}).get("carousel_images")
|
| 215 |
+
return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def normalize_post(node):
|
| 219 |
+
"""A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity.
|
| 220 |
+
|
| 221 |
+
⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
|
| 222 |
+
Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
|
| 223 |
+
fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and
|
| 224 |
+
`ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
|
| 225 |
+
manufacture a second measurement that a rollup could average or double-count, which is a worse
|
| 226 |
+
outcome than the missing column it would paper over.
|
| 227 |
+
|
| 228 |
+
⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
|
| 229 |
+
the probe measured them as the same shape. That equality is what lets the comments→posts link
|
| 230 |
+
join with NO normaliser, which the Instagram side never had.
|
| 231 |
+
⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
|
| 232 |
+
⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and
|
| 233 |
+
countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
|
| 234 |
+
no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
|
| 235 |
+
"""
|
| 236 |
+
node = node if isinstance(node, dict) else {}
|
| 237 |
+
shortcode = _tt_str(node, "shortcode", "post_id")
|
| 238 |
+
if not shortcode:
|
| 239 |
+
return None
|
| 240 |
+
return _drop_blanks({
|
| 241 |
+
"platform": engine.PLATFORM_TIKTOK,
|
| 242 |
+
"shortcode": shortcode,
|
| 243 |
+
# ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12.
|
| 244 |
+
# The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
|
| 245 |
+
# @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so
|
| 246 |
+
# one name means one thing across both corpora. Reading the display name silently broke the
|
| 247 |
+
# only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
|
| 248 |
+
# NOTHING, so a person could not filter posts by creator and a rollup would count zero.
|
| 249 |
+
# ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
|
| 250 |
+
# different fact, and filling a join key with it is worse than leaving it blank — a blank
|
| 251 |
+
# is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
|
| 252 |
+
# The URL is the honest second source: it carries the handle by construction.
|
| 253 |
+
"influencer_key": (_tt_str(node, "account_id")
|
| 254 |
+
or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
|
| 255 |
+
"posted_at": _tt_day(node, "create_time"),
|
| 256 |
+
"type": tt_post_type(node),
|
| 257 |
+
"caption": _tt_str(node, "description"),
|
| 258 |
+
"url": _tt_str(node, "url"),
|
| 259 |
+
"hashtags": _bd_list(node, "hashtags"),
|
| 260 |
+
"tagged_location": _tt_str(node, "commerce_info"),
|
| 261 |
+
"views": _ig_int(_first(node, "play_count")),
|
| 262 |
+
"likes": _ig_int(_first(node, "digg_count")),
|
| 263 |
+
"comments": _ig_int(_first(node, "comment_count")),
|
| 264 |
+
"shares": _ig_int(_first(node, "num_share_count")),
|
| 265 |
+
"saves": _ig_int(_first(node, "collect_count")),
|
| 266 |
+
"video_duration": _ig_int(_first(node, "video_duration")),
|
| 267 |
+
"source_payload": _bd_source_payload(node),
|
| 268 |
+
})
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def normalize_comment(node):
|
| 272 |
+
"""A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id.
|
| 273 |
+
|
| 274 |
+
⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
|
| 275 |
+
`replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
|
| 276 |
+
`source_payload`. Reading the array's length instead would be a second, disagreeing answer to
|
| 277 |
+
a question the vendor already answers — and it would disagree, because a page of replies is not
|
| 278 |
+
all of them.
|
| 279 |
+
⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
|
| 280 |
+
text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor
|
| 281 |
+
that changes its mind cannot change ours.
|
| 282 |
+
⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
|
| 283 |
+
PII) stay in `source_payload` and are promoted to no column, which is the same posture the
|
| 284 |
+
Instagram comment schema takes for the same D-24 reason.
|
| 285 |
+
"""
|
| 286 |
+
node = node if isinstance(node, dict) else {}
|
| 287 |
+
comment_key = _tt_str(node, "comment_id")
|
| 288 |
+
if not comment_key:
|
| 289 |
+
return None
|
| 290 |
+
return _drop_blanks({
|
| 291 |
+
"platform": engine.PLATFORM_TIKTOK,
|
| 292 |
+
"comment_key": comment_key,
|
| 293 |
+
"shortcode": _tt_str(node, "post_id"),
|
| 294 |
+
# ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
|
| 295 |
+
# vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
|
| 296 |
+
# primary first, so a row carrying the rich form is not silently served the plain one.
|
| 297 |
+
# ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`.
|
| 298 |
+
"text": _tt_str(node, "comment_text", "comment_text_only"),
|
| 299 |
+
"commented_at": _tt_day(node, "date_created"),
|
| 300 |
+
"likes": _ig_int(_first(node, "num_likes")),
|
| 301 |
+
"replies": _ig_int(_first(node, "num_replies")),
|
| 302 |
+
"source_payload": _bd_source_payload(node),
|
| 303 |
+
})
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them
|
| 307 |
+
#: rather than naming three functions, and so adding a fourth dataset is one entry.
|
| 308 |
+
TT_NORMALIZERS = {
|
| 309 |
+
"tt_profile": normalize_profile,
|
| 310 |
+
"tt_post_metrics": normalize_post,
|
| 311 |
+
"tt_comments": normalize_comment,
|
| 312 |
+
}
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
# ---------------------------------------------------------------------------------------------
|
| 316 |
+
# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05)
|
| 317 |
+
# ---------------------------------------------------------------------------------------------
|
| 318 |
+
|
| 319 |
+
def tt_handle(url):
|
| 320 |
+
"""A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither.
|
| 321 |
+
|
| 322 |
+
Deliberately permissive about the input and strict about the output, because the two callers
|
| 323 |
+
hand it different things: an automation stores whatever a person typed in the profile column
|
| 324 |
+
(`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
|
| 325 |
+
`account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
|
| 326 |
+
resolve to two different handles.
|
| 327 |
+
"""
|
| 328 |
+
s = str(url or "").strip()
|
| 329 |
+
if not s:
|
| 330 |
+
return ""
|
| 331 |
+
if "tiktok.com" in s.lower():
|
| 332 |
+
# Everything after the first `@`, up to the next path segment or query.
|
| 333 |
+
tail = s.split("@", 1)[1] if "@" in s else ""
|
| 334 |
+
s = tail.split("/")[0].split("?")[0].split("#")[0]
|
| 335 |
+
s = s.strip().lstrip("@").strip()
|
| 336 |
+
# A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
|
| 337 |
+
return s if s and all(c.isalnum() or c in "._" for c in s) else ""
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def tt_post_urls(node, limit=0):
|
| 341 |
+
"""⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
|
| 342 |
+
|
| 343 |
+
⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
|
| 344 |
+
we have already bought, so the posts read is a scrape of links we hold, never a search for them.
|
| 345 |
+
The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
|
| 346 |
+
so a design that reached for either would not merely be dearer, it would not work.
|
| 347 |
+
|
| 348 |
+
⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
|
| 349 |
+
array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
|
| 350 |
+
read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's
|
| 351 |
+
`ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
|
| 352 |
+
(measured below), so the reader built against the quoted sentence found nothing, forever, in
|
| 353 |
+
silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
|
| 354 |
+
of X"* are different claims, and prose cannot be told apart by a reader downstream — which is
|
| 355 |
+
why the correction names the method, not just the value.
|
| 356 |
+
|
| 357 |
+
⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
|
| 358 |
+
and preferring whichever happened to be longer is how one creator's window silently differs
|
| 359 |
+
from another's.
|
| 360 |
+
⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`,
|
| 361 |
+
validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is
|
| 362 |
+
never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
|
| 363 |
+
not a number this function may invent.
|
| 364 |
+
"""
|
| 365 |
+
raw = (node or {}).get("top_videos")
|
| 366 |
+
out = []
|
| 367 |
+
for item in raw if isinstance(raw, list) else []:
|
| 368 |
+
# ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
|
| 369 |
+
# `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
|
| 370 |
+
# live handle returns **19 DICTS**, keyed
|
| 371 |
+
# `video_url · video_id · playcount · diggcount · commentcount · share_count ·
|
| 372 |
+
# favorites_count · create_date · cover_image`.
|
| 373 |
+
# The docstring above cites the $0 probe as having "measured" permalinks — it had not, and
|
| 374 |
+
# could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
|
| 375 |
+
# so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
|
| 376 |
+
# POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
|
| 377 |
+
# declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
|
| 378 |
+
# the first. [[reachable-is-not-the-same-as-built]]
|
| 379 |
+
# ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
|
| 380 |
+
# list on every real profile — post capture could not have worked for anybody, and the
|
| 381 |
+
# T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
|
| 382 |
+
# written from a schema tests the schema.
|
| 383 |
+
# ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
|
| 384 |
+
# because it costs nothing and is what a future corpus revision would most likely use. The
|
| 385 |
+
# bare-string branch stays for the same reason — this widens what is ACCEPTED and invents
|
| 386 |
+
# nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
|
| 387 |
+
# before, rather than to a URL built out of a guess.
|
| 388 |
+
# ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
|
| 389 |
+
# whichever array happened to be longer is how one creator's window silently differs from
|
| 390 |
+
# another's, and that reasoning is unchanged by this correction.
|
| 391 |
+
if isinstance(item, dict):
|
| 392 |
+
u = str(item.get("video_url") or item.get("url") or "").strip()
|
| 393 |
+
else:
|
| 394 |
+
u = str(item or "").strip()
|
| 395 |
+
if u.lower().startswith("http") and u not in out:
|
| 396 |
+
out.append(u)
|
| 397 |
+
return out[:limit] if limit and limit > 0 else out
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def pull_posts_tt(post_urls, log=print, deferred=None):
|
| 401 |
+
"""The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised.
|
| 402 |
+
|
| 403 |
+
⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
|
| 404 |
+
measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
|
| 405 |
+
running at 67 minutes. Nothing here loops per URL.
|
| 406 |
+
"""
|
| 407 |
+
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 408 |
+
if not urls:
|
| 409 |
+
return [], ""
|
| 410 |
+
rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
|
| 411 |
+
if note:
|
| 412 |
+
log(f"[aios-tt] posts: {note}")
|
| 413 |
+
return [], note
|
| 414 |
+
out = [r for r in (normalize_post(n) for n in rows) if r]
|
| 415 |
+
return out, ""
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def pull_comments_tt(post_urls, log=print, deferred=None):
|
| 419 |
+
"""The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised.
|
| 420 |
+
|
| 421 |
+
⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
|
| 422 |
+
both networks: a comments scrape ingests identifiable third parties who never entered anybody's
|
| 423 |
+
list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
|
| 424 |
+
none of them to a column; this function adds no new exposure, it just has to be asked for.
|
| 425 |
+
"""
|
| 426 |
+
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 427 |
+
if not urls:
|
| 428 |
+
return [], ""
|
| 429 |
+
rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
|
| 430 |
+
if note:
|
| 431 |
+
log(f"[aios-tt] comments: {note}")
|
| 432 |
+
return [], note
|
| 433 |
+
out = [r for r in (normalize_comment(n) for n in rows) if r]
|
| 434 |
+
return out, ""
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
|
| 438 |
+
#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
|
| 439 |
+
#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
|
| 440 |
+
#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
|
| 441 |
+
#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
|
| 442 |
+
TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def _media_deferrals(deferred):
|
| 446 |
+
"""The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's.
|
| 447 |
+
|
| 448 |
+
⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
|
| 449 |
+
handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
|
| 450 |
+
kind here, so the id already carries everything a mapper choice depends on — and importing
|
| 451 |
+
Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
|
| 452 |
+
"""
|
| 453 |
+
out = []
|
| 454 |
+
for d in deferred or []:
|
| 455 |
+
if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
|
| 456 |
+
out.append(dict(d))
|
| 457 |
+
return out
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
| 461 |
+
max_posts=0, post_metrics=False, comment_metrics=False):
|
| 462 |
+
"""ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
|
| 463 |
+
|
| 464 |
+
`{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
|
| 465 |
+
so the engine's enrich branch treats every network identically and no caller learns a new
|
| 466 |
+
shape.
|
| 467 |
+
|
| 468 |
+
⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
|
| 469 |
+
do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` —
|
| 470 |
+
no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
|
| 471 |
+
then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
|
| 472 |
+
rather than by a rule: their input IS a post permalink, so asking for comments with post capture
|
| 473 |
+
off is a request with no subject, and it returns none instead of quietly buying posts nobody
|
| 474 |
+
asked for.
|
| 475 |
+
|
| 476 |
+
⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
|
| 477 |
+
pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
|
| 478 |
+
(`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
|
| 479 |
+
count and no posts must not paint green over a posts table that did not grow"*). So: posts not
|
| 480 |
+
ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came →
|
| 481 |
+
`partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
|
| 482 |
+
never "did the function finish".
|
| 483 |
+
|
| 484 |
+
⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
|
| 485 |
+
paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
|
| 486 |
+
with its own note explaining that a multi-provider chain is a promise something walks it and
|
| 487 |
+
that nothing walks Instagram's second name today either. So a refusal here is final, and it
|
| 488 |
+
says so instead of implying a retry somewhere.
|
| 489 |
+
"""
|
| 490 |
+
handle = tt_handle(url)
|
| 491 |
+
if not handle:
|
| 492 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
|
| 493 |
+
"note": f"{url!r} is not a TikTok profile URL or handle"}
|
| 494 |
+
|
| 495 |
+
# ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
|
| 496 |
+
# one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
|
| 497 |
+
# happen; a miss falls through to the single-URL call below.
|
| 498 |
+
cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
|
| 499 |
+
_deferred = []
|
| 500 |
+
if isinstance(cached, dict) and cached:
|
| 501 |
+
rows, note = [cached], ""
|
| 502 |
+
else:
|
| 503 |
+
rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
|
| 504 |
+
|
| 505 |
+
node = rows[0] if rows else {}
|
| 506 |
+
profile = normalize_profile(node, handle) if node else {}
|
| 507 |
+
# ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
|
| 508 |
+
# drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and
|
| 509 |
+
# carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
|
| 510 |
+
# this reason, and answering "0 followers" instead is the failure it exists to prevent.
|
| 511 |
+
unreadable = profile.get("followers") is None and profile.get("following") is None
|
| 512 |
+
if note or unreadable:
|
| 513 |
+
# ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
|
| 514 |
+
# building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
|
| 515 |
+
# same record. That was live on the Instagram profile path until 2026-08-09 — measured on
|
| 516 |
+
# nurilab as two runs, two fresh snapshots, both abandoned — and it is not being
|
| 517 |
+
# reintroduced here by omission.
|
| 518 |
+
if isinstance(pending_profile, list):
|
| 519 |
+
for d in _deferred:
|
| 520 |
+
pending_profile.append({**d, "kind": "profile", "influencer": handle})
|
| 521 |
+
why = note or ("the scrape answered, but no follower/following counts were readable in it "
|
| 522 |
+
"(the field names may have moved - see tiktok-capture.md)")
|
| 523 |
+
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 524 |
+
"deferredProfile": [d.get("snapshotId") for d in _deferred],
|
| 525 |
+
"note": why}
|
| 526 |
+
|
| 527 |
+
# --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
|
| 528 |
+
if not post_metrics:
|
| 529 |
+
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 530 |
+
"via": "brightdata",
|
| 531 |
+
"note": note or "profile read; post capture is off for this step"}
|
| 532 |
+
urls = tt_post_urls(node, limit=max_posts)
|
| 533 |
+
if not urls:
|
| 534 |
+
# ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
|
| 535 |
+
# saying so is what stops the next run paying to be told the same thing.
|
| 536 |
+
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 537 |
+
"via": "brightdata",
|
| 538 |
+
"note": note or "profile read; this account's row carried no post links"}
|
| 539 |
+
posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
|
| 540 |
+
comments, c_note = ([], "")
|
| 541 |
+
if comment_metrics and posts:
|
| 542 |
+
# The comments dataset is keyed on a POST permalink, so it reads the posts we just bought —
|
| 543 |
+
# `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
|
| 544 |
+
# is not silently asked about again one rung later.
|
| 545 |
+
comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
|
| 546 |
+
log=log, deferred=_deferred)
|
| 547 |
+
# ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
|
| 548 |
+
# the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
|
| 549 |
+
# `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
|
| 550 |
+
# deferral is impossible — the profile branch above returns `blocked` on any note — so **every
|
| 551 |
+
# id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
|
| 552 |
+
# collector writes preset profile cells onto somebody's record from post rows. The engine keeps
|
| 553 |
+
# the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
|
| 554 |
+
# ⇒ So this returns them under their OWN key, filtered by dataset identity
|
| 555 |
+
# (`_media_deferrals`), and the engine files them in the metric queue with the handle it
|
| 556 |
+
# already holds. Returning rather than appending also keeps the queue's vocabulary out of a
|
| 557 |
+
# connector: this module knows which CORPUS deferred, never what the engine calls it.
|
| 558 |
+
# ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
|
| 559 |
+
# matters most — that is exactly the run where the vendor took too long, so a caller reading
|
| 560 |
+
# the ids only from the success path would lose every batch it actually paid for.
|
| 561 |
+
deferred_media = _media_deferrals(_deferred)
|
| 562 |
+
if not posts:
|
| 563 |
+
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 564 |
+
"via": "brightdata", "deferredMedia": deferred_media,
|
| 565 |
+
"note": p_note or note or "profile read; the post source returned nothing"}
|
| 566 |
+
return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
|
| 567 |
+
"via": "brightdata", "deferredMedia": deferred_media,
|
| 568 |
+
"note": c_note or note or ""}
|
api/deps.py
CHANGED
|
@@ -110,14 +110,7 @@ def _user_for(claims):
|
|
| 110 |
# (login validated the slug's existence, never the membership) — with
|
| 111 |
# AIOS_ENABLE_QA_TENANT=1 that was a working cross-tenant session. A signature proves
|
| 112 |
# the cookie is ours; THIS proves the account belongs where the cookie says it does.
|
| 113 |
-
|
| 114 |
-
# `pub.get("tenant", "royal-imports")` and could never fire: `users._public` normalises a
|
| 115 |
-
# missing tenant to `royal-imports` before this line ever sees it. A dead default that
|
| 116 |
-
# names a real tenant reads like a live fallback, and the cross-tenant audit spent a row
|
| 117 |
-
# proving it was not one. Bare `.get` is also the safer shape if that ever stops being
|
| 118 |
-
# true: `None != claim_tenant` denies, where the old default would have ADMITTED a
|
| 119 |
-
# tenant-less record into tenant #0's session.
|
| 120 |
-
if claim_tenant != pub.get("tenant"):
|
| 121 |
return None
|
| 122 |
return pub
|
| 123 |
if (uname == "admin" and os.environ.get("APP_PASSWORD", "")
|
|
|
|
| 110 |
# (login validated the slug's existence, never the membership) — with
|
| 111 |
# AIOS_ENABLE_QA_TENANT=1 that was a working cross-tenant session. A signature proves
|
| 112 |
# the cookie is ours; THIS proves the account belongs where the cookie says it does.
|
| 113 |
+
if claim_tenant != pub.get("tenant", "royal-imports"):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
return None
|
| 115 |
return pub
|
| 116 |
if (uname == "admin" and os.environ.get("APP_PASSWORD", "")
|
api/main.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/odoo_relational.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_admin.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_alerts.py
CHANGED
|
@@ -1,664 +1,537 @@
|
|
| 1 |
-
"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
|
| 2 |
-
|
| 3 |
-
GET /api/v1/alerts -> {alerts:[...]}
|
| 4 |
-
POST /api/v1/alerts <- {viewId, topic, label?}
|
| 5 |
-
DELETE /api/v1/alerts/{alert_id}
|
| 6 |
-
POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
|
| 7 |
-
GET /api/v1/notifications -> {unread, items:[...]}
|
| 8 |
-
POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
|
| 9 |
-
|
| 10 |
-
The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
|
| 11 |
-
ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
|
| 12 |
-
This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
|
| 13 |
-
|
| 14 |
-
⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
|
| 15 |
-
for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
|
| 16 |
-
full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
|
| 17 |
-
book, and the notification would name customers that user may not see — a permission leak wearing
|
| 18 |
-
a notification's clothes. The owner's own scope is the only correct basis for their alert.
|
| 19 |
-
|
| 20 |
-
⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
|
| 21 |
-
`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
|
| 22 |
-
alert can see is by construction a row its owner could open. Re-implementing the filter here
|
| 23 |
-
would be a second definition of "matches", and those two would drift.
|
| 24 |
-
"""
|
| 25 |
-
import re
|
| 26 |
-
|
| 27 |
-
from fastapi import APIRouter, Body, Depends
|
| 28 |
-
|
| 29 |
-
import core.alerts as alerts
|
| 30 |
-
from deps import Session, err, require_session
|
| 31 |
-
|
| 32 |
-
router = APIRouter(prefix="/api/v1")
|
| 33 |
-
|
| 34 |
-
#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
|
| 35 |
-
_TOPICS = ("customer", "product")
|
| 36 |
-
|
| 37 |
-
# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ────────────────────────
|
| 38 |
-
#
|
| 39 |
-
# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
|
| 40 |
-
#
|
| 41 |
-
# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
|
| 42 |
-
# keys onto the record at write time would give them to notifications minted AFTER the deploy and
|
| 43 |
-
# to nothing else — every notification already sitting in every tenant's inbox would open nothing,
|
| 44 |
-
# and the feature would be correct in the source and absent from the product
|
| 45 |
-
# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
|
| 46 |
-
# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
|
| 47 |
-
# another lane's file this wave — but that is the convenience, not the reason.
|
| 48 |
-
#
|
| 49 |
-
# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
|
| 50 |
-
# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
|
| 51 |
-
# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
|
| 52 |
-
# here means the client branches on ONE field instead of re-deriving the same split.
|
| 53 |
-
#
|
| 54 |
-
# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'`
|
| 55 |
-
# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by
|
| 56 |
-
# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
|
| 57 |
-
# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
|
| 58 |
-
# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
|
| 59 |
-
# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
|
| 60 |
-
# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
|
| 61 |
-
# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
|
| 62 |
-
# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed
|
| 63 |
-
# at was deleted two waves ago.
|
| 64 |
-
|
| 65 |
-
#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them
|
| 66 |
-
#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
|
| 67 |
-
NOTIF_KIND_ALERT = "alert"
|
| 68 |
-
NOTIF_KIND_AUTOMATION = "automation"
|
| 69 |
-
NOTIF_KIND_SHARE = "share"
|
| 70 |
-
|
| 71 |
-
#: C3's `target.module` vocabulary, and the automation sub-selection.
|
| 72 |
-
TARGET_MODULE_DATABASE = "database"
|
| 73 |
-
TARGET_MODULE_AUTOMATION = "automation"
|
| 74 |
-
TARGET_TAB_RUNS = "runs"
|
| 75 |
-
|
| 76 |
-
#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
|
| 77 |
-
#: no producer is a string that reads as a feature — the reason this constant is named here and
|
| 78 |
-
#: cited from `routes_shares` rather than typed twice).
|
| 79 |
-
SHARE_TOPIC = "share"
|
| 80 |
-
|
| 81 |
-
#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
|
| 82 |
-
AUTOMATION_TOPIC = "automation"
|
| 83 |
-
|
| 84 |
-
_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
def route_for_topic(topic):
|
| 88 |
-
"""A grid SCOPE key -> the registry route that renders it, or None.
|
| 89 |
-
|
| 90 |
-
⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
|
| 91 |
-
(`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
|
| 92 |
-
pair that differ — the registry names the surface (`customer_data`) while the grid names the
|
| 93 |
-
scope (`customer`) — so a topic passed through as a route sends every click to a page that
|
| 94 |
-
does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
|
| 95 |
-
rather than plausible, because an absent target renders as a row that does not pretend to be
|
| 96 |
-
clickable, and a wrong one renders as a click that silently goes nowhere.
|
| 97 |
-
"""
|
| 98 |
-
t = str(topic or "").strip()
|
| 99 |
-
if t == "customer":
|
| 100 |
-
return "customer_data"
|
| 101 |
-
if t == "product":
|
| 102 |
-
return "product_data"
|
| 103 |
-
if _UT_TOPIC.match(t):
|
| 104 |
-
return t
|
| 105 |
-
return None
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
def _refusal_code(exc):
|
| 109 |
-
"""The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
|
| 110 |
-
|
| 111 |
-
⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden`
|
| 112 |
-
(403), `window_required` (409) and `store_not_ready` (503) — and each already names its own
|
| 113 |
-
cause. Anything that reduces all four to one word is throwing away the only information the
|
| 114 |
-
reader could have acted on. Returns `""` for a plain exception, so a caller can tell
|
| 115 |
-
"refused, and here is why" apart from "broke, and we do not know why".
|
| 116 |
-
"""
|
| 117 |
-
detail = getattr(exc, "detail", None)
|
| 118 |
-
if isinstance(detail, dict):
|
| 119 |
-
inner = detail.get("error")
|
| 120 |
-
if isinstance(inner, dict):
|
| 121 |
-
return str(inner.get("code") or "")
|
| 122 |
-
return ""
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
def notification_view(item):
|
| 126 |
-
"""One STORED notification -> the shape the Inbox renders. PURE, and total.
|
| 127 |
-
|
| 128 |
-
Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
|
| 129 |
-
no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
|
| 130 |
-
silently omits what it does not understand is the one failure a reader cannot detect.
|
| 131 |
-
"""
|
| 132 |
-
if not isinstance(item, dict):
|
| 133 |
-
return item
|
| 134 |
-
topic = str(item.get("topic") or "").strip()
|
| 135 |
-
alert_id = str(item.get("alertId") or "").strip()
|
| 136 |
-
|
| 137 |
-
# ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
|
| 138 |
-
# says `automation` but whose producer key never arrived (a truncated payload, a server
|
| 139 |
-
# mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work
|
| 140 |
-
# and silently does not, which is this repo's most-repeated failure shape. Failing the test
|
| 141 |
-
# drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
|
| 142 |
-
if topic == AUTOMATION_TOPIC and alert_id:
|
| 143 |
-
kind = NOTIF_KIND_AUTOMATION
|
| 144 |
-
target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
|
| 145 |
-
elif topic == SHARE_TOPIC and alert_id:
|
| 146 |
-
# ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
|
| 147 |
-
# `row_id=<the view to select>`.
|
| 148 |
-
#
|
| 149 |
-
# ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
|
| 150 |
-
# wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
|
| 151 |
-
# target read `{module: "database", id: "view_42"}` — an instruction to open a database
|
| 152 |
-
# called `view_42`. It looked right in the payload and would have opened nothing. The
|
| 153 |
-
# producer resolves the object to its topic and hands over the route; this branch only
|
| 154 |
-
# shapes what it is given.
|
| 155 |
-
kind = NOTIF_KIND_SHARE
|
| 156 |
-
row_id = str(item.get("rowId") or "").strip()
|
| 157 |
-
target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
|
| 158 |
-
**({"tab": row_id} if row_id else {})}
|
| 159 |
-
else:
|
| 160 |
-
kind = NOTIF_KIND_ALERT
|
| 161 |
-
route = route_for_topic(topic)
|
| 162 |
-
view_id = str(item.get("viewId") or "").strip()
|
| 163 |
-
target = None if route is None else (
|
| 164 |
-
{"module": TARGET_MODULE_DATABASE, "id": route,
|
| 165 |
-
**({"tab": view_id} if view_id else {})})
|
| 166 |
-
|
| 167 |
-
# The email split: `subject` is the HEADER (what this is about — the alert, the automation,
|
| 168 |
-
# the database), `label` stays the BODY (what happened — the record that entered, the run
|
| 169 |
-
# summary). They were one field, which is why a notification read as a sentence with no
|
| 170 |
-
# sender and the pane could not be laid out like mail.
|
| 171 |
-
subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
|
| 172 |
-
# ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
|
| 173 |
-
# is D-101's dead one; leaving it through would give the client two vocabularies for one
|
| 174 |
-
# question, which is the defect this wave's item 6 is about in a different file.
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
is
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
#
|
| 224 |
-
#
|
| 225 |
-
#
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
`
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
#
|
| 512 |
-
#
|
| 513 |
-
#
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
#
|
| 537 |
-
|
| 538 |
-
# 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
|
| 539 |
-
# wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
|
| 540 |
-
#
|
| 541 |
-
# ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
|
| 542 |
-
# arity until the line RUNS, and this line runs only on the success path of a feature whose
|
| 543 |
-
# every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
|
| 544 |
-
# well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
|
| 545 |
-
# (`no_filter` reaches the user) and the transport — never a creation. A gate can be green,
|
| 546 |
-
# thorough and honest about everything except the one path the feature exists for.
|
| 547 |
-
#
|
| 548 |
-
# `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
|
| 549 |
-
# column up in; building a second dict here would be a second answer to one question, which
|
| 550 |
-
# is this wave's other headline defect in a different file. Its leading underscore is a real
|
| 551 |
-
# smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
|
| 552 |
-
columns = filter_eval._columns_map(g.get("fields") or [])
|
| 553 |
-
|
| 554 |
-
def _any_active(ns):
|
| 555 |
-
for n in ns or ():
|
| 556 |
-
if isinstance(n, dict) and isinstance(n.get("children"), list):
|
| 557 |
-
if _any_active(n["children"]):
|
| 558 |
-
return True
|
| 559 |
-
elif filter_eval.is_rule_active(n, columns):
|
| 560 |
-
return True
|
| 561 |
-
return False
|
| 562 |
-
|
| 563 |
-
if not _any_active(nodes):
|
| 564 |
-
raise err(400, "no_filter",
|
| 565 |
-
"this view has no active filter, so no record can ever ENTER it — add a "
|
| 566 |
-
"condition to the view first, then create the alert")
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
@router.delete("/alerts/{alert_id}")
|
| 570 |
-
def delete_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 571 |
-
rec = next((r for r in alerts.list_alerts(st=session.runtime)
|
| 572 |
-
if str(r.get("id")) == str(alert_id)), None)
|
| 573 |
-
if rec is None:
|
| 574 |
-
raise err(404, "no_alert", "that alert does not exist")
|
| 575 |
-
if str(rec.get("owner")) != str(session.uname) and not session.admin:
|
| 576 |
-
raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
|
| 577 |
-
alerts.delete(alert_id, st=session.runtime)
|
| 578 |
-
return {"ok": True}
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
@router.post("/alerts/{alert_id}/run")
|
| 582 |
-
def run_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 583 |
-
rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
|
| 584 |
-
st=session.runtime)
|
| 585 |
-
if str(r.get("id")) == str(alert_id)), None)
|
| 586 |
-
if rec is None:
|
| 587 |
-
raise err(404, "no_alert", "that alert does not exist")
|
| 588 |
-
return _evaluate(session, rec)
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
@router.get("/notifications")
|
| 592 |
-
def notifications(session: Session = Depends(require_session)):
|
| 593 |
-
"""The inbox — RE-EVALUATED on read, which is a deliberate design choice.
|
| 594 |
-
|
| 595 |
-
⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
|
| 596 |
-
plan was a push hook: the automation engine calls `after_write` when it lands rows. But
|
| 597 |
-
`run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
|
| 598 |
-
evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
|
| 599 |
-
a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
|
| 600 |
-
construction that leaks scope.
|
| 601 |
-
|
| 602 |
-
Pulling on read has none of that: the caller IS a session, the assemblies are already
|
| 603 |
-
scope-cached, and the user cannot observe the difference — an inbox is only ever read by
|
| 604 |
-
someone opening it. The cost is that a notification is minted when you LOOK rather than when
|
| 605 |
-
the row landed, so the `at` stamp is detection time, not arrival time.
|
| 606 |
-
|
| 607 |
-
`after_write` stays exported for the day the engine can hand over a real identity.
|
| 608 |
-
|
| 609 |
-
⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
|
| 610 |
-
⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
|
| 611 |
-
**20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts**
|
| 612 |
-
(censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
|
| 613 |
-
re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
|
| 614 |
-
body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
|
| 615 |
-
grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which
|
| 616 |
-
is the difference between "fine" and "three seconds per alert" the day somebody uses the
|
| 617 |
-
feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
|
| 618 |
-
this does: every alert is still evaluated, against the same rows, in the same order.
|
| 619 |
-
"""
|
| 620 |
-
assemblies = {}
|
| 621 |
-
for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
|
| 622 |
-
try:
|
| 623 |
-
_evaluate(session, rec, assemblies=assemblies)
|
| 624 |
-
except Exception: # noqa: BLE001
|
| 625 |
-
continue # one bad alert must not empty the pane
|
| 626 |
-
# ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
|
| 627 |
-
# this wave carries a `target` too. See `notification_view`'s header for why it is derived.
|
| 628 |
-
return inbox_view(alerts.inbox(session.uname, st=session.runtime))
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
@router.post("/notifications/read")
|
| 632 |
-
def read_notifications(body: dict = Body(default=None),
|
| 633 |
-
session: Session = Depends(require_session)):
|
| 634 |
-
body = body or {}
|
| 635 |
-
ids = body.get("ids")
|
| 636 |
-
if ids is not None and not isinstance(ids, list):
|
| 637 |
-
raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
|
| 638 |
-
# ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
|
| 639 |
-
# module re-renders from it — an un-enriched answer here would strip `target` off every row
|
| 640 |
-
# the moment somebody marked one read, i.e. the feature would work until first use.
|
| 641 |
-
return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
|
| 642 |
-
st=session.runtime))
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
def after_write(session: Session, topic_key: str):
|
| 646 |
-
"""THE WRITE HOOK — call after a write that could change what a view matches.
|
| 647 |
-
|
| 648 |
-
Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
|
| 649 |
-
upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
|
| 650 |
-
edit that triggered it.
|
| 651 |
-
|
| 652 |
-
⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes
|
| 653 |
-
one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
|
| 654 |
-
⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
|
| 655 |
-
push hook was resolved the other way). Booked rather than wired: minting a session inside the
|
| 656 |
-
engine's worker thread is the ad-hoc identity construction this file exists to avoid.
|
| 657 |
-
"""
|
| 658 |
-
try:
|
| 659 |
-
assemblies = {}
|
| 660 |
-
return alerts.after_write(topic_key, st=session.runtime,
|
| 661 |
-
runner=lambda rec: _evaluate(session, rec,
|
| 662 |
-
assemblies=assemblies))
|
| 663 |
-
except Exception: # noqa: BLE001
|
| 664 |
-
return {"evaluated": 0}
|
|
|
|
| 1 |
+
"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
|
| 2 |
+
|
| 3 |
+
GET /api/v1/alerts -> {alerts:[...]}
|
| 4 |
+
POST /api/v1/alerts <- {viewId, topic, label?}
|
| 5 |
+
DELETE /api/v1/alerts/{alert_id}
|
| 6 |
+
POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
|
| 7 |
+
GET /api/v1/notifications -> {unread, items:[...]}
|
| 8 |
+
POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
|
| 9 |
+
|
| 10 |
+
The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
|
| 11 |
+
ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
|
| 12 |
+
This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
|
| 13 |
+
|
| 14 |
+
⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
|
| 15 |
+
for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
|
| 16 |
+
full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
|
| 17 |
+
book, and the notification would name customers that user may not see — a permission leak wearing
|
| 18 |
+
a notification's clothes. The owner's own scope is the only correct basis for their alert.
|
| 19 |
+
|
| 20 |
+
⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
|
| 21 |
+
`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
|
| 22 |
+
alert can see is by construction a row its owner could open. Re-implementing the filter here
|
| 23 |
+
would be a second definition of "matches", and those two would drift.
|
| 24 |
+
"""
|
| 25 |
+
import re
|
| 26 |
+
|
| 27 |
+
from fastapi import APIRouter, Body, Depends
|
| 28 |
+
|
| 29 |
+
import core.alerts as alerts
|
| 30 |
+
from deps import Session, err, require_session
|
| 31 |
+
|
| 32 |
+
router = APIRouter(prefix="/api/v1")
|
| 33 |
+
|
| 34 |
+
#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
|
| 35 |
+
_TOPICS = ("customer", "product")
|
| 36 |
+
|
| 37 |
+
# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ────────────────────────
|
| 38 |
+
#
|
| 39 |
+
# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
|
| 40 |
+
#
|
| 41 |
+
# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
|
| 42 |
+
# keys onto the record at write time would give them to notifications minted AFTER the deploy and
|
| 43 |
+
# to nothing else — every notification already sitting in every tenant's inbox would open nothing,
|
| 44 |
+
# and the feature would be correct in the source and absent from the product
|
| 45 |
+
# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
|
| 46 |
+
# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
|
| 47 |
+
# another lane's file this wave — but that is the convenience, not the reason.
|
| 48 |
+
#
|
| 49 |
+
# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
|
| 50 |
+
# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
|
| 51 |
+
# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
|
| 52 |
+
# here means the client branches on ONE field instead of re-deriving the same split.
|
| 53 |
+
#
|
| 54 |
+
# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'`
|
| 55 |
+
# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by
|
| 56 |
+
# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
|
| 57 |
+
# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
|
| 58 |
+
# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
|
| 59 |
+
# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
|
| 60 |
+
# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
|
| 61 |
+
# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
|
| 62 |
+
# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed
|
| 63 |
+
# at was deleted two waves ago.
|
| 64 |
+
|
| 65 |
+
#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them
|
| 66 |
+
#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
|
| 67 |
+
NOTIF_KIND_ALERT = "alert"
|
| 68 |
+
NOTIF_KIND_AUTOMATION = "automation"
|
| 69 |
+
NOTIF_KIND_SHARE = "share"
|
| 70 |
+
|
| 71 |
+
#: C3's `target.module` vocabulary, and the automation sub-selection.
|
| 72 |
+
TARGET_MODULE_DATABASE = "database"
|
| 73 |
+
TARGET_MODULE_AUTOMATION = "automation"
|
| 74 |
+
TARGET_TAB_RUNS = "runs"
|
| 75 |
+
|
| 76 |
+
#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
|
| 77 |
+
#: no producer is a string that reads as a feature — the reason this constant is named here and
|
| 78 |
+
#: cited from `routes_shares` rather than typed twice).
|
| 79 |
+
SHARE_TOPIC = "share"
|
| 80 |
+
|
| 81 |
+
#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
|
| 82 |
+
AUTOMATION_TOPIC = "automation"
|
| 83 |
+
|
| 84 |
+
_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def route_for_topic(topic):
|
| 88 |
+
"""A grid SCOPE key -> the registry route that renders it, or None.
|
| 89 |
+
|
| 90 |
+
⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
|
| 91 |
+
(`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
|
| 92 |
+
pair that differ — the registry names the surface (`customer_data`) while the grid names the
|
| 93 |
+
scope (`customer`) — so a topic passed through as a route sends every click to a page that
|
| 94 |
+
does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
|
| 95 |
+
rather than plausible, because an absent target renders as a row that does not pretend to be
|
| 96 |
+
clickable, and a wrong one renders as a click that silently goes nowhere.
|
| 97 |
+
"""
|
| 98 |
+
t = str(topic or "").strip()
|
| 99 |
+
if t == "customer":
|
| 100 |
+
return "customer_data"
|
| 101 |
+
if t == "product":
|
| 102 |
+
return "product_data"
|
| 103 |
+
if _UT_TOPIC.match(t):
|
| 104 |
+
return t
|
| 105 |
+
return None
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _refusal_code(exc):
|
| 109 |
+
"""The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
|
| 110 |
+
|
| 111 |
+
⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden`
|
| 112 |
+
(403), `window_required` (409) and `store_not_ready` (503) — and each already names its own
|
| 113 |
+
cause. Anything that reduces all four to one word is throwing away the only information the
|
| 114 |
+
reader could have acted on. Returns `""` for a plain exception, so a caller can tell
|
| 115 |
+
"refused, and here is why" apart from "broke, and we do not know why".
|
| 116 |
+
"""
|
| 117 |
+
detail = getattr(exc, "detail", None)
|
| 118 |
+
if isinstance(detail, dict):
|
| 119 |
+
inner = detail.get("error")
|
| 120 |
+
if isinstance(inner, dict):
|
| 121 |
+
return str(inner.get("code") or "")
|
| 122 |
+
return ""
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def notification_view(item):
|
| 126 |
+
"""One STORED notification -> the shape the Inbox renders. PURE, and total.
|
| 127 |
+
|
| 128 |
+
Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
|
| 129 |
+
no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
|
| 130 |
+
silently omits what it does not understand is the one failure a reader cannot detect.
|
| 131 |
+
"""
|
| 132 |
+
if not isinstance(item, dict):
|
| 133 |
+
return item
|
| 134 |
+
topic = str(item.get("topic") or "").strip()
|
| 135 |
+
alert_id = str(item.get("alertId") or "").strip()
|
| 136 |
+
|
| 137 |
+
# ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
|
| 138 |
+
# says `automation` but whose producer key never arrived (a truncated payload, a server
|
| 139 |
+
# mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work
|
| 140 |
+
# and silently does not, which is this repo's most-repeated failure shape. Failing the test
|
| 141 |
+
# drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
|
| 142 |
+
if topic == AUTOMATION_TOPIC and alert_id:
|
| 143 |
+
kind = NOTIF_KIND_AUTOMATION
|
| 144 |
+
target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
|
| 145 |
+
elif topic == SHARE_TOPIC and alert_id:
|
| 146 |
+
# ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
|
| 147 |
+
# `row_id=<the view to select>`.
|
| 148 |
+
#
|
| 149 |
+
# ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
|
| 150 |
+
# wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
|
| 151 |
+
# target read `{module: "database", id: "view_42"}` — an instruction to open a database
|
| 152 |
+
# called `view_42`. It looked right in the payload and would have opened nothing. The
|
| 153 |
+
# producer resolves the object to its topic and hands over the route; this branch only
|
| 154 |
+
# shapes what it is given.
|
| 155 |
+
kind = NOTIF_KIND_SHARE
|
| 156 |
+
row_id = str(item.get("rowId") or "").strip()
|
| 157 |
+
target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
|
| 158 |
+
**({"tab": row_id} if row_id else {})}
|
| 159 |
+
else:
|
| 160 |
+
kind = NOTIF_KIND_ALERT
|
| 161 |
+
route = route_for_topic(topic)
|
| 162 |
+
view_id = str(item.get("viewId") or "").strip()
|
| 163 |
+
target = None if route is None else (
|
| 164 |
+
{"module": TARGET_MODULE_DATABASE, "id": route,
|
| 165 |
+
**({"tab": view_id} if view_id else {})})
|
| 166 |
+
|
| 167 |
+
# The email split: `subject` is the HEADER (what this is about — the alert, the automation,
|
| 168 |
+
# the database), `label` stays the BODY (what happened — the record that entered, the run
|
| 169 |
+
# summary). They were one field, which is why a notification read as a sentence with no
|
| 170 |
+
# sender and the pane could not be laid out like mail.
|
| 171 |
+
subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
|
| 172 |
+
# ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
|
| 173 |
+
# is D-101's dead one; leaving it through would give the client two vocabularies for one
|
| 174 |
+
# question, which is the defect this wave's item 6 is about in a different file.
|
| 175 |
+
out = {**item, "read": bool(item.get("read")), "kind": kind,
|
| 176 |
+
"subject": subject or "Notification"}
|
| 177 |
+
if target is not None:
|
| 178 |
+
out["target"] = target
|
| 179 |
+
return out
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def inbox_view(box):
|
| 183 |
+
"""`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
|
| 184 |
+
|
| 185 |
+
⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
|
| 186 |
+
recount here would make the badge a function of whatever this page happened to include, which
|
| 187 |
+
is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
|
| 188 |
+
"""
|
| 189 |
+
if not isinstance(box, dict):
|
| 190 |
+
return box
|
| 191 |
+
items = box.get("items")
|
| 192 |
+
if not isinstance(items, list):
|
| 193 |
+
return box
|
| 194 |
+
return {**box, "items": [notification_view(n) for n in items]}
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _topic_or_400(raw):
|
| 198 |
+
topic = str(raw or "").strip().lower()
|
| 199 |
+
if topic.startswith("ut_") or topic in _TOPICS:
|
| 200 |
+
return topic
|
| 201 |
+
raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def _owner_session(session: Session, owner: str):
|
| 205 |
+
"""A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
|
| 206 |
+
|
| 207 |
+
⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an
|
| 208 |
+
owner session is built by swapping the `user` RECORD and letting both derive themselves. An
|
| 209 |
+
earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
|
| 210 |
+
first write hook of the wave; the properties are the single definition of who a session is,
|
| 211 |
+
and going around them is how a session with an admin flag and a non-admin record exists.
|
| 212 |
+
|
| 213 |
+
Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather
|
| 214 |
+
than evaluating as somebody else, which is the fail-closed direction.
|
| 215 |
+
"""
|
| 216 |
+
import core.users as users
|
| 217 |
+
|
| 218 |
+
if str(owner) == str(session.uname):
|
| 219 |
+
return session
|
| 220 |
+
rec = (users.registry() or {}).get(str(owner))
|
| 221 |
+
if not isinstance(rec, dict) or not rec.get("active", True):
|
| 222 |
+
return None
|
| 223 |
+
# `_public` is THE definition of what a session may know about its own account (never a hash
|
| 224 |
+
# or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a
|
| 225 |
+
# second definition, and the one that leaks is always the copy.
|
| 226 |
+
return Session(tenant=session.tenant, user=users._public(str(owner), rec),
|
| 227 |
+
claims=session.claims, runtime=session.runtime)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _evaluate(session: Session, rec: dict, assemblies=None):
|
| 231 |
+
"""Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
|
| 232 |
+
|
| 233 |
+
⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
|
| 234 |
+
of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
|
| 235 |
+
one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two
|
| 236 |
+
alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
|
| 237 |
+
because each `_evaluate` was a closed call.
|
| 238 |
+
⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
|
| 239 |
+
note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
|
| 240 |
+
wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
|
| 241 |
+
must never share an entry. Getting that key wrong is the one way this optimisation could leak.
|
| 242 |
+
⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
|
| 243 |
+
they evaluate ONE alert and a memo for a single call is pure overhead.
|
| 244 |
+
"""
|
| 245 |
+
import aios_grid
|
| 246 |
+
from harness import filter_eval
|
| 247 |
+
|
| 248 |
+
owner_sess = _owner_session(session, rec.get("owner"))
|
| 249 |
+
if owner_sess is None:
|
| 250 |
+
return {"skipped": "owner_unavailable"}
|
| 251 |
+
topic = str(rec.get("topic") or "")
|
| 252 |
+
memo_key = (topic, str(owner_sess.uname))
|
| 253 |
+
g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
|
| 254 |
+
if g is None:
|
| 255 |
+
try:
|
| 256 |
+
if topic.startswith("ut_"):
|
| 257 |
+
from routes_tables import ut_assembly
|
| 258 |
+
# ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
|
| 259 |
+
# `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
|
| 260 |
+
# one-shot field-name correction acks for every `ut_` topic that has an alert —
|
| 261 |
+
# taking them from the `/workspace` refresh that exists to show them to the person
|
| 262 |
+
# who made the edit. The customer branch below has always passed False; this one
|
| 263 |
+
# inherited a default nobody re-read. An inbox poll must never consume a one-shot.
|
| 264 |
+
g = ut_assembly(owner_sess, topic,
|
| 265 |
+
storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
|
| 266 |
+
consume_corrections=False)
|
| 267 |
+
else:
|
| 268 |
+
from routes_customers import grid_assembly
|
| 269 |
+
g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
|
| 270 |
+
except Exception as e: # noqa: BLE001
|
| 271 |
+
# ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
|
| 272 |
+
# raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
|
| 273 |
+
# keeps a blanket catch — but it now reports the refusal's OWN code where there is
|
| 274 |
+
# one. `type(e).__name__` said `HTTPException` for four different causes, and
|
| 275 |
+
# `lastError` is the only place a user ever learns why an alert stopped firing.
|
| 276 |
+
#
|
| 277 |
+
# ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
|
| 278 |
+
# evaluation genuinely needs the rows to run the filter over. So an alert on a
|
| 279 |
+
# read-through grid is created (T22) and then skips at evaluation with
|
| 280 |
+
# `window_required` naming why — which is D-184's remaining half, and it is a
|
| 281 |
+
# SENTENCE now rather than silence.
|
| 282 |
+
return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
|
| 283 |
+
if isinstance(assemblies, dict):
|
| 284 |
+
assemblies[memo_key] = g
|
| 285 |
+
|
| 286 |
+
view = (g.get("views") or {}).get(str(rec.get("viewId")))
|
| 287 |
+
if not isinstance(view, dict):
|
| 288 |
+
# Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
|
| 289 |
+
# deleting the alert: an alert that silently vanishes is indistinguishable from one that
|
| 290 |
+
# never fires, and the user cannot debug what is not there.
|
| 291 |
+
return {"skipped": "view_missing"}
|
| 292 |
+
|
| 293 |
+
# The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived
|
| 294 |
+
# and overlay values on a row. Evaluating a filter against raw pool dicts would silently
|
| 295 |
+
# never match any condition on a user-created or measure column.
|
| 296 |
+
rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"),
|
| 297 |
+
derived=g.get("derived"))
|
| 298 |
+
config = view.get("config") or view
|
| 299 |
+
ctx = filter_eval.EvalCtx(
|
| 300 |
+
cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
|
| 301 |
+
for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
|
| 302 |
+
measure_sets=g.get("measure_sets") or {},
|
| 303 |
+
today=g.get("today"))
|
| 304 |
+
pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
|
| 305 |
+
member_pids=config.get("memberPids"))
|
| 306 |
+
labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
|
| 307 |
+
return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
|
| 308 |
+
labels=labels, partial=False, st=session.runtime)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
@router.get("/alerts")
|
| 312 |
+
def list_alerts(session: Session = Depends(require_session)):
|
| 313 |
+
return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
|
| 314 |
+
st=session.runtime)}
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
@router.post("/alerts")
|
| 318 |
+
def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 319 |
+
body = body or {}
|
| 320 |
+
view_id = str(body.get("viewId") or "").strip()
|
| 321 |
+
if not view_id:
|
| 322 |
+
raise err(400, "bad_view", "an alert needs the id of the view it watches")
|
| 323 |
+
topic = _topic_or_400(body.get("topic"))
|
| 324 |
+
_require_filtered_view(session, topic, view_id)
|
| 325 |
+
import uuid
|
| 326 |
+
aid = f"al_{uuid.uuid4().hex[:12]}"
|
| 327 |
+
rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
|
| 328 |
+
label=body.get("label") or "", st=session.runtime)
|
| 329 |
+
# SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
|
| 330 |
+
# Deferring this to the first write hook would mean the next edit announces the whole view.
|
| 331 |
+
outcome = _evaluate(session, rec)
|
| 332 |
+
return {"alert": {**rec, "seeded": True}, "first": outcome}
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def _require_filtered_view(session: Session, topic: str, view_id: str):
|
| 336 |
+
"""400 unless `view_id` exists on `topic` AND actually narrows something.
|
| 337 |
+
|
| 338 |
+
⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
|
| 339 |
+
that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
|
| 340 |
+
(`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
|
| 341 |
+
entrant again — there is nothing left to enter. The owner's words are *"when a Record gets
|
| 342 |
+
into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
|
| 343 |
+
discovered by never being notified.
|
| 344 |
+
|
| 345 |
+
`is_rule_active` is the SAME activeness predicate the engine and the column tints use — a
|
| 346 |
+
half-typed rule is not a filter, and this must agree with what actually narrows or it would
|
| 347 |
+
accept a view whose one rule the engine then ignores.
|
| 348 |
+
|
| 349 |
+
⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
|
| 350 |
+
the second one hid the first.
|
| 351 |
+
|
| 352 |
+
(1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
|
| 353 |
+
`g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
|
| 354 |
+
read-through grid built the whole pool — and `scoped_pool` refuses that with
|
| 355 |
+
`409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
|
| 356 |
+
(W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
|
| 357 |
+
runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
|
| 358 |
+
closed** — an alert on a read-through grid can now be made at all.
|
| 359 |
+
(2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
|
| 360 |
+
is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
|
| 361 |
+
`503 store_not_ready` — four refusals that each say what is wrong — were all replaced by
|
| 362 |
+
*"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED
|
| 363 |
+
A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design
|
| 364 |
+
(a 5xx body is the server's internals), substituting *"Something went wrong on our
|
| 365 |
+
side."* — which is the owner's screenshot, word for word. A knowable cause returned as a
|
| 366 |
+
5xx is invisible by construction, so re-wording the 503 could never have fixed this.
|
| 367 |
+
⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
|
| 368 |
+
honest. What it may no longer do is catch a refusal that already knows its own name.
|
| 369 |
+
"""
|
| 370 |
+
from fastapi import HTTPException
|
| 371 |
+
|
| 372 |
+
from harness import filter_eval
|
| 373 |
+
|
| 374 |
+
try:
|
| 375 |
+
if topic.startswith("ut_"):
|
| 376 |
+
from routes_tables import ut_assembly
|
| 377 |
+
# ⚠ `consume_corrections=False` — the customer branch has always passed it and this
|
| 378 |
+
# one inherited a default nobody re-read. Creating an alert must not eat the one-shot
|
| 379 |
+
# field-name correction acks belonging to the `/workspace` refresh that exists to show
|
| 380 |
+
# them to the person who made the edit. Same defect `_evaluate`'s header records.
|
| 381 |
+
g = ut_assembly(session, topic,
|
| 382 |
+
storage_key=f"{session.tenant}:{topic}:{session.uname}",
|
| 383 |
+
consume_corrections=False, with_rows=False)
|
| 384 |
+
else:
|
| 385 |
+
from routes_customers import grid_assembly
|
| 386 |
+
g = grid_assembly(session, scope=topic, consume_corrections=False)
|
| 387 |
+
except HTTPException:
|
| 388 |
+
raise # it already names its own cause
|
| 389 |
+
except Exception as e: # noqa: BLE001
|
| 390 |
+
# Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it,
|
| 391 |
+
# the one path that reaches this branch is also the one path with nothing to debug from.
|
| 392 |
+
raise err(503, "unavailable",
|
| 393 |
+
f"the table could not be read ({type(e).__name__}) — try again in a moment")
|
| 394 |
+
view = (g.get("views") or {}).get(str(view_id))
|
| 395 |
+
if not isinstance(view, dict):
|
| 396 |
+
raise err(404, "no_view", "that view does not exist on this table")
|
| 397 |
+
nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
|
| 398 |
+
|
| 399 |
+
# ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT, AND THAT IS OWNER ITEM 17.**
|
| 400 |
+
#
|
| 401 |
+
# `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
|
| 402 |
+
# caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
|
| 403 |
+
# rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
|
| 404 |
+
#
|
| 405 |
+
# ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
|
| 406 |
+
# view HAS a condition — and a view with a condition is the only kind an alert is allowed on.
|
| 407 |
+
# A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
|
| 408 |
+
# calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
|
| 409 |
+
# worked was the refusal path: "Alert me about new records" had never once created an alert
|
| 410 |
+
# on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the
|
| 411 |
+
# 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
|
| 412 |
+
# wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
|
| 413 |
+
#
|
| 414 |
+
# ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
|
| 415 |
+
# arity until the line RUNS, and this line runs only on the success path of a feature whose
|
| 416 |
+
# every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
|
| 417 |
+
# well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
|
| 418 |
+
# (`no_filter` reaches the user) and the transport — never a creation. A gate can be green,
|
| 419 |
+
# thorough and honest about everything except the one path the feature exists for.
|
| 420 |
+
#
|
| 421 |
+
# `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
|
| 422 |
+
# column up in; building a second dict here would be a second answer to one question, which
|
| 423 |
+
# is this wave's other headline defect in a different file. Its leading underscore is a real
|
| 424 |
+
# smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
|
| 425 |
+
columns = filter_eval._columns_map(g.get("fields") or [])
|
| 426 |
+
|
| 427 |
+
def _any_active(ns):
|
| 428 |
+
for n in ns or ():
|
| 429 |
+
if isinstance(n, dict) and isinstance(n.get("children"), list):
|
| 430 |
+
if _any_active(n["children"]):
|
| 431 |
+
return True
|
| 432 |
+
elif filter_eval.is_rule_active(n, columns):
|
| 433 |
+
return True
|
| 434 |
+
return False
|
| 435 |
+
|
| 436 |
+
if not _any_active(nodes):
|
| 437 |
+
raise err(400, "no_filter",
|
| 438 |
+
"this view has no active filter, so no record can ever ENTER it — add a "
|
| 439 |
+
"condition to the view first, then create the alert")
|
| 440 |
+
|
| 441 |
+
|
| 442 |
+
@router.delete("/alerts/{alert_id}")
|
| 443 |
+
def delete_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 444 |
+
rec = next((r for r in alerts.list_alerts(st=session.runtime)
|
| 445 |
+
if str(r.get("id")) == str(alert_id)), None)
|
| 446 |
+
if rec is None:
|
| 447 |
+
raise err(404, "no_alert", "that alert does not exist")
|
| 448 |
+
if str(rec.get("owner")) != str(session.uname) and not session.admin:
|
| 449 |
+
raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
|
| 450 |
+
alerts.delete(alert_id, st=session.runtime)
|
| 451 |
+
return {"ok": True}
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
@router.post("/alerts/{alert_id}/run")
|
| 455 |
+
def run_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 456 |
+
rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
|
| 457 |
+
st=session.runtime)
|
| 458 |
+
if str(r.get("id")) == str(alert_id)), None)
|
| 459 |
+
if rec is None:
|
| 460 |
+
raise err(404, "no_alert", "that alert does not exist")
|
| 461 |
+
return _evaluate(session, rec)
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
@router.get("/notifications")
|
| 465 |
+
def notifications(session: Session = Depends(require_session)):
|
| 466 |
+
"""The inbox — RE-EVALUATED on read, which is a deliberate design choice.
|
| 467 |
+
|
| 468 |
+
⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
|
| 469 |
+
plan was a push hook: the automation engine calls `after_write` when it lands rows. But
|
| 470 |
+
`run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
|
| 471 |
+
evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
|
| 472 |
+
a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
|
| 473 |
+
construction that leaks scope.
|
| 474 |
+
|
| 475 |
+
Pulling on read has none of that: the caller IS a session, the assemblies are already
|
| 476 |
+
scope-cached, and the user cannot observe the difference — an inbox is only ever read by
|
| 477 |
+
someone opening it. The cost is that a notification is minted when you LOOK rather than when
|
| 478 |
+
the row landed, so the `at` stamp is detection time, not arrival time.
|
| 479 |
+
|
| 480 |
+
`after_write` stays exported for the day the engine can hand over a real identity.
|
| 481 |
+
|
| 482 |
+
⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
|
| 483 |
+
⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
|
| 484 |
+
**20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts**
|
| 485 |
+
(censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
|
| 486 |
+
re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
|
| 487 |
+
body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
|
| 488 |
+
grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which
|
| 489 |
+
is the difference between "fine" and "three seconds per alert" the day somebody uses the
|
| 490 |
+
feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
|
| 491 |
+
this does: every alert is still evaluated, against the same rows, in the same order.
|
| 492 |
+
"""
|
| 493 |
+
assemblies = {}
|
| 494 |
+
for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
|
| 495 |
+
try:
|
| 496 |
+
_evaluate(session, rec, assemblies=assemblies)
|
| 497 |
+
except Exception: # noqa: BLE001
|
| 498 |
+
continue # one bad alert must not empty the pane
|
| 499 |
+
# ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
|
| 500 |
+
# this wave carries a `target` too. See `notification_view`'s header for why it is derived.
|
| 501 |
+
return inbox_view(alerts.inbox(session.uname, st=session.runtime))
|
| 502 |
+
|
| 503 |
+
|
| 504 |
+
@router.post("/notifications/read")
|
| 505 |
+
def read_notifications(body: dict = Body(default=None),
|
| 506 |
+
session: Session = Depends(require_session)):
|
| 507 |
+
body = body or {}
|
| 508 |
+
ids = body.get("ids")
|
| 509 |
+
if ids is not None and not isinstance(ids, list):
|
| 510 |
+
raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
|
| 511 |
+
# ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
|
| 512 |
+
# module re-renders from it — an un-enriched answer here would strip `target` off every row
|
| 513 |
+
# the moment somebody marked one read, i.e. the feature would work until first use.
|
| 514 |
+
return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
|
| 515 |
+
st=session.runtime))
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
def after_write(session: Session, topic_key: str):
|
| 519 |
+
"""THE WRITE HOOK — call after a write that could change what a view matches.
|
| 520 |
+
|
| 521 |
+
Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
|
| 522 |
+
upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
|
| 523 |
+
edit that triggered it.
|
| 524 |
+
|
| 525 |
+
⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes
|
| 526 |
+
one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
|
| 527 |
+
⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
|
| 528 |
+
push hook was resolved the other way). Booked rather than wired: minting a session inside the
|
| 529 |
+
engine's worker thread is the ad-hoc identity construction this file exists to avoid.
|
| 530 |
+
"""
|
| 531 |
+
try:
|
| 532 |
+
assemblies = {}
|
| 533 |
+
return alerts.after_write(topic_key, st=session.runtime,
|
| 534 |
+
runner=lambda rec: _evaluate(session, rec,
|
| 535 |
+
assemblies=assemblies))
|
| 536 |
+
except Exception: # noqa: BLE001
|
| 537 |
+
return {"evaluated": 0}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/routes_automation.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_changes.py
CHANGED
|
@@ -55,7 +55,7 @@ So: a valid session, a scope sanitised to a bucket-safe shape, and — for a `ut
|
|
| 55 |
cached verdict that this session may open it.
|
| 56 |
"""
|
| 57 |
import time
|
| 58 |
-
from fastapi import APIRouter, Depends
|
| 59 |
|
| 60 |
import core.shared_overlay as shared_overlay
|
| 61 |
import core.store as store
|
|
@@ -224,87 +224,3 @@ def changes(scope: str = "", session: Session = Depends(require_session)):
|
|
| 224 |
# a token frozen at 0 would promise liveness the backend is not delivering.
|
| 225 |
tokens[label] = rev.get("token") if isinstance(rev, dict) else None
|
| 226 |
return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()}
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
# ── ⭐⭐ D-144 (W33-T32) — A CAUGHT RENDER ERROR TELLS SOMEBODY ────────────────────────────────
|
| 230 |
-
#
|
| 231 |
-
# ⛔ THE DEBT, in its own words: *"A caught render error tells nobody."* W30-T39 gave the app its
|
| 232 |
-
# first error boundary — measured at the time, `componentDidCatch|getDerivedStateFromError` was
|
| 233 |
-
# **0 hits** across all of `web/src`, so any throw anywhere blanked the whole product. The
|
| 234 |
-
# boundary CONTAINS the failure; it does not REPORT it, so the next user complaint still arrives
|
| 235 |
-
# as a sentence rather than as a stack. That is why owner item 2's crash could not be diagnosed
|
| 236 |
-
# and had to be fixed at both candidate causes.
|
| 237 |
-
#
|
| 238 |
-
# ⚠ IT LIVES HERE, IN THE CHANGE-TOKEN ROUTER, AND THE REASON IS WORTH ONE LINE. It is not about
|
| 239 |
-
# change tokens. It is here because this router is already mounted, already in one lane's fence,
|
| 240 |
-
# and a NEW `routes_*.py` costs a cross-fence mount (contract C2) that this wave has already spent
|
| 241 |
-
# three times — and a fourth finished-but-unmounted router is precisely the failure wave 23 shipped
|
| 242 |
-
# three of. Subject purity would have been bought with the exact risk the contract exists to
|
| 243 |
-
# prevent. ⚠ It performs no `store.get()`, so this file's zero-deep-copy claim (and
|
| 244 |
-
# `verify_live_workspace`'s count of it) is untouched: `telemetry.log` appends one line to a
|
| 245 |
-
# container-local JSONL.
|
| 246 |
-
#
|
| 247 |
-
# ⛔⛔ AND IT CARRIES NO ROW DATA, WHICH IS THE DEBT'S OWN HARD CONDITION. The body is rebuilt KEY
|
| 248 |
-
# BY KEY from exactly two strings; there is no `**body` anywhere below. A crash reporter that
|
| 249 |
-
# forwarded props or state would quietly ship a customer's cell values into a log — the one way
|
| 250 |
-
# this feature could do more harm than the silence it replaces.
|
| 251 |
-
_CLIENT_ERR_MAX = 4 * 1024
|
| 252 |
-
_CLIENT_ERR_WINDOW_S = 60
|
| 253 |
-
_CLIENT_ERR_PER_WINDOW = 12
|
| 254 |
-
_CLIENT_ERR_HITS: dict = {}
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
@router.post("/client-error")
|
| 258 |
-
async def client_error(request: Request, session: Session = Depends(require_session)):
|
| 259 |
-
"""One caught render error, forwarded to telemetry. `{message, componentStack}`.
|
| 260 |
-
|
| 261 |
-
⚠ SESSION-GATED, deliberately. A public crash sink is an unauthenticated append to a log file,
|
| 262 |
-
which is a disk-filling primitive with a URL. Only signed-in accounts can crash this app in a
|
| 263 |
-
way we need to hear about.
|
| 264 |
-
⚠ RATE-LIMITED PER ACCOUNT, because the failure mode is a render LOOP: a boundary that catches,
|
| 265 |
-
resets and re-throws would otherwise post several times a second, for ever, from every open
|
| 266 |
-
tab. Beyond the window the report is DROPPED and the request still answers 200 — the client
|
| 267 |
-
must never learn to retry a report.
|
| 268 |
-
"""
|
| 269 |
-
now = time.time()
|
| 270 |
-
who = session.uname or "?"
|
| 271 |
-
seen = [t for t in _CLIENT_ERR_HITS.get(who, ()) if now - t < _CLIENT_ERR_WINDOW_S]
|
| 272 |
-
seen.append(now)
|
| 273 |
-
_CLIENT_ERR_HITS[who] = seen
|
| 274 |
-
if len(_CLIENT_ERR_HITS) > 2048:
|
| 275 |
-
for k in [k for k, v in _CLIENT_ERR_HITS.items()
|
| 276 |
-
if not v or now - v[-1] > _CLIENT_ERR_WINDOW_S]:
|
| 277 |
-
_CLIENT_ERR_HITS.pop(k, None)
|
| 278 |
-
if len(seen) > _CLIENT_ERR_PER_WINDOW:
|
| 279 |
-
return {"ok": True, "recorded": False}
|
| 280 |
-
|
| 281 |
-
# The body, BOUNDED BY THE STREAM. `Body(...)` would already have been read and parsed before
|
| 282 |
-
# this line ran, and `content-length` is caller-supplied — the same argument `routes_forms`
|
| 283 |
-
# and `routes_publish` both carry.
|
| 284 |
-
size, chunks = 0, []
|
| 285 |
-
async for chunk in request.stream():
|
| 286 |
-
size += len(chunk)
|
| 287 |
-
if size > _CLIENT_ERR_MAX:
|
| 288 |
-
raise err(413, "body_too_large", "that report is too large")
|
| 289 |
-
chunks.append(chunk)
|
| 290 |
-
import json as _json
|
| 291 |
-
try:
|
| 292 |
-
body = _json.loads(b"".join(chunks) or b"{}")
|
| 293 |
-
except ValueError:
|
| 294 |
-
body = {}
|
| 295 |
-
if not isinstance(body, dict):
|
| 296 |
-
body = {}
|
| 297 |
-
|
| 298 |
-
# ⛔ TWO KEYS, BOTH TRUNCATED, NOTHING ELSE. See the block note above.
|
| 299 |
-
message = str(body.get("message") or "")[:500]
|
| 300 |
-
stack = str(body.get("componentStack") or "")[:4000]
|
| 301 |
-
if not message and not stack:
|
| 302 |
-
return {"ok": True, "recorded": False}
|
| 303 |
-
try:
|
| 304 |
-
import harness.telemetry as _tel
|
| 305 |
-
_tel.log("client_error", user=session.uname, tenant=session.tenant,
|
| 306 |
-
message=message, componentStack=stack)
|
| 307 |
-
except Exception: # noqa: BLE001
|
| 308 |
-
# Telemetry may never break a request — `deps.py` carries the same rule at its own site.
|
| 309 |
-
return {"ok": True, "recorded": False}
|
| 310 |
-
return {"ok": True, "recorded": True}
|
|
|
|
| 55 |
cached verdict that this session may open it.
|
| 56 |
"""
|
| 57 |
import time
|
| 58 |
+
from fastapi import APIRouter, Depends
|
| 59 |
|
| 60 |
import core.shared_overlay as shared_overlay
|
| 61 |
import core.store as store
|
|
|
|
| 224 |
# a token frozen at 0 would promise liveness the backend is not delivering.
|
| 225 |
tokens[label] = rev.get("token") if isinstance(rev, dict) else None
|
| 226 |
return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/routes_connectors.py
CHANGED
|
@@ -30,7 +30,7 @@ from deps import Session, require_session
|
|
| 30 |
#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
|
| 31 |
#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
|
| 32 |
#: second correct one here would leave the same two-copies shape that produced the defect.
|
| 33 |
-
from routes_keychain import
|
| 34 |
|
| 35 |
# ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit).
|
| 36 |
# This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory`
|
|
@@ -396,31 +396,15 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 396 |
# reader. A token connector is the case R4 is really about: a member's own Stripe
|
| 397 |
# key is theirs, and the card must say which kind of connection it is looking at.
|
| 398 |
"scope": scopes.get(t["entry"]) or "business",
|
| 399 |
-
#
|
| 400 |
-
#
|
| 401 |
-
#
|
| 402 |
-
#
|
| 403 |
-
#
|
| 404 |
-
#
|
| 405 |
-
#
|
| 406 |
-
#
|
| 407 |
-
|
| 408 |
-
# `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"`
|
| 409 |
-
# row, and its own comment says why in the right words: *"the line is TENANT-WIDE
|
| 410 |
-
# vs PER-USER, not admin vs everyone … what they must never be offered is the
|
| 411 |
-
# KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion
|
| 412 |
-
# is CORRECT and stays green. A member's personal connection is a different door
|
| 413 |
-
# into the same room, so it gets its own word.
|
| 414 |
-
#
|
| 415 |
-
# ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE.
|
| 416 |
-
# `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so
|
| 417 |
-
# offering a personal door for one of those types would be offering a button that
|
| 418 |
-
# 400s — the exact failure R6 removed from the Google card. Today that means
|
| 419 |
-
# Stripe and Shopify open for a member and Meta Ads does not
|
| 420 |
-
# ([[one-question-two-normalizers]]).
|
| 421 |
-
"manage": ("keychain" if is_admin
|
| 422 |
-
else "" if t["entry"] in TENANT_WIDE_TYPES
|
| 423 |
-
else "keychain_personal"),
|
| 424 |
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
|
| 425 |
# key is real and useful, and it is not the same as the data being on screen.
|
| 426 |
"note": "Storing the key connects the account; reading its data lands in a later "
|
|
@@ -582,22 +566,9 @@ def connectors_directory(session: Session = Depends(require_session)):
|
|
| 582 |
"""C11: every connector this platform knows about, with its state for THIS reader.
|
| 583 |
|
| 584 |
Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the
|
| 585 |
-
workspace is connected to
|
| 586 |
-
of R7's per-user slots.
|
| 587 |
-
|
| 588 |
-
⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and
|
| 589 |
-
then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction
|
| 590 |
-
inside a single paragraph, which is how the promise survived three waves without a door behind
|
| 591 |
-
it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row:
|
| 592 |
-
|
| 593 |
-
· `keychain` — the tenant-wide credential door. **Admins only**, and it must stay
|
| 594 |
-
that way: it edits the key the whole workspace reads through.
|
| 595 |
-
· `keychain_personal` — the same room, for a connection that is this reader's own. Offered
|
| 596 |
-
to a non-admin **only** where a personal credential is actually
|
| 597 |
-
allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so
|
| 598 |
-
Odoo and Meta Ads never get it).
|
| 599 |
-
· `""` — nothing this reader can do here; the client renders no control
|
| 600 |
-
rather than a door that would 403.
|
| 601 |
"""
|
| 602 |
# ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
|
| 603 |
# careful and is silently False for every admin on earth — the manage actions would simply
|
|
|
|
| 30 |
#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
|
| 31 |
#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
|
| 32 |
#: second correct one here would leave the same two-copies shape that produced the defect.
|
| 33 |
+
from routes_keychain import env_odoo_available
|
| 34 |
|
| 35 |
# ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit).
|
| 36 |
# This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory`
|
|
|
|
| 396 |
# reader. A token connector is the case R4 is really about: a member's own Stripe
|
| 397 |
# key is theirs, and the card must say which kind of connection it is looking at.
|
| 398 |
"scope": scopes.get(t["entry"]) or "business",
|
| 399 |
+
# ⚠ `manage` STAYS ADMIN-ONLY HERE, deliberately, and it is a smaller door than
|
| 400 |
+
# R4 allows. A member may now hold a PERSONAL connection — the room for it is
|
| 401 |
+
# Settings → Keychains, which W32-T11 opens to them — but this card's action word
|
| 402 |
+
# has been "the tenant-wide credential door" since wave 23 and a sibling gate
|
| 403 |
+
# asserts a member is offered none. Widening it is a one-word change once that
|
| 404 |
+
# assertion is retargeted; booked rather than done, because reddening another
|
| 405 |
+
# lane's gate to add a second door to a room that is already reachable is a bad
|
| 406 |
+
# trade.
|
| 407 |
+
"manage": "keychain" if is_admin else "",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 408 |
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
|
| 409 |
# key is real and useful, and it is not the same as the data being on screen.
|
| 410 |
"note": "Storing the key connects the account; reading its data lands in a later "
|
|
|
|
| 566 |
"""C11: every connector this platform knows about, with its state for THIS reader.
|
| 567 |
|
| 568 |
Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the
|
| 569 |
+
workspace is connected to (and connect their OWN per-user accounts, which is the whole point
|
| 570 |
+
of R7's per-user slots). `manage` is what carries admin-ness — it is empty for a non-admin,
|
| 571 |
+
so the client renders the card without an action rather than offering a door that 403s.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 572 |
"""
|
| 573 |
# ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
|
| 574 |
# careful and is silently False for every admin on earth — the manage actions would simply
|
api/routes_customers.py
CHANGED
|
@@ -209,17 +209,6 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 209 |
# The derived channel: cohort membership cells + measure column values, ONE dict — the
|
| 210 |
# same read-only channel the embed host hands to rows_from_pool.
|
| 211 |
derived = aios_grid.cohort_cells(lists)
|
| 212 |
-
# ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried
|
| 213 |
-
# NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id`
|
| 214 |
-
# as its join key — so the merge would have lost the one value every Odoo document joins on.
|
| 215 |
-
#
|
| 216 |
-
# ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS
|
| 217 |
-
# the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is
|
| 218 |
-
# asserted against Odoo in that module). Adding it to the pool would be a SECOND source for
|
| 219 |
-
# one fact, which is the class of defect item 12 is about. This channel exists for exactly
|
| 220 |
-
# this — a value the host knows per render and the pool has no business storing.
|
| 221 |
-
for pid in pids:
|
| 222 |
-
derived.setdefault(pid, {})["partner_id"] = pid
|
| 223 |
for pid, cells in measure_resolve.column_values(
|
| 224 |
fields, team_id, pids, today, stamp, rt.measure_memo,
|
| 225 |
on_error=_measure_err).items():
|
|
|
|
| 209 |
# The derived channel: cohort membership cells + measure column values, ONE dict — the
|
| 210 |
# same read-only channel the embed host hands to rows_from_pool.
|
| 211 |
derived = aios_grid.cohort_cells(lists)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
for pid, cells in measure_resolve.column_values(
|
| 213 |
fields, team_id, pids, today, stamp, rt.measure_memo,
|
| 214 |
on_error=_measure_err).items():
|
api/routes_keychain.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_oauth.py
CHANGED
|
@@ -1,114 +1,114 @@
|
|
| 1 |
-
"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
|
| 2 |
-
|
| 3 |
-
Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
|
| 4 |
-
shapes and status codes here; every decision that could be wrong lives in the module a gate
|
| 5 |
-
can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
|
| 6 |
-
so the day a second provider lands here is the day nothing in this file changes.
|
| 7 |
-
|
| 8 |
-
MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
|
| 9 |
-
`main.py`, and `routes_automation` is already included there — so this router rides inside it
|
| 10 |
-
(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
|
| 11 |
-
alters no path.
|
| 12 |
-
|
| 13 |
-
⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
|
| 14 |
-
screen** — it is a top-level navigation the client reaches by `<a href>`, never JSON. The
|
| 15 |
-
callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
|
| 16 |
-
`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever
|
| 17 |
-
went wrong rides in the query string; a dead-end error page where the app used to be reads as
|
| 18 |
-
"the product broke", not "the connect failed".
|
| 19 |
-
"""
|
| 20 |
-
import os
|
| 21 |
-
|
| 22 |
-
from fastapi import APIRouter, Depends, Request
|
| 23 |
-
from fastapi.responses import RedirectResponse
|
| 24 |
-
|
| 25 |
-
import oauth_connect
|
| 26 |
-
from deps import Session, err, require_session
|
| 27 |
-
|
| 28 |
-
router = APIRouter(prefix="/oauth")
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
def _redirect_uri(request: Request, provider: str) -> str:
|
| 32 |
-
"""The redirect URI this deployment registers at the provider — env-pinned when the
|
| 33 |
-
container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
|
| 34 |
-
the request. MUST match a console-registered URI verbatim, so it is computed in exactly
|
| 35 |
-
one place.
|
| 36 |
-
|
| 37 |
-
⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
|
| 38 |
-
the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
|
| 39 |
-
the request-derived fallback below is effectively dev-only.
|
| 40 |
-
⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
|
| 41 |
-
this returns is where the provider sends the user BACK, and the session cookie is host-only
|
| 42 |
-
(`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host
|
| 43 |
-
the user actually browsed plants the session on the wrong hostname and they return logged
|
| 44 |
-
out. Moving the app to a new hostname means moving this value AND re-registering the
|
| 45 |
-
resulting URI in the provider console; one without the other fails closed.
|
| 46 |
-
Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
|
| 47 |
-
base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
|
| 48 |
-
if not base:
|
| 49 |
-
base = f"{request.url.scheme}://{request.url.netloc}"
|
| 50 |
-
return f"{base}/api/v1/oauth/{provider}/callback"
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
@router.get("/status")
|
| 54 |
-
def oauth_status(session: Session = Depends(require_session)):
|
| 55 |
-
"""C5's status shape for the SESSION user, one entry per registry provider:
|
| 56 |
-
`{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
|
| 57 |
-
`ready` reads through."""
|
| 58 |
-
return oauth_connect.status(session.runtime, session.uname)
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
def _offered_or_404(provider: str):
|
| 62 |
-
"""⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
|
| 63 |
-
|
| 64 |
-
The owner pasted the failure this replaces: clicking Connect on the Google card answered
|
| 65 |
-
`503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is
|
| 66 |
-
not offered at all, so the honest status is the one for a URL that does not exist. `404`
|
| 67 |
-
rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
|
| 68 |
-
coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
|
| 69 |
-
|
| 70 |
-
⚠ Every door in this router goes through it, START included, because the JSON the owner saw
|
| 71 |
-
came from the start route and a guard on one leg is a guard on one leg.
|
| 72 |
-
"""
|
| 73 |
-
if not oauth_connect.offered(provider):
|
| 74 |
-
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
@router.get("/{provider}/start")
|
| 78 |
-
def oauth_start(provider: str, request: Request, next: str = "",
|
| 79 |
-
session: Session = Depends(require_session)):
|
| 80 |
-
"""302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the
|
| 81 |
-
RELATIVE path the callback returns the browser to; it rides inside the single-use state,
|
| 82 |
-
sanitised, so the round trip cannot be steered off-origin."""
|
| 83 |
-
_offered_or_404(provider)
|
| 84 |
-
url, problem = oauth_connect.start(provider, session.uname,
|
| 85 |
-
_redirect_uri(request, provider), next_path=next)
|
| 86 |
-
if problem:
|
| 87 |
-
raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
|
| 88 |
-
return RedirectResponse(url, status_code=302)
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
@router.get("/{provider}/callback")
|
| 92 |
-
def oauth_callback(provider: str, request: Request,
|
| 93 |
-
session: Session = Depends(require_session),
|
| 94 |
-
state: str = "", code: str = "", error: str = ""):
|
| 95 |
-
"""The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
|
| 96 |
-
the browser back to the state's return path — connected or not (see module header)."""
|
| 97 |
-
_offered_or_404(provider)
|
| 98 |
-
if error:
|
| 99 |
-
home = "/#/"
|
| 100 |
-
return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
|
| 101 |
-
email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
|
| 102 |
-
sep = "&" if "?" in home else "?"
|
| 103 |
-
if problem:
|
| 104 |
-
return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
|
| 105 |
-
return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
@router.post("/{provider}/disconnect")
|
| 109 |
-
def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
|
| 110 |
-
_offered_or_404(provider)
|
| 111 |
-
if oauth_connect.provider_def(provider) is None:
|
| 112 |
-
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 113 |
-
oauth_connect.disconnect(session.runtime, session.uname, provider)
|
| 114 |
-
return {"disconnected": provider}
|
|
|
|
| 1 |
+
"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
|
| 2 |
+
|
| 3 |
+
Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
|
| 4 |
+
shapes and status codes here; every decision that could be wrong lives in the module a gate
|
| 5 |
+
can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
|
| 6 |
+
so the day a second provider lands here is the day nothing in this file changes.
|
| 7 |
+
|
| 8 |
+
MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
|
| 9 |
+
`main.py`, and `routes_automation` is already included there — so this router rides inside it
|
| 10 |
+
(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
|
| 11 |
+
alters no path.
|
| 12 |
+
|
| 13 |
+
⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
|
| 14 |
+
screen** — it is a top-level navigation the client reaches by `<a href>`, never JSON. The
|
| 15 |
+
callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
|
| 16 |
+
`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever
|
| 17 |
+
went wrong rides in the query string; a dead-end error page where the app used to be reads as
|
| 18 |
+
"the product broke", not "the connect failed".
|
| 19 |
+
"""
|
| 20 |
+
import os
|
| 21 |
+
|
| 22 |
+
from fastapi import APIRouter, Depends, Request
|
| 23 |
+
from fastapi.responses import RedirectResponse
|
| 24 |
+
|
| 25 |
+
import oauth_connect
|
| 26 |
+
from deps import Session, err, require_session
|
| 27 |
+
|
| 28 |
+
router = APIRouter(prefix="/oauth")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _redirect_uri(request: Request, provider: str) -> str:
|
| 32 |
+
"""The redirect URI this deployment registers at the provider — env-pinned when the
|
| 33 |
+
container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
|
| 34 |
+
the request. MUST match a console-registered URI verbatim, so it is computed in exactly
|
| 35 |
+
one place.
|
| 36 |
+
|
| 37 |
+
⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
|
| 38 |
+
the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
|
| 39 |
+
the request-derived fallback below is effectively dev-only.
|
| 40 |
+
⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
|
| 41 |
+
this returns is where the provider sends the user BACK, and the session cookie is host-only
|
| 42 |
+
(`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host
|
| 43 |
+
the user actually browsed plants the session on the wrong hostname and they return logged
|
| 44 |
+
out. Moving the app to a new hostname means moving this value AND re-registering the
|
| 45 |
+
resulting URI in the provider console; one without the other fails closed.
|
| 46 |
+
Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
|
| 47 |
+
base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
|
| 48 |
+
if not base:
|
| 49 |
+
base = f"{request.url.scheme}://{request.url.netloc}"
|
| 50 |
+
return f"{base}/api/v1/oauth/{provider}/callback"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@router.get("/status")
|
| 54 |
+
def oauth_status(session: Session = Depends(require_session)):
|
| 55 |
+
"""C5's status shape for the SESSION user, one entry per registry provider:
|
| 56 |
+
`{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
|
| 57 |
+
`ready` reads through."""
|
| 58 |
+
return oauth_connect.status(session.runtime, session.uname)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _offered_or_404(provider: str):
|
| 62 |
+
"""⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
|
| 63 |
+
|
| 64 |
+
The owner pasted the failure this replaces: clicking Connect on the Google card answered
|
| 65 |
+
`503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is
|
| 66 |
+
not offered at all, so the honest status is the one for a URL that does not exist. `404`
|
| 67 |
+
rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
|
| 68 |
+
coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
|
| 69 |
+
|
| 70 |
+
⚠ Every door in this router goes through it, START included, because the JSON the owner saw
|
| 71 |
+
came from the start route and a guard on one leg is a guard on one leg.
|
| 72 |
+
"""
|
| 73 |
+
if not oauth_connect.offered(provider):
|
| 74 |
+
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
@router.get("/{provider}/start")
|
| 78 |
+
def oauth_start(provider: str, request: Request, next: str = "",
|
| 79 |
+
session: Session = Depends(require_session)):
|
| 80 |
+
"""302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the
|
| 81 |
+
RELATIVE path the callback returns the browser to; it rides inside the single-use state,
|
| 82 |
+
sanitised, so the round trip cannot be steered off-origin."""
|
| 83 |
+
_offered_or_404(provider)
|
| 84 |
+
url, problem = oauth_connect.start(provider, session.uname,
|
| 85 |
+
_redirect_uri(request, provider), next_path=next)
|
| 86 |
+
if problem:
|
| 87 |
+
raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
|
| 88 |
+
return RedirectResponse(url, status_code=302)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
@router.get("/{provider}/callback")
|
| 92 |
+
def oauth_callback(provider: str, request: Request,
|
| 93 |
+
session: Session = Depends(require_session),
|
| 94 |
+
state: str = "", code: str = "", error: str = ""):
|
| 95 |
+
"""The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
|
| 96 |
+
the browser back to the state's return path — connected or not (see module header)."""
|
| 97 |
+
_offered_or_404(provider)
|
| 98 |
+
if error:
|
| 99 |
+
home = "/#/"
|
| 100 |
+
return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
|
| 101 |
+
email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
|
| 102 |
+
sep = "&" if "?" in home else "?"
|
| 103 |
+
if problem:
|
| 104 |
+
return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
|
| 105 |
+
return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@router.post("/{provider}/disconnect")
|
| 109 |
+
def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
|
| 110 |
+
_offered_or_404(provider)
|
| 111 |
+
if oauth_connect.provider_def(provider) is None:
|
| 112 |
+
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 113 |
+
oauth_connect.disconnect(session.runtime, session.uname, provider)
|
| 114 |
+
return {"disconnected": provider}
|
api/routes_odoo_tables.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_query.py
CHANGED
|
@@ -1,713 +1,693 @@
|
|
| 1 |
-
"""routes_query.py — WAVE 32 item 7 (ruling R1, contract C5): the Query module's doors.
|
| 2 |
-
|
| 3 |
-
POST /api/v1/query/build {question, scope} -> {spec, explain, refused} WRITES NOTHING
|
| 4 |
-
POST /api/v1/query/save {question, scope, spec} -> the saved view
|
| 5 |
-
GET /api/v1/query -> the AI-built views this caller may open
|
| 6 |
-
DELETE /api/v1/query/{qid} -> forget one
|
| 7 |
-
|
| 8 |
-
**The AI turns a question about ONE granted database into a view SPEC. It may only build what the
|
| 9 |
-
backend already supports** (R1), and the wall on which database it may name is the EXISTING one.
|
| 10 |
-
|
| 11 |
-
⛔ THE FOUNDATION IS NOT `harness/semantic.py` — MEASURED, W32-T50
|
| 12 |
-
(`.claude/wiki/waves/wave32/proto/query-spike/README.md`). That layer is bound to 8 hand-authored
|
| 13 |
-
YAML topics over Odoo mirror tables: **0** are `ut_`-bound, and a `ut_` key answers
|
| 14 |
-
`ModelError: topic 'ut_…' has no store binding` from both `store_columns` and `store_query`. Its
|
| 15 |
-
own `save_view` writes `platform/data/store/views.json`, a chart workspace the product's grid
|
| 16 |
-
cannot read. What it lends this file is its POSTURE, quoted from its system prompt: *the model
|
| 17 |
-
never writes SQL and never invents a field name — it only picks registry keys.*
|
| 18 |
-
|
| 19 |
-
The real foundation is the view stack that already ships, and this module is `routes_templates.py`
|
| 20 |
-
with a model where the curated registry used to be:
|
| 21 |
-
|
| 22 |
-
· the wall `routes_templates._target_or_refuse` (imported, never re-implemented)
|
| 23 |
-
· the column contract `core.view_templates.columns_named` / `missing_columns`
|
| 24 |
-
· the view skeleton `core.view_templates._view` (one definition of what a view is)
|
| 25 |
-
· the validators `aios_grid._clean_display` / `clean_filter_tree`
|
| 26 |
-
· the writer `core.table_store.make(ws).save_view(..., shared=False)`
|
| 27 |
-
|
| 28 |
-
⛔⛔ THE VALIDATORS DROP; THEY DO NOT REFUSE — and C5 requires a REFUSAL. `_clean_display` returns
|
| 29 |
-
None for an unknown mode, and a view with no display block IS A GRID; `clean_filter_tree` drops a
|
| 30 |
-
leaf naming an unknown column, and a view whose only condition was dropped SHOWS EVERY ROW under a
|
| 31 |
-
name promising a shortlist (`view_templates`' own scar, and `_seed_wave17`'s before it). So
|
| 32 |
-
membership is tested HERE, first and explicitly, and the cleaners run second as a belt. They are
|
| 33 |
-
the vocabulary ORACLE, never the refusal.
|
| 34 |
-
|
| 35 |
-
⛔ THE REFUSAL PATH IS A MODEL PROPERTY, so this module declares its OWN provider order rather than
|
| 36 |
-
inheriting `analyst.available_providers()`' cheap-first one. Measured over 4 arms x 9 questions
|
| 37 |
-
(`proto/query-spike/refusal-arms.json`): `cerebras/gpt-oss-120b` refuses 4/4 unanswerable questions
|
| 38 |
-
with a usable sentence; `groq/llama-3.3-70b` refuses **0/4** and instead emits a fully valid spec
|
| 39 |
-
answering a different question ("Revenue by Creator" against a database holding no revenue).
|
| 40 |
-
Changing refusal from an enum value to its own tool changed cerebras not at all and made groq
|
| 41 |
-
worse.
|
| 42 |
-
⚠ AND THE LADDER STILL FALLS THROUGH ON A 429, WHICH IS A COMPROMISE, NOT AN OVERSIGHT — stated
|
| 43 |
-
here because the first draft of this header claimed the opposite. A door that dies whenever one
|
| 44 |
-
free tier rate-limits is worse than one that answers with a weaker model behind the SAME validator
|
| 45 |
-
and the SAME human read-back. What "fail closed" means precisely: when the whole ladder is
|
| 46 |
-
unreachable the answer is a sentence and never a view. Which provider answered rides back in the
|
| 47 |
-
response so a wrong answer stays diagnosable (`_call_model`'s note).
|
| 48 |
-
|
| 49 |
-
⛔ AND A PROVIDER `400` CAN BE THE MODEL'S REFUSAL. Groq validates the model's tool arguments
|
| 50 |
-
against the tool schema server-side and answers `400 tool_use_failed` carrying the model's real
|
| 51 |
-
output in `failed_generation` — so a refusal that omits a `required` property arrives as an HTTP
|
| 52 |
-
error. `analyst._live_chat` calls `raise_for_status()` and discards that body, which is why this
|
| 53 |
-
file has its own small ladder: `required` is `["kind"]` ALONE (a refusal must be a legal argument
|
| 54 |
-
object), and a 400 body is READ before it is treated as transport.
|
| 55 |
-
|
| 56 |
-
⚠ WHAT NO VALIDATOR CAN SEE, stated so nobody thinks the gate covers it: a spec whose every column
|
| 57 |
-
is real, every op legal, kind in the vocabulary and target granted, which answers a DIFFERENT
|
| 58 |
-
QUESTION. That class is invisible to server-side validation. The mechanism against it is C5's own
|
| 59 |
-
shape — `build` returns `{spec, explain}` and WRITES NOTHING, `explain` is DERIVED from the
|
| 60 |
-
validated spec (never asked of the model, which would let it flatter its own output), and saving is
|
| 61 |
-
a second, explicit call. The person is the last validator, by design.
|
| 62 |
-
"""
|
| 63 |
-
import datetime as _dt
|
| 64 |
-
import hashlib
|
| 65 |
-
import json
|
| 66 |
-
import os
|
| 67 |
-
import re
|
| 68 |
-
|
| 69 |
-
from fastapi import APIRouter, Body, Depends
|
| 70 |
-
|
| 71 |
-
from deps import Session, err, require_session
|
| 72 |
-
|
| 73 |
-
router = APIRouter(prefix="/api/v1")
|
| 74 |
-
|
| 75 |
-
#: The store key holding the REGISTRY of AI-built views, per tenant: {qid: {...}}. The view itself
|
| 76 |
-
#: is a real saved view in its target's own workspace — this is only the index, so listing the
|
| 77 |
-
#: Query module never has to read every granted table's workspace to find them. Same split
|
| 78 |
-
#: `core/alerts.py` draws between definitions and the inbox.
|
| 79 |
-
QUERY_KEY = 'query_views'
|
| 80 |
-
|
| 81 |
-
MAX_QUESTION = 500
|
| 82 |
-
MAX_VISIBLE = 24
|
| 83 |
-
MAX_REGISTRY = 200 #: a working list, not an archive (alerts' MAX_NOTIFICATIONS rule)
|
| 84 |
-
|
| 85 |
-
#: ⭐ THE KINDS THE AI MAY EMIT. Held here as a NAMED SET, and `verify_query.py` asserts it is a
|
| 86 |
-
#: subset of BOTH `aios_grid.DISPLAY_MODES` (the stored vocabulary) and the client's
|
| 87 |
-
#: `CREATABLE_MODES` (which means "has a mounted renderer" — `verify_icons::mode_parity` enforces
|
| 88 |
-
#: that both ways), AND that the excluded remainder is exactly `QUERY_EXCLUDED`. So a mode that
|
| 89 |
-
#: lands in `CREATABLE_MODES` later REDS the gate instead of being silently un-offered — the
|
| 90 |
-
#: `swipe`/`form` failure ("whole, correct and unreachable") in reverse.
|
| 91 |
-
QUERY_KINDS = ('grid', 'chart', 'calendar', 'kanban', 'timeseries', 'map', 'list')
|
| 92 |
-
|
| 93 |
-
#: Creatable modes the AI deliberately may NOT emit, each with the reason it cannot be derived
|
| 94 |
-
#: from a question about existing records. A reason per member, because "excluded" with no cause
|
| 95 |
-
#: is how a hold outlives its own justification.
|
| 96 |
-
QUERY_EXCLUDED = {
|
| 97 |
-
'form': "a form is a door records come IN through and shows none — a question about existing "
|
| 98 |
-
"records cannot describe one, and its spec carries emails and a public token",
|
| 99 |
-
'catalog': "a catalog is a published artifact needing page authoring and product codes "
|
| 100 |
-
"(MAX_CATALOG_CODES is spent in page order), not a shape of the current rows",
|
| 101 |
-
'swipe': "a swipe deck needs an empty single-select plus a chosen option per direction — two "
|
| 102 |
-
"product decisions a question does not contain",
|
| 103 |
-
}
|
| 104 |
-
|
| 105 |
-
#: Mode-required refs, and the field FAMILY each one demands. ⚠ STRICTER THAN `_clean_display` ON
|
| 106 |
-
#: PURPOSE. That function accepts any ref naming a real field and leaves the meaning to the client,
|
| 107 |
-
#: which "degrades a wrong-typed ref to that surface's default rather than to an error" — correct
|
| 108 |
-
#: for a person clicking a picker who SEES the result, wrong for a generated spec nobody has looked
|
| 109 |
-
#: at yet. Measured: a free model stacked a kanban on `category` (a text column) for a contract
|
| 110 |
-
#: with ZERO select columns, and every validator passed it.
|
| 111 |
-
_MODE_REFS = {
|
| 112 |
-
'kanban': [('stackField', ('select',), True)],
|
| 113 |
-
'calendar': [('dateField', ('date',), True)],
|
| 114 |
-
'timeseries': [('dateField', ('date',), True)],
|
| 115 |
-
'map': [('colorField', ('select',), False), ('sizeField', ('int', 'currency', 'pct'), False)],
|
| 116 |
-
}
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
def _now_iso():
|
| 120 |
-
"""UTC WITH the offset — a naive stamp is unsubtractable in a browser (D-18, alerts' rule)."""
|
| 121 |
-
return _dt.datetime.now(_dt.timezone.utc).isoformat()
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
def _grid():
|
| 125 |
-
import aios_grid
|
| 126 |
-
return aios_grid
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
def _templates():
|
| 130 |
-
import core.view_templates as view_templates
|
| 131 |
-
return view_templates
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
def _target(session, scope):
|
| 135 |
-
"""`(fields, source)` for a database this session may open — or a refusal.
|
| 136 |
-
|
| 137 |
-
⛔ THE WALL IS `routes_templates._target_or_refuse`, IMPORTED AND NOT COPIED. It is the same
|
| 138 |
-
predicate the nav uses, per topic (`ut_*` -> `user_tables.may_open`; a built-in -> the module
|
| 139 |
-
grant `session.require`; anything else a 404 that is not a directory of what exists), and T51's
|
| 140 |
-
instruction is explicit that the wall is re-used, never re-implemented beside itself. The
|
| 141 |
-
private name is deliberate: an alias would mean editing a file no lane's fence covers.
|
| 142 |
-
|
| 143 |
-
It returns field KEYS; this door also needs each field's TYPE (for the prompt and for
|
| 144 |
-
`_MODE_REFS`), so the types are read from the same source the wall consulted — never from a
|
| 145 |
-
list assembled here.
|
| 146 |
-
"""
|
| 147 |
-
import routes_templates
|
| 148 |
-
keys, source = routes_templates._target_or_refuse(session, scope)
|
| 149 |
-
key = str(scope or '').strip()
|
| 150 |
-
defs = []
|
| 151 |
-
if key.startswith('ut_'):
|
| 152 |
-
import core.user_tables as user_tables
|
| 153 |
-
defn = user_tables.get(key, st=session.runtime) or {}
|
| 154 |
-
by_key = {str(f.get('key')): f for f in (defn.get('fields') or []) if f.get('key')}
|
| 155 |
-
else:
|
| 156 |
-
g = _grid()
|
| 157 |
-
raw = g.product_fields() if key == 'product_data' else g.FIELDS
|
| 158 |
-
by_key = {str(f.get('key')): f for f in raw if f.get('key')}
|
| 159 |
-
for k in keys:
|
| 160 |
-
f = by_key.get(k) or {}
|
| 161 |
-
defs.append({'key': k, 'label': str(f.get('label') or k)[:80],
|
| 162 |
-
'type': str(f.get('type') or 'text')})
|
| 163 |
-
return defs, source
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
# ── the model ───────────────────────────────────────────────────────────────────────────────
|
| 167 |
-
|
| 168 |
-
#: This module's OWN provider order — cerebras first. See the header: the refusal is a model
|
| 169 |
-
#: property, and the ladder's cheap-first order is exactly wrong for a contract that includes one.
|
| 170 |
-
#: Override with `AIOS_QUERY_PROVIDER`. The credentials themselves stay in ONE place
|
| 171 |
-
#: (`analyst.PROVIDERS`), so a rotated key or a new model reaches this file too.
|
| 172 |
-
QUERY_PROVIDER_ORDER = ('cerebras', 'groq', 'openrouter')
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
def _providers():
|
| 176 |
-
import harness.analyst as analyst
|
| 177 |
-
pin = os.environ.get("AIOS_QUERY_PROVIDER")
|
| 178 |
-
by_name = {p["name"]: p for p in analyst.PROVIDERS}
|
| 179 |
-
order = [pin] if pin else list(QUERY_PROVIDER_ORDER)
|
| 180 |
-
return [by_name[n] for n in order
|
| 181 |
-
if n in by_name and os.environ.get(by_name[n]["env"])]
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
def _spec_schema(field_keys):
|
| 185 |
-
"""The tool schema the model fills. ⛔ `required` IS `["kind"]` ALONE — see the header: a
|
| 186 |
-
refusal carries no name, and groq rejects a schema-violating refusal with a 400 that reads
|
| 187 |
-
exactly like a transport failure."""
|
| 188 |
-
ops = sorted(_grid().FILTER_OPS)
|
| 189 |
-
col = {"type": "string", "enum": sorted(field_keys)}
|
| 190 |
-
return {
|
| 191 |
-
"type": "object",
|
| 192 |
-
"properties": {
|
| 193 |
-
"kind": {"type": "string", "enum": list(QUERY_KINDS) + ["refused"],
|
| 194 |
-
"description": "refused = this database cannot answer the question"},
|
| 195 |
-
"name": {"type": "string", "description": "a short title for the view"},
|
| 196 |
-
"refusal": {"type": "string",
|
| 197 |
-
"description": "when kind=refused: ONE plain sentence naming what is "
|
| 198 |
-
"missing"},
|
| 199 |
-
"visible": {"type": "array", "items": col},
|
| 200 |
-
"filters": {"type": "array", "items": {
|
| 201 |
-
"type": "object",
|
| 202 |
-
"properties": {"colId": col, "op": {"type": "string", "enum": ops},
|
| 203 |
-
"value": {"type": "string"}},
|
| 204 |
-
"required": ["colId", "op"]}},
|
| 205 |
-
"filterConj": {"type": "string", "enum": ["and", "or"]},
|
| 206 |
-
"sorts": {"type": "array", "items": {
|
| 207 |
-
"type": "object",
|
| 208 |
-
"properties": {"colId": col,
|
| 209 |
-
"dir": {"type": "string", "enum": ["asc", "desc"]}},
|
| 210 |
-
"required": ["colId", "dir"]}},
|
| 211 |
-
"groupBy": col,
|
| 212 |
-
"stackField": col,
|
| 213 |
-
"dateField": col,
|
| 214 |
-
"colorField": col,
|
| 215 |
-
"sizeField": col,
|
| 216 |
-
},
|
| 217 |
-
"required": ["kind"],
|
| 218 |
-
}
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
def _system_prompt(fields):
|
| 222 |
-
"""The standing instructions. The vocabulary is DERIVED (the kinds set, `FILTER_OPS`, the
|
| 223 |
-
target's own columns) — the model is never told about a key this server cannot honour."""
|
| 224 |
-
cols = json.dumps([{"key": f["key"], "label": f["label"], "type": f["type"]}
|
| 225 |
-
for f in fields], separators=(",", ":"))
|
| 226 |
-
kinds = ", ".join(QUERY_KINDS)
|
| 227 |
-
ops = ", ".join(sorted(_grid().FILTER_OPS))
|
| 228 |
-
selects = [f["key"] for f in fields if f["type"] in ('select', 'multiselect')] or ["(none)"]
|
| 229 |
-
dates = [f["key"] for f in fields if f["type"] == 'date'] or ["(none)"]
|
| 230 |
-
return f"""You turn a question about ONE database into a VIEW SPEC. You never write SQL and you
|
| 231 |
-
never invent a column or a kind of view.
|
| 232 |
-
|
| 233 |
-
THE DATABASE ({len(fields)} columns) — you may name ONLY these column keys:
|
| 234 |
-
{cols}
|
| 235 |
-
|
| 236 |
-
VIEW KINDS you may name, and nothing else: {kinds}
|
| 237 |
-
FILTER OPS you may name, and nothing else: {ops}
|
| 238 |
-
The only select-type columns (a kanban MUST stack on one of these): {", ".join(selects)}
|
| 239 |
-
The only date columns (a calendar or timeseries MUST use one of these): {", ".join(dates)}
|
| 240 |
-
|
| 241 |
-
RULES:
|
| 242 |
-
1. Emit ONE call to build_view. Every colId in visible/filters/sorts/groupBy and every field ref
|
| 243 |
-
MUST be one of the keys above.
|
| 244 |
-
2. Pick the kind by what the question asks for: a list of records -> grid (or list); totals or
|
| 245 |
-
counts by a category -> chart with groupBy; cards by a stage -> kanban (stackField REQUIRED,
|
| 246 |
-
and it must be a select column); something over time -> calendar or timeseries (dateField
|
| 247 |
-
REQUIRED, and it must be a date column); geography -> map.
|
| 248 |
-
3. `visible` is the columns the question is about, most important first — not every column.
|
| 249 |
-
4. Set kind="refused" and put ONE plain sentence in `refusal` when the question needs a column
|
| 250 |
-
this database does not have, a number it does not store, or a picture that is not in the kind
|
| 251 |
-
list. REFUSING IS A CORRECT ANSWER. A view that answers a DIFFERENT question than the one
|
| 252 |
-
asked is the worst possible answer — worse than saying you cannot build it. When in doubt,
|
| 253 |
-
refuse.
|
| 254 |
-
5. Every `value` is a string."""
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
def _call_model(question, fields, chat=None):
|
| 258 |
-
"""`(spec, error_sentence, provider)` — one bounded call down THIS module's ladder.
|
| 259 |
-
|
| 260 |
-
`chat` is injectable so `verify_query.py` proves this door end to end with no key and no
|
| 261 |
-
tokens (`analyst.ask`'s own posture, and the reason its loop is testable offline).
|
| 262 |
-
|
| 263 |
-
⚠ THE LADDER IS A DELIBERATE COMPROMISE, AND THE PROVIDER RIDES BACK BECAUSE OF IT. Only the
|
| 264 |
-
FIRST provider is measured to refuse honestly (README section 4b); the ones below it answer
|
| 265 |
-
unanswerable questions with plausible specs. Falling through on a 429 is still the right call —
|
| 266 |
-
a Query module that dies whenever one free tier rate-limits is worse than one that answers with
|
| 267 |
-
a weaker model behind the same validator and the same human read-back — but *which* model
|
| 268 |
-
answered must be knowable, or a wrong answer is undiagnosable after the fact. Failing closed is
|
| 269 |
-
what happens when the whole ladder is unreachable, not when the best rung is.
|
| 270 |
-
"""
|
| 271 |
-
tools = [{"type": "function", "function": {
|
| 272 |
-
"name": "build_view",
|
| 273 |
-
"description": "Emit the view spec, or refuse.",
|
| 274 |
-
"parameters": _spec_schema([f["key"] for f in fields])}}]
|
| 275 |
-
messages = [{"role": "system", "content": _system_prompt(fields)},
|
| 276 |
-
{"role": "user", "content": question}]
|
| 277 |
-
if chat is not None:
|
| 278 |
-
return chat(messages, tools), None, "injected"
|
| 279 |
-
|
| 280 |
-
provs = _providers()
|
| 281 |
-
if not provs:
|
| 282 |
-
# No key at all: say so. An AI feature that silently does nothing is indistinguishable
|
| 283 |
-
# from one that was never built ([[flag-shipped-without-its-writer]]).
|
| 284 |
-
return None, ("the assistant is not configured on this deployment, so a view cannot be "
|
| 285 |
-
"built from a question yet"), None
|
| 286 |
-
import requests
|
| 287 |
-
last = None
|
| 288 |
-
for p in provs:
|
| 289 |
-
try:
|
| 290 |
-
r = requests.post(p["url"], timeout=60,
|
| 291 |
-
headers={"Authorization": f"Bearer {os.environ[p['env']]}"},
|
| 292 |
-
json={"model": p["model"], "messages": messages, "tools": tools,
|
| 293 |
-
"tool_choice": "required", "temperature": 0.1,
|
| 294 |
-
"max_tokens": 1200})
|
| 295 |
-
except Exception as e: # network blip — try the next provider
|
| 296 |
-
last = f"{p['name']}: {type(e).__name__}"
|
| 297 |
-
continue
|
| 298 |
-
if r.status_code == 400:
|
| 299 |
-
# ⛔ THE MODEL'S ANSWER CAN BE IN HERE. Groq validates tool arguments against the
|
| 300 |
-
# schema and returns the generation it rejected. Reading it turns a "transport
|
| 301 |
-
# failure" back into the refusal it actually was.
|
| 302 |
-
spec = _spec_from_400(r.text)
|
| 303 |
-
if spec is not None:
|
| 304 |
-
return spec, None, p["name"]
|
| 305 |
-
last = f"{p['name']}: 400"
|
| 306 |
-
continue
|
| 307 |
-
if r.status_code != 200:
|
| 308 |
-
last = f"{p['name']}: HTTP {r.status_code}"
|
| 309 |
-
continue # 429 included: the NEXT provider, not a sleep
|
| 310 |
-
try:
|
| 311 |
-
msg = r.json()["choices"][0]["message"]
|
| 312 |
-
calls = msg.get("tool_calls") or []
|
| 313 |
-
if not calls:
|
| 314 |
-
last = f"{p['name']}: no tool call"
|
| 315 |
-
continue
|
| 316 |
-
return json.loads(calls[0]["function"].get("arguments") or "{}"), None, p["name"]
|
| 317 |
-
except Exception as e:
|
| 318 |
-
last = f"{p['name']}: unreadable answer ({type(e).__name__})"
|
| 319 |
-
continue
|
| 320 |
-
# ⛔ FAIL CLOSED. The first provider in the order is the one measured to refuse honestly; if it
|
| 321 |
-
# is rate-limited and the rest are too, the answer is a sentence, never a fabricated view.
|
| 322 |
-
return None, ("the assistant could not be reached just now (" + (last or "no provider") +
|
| 323 |
-
") — try the question again in a moment"), None
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
_FAILED_GEN = re.compile(r'<function=build_view>(\{.*?\})\s*</function>', re.S)
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
def _spec_from_400(body):
|
| 330 |
-
"""The spec out of a `tool_use_failed` body's `failed_generation`, or None."""
|
| 331 |
-
try:
|
| 332 |
-
err_obj = (json.loads(body) or {}).get("error") or {}
|
| 333 |
-
except Exception:
|
| 334 |
-
return None
|
| 335 |
-
gen = str(err_obj.get("failed_generation") or "")
|
| 336 |
-
if not gen:
|
| 337 |
-
return None
|
| 338 |
-
m = _FAILED_GEN.search(gen)
|
| 339 |
-
raw = m.group(1) if m else gen.strip()
|
| 340 |
-
try:
|
| 341 |
-
spec = json.loads(raw)
|
| 342 |
-
except Exception:
|
| 343 |
-
return None
|
| 344 |
-
return spec if isinstance(spec, dict) else None
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
# ── validation: the REFUSAL this contract promises ───────────────────────────────────────────
|
| 348 |
-
|
| 349 |
-
def _validate(spec, fields):
|
| 350 |
-
"""`(clean, refusal, code)` — exactly one of `clean` / `refusal` is set.
|
| 351 |
-
|
| 352 |
-
Membership FIRST and explicitly (the header's whole subject), then the product's own cleaners
|
| 353 |
-
as a belt. A refusal is a SENTENCE, because C5's user-facing promise is a plain sentence and
|
| 354 |
-
never a broken view.
|
| 355 |
-
"""
|
| 356 |
-
g = _grid()
|
| 357 |
-
if not isinstance(spec, dict):
|
| 358 |
-
return None, "the assistant did not answer with a view", "no_spec"
|
| 359 |
-
by_key = {f['key']: f['type'] for f in fields}
|
| 360 |
-
keys = set(by_key)
|
| 361 |
-
kind = spec.get('kind')
|
| 362 |
-
|
| 363 |
-
if kind == 'refused':
|
| 364 |
-
return None, (str(spec.get('refusal') or '').strip()
|
| 365 |
-
or "this database cannot answer that question"), "model_refused"
|
| 366 |
-
if kind in QUERY_EXCLUDED:
|
| 367 |
-
return None, (f"a {kind} view cannot be built from a question — "
|
| 368 |
-
+ QUERY_EXCLUDED[kind]), "unsupported_kind"
|
| 369 |
-
if kind not in QUERY_KINDS:
|
| 370 |
-
# ⛔ NOT `_clean_display`'s job: it would DROP this and hand back a grid under a name
|
| 371 |
-
# promising something else.
|
| 372 |
-
return None, (f"{kind!r} is not a kind of view this product can build"
|
| 373 |
-
if kind else "the assistant did not name a kind of view"), "unsupported_kind"
|
| 374 |
-
|
| 375 |
-
named = set(spec.get('visible') or ())
|
| 376 |
-
for s in spec.get('sorts') or ():
|
| 377 |
-
if isinstance(s, dict) and s.get('colId'):
|
| 378 |
-
named.add(str(s['colId']))
|
| 379 |
-
for f in spec.get('filters') or ():
|
| 380 |
-
if isinstance(f, dict) and f.get('colId'):
|
| 381 |
-
named.add(str(f['colId']))
|
| 382 |
-
if spec.get('groupBy'):
|
| 383 |
-
named.add(str(spec['groupBy']))
|
| 384 |
-
for ref in ('stackField', 'dateField', 'colorField', 'sizeField'):
|
| 385 |
-
if spec.get(ref):
|
| 386 |
-
named.add(str(spec[ref]))
|
| 387 |
-
missing = sorted(named - keys)
|
| 388 |
-
if missing:
|
| 389 |
-
# NAMED, not counted (`routes_templates`' rule): the column keys are what tell the reader
|
| 390 |
-
# this is the wrong database for this question.
|
| 391 |
-
return None, ("this database does not have the columns that answer needs: "
|
| 392 |
-
+ ", ".join(missing)), "unknown_columns"
|
| 393 |
-
|
| 394 |
-
visible = [k for k in (spec.get('visible') or ()) if k in keys][:MAX_VISIBLE]
|
| 395 |
-
if not visible:
|
| 396 |
-
# An empty projection is not a view. `_default_view_config` would fill one in, which is
|
| 397 |
-
# how a question about two columns becomes a 285-column wall.
|
| 398 |
-
return None, ("that question did not name anything to show — try naming the columns you "
|
| 399 |
-
"want to see"), "no_columns"
|
| 400 |
-
|
| 401 |
-
# ⛔ D-137, GUARDED WHERE IT REACHES THIS FEATURE. "The chart cannot key a SET-LIKE or CHECKBOX
|
| 402 |
-
# column the way the grid does" — a set row lands in several buckets, so a sum means something
|
| 403 |
-
# else, and a checkbox reads "(blank)". The defect lives in the CLIENT's `chartData` bucket
|
| 404 |
-
# keying, which is in no lane's fence this wave (booked, with that reason). What IS in this
|
| 405 |
-
# lane's power is refusing to GENERATE the broken case: the AI must only build what the backend
|
| 406 |
-
# and the renderer already support, which is R1's own sentence. Delete this guard when D-137
|
| 407 |
-
# closes — the refusal names the debt so the reader knows it is a limit, not a rule.
|
| 408 |
-
if kind == 'chart' and by_key.get(spec.get('groupBy')) in ('multiselect', 'checkbox'):
|
| 409 |
-
return None, (f"a chart cannot group by {spec.get('groupBy')!r} yet — a column that holds "
|
| 410 |
-
f"several values at once, or a tick box, does not bucket the way a chart "
|
| 411 |
-
f"needs. Try a table grouped by it instead"), "chart_key_unsupported"
|
| 412 |
-
|
| 413 |
-
for ref, families, required in _MODE_REFS.get(kind, ()):
|
| 414 |
-
val = spec.get(ref)
|
| 415 |
-
if val and by_key.get(val) not in families:
|
| 416 |
-
return None, (f"a {kind} view needs {ref[:-5]} to be a "
|
| 417 |
-
f"{' or '.join(families)} column, and {val!r} is a "
|
| 418 |
-
f"{by_key.get(val)} column"), "wrong_ref_type"
|
| 419 |
-
if required and not val:
|
| 420 |
-
return None, (f"a {kind} view needs a {' or '.join(families)} column and this "
|
| 421 |
-
f"database has none that fits"), "missing_ref"
|
| 422 |
-
|
| 423 |
-
raw_filters = [f for f in (spec.get('filters') or ()) if isinstance(f, dict)]
|
| 424 |
-
filters = g.clean_filter_tree(raw_filters, keys)
|
| 425 |
-
if len(filters) != len(raw_filters):
|
| 426 |
-
# The belt caught something membership did not — an unknown op. Refuse rather than save a
|
| 427 |
-
# view whose condition was silently dropped (the widening this file exists to prevent).
|
| 428 |
-
return None, ("part of that filter is not something this product can express — try "
|
| 429 |
-
"asking it more simply"), "filter_dropped"
|
| 430 |
-
|
| 431 |
-
display = {'mode': kind}
|
| 432 |
-
for ref, _families, _req in _MODE_REFS.get(kind, ()):
|
| 433 |
-
if spec.get(ref):
|
| 434 |
-
display[ref] = spec[ref]
|
| 435 |
-
cleaned_display = g._clean_display(display, keys) if kind != 'grid' else None
|
| 436 |
-
if kind != 'grid' and not cleaned_display:
|
| 437 |
-
return None, (f"this product could not build a {kind} view from that question"
|
| 438 |
-
), "display_dropped"
|
| 439 |
-
|
| 440 |
-
clean = {
|
| 441 |
-
'kind': kind,
|
| 442 |
-
'name': ' '.join(str(spec.get('name') or 'Query').split())[:60] or 'Query',
|
| 443 |
-
'visible': visible,
|
| 444 |
-
'filters': filters,
|
| 445 |
-
'filterConj': 'or' if spec.get('filterConj') == 'or' else 'and',
|
| 446 |
-
'sorts': [{'colId': s['colId'], 'dir': 'desc' if s.get('dir') == 'desc' else 'asc'}
|
| 447 |
-
for s in (spec.get('sorts') or ()) if isinstance(s, dict)
|
| 448 |
-
and s.get('colId') in keys][:3],
|
| 449 |
-
'groupBy': spec['groupBy'] if spec.get('groupBy') in keys else None,
|
| 450 |
-
'display': cleaned_display,
|
| 451 |
-
}
|
| 452 |
-
return clean, None, None
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
def _explain(clean, fields):
|
| 456 |
-
"""A plain-language read-back of what the view WILL do — DERIVED from the validated spec.
|
| 457 |
-
|
| 458 |
-
⛔ NEVER ASKED OF THE MODEL. A model describing its own output describes what it MEANT, which
|
| 459 |
-
is precisely the failure this sentence exists to expose: a spec whose every column is real and
|
| 460 |
-
which answers a different question is invisible to validation, so the only thing that catches
|
| 461 |
-
it is a person reading an honest description of the spec that was actually accepted.
|
| 462 |
-
"""
|
| 463 |
-
label = {f['key']: f['label'] for f in fields}
|
| 464 |
-
ops = {'eq': 'is', 'neq': 'is not', 'gt': 'is over', 'gte': 'is at least',
|
| 465 |
-
'lt': 'is under', 'lte': 'is at most', 'contains': 'contains',
|
| 466 |
-
'doesNotContain': 'does not contain', 'isEmpty': 'is empty',
|
| 467 |
-
'isNotEmpty': 'is not empty', 'between': 'is between', 'within': 'is within',
|
| 468 |
-
'topN': 'is in the top', 'bottomN': 'is in the bottom'}
|
| 469 |
-
kinds = {'grid': 'a table', 'list': 'a list', 'chart': 'a chart', 'kanban': 'a board',
|
| 470 |
-
'calendar': 'a calendar', 'timeseries': 'a time series', 'map': 'a map'}
|
| 471 |
-
parts = [f"{kinds.get(clean['kind'], clean['kind'])} of "
|
| 472 |
-
+ ", ".join(label.get(k, k) for k in clean['visible'][:6])
|
| 473 |
-
+ (f" and {len(clean['visible']) - 6} more columns"
|
| 474 |
-
if len(clean['visible']) > 6 else "")]
|
| 475 |
-
if clean['filters']:
|
| 476 |
-
joiner = " or " if clean['filterConj'] == 'or' else " and "
|
| 477 |
-
parts.append("showing only records where " + joiner.join(
|
| 478 |
-
f"{label.get(f['colId'], f['colId'])} {ops.get(f['op'], f['op'])}"
|
| 479 |
-
+ (f" {f['value']}" if f.get('value') else "")
|
| 480 |
-
for f in clean['filters'] if isinstance(f, dict) and f.get('colId')))
|
| 481 |
-
else:
|
| 482 |
-
parts.append("showing every record")
|
| 483 |
-
if clean['groupBy']:
|
| 484 |
-
parts.append(f"grouped by {label.get(clean['groupBy'], clean['groupBy'])}")
|
| 485 |
-
if clean['sorts']:
|
| 486 |
-
s = clean['sorts'][0]
|
| 487 |
-
parts.append(f"sorted by {label.get(s['colId'], s['colId'])}"
|
| 488 |
-
+ (" (highest first)" if s['dir'] == 'desc' else " (lowest first)"))
|
| 489 |
-
for ref, word in (('stackField', 'in columns by'), ('dateField', 'placed in time by'),
|
| 490 |
-
('colorField', 'coloured by'), ('sizeField', 'sized by')):
|
| 491 |
-
val = (clean.get('display') or {}).get(ref)
|
| 492 |
-
if val:
|
| 493 |
-
parts.append(f"{word} {label.get(val, val)}")
|
| 494 |
-
return ", ".join(parts) + "."
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
# ── the registry ────────────────────────────────────────────────────────────────────────────
|
| 498 |
-
|
| 499 |
-
def _qid(scope, question):
|
| 500 |
-
"""PINNED per (database, question), the `view_templates` discipline: asking the same thing
|
| 501 |
-
twice UPDATES one view instead of minting "Query 2" — `save_view` de-duplicates NAMES by
|
| 502 |
-
appending a number, so an unpinned id fills the store with numbered near-duplicates."""
|
| 503 |
-
h = hashlib.sha1(f"{scope}|{' '.join(str(question).split()).lower()}".encode("utf-8"))
|
| 504 |
-
return "q_" + h.hexdigest()[:12]
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
def _registry(session):
|
| 508 |
-
try:
|
| 509 |
-
recs = session.runtime.get(QUERY_KEY) or {}
|
| 510 |
-
except Exception:
|
| 511 |
-
return {}
|
| 512 |
-
return recs if isinstance(recs, dict) else {}
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
def _mine(session, rec):
|
| 516 |
-
return session.admin or str(rec.get('createdBy') or '') == session.uname
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
#: The BUILT-IN topics `routes_templates._target_or_refuse` accepts beside any `ut_*` key. ⛔ THE
|
| 520 |
-
#: CLIENT'S PICKER MUST MIRROR THIS, and `GET /query` publishes it (`builtins`) rather than leaving
|
| 521 |
-
#: the page to re-derive it: the nav hands `QueryPage` every granted entry, which is a WIDER set
|
| 522 |
-
#: than this door accepts, so a picker filtered on "not a surface" alone would offer a database the
|
| 523 |
-
#: build door 404s. `view_templates`' own rule — "a picker that offered a template the apply door
|
| 524 |
-
#: would then refuse is a control that lies" — and it is the same predicate twice, published once.
|
| 525 |
-
BUILTIN_SCOPES = ('customer_data', 'product_data')
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
@router.get("/query")
|
| 529 |
-
def list_queries(session: Session = Depends(require_session)):
|
| 530 |
-
"""The AI-built views this caller may open: their own, still-granted, still-existing.
|
| 531 |
-
|
| 532 |
-
⚠ PRUNED ON READ against the CURRENT wall, never merely on write — a grant can be withdrawn
|
| 533 |
-
and a database deleted after a view was saved, and the Query module is chrome: it may render
|
| 534 |
-
nothing the server would not grant today (`nav.ts`'s chrome law).
|
| 535 |
-
"""
|
| 536 |
-
import routes_templates
|
| 537 |
-
out = []
|
| 538 |
-
for qid, rec in sorted(_registry(session).items(),
|
| 539 |
-
key=lambda kv: str((kv[1] or {}).get('createdAt') or ''),
|
| 540 |
-
reverse=True):
|
| 541 |
-
if not isinstance(rec, dict) or not _mine(session, rec):
|
| 542 |
-
continue
|
| 543 |
-
try:
|
| 544 |
-
routes_templates._target_or_refuse(session, rec.get('scope'))
|
| 545 |
-
except Exception:
|
| 546 |
-
continue # no longer granted, or gone
|
| 547 |
-
out.append({"id": qid, "scope": rec.get('scope'), "viewId": rec.get('viewId'),
|
| 548 |
-
"name": rec.get('name'), "kind": rec.get('kind'),
|
| 549 |
-
"question": rec.get('question'), "explain": rec.get('explain'),
|
| 550 |
-
"createdAt": rec.get('createdAt')})
|
| 551 |
-
return {"views": out, "kinds": list(QUERY_KINDS), "builtins": list(BUILTIN_SCOPES)}
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
@router.post("/query/build")
|
| 555 |
-
def build_query(body: dict = Body(default=None),
|
| 556 |
-
session: Session = Depends(require_session)):
|
| 557 |
-
"""A question + a granted database -> `{spec, explain, refused}`. **WRITES NOTHING.**
|
| 558 |
-
|
| 559 |
-
A refusal is a 200 carrying a sentence, not a 5xx: "this database cannot answer that" is a
|
| 560 |
-
successful answer to the person asking, and an error envelope would paint the error page R1
|
| 561 |
-
forbids. The 4xx cases are the ones that ARE the caller's fault — an unnamed database, an
|
| 562 |
-
over-long question, a database this session may not open.
|
| 563 |
-
|
| 564 |
-
⛔ THE MODEL SEAM IS **NOT** ON THIS SIGNATURE, AND THAT IS A CORRECTION. The first draft took
|
| 565 |
-
`chat=None` as a parameter of the ROUTE — and FastAPI reads an un-annotated defaulted argument
|
| 566 |
-
as a **QUERY PARAMETER**, so `chat` appeared in `openapi()["paths"]` (measured) and
|
| 567 |
-
`POST /query/build?chat=x` would have reached `_call_model(..., chat="x")` and 500'd on
|
| 568 |
-
`"x"(messages, tools)`. The gate never saw it because the gate calls the handler directly —
|
| 569 |
-
which is exactly the shape of a test seam that becomes an input. The seam now lives on the
|
| 570 |
-
module-level `_build` below: the gate calls that, and the wire cannot.
|
| 571 |
-
"""
|
| 572 |
-
body = body if isinstance(body, dict) else {}
|
| 573 |
-
return _build(body.get("question"), body.get("scope"), session)
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
def _build(question, scope, session, chat=None):
|
| 577 |
-
"""`build_query`'s body, with the model injectable — see that route's note on why the seam is
|
| 578 |
-
here and not on the signature FastAPI reads."""
|
| 579 |
-
question = ' '.join(str(question or "").split())
|
| 580 |
-
if not question:
|
| 581 |
-
raise err(400, "bad_request", "no question was asked")
|
| 582 |
-
if len(question) > MAX_QUESTION:
|
| 583 |
-
raise err(400, "question_too_long",
|
| 584 |
-
f"a question must be under {MAX_QUESTION} characters")
|
| 585 |
-
fields, _source = _target(session, scope)
|
| 586 |
-
if not fields:
|
| 587 |
-
raise err(400, "no_columns", "that database has no columns to build a view from")
|
| 588 |
-
|
| 589 |
-
spec, transport, provider = _call_model(question, fields, chat=chat)
|
| 590 |
-
if transport:
|
| 591 |
-
return {"spec": None, "explain": None, "refused": transport, "provider": provider}
|
| 592 |
-
clean, refusal, code = _validate(spec, fields)
|
| 593 |
-
if refusal:
|
| 594 |
-
return {"spec": None, "explain": None, "refused": refusal, "reason": code,
|
| 595 |
-
"provider": provider}
|
| 596 |
-
return {"spec": clean, "explain": _explain(clean, fields),
|
| 597 |
-
"refused": None, "scope": scope,
|
| 598 |
-
"id": _qid(scope, question), "provider": provider
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
def
|
| 603 |
-
"""
|
| 604 |
-
|
| 605 |
-
⛔
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
question that made it ([[a-record-can-outlive-its-subject]]).
|
| 695 |
-
"""
|
| 696 |
-
rec = _registry(session).get(str(qid))
|
| 697 |
-
if not isinstance(rec, dict) or not _mine(session, rec):
|
| 698 |
-
raise err(404, "unknown_query", "that view does not exist")
|
| 699 |
-
_target(session, rec.get('scope')) # the wall, again, on the write path
|
| 700 |
-
vt = _templates()
|
| 701 |
-
ws_key = vt.workspace_key(rec.get('scope'))
|
| 702 |
-
if ws_key and session.runtime.available():
|
| 703 |
-
import core.table_store as table_store
|
| 704 |
-
table_store.make(ws_key, st=session.runtime).delete_view(
|
| 705 |
-
session.uname, rec.get('viewId') or str(qid))
|
| 706 |
-
|
| 707 |
-
def _up(data):
|
| 708 |
-
data = data if isinstance(data, dict) else {}
|
| 709 |
-
data.pop(str(qid), None)
|
| 710 |
-
return data
|
| 711 |
-
|
| 712 |
-
session.runtime.update(QUERY_KEY, _up, flush='async')
|
| 713 |
-
return {"deleted": str(qid)}
|
|
|
|
| 1 |
+
"""routes_query.py — WAVE 32 item 7 (ruling R1, contract C5): the Query module's doors.
|
| 2 |
+
|
| 3 |
+
POST /api/v1/query/build {question, scope} -> {spec, explain, refused} WRITES NOTHING
|
| 4 |
+
POST /api/v1/query/save {question, scope, spec} -> the saved view
|
| 5 |
+
GET /api/v1/query -> the AI-built views this caller may open
|
| 6 |
+
DELETE /api/v1/query/{qid} -> forget one
|
| 7 |
+
|
| 8 |
+
**The AI turns a question about ONE granted database into a view SPEC. It may only build what the
|
| 9 |
+
backend already supports** (R1), and the wall on which database it may name is the EXISTING one.
|
| 10 |
+
|
| 11 |
+
⛔ THE FOUNDATION IS NOT `harness/semantic.py` — MEASURED, W32-T50
|
| 12 |
+
(`.claude/wiki/waves/wave32/proto/query-spike/README.md`). That layer is bound to 8 hand-authored
|
| 13 |
+
YAML topics over Odoo mirror tables: **0** are `ut_`-bound, and a `ut_` key answers
|
| 14 |
+
`ModelError: topic 'ut_…' has no store binding` from both `store_columns` and `store_query`. Its
|
| 15 |
+
own `save_view` writes `platform/data/store/views.json`, a chart workspace the product's grid
|
| 16 |
+
cannot read. What it lends this file is its POSTURE, quoted from its system prompt: *the model
|
| 17 |
+
never writes SQL and never invents a field name — it only picks registry keys.*
|
| 18 |
+
|
| 19 |
+
The real foundation is the view stack that already ships, and this module is `routes_templates.py`
|
| 20 |
+
with a model where the curated registry used to be:
|
| 21 |
+
|
| 22 |
+
· the wall `routes_templates._target_or_refuse` (imported, never re-implemented)
|
| 23 |
+
· the column contract `core.view_templates.columns_named` / `missing_columns`
|
| 24 |
+
· the view skeleton `core.view_templates._view` (one definition of what a view is)
|
| 25 |
+
· the validators `aios_grid._clean_display` / `clean_filter_tree`
|
| 26 |
+
· the writer `core.table_store.make(ws).save_view(..., shared=False)`
|
| 27 |
+
|
| 28 |
+
⛔⛔ THE VALIDATORS DROP; THEY DO NOT REFUSE — and C5 requires a REFUSAL. `_clean_display` returns
|
| 29 |
+
None for an unknown mode, and a view with no display block IS A GRID; `clean_filter_tree` drops a
|
| 30 |
+
leaf naming an unknown column, and a view whose only condition was dropped SHOWS EVERY ROW under a
|
| 31 |
+
name promising a shortlist (`view_templates`' own scar, and `_seed_wave17`'s before it). So
|
| 32 |
+
membership is tested HERE, first and explicitly, and the cleaners run second as a belt. They are
|
| 33 |
+
the vocabulary ORACLE, never the refusal.
|
| 34 |
+
|
| 35 |
+
⛔ THE REFUSAL PATH IS A MODEL PROPERTY, so this module declares its OWN provider order rather than
|
| 36 |
+
inheriting `analyst.available_providers()`' cheap-first one. Measured over 4 arms x 9 questions
|
| 37 |
+
(`proto/query-spike/refusal-arms.json`): `cerebras/gpt-oss-120b` refuses 4/4 unanswerable questions
|
| 38 |
+
with a usable sentence; `groq/llama-3.3-70b` refuses **0/4** and instead emits a fully valid spec
|
| 39 |
+
answering a different question ("Revenue by Creator" against a database holding no revenue).
|
| 40 |
+
Changing refusal from an enum value to its own tool changed cerebras not at all and made groq
|
| 41 |
+
worse.
|
| 42 |
+
⚠ AND THE LADDER STILL FALLS THROUGH ON A 429, WHICH IS A COMPROMISE, NOT AN OVERSIGHT — stated
|
| 43 |
+
here because the first draft of this header claimed the opposite. A door that dies whenever one
|
| 44 |
+
free tier rate-limits is worse than one that answers with a weaker model behind the SAME validator
|
| 45 |
+
and the SAME human read-back. What "fail closed" means precisely: when the whole ladder is
|
| 46 |
+
unreachable the answer is a sentence and never a view. Which provider answered rides back in the
|
| 47 |
+
response so a wrong answer stays diagnosable (`_call_model`'s note).
|
| 48 |
+
|
| 49 |
+
⛔ AND A PROVIDER `400` CAN BE THE MODEL'S REFUSAL. Groq validates the model's tool arguments
|
| 50 |
+
against the tool schema server-side and answers `400 tool_use_failed` carrying the model's real
|
| 51 |
+
output in `failed_generation` — so a refusal that omits a `required` property arrives as an HTTP
|
| 52 |
+
error. `analyst._live_chat` calls `raise_for_status()` and discards that body, which is why this
|
| 53 |
+
file has its own small ladder: `required` is `["kind"]` ALONE (a refusal must be a legal argument
|
| 54 |
+
object), and a 400 body is READ before it is treated as transport.
|
| 55 |
+
|
| 56 |
+
⚠ WHAT NO VALIDATOR CAN SEE, stated so nobody thinks the gate covers it: a spec whose every column
|
| 57 |
+
is real, every op legal, kind in the vocabulary and target granted, which answers a DIFFERENT
|
| 58 |
+
QUESTION. That class is invisible to server-side validation. The mechanism against it is C5's own
|
| 59 |
+
shape — `build` returns `{spec, explain}` and WRITES NOTHING, `explain` is DERIVED from the
|
| 60 |
+
validated spec (never asked of the model, which would let it flatter its own output), and saving is
|
| 61 |
+
a second, explicit call. The person is the last validator, by design.
|
| 62 |
+
"""
|
| 63 |
+
import datetime as _dt
|
| 64 |
+
import hashlib
|
| 65 |
+
import json
|
| 66 |
+
import os
|
| 67 |
+
import re
|
| 68 |
+
|
| 69 |
+
from fastapi import APIRouter, Body, Depends
|
| 70 |
+
|
| 71 |
+
from deps import Session, err, require_session
|
| 72 |
+
|
| 73 |
+
router = APIRouter(prefix="/api/v1")
|
| 74 |
+
|
| 75 |
+
#: The store key holding the REGISTRY of AI-built views, per tenant: {qid: {...}}. The view itself
|
| 76 |
+
#: is a real saved view in its target's own workspace — this is only the index, so listing the
|
| 77 |
+
#: Query module never has to read every granted table's workspace to find them. Same split
|
| 78 |
+
#: `core/alerts.py` draws between definitions and the inbox.
|
| 79 |
+
QUERY_KEY = 'query_views'
|
| 80 |
+
|
| 81 |
+
MAX_QUESTION = 500
|
| 82 |
+
MAX_VISIBLE = 24
|
| 83 |
+
MAX_REGISTRY = 200 #: a working list, not an archive (alerts' MAX_NOTIFICATIONS rule)
|
| 84 |
+
|
| 85 |
+
#: ⭐ THE KINDS THE AI MAY EMIT. Held here as a NAMED SET, and `verify_query.py` asserts it is a
|
| 86 |
+
#: subset of BOTH `aios_grid.DISPLAY_MODES` (the stored vocabulary) and the client's
|
| 87 |
+
#: `CREATABLE_MODES` (which means "has a mounted renderer" — `verify_icons::mode_parity` enforces
|
| 88 |
+
#: that both ways), AND that the excluded remainder is exactly `QUERY_EXCLUDED`. So a mode that
|
| 89 |
+
#: lands in `CREATABLE_MODES` later REDS the gate instead of being silently un-offered — the
|
| 90 |
+
#: `swipe`/`form` failure ("whole, correct and unreachable") in reverse.
|
| 91 |
+
QUERY_KINDS = ('grid', 'chart', 'calendar', 'kanban', 'timeseries', 'map', 'list')
|
| 92 |
+
|
| 93 |
+
#: Creatable modes the AI deliberately may NOT emit, each with the reason it cannot be derived
|
| 94 |
+
#: from a question about existing records. A reason per member, because "excluded" with no cause
|
| 95 |
+
#: is how a hold outlives its own justification.
|
| 96 |
+
QUERY_EXCLUDED = {
|
| 97 |
+
'form': "a form is a door records come IN through and shows none — a question about existing "
|
| 98 |
+
"records cannot describe one, and its spec carries emails and a public token",
|
| 99 |
+
'catalog': "a catalog is a published artifact needing page authoring and product codes "
|
| 100 |
+
"(MAX_CATALOG_CODES is spent in page order), not a shape of the current rows",
|
| 101 |
+
'swipe': "a swipe deck needs an empty single-select plus a chosen option per direction — two "
|
| 102 |
+
"product decisions a question does not contain",
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
#: Mode-required refs, and the field FAMILY each one demands. ⚠ STRICTER THAN `_clean_display` ON
|
| 106 |
+
#: PURPOSE. That function accepts any ref naming a real field and leaves the meaning to the client,
|
| 107 |
+
#: which "degrades a wrong-typed ref to that surface's default rather than to an error" — correct
|
| 108 |
+
#: for a person clicking a picker who SEES the result, wrong for a generated spec nobody has looked
|
| 109 |
+
#: at yet. Measured: a free model stacked a kanban on `category` (a text column) for a contract
|
| 110 |
+
#: with ZERO select columns, and every validator passed it.
|
| 111 |
+
_MODE_REFS = {
|
| 112 |
+
'kanban': [('stackField', ('select',), True)],
|
| 113 |
+
'calendar': [('dateField', ('date',), True)],
|
| 114 |
+
'timeseries': [('dateField', ('date',), True)],
|
| 115 |
+
'map': [('colorField', ('select',), False), ('sizeField', ('int', 'currency', 'pct'), False)],
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _now_iso():
|
| 120 |
+
"""UTC WITH the offset — a naive stamp is unsubtractable in a browser (D-18, alerts' rule)."""
|
| 121 |
+
return _dt.datetime.now(_dt.timezone.utc).isoformat()
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _grid():
|
| 125 |
+
import aios_grid
|
| 126 |
+
return aios_grid
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _templates():
|
| 130 |
+
import core.view_templates as view_templates
|
| 131 |
+
return view_templates
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _target(session, scope):
|
| 135 |
+
"""`(fields, source)` for a database this session may open — or a refusal.
|
| 136 |
+
|
| 137 |
+
⛔ THE WALL IS `routes_templates._target_or_refuse`, IMPORTED AND NOT COPIED. It is the same
|
| 138 |
+
predicate the nav uses, per topic (`ut_*` -> `user_tables.may_open`; a built-in -> the module
|
| 139 |
+
grant `session.require`; anything else a 404 that is not a directory of what exists), and T51's
|
| 140 |
+
instruction is explicit that the wall is re-used, never re-implemented beside itself. The
|
| 141 |
+
private name is deliberate: an alias would mean editing a file no lane's fence covers.
|
| 142 |
+
|
| 143 |
+
It returns field KEYS; this door also needs each field's TYPE (for the prompt and for
|
| 144 |
+
`_MODE_REFS`), so the types are read from the same source the wall consulted — never from a
|
| 145 |
+
list assembled here.
|
| 146 |
+
"""
|
| 147 |
+
import routes_templates
|
| 148 |
+
keys, source = routes_templates._target_or_refuse(session, scope)
|
| 149 |
+
key = str(scope or '').strip()
|
| 150 |
+
defs = []
|
| 151 |
+
if key.startswith('ut_'):
|
| 152 |
+
import core.user_tables as user_tables
|
| 153 |
+
defn = user_tables.get(key, st=session.runtime) or {}
|
| 154 |
+
by_key = {str(f.get('key')): f for f in (defn.get('fields') or []) if f.get('key')}
|
| 155 |
+
else:
|
| 156 |
+
g = _grid()
|
| 157 |
+
raw = g.product_fields() if key == 'product_data' else g.FIELDS
|
| 158 |
+
by_key = {str(f.get('key')): f for f in raw if f.get('key')}
|
| 159 |
+
for k in keys:
|
| 160 |
+
f = by_key.get(k) or {}
|
| 161 |
+
defs.append({'key': k, 'label': str(f.get('label') or k)[:80],
|
| 162 |
+
'type': str(f.get('type') or 'text')})
|
| 163 |
+
return defs, source
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
# ── the model ───────────────────────────────────────────────────────────────────────────────
|
| 167 |
+
|
| 168 |
+
#: This module's OWN provider order — cerebras first. See the header: the refusal is a model
|
| 169 |
+
#: property, and the ladder's cheap-first order is exactly wrong for a contract that includes one.
|
| 170 |
+
#: Override with `AIOS_QUERY_PROVIDER`. The credentials themselves stay in ONE place
|
| 171 |
+
#: (`analyst.PROVIDERS`), so a rotated key or a new model reaches this file too.
|
| 172 |
+
QUERY_PROVIDER_ORDER = ('cerebras', 'groq', 'openrouter')
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _providers():
|
| 176 |
+
import harness.analyst as analyst
|
| 177 |
+
pin = os.environ.get("AIOS_QUERY_PROVIDER")
|
| 178 |
+
by_name = {p["name"]: p for p in analyst.PROVIDERS}
|
| 179 |
+
order = [pin] if pin else list(QUERY_PROVIDER_ORDER)
|
| 180 |
+
return [by_name[n] for n in order
|
| 181 |
+
if n in by_name and os.environ.get(by_name[n]["env"])]
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def _spec_schema(field_keys):
|
| 185 |
+
"""The tool schema the model fills. ⛔ `required` IS `["kind"]` ALONE — see the header: a
|
| 186 |
+
refusal carries no name, and groq rejects a schema-violating refusal with a 400 that reads
|
| 187 |
+
exactly like a transport failure."""
|
| 188 |
+
ops = sorted(_grid().FILTER_OPS)
|
| 189 |
+
col = {"type": "string", "enum": sorted(field_keys)}
|
| 190 |
+
return {
|
| 191 |
+
"type": "object",
|
| 192 |
+
"properties": {
|
| 193 |
+
"kind": {"type": "string", "enum": list(QUERY_KINDS) + ["refused"],
|
| 194 |
+
"description": "refused = this database cannot answer the question"},
|
| 195 |
+
"name": {"type": "string", "description": "a short title for the view"},
|
| 196 |
+
"refusal": {"type": "string",
|
| 197 |
+
"description": "when kind=refused: ONE plain sentence naming what is "
|
| 198 |
+
"missing"},
|
| 199 |
+
"visible": {"type": "array", "items": col},
|
| 200 |
+
"filters": {"type": "array", "items": {
|
| 201 |
+
"type": "object",
|
| 202 |
+
"properties": {"colId": col, "op": {"type": "string", "enum": ops},
|
| 203 |
+
"value": {"type": "string"}},
|
| 204 |
+
"required": ["colId", "op"]}},
|
| 205 |
+
"filterConj": {"type": "string", "enum": ["and", "or"]},
|
| 206 |
+
"sorts": {"type": "array", "items": {
|
| 207 |
+
"type": "object",
|
| 208 |
+
"properties": {"colId": col,
|
| 209 |
+
"dir": {"type": "string", "enum": ["asc", "desc"]}},
|
| 210 |
+
"required": ["colId", "dir"]}},
|
| 211 |
+
"groupBy": col,
|
| 212 |
+
"stackField": col,
|
| 213 |
+
"dateField": col,
|
| 214 |
+
"colorField": col,
|
| 215 |
+
"sizeField": col,
|
| 216 |
+
},
|
| 217 |
+
"required": ["kind"],
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _system_prompt(fields):
|
| 222 |
+
"""The standing instructions. The vocabulary is DERIVED (the kinds set, `FILTER_OPS`, the
|
| 223 |
+
target's own columns) — the model is never told about a key this server cannot honour."""
|
| 224 |
+
cols = json.dumps([{"key": f["key"], "label": f["label"], "type": f["type"]}
|
| 225 |
+
for f in fields], separators=(",", ":"))
|
| 226 |
+
kinds = ", ".join(QUERY_KINDS)
|
| 227 |
+
ops = ", ".join(sorted(_grid().FILTER_OPS))
|
| 228 |
+
selects = [f["key"] for f in fields if f["type"] in ('select', 'multiselect')] or ["(none)"]
|
| 229 |
+
dates = [f["key"] for f in fields if f["type"] == 'date'] or ["(none)"]
|
| 230 |
+
return f"""You turn a question about ONE database into a VIEW SPEC. You never write SQL and you
|
| 231 |
+
never invent a column or a kind of view.
|
| 232 |
+
|
| 233 |
+
THE DATABASE ({len(fields)} columns) — you may name ONLY these column keys:
|
| 234 |
+
{cols}
|
| 235 |
+
|
| 236 |
+
VIEW KINDS you may name, and nothing else: {kinds}
|
| 237 |
+
FILTER OPS you may name, and nothing else: {ops}
|
| 238 |
+
The only select-type columns (a kanban MUST stack on one of these): {", ".join(selects)}
|
| 239 |
+
The only date columns (a calendar or timeseries MUST use one of these): {", ".join(dates)}
|
| 240 |
+
|
| 241 |
+
RULES:
|
| 242 |
+
1. Emit ONE call to build_view. Every colId in visible/filters/sorts/groupBy and every field ref
|
| 243 |
+
MUST be one of the keys above.
|
| 244 |
+
2. Pick the kind by what the question asks for: a list of records -> grid (or list); totals or
|
| 245 |
+
counts by a category -> chart with groupBy; cards by a stage -> kanban (stackField REQUIRED,
|
| 246 |
+
and it must be a select column); something over time -> calendar or timeseries (dateField
|
| 247 |
+
REQUIRED, and it must be a date column); geography -> map.
|
| 248 |
+
3. `visible` is the columns the question is about, most important first — not every column.
|
| 249 |
+
4. Set kind="refused" and put ONE plain sentence in `refusal` when the question needs a column
|
| 250 |
+
this database does not have, a number it does not store, or a picture that is not in the kind
|
| 251 |
+
list. REFUSING IS A CORRECT ANSWER. A view that answers a DIFFERENT question than the one
|
| 252 |
+
asked is the worst possible answer — worse than saying you cannot build it. When in doubt,
|
| 253 |
+
refuse.
|
| 254 |
+
5. Every `value` is a string."""
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _call_model(question, fields, chat=None):
|
| 258 |
+
"""`(spec, error_sentence, provider)` — one bounded call down THIS module's ladder.
|
| 259 |
+
|
| 260 |
+
`chat` is injectable so `verify_query.py` proves this door end to end with no key and no
|
| 261 |
+
tokens (`analyst.ask`'s own posture, and the reason its loop is testable offline).
|
| 262 |
+
|
| 263 |
+
⚠ THE LADDER IS A DELIBERATE COMPROMISE, AND THE PROVIDER RIDES BACK BECAUSE OF IT. Only the
|
| 264 |
+
FIRST provider is measured to refuse honestly (README section 4b); the ones below it answer
|
| 265 |
+
unanswerable questions with plausible specs. Falling through on a 429 is still the right call —
|
| 266 |
+
a Query module that dies whenever one free tier rate-limits is worse than one that answers with
|
| 267 |
+
a weaker model behind the same validator and the same human read-back — but *which* model
|
| 268 |
+
answered must be knowable, or a wrong answer is undiagnosable after the fact. Failing closed is
|
| 269 |
+
what happens when the whole ladder is unreachable, not when the best rung is.
|
| 270 |
+
"""
|
| 271 |
+
tools = [{"type": "function", "function": {
|
| 272 |
+
"name": "build_view",
|
| 273 |
+
"description": "Emit the view spec, or refuse.",
|
| 274 |
+
"parameters": _spec_schema([f["key"] for f in fields])}}]
|
| 275 |
+
messages = [{"role": "system", "content": _system_prompt(fields)},
|
| 276 |
+
{"role": "user", "content": question}]
|
| 277 |
+
if chat is not None:
|
| 278 |
+
return chat(messages, tools), None, "injected"
|
| 279 |
+
|
| 280 |
+
provs = _providers()
|
| 281 |
+
if not provs:
|
| 282 |
+
# No key at all: say so. An AI feature that silently does nothing is indistinguishable
|
| 283 |
+
# from one that was never built ([[flag-shipped-without-its-writer]]).
|
| 284 |
+
return None, ("the assistant is not configured on this deployment, so a view cannot be "
|
| 285 |
+
"built from a question yet"), None
|
| 286 |
+
import requests
|
| 287 |
+
last = None
|
| 288 |
+
for p in provs:
|
| 289 |
+
try:
|
| 290 |
+
r = requests.post(p["url"], timeout=60,
|
| 291 |
+
headers={"Authorization": f"Bearer {os.environ[p['env']]}"},
|
| 292 |
+
json={"model": p["model"], "messages": messages, "tools": tools,
|
| 293 |
+
"tool_choice": "required", "temperature": 0.1,
|
| 294 |
+
"max_tokens": 1200})
|
| 295 |
+
except Exception as e: # network blip — try the next provider
|
| 296 |
+
last = f"{p['name']}: {type(e).__name__}"
|
| 297 |
+
continue
|
| 298 |
+
if r.status_code == 400:
|
| 299 |
+
# ⛔ THE MODEL'S ANSWER CAN BE IN HERE. Groq validates tool arguments against the
|
| 300 |
+
# schema and returns the generation it rejected. Reading it turns a "transport
|
| 301 |
+
# failure" back into the refusal it actually was.
|
| 302 |
+
spec = _spec_from_400(r.text)
|
| 303 |
+
if spec is not None:
|
| 304 |
+
return spec, None, p["name"]
|
| 305 |
+
last = f"{p['name']}: 400"
|
| 306 |
+
continue
|
| 307 |
+
if r.status_code != 200:
|
| 308 |
+
last = f"{p['name']}: HTTP {r.status_code}"
|
| 309 |
+
continue # 429 included: the NEXT provider, not a sleep
|
| 310 |
+
try:
|
| 311 |
+
msg = r.json()["choices"][0]["message"]
|
| 312 |
+
calls = msg.get("tool_calls") or []
|
| 313 |
+
if not calls:
|
| 314 |
+
last = f"{p['name']}: no tool call"
|
| 315 |
+
continue
|
| 316 |
+
return json.loads(calls[0]["function"].get("arguments") or "{}"), None, p["name"]
|
| 317 |
+
except Exception as e:
|
| 318 |
+
last = f"{p['name']}: unreadable answer ({type(e).__name__})"
|
| 319 |
+
continue
|
| 320 |
+
# ⛔ FAIL CLOSED. The first provider in the order is the one measured to refuse honestly; if it
|
| 321 |
+
# is rate-limited and the rest are too, the answer is a sentence, never a fabricated view.
|
| 322 |
+
return None, ("the assistant could not be reached just now (" + (last or "no provider") +
|
| 323 |
+
") — try the question again in a moment"), None
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
_FAILED_GEN = re.compile(r'<function=build_view>(\{.*?\})\s*</function>', re.S)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _spec_from_400(body):
|
| 330 |
+
"""The spec out of a `tool_use_failed` body's `failed_generation`, or None."""
|
| 331 |
+
try:
|
| 332 |
+
err_obj = (json.loads(body) or {}).get("error") or {}
|
| 333 |
+
except Exception:
|
| 334 |
+
return None
|
| 335 |
+
gen = str(err_obj.get("failed_generation") or "")
|
| 336 |
+
if not gen:
|
| 337 |
+
return None
|
| 338 |
+
m = _FAILED_GEN.search(gen)
|
| 339 |
+
raw = m.group(1) if m else gen.strip()
|
| 340 |
+
try:
|
| 341 |
+
spec = json.loads(raw)
|
| 342 |
+
except Exception:
|
| 343 |
+
return None
|
| 344 |
+
return spec if isinstance(spec, dict) else None
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
# ── validation: the REFUSAL this contract promises ───────────────────────────────────────────
|
| 348 |
+
|
| 349 |
+
def _validate(spec, fields):
|
| 350 |
+
"""`(clean, refusal, code)` — exactly one of `clean` / `refusal` is set.
|
| 351 |
+
|
| 352 |
+
Membership FIRST and explicitly (the header's whole subject), then the product's own cleaners
|
| 353 |
+
as a belt. A refusal is a SENTENCE, because C5's user-facing promise is a plain sentence and
|
| 354 |
+
never a broken view.
|
| 355 |
+
"""
|
| 356 |
+
g = _grid()
|
| 357 |
+
if not isinstance(spec, dict):
|
| 358 |
+
return None, "the assistant did not answer with a view", "no_spec"
|
| 359 |
+
by_key = {f['key']: f['type'] for f in fields}
|
| 360 |
+
keys = set(by_key)
|
| 361 |
+
kind = spec.get('kind')
|
| 362 |
+
|
| 363 |
+
if kind == 'refused':
|
| 364 |
+
return None, (str(spec.get('refusal') or '').strip()
|
| 365 |
+
or "this database cannot answer that question"), "model_refused"
|
| 366 |
+
if kind in QUERY_EXCLUDED:
|
| 367 |
+
return None, (f"a {kind} view cannot be built from a question — "
|
| 368 |
+
+ QUERY_EXCLUDED[kind]), "unsupported_kind"
|
| 369 |
+
if kind not in QUERY_KINDS:
|
| 370 |
+
# ⛔ NOT `_clean_display`'s job: it would DROP this and hand back a grid under a name
|
| 371 |
+
# promising something else.
|
| 372 |
+
return None, (f"{kind!r} is not a kind of view this product can build"
|
| 373 |
+
if kind else "the assistant did not name a kind of view"), "unsupported_kind"
|
| 374 |
+
|
| 375 |
+
named = set(spec.get('visible') or ())
|
| 376 |
+
for s in spec.get('sorts') or ():
|
| 377 |
+
if isinstance(s, dict) and s.get('colId'):
|
| 378 |
+
named.add(str(s['colId']))
|
| 379 |
+
for f in spec.get('filters') or ():
|
| 380 |
+
if isinstance(f, dict) and f.get('colId'):
|
| 381 |
+
named.add(str(f['colId']))
|
| 382 |
+
if spec.get('groupBy'):
|
| 383 |
+
named.add(str(spec['groupBy']))
|
| 384 |
+
for ref in ('stackField', 'dateField', 'colorField', 'sizeField'):
|
| 385 |
+
if spec.get(ref):
|
| 386 |
+
named.add(str(spec[ref]))
|
| 387 |
+
missing = sorted(named - keys)
|
| 388 |
+
if missing:
|
| 389 |
+
# NAMED, not counted (`routes_templates`' rule): the column keys are what tell the reader
|
| 390 |
+
# this is the wrong database for this question.
|
| 391 |
+
return None, ("this database does not have the columns that answer needs: "
|
| 392 |
+
+ ", ".join(missing)), "unknown_columns"
|
| 393 |
+
|
| 394 |
+
visible = [k for k in (spec.get('visible') or ()) if k in keys][:MAX_VISIBLE]
|
| 395 |
+
if not visible:
|
| 396 |
+
# An empty projection is not a view. `_default_view_config` would fill one in, which is
|
| 397 |
+
# how a question about two columns becomes a 285-column wall.
|
| 398 |
+
return None, ("that question did not name anything to show — try naming the columns you "
|
| 399 |
+
"want to see"), "no_columns"
|
| 400 |
+
|
| 401 |
+
# ⛔ D-137, GUARDED WHERE IT REACHES THIS FEATURE. "The chart cannot key a SET-LIKE or CHECKBOX
|
| 402 |
+
# column the way the grid does" — a set row lands in several buckets, so a sum means something
|
| 403 |
+
# else, and a checkbox reads "(blank)". The defect lives in the CLIENT's `chartData` bucket
|
| 404 |
+
# keying, which is in no lane's fence this wave (booked, with that reason). What IS in this
|
| 405 |
+
# lane's power is refusing to GENERATE the broken case: the AI must only build what the backend
|
| 406 |
+
# and the renderer already support, which is R1's own sentence. Delete this guard when D-137
|
| 407 |
+
# closes — the refusal names the debt so the reader knows it is a limit, not a rule.
|
| 408 |
+
if kind == 'chart' and by_key.get(spec.get('groupBy')) in ('multiselect', 'checkbox'):
|
| 409 |
+
return None, (f"a chart cannot group by {spec.get('groupBy')!r} yet — a column that holds "
|
| 410 |
+
f"several values at once, or a tick box, does not bucket the way a chart "
|
| 411 |
+
f"needs. Try a table grouped by it instead"), "chart_key_unsupported"
|
| 412 |
+
|
| 413 |
+
for ref, families, required in _MODE_REFS.get(kind, ()):
|
| 414 |
+
val = spec.get(ref)
|
| 415 |
+
if val and by_key.get(val) not in families:
|
| 416 |
+
return None, (f"a {kind} view needs {ref[:-5]} to be a "
|
| 417 |
+
f"{' or '.join(families)} column, and {val!r} is a "
|
| 418 |
+
f"{by_key.get(val)} column"), "wrong_ref_type"
|
| 419 |
+
if required and not val:
|
| 420 |
+
return None, (f"a {kind} view needs a {' or '.join(families)} column and this "
|
| 421 |
+
f"database has none that fits"), "missing_ref"
|
| 422 |
+
|
| 423 |
+
raw_filters = [f for f in (spec.get('filters') or ()) if isinstance(f, dict)]
|
| 424 |
+
filters = g.clean_filter_tree(raw_filters, keys)
|
| 425 |
+
if len(filters) != len(raw_filters):
|
| 426 |
+
# The belt caught something membership did not — an unknown op. Refuse rather than save a
|
| 427 |
+
# view whose condition was silently dropped (the widening this file exists to prevent).
|
| 428 |
+
return None, ("part of that filter is not something this product can express — try "
|
| 429 |
+
"asking it more simply"), "filter_dropped"
|
| 430 |
+
|
| 431 |
+
display = {'mode': kind}
|
| 432 |
+
for ref, _families, _req in _MODE_REFS.get(kind, ()):
|
| 433 |
+
if spec.get(ref):
|
| 434 |
+
display[ref] = spec[ref]
|
| 435 |
+
cleaned_display = g._clean_display(display, keys) if kind != 'grid' else None
|
| 436 |
+
if kind != 'grid' and not cleaned_display:
|
| 437 |
+
return None, (f"this product could not build a {kind} view from that question"
|
| 438 |
+
), "display_dropped"
|
| 439 |
+
|
| 440 |
+
clean = {
|
| 441 |
+
'kind': kind,
|
| 442 |
+
'name': ' '.join(str(spec.get('name') or 'Query').split())[:60] or 'Query',
|
| 443 |
+
'visible': visible,
|
| 444 |
+
'filters': filters,
|
| 445 |
+
'filterConj': 'or' if spec.get('filterConj') == 'or' else 'and',
|
| 446 |
+
'sorts': [{'colId': s['colId'], 'dir': 'desc' if s.get('dir') == 'desc' else 'asc'}
|
| 447 |
+
for s in (spec.get('sorts') or ()) if isinstance(s, dict)
|
| 448 |
+
and s.get('colId') in keys][:3],
|
| 449 |
+
'groupBy': spec['groupBy'] if spec.get('groupBy') in keys else None,
|
| 450 |
+
'display': cleaned_display,
|
| 451 |
+
}
|
| 452 |
+
return clean, None, None
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
def _explain(clean, fields):
|
| 456 |
+
"""A plain-language read-back of what the view WILL do — DERIVED from the validated spec.
|
| 457 |
+
|
| 458 |
+
⛔ NEVER ASKED OF THE MODEL. A model describing its own output describes what it MEANT, which
|
| 459 |
+
is precisely the failure this sentence exists to expose: a spec whose every column is real and
|
| 460 |
+
which answers a different question is invisible to validation, so the only thing that catches
|
| 461 |
+
it is a person reading an honest description of the spec that was actually accepted.
|
| 462 |
+
"""
|
| 463 |
+
label = {f['key']: f['label'] for f in fields}
|
| 464 |
+
ops = {'eq': 'is', 'neq': 'is not', 'gt': 'is over', 'gte': 'is at least',
|
| 465 |
+
'lt': 'is under', 'lte': 'is at most', 'contains': 'contains',
|
| 466 |
+
'doesNotContain': 'does not contain', 'isEmpty': 'is empty',
|
| 467 |
+
'isNotEmpty': 'is not empty', 'between': 'is between', 'within': 'is within',
|
| 468 |
+
'topN': 'is in the top', 'bottomN': 'is in the bottom'}
|
| 469 |
+
kinds = {'grid': 'a table', 'list': 'a list', 'chart': 'a chart', 'kanban': 'a board',
|
| 470 |
+
'calendar': 'a calendar', 'timeseries': 'a time series', 'map': 'a map'}
|
| 471 |
+
parts = [f"{kinds.get(clean['kind'], clean['kind'])} of "
|
| 472 |
+
+ ", ".join(label.get(k, k) for k in clean['visible'][:6])
|
| 473 |
+
+ (f" and {len(clean['visible']) - 6} more columns"
|
| 474 |
+
if len(clean['visible']) > 6 else "")]
|
| 475 |
+
if clean['filters']:
|
| 476 |
+
joiner = " or " if clean['filterConj'] == 'or' else " and "
|
| 477 |
+
parts.append("showing only records where " + joiner.join(
|
| 478 |
+
f"{label.get(f['colId'], f['colId'])} {ops.get(f['op'], f['op'])}"
|
| 479 |
+
+ (f" {f['value']}" if f.get('value') else "")
|
| 480 |
+
for f in clean['filters'] if isinstance(f, dict) and f.get('colId')))
|
| 481 |
+
else:
|
| 482 |
+
parts.append("showing every record")
|
| 483 |
+
if clean['groupBy']:
|
| 484 |
+
parts.append(f"grouped by {label.get(clean['groupBy'], clean['groupBy'])}")
|
| 485 |
+
if clean['sorts']:
|
| 486 |
+
s = clean['sorts'][0]
|
| 487 |
+
parts.append(f"sorted by {label.get(s['colId'], s['colId'])}"
|
| 488 |
+
+ (" (highest first)" if s['dir'] == 'desc' else " (lowest first)"))
|
| 489 |
+
for ref, word in (('stackField', 'in columns by'), ('dateField', 'placed in time by'),
|
| 490 |
+
('colorField', 'coloured by'), ('sizeField', 'sized by')):
|
| 491 |
+
val = (clean.get('display') or {}).get(ref)
|
| 492 |
+
if val:
|
| 493 |
+
parts.append(f"{word} {label.get(val, val)}")
|
| 494 |
+
return ", ".join(parts) + "."
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
# ── the registry ────────────────────────────────────────────────────────────────────────────
|
| 498 |
+
|
| 499 |
+
def _qid(scope, question):
|
| 500 |
+
"""PINNED per (database, question), the `view_templates` discipline: asking the same thing
|
| 501 |
+
twice UPDATES one view instead of minting "Query 2" — `save_view` de-duplicates NAMES by
|
| 502 |
+
appending a number, so an unpinned id fills the store with numbered near-duplicates."""
|
| 503 |
+
h = hashlib.sha1(f"{scope}|{' '.join(str(question).split()).lower()}".encode("utf-8"))
|
| 504 |
+
return "q_" + h.hexdigest()[:12]
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
def _registry(session):
|
| 508 |
+
try:
|
| 509 |
+
recs = session.runtime.get(QUERY_KEY) or {}
|
| 510 |
+
except Exception:
|
| 511 |
+
return {}
|
| 512 |
+
return recs if isinstance(recs, dict) else {}
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
def _mine(session, rec):
|
| 516 |
+
return session.admin or str(rec.get('createdBy') or '') == session.uname
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
#: The BUILT-IN topics `routes_templates._target_or_refuse` accepts beside any `ut_*` key. ⛔ THE
|
| 520 |
+
#: CLIENT'S PICKER MUST MIRROR THIS, and `GET /query` publishes it (`builtins`) rather than leaving
|
| 521 |
+
#: the page to re-derive it: the nav hands `QueryPage` every granted entry, which is a WIDER set
|
| 522 |
+
#: than this door accepts, so a picker filtered on "not a surface" alone would offer a database the
|
| 523 |
+
#: build door 404s. `view_templates`' own rule — "a picker that offered a template the apply door
|
| 524 |
+
#: would then refuse is a control that lies" — and it is the same predicate twice, published once.
|
| 525 |
+
BUILTIN_SCOPES = ('customer_data', 'product_data')
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
@router.get("/query")
|
| 529 |
+
def list_queries(session: Session = Depends(require_session)):
|
| 530 |
+
"""The AI-built views this caller may open: their own, still-granted, still-existing.
|
| 531 |
+
|
| 532 |
+
⚠ PRUNED ON READ against the CURRENT wall, never merely on write — a grant can be withdrawn
|
| 533 |
+
and a database deleted after a view was saved, and the Query module is chrome: it may render
|
| 534 |
+
nothing the server would not grant today (`nav.ts`'s chrome law).
|
| 535 |
+
"""
|
| 536 |
+
import routes_templates
|
| 537 |
+
out = []
|
| 538 |
+
for qid, rec in sorted(_registry(session).items(),
|
| 539 |
+
key=lambda kv: str((kv[1] or {}).get('createdAt') or ''),
|
| 540 |
+
reverse=True):
|
| 541 |
+
if not isinstance(rec, dict) or not _mine(session, rec):
|
| 542 |
+
continue
|
| 543 |
+
try:
|
| 544 |
+
routes_templates._target_or_refuse(session, rec.get('scope'))
|
| 545 |
+
except Exception:
|
| 546 |
+
continue # no longer granted, or gone
|
| 547 |
+
out.append({"id": qid, "scope": rec.get('scope'), "viewId": rec.get('viewId'),
|
| 548 |
+
"name": rec.get('name'), "kind": rec.get('kind'),
|
| 549 |
+
"question": rec.get('question'), "explain": rec.get('explain'),
|
| 550 |
+
"createdAt": rec.get('createdAt')})
|
| 551 |
+
return {"views": out, "kinds": list(QUERY_KINDS), "builtins": list(BUILTIN_SCOPES)}
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
@router.post("/query/build")
|
| 555 |
+
def build_query(body: dict = Body(default=None),
|
| 556 |
+
session: Session = Depends(require_session)):
|
| 557 |
+
"""A question + a granted database -> `{spec, explain, refused}`. **WRITES NOTHING.**
|
| 558 |
+
|
| 559 |
+
A refusal is a 200 carrying a sentence, not a 5xx: "this database cannot answer that" is a
|
| 560 |
+
successful answer to the person asking, and an error envelope would paint the error page R1
|
| 561 |
+
forbids. The 4xx cases are the ones that ARE the caller's fault — an unnamed database, an
|
| 562 |
+
over-long question, a database this session may not open.
|
| 563 |
+
|
| 564 |
+
⛔ THE MODEL SEAM IS **NOT** ON THIS SIGNATURE, AND THAT IS A CORRECTION. The first draft took
|
| 565 |
+
`chat=None` as a parameter of the ROUTE — and FastAPI reads an un-annotated defaulted argument
|
| 566 |
+
as a **QUERY PARAMETER**, so `chat` appeared in `openapi()["paths"]` (measured) and
|
| 567 |
+
`POST /query/build?chat=x` would have reached `_call_model(..., chat="x")` and 500'd on
|
| 568 |
+
`"x"(messages, tools)`. The gate never saw it because the gate calls the handler directly —
|
| 569 |
+
which is exactly the shape of a test seam that becomes an input. The seam now lives on the
|
| 570 |
+
module-level `_build` below: the gate calls that, and the wire cannot.
|
| 571 |
+
"""
|
| 572 |
+
body = body if isinstance(body, dict) else {}
|
| 573 |
+
return _build(body.get("question"), body.get("scope"), session)
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
def _build(question, scope, session, chat=None):
|
| 577 |
+
"""`build_query`'s body, with the model injectable — see that route's note on why the seam is
|
| 578 |
+
here and not on the signature FastAPI reads."""
|
| 579 |
+
question = ' '.join(str(question or "").split())
|
| 580 |
+
if not question:
|
| 581 |
+
raise err(400, "bad_request", "no question was asked")
|
| 582 |
+
if len(question) > MAX_QUESTION:
|
| 583 |
+
raise err(400, "question_too_long",
|
| 584 |
+
f"a question must be under {MAX_QUESTION} characters")
|
| 585 |
+
fields, _source = _target(session, scope)
|
| 586 |
+
if not fields:
|
| 587 |
+
raise err(400, "no_columns", "that database has no columns to build a view from")
|
| 588 |
+
|
| 589 |
+
spec, transport, provider = _call_model(question, fields, chat=chat)
|
| 590 |
+
if transport:
|
| 591 |
+
return {"spec": None, "explain": None, "refused": transport, "provider": provider}
|
| 592 |
+
clean, refusal, code = _validate(spec, fields)
|
| 593 |
+
if refusal:
|
| 594 |
+
return {"spec": None, "explain": None, "refused": refusal, "reason": code,
|
| 595 |
+
"provider": provider}
|
| 596 |
+
return {"spec": clean, "explain": _explain(clean, fields),
|
| 597 |
+
"refused": None, "scope": scope,
|
| 598 |
+
"id": _qid(scope, question), "provider": provider}
|
| 599 |
+
|
| 600 |
+
|
| 601 |
+
@router.post("/query/save")
|
| 602 |
+
def save_query(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 603 |
+
"""Save an accepted spec as the CALLING USER's own view, and index it under Query.
|
| 604 |
+
|
| 605 |
+
⛔ THE SPEC IS RE-VALIDATED HERE, against the target's contract as it is NOW. The build door
|
| 606 |
+
wrote nothing, so this is the first write and it is the only place authorisation and validity
|
| 607 |
+
have to hold together — and a client is not a wall. An unsupported kind or an unknown column
|
| 608 |
+
is a **400 with a named code** on this door (the caller sent it), where the same fact is a
|
| 609 |
+
plain sentence on `build` (the model produced it).
|
| 610 |
+
"""
|
| 611 |
+
body = body if isinstance(body, dict) else {}
|
| 612 |
+
scope = str(body.get("scope") or '').strip()
|
| 613 |
+
question = ' '.join(str(body.get("question") or "").split())[:MAX_QUESTION]
|
| 614 |
+
fields, _source = _target(session, scope)
|
| 615 |
+
clean, refusal, code = _validate(body.get("spec"), fields)
|
| 616 |
+
if refusal:
|
| 617 |
+
raise err(400, code or "bad_spec", refusal)
|
| 618 |
+
if not session.runtime.available():
|
| 619 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved")
|
| 620 |
+
|
| 621 |
+
vt = _templates()
|
| 622 |
+
ws_key = vt.workspace_key(scope)
|
| 623 |
+
if not ws_key:
|
| 624 |
+
raise err(404, "unknown_table", "that database does not exist")
|
| 625 |
+
qid = _qid(scope, question) if question else _qid(scope, clean['name'])
|
| 626 |
+
|
| 627 |
+
# ⛔ THE SKELETON IS `view_templates._view`, NOT A SECOND ONE. "A view assembled from a
|
| 628 |
+
# DIFFERENT skeleton is a second definition of what a view is — and the two drift on the day
|
| 629 |
+
# one of them gains a key." It also puts the mode at `config.display.mode`, which is the key
|
| 630 |
+
# the grid actually reads (`config.displayMode` was the first draft, and nothing reads it).
|
| 631 |
+
payload = vt._view(qid, clean['name'], f"Built by the assistant from: {question}"[:300],
|
| 632 |
+
clean['visible'], mode=clean['kind'],
|
| 633 |
+
sorts=clean['sorts'], filters=clean['filters'],
|
| 634 |
+
group_by=clean['groupBy'], conj=clean['filterConj'])
|
| 635 |
+
if clean.get('display'):
|
| 636 |
+
payload['config']['display'] = dict(clean['display'])
|
| 637 |
+
payload['createdBy'] = session.uname
|
| 638 |
+
|
| 639 |
+
# ⛔ THE CAP REFUSES; IT DOES NOT EVICT — and the first draft evicted. Dropping the oldest entry
|
| 640 |
+
# would leave its saved VIEW behind in the table's own workspace: "stranded in the target's view
|
| 641 |
+
# list with no way back to the question that made it", which is the thing `delete_query` below
|
| 642 |
+
# exists to prevent. And an eviction nobody was told about is W30/R6's second sentence broken —
|
| 643 |
+
# a limit that cannot be removed must be REPORTED with its cause. So the 201st is a refusal
|
| 644 |
+
# naming the number and the way out, checked BEFORE the view is written.
|
| 645 |
+
existing = _registry(session)
|
| 646 |
+
if qid not in existing and len(existing) >= MAX_REGISTRY:
|
| 647 |
+
raise err(400, "query_limit",
|
| 648 |
+
f"you already have {len(existing)} saved questions, which is the limit "
|
| 649 |
+
f"({MAX_REGISTRY}). Delete one you no longer need and save this again.")
|
| 650 |
+
|
| 651 |
+
import core.table_store as table_store
|
| 652 |
+
ops = table_store.make(ws_key, st=session.runtime)
|
| 653 |
+
saved = ops.save_view(session.uname, payload, shared=False)
|
| 654 |
+
|
| 655 |
+
entry = {"scope": scope, "viewId": qid, "name": saved.get('name') or clean['name'],
|
| 656 |
+
"kind": clean['kind'], "question": question,
|
| 657 |
+
"explain": _explain(clean, fields), "createdBy": session.uname,
|
| 658 |
+
"createdAt": _now_iso()}
|
| 659 |
+
|
| 660 |
+
def _up(data):
|
| 661 |
+
data = data if isinstance(data, dict) else {}
|
| 662 |
+
data[qid] = entry
|
| 663 |
+
return data
|
| 664 |
+
|
| 665 |
+
session.runtime.update(QUERY_KEY, _up, flush='async')
|
| 666 |
+
return {"id": qid, **entry}
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
@router.delete("/query/{qid}")
|
| 670 |
+
def delete_query(qid: str, session: Session = Depends(require_session)):
|
| 671 |
+
"""Forget an AI-built view — the index entry AND the view itself.
|
| 672 |
+
|
| 673 |
+
Leaving the view behind would strand it in the target's view list with no way back to the
|
| 674 |
+
question that made it ([[a-record-can-outlive-its-subject]]).
|
| 675 |
+
"""
|
| 676 |
+
rec = _registry(session).get(str(qid))
|
| 677 |
+
if not isinstance(rec, dict) or not _mine(session, rec):
|
| 678 |
+
raise err(404, "unknown_query", "that view does not exist")
|
| 679 |
+
_target(session, rec.get('scope')) # the wall, again, on the write path
|
| 680 |
+
vt = _templates()
|
| 681 |
+
ws_key = vt.workspace_key(rec.get('scope'))
|
| 682 |
+
if ws_key and session.runtime.available():
|
| 683 |
+
import core.table_store as table_store
|
| 684 |
+
table_store.make(ws_key, st=session.runtime).delete_view(
|
| 685 |
+
session.uname, rec.get('viewId') or str(qid))
|
| 686 |
+
|
| 687 |
+
def _up(data):
|
| 688 |
+
data = data if isinstance(data, dict) else {}
|
| 689 |
+
data.pop(str(qid), None)
|
| 690 |
+
return data
|
| 691 |
+
|
| 692 |
+
session.runtime.update(QUERY_KEY, _up, flush='async')
|
| 693 |
+
return {"deleted": str(qid)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/routes_records.py
CHANGED
|
@@ -1,211 +1,164 @@
|
|
| 1 |
-
"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE.
|
| 2 |
-
|
| 3 |
-
⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
|
| 4 |
-
customer-shaped wall bolted to the module import line: `_in_book` asked
|
| 5 |
-
`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
|
| 6 |
-
typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
|
| 7 |
-
panel answered "that customer is not in your book" — the owner's report. The dangerous half is
|
| 8 |
-
the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
|
| 9 |
-
is filed against somebody's customer where the whole team can read it.
|
| 10 |
-
|
| 11 |
-
THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
|
| 12 |
-
events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
|
| 13 |
-
per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving
|
| 14 |
-
one here:
|
| 15 |
-
|
| 16 |
-
customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
|
| 17 |
-
product `routes_products.scoped_pool` behind the `product_data` grant
|
| 18 |
-
ut_<slug> `routes_tables.
|
| 19 |
-
(404 unknown / 403 not yours — a user table has no module grant)
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
⚠
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
"""
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
#
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
"
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
from core import record_comments
|
| 146 |
-
|
| 147 |
-
scope = _scope_or_400(scope)
|
| 148 |
-
_in_book(pid, session, scope)
|
| 149 |
-
try:
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
)
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
scope = _scope_or_400(scope)
|
| 167 |
-
_in_book(pid, session, scope)
|
| 168 |
-
try:
|
| 169 |
-
comment = record_comments.add_comment(
|
| 170 |
-
session.runtime,
|
| 171 |
-
pid,
|
| 172 |
-
(body or {}).get("body"),
|
| 173 |
-
session.uname,
|
| 174 |
-
session.user.get("name") or session.uname,
|
| 175 |
-
scope=scope,
|
| 176 |
-
)
|
| 177 |
-
except ValueError as exc:
|
| 178 |
-
raise err(400, "bad_comment", str(exc))
|
| 179 |
-
except record_comments.CommentsUnavailable:
|
| 180 |
-
raise _unavailable()
|
| 181 |
-
return {"comment": comment}
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
@router.delete("/records/{pid}/comments/{comment_id}")
|
| 185 |
-
@router.delete("/customers/{pid}/comments/{comment_id}")
|
| 186 |
-
def remove_comment(
|
| 187 |
-
pid: int,
|
| 188 |
-
comment_id: str,
|
| 189 |
-
scope: str = Query(default="customer"),
|
| 190 |
-
session: Session = Depends(require_session),
|
| 191 |
-
):
|
| 192 |
-
from core import record_comments
|
| 193 |
-
|
| 194 |
-
scope = _scope_or_400(scope)
|
| 195 |
-
_in_book(pid, session, scope)
|
| 196 |
-
try:
|
| 197 |
-
deleted = record_comments.delete_comment(
|
| 198 |
-
session.runtime,
|
| 199 |
-
pid,
|
| 200 |
-
comment_id,
|
| 201 |
-
session.uname,
|
| 202 |
-
admin=session.admin,
|
| 203 |
-
scope=scope,
|
| 204 |
-
)
|
| 205 |
-
except record_comments.CommentForbidden:
|
| 206 |
-
raise err(403, "comment_forbidden", "only the author may delete this comment")
|
| 207 |
-
except record_comments.CommentsUnavailable:
|
| 208 |
-
raise _unavailable()
|
| 209 |
-
if not deleted:
|
| 210 |
-
raise err(404, "comment_not_found", "that comment no longer exists")
|
| 211 |
-
return {"ok": True, "id": comment_id}
|
|
|
|
| 1 |
+
"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE.
|
| 2 |
+
|
| 3 |
+
⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
|
| 4 |
+
customer-shaped wall bolted to the module import line: `_in_book` asked
|
| 5 |
+
`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
|
| 6 |
+
typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
|
| 7 |
+
panel answered "that customer is not in your book" — the owner's report. The dangerous half is
|
| 8 |
+
the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
|
| 9 |
+
is filed against somebody's customer where the whole team can read it.
|
| 10 |
+
|
| 11 |
+
THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
|
| 12 |
+
events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
|
| 13 |
+
per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving
|
| 14 |
+
one here:
|
| 15 |
+
|
| 16 |
+
customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
|
| 17 |
+
product `routes_products.scoped_pool` behind the `product_data` grant
|
| 18 |
+
ut_<slug> `routes_tables.scoped_pool`, whose `_defn_or_refuse` IS the wall
|
| 19 |
+
(404 unknown / 403 not yours — a user table has no module grant)
|
| 20 |
+
|
| 21 |
+
⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
|
| 22 |
+
section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
|
| 23 |
+
another session's gate mid-wave — the WALL is the query parameter, not the word.
|
| 24 |
+
|
| 25 |
+
⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
|
| 26 |
+
shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
|
| 27 |
+
400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
|
| 28 |
+
the same reason: a typo served as `customer` answers a question nobody asked).
|
| 29 |
+
"""
|
| 30 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 31 |
+
|
| 32 |
+
from deps import Session, err, require_session
|
| 33 |
+
|
| 34 |
+
router = APIRouter(prefix="/api/v1")
|
| 35 |
+
|
| 36 |
+
#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table
|
| 37 |
+
#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
|
| 38 |
+
_CUSTOMER_SCOPES = ("", "customer", "cohort")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _scope_or_400(raw):
|
| 42 |
+
scope = str(raw or "customer").strip().lower()
|
| 43 |
+
if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
|
| 44 |
+
return "customer" if scope in _CUSTOMER_SCOPES else scope
|
| 45 |
+
raise err(400, "bad_scope",
|
| 46 |
+
"scope must be customer, cohort, product or a ut_ database — refusing to guess")
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _pool_or_refuse(session: Session, scope: str):
|
| 50 |
+
"""The pids this session may attach comments to ON THIS DATABASE — grant wall included.
|
| 51 |
+
|
| 52 |
+
Returns the frozenset. Raises the topic's own 403/404/503, so a caller who may not open the
|
| 53 |
+
surface never learns anything about the row they asked about.
|
| 54 |
+
"""
|
| 55 |
+
if scope == "product":
|
| 56 |
+
from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
|
| 57 |
+
|
| 58 |
+
session.require(PRODUCT_MODULE)
|
| 59 |
+
pids, _team, _rows, _fields = scoped_pool(session)
|
| 60 |
+
return pids
|
| 61 |
+
if scope.startswith("ut_"):
|
| 62 |
+
# No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pool` IS
|
| 63 |
+
# the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
|
| 64 |
+
# rows routes do.
|
| 65 |
+
from routes_tables import scoped_pool
|
| 66 |
+
|
| 67 |
+
pids, _rows, _fields, _defn = scoped_pool(session, scope)
|
| 68 |
+
return pids
|
| 69 |
+
from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
|
| 70 |
+
|
| 71 |
+
session.require(CUSTOMER_MODULE)
|
| 72 |
+
return frozenset(allowed_pids(session))
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _in_book(pid, session, scope):
|
| 76 |
+
if pid not in _pool_or_refuse(session, scope):
|
| 77 |
+
# 403, not 404: the record may exist, but this session may not inspect it.
|
| 78 |
+
raise err(403, "out_of_scope", "that record is not in your book")
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _unavailable():
|
| 82 |
+
return err(
|
| 83 |
+
503,
|
| 84 |
+
"store_unavailable",
|
| 85 |
+
"record comments are temporarily unavailable — no change was saved",
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD
|
| 90 |
+
# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
|
| 91 |
+
# wall was always the query param). The old path stays as an ALIAS because the shipped client
|
| 92 |
+
# still calls it; verify_api pins the canonical path AND that the alias answers, so removing
|
| 93 |
+
# the alias later is a decision, never an accident.
|
| 94 |
+
@router.get("/records/{pid}/comments")
|
| 95 |
+
@router.get("/customers/{pid}/comments")
|
| 96 |
+
def comments(pid: int, scope: str = Query(default="customer"),
|
| 97 |
+
session: Session = Depends(require_session)):
|
| 98 |
+
from core import record_comments
|
| 99 |
+
|
| 100 |
+
scope = _scope_or_400(scope)
|
| 101 |
+
_in_book(pid, session, scope)
|
| 102 |
+
try:
|
| 103 |
+
rows = record_comments.list_comments(session.runtime, pid, scope=scope)
|
| 104 |
+
except record_comments.CommentsUnavailable:
|
| 105 |
+
raise _unavailable()
|
| 106 |
+
return {"comments": rows}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
@router.post("/records/{pid}/comments", status_code=201)
|
| 110 |
+
@router.post("/customers/{pid}/comments", status_code=201)
|
| 111 |
+
def create_comment(
|
| 112 |
+
pid: int,
|
| 113 |
+
body: dict = Body(default=None),
|
| 114 |
+
scope: str = Query(default="customer"),
|
| 115 |
+
session: Session = Depends(require_session),
|
| 116 |
+
):
|
| 117 |
+
from core import record_comments
|
| 118 |
+
|
| 119 |
+
scope = _scope_or_400(scope)
|
| 120 |
+
_in_book(pid, session, scope)
|
| 121 |
+
try:
|
| 122 |
+
comment = record_comments.add_comment(
|
| 123 |
+
session.runtime,
|
| 124 |
+
pid,
|
| 125 |
+
(body or {}).get("body"),
|
| 126 |
+
session.uname,
|
| 127 |
+
session.user.get("name") or session.uname,
|
| 128 |
+
scope=scope,
|
| 129 |
+
)
|
| 130 |
+
except ValueError as exc:
|
| 131 |
+
raise err(400, "bad_comment", str(exc))
|
| 132 |
+
except record_comments.CommentsUnavailable:
|
| 133 |
+
raise _unavailable()
|
| 134 |
+
return {"comment": comment}
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@router.delete("/records/{pid}/comments/{comment_id}")
|
| 138 |
+
@router.delete("/customers/{pid}/comments/{comment_id}")
|
| 139 |
+
def remove_comment(
|
| 140 |
+
pid: int,
|
| 141 |
+
comment_id: str,
|
| 142 |
+
scope: str = Query(default="customer"),
|
| 143 |
+
session: Session = Depends(require_session),
|
| 144 |
+
):
|
| 145 |
+
from core import record_comments
|
| 146 |
+
|
| 147 |
+
scope = _scope_or_400(scope)
|
| 148 |
+
_in_book(pid, session, scope)
|
| 149 |
+
try:
|
| 150 |
+
deleted = record_comments.delete_comment(
|
| 151 |
+
session.runtime,
|
| 152 |
+
pid,
|
| 153 |
+
comment_id,
|
| 154 |
+
session.uname,
|
| 155 |
+
admin=session.admin,
|
| 156 |
+
scope=scope,
|
| 157 |
+
)
|
| 158 |
+
except record_comments.CommentForbidden:
|
| 159 |
+
raise err(403, "comment_forbidden", "only the author may delete this comment")
|
| 160 |
+
except record_comments.CommentsUnavailable:
|
| 161 |
+
raise _unavailable()
|
| 162 |
+
if not deleted:
|
| 163 |
+
raise err(404, "comment_not_found", "that comment no longer exists")
|
| 164 |
+
return {"ok": True, "id": comment_id}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/routes_shares.py
CHANGED
|
@@ -1,393 +1,385 @@
|
|
| 1 |
-
"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
|
| 2 |
-
|
| 3 |
-
GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
|
| 4 |
-
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
-
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
| 6 |
-
|
| 7 |
-
`kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail
|
| 8 |
-
already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
|
| 9 |
-
(R10: "the same picker views use").
|
| 10 |
-
|
| 11 |
-
⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
|
| 12 |
-
`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
|
| 13 |
-
CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could
|
| 14 |
-
widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
|
| 15 |
-
for non-administrators; that is a courtesy, and this check is the wall.
|
| 16 |
-
|
| 17 |
-
⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means
|
| 18 |
-
every account that can already open the surface: `require_session` plus the topic's own gate run
|
| 19 |
-
first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
|
| 20 |
-
closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
|
| 21 |
-
could already reach the data ([[aios-permissioning]]).
|
| 22 |
-
|
| 23 |
-
⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
|
| 24 |
-
MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
|
| 25 |
-
and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
|
| 26 |
-
DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes
|
| 27 |
-
`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
|
| 28 |
-
by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every
|
| 29 |
-
column — and an `*` database grant admits every account in the tenant to all of it.
|
| 30 |
-
That is a real capability, deliberately kept; what was wrong was a docstring promising a second
|
| 31 |
-
wall that does not exist for this kind. Scoping user tables is booked, not done
|
| 32 |
-
(`waves/wave32/sharing-audit.md` S-8).
|
| 33 |
-
|
| 34 |
-
⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
|
| 35 |
-
registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
|
| 36 |
-
— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here
|
| 37 |
-
whose object is invisible under that one is a row in a list that opens a refusal, which is what
|
| 38 |
-
made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
|
| 39 |
-
the two vocabularies are still two.
|
| 40 |
-
"""
|
| 41 |
-
from fastapi import APIRouter, Body, Depends
|
| 42 |
-
|
| 43 |
-
import core.shares as shares
|
| 44 |
-
import core.users as users
|
| 45 |
-
from deps import Session, err, require_session
|
| 46 |
-
# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
|
| 47 |
-
# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
|
| 48 |
-
# reader agreeing about one string is the whole difference between an Inbox row that opens the
|
| 49 |
-
# shared database and one that is quietly unclickable.
|
| 50 |
-
from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
|
| 51 |
-
|
| 52 |
-
router = APIRouter(prefix="/api/v1")
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def _kind_or_400(raw):
|
| 56 |
-
try:
|
| 57 |
-
return shares._check_kind(raw)
|
| 58 |
-
except ValueError as e:
|
| 59 |
-
raise err(400, "bad_kind", str(e))
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ─────────────
|
| 63 |
-
#
|
| 64 |
-
# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
|
| 65 |
-
# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
|
| 66 |
-
# strings off the URL and the only dependency was `require_session`, so any signed-in account
|
| 67 |
-
# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
|
| 68 |
-
# an object with no grant record skipped the check entirely and the caller was stamped OWNER —
|
| 69 |
-
# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
|
| 70 |
-
# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
|
| 71 |
-
#
|
| 72 |
-
# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
|
| 73 |
-
# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
|
| 74 |
-
# victim next opens the dialog.
|
| 75 |
-
|
| 76 |
-
#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
|
| 77 |
-
#: route is not told which topic — so resolving one means asking each.
|
| 78 |
-
_BUILTIN_TOPICS = ("customer", "product")
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def _topics(session):
|
| 82 |
-
"""Every topic whose workspace could hold a view or folder for this tenant.
|
| 83 |
-
|
| 84 |
-
⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on
|
| 85 |
-
tenant #0) to answer a question about KEYS (D-185).
|
| 86 |
-
"""
|
| 87 |
-
try:
|
| 88 |
-
import core.user_tables as ut
|
| 89 |
-
return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
|
| 90 |
-
except Exception: # noqa: BLE001
|
| 91 |
-
return _BUILTIN_TOPICS
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
def _owns_object(session, kind, oid):
|
| 95 |
-
"""May this caller CLAIM an object that has no grant record yet — i.e. do they own it?
|
| 96 |
-
|
| 97 |
-
⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
|
| 98 |
-
each topic's workspace in turn, which is N store reads; making every share call pay that
|
| 99 |
-
would put a loop on a route the manage-access dialog opens. The dangerous path is the one
|
| 100 |
-
where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs
|
| 101 |
-
exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
|
| 102 |
-
"""
|
| 103 |
-
if session.admin:
|
| 104 |
-
return True
|
| 105 |
-
if kind == "database":
|
| 106 |
-
# ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
|
| 107 |
-
# admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
|
| 108 |
-
# here would be the second definition this wave keeps finding.
|
| 109 |
-
try:
|
| 110 |
-
import core.user_tables as ut
|
| 111 |
-
return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
|
| 112 |
-
st=session.runtime))
|
| 113 |
-
except Exception: # noqa: BLE001
|
| 114 |
-
return False
|
| 115 |
-
try:
|
| 116 |
-
import core.table_store as table_store
|
| 117 |
-
except Exception: # noqa: BLE001
|
| 118 |
-
return False
|
| 119 |
-
for topic in _topics(session):
|
| 120 |
-
try:
|
| 121 |
-
ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
|
| 122 |
-
hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
|
| 123 |
-
except Exception: # noqa: BLE001
|
| 124 |
-
continue
|
| 125 |
-
if hit:
|
| 126 |
-
# `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
|
| 127 |
-
# person whose personal stratum holds it — anybody else reaching this line is
|
| 128 |
-
# exactly the case S-1 describes.
|
| 129 |
-
return str(hit[0]) == str(session.uname)
|
| 130 |
-
return False
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
def _can_see_object(session, kind, oid):
|
| 134 |
-
"""May this caller READ an object's grant list — i.e. can they reach the object at all?
|
| 135 |
-
|
| 136 |
-
⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
|
| 137 |
-
REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
|
| 138 |
-
ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
|
| 139 |
-
PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
|
| 140 |
-
`permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit
|
| 141 |
-
in the grid** — answered `404` when bob opened its manage-access dialog. Measured before
|
| 142 |
-
fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
|
| 143 |
-
`404 no_object`.
|
| 144 |
-
⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
|
| 145 |
-
and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
|
| 146 |
-
`permissions` (system B) — which is the one that actually decides who may OPEN it.
|
| 147 |
-
⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
|
| 148 |
-
gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too —
|
| 149 |
-
'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
|
| 150 |
-
legitimate collaborator their view does not exist.
|
| 151 |
-
|
| 152 |
-
⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
|
| 153 |
-
owner — that is S-1, and widening this predicate onto `put_share` would re-open it.
|
| 154 |
-
"""
|
| 155 |
-
if _owns_object(session, kind, oid):
|
| 156 |
-
return True
|
| 157 |
-
if kind != "view":
|
| 158 |
-
# A folder carries no per-object visibility flag of its own, and a database's `may_open`
|
| 159 |
-
# (inside `_owns_object`) already admits grantees. Nothing wider to ask.
|
| 160 |
-
return False
|
| 161 |
-
try:
|
| 162 |
-
import core.table_store as table_store
|
| 163 |
-
for topic in _topics(session):
|
| 164 |
-
hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
|
| 165 |
-
if hit:
|
| 166 |
-
return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
|
| 167 |
-
session.uname, is_admin=session.admin))
|
| 168 |
-
except Exception: # noqa: BLE001
|
| 169 |
-
return False
|
| 170 |
-
return False
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
def _entries_or_400(session, entries):
|
| 174 |
-
"""Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME.
|
| 175 |
-
|
| 176 |
-
⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
|
| 177 |
-
a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
|
| 178 |
-
of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**,
|
| 179 |
-
so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
|
| 180 |
-
sharer believes the person has access. That is item 18's plain reading.
|
| 181 |
-
⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
|
| 182 |
-
(`_people`). One route, two populations, and the write door was the permissive one.
|
| 183 |
-
⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for
|
| 184 |
-
"every account that can already open the surface".
|
| 185 |
-
"""
|
| 186 |
-
known = {p["username"].strip().lower() for p in _people(session.tenant)}
|
| 187 |
-
unknown = []
|
| 188 |
-
for e in entries or ():
|
| 189 |
-
if not isinstance(e, dict):
|
| 190 |
-
continue
|
| 191 |
-
user = str(e.get("user") or "").strip().lower()
|
| 192 |
-
if user and user != shares.EVERYONE and user not in known:
|
| 193 |
-
unknown.append(user)
|
| 194 |
-
if unknown:
|
| 195 |
-
raise err(400, "unknown_people",
|
| 196 |
-
"no active account in this workspace is named "
|
| 197 |
-
+ ", ".join(sorted(set(unknown)))
|
| 198 |
-
+ " — nothing was shared. Pick people from the list rather than typing a name.")
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
@router.get("/share/mine")
|
| 202 |
-
def my_shares(session: Session = Depends(require_session)):
|
| 203 |
-
"""Everything shared WITH me, by kind — the "Shared with me" rail section (R10).
|
| 204 |
-
|
| 205 |
-
Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
|
| 206 |
-
in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
|
| 207 |
-
that is not malformed at all.
|
| 208 |
-
"""
|
| 209 |
-
return shares.shared_with(session.uname, st=session.runtime)
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
@router.get("/share/{kind}/{oid}")
|
| 213 |
-
def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
| 214 |
-
kind = _kind_or_400(kind)
|
| 215 |
-
rec = shares.grants(kind, oid, st=session.runtime)
|
| 216 |
-
role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
|
| 217 |
-
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 218 |
-
st=session.runtime)
|
| 219 |
-
# ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 220 |
-
# who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
|
| 221 |
-
# to any signed-in session, about objects it cannot open. Now a caller with no role on an
|
| 222 |
-
# object must prove they can reach it, and gets a 404 otherwise: the same answer a
|
| 223 |
-
# non-existent id gives, so the route cannot be used to probe which ids are real.
|
| 224 |
-
# ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
|
| 225 |
-
# caller who has no relationship with the object at all.
|
| 226 |
-
if role is None and not _can_see_object(session, kind, oid):
|
| 227 |
-
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 228 |
-
return {
|
| 229 |
-
**rec,
|
| 230 |
-
"role": role,
|
| 231 |
-
"mayAdminister": may_admin,
|
| 232 |
-
# ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 233 |
-
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
| 234 |
-
# display names — that list's shape cannot change without migrating cell values — so
|
| 235 |
-
# this route serves objects of its own. Existing grants that were written as lowercased
|
| 236 |
-
# display names are normalised by the wave-21 cleanup script.
|
| 237 |
-
# ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller
|
| 238 |
-
# who may open the editor. A read-only grantee gets the grant list (their fair question is
|
| 239 |
-
# "who else has this?") and not a directory of every account in the workspace.
|
| 240 |
-
"people": _people(session.tenant) if may_admin else [],
|
| 241 |
-
}
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
def _people(tenant):
|
| 245 |
-
"""[{username, name}] for this tenant — same population as `assignable_people`, with the
|
| 246 |
-
BINDING identity alongside the display one."""
|
| 247 |
-
try:
|
| 248 |
-
reg = users.registry() or {}
|
| 249 |
-
except Exception:
|
| 250 |
-
return []
|
| 251 |
-
want = str(tenant or '').strip().lower()
|
| 252 |
-
out = []
|
| 253 |
-
for uname, u in reg.items():
|
| 254 |
-
if not isinstance(u, dict) or u.get('active') is False:
|
| 255 |
-
continue
|
| 256 |
-
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 257 |
-
continue
|
| 258 |
-
out.append({"username": str(uname), "name": str(u.get('name') or uname)})
|
| 259 |
-
return sorted(out, key=lambda p: p["name"].lower())
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
@router.put("/share/{kind}/{oid}")
|
| 263 |
-
def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
| 264 |
-
session: Session = Depends(require_session)):
|
| 265 |
-
kind = _kind_or_400(kind)
|
| 266 |
-
body = body or {}
|
| 267 |
-
rec = shares.grants(kind, oid, st=session.runtime)
|
| 268 |
-
# An object with NO grant record yet has no owner — the first person to share it claims it.
|
| 269 |
-
# That is safe because reaching this route at all means passing the surface's own wall, and
|
| 270 |
-
# the alternative (refusing until somebody seeds an owner) would make a brand-new folder
|
| 271 |
-
# unshareable by the person who just made it.
|
| 272 |
-
if rec["owner"]:
|
| 273 |
-
if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 274 |
-
st=session.runtime):
|
| 275 |
-
raise err(403, "not_owner",
|
| 276 |
-
"only the owner of this item (or an administrator) can change who it is "
|
| 277 |
-
"shared with")
|
| 278 |
-
# ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
|
| 279 |
-
# is still claimed by the first person to share it — that rule is right, and refusing until
|
| 280 |
-
# somebody seeds an owner would make a brand-new folder unshareable by the person who just
|
| 281 |
-
# made it. What was missing is the half the old comment ASSERTED and the code never did: the
|
| 282 |
-
# claimant has to be able to reach the object. Without this, any signed-in account could
|
| 283 |
-
# stamp itself owner of an id it had never seen and lock the real creator out for good.
|
| 284 |
-
elif not _owns_object(session, kind, oid):
|
| 285 |
-
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 286 |
-
entries = body.get("entries")
|
| 287 |
-
if not isinstance(entries, list):
|
| 288 |
-
raise err(400, "bad_entries",
|
| 289 |
-
"entries must be a list of {user, role} — send [] to un-share, which is how "
|
| 290 |
-
"revoking is expressed")
|
| 291 |
-
_entries_or_400(session, entries)
|
| 292 |
-
out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
|
| 293 |
-
st=session.runtime)
|
| 294 |
-
_notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
|
| 295 |
-
return out
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
def _notify_new_grantees(session, kind, oid, before, after):
|
| 299 |
-
"""⭐⭐ W32-T28 (owner item 18's last clause, contract C3)
|
| 300 |
-
|
| 301 |
-
Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
|
| 302 |
-
silent: the grant landed in a rail section the receiver had to notice on their own, which is
|
| 303 |
-
why "I shared it with you" and "I never saw it" were both true.
|
| 304 |
-
|
| 305 |
-
⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
|
| 306 |
-
because an alert is a live question about rows; a share is an EVENT that happened once, and
|
| 307 |
-
polling for it would mean re-deriving "was this new?" on every inbox open — the diff below
|
| 308 |
-
only exists here, at the moment the set changes.
|
| 309 |
-
|
| 310 |
-
⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
|
| 311 |
-
so every save re-sends everyone who was already there. Diffing against `before` is what stops
|
| 312 |
-
a rename or a role change from ringing the bell for people whose access did not change.
|
| 313 |
-
⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
|
| 314 |
-
the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
|
| 315 |
-
⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the
|
| 316 |
-
grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
|
| 317 |
-
"""
|
| 318 |
-
try:
|
| 319 |
-
was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
|
| 320 |
-
fresh = [str(e.get("user")) for e in (after or ())
|
| 321 |
-
if isinstance(e, dict) and e.get("user") not in was
|
| 322 |
-
and e.get("user") != shares.EVERYONE]
|
| 323 |
-
if not fresh:
|
| 324 |
-
return
|
| 325 |
-
import core.alerts as alerts
|
| 326 |
-
|
| 327 |
-
label, route, view_id = _object_ref(session, kind, oid)
|
| 328 |
-
if not route:
|
| 329 |
-
# ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and
|
| 330 |
-
# `notification_view` would have to invent a target. Silence is the honest answer
|
| 331 |
-
# here; the rail still shows the grant under "Shared with me".
|
| 332 |
-
return
|
| 333 |
-
sharer = str(session.user.get("name") or session.uname)
|
| 334 |
-
for user in fresh:
|
| 335 |
-
# ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
|
| 336 |
-
# agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
|
| 337 |
-
# `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
|
| 338 |
-
# from there rather than typed again — one vocabulary, one owner.
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
name = str((row or {}).get("name") or "").strip()
|
| 387 |
-
# ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
|
| 388 |
-
# the grid and stops there rather than naming a view the receiver did not get.
|
| 389 |
-
return (name or ("A view" if kind == "view" else "A folder"),
|
| 390 |
-
route, str(oid) if kind == "view" else "")
|
| 391 |
-
except Exception: # noqa: BLE001
|
| 392 |
-
pass
|
| 393 |
-
return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
|
|
|
|
| 1 |
+
"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
|
| 2 |
+
|
| 3 |
+
GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
|
| 4 |
+
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
+
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
| 6 |
+
|
| 7 |
+
`kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail
|
| 8 |
+
already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
|
| 9 |
+
(R10: "the same picker views use").
|
| 10 |
+
|
| 11 |
+
⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
|
| 12 |
+
`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
|
| 13 |
+
CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could
|
| 14 |
+
widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
|
| 15 |
+
for non-administrators; that is a courtesy, and this check is the wall.
|
| 16 |
+
|
| 17 |
+
⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means
|
| 18 |
+
every account that can already open the surface: `require_session` plus the topic's own gate run
|
| 19 |
+
first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
|
| 20 |
+
closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
|
| 21 |
+
could already reach the data ([[aios-permissioning]]).
|
| 22 |
+
|
| 23 |
+
⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
|
| 24 |
+
MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
|
| 25 |
+
and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
|
| 26 |
+
DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes
|
| 27 |
+
`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
|
| 28 |
+
by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every
|
| 29 |
+
column — and an `*` database grant admits every account in the tenant to all of it.
|
| 30 |
+
That is a real capability, deliberately kept; what was wrong was a docstring promising a second
|
| 31 |
+
wall that does not exist for this kind. Scoping user tables is booked, not done
|
| 32 |
+
(`waves/wave32/sharing-audit.md` S-8).
|
| 33 |
+
|
| 34 |
+
⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
|
| 35 |
+
registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
|
| 36 |
+
— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here
|
| 37 |
+
whose object is invisible under that one is a row in a list that opens a refusal, which is what
|
| 38 |
+
made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
|
| 39 |
+
the two vocabularies are still two.
|
| 40 |
+
"""
|
| 41 |
+
from fastapi import APIRouter, Body, Depends
|
| 42 |
+
|
| 43 |
+
import core.shares as shares
|
| 44 |
+
import core.users as users
|
| 45 |
+
from deps import Session, err, require_session
|
| 46 |
+
# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
|
| 47 |
+
# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
|
| 48 |
+
# reader agreeing about one string is the whole difference between an Inbox row that opens the
|
| 49 |
+
# shared database and one that is quietly unclickable.
|
| 50 |
+
from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
|
| 51 |
+
|
| 52 |
+
router = APIRouter(prefix="/api/v1")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _kind_or_400(raw):
|
| 56 |
+
try:
|
| 57 |
+
return shares._check_kind(raw)
|
| 58 |
+
except ValueError as e:
|
| 59 |
+
raise err(400, "bad_kind", str(e))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ─────────────
|
| 63 |
+
#
|
| 64 |
+
# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
|
| 65 |
+
# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
|
| 66 |
+
# strings off the URL and the only dependency was `require_session`, so any signed-in account
|
| 67 |
+
# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
|
| 68 |
+
# an object with no grant record skipped the check entirely and the caller was stamped OWNER —
|
| 69 |
+
# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
|
| 70 |
+
# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
|
| 71 |
+
#
|
| 72 |
+
# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
|
| 73 |
+
# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
|
| 74 |
+
# victim next opens the dialog.
|
| 75 |
+
|
| 76 |
+
#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
|
| 77 |
+
#: route is not told which topic — so resolving one means asking each.
|
| 78 |
+
_BUILTIN_TOPICS = ("customer", "product")
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _topics(session):
|
| 82 |
+
"""Every topic whose workspace could hold a view or folder for this tenant.
|
| 83 |
+
|
| 84 |
+
⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on
|
| 85 |
+
tenant #0) to answer a question about KEYS (D-185).
|
| 86 |
+
"""
|
| 87 |
+
try:
|
| 88 |
+
import core.user_tables as ut
|
| 89 |
+
return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
|
| 90 |
+
except Exception: # noqa: BLE001
|
| 91 |
+
return _BUILTIN_TOPICS
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _owns_object(session, kind, oid):
|
| 95 |
+
"""May this caller CLAIM an object that has no grant record yet — i.e. do they own it?
|
| 96 |
+
|
| 97 |
+
⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
|
| 98 |
+
each topic's workspace in turn, which is N store reads; making every share call pay that
|
| 99 |
+
would put a loop on a route the manage-access dialog opens. The dangerous path is the one
|
| 100 |
+
where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs
|
| 101 |
+
exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
|
| 102 |
+
"""
|
| 103 |
+
if session.admin:
|
| 104 |
+
return True
|
| 105 |
+
if kind == "database":
|
| 106 |
+
# ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
|
| 107 |
+
# admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
|
| 108 |
+
# here would be the second definition this wave keeps finding.
|
| 109 |
+
try:
|
| 110 |
+
import core.user_tables as ut
|
| 111 |
+
return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
|
| 112 |
+
st=session.runtime))
|
| 113 |
+
except Exception: # noqa: BLE001
|
| 114 |
+
return False
|
| 115 |
+
try:
|
| 116 |
+
import core.table_store as table_store
|
| 117 |
+
except Exception: # noqa: BLE001
|
| 118 |
+
return False
|
| 119 |
+
for topic in _topics(session):
|
| 120 |
+
try:
|
| 121 |
+
ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
|
| 122 |
+
hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
|
| 123 |
+
except Exception: # noqa: BLE001
|
| 124 |
+
continue
|
| 125 |
+
if hit:
|
| 126 |
+
# `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
|
| 127 |
+
# person whose personal stratum holds it — anybody else reaching this line is
|
| 128 |
+
# exactly the case S-1 describes.
|
| 129 |
+
return str(hit[0]) == str(session.uname)
|
| 130 |
+
return False
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _can_see_object(session, kind, oid):
|
| 134 |
+
"""May this caller READ an object's grant list — i.e. can they reach the object at all?
|
| 135 |
+
|
| 136 |
+
⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
|
| 137 |
+
REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
|
| 138 |
+
ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
|
| 139 |
+
PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
|
| 140 |
+
`permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit
|
| 141 |
+
in the grid** — answered `404` when bob opened its manage-access dialog. Measured before
|
| 142 |
+
fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
|
| 143 |
+
`404 no_object`.
|
| 144 |
+
⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
|
| 145 |
+
and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
|
| 146 |
+
`permissions` (system B) — which is the one that actually decides who may OPEN it.
|
| 147 |
+
⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
|
| 148 |
+
gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too —
|
| 149 |
+
'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
|
| 150 |
+
legitimate collaborator their view does not exist.
|
| 151 |
+
|
| 152 |
+
⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
|
| 153 |
+
owner — that is S-1, and widening this predicate onto `put_share` would re-open it.
|
| 154 |
+
"""
|
| 155 |
+
if _owns_object(session, kind, oid):
|
| 156 |
+
return True
|
| 157 |
+
if kind != "view":
|
| 158 |
+
# A folder carries no per-object visibility flag of its own, and a database's `may_open`
|
| 159 |
+
# (inside `_owns_object`) already admits grantees. Nothing wider to ask.
|
| 160 |
+
return False
|
| 161 |
+
try:
|
| 162 |
+
import core.table_store as table_store
|
| 163 |
+
for topic in _topics(session):
|
| 164 |
+
hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
|
| 165 |
+
if hit:
|
| 166 |
+
return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
|
| 167 |
+
session.uname, is_admin=session.admin))
|
| 168 |
+
except Exception: # noqa: BLE001
|
| 169 |
+
return False
|
| 170 |
+
return False
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _entries_or_400(session, entries):
|
| 174 |
+
"""Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME.
|
| 175 |
+
|
| 176 |
+
⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
|
| 177 |
+
a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
|
| 178 |
+
of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**,
|
| 179 |
+
so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
|
| 180 |
+
sharer believes the person has access. That is item 18's plain reading.
|
| 181 |
+
⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
|
| 182 |
+
(`_people`). One route, two populations, and the write door was the permissive one.
|
| 183 |
+
⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for
|
| 184 |
+
"every account that can already open the surface".
|
| 185 |
+
"""
|
| 186 |
+
known = {p["username"].strip().lower() for p in _people(session.tenant)}
|
| 187 |
+
unknown = []
|
| 188 |
+
for e in entries or ():
|
| 189 |
+
if not isinstance(e, dict):
|
| 190 |
+
continue
|
| 191 |
+
user = str(e.get("user") or "").strip().lower()
|
| 192 |
+
if user and user != shares.EVERYONE and user not in known:
|
| 193 |
+
unknown.append(user)
|
| 194 |
+
if unknown:
|
| 195 |
+
raise err(400, "unknown_people",
|
| 196 |
+
"no active account in this workspace is named "
|
| 197 |
+
+ ", ".join(sorted(set(unknown)))
|
| 198 |
+
+ " — nothing was shared. Pick people from the list rather than typing a name.")
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
@router.get("/share/mine")
|
| 202 |
+
def my_shares(session: Session = Depends(require_session)):
|
| 203 |
+
"""Everything shared WITH me, by kind — the "Shared with me" rail section (R10).
|
| 204 |
+
|
| 205 |
+
Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
|
| 206 |
+
in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
|
| 207 |
+
that is not malformed at all.
|
| 208 |
+
"""
|
| 209 |
+
return shares.shared_with(session.uname, st=session.runtime)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
@router.get("/share/{kind}/{oid}")
|
| 213 |
+
def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
| 214 |
+
kind = _kind_or_400(kind)
|
| 215 |
+
rec = shares.grants(kind, oid, st=session.runtime)
|
| 216 |
+
role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
|
| 217 |
+
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 218 |
+
st=session.runtime)
|
| 219 |
+
# ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 220 |
+
# who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
|
| 221 |
+
# to any signed-in session, about objects it cannot open. Now a caller with no role on an
|
| 222 |
+
# object must prove they can reach it, and gets a 404 otherwise: the same answer a
|
| 223 |
+
# non-existent id gives, so the route cannot be used to probe which ids are real.
|
| 224 |
+
# ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
|
| 225 |
+
# caller who has no relationship with the object at all.
|
| 226 |
+
if role is None and not _can_see_object(session, kind, oid):
|
| 227 |
+
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 228 |
+
return {
|
| 229 |
+
**rec,
|
| 230 |
+
"role": role,
|
| 231 |
+
"mayAdminister": may_admin,
|
| 232 |
+
# ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 233 |
+
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
| 234 |
+
# display names — that list's shape cannot change without migrating cell values — so
|
| 235 |
+
# this route serves objects of its own. Existing grants that were written as lowercased
|
| 236 |
+
# display names are normalised by the wave-21 cleanup script.
|
| 237 |
+
# ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller
|
| 238 |
+
# who may open the editor. A read-only grantee gets the grant list (their fair question is
|
| 239 |
+
# "who else has this?") and not a directory of every account in the workspace.
|
| 240 |
+
"people": _people(session.tenant) if may_admin else [],
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _people(tenant):
|
| 245 |
+
"""[{username, name}] for this tenant — same population as `assignable_people`, with the
|
| 246 |
+
BINDING identity alongside the display one."""
|
| 247 |
+
try:
|
| 248 |
+
reg = users.registry() or {}
|
| 249 |
+
except Exception:
|
| 250 |
+
return []
|
| 251 |
+
want = str(tenant or '').strip().lower()
|
| 252 |
+
out = []
|
| 253 |
+
for uname, u in reg.items():
|
| 254 |
+
if not isinstance(u, dict) or u.get('active') is False:
|
| 255 |
+
continue
|
| 256 |
+
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 257 |
+
continue
|
| 258 |
+
out.append({"username": str(uname), "name": str(u.get('name') or uname)})
|
| 259 |
+
return sorted(out, key=lambda p: p["name"].lower())
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
@router.put("/share/{kind}/{oid}")
|
| 263 |
+
def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
| 264 |
+
session: Session = Depends(require_session)):
|
| 265 |
+
kind = _kind_or_400(kind)
|
| 266 |
+
body = body or {}
|
| 267 |
+
rec = shares.grants(kind, oid, st=session.runtime)
|
| 268 |
+
# An object with NO grant record yet has no owner — the first person to share it claims it.
|
| 269 |
+
# That is safe because reaching this route at all means passing the surface's own wall, and
|
| 270 |
+
# the alternative (refusing until somebody seeds an owner) would make a brand-new folder
|
| 271 |
+
# unshareable by the person who just made it.
|
| 272 |
+
if rec["owner"]:
|
| 273 |
+
if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 274 |
+
st=session.runtime):
|
| 275 |
+
raise err(403, "not_owner",
|
| 276 |
+
"only the owner of this item (or an administrator) can change who it is "
|
| 277 |
+
"shared with")
|
| 278 |
+
# ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
|
| 279 |
+
# is still claimed by the first person to share it — that rule is right, and refusing until
|
| 280 |
+
# somebody seeds an owner would make a brand-new folder unshareable by the person who just
|
| 281 |
+
# made it. What was missing is the half the old comment ASSERTED and the code never did: the
|
| 282 |
+
# claimant has to be able to reach the object. Without this, any signed-in account could
|
| 283 |
+
# stamp itself owner of an id it had never seen and lock the real creator out for good.
|
| 284 |
+
elif not _owns_object(session, kind, oid):
|
| 285 |
+
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 286 |
+
entries = body.get("entries")
|
| 287 |
+
if not isinstance(entries, list):
|
| 288 |
+
raise err(400, "bad_entries",
|
| 289 |
+
"entries must be a list of {user, role} — send [] to un-share, which is how "
|
| 290 |
+
"revoking is expressed")
|
| 291 |
+
_entries_or_400(session, entries)
|
| 292 |
+
out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
|
| 293 |
+
st=session.runtime)
|
| 294 |
+
_notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
|
| 295 |
+
return out
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def _notify_new_grantees(session, kind, oid, before, after):
|
| 299 |
+
"""⭐⭐ W32-T28 (owner item 18's last clause, contract C3) �� tell the RECEIVER, in their Inbox.
|
| 300 |
+
|
| 301 |
+
Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
|
| 302 |
+
silent: the grant landed in a rail section the receiver had to notice on their own, which is
|
| 303 |
+
why "I shared it with you" and "I never saw it" were both true.
|
| 304 |
+
|
| 305 |
+
⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
|
| 306 |
+
because an alert is a live question about rows; a share is an EVENT that happened once, and
|
| 307 |
+
polling for it would mean re-deriving "was this new?" on every inbox open — the diff below
|
| 308 |
+
only exists here, at the moment the set changes.
|
| 309 |
+
|
| 310 |
+
⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
|
| 311 |
+
so every save re-sends everyone who was already there. Diffing against `before` is what stops
|
| 312 |
+
a rename or a role change from ringing the bell for people whose access did not change.
|
| 313 |
+
⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
|
| 314 |
+
the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
|
| 315 |
+
⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the
|
| 316 |
+
grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
|
| 317 |
+
"""
|
| 318 |
+
try:
|
| 319 |
+
was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
|
| 320 |
+
fresh = [str(e.get("user")) for e in (after or ())
|
| 321 |
+
if isinstance(e, dict) and e.get("user") not in was
|
| 322 |
+
and e.get("user") != shares.EVERYONE]
|
| 323 |
+
if not fresh:
|
| 324 |
+
return
|
| 325 |
+
import core.alerts as alerts
|
| 326 |
+
|
| 327 |
+
label, route, view_id = _object_ref(session, kind, oid)
|
| 328 |
+
if not route:
|
| 329 |
+
# ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and
|
| 330 |
+
# `notification_view` would have to invent a target. Silence is the honest answer
|
| 331 |
+
# here; the rail still shows the grant under "Shared with me".
|
| 332 |
+
return
|
| 333 |
+
sharer = str(session.user.get("name") or session.uname)
|
| 334 |
+
for user in fresh:
|
| 335 |
+
# ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
|
| 336 |
+
# agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
|
| 337 |
+
# `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
|
| 338 |
+
# from there rather than typed again — one vocabulary, one owner.
|
| 339 |
+
alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
|
| 340 |
+
detail=f"{sharer} shared this with you", st=session.runtime)
|
| 341 |
+
except Exception: # noqa: BLE001
|
| 342 |
+
return
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def _object_ref(session, kind, oid):
|
| 346 |
+
"""`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS.
|
| 347 |
+
|
| 348 |
+
⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
|
| 349 |
+
WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
|
| 350 |
+
`target: {module: "database", id: "view_42"}` — an instruction to open a database named
|
| 351 |
+
`view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
|
| 352 |
+
addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
|
| 353 |
+
travel. Caught by looking at the notification the driver actually produced, not by reading
|
| 354 |
+
the code back.
|
| 355 |
+
|
| 356 |
+
⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
|
| 357 |
+
receiver nothing they can act on, and the id is already in the target.
|
| 358 |
+
⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
|
| 359 |
+
a row that opens nowhere.
|
| 360 |
+
"""
|
| 361 |
+
try:
|
| 362 |
+
if kind == "database":
|
| 363 |
+
import core.user_tables as ut
|
| 364 |
+
defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
|
| 365 |
+
# A user table IS its own route key in both vocabularies (`route_for_topic`).
|
| 366 |
+
return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
|
| 367 |
+
import core.table_store as table_store
|
| 368 |
+
from routes_alerts import route_for_topic
|
| 369 |
+
for topic in _topics(session):
|
| 370 |
+
ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
|
| 371 |
+
hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
|
| 372 |
+
if not hit:
|
| 373 |
+
continue
|
| 374 |
+
route = route_for_topic(topic)
|
| 375 |
+
if not route:
|
| 376 |
+
break
|
| 377 |
+
row = hit[1] if len(hit) > 1 else {}
|
| 378 |
+
name = str((row or {}).get("name") or "").strip()
|
| 379 |
+
# ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
|
| 380 |
+
# the grid and stops there rather than naming a view the receiver did not get.
|
| 381 |
+
return (name or ("A view" if kind == "view" else "A folder"),
|
| 382 |
+
route, str(oid) if kind == "view" else "")
|
| 383 |
+
except Exception: # noqa: BLE001
|
| 384 |
+
pass
|
| 385 |
+
return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/routes_tables.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
jobs/web_agent_job.py
CHANGED
|
@@ -113,21 +113,6 @@ INSTALL_DEPS = os.environ.get("WEB_AGENT_INSTALL_DEPS", "1") != "0"
|
|
| 113 |
INSTALL_BROWSER = os.environ.get("WEB_AGENT_INSTALL_BROWSER", "1") != "0"
|
| 114 |
MAX_VALUE_CHARS = 8000 # one cell's worth; a page is not a payload
|
| 115 |
MAX_STEPS = 20 # matches the engine's MAX_ACTIONS — one flow, one job
|
| 116 |
-
#: ⛔ D-193 — HOW OLD A CHECKPOINT MAY BE AND STILL BE SERVED, in hours. **Deliberately SHORT.**
|
| 117 |
-
#: The key is content-addressed (`runId` + a hash of the steps), so a scheduled automation with a
|
| 118 |
-
#: stable run id hits the same key every time it runs; with no ceiling it would be served the same
|
| 119 |
-
#: stored page for as long as the file existed, which is a measurement of a web page presented with
|
| 120 |
-
#: today's date on it. Six hours makes a same-morning retry cheap and a next-day run honest.
|
| 121 |
-
#: ⚠ A CEILING, NOT A CACHE POLICY. The point of the checkpoint is that a RETRY of a job that
|
| 122 |
-
#: already paid for a fetch does not pay again; that window is minutes, not days.
|
| 123 |
-
try:
|
| 124 |
-
CHECKPOINT_MAX_AGE_H = float(os.environ.get("WEB_AGENT_CHECKPOINT_MAX_AGE_H", "6") or 6)
|
| 125 |
-
except (TypeError, ValueError):
|
| 126 |
-
CHECKPOINT_MAX_AGE_H = 6.0
|
| 127 |
-
#: How many expired `runs/*.json` one job may delete, in ONE commit. ⛔ Bounded because the repo
|
| 128 |
-
#: already spends a commit per run against HF's 256-commits/hr budget, and a sweep that spent one
|
| 129 |
-
#: commit per FILE would be a denial of service on the store it is tidying.
|
| 130 |
-
CHECKPOINT_SWEEP_MAX = 200
|
| 131 |
|
| 132 |
|
| 133 |
def _fail(sentence: str, *, kind: str = "refused", steps=None) -> None:
|
|
@@ -177,8 +162,6 @@ def _checkpoint(blob: str, payload: dict) -> None:
|
|
| 177 |
repo_id=CHECKPOINT_REPO, repo_type="dataset",
|
| 178 |
commit_message=f"web-agent run {run_id}")
|
| 179 |
say(f"CHECKPOINT wrote runs/{run_id}.json to {CHECKPOINT_REPO}")
|
| 180 |
-
# ⚠ AFTER the write, never before: the sweep is housekeeping and the result is the job.
|
| 181 |
-
_sweep_expired()
|
| 182 |
except Exception as exc: # noqa: BLE001 — a boundary
|
| 183 |
# ⛔ CATCH EVERYTHING. This is a report-don't-raise boundary and the ways an upload can
|
| 184 |
# fail are not enumerable (429 commit budget, 403 scope, DNS, a repo that does not
|
|
@@ -188,94 +171,8 @@ def _checkpoint(blob: str, payload: dict) -> None:
|
|
| 188 |
f"the stdout frame still carries the result")
|
| 189 |
|
| 190 |
|
| 191 |
-
def _too_old(payload: dict) -> str:
|
| 192 |
-
""""" if this checkpoint may still be served, else the SENTENCE saying why it may not.
|
| 193 |
-
|
| 194 |
-
⛔ D-193 — A RESUME THAT IS A YEAR OLD IS NOT A RESUME, IT IS A STALE ANSWER SERVED AS FRESH.
|
| 195 |
-
The key is deliberately content-addressed (`runId` + a hash of the steps), so a scheduled
|
| 196 |
-
automation reusing a stable run id resumed the SAME STORED PAGE for as long as the file
|
| 197 |
-
existed. `finishedAt` was written by `_emit` from the first day and read by nothing.
|
| 198 |
-
⚠ AN UNREADABLE OR ABSENT TIMESTAMP IS TREATED AS TOO OLD, not as fresh. A checkpoint written
|
| 199 |
-
before this field existed carries no date, and "no date" must not be the one input that gets
|
| 200 |
-
served forever — fail toward re-fetching, which costs a job, rather than toward a wrong answer,
|
| 201 |
-
which costs a number nobody can explain.
|
| 202 |
-
"""
|
| 203 |
-
stamp = str(payload.get("finishedAt") or "")
|
| 204 |
-
if not stamp:
|
| 205 |
-
return "it carries no finishedAt, so its age cannot be established"
|
| 206 |
-
try:
|
| 207 |
-
when = datetime.fromisoformat(stamp)
|
| 208 |
-
if when.tzinfo is None:
|
| 209 |
-
when = when.replace(tzinfo=timezone.utc)
|
| 210 |
-
except Exception: # noqa: BLE001
|
| 211 |
-
return f"its finishedAt ({stamp[:40]}) is not a date this job can read"
|
| 212 |
-
hours = (datetime.now(timezone.utc) - when).total_seconds() / 3600.0
|
| 213 |
-
if hours > CHECKPOINT_MAX_AGE_H:
|
| 214 |
-
return (f"it finished {hours:.1f} h ago and the ceiling is {CHECKPOINT_MAX_AGE_H} h "
|
| 215 |
-
f"(WEB_AGENT_CHECKPOINT_MAX_AGE_H)")
|
| 216 |
-
return ""
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
def _sweep_expired() -> None:
|
| 220 |
-
"""Delete checkpoints past the ceiling, ONE COMMIT, at most `CHECKPOINT_SWEEP_MAX` of them.
|
| 221 |
-
|
| 222 |
-
⛔ D-193's SECOND CLAUSE, and it is a different problem from the first. Ageing out the READ
|
| 223 |
-
stops a stale answer being served; it does not stop the repo growing one file per run forever.
|
| 224 |
-
⛔ ONE COMMIT FOR THE WHOLE SWEEP. `upload_file`/`delete_file` each spend a commit, and this
|
| 225 |
-
repo already spends one per run against HF's 256-per-hour budget — a per-file sweep would
|
| 226 |
-
exhaust the budget it exists to protect, and the failure would land on the next run's WRITE.
|
| 227 |
-
`create_commit` with a list of delete operations is one commit whatever the count.
|
| 228 |
-
⚠ AGE IS READ FROM THE LISTING, not by downloading each file. `last_commit` is what the API
|
| 229 |
-
gives cheaply; downloading N files to read a timestamp inside them would make a tidy-up cost
|
| 230 |
-
more than the runs it is tidying.
|
| 231 |
-
⚠ REPORT-DON'T-RAISE, like `_checkpoint`: this runs AFTER the result is in hand, and a
|
| 232 |
-
housekeeping failure must never destroy a result that has already been paid for.
|
| 233 |
-
"""
|
| 234 |
-
if not (CHECKPOINT_REPO and HF_TOKEN):
|
| 235 |
-
return
|
| 236 |
-
try:
|
| 237 |
-
from huggingface_hub import CommitOperationDelete, HfApi
|
| 238 |
-
api = HfApi(token=HF_TOKEN)
|
| 239 |
-
cutoff = datetime.now(timezone.utc).timestamp() - CHECKPOINT_MAX_AGE_H * 3600.0
|
| 240 |
-
doomed = []
|
| 241 |
-
for info in api.list_repo_tree(CHECKPOINT_REPO, repo_type="dataset", path_in_repo="runs",
|
| 242 |
-
expand=True, recursive=False):
|
| 243 |
-
path = getattr(info, "path", "")
|
| 244 |
-
when = getattr(getattr(info, "last_commit", None), "date", None)
|
| 245 |
-
if not path.endswith(".json") or when is None:
|
| 246 |
-
continue
|
| 247 |
-
if when.timestamp() < cutoff:
|
| 248 |
-
doomed.append(path)
|
| 249 |
-
if len(doomed) >= CHECKPOINT_SWEEP_MAX:
|
| 250 |
-
break
|
| 251 |
-
if not doomed:
|
| 252 |
-
return
|
| 253 |
-
api.create_commit(repo_id=CHECKPOINT_REPO, repo_type="dataset",
|
| 254 |
-
operations=[CommitOperationDelete(path_in_repo=p) for p in doomed],
|
| 255 |
-
commit_message=f"sweep {len(doomed)} checkpoints past "
|
| 256 |
-
f"{CHECKPOINT_MAX_AGE_H} h")
|
| 257 |
-
say(f"CHECKPOINT swept {len(doomed)} expired run(s) in one commit")
|
| 258 |
-
except Exception as exc: # noqa: BLE001 — a boundary
|
| 259 |
-
say(f"CHECKPOINT sweep skipped ({type(exc).__name__}: {str(exc)[:160]}) — "
|
| 260 |
-
f"the result is unaffected")
|
| 261 |
-
|
| 262 |
-
|
| 263 |
def _resume(run_id: str):
|
| 264 |
-
"""A prior run's
|
| 265 |
-
|
| 266 |
-
⛔ D-199 — A FAILED RUN'S CHECKPOINT USED TO REPLAY FOREVER. `_fail` checkpoints too
|
| 267 |
-
(`_fail` → `_emit` → `_checkpoint`), so the stored envelope for a failure is exactly as
|
| 268 |
-
truthy as one for a success; the caller tested `if prior:` and nothing ever looked at `ok`.
|
| 269 |
-
Under a stable `runId` a failed fetch could then never be re-attempted — it replayed the
|
| 270 |
-
stored failure unchanged, with `resumedFromCheckpoint: True` on it. The defect was LATENT
|
| 271 |
-
only because no deployment set `WEB_AGENT_REPO`, i.e. it armed itself on the day somebody
|
| 272 |
-
switched checkpointing on, which is the day nobody would be reading this function.
|
| 273 |
-
⚠ THE TEST BELONGS HERE, NOT AT THE CALL SITE. `_resume` is the one place that decides what a
|
| 274 |
-
resumable checkpoint IS; a caller-side `if prior and prior.get("ok")` would be a second
|
| 275 |
-
definition, and the next caller would be written from the first one.
|
| 276 |
-
⚠ A REFUSED CHECKPOINT SAYS SO IN THE LOG. Silently re-running looks identical to never
|
| 277 |
-
having had a checkpoint, and those two need to be distinguishable at 3am.
|
| 278 |
-
"""
|
| 279 |
if not (CHECKPOINT_REPO and HF_TOKEN and run_id):
|
| 280 |
return None
|
| 281 |
try:
|
|
@@ -284,21 +181,9 @@ def _resume(run_id: str):
|
|
| 284 |
filename=f"runs/{run_id}.json", token=HF_TOKEN,
|
| 285 |
force_download=True)
|
| 286 |
with open(path, encoding="utf-8") as fh:
|
| 287 |
-
|
| 288 |
except Exception: # noqa: BLE001
|
| 289 |
return None # absent is the normal case
|
| 290 |
-
if not isinstance(prior, dict):
|
| 291 |
-
say(f"RESUME refused — runs/{run_id}.json is not an object; re-running")
|
| 292 |
-
return None
|
| 293 |
-
if not prior.get("ok"):
|
| 294 |
-
say(f"RESUME refused — the stored run FAILED "
|
| 295 |
-
f"({str(prior.get('error') or 'no reason recorded')[:120]}); re-running")
|
| 296 |
-
return None
|
| 297 |
-
stale = _too_old(prior)
|
| 298 |
-
if stale:
|
| 299 |
-
say(f"RESUME refused — {stale}; re-running")
|
| 300 |
-
return None
|
| 301 |
-
return prior
|
| 302 |
|
| 303 |
|
| 304 |
# ── the plan ────────────────────────────────────────────────────────────────────────────────
|
|
|
|
| 113 |
INSTALL_BROWSER = os.environ.get("WEB_AGENT_INSTALL_BROWSER", "1") != "0"
|
| 114 |
MAX_VALUE_CHARS = 8000 # one cell's worth; a page is not a payload
|
| 115 |
MAX_STEPS = 20 # matches the engine's MAX_ACTIONS — one flow, one job
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
def _fail(sentence: str, *, kind: str = "refused", steps=None) -> None:
|
|
|
|
| 162 |
repo_id=CHECKPOINT_REPO, repo_type="dataset",
|
| 163 |
commit_message=f"web-agent run {run_id}")
|
| 164 |
say(f"CHECKPOINT wrote runs/{run_id}.json to {CHECKPOINT_REPO}")
|
|
|
|
|
|
|
| 165 |
except Exception as exc: # noqa: BLE001 — a boundary
|
| 166 |
# ⛔ CATCH EVERYTHING. This is a report-don't-raise boundary and the ways an upload can
|
| 167 |
# fail are not enumerable (429 commit budget, 403 scope, DNS, a repo that does not
|
|
|
|
| 171 |
f"the stdout frame still carries the result")
|
| 172 |
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
def _resume(run_id: str):
|
| 175 |
+
"""A prior run's result, if this exact run already finished. None otherwise."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
if not (CHECKPOINT_REPO and HF_TOKEN and run_id):
|
| 177 |
return None
|
| 178 |
try:
|
|
|
|
| 181 |
filename=f"runs/{run_id}.json", token=HF_TOKEN,
|
| 182 |
force_download=True)
|
| 183 |
with open(path, encoding="utf-8") as fh:
|
| 184 |
+
return json.load(fh)
|
| 185 |
except Exception: # noqa: BLE001
|
| 186 |
return None # absent is the normal case
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
|
| 188 |
|
| 189 |
# ── the plan ────────────────────────────────────────────────────────────────────────────────
|
platform/aios_grid.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/aios_grid_fields.json
CHANGED
|
@@ -1,474 +1,464 @@
|
|
| 1 |
-
{
|
| 2 |
-
"_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
|
| 3 |
-
"fields": [
|
| 4 |
-
{
|
| 5 |
-
"key": "customer",
|
| 6 |
-
"label": "Customer",
|
| 7 |
-
"type": "text",
|
| 8 |
-
"source": "odoo",
|
| 9 |
-
"pinned": true,
|
| 10 |
-
"default": true,
|
| 11 |
-
"description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 12 |
-
},
|
| 13 |
-
{
|
| 14 |
-
"key": "
|
| 15 |
-
"label": "Odoo
|
| 16 |
-
"type": "
|
| 17 |
-
"source": "odoo",
|
| 18 |
-
"
|
| 19 |
-
"
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
"
|
| 24 |
-
"
|
| 25 |
-
"
|
| 26 |
-
"
|
| 27 |
-
"
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
"
|
| 33 |
-
"
|
| 34 |
-
"
|
| 35 |
-
"
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
"
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
"
|
| 44 |
-
"
|
| 45 |
-
"
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
"
|
| 54 |
-
"
|
| 55 |
-
"
|
| 56 |
-
"
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
"
|
| 62 |
-
"
|
| 63 |
-
"
|
| 64 |
-
"
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
"
|
| 70 |
-
"
|
| 71 |
-
"
|
| 72 |
-
"
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
"
|
| 78 |
-
"
|
| 79 |
-
"
|
| 80 |
-
"
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
"
|
| 86 |
-
"
|
| 87 |
-
"
|
| 88 |
-
"
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
"
|
| 94 |
-
"
|
| 95 |
-
"
|
| 96 |
-
"
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
"
|
| 102 |
-
"
|
| 103 |
-
"
|
| 104 |
-
"
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
"
|
| 110 |
-
"
|
| 111 |
-
"
|
| 112 |
-
"
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
"
|
| 118 |
-
"
|
| 119 |
-
"
|
| 120 |
-
"
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
"
|
| 126 |
-
"
|
| 127 |
-
"
|
| 128 |
-
"
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
"
|
| 134 |
-
"
|
| 135 |
-
"
|
| 136 |
-
"
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
"
|
| 142 |
-
"
|
| 143 |
-
"
|
| 144 |
-
"
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
"
|
| 150 |
-
"
|
| 151 |
-
"
|
| 152 |
-
"
|
| 153 |
-
"
|
| 154 |
-
"
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
"
|
| 159 |
-
"
|
| 160 |
-
"
|
| 161 |
-
"
|
| 162 |
-
"
|
| 163 |
-
"
|
| 164 |
-
"
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
"
|
| 169 |
-
"
|
| 170 |
-
"
|
| 171 |
-
"
|
| 172 |
-
"
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
"
|
| 178 |
-
"
|
| 179 |
-
"
|
| 180 |
-
"
|
| 181 |
-
"
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
"
|
| 186 |
-
"
|
| 187 |
-
"
|
| 188 |
-
"
|
| 189 |
-
"
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
"
|
| 195 |
-
"
|
| 196 |
-
"
|
| 197 |
-
"
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
"
|
| 203 |
-
"
|
| 204 |
-
"
|
| 205 |
-
"
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
"
|
| 211 |
-
"
|
| 212 |
-
"
|
| 213 |
-
"
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
"
|
| 219 |
-
"
|
| 220 |
-
"
|
| 221 |
-
"
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
"
|
| 227 |
-
"
|
| 228 |
-
"
|
| 229 |
-
"
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
"
|
| 235 |
-
"
|
| 236 |
-
"
|
| 237 |
-
"
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
"
|
| 243 |
-
"
|
| 244 |
-
"
|
| 245 |
-
"
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
"
|
| 251 |
-
"
|
| 252 |
-
"
|
| 253 |
-
"
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
"
|
| 259 |
-
"
|
| 260 |
-
"
|
| 261 |
-
"
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
"
|
| 267 |
-
"
|
| 268 |
-
"
|
| 269 |
-
"
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
"
|
| 275 |
-
"
|
| 276 |
-
"
|
| 277 |
-
"
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
"
|
| 283 |
-
"
|
| 284 |
-
"
|
| 285 |
-
"
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
"
|
| 306 |
-
"
|
| 307 |
-
"
|
| 308 |
-
"
|
| 309 |
-
"
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
"
|
| 315 |
-
"
|
| 316 |
-
"
|
| 317 |
-
"
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
"
|
| 323 |
-
"
|
| 324 |
-
"
|
| 325 |
-
"
|
| 326 |
-
"
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
"
|
| 331 |
-
"
|
| 332 |
-
"
|
| 333 |
-
"
|
| 334 |
-
"
|
| 335 |
-
"
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
"
|
| 340 |
-
"
|
| 341 |
-
"
|
| 342 |
-
"
|
| 343 |
-
"
|
| 344 |
-
"
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
"
|
| 349 |
-
"
|
| 350 |
-
"
|
| 351 |
-
"
|
| 352 |
-
"
|
| 353 |
-
"
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
"
|
| 358 |
-
"
|
| 359 |
-
"
|
| 360 |
-
"
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
"
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
"
|
| 367 |
-
"
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
"
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
"
|
| 374 |
-
"
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
"
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
"
|
| 381 |
-
"
|
| 382 |
-
"
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
"
|
| 388 |
-
"
|
| 389 |
-
"
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
"
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
"
|
| 396 |
-
"
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
"
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
"
|
| 403 |
-
"
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
"
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
"
|
| 410 |
-
"
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
"
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
"
|
| 417 |
-
"
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
"
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
"
|
| 424 |
-
"
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
"
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
"
|
| 431 |
-
"
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
"
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
"
|
| 438 |
-
"
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
"
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
"
|
| 445 |
-
"
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
"
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
"
|
| 452 |
-
"
|
| 453 |
-
"
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
"
|
| 459 |
-
"
|
| 460 |
-
"
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
{
|
| 466 |
-
"key": "stock_bucket",
|
| 467 |
-
"label": "Stock status",
|
| 468 |
-
"type": "select",
|
| 469 |
-
"source": "odoo",
|
| 470 |
-
"description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
|
| 471 |
-
}
|
| 472 |
-
]
|
| 473 |
-
}
|
| 474 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
|
| 3 |
+
"fields": [
|
| 4 |
+
{
|
| 5 |
+
"key": "customer",
|
| 6 |
+
"label": "Customer",
|
| 7 |
+
"type": "text",
|
| 8 |
+
"source": "odoo",
|
| 9 |
+
"pinned": true,
|
| 10 |
+
"default": true,
|
| 11 |
+
"description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"key": "odoo_status",
|
| 15 |
+
"label": "Odoo record",
|
| 16 |
+
"type": "status",
|
| 17 |
+
"source": "odoo",
|
| 18 |
+
"default": false,
|
| 19 |
+
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"key": "agent",
|
| 23 |
+
"label": "Agent",
|
| 24 |
+
"type": "text",
|
| 25 |
+
"source": "odoo",
|
| 26 |
+
"default": true,
|
| 27 |
+
"description": "The sales agent who owns this account."
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"key": "dba",
|
| 31 |
+
"label": "DBA",
|
| 32 |
+
"type": "select",
|
| 33 |
+
"source": "odoo",
|
| 34 |
+
"default": false,
|
| 35 |
+
"options": [
|
| 36 |
+
"Fisch",
|
| 37 |
+
"Royal",
|
| 38 |
+
"Both"
|
| 39 |
+
],
|
| 40 |
+
"description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"key": "salesperson",
|
| 44 |
+
"label": "Salesperson",
|
| 45 |
+
"type": "text",
|
| 46 |
+
"source": "odoo",
|
| 47 |
+
"default": false,
|
| 48 |
+
"description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"key": "street",
|
| 52 |
+
"label": "Street",
|
| 53 |
+
"type": "text",
|
| 54 |
+
"source": "odoo",
|
| 55 |
+
"default": false,
|
| 56 |
+
"description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"key": "street2",
|
| 60 |
+
"label": "Street 2",
|
| 61 |
+
"type": "text",
|
| 62 |
+
"source": "odoo",
|
| 63 |
+
"default": false,
|
| 64 |
+
"description": "Second address line (suite, unit, floor) on the customer's Odoo address."
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"key": "city",
|
| 68 |
+
"label": "City",
|
| 69 |
+
"type": "text",
|
| 70 |
+
"source": "odoo",
|
| 71 |
+
"default": true,
|
| 72 |
+
"description": "City on the customer's Odoo address."
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"key": "state",
|
| 76 |
+
"label": "State",
|
| 77 |
+
"type": "text",
|
| 78 |
+
"source": "odoo",
|
| 79 |
+
"default": true,
|
| 80 |
+
"description": "State or province on the customer's Odoo address."
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"key": "country",
|
| 84 |
+
"label": "Country",
|
| 85 |
+
"type": "text",
|
| 86 |
+
"source": "odoo",
|
| 87 |
+
"default": false,
|
| 88 |
+
"description": "Country on the customer's Odoo address."
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"key": "zip",
|
| 92 |
+
"label": "ZIP",
|
| 93 |
+
"type": "text",
|
| 94 |
+
"source": "odoo",
|
| 95 |
+
"default": false,
|
| 96 |
+
"description": "Postal code on the customer's Odoo address."
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"key": "customer_since",
|
| 100 |
+
"label": "Customer since",
|
| 101 |
+
"type": "date",
|
| 102 |
+
"source": "odoo",
|
| 103 |
+
"default": false,
|
| 104 |
+
"description": "When this customer was first set up in Odoo."
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"key": "tags",
|
| 108 |
+
"label": "Tags",
|
| 109 |
+
"type": "text",
|
| 110 |
+
"source": "odoo",
|
| 111 |
+
"default": false,
|
| 112 |
+
"description": "Odoo labels on this customer, comma-separated."
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"key": "pricelist",
|
| 116 |
+
"label": "Price list",
|
| 117 |
+
"type": "text",
|
| 118 |
+
"source": "odoo",
|
| 119 |
+
"default": false,
|
| 120 |
+
"description": "The price list this customer buys on."
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"key": "payment_terms",
|
| 124 |
+
"label": "Payment terms",
|
| 125 |
+
"type": "text",
|
| 126 |
+
"source": "odoo",
|
| 127 |
+
"default": false,
|
| 128 |
+
"description": "Payment terms on this customer's account — Net 30, for example."
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
"key": "last_order",
|
| 132 |
+
"label": "Last order",
|
| 133 |
+
"type": "date",
|
| 134 |
+
"source": "odoo",
|
| 135 |
+
"default": true,
|
| 136 |
+
"description": "Date of the most recent confirmed order."
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"key": "overdue_days",
|
| 140 |
+
"label": "Overdue days",
|
| 141 |
+
"type": "int",
|
| 142 |
+
"source": "odoo",
|
| 143 |
+
"default": true,
|
| 144 |
+
"description": "How many days late this customer is running against their own usual ordering rhythm."
|
| 145 |
+
},
|
| 146 |
+
{
|
| 147 |
+
"_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
|
| 148 |
+
"key": "est_missed",
|
| 149 |
+
"label": "Est. missed $",
|
| 150 |
+
"type": "currency",
|
| 151 |
+
"source": "odoo",
|
| 152 |
+
"default": true,
|
| 153 |
+
"agg": "sum",
|
| 154 |
+
"filterable": false,
|
| 155 |
+
"description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
|
| 159 |
+
"key": "ar_open",
|
| 160 |
+
"label": "AR current $",
|
| 161 |
+
"type": "currency",
|
| 162 |
+
"source": "odoo",
|
| 163 |
+
"default": false,
|
| 164 |
+
"description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"key": "ar_overdue",
|
| 168 |
+
"label": "AR overdue $",
|
| 169 |
+
"type": "currency",
|
| 170 |
+
"source": "odoo",
|
| 171 |
+
"default": false,
|
| 172 |
+
"description": "Invoiced money past due — same basis as the Collections page."
|
| 173 |
+
},
|
| 174 |
+
{
|
| 175 |
+
"_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
|
| 176 |
+
"key": "ar_outstanding",
|
| 177 |
+
"label": "AR outstanding $",
|
| 178 |
+
"type": "currency",
|
| 179 |
+
"source": "odoo",
|
| 180 |
+
"default": false,
|
| 181 |
+
"description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"key": "ar_exposure",
|
| 185 |
+
"label": "Credit exposure $",
|
| 186 |
+
"type": "currency",
|
| 187 |
+
"source": "odoo",
|
| 188 |
+
"default": false,
|
| 189 |
+
"description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"key": "ar_aged_1_30",
|
| 193 |
+
"label": "1-30 days $",
|
| 194 |
+
"type": "currency",
|
| 195 |
+
"source": "odoo",
|
| 196 |
+
"default": false,
|
| 197 |
+
"description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"key": "ar_aged_31_60",
|
| 201 |
+
"label": "31-60 days $",
|
| 202 |
+
"type": "currency",
|
| 203 |
+
"source": "odoo",
|
| 204 |
+
"default": false,
|
| 205 |
+
"description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"key": "ar_aged_61_90",
|
| 209 |
+
"label": "61-90 days $",
|
| 210 |
+
"type": "currency",
|
| 211 |
+
"source": "odoo",
|
| 212 |
+
"default": false,
|
| 213 |
+
"description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
|
| 214 |
+
},
|
| 215 |
+
{
|
| 216 |
+
"key": "ar_aged_90_plus",
|
| 217 |
+
"label": "90+ days $",
|
| 218 |
+
"type": "currency",
|
| 219 |
+
"source": "odoo",
|
| 220 |
+
"default": false,
|
| 221 |
+
"description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"key": "days_to_pay",
|
| 225 |
+
"label": "Days to pay",
|
| 226 |
+
"type": "int",
|
| 227 |
+
"source": "odoo",
|
| 228 |
+
"default": false,
|
| 229 |
+
"description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"key": "top_category",
|
| 233 |
+
"label": "Top category",
|
| 234 |
+
"type": "text",
|
| 235 |
+
"source": "odoo",
|
| 236 |
+
"default": false,
|
| 237 |
+
"description": "The category this customer spent the most on in the last 12 months."
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"key": "top_category_pct",
|
| 241 |
+
"label": "Top category %",
|
| 242 |
+
"type": "pct",
|
| 243 |
+
"source": "odoo",
|
| 244 |
+
"default": false,
|
| 245 |
+
"description": "Share of last-12-months spend that went to the top category."
|
| 246 |
+
},
|
| 247 |
+
{
|
| 248 |
+
"key": "sku_count",
|
| 249 |
+
"label": "SKUs bought",
|
| 250 |
+
"type": "int",
|
| 251 |
+
"source": "odoo",
|
| 252 |
+
"default": false,
|
| 253 |
+
"description": "Distinct products bought in the last 12 months."
|
| 254 |
+
},
|
| 255 |
+
{
|
| 256 |
+
"key": "top_sku",
|
| 257 |
+
"label": "Top SKU",
|
| 258 |
+
"type": "text",
|
| 259 |
+
"source": "odoo",
|
| 260 |
+
"default": false,
|
| 261 |
+
"description": "The product this customer spent the most on in the last 12 months."
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"key": "days_since",
|
| 265 |
+
"label": "Days since order",
|
| 266 |
+
"type": "int",
|
| 267 |
+
"source": "odoo",
|
| 268 |
+
"default": false,
|
| 269 |
+
"description": "Days since the last confirmed order."
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"key": "typical_gap_days",
|
| 273 |
+
"label": "Typical gap days",
|
| 274 |
+
"type": "int",
|
| 275 |
+
"source": "odoo",
|
| 276 |
+
"default": false,
|
| 277 |
+
"description": "Days this customer usually goes between orders, from their own history."
|
| 278 |
+
},
|
| 279 |
+
{
|
| 280 |
+
"key": "notes",
|
| 281 |
+
"label": "Notes",
|
| 282 |
+
"type": "text",
|
| 283 |
+
"source": "overlay",
|
| 284 |
+
"default": false,
|
| 285 |
+
"description": "Your notes on this customer. Saved in this app only, visible only to you."
|
| 286 |
+
}
|
| 287 |
+
],
|
| 288 |
+
"_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.",
|
| 289 |
+
"_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).",
|
| 290 |
+
"product_data": {
|
| 291 |
+
"identity": "pid",
|
| 292 |
+
"business_key": "code",
|
| 293 |
+
"fields": [
|
| 294 |
+
{
|
| 295 |
+
"key": "code",
|
| 296 |
+
"label": "SKU",
|
| 297 |
+
"type": "text",
|
| 298 |
+
"source": "odoo",
|
| 299 |
+
"pinned": true,
|
| 300 |
+
"default": true,
|
| 301 |
+
"description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
|
| 302 |
+
},
|
| 303 |
+
{
|
| 304 |
+
"key": "product",
|
| 305 |
+
"label": "Product",
|
| 306 |
+
"type": "text",
|
| 307 |
+
"source": "odoo",
|
| 308 |
+
"default": true,
|
| 309 |
+
"description": "Product name as it appears in Odoo."
|
| 310 |
+
},
|
| 311 |
+
{
|
| 312 |
+
"key": "category",
|
| 313 |
+
"label": "Category",
|
| 314 |
+
"type": "select",
|
| 315 |
+
"source": "odoo",
|
| 316 |
+
"default": true,
|
| 317 |
+
"description": "Product category; '(uncategorized)' when Odoo carries none."
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"key": "supplier",
|
| 321 |
+
"label": "Supplier",
|
| 322 |
+
"type": "text",
|
| 323 |
+
"source": "overlay",
|
| 324 |
+
"default": true,
|
| 325 |
+
"description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.",
|
| 326 |
+
"shared": true
|
| 327 |
+
},
|
| 328 |
+
{
|
| 329 |
+
"key": "origin_country",
|
| 330 |
+
"label": "Country",
|
| 331 |
+
"type": "text",
|
| 332 |
+
"source": "overlay",
|
| 333 |
+
"default": false,
|
| 334 |
+
"description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.",
|
| 335 |
+
"shared": true
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"key": "lead_days",
|
| 339 |
+
"label": "Lead time (days)",
|
| 340 |
+
"type": "int",
|
| 341 |
+
"source": "overlay",
|
| 342 |
+
"default": true,
|
| 343 |
+
"description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.",
|
| 344 |
+
"shared": true
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"key": "first_cost",
|
| 348 |
+
"label": "First cost",
|
| 349 |
+
"type": "currency",
|
| 350 |
+
"source": "overlay",
|
| 351 |
+
"default": false,
|
| 352 |
+
"description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.",
|
| 353 |
+
"shared": true
|
| 354 |
+
},
|
| 355 |
+
{
|
| 356 |
+
"key": "price_fisch",
|
| 357 |
+
"label": "Fisch price",
|
| 358 |
+
"type": "currency",
|
| 359 |
+
"source": "odoo",
|
| 360 |
+
"description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"key": "price_royal_1",
|
| 364 |
+
"label": "Royal 1 price",
|
| 365 |
+
"type": "currency",
|
| 366 |
+
"source": "odoo",
|
| 367 |
+
"description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 368 |
+
},
|
| 369 |
+
{
|
| 370 |
+
"key": "price_royal_2",
|
| 371 |
+
"label": "Royal 2 price",
|
| 372 |
+
"type": "currency",
|
| 373 |
+
"source": "odoo",
|
| 374 |
+
"description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"key": "rev_ytd",
|
| 378 |
+
"label": "Revenue YTD",
|
| 379 |
+
"type": "currency",
|
| 380 |
+
"source": "odoo",
|
| 381 |
+
"default": true,
|
| 382 |
+
"description": "Year-to-date revenue for this SKU, BU-scoped when the caller is."
|
| 383 |
+
},
|
| 384 |
+
{
|
| 385 |
+
"key": "rev_ly",
|
| 386 |
+
"label": "Revenue LY",
|
| 387 |
+
"type": "currency",
|
| 388 |
+
"source": "odoo",
|
| 389 |
+
"description": "Same period last year — seasonal wholesale compares like for like."
|
| 390 |
+
},
|
| 391 |
+
{
|
| 392 |
+
"key": "yoy_pct",
|
| 393 |
+
"label": "YoY %",
|
| 394 |
+
"type": "pct",
|
| 395 |
+
"source": "odoo",
|
| 396 |
+
"description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
|
| 397 |
+
},
|
| 398 |
+
{
|
| 399 |
+
"key": "qty_ytd",
|
| 400 |
+
"label": "Units YTD",
|
| 401 |
+
"type": "int",
|
| 402 |
+
"source": "odoo",
|
| 403 |
+
"description": "Units sold year to date."
|
| 404 |
+
},
|
| 405 |
+
{
|
| 406 |
+
"key": "orders_ytd",
|
| 407 |
+
"label": "Orders YTD",
|
| 408 |
+
"type": "int",
|
| 409 |
+
"source": "odoo",
|
| 410 |
+
"description": "Distinct orders containing this SKU, year to date."
|
| 411 |
+
},
|
| 412 |
+
{
|
| 413 |
+
"key": "on_hand",
|
| 414 |
+
"label": "On hand",
|
| 415 |
+
"type": "int",
|
| 416 |
+
"source": "odoo",
|
| 417 |
+
"description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
|
| 418 |
+
},
|
| 419 |
+
{
|
| 420 |
+
"key": "unit_cost",
|
| 421 |
+
"label": "Unit cost",
|
| 422 |
+
"type": "currency",
|
| 423 |
+
"source": "odoo",
|
| 424 |
+
"description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
|
| 425 |
+
},
|
| 426 |
+
{
|
| 427 |
+
"key": "inv_value",
|
| 428 |
+
"label": "Stock value",
|
| 429 |
+
"type": "currency",
|
| 430 |
+
"source": "odoo",
|
| 431 |
+
"description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
|
| 432 |
+
},
|
| 433 |
+
{
|
| 434 |
+
"key": "qty_ltm",
|
| 435 |
+
"label": "Units LTM",
|
| 436 |
+
"type": "int",
|
| 437 |
+
"source": "odoo",
|
| 438 |
+
"description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
|
| 439 |
+
},
|
| 440 |
+
{
|
| 441 |
+
"key": "dos",
|
| 442 |
+
"label": "Days of supply",
|
| 443 |
+
"type": "int",
|
| 444 |
+
"source": "odoo",
|
| 445 |
+
"description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
|
| 446 |
+
},
|
| 447 |
+
{
|
| 448 |
+
"key": "cover_gap_d",
|
| 449 |
+
"label": "Cover gap (days)",
|
| 450 |
+
"type": "int",
|
| 451 |
+
"source": "odoo",
|
| 452 |
+
"default": false,
|
| 453 |
+
"description": "Days of supply minus lead time. Negative means it runs out before a reorder lands."
|
| 454 |
+
},
|
| 455 |
+
{
|
| 456 |
+
"key": "stock_bucket",
|
| 457 |
+
"label": "Stock status",
|
| 458 |
+
"type": "select",
|
| 459 |
+
"source": "odoo",
|
| 460 |
+
"description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
|
| 461 |
+
}
|
| 462 |
+
]
|
| 463 |
+
}
|
| 464 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
platform/core/alerts.py
CHANGED
|
@@ -209,7 +209,7 @@ def _queue(rec, pids, labels, st=None):
|
|
| 209 |
_st(st).update(NOTIFICATIONS_KEY, _apply, flush='async')
|
| 210 |
|
| 211 |
|
| 212 |
-
def notify(owner, label, *, topic='automation', key='', row_id='', detail='',
|
| 213 |
"""⭐ WAVE 23 (C6) — put ONE notification in a named user's inbox, for a producer that is not
|
| 214 |
a view-alert evaluation.
|
| 215 |
|
|
@@ -222,24 +222,6 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', ac
|
|
| 222 |
The inbox shape is UNCHANGED, deliberately: `AlertsPane` reads one list, and a second row
|
| 223 |
shape would mean a client that must branch on where a notification came from. `alertId`
|
| 224 |
carries the producer's key so a click-through can route (W23-W5).
|
| 225 |
-
|
| 226 |
-
⭐⭐ W33-T28 (`ASK C-14`) — `actor` IS THE ONLY THING THAT KNOWS WHO. Owner item: *"the Inbox
|
| 227 |
-
reads like email"*, and a verifier reading the carried wave-32 surface found the SENDER position
|
| 228 |
-
occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you", i.e. a
|
| 229 |
-
CATEGORY where mail puts a who — with no sender field anywhere on the wire, in the model or in
|
| 230 |
-
the markup. An alert and an automation have no person behind them and are honestly named by
|
| 231 |
-
their machine; **a SHARE has a real person, and only the PRODUCER knows their name.** By the time
|
| 232 |
-
the read path sees the row that fact is gone, so it has to be carried at write time.
|
| 233 |
-
|
| 234 |
-
⛔ ADDITIVE, AND THAT IS THE WHOLE DESIGN. No existing caller changes, no stored row changes
|
| 235 |
-
shape, and every reader already tolerates the key's absence — so an inbox written before this
|
| 236 |
-
line reads exactly as it did. The alternative C rejected is worth recording because it is the
|
| 237 |
-
tempting one: recovering the name by REGEXING the body prose (`"<name> shared this with you"`).
|
| 238 |
-
A sender parsed out of a sentence breaks the first time the sentence is reworded, and it breaks
|
| 239 |
-
SILENTLY, in the header — [[one-question-two-normalizers]] with the second normaliser being
|
| 240 |
-
English.
|
| 241 |
-
⚠ Blank is a legitimate value and means *"no person did this"*, NOT "unknown". The client falls
|
| 242 |
-
back BY KIND rather than to an empty From column, so an automation never borrows a human's name.
|
| 243 |
"""
|
| 244 |
owner = str(owner or '').strip()
|
| 245 |
if not owner or not label:
|
|
@@ -248,8 +230,7 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', ac
|
|
| 248 |
item = {'id': f"{key or topic}:{row_id or 'n'}:{at}",
|
| 249 |
'alertId': str(key or ''), 'viewId': '', 'topic': str(topic),
|
| 250 |
'rowId': str(row_id or ''), 'label': str(detail or label),
|
| 251 |
-
'alertLabel': str(label), 'at': at, 'read': False
|
| 252 |
-
'actor': str(actor or '')}
|
| 253 |
|
| 254 |
def _apply(data):
|
| 255 |
box = list(data.get(owner) or [])
|
|
|
|
| 209 |
_st(st).update(NOTIFICATIONS_KEY, _apply, flush='async')
|
| 210 |
|
| 211 |
|
| 212 |
+
def notify(owner, label, *, topic='automation', key='', row_id='', detail='', st=None):
|
| 213 |
"""⭐ WAVE 23 (C6) — put ONE notification in a named user's inbox, for a producer that is not
|
| 214 |
a view-alert evaluation.
|
| 215 |
|
|
|
|
| 222 |
The inbox shape is UNCHANGED, deliberately: `AlertsPane` reads one list, and a second row
|
| 223 |
shape would mean a client that must branch on where a notification came from. `alertId`
|
| 224 |
carries the producer's key so a click-through can route (W23-W5).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
"""
|
| 226 |
owner = str(owner or '').strip()
|
| 227 |
if not owner or not label:
|
|
|
|
| 230 |
item = {'id': f"{key or topic}:{row_id or 'n'}:{at}",
|
| 231 |
'alertId': str(key or ''), 'viewId': '', 'topic': str(topic),
|
| 232 |
'rowId': str(row_id or ''), 'label': str(detail or label),
|
| 233 |
+
'alertLabel': str(label), 'at': at, 'read': False}
|
|
|
|
| 234 |
|
| 235 |
def _apply(data):
|
| 236 |
box = list(data.get(owner) or [])
|
platform/core/grid_events.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/core/perms.py
CHANGED
|
@@ -199,116 +199,6 @@ def nav_pages(user):
|
|
| 199 |
return out
|
| 200 |
|
| 201 |
|
| 202 |
-
# ------------------------------------------------------------------ governable modules (C3)
|
| 203 |
-
#: ⭐⭐ WAVE 33, OWNER ITEM 11 — *"its an error that ALL fucking tenant can see both Customer and
|
| 204 |
-
#: Product database under Manage User, this is only for Royal Imports tenant. Make sure per tenant
|
| 205 |
-
#: configuration always work, this is my fourth time I flag it."*
|
| 206 |
-
#:
|
| 207 |
-
#: The cause was two lines in `routes_admin.py`: `_PERM_MODULES = ("customer_data","product_data")`
|
| 208 |
-
#: and a `_MODULE_LABELS` dict beside it. `get_perms` built its answer from the literal and never
|
| 209 |
-
#: touched `session.runtime`, so the permission editor was TENANT-BLIND BY CONSTRUCTION — every
|
| 210 |
-
#: tenant was handed tenant #0's two grid modules whether or not it is provisioned for either.
|
| 211 |
-
#:
|
| 212 |
-
#: ⛔ WHY THE DERIVATION LIVES HERE AND NOT AT THE ROUTE. `routes_nav.py::nav` already applies
|
| 213 |
-
#: exactly this rule and has since wave 18; the perms route is the ONE surface that skipped it.
|
| 214 |
-
#: A second copy of the rule at a second call site is how the two came apart in the first place,
|
| 215 |
-
#: so the rule is stated once, as a pure function of (runtime, catalogue), and both the route and
|
| 216 |
-
#: `verify_perm_scope` ask THIS.
|
| 217 |
-
def tenant_modules(runtime):
|
| 218 |
-
"""The registry keys this TENANT is provisioned for. None means "every key".
|
| 219 |
-
|
| 220 |
-
⛔ FAIL-CLOSED ON THE EMPTY LIST, and that is not the obvious reading. `routes_admin`'s own
|
| 221 |
-
docstring records two shipped fail-OPENs of exactly this shape (`modules: []` and
|
| 222 |
-
`bus: [<unknown id>]` both read as UNRESTRICTED), so ONLY the literal `'all'` — or an absent
|
| 223 |
-
key, which is how the compiled-in tenant #0 record is written — means every module. A
|
| 224 |
-
bucket-provisioned tenant gets `{"modules": rec.get("modules", [])}`
|
| 225 |
-
(`harness/runtime.py::_builder_from_record`), so `[]` there means the tenant was provisioned
|
| 226 |
-
for NOTHING and must be told so, never handed the catalogue.
|
| 227 |
-
|
| 228 |
-
Legacy keys are migrated on read for the same reason `allowed_modules` does it: a tenant
|
| 229 |
-
record written before a rename must not silently lose the module it names.
|
| 230 |
-
"""
|
| 231 |
-
cfg = getattr(getattr(runtime, 'tenant', None), 'config', None) or {}
|
| 232 |
-
raw = cfg.get('modules', 'all')
|
| 233 |
-
if raw is None or raw == 'all':
|
| 234 |
-
return None
|
| 235 |
-
if isinstance(raw, str):
|
| 236 |
-
raw = [raw]
|
| 237 |
-
return {_LEGACY_KEYS.get(str(k).strip(), str(k).strip())
|
| 238 |
-
for k in raw if str(k or '').strip()}
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
def tenant_has_module(runtime, key):
|
| 242 |
-
"""Is `key` in this tenant's provisioned set? PARENT-AWARE, like `may_open`.
|
| 243 |
-
|
| 244 |
-
`routes_nav` keeps a page whose `parent` is granted even when the child key is not, and a
|
| 245 |
-
governable list that disagreed with the nav would offer an admin a wall for a database the
|
| 246 |
-
tenant cannot open — or hide one it can.
|
| 247 |
-
"""
|
| 248 |
-
allowed = tenant_modules(runtime)
|
| 249 |
-
if allowed is None or key in allowed:
|
| 250 |
-
return True
|
| 251 |
-
parent = registry.BY_KEY.get(key, {}).get('parent')
|
| 252 |
-
return parent in allowed if parent else False
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
def tenant_governable_modules(runtime, topics, ut_entries=()):
|
| 256 |
-
"""The databases the permission editor may render for THIS tenant, in stable order.
|
| 257 |
-
|
| 258 |
-
⚠ NAMED `tenant_*` TO STAY DISTINCT FROM `registry.governable_modules()`, which lane E
|
| 259 |
-
published under contract C3 and which this consumes. They answer different questions —
|
| 260 |
-
registry's is "which topics are governable at all, platform-wide"; this one is "which
|
| 261 |
-
databases does THIS tenant govern" — and two same-named functions one import apart is the
|
| 262 |
-
[[one-question-two-normalizers]] shape waiting to happen.
|
| 263 |
-
|
| 264 |
-
`topics` — the registry-backed grid modules the editor can build pickers for, as
|
| 265 |
-
`[{'key': ..., 'label'?: ...}]`, i.e. `registry.governable_modules()` narrowed
|
| 266 |
-
to the topics the caller has a field-schema arm for. The CALLER owns that
|
| 267 |
-
narrowing because the schema behind each arm is code
|
| 268 |
-
(`routes_admin::_module_fields`), not a policy list. Tenant-blind on the way
|
| 269 |
-
in; this function is the only place the tenant is applied.
|
| 270 |
-
`ut_entries` — this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape.
|
| 271 |
-
|
| 272 |
-
Returns `[{'key', 'label', 'enforced'}]`.
|
| 273 |
-
|
| 274 |
-
⛔⛔ `enforced` IS THE HONEST HALF AND IT IS NOT DECORATION. `perm_scope` walls the REGISTRY
|
| 275 |
-
topics — `grid_assembly` calls `may_access`, `visible_fields` and `apply_row_scope` on every
|
| 276 |
-
read. It is never consulted for a `ut_*` database: `routes_tables`' visibility is
|
| 277 |
-
`user_tables.may_open` (creator, admin, or a `core.shares` grant) and there is no module
|
| 278 |
-
grant in that path at all. So a stored `ut_*` wall would be INERT — the editor would say
|
| 279 |
-
DENY, the table routes would keep serving, and nothing anywhere would say so. That is the
|
| 280 |
-
same class as the empty-dropdown defect (wave 26 item 24): a control that answers when it
|
| 281 |
-
cannot. The database is LISTED, because the owner asked to see every database; the wall is
|
| 282 |
-
declared unenforced, and `routes_admin::_clean_perms` REFUSES to store one against it rather
|
| 283 |
-
than accepting a rule nobody applies. Arming `perm_scope` over `ut_*` is a change to
|
| 284 |
-
`routes_tables`/`user_tables` (lane A's fence) — booked, not faked.
|
| 285 |
-
"""
|
| 286 |
-
out = []
|
| 287 |
-
for t in topics or ():
|
| 288 |
-
key = str((t or {}).get('key') or '').strip()
|
| 289 |
-
if not key or not tenant_has_module(runtime, key):
|
| 290 |
-
continue
|
| 291 |
-
# The LABEL comes from the registry, never from a dict beside the list. `_MODULE_LABELS`
|
| 292 |
-
# was a hardcoded copy of two registry labels, so wave 33's R2 relabel ("Odoo customers" /
|
| 293 |
-
# "Odoo products") would have moved one of them and left the permission editor showing the
|
| 294 |
-
# old name with nothing to notice.
|
| 295 |
-
label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key)
|
| 296 |
-
out.append({'key': key, 'label': label, 'enforced': True})
|
| 297 |
-
seen = {r['key'] for r in out}
|
| 298 |
-
for e in ut_entries or ():
|
| 299 |
-
key = str((e or {}).get('key') or '').strip()
|
| 300 |
-
if not key or key in seen:
|
| 301 |
-
continue
|
| 302 |
-
seen.add(key)
|
| 303 |
-
out.append({'key': key, 'label': (e.get('label') or key), 'enforced': False})
|
| 304 |
-
return out
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
def enforced_module_keys(modules):
|
| 308 |
-
"""The subset of `governable_modules`' answer a permission block may actually be stored for."""
|
| 309 |
-
return {m['key'] for m in (modules or ()) if m.get('enforced')}
|
| 310 |
-
|
| 311 |
-
|
| 312 |
def landing_page(user):
|
| 313 |
"""Where a restricted user lands: the first MAIN-chrome page in their grant. None for
|
| 314 |
unrestricted users (the client lands on the Analyst, as the host does).
|
|
|
|
| 199 |
return out
|
| 200 |
|
| 201 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
def landing_page(user):
|
| 203 |
"""Where a restricted user lands: the first MAIN-chrome page in their grant. None for
|
| 204 |
unrestricted users (the client lands on the Analyst, as the host does).
|
platform/core/registry.py
CHANGED
|
@@ -22,22 +22,6 @@ Fields:
|
|
| 22 |
2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
|
| 23 |
Collections + Analyst/plumbing): products, assortment, financial, management,
|
| 24 |
pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
|
| 25 |
-
subject set -> the REAL-WORLD POPULATION this row presents, as a source-qualified id
|
| 26 |
-
('odoo:res.partner'). ⭐ W33-T41 / item 12a: it is what makes "two databases present
|
| 27 |
-
the same thing" a statement code can refuse, and it exists because NO OTHER FIELD
|
| 28 |
-
CAN CARRY THAT TEST. Not the key (`product_data` and `ut_odoo_products` are different
|
| 29 |
-
keys by construction — that is the bug, not the check), and NOT THE LABEL: the owner's
|
| 30 |
-
own complaint is "why do we have 'Odoo products' ALREADY with the current Products
|
| 31 |
-
database?", i.e. the twin was already carrying the label the legacy row is about to
|
| 32 |
-
take (W33-T42). Between T42 and T44 the two rows legitimately share a label, so a
|
| 33 |
-
label-keyed check would refuse the rename that fixes the bug.
|
| 34 |
-
⚠ A row without `subject` claims nothing and conflicts with nothing — silence is not
|
| 35 |
-
a claim, so adding this field cannot retro-refuse anything already shipped.
|
| 36 |
-
topic set -> this row is a GOVERNABLE DATABASE and this is the word the rest of the codebase
|
| 37 |
-
already calls it by (`routes_grid._SCOPES`, `routes_alerts._TOPICS`, `scope=` on the
|
| 38 |
-
grid routes). ⭐ W33-T46 / contract C3: it is what `governable_modules()` keys on, and
|
| 39 |
-
it exists so `routes_admin`'s permission list can be DERIVED from the registry instead
|
| 40 |
-
of being the hardcoded two-element tuple that made every tenant see Customer + Product.
|
| 41 |
note one-line scope note shown in the UI / docs
|
| 42 |
"""
|
| 43 |
# WAVE-9 I8 — `'source'`: which connected system a module's data comes from.
|
|
@@ -81,14 +65,7 @@ REGISTRY = [
|
|
| 81 |
# myday->customer_list was done. The STORE keys are deliberately NOT renamed:
|
| 82 |
# 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
|
| 83 |
# would orphan every saved view. Label-only changes need none of that machinery.
|
| 84 |
-
|
| 85 |
-
# naming convention of every other database from this connector ("Odoo orders", "Odoo vendor
|
| 86 |
-
# bills"). ⭐ STILL LABEL-ONLY, and the note above is the precedent — the KEY does not move, so
|
| 87 |
-
# no grant, saved view, cohort, formula, shared column or `?page=` deep link is touched.
|
| 88 |
-
# ⛔ `nav_meta` could not have done this: `routes_nav.py` raises `400 name_not_allowed` for any
|
| 89 |
-
# key that does not start with `ut_`, so a tenant rename cannot reach a registry module at all.
|
| 90 |
-
{'key': 'customer_data', 'label': 'Odoo customers', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
|
| 91 |
-
'subject': 'odoo:res.partner', 'topic': 'customer',
|
| 92 |
'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
|
| 93 |
'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
|
| 94 |
'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
|
|
@@ -124,10 +101,7 @@ REGISTRY = [
|
|
| 124 |
'group_only': True,
|
| 125 |
'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
|
| 126 |
'not part of this family.'},
|
| 127 |
-
|
| 128 |
-
# customer_data note above; label-only, the key stays `product_data`.
|
| 129 |
-
{'key': 'product_data', 'label': 'Odoo products', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
|
| 130 |
-
'subject': 'odoo:product.product', 'topic': 'product',
|
| 131 |
'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
|
| 132 |
'stock columns, with saved views and custom fields. Identity = the SKU code.'},
|
| 133 |
{'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
|
@@ -223,90 +197,3 @@ BY_KEY = {m['key']: m for m in REGISTRY}
|
|
| 223 |
|
| 224 |
def brand_filterable(key) -> bool:
|
| 225 |
return bool(BY_KEY.get(key, {}).get('brand'))
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
# ═════════════════════════════════════════════════════════════════════════════════════════════
|
| 229 |
-
# ⭐⭐ W33-T41 / item 12a — ONE SUBJECT, ONE DATABASE
|
| 230 |
-
# ════════════════════════��════════════════════════════════════════════════════════════════════
|
| 231 |
-
#
|
| 232 |
-
# The owner: *"why do we have 'Odoo products' already with the current Products database? The
|
| 233 |
-
# Unique ID is redundant, make sure this never happens. One unique ID per database always."*
|
| 234 |
-
#
|
| 235 |
-
# ⛔ THE CHECK HAD TO GO WHERE THE BUG WAS MADE, AND THAT IS NOT WHERE ANYONE WOULD LOOK.
|
| 236 |
-
# `core.user_tables.create` — the UI door — HAS had a collision loop all along: it slugs the
|
| 237 |
-
# label to a key and appends `_2 … _49` until the key is free. It never fired here for two
|
| 238 |
-
# reasons, and both matter. (1) It compares KEYS inside ONE tenant's `user_tables` document,
|
| 239 |
-
# and `customer_data` / `product_data` are compiled REGISTRY rows that never enter that
|
| 240 |
-
# document at all — the two halves of the duplicate live in different namespaces and only meet
|
| 241 |
-
# in `routes_nav.py::nav`, where the registry pages and `user_tables.nav_entries` are merged
|
| 242 |
-
# for display. (2) The connector path never calls `create()`: `odoo_relational.
|
| 243 |
-
# _ensure_table_inplace` writes `cur[key] = {...}` straight into the document. So a check bolted
|
| 244 |
-
# onto `create()` would have been green through the entire life of this defect.
|
| 245 |
-
#
|
| 246 |
-
# ⚠ THE SUFFIX LOOP IS ALSO WHY A RELABEL ALONE WOULD HAVE MADE THIS WORSE, MECHANICALLY: a
|
| 247 |
-
# naive "just rename Product to Odoo products" mints `ut_odoo_products_2` — the owner's
|
| 248 |
-
# redundant id again, from the fix for it.
|
| 249 |
-
#
|
| 250 |
-
#: subject -> the ONE registry key entitled to present it. Derived, never typed twice.
|
| 251 |
-
SUBJECT_CLAIMS = {m['subject']: m['key'] for m in REGISTRY if m.get('subject')}
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
# ═════════════════════════════════════════════════════════════════════════════════════════════
|
| 255 |
-
# ⭐⭐ W33-T46 / CONTRACT C3 — THE GOVERNABLE MODULE LIST IS DERIVED, NOT A LITERAL
|
| 256 |
-
# ═════════════════════════════════════════════════════════════════════════════════════════════
|
| 257 |
-
#
|
| 258 |
-
# The owner, on his FOURTH flag of the same thing: *"its an error that ALL fucking tenant can see
|
| 259 |
-
# both Customer and Product database under Manage User, this is only for Royal Imports tenant.
|
| 260 |
-
# Make sure per tenant configuration always work."*
|
| 261 |
-
#
|
| 262 |
-
# ⛔ THE CAUSE IS A TWO-ELEMENT TUPLE. `routes_admin._PERM_MODULES = ("customer_data",
|
| 263 |
-
# "product_data")` — a compiled literal that `get_perms` builds its whole list from without ever
|
| 264 |
-
# touching `session.runtime`. It is tenant-blind BY CONSTRUCTION, which is why no amount of
|
| 265 |
-
# per-tenant work elsewhere ever fixed it, and why the answer is not "filter the tuple" but
|
| 266 |
-
# "delete the tuple": a second hand-typed copy of a fact the registry already holds is a thing
|
| 267 |
-
# that CAN go stale, and this one did, four times.
|
| 268 |
-
#
|
| 269 |
-
# ⚠ THE LINE BETWEEN THE TWO HALVES, stated because both sides need it: this map is
|
| 270 |
-
# REGISTRY-SIDE and TENANT-BLIND — it answers *"which compiled modules are governable, and what
|
| 271 |
-
# are they called"*, nothing more. Applying the tenant's own `config["modules"]` filter and
|
| 272 |
-
# merging that tenant's `ut_*` databases is the CALLER's half (`routes_admin`, D's fence), which
|
| 273 |
-
# is exactly what `routes_nav.py` already does correctly and `routes_admin` never did.
|
| 274 |
-
def governable_modules():
|
| 275 |
-
"""`{topic: {'key','label','subject'}}` — every registry row that is a governable DATABASE.
|
| 276 |
-
|
| 277 |
-
A fresh dict per call, deliberately: a module-level mapping handed to a route is one
|
| 278 |
-
`.pop()` away from being a different list for every request after the first.
|
| 279 |
-
"""
|
| 280 |
-
return {m['topic']: {'key': m['key'], 'label': m['label'], 'subject': m.get('subject')}
|
| 281 |
-
for m in REGISTRY
|
| 282 |
-
if m.get('topic') and not m.get('archived') and not m.get('group_only')}
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
def subject_of(key):
|
| 286 |
-
"""The subject a REGISTRY row claims, or None. (`ut_*` databases carry theirs on the stored
|
| 287 |
-
definition instead — they are per-tenant data, not compiled rows.)"""
|
| 288 |
-
return (BY_KEY.get(key) or {}).get('subject')
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
def subject_conflict(subject, key, claimed=None):
|
| 292 |
-
"""The OTHER key already presenting `subject`, or None.
|
| 293 |
-
|
| 294 |
-
`claimed` is an optional {subject: key} of claims outside the registry — in practice the
|
| 295 |
-
caller's own tenant document, whose stored definitions carry a `subject` stamp. It is a
|
| 296 |
-
PARAMETER rather than a lookup on purpose: the store is per-tenant, two tenants both holding
|
| 297 |
-
an "Odoo customers" is entirely legitimate, and a module-level cache of somebody's tables is
|
| 298 |
-
how D-169 happened. This function knows nothing about which tenant it is serving.
|
| 299 |
-
|
| 300 |
-
⭐ THE TEST IS "A DIFFERENT KEY", NOT "IS THIS SUBJECT TAKEN". `_ensure_table_inplace` and
|
| 301 |
-
`ut_ensure` re-run on every boot and every 1800 s resync, adopting their table BY KEY — so a
|
| 302 |
-
check phrased "is this subject already claimed?" goes red on the second boot against the very
|
| 303 |
-
table it created on the first, and the container fails to start. Idempotency is not an
|
| 304 |
-
exception to the rule here; it is the common case.
|
| 305 |
-
"""
|
| 306 |
-
if not subject:
|
| 307 |
-
return None # silence is not a claim
|
| 308 |
-
for holder in (SUBJECT_CLAIMS, claimed or {}):
|
| 309 |
-
other = holder.get(subject)
|
| 310 |
-
if other and other != key:
|
| 311 |
-
return other
|
| 312 |
-
return None
|
|
|
|
| 22 |
2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
|
| 23 |
Collections + Analyst/plumbing): products, assortment, financial, management,
|
| 24 |
pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
note one-line scope note shown in the UI / docs
|
| 26 |
"""
|
| 27 |
# WAVE-9 I8 — `'source'`: which connected system a module's data comes from.
|
|
|
|
| 65 |
# myday->customer_list was done. The STORE keys are deliberately NOT renamed:
|
| 66 |
# 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
|
| 67 |
# would orphan every saved view. Label-only changes need none of that machinery.
|
| 68 |
+
{'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
|
| 70 |
'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
|
| 71 |
'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
|
|
|
|
| 101 |
'group_only': True,
|
| 102 |
'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
|
| 103 |
'not part of this family.'},
|
| 104 |
+
{'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
|
|
|
|
|
|
|
|
|
|
| 105 |
'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
|
| 106 |
'stock columns, with saved views and custom fields. Identity = the SKU code.'},
|
| 107 |
{'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
|
|
|
| 197 |
|
| 198 |
def brand_filterable(key) -> bool:
|
| 199 |
return bool(BY_KEY.get(key, {}).get('brand'))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
platform/core/shares.py
CHANGED
|
@@ -43,32 +43,10 @@ SHARES_KEY = 'object_shares'
|
|
| 43 |
#: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
|
| 44 |
KINDS = ('view', 'folder', 'database')
|
| 45 |
|
| 46 |
-
#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform"
|
| 47 |
-
#:
|
| 48 |
-
#:
|
| 49 |
-
#:
|
| 50 |
-
#: ⛔⛔ **AND WHAT THAT MEANS DEPENDS ON THE KIND — THE LINE THIS NOTE USED TO CARRY WAS FALSE FOR
|
| 51 |
-
#: ONE OF THE THREE** (W33-T30, `waves/wave32/sharing-audit.md` S-8). It read *"the module/table
|
| 52 |
-
#: wall runs FIRST and this never widens past it"*, flatly, and the audit's own words for that are
|
| 53 |
-
#: *"the third docstring in this audit describing a check that is not on the path"*. Corrected
|
| 54 |
-
#: here rather than deleted, because the sentence is TRUE of two kinds and the difference is the
|
| 55 |
-
#: whole point:
|
| 56 |
-
#: * `kind='view'` / `kind='folder'` on a GOVERNED module (`customer_data`, `product_data`) —
|
| 57 |
-
#: the sentence holds. `require_session` plus the topic's own gate run first, and the
|
| 58 |
-
#: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
|
| 59 |
-
#: merged, so a grant can only narrow-or-equal what that account could already reach.
|
| 60 |
-
#: * `kind='database'` on a `ut_*` table — **THE SENTENCE IS FALSE AND THIS REGISTRY IS THE
|
| 61 |
-
#: ONLY WALL.** `routes_admin._PERM_MODULES` is `("customer_data", "product_data")` and
|
| 62 |
-
#: `_clean_perms` 400s anything else, so no row filter and no hidden field can even be
|
| 63 |
-
#: DECLARED for a user table; `routes_tables.py` makes zero `perm_scope` calls and passes
|
| 64 |
-
#: `hidden_keys=frozenset()`. A `database` grant is therefore ALL-OR-NOTHING — every row,
|
| 65 |
-
#: every column — and an `*` database grant admits every account in the tenant to all of it.
|
| 66 |
-
#:
|
| 67 |
-
#: ⚠ THAT IS A REAL CAPABILITY, DELIBERATELY KEPT, not a hole to plug in passing. What was wrong
|
| 68 |
-
#: was a docstring promising a second wall that does not exist for this kind; scoping user tables
|
| 69 |
-
#: is booked (S-8), not done. `routes_shares.py`'s module docstring says the same thing at the
|
| 70 |
-
#: OTHER door — the audit's fix is "say so at both", and one door saying it is how the next reader
|
| 71 |
-
#: gets the confident half [[one-question-two-normalizers]].
|
| 72 |
EVERYONE = '*'
|
| 73 |
|
| 74 |
ROLES = ('view', 'edit')
|
|
|
|
| 43 |
#: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
|
| 44 |
KINDS = ('view', 'folder', 'database')
|
| 45 |
|
| 46 |
+
#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform" —
|
| 47 |
+
#: the module/table wall runs FIRST and this never widens past it. Spelled as a single character
|
| 48 |
+
#: so it can never collide with a username (usernames are lower-case and non-empty by
|
| 49 |
+
#: `core/users.py`, and are checked against this explicitly below).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
EVERYONE = '*'
|
| 51 |
|
| 52 |
ROLES = ('view', 'edit')
|
platform/core/store.py
CHANGED
|
@@ -1,838 +1,817 @@
|
|
| 1 |
-
"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo — the platform's ONLY writable store
|
| 2 |
-
(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
|
| 3 |
-
|
| 4 |
-
Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
|
| 5 |
-
repo is created private on first write. Reads use a short in-process cache that writes refresh, so
|
| 6 |
-
within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
|
| 7 |
-
|
| 8 |
-
WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
|
| 9 |
-
functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) — every existing
|
| 10 |
-
`import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
|
| 11 |
-
`for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
|
| 12 |
-
(nurilab, gtmlab, …); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
|
| 13 |
-
instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
|
| 14 |
-
write burst can never starve another's flush window.
|
| 15 |
-
"""
|
| 16 |
-
import io
|
| 17 |
-
import os
|
| 18 |
-
import json
|
| 19 |
-
import threading
|
| 20 |
-
import time as _time
|
| 21 |
-
from pathlib import Path
|
| 22 |
-
|
| 23 |
-
try:
|
| 24 |
-
from dotenv import load_dotenv
|
| 25 |
-
load_dotenv(Path(__file__).resolve().parents[1] / '.env')
|
| 26 |
-
except Exception:
|
| 27 |
-
pass
|
| 28 |
-
|
| 29 |
-
from huggingface_hub import HfApi, hf_hub_download
|
| 30 |
-
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
|
| 31 |
-
|
| 32 |
-
REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
|
| 33 |
-
_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
|
| 34 |
-
_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
|
| 35 |
-
|
| 36 |
-
#: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP.
|
| 37 |
-
#:
|
| 38 |
-
#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a
|
| 39 |
-
#: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude
|
| 40 |
-
#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a
|
| 41 |
-
#: change, which is the truthful answer: the client's copy predates this process entirely.
|
| 42 |
-
#:
|
| 43 |
-
#: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality.
|
| 44 |
-
REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}'
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def _token():
|
| 48 |
-
return os.environ.get('HF_TOKEN') or None
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
class _Projected(dict):
|
| 52 |
-
"""One top-level value of a PROJECTED read — a dict that RAISES for the keys that were dropped.
|
| 53 |
-
|
| 54 |
-
⛔⛔ W32-T01: THE ALTERNATIVE IS A SILENT WRONG ANSWER, WHICH IS WHY THIS CLASS EXISTS AT ALL.
|
| 55 |
-
A projection built as a plain dict answers `defn.get('rows') or {}` with `{}` — indistinguishable
|
| 56 |
-
from a database that genuinely has no rows. The caller then paints an empty grid, exports an
|
| 57 |
-
empty CSV, or (worse) writes that emptiness back. This repo has shipped that exact shape before
|
| 58 |
-
([[empty-answer-vs-unfinished-answer]], and a grouped view that painted zero rows).
|
| 59 |
-
|
| 60 |
-
So a dropped key is not absent — it is UNREAD, and asking for it raises with the reason and the
|
| 61 |
-
fix in the message. `KeyError` is the type on purpose: `dict.__getitem__`'s own failure mode, so
|
| 62 |
-
nothing has to learn a new exception; and the message names `get_projection` so the traceback
|
| 63 |
-
points at the caller's mistake rather than at this file.
|
| 64 |
-
|
| 65 |
-
⚠ Still a real `dict` for every OTHER purpose — `json.dumps`, `dict(x)`, `**x`, iteration and
|
| 66 |
-
`in` all behave normally, because a projection has to serialise onto the wire like any
|
| 67 |
-
definition. Only the named keys are booby-trapped.
|
| 68 |
-
"""
|
| 69 |
-
|
| 70 |
-
__slots__ = ('_dropped',)
|
| 71 |
-
|
| 72 |
-
def __init__(self, data=None, dropped=()):
|
| 73 |
-
super().__init__(data or {})
|
| 74 |
-
object.__setattr__(self, '_dropped', frozenset(dropped))
|
| 75 |
-
|
| 76 |
-
def _refuse(self, key):
|
| 77 |
-
raise KeyError(
|
| 78 |
-
f'{key!r} was DROPPED by a projected store read and is not absent — this document came '
|
| 79 |
-
f'from Store.get_projection(drop={sorted(self._dropped)!r}). Reading it as empty would '
|
| 80 |
-
f'be a silent wrong answer. Use the whole read (Store.get / user_tables.all_tables) if '
|
| 81 |
-
f'you need {key!r}.')
|
| 82 |
-
|
| 83 |
-
def __getitem__(self, key):
|
| 84 |
-
if key in self._dropped:
|
| 85 |
-
self._refuse(key)
|
| 86 |
-
return super().__getitem__(key)
|
| 87 |
-
|
| 88 |
-
def get(self, key, default=None):
|
| 89 |
-
if key in self._dropped:
|
| 90 |
-
self._refuse(key)
|
| 91 |
-
return super().get(key, default)
|
| 92 |
-
|
| 93 |
-
def pop(self, key, *a):
|
| 94 |
-
if key in self._dropped:
|
| 95 |
-
self._refuse(key)
|
| 96 |
-
return super().pop(key, *a)
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
def _project(doc, drop):
|
| 100 |
-
"""A deep COPY of `doc` with `drop`'s keys removed from every top-level value.
|
| 101 |
-
|
| 102 |
-
⭐ The order is the optimisation: build the retained structure FIRST (a shallow comprehension
|
| 103 |
-
that never touches the dropped values), then deep-copy THAT. One `json.dumps` over 0.1% of the
|
| 104 |
-
bytes, instead of one over all of them.
|
| 105 |
-
"""
|
| 106 |
-
if not isinstance(doc, dict):
|
| 107 |
-
return json.loads(json.dumps(doc))
|
| 108 |
-
kept = {k: ({kk: vv for kk, vv in v.items() if kk not in drop} if isinstance(v, dict) else v)
|
| 109 |
-
for k, v in doc.items()}
|
| 110 |
-
copied = json.loads(json.dumps(kept))
|
| 111 |
-
return {k: (_Projected(v, drop) if isinstance(v, dict) else v) for k, v in copied.items()}
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
"""
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
self._lock
|
| 146 |
-
|
| 147 |
-
#
|
| 148 |
-
#
|
| 149 |
-
#
|
| 150 |
-
#
|
| 151 |
-
#
|
| 152 |
-
#
|
| 153 |
-
#
|
| 154 |
-
#
|
| 155 |
-
|
| 156 |
-
self.
|
| 157 |
-
self.
|
| 158 |
-
self.
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
return {}
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
with self._lock:
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
return
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
def
|
| 523 |
-
"""
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
#
|
| 582 |
-
#
|
| 583 |
-
#
|
| 584 |
-
#
|
| 585 |
-
#
|
| 586 |
-
#
|
| 587 |
-
#
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
#
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
#
|
| 613 |
-
#
|
| 614 |
-
#
|
| 615 |
-
if
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
data
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
#
|
| 683 |
-
#
|
| 684 |
-
#
|
| 685 |
-
#
|
| 686 |
-
#
|
| 687 |
-
#
|
| 688 |
-
#
|
| 689 |
-
#
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
|
| 760 |
-
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
|
| 772 |
-
|
| 773 |
-
|
| 774 |
-
|
| 775 |
-
|
| 776 |
-
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
|
| 799 |
-
|
| 800 |
-
|
| 801 |
-
|
| 802 |
-
|
| 803 |
-
|
| 804 |
-
|
| 805 |
-
|
| 806 |
-
|
| 807 |
-
|
| 808 |
-
|
| 809 |
-
|
| 810 |
-
|
| 811 |
-
|
| 812 |
-
|
| 813 |
-
|
| 814 |
-
|
| 815 |
-
|
| 816 |
-
|
| 817 |
-
|
| 818 |
-
|
| 819 |
-
⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS
|
| 820 |
-
HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely
|
| 821 |
-
so a capability one backend lacks cannot be reached through the shared door and die at the
|
| 822 |
-
cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change
|
| 823 |
-
and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half
|
| 824 |
-
had no owner: a method is a capability the HF store has, a module function is a promise both
|
| 825 |
-
backends keep.
|
| 826 |
-
"""
|
| 827 |
-
return _d().revision(name)
|
| 828 |
-
|
| 829 |
-
|
| 830 |
-
def _flush_all_at_exit():
|
| 831 |
-
with _INSTANCES_LOCK:
|
| 832 |
-
instances = list(_INSTANCES.values())
|
| 833 |
-
for inst in instances:
|
| 834 |
-
inst._flush_now_at_exit()
|
| 835 |
-
|
| 836 |
-
|
| 837 |
-
import atexit # noqa: E402 (registered after the class it needs)
|
| 838 |
-
atexit.register(_flush_all_at_exit)
|
|
|
|
| 1 |
+
"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo — the platform's ONLY writable store
|
| 2 |
+
(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
|
| 3 |
+
|
| 4 |
+
Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
|
| 5 |
+
repo is created private on first write. Reads use a short in-process cache that writes refresh, so
|
| 6 |
+
within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
|
| 7 |
+
|
| 8 |
+
WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
|
| 9 |
+
functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) — every existing
|
| 10 |
+
`import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
|
| 11 |
+
`for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
|
| 12 |
+
(nurilab, gtmlab, …); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
|
| 13 |
+
instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
|
| 14 |
+
write burst can never starve another's flush window.
|
| 15 |
+
"""
|
| 16 |
+
import io
|
| 17 |
+
import os
|
| 18 |
+
import json
|
| 19 |
+
import threading
|
| 20 |
+
import time as _time
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
from dotenv import load_dotenv
|
| 25 |
+
load_dotenv(Path(__file__).resolve().parents[1] / '.env')
|
| 26 |
+
except Exception:
|
| 27 |
+
pass
|
| 28 |
+
|
| 29 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 30 |
+
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
|
| 31 |
+
|
| 32 |
+
REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
|
| 33 |
+
_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
|
| 34 |
+
_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
|
| 35 |
+
|
| 36 |
+
#: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP.
|
| 37 |
+
#:
|
| 38 |
+
#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a
|
| 39 |
+
#: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude
|
| 40 |
+
#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a
|
| 41 |
+
#: change, which is the truthful answer: the client's copy predates this process entirely.
|
| 42 |
+
#:
|
| 43 |
+
#: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality.
|
| 44 |
+
REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}'
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _token():
|
| 48 |
+
return os.environ.get('HF_TOKEN') or None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class _Projected(dict):
|
| 52 |
+
"""One top-level value of a PROJECTED read — a dict that RAISES for the keys that were dropped.
|
| 53 |
+
|
| 54 |
+
⛔⛔ W32-T01: THE ALTERNATIVE IS A SILENT WRONG ANSWER, WHICH IS WHY THIS CLASS EXISTS AT ALL.
|
| 55 |
+
A projection built as a plain dict answers `defn.get('rows') or {}` with `{}` — indistinguishable
|
| 56 |
+
from a database that genuinely has no rows. The caller then paints an empty grid, exports an
|
| 57 |
+
empty CSV, or (worse) writes that emptiness back. This repo has shipped that exact shape before
|
| 58 |
+
([[empty-answer-vs-unfinished-answer]], and a grouped view that painted zero rows).
|
| 59 |
+
|
| 60 |
+
So a dropped key is not absent — it is UNREAD, and asking for it raises with the reason and the
|
| 61 |
+
fix in the message. `KeyError` is the type on purpose: `dict.__getitem__`'s own failure mode, so
|
| 62 |
+
nothing has to learn a new exception; and the message names `get_projection` so the traceback
|
| 63 |
+
points at the caller's mistake rather than at this file.
|
| 64 |
+
|
| 65 |
+
⚠ Still a real `dict` for every OTHER purpose — `json.dumps`, `dict(x)`, `**x`, iteration and
|
| 66 |
+
`in` all behave normally, because a projection has to serialise onto the wire like any
|
| 67 |
+
definition. Only the named keys are booby-trapped.
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
__slots__ = ('_dropped',)
|
| 71 |
+
|
| 72 |
+
def __init__(self, data=None, dropped=()):
|
| 73 |
+
super().__init__(data or {})
|
| 74 |
+
object.__setattr__(self, '_dropped', frozenset(dropped))
|
| 75 |
+
|
| 76 |
+
def _refuse(self, key):
|
| 77 |
+
raise KeyError(
|
| 78 |
+
f'{key!r} was DROPPED by a projected store read and is not absent — this document came '
|
| 79 |
+
f'from Store.get_projection(drop={sorted(self._dropped)!r}). Reading it as empty would '
|
| 80 |
+
f'be a silent wrong answer. Use the whole read (Store.get / user_tables.all_tables) if '
|
| 81 |
+
f'you need {key!r}.')
|
| 82 |
+
|
| 83 |
+
def __getitem__(self, key):
|
| 84 |
+
if key in self._dropped:
|
| 85 |
+
self._refuse(key)
|
| 86 |
+
return super().__getitem__(key)
|
| 87 |
+
|
| 88 |
+
def get(self, key, default=None):
|
| 89 |
+
if key in self._dropped:
|
| 90 |
+
self._refuse(key)
|
| 91 |
+
return super().get(key, default)
|
| 92 |
+
|
| 93 |
+
def pop(self, key, *a):
|
| 94 |
+
if key in self._dropped:
|
| 95 |
+
self._refuse(key)
|
| 96 |
+
return super().pop(key, *a)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _project(doc, drop):
|
| 100 |
+
"""A deep COPY of `doc` with `drop`'s keys removed from every top-level value.
|
| 101 |
+
|
| 102 |
+
⭐ The order is the optimisation: build the retained structure FIRST (a shallow comprehension
|
| 103 |
+
that never touches the dropped values), then deep-copy THAT. One `json.dumps` over 0.1% of the
|
| 104 |
+
bytes, instead of one over all of them.
|
| 105 |
+
"""
|
| 106 |
+
if not isinstance(doc, dict):
|
| 107 |
+
return json.loads(json.dumps(doc))
|
| 108 |
+
kept = {k: ({kk: vv for kk, vv in v.items() if kk not in drop} if isinstance(v, dict) else v)
|
| 109 |
+
for k, v in doc.items()}
|
| 110 |
+
copied = json.loads(json.dumps(kept))
|
| 111 |
+
return {k: (_Projected(v, drop) if isinstance(v, dict) else v) for k, v in copied.items()}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class Store:
|
| 115 |
+
"""One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush.
|
| 116 |
+
|
| 117 |
+
Every method body is the pre-wave-18 module function, verbatim except globals→fields.
|
| 118 |
+
The behaviour contracts that matter are stated on each method and are UNCHANGED:
|
| 119 |
+
strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads.
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
def __init__(self, repo):
|
| 123 |
+
self.repo = str(repo)
|
| 124 |
+
self._lock = threading.RLock()
|
| 125 |
+
self._cache = {}
|
| 126 |
+
# --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------
|
| 127 |
+
# One typed character in a Notes cell used to cost THREE synchronous network calls
|
| 128 |
+
# inside the component round-trip. update(..., flush='async') applies the mutation to
|
| 129 |
+
# the in-process cache synchronously (read-your-writes: every reader consults the
|
| 130 |
+
# cache first) and a background worker coalesces uploads. The restart-wipe protection
|
| 131 |
+
# is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
|
| 132 |
+
# strict-reads. Known, accepted: with a second concurrent writer the stale window
|
| 133 |
+
# widens from ~one round-trip to ~the flush gap.
|
| 134 |
+
self._owned = {} # name -> True once cache reflects our own last accepted write
|
| 135 |
+
self._dirty = {} # name -> cache is ahead of the hub
|
| 136 |
+
self._flushers = {} # name -> live worker thread
|
| 137 |
+
self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
|
| 138 |
+
self._last_flush = {} # name -> monotonic time of the last completed upload
|
| 139 |
+
# --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------
|
| 140 |
+
# name -> how many accepted writes this process has applied to that bucket, and when.
|
| 141 |
+
# Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for
|
| 142 |
+
# why that constraint is the whole feature rather than an optimisation.
|
| 143 |
+
self._revs = {}
|
| 144 |
+
# --- ⭐⭐ W31-T13 (D-177(d)) — THE PER-KEY WRITE LOCK ------------------------------
|
| 145 |
+
# `self._lock` guards the CACHE. It used to be held across a sync `update`'s whole
|
| 146 |
+
# read-modify-write — a hub download, the caller's `fn`, and a blocking hub upload — so
|
| 147 |
+
# one write froze every reader in the process for two network round trips. `get()` takes
|
| 148 |
+
# the same lock, and `/nav`, `/workspace` and every permission check are `get()`.
|
| 149 |
+
# ⛔ IT IS NOT SAFE TO SIMPLY DROP THE LOCK. Two sync writers of one key would interleave
|
| 150 |
+
# read-modify-write and one update would vanish — corruption, not slowness. So the two
|
| 151 |
+
# jobs are separated: this lock serialises WRITERS OF ONE KEY (held across the network,
|
| 152 |
+
# never touching readers), and `self._lock` is taken only for the microseconds a cache
|
| 153 |
+
# read or write needs. Readers never wait on a network call again.
|
| 154 |
+
# ⚠ PER KEY, not one write lock: two tenants' buckets, or `user_tables` and `nav_meta`,
|
| 155 |
+
# have no reason to serialise against each other.
|
| 156 |
+
self._key_locks = {}
|
| 157 |
+
self._key_locks_guard = threading.Lock()
|
| 158 |
+
self._rev_at = {}
|
| 159 |
+
|
| 160 |
+
def available(self):
|
| 161 |
+
"""True when a token is configured — callers degrade gracefully without one."""
|
| 162 |
+
return bool(_token())
|
| 163 |
+
|
| 164 |
+
def _download(self, name):
|
| 165 |
+
"""Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent.
|
| 166 |
+
Any other failure (network, auth, rate-limit, corrupt JSON) RAISES — a transient read
|
| 167 |
+
error must never be mistaken for an empty store, or a caller could overwrite real data
|
| 168 |
+
with an empty dict (this is what wiped the user registry on restart)."""
|
| 169 |
+
try:
|
| 170 |
+
path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset',
|
| 171 |
+
token=_token(), force_download=True)
|
| 172 |
+
with open(path, encoding='utf-8') as f:
|
| 173 |
+
return json.load(f)
|
| 174 |
+
except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
|
| 175 |
+
return {}
|
| 176 |
+
except ValueError as e:
|
| 177 |
+
# Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in
|
| 178 |
+
# ValueError("Force download failed…") with the 404 as __cause__ — so a brand-new
|
| 179 |
+
# tenant repo's very first strict read raised here and aborted its very first write.
|
| 180 |
+
# A genuinely-absent repo is the same fact as a genuinely-absent file: {}.
|
| 181 |
+
cause = getattr(e, '__cause__', None)
|
| 182 |
+
if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)):
|
| 183 |
+
return {}
|
| 184 |
+
raise
|
| 185 |
+
|
| 186 |
+
def get(self, name, fresh=False):
|
| 187 |
+
"""Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error,
|
| 188 |
+
hand back the last known-good cached value if we have one, else {}. Mutating callers go
|
| 189 |
+
through update()/_read_strict, which RAISE instead."""
|
| 190 |
+
with self._lock:
|
| 191 |
+
if not fresh and name in self._cache:
|
| 192 |
+
return json.loads(json.dumps(self._cache[name])) # hand back a copy
|
| 193 |
+
try:
|
| 194 |
+
data = self._download(name)
|
| 195 |
+
except Exception as e:
|
| 196 |
+
try: # lazy import — core stays layer-clean
|
| 197 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 198 |
+
_tel.error(f'store:get:{name}', e,
|
| 199 |
+
fallback='cached' if name in self._cache else 'empty')
|
| 200 |
+
except Exception:
|
| 201 |
+
pass
|
| 202 |
+
if name in self._cache:
|
| 203 |
+
return json.loads(json.dumps(self._cache[name]))
|
| 204 |
+
return {}
|
| 205 |
+
self._cache[name] = data
|
| 206 |
+
return json.loads(json.dumps(data))
|
| 207 |
+
|
| 208 |
+
def get_projection(self, name, drop=()):
|
| 209 |
+
"""Like `get`, but each top-level VALUE comes back WITHOUT the keys named in `drop`.
|
| 210 |
+
|
| 211 |
+
⭐⭐ W32-T01 (D-185) — THE POINT IS THE COPY, NOT THE PAYLOAD SIZE ON THE WIRE.
|
| 212 |
+
|
| 213 |
+
`get()` is honest and expensive: it hands back `json.loads(json.dumps(...))` so a caller
|
| 214 |
+
cannot mutate the cache, and on tenant #0's `user_tables` that copy is **28.6 MB / 81,330
|
| 215 |
+
rows / ~703 ms warm**, of which **99.9% is `rows`** — which no permission check, no nav
|
| 216 |
+
render and no workspace envelope ever reads. `/nav` used to pay ~13 of those copies per
|
| 217 |
+
request; W31-T10 made it ONE, and that one is still 703 ms of pure copying. This is the
|
| 218 |
+
floor D-185 named: copy only what was asked for.
|
| 219 |
+
|
| 220 |
+
⛔⛔ **THE PROJECTION IS NEVER CACHED, AND THAT LINE IS THE WHOLE SAFETY ARGUMENT.** A
|
| 221 |
+
rows-less document in `self._cache` would be served to the next WHOLE reader, and the next
|
| 222 |
+
`update()` would upload it — every row in the tenant deleted, by a read. So the cold path
|
| 223 |
+
downloads whole, caches WHOLE (exactly as `get` does), and projects the RETURN value only.
|
| 224 |
+
`verify_store_projection.py` asserts the cache still contains `rows` after a projected read,
|
| 225 |
+
with an NC that reds when the projection is cached instead.
|
| 226 |
+
|
| 227 |
+
⚠ **`self._lock` is held across the copy, same as `get`** — the retained structure is
|
| 228 |
+
~0.1% of the bytes, so this is microseconds where `get` is most of a second. Narrowing it
|
| 229 |
+
would let a concurrent writer mutate a value mid-serialise for no measurable gain.
|
| 230 |
+
|
| 231 |
+
⚠ **A dropped key RAISES on read rather than reading as absent** (`_Projected`). A
|
| 232 |
+
projection that answered `{}` for `rows` would paint an empty grid — the exact
|
| 233 |
+
[[empty-answer-vs-unfinished-answer]] shape this repo has shipped before — and the caller
|
| 234 |
+
that most wants rows is the one least able to tell. See `_Projected`.
|
| 235 |
+
|
| 236 |
+
⚠ Lenient, like `get`: a transient fetch error falls back to the cached document (projected)
|
| 237 |
+
and then to `{}`. Mutating callers still go through `update()`/`_read_strict`, which never
|
| 238 |
+
project — a write must always read whole.
|
| 239 |
+
"""
|
| 240 |
+
drop = frozenset(str(d) for d in drop)
|
| 241 |
+
with self._lock:
|
| 242 |
+
if name in self._cache:
|
| 243 |
+
return _project(self._cache[name], drop)
|
| 244 |
+
try:
|
| 245 |
+
data = self._download(name)
|
| 246 |
+
except Exception as e:
|
| 247 |
+
try: # lazy import — core stays layer-clean
|
| 248 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 249 |
+
_tel.error(f'store:get_projection:{name}', e,
|
| 250 |
+
fallback='cached' if name in self._cache else 'empty')
|
| 251 |
+
except Exception:
|
| 252 |
+
pass
|
| 253 |
+
with self._lock:
|
| 254 |
+
if name in self._cache:
|
| 255 |
+
return _project(self._cache[name], drop)
|
| 256 |
+
return {}
|
| 257 |
+
with self._lock:
|
| 258 |
+
self._cache[name] = data # ⛔ the WHOLE document, never the projection
|
| 259 |
+
return _project(data, drop)
|
| 260 |
+
|
| 261 |
+
def _key_lock(self, name):
|
| 262 |
+
"""The write lock for ONE key (W31-T13). Created on demand; never removed."""
|
| 263 |
+
n = str(name)
|
| 264 |
+
with self._key_locks_guard:
|
| 265 |
+
lk = self._key_locks.get(n)
|
| 266 |
+
if lk is None:
|
| 267 |
+
lk = self._key_locks[n] = threading.RLock()
|
| 268 |
+
return lk
|
| 269 |
+
|
| 270 |
+
def _read_strict(self, name):
|
| 271 |
+
"""Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
|
| 272 |
+
absent file), so a failed read ABORTS a read-modify-write instead of merging into {}.
|
| 273 |
+
|
| 274 |
+
⭐ W31-T13 — THE DOWNLOAD IS OUTSIDE `self._lock` NOW, and the re-check after it is the
|
| 275 |
+
whole of what makes that safe. A hub read takes a round trip; holding the cache lock
|
| 276 |
+
across it is what froze every reader. But moving it out opens a window in which an ASYNC
|
| 277 |
+
writer can accept a row into the cache while our download is in flight — and assigning our
|
| 278 |
+
(older) hub copy afterwards would erase it. That is D-177's original symptom exactly
|
| 279 |
+
(`POST /rows` → 201, row gone), re-created by the lock split rather than by a call site.
|
| 280 |
+
So the dirty mark is re-read AFTER the download, under the short lock, and a key that
|
| 281 |
+
became dirty keeps its cache.
|
| 282 |
+
"""
|
| 283 |
+
data = self._download(name) # network, no lock held
|
| 284 |
+
with self._lock:
|
| 285 |
+
if self._dirty.get(name) and self._owned.get(name) and name in self._cache:
|
| 286 |
+
# Our copy is stale by construction: a pending upload is going to overwrite the
|
| 287 |
+
# hub anyway, so the fresh read cannot preserve a remote write — it can only
|
| 288 |
+
# destroy a local one we have already acknowledged. Same argument, and the same
|
| 289 |
+
# three conditions, as `update`'s pending-async branch below.
|
| 290 |
+
return json.loads(json.dumps(self._cache[name]))
|
| 291 |
+
self._cache[name] = data
|
| 292 |
+
return json.loads(json.dumps(data))
|
| 293 |
+
|
| 294 |
+
def exists(self, name):
|
| 295 |
+
"""True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
|
| 296 |
+
uncertainty — a seeder must never clobber a registry we merely failed to reach."""
|
| 297 |
+
try:
|
| 298 |
+
files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset')
|
| 299 |
+
return f'{name}.json' in files
|
| 300 |
+
except RepositoryNotFoundError:
|
| 301 |
+
return False
|
| 302 |
+
except Exception:
|
| 303 |
+
return True
|
| 304 |
+
|
| 305 |
+
# ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6)
|
| 306 |
+
def _bump(self, name):
|
| 307 |
+
"""Advance one bucket's write counter. THE CALLER HOLDS THE LOCK.
|
| 308 |
+
|
| 309 |
+
Called from the two places an accepted write becomes visible to readers of this process
|
| 310 |
+
— `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT
|
| 311 |
+
from `_flush_worker`: that uploads data a bump already accounted for, and counting it
|
| 312 |
+
again would report a change to every browser each time the coalescing worker woke.
|
| 313 |
+
"""
|
| 314 |
+
n = str(name)
|
| 315 |
+
self._revs[n] = self._revs.get(n, 0) + 1
|
| 316 |
+
self._rev_at[n] = _time.time()
|
| 317 |
+
|
| 318 |
+
def revision(self, name):
|
| 319 |
+
"""`{'rev', 'updated_at', 'token'}` for ONE bucket — a dict read, nothing else.
|
| 320 |
+
|
| 321 |
+
⛔ THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why
|
| 322 |
+
this method exists at all. A browser asking "has anything changed" by re-fetching rows
|
| 323 |
+
costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way
|
| 324 |
+
in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s
|
| 325 |
+
per bucket — so two tabs polling saturate the server and every unrelated write queues
|
| 326 |
+
behind them. This performs no download, no `get()`, and no copy of the value.
|
| 327 |
+
⇒ Anything added here that reads a bucket destroys the whole point (`verify_live_workspace`
|
| 328 |
+
counts `get`/`_read_strict`/`_download` on the serving path and goes red on one).
|
| 329 |
+
|
| 330 |
+
`rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend
|
| 331 |
+
ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops
|
| 332 |
+
being inert the moment D-4 flips, with the wire shape unchanged.
|
| 333 |
+
|
| 334 |
+
⚠ WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write
|
| 335 |
+
by a SECOND container against the same HF dataset is invisible here. That is not a limit
|
| 336 |
+
this introduces — `get()` caches a bucket for the life of the process and never re-reads
|
| 337 |
+
it, so the other container's write is invisible to every reader on this backend already.
|
| 338 |
+
The token is exactly as fresh as the store it reports on. Postgres has no such gap.
|
| 339 |
+
⚠ Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted:
|
| 340 |
+
an attachment is fetched on explicit user action, never polled.
|
| 341 |
+
"""
|
| 342 |
+
n = str(name)
|
| 343 |
+
with self._lock:
|
| 344 |
+
rev = int(self._revs.get(n, 0))
|
| 345 |
+
at = self._rev_at.get(n)
|
| 346 |
+
return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'}
|
| 347 |
+
|
| 348 |
+
def _upload(self, name, data):
|
| 349 |
+
"""The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra
|
| 350 |
+
API round-trip on every put(), pure latency after the first success."""
|
| 351 |
+
api = HfApi(token=_token())
|
| 352 |
+
if not self._repo_ok:
|
| 353 |
+
api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
|
| 354 |
+
self._repo_ok = True
|
| 355 |
+
buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
|
| 356 |
+
api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo,
|
| 357 |
+
repo_type='dataset', commit_message=f'update {name}')
|
| 358 |
+
|
| 359 |
+
def upload_bytes(self, path_in_repo, data, message=None):
|
| 360 |
+
"""Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
|
| 361 |
+
Bytes are never cached in-process: large, cold, fetched on explicit user action."""
|
| 362 |
+
if not self.available():
|
| 363 |
+
raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
|
| 364 |
+
with self._lock:
|
| 365 |
+
api = HfApi(token=_token())
|
| 366 |
+
if not self._repo_ok:
|
| 367 |
+
api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
|
| 368 |
+
self._repo_ok = True
|
| 369 |
+
api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
|
| 370 |
+
repo_id=self.repo, repo_type='dataset',
|
| 371 |
+
commit_message=message or f'add {path_in_repo}')
|
| 372 |
+
|
| 373 |
+
def download_bytes(self, path_in_repo):
|
| 374 |
+
"""Raw bytes back, or None when absent/unreachable — a missing attachment degrades to
|
| 375 |
+
"that file is gone", never to a broken page."""
|
| 376 |
+
if not self.available():
|
| 377 |
+
return None
|
| 378 |
+
try:
|
| 379 |
+
local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset',
|
| 380 |
+
token=_token())
|
| 381 |
+
return Path(local).read_bytes()
|
| 382 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 383 |
+
return None
|
| 384 |
+
except Exception:
|
| 385 |
+
return None
|
| 386 |
+
|
| 387 |
+
def delete_path(self, path_in_repo):
|
| 388 |
+
"""Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal."""
|
| 389 |
+
if not self.available():
|
| 390 |
+
return False
|
| 391 |
+
try:
|
| 392 |
+
HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo,
|
| 393 |
+
repo_type='dataset',
|
| 394 |
+
commit_message=f'delete {path_in_repo}')
|
| 395 |
+
return True
|
| 396 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 397 |
+
return True
|
| 398 |
+
except Exception:
|
| 399 |
+
return False
|
| 400 |
+
|
| 401 |
+
def put(self, name, data):
|
| 402 |
+
"""Persist {name}.json (creates the private repo on first write). Updates the cache."""
|
| 403 |
+
if not self.available():
|
| 404 |
+
raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
|
| 405 |
+
with self._lock:
|
| 406 |
+
self._upload(name, data)
|
| 407 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 408 |
+
self._owned[name] = True
|
| 409 |
+
self._last_flush[name] = _time.monotonic()
|
| 410 |
+
self._bump(name) # C6 — AFTER the upload: a raise here is not a write
|
| 411 |
+
return self._cache[name]
|
| 412 |
+
|
| 413 |
+
def _flush_worker(self, name):
|
| 414 |
+
"""Coalescing uploader for one key. Runs while dirty; retries with backoff on failure —
|
| 415 |
+
the cache stays the truth and the dirty mark survives, so nothing is lost silently.
|
| 416 |
+
|
| 417 |
+
⭐⭐ W31-T13 — THIS WORKER IS THE SECOND WRITER, AND THE PER-KEY LOCK MUST COVER IT.
|
| 418 |
+
⛔ THE RACE THE LOCK SPLIT OPENED, spelled out because it is silent and survives a restart
|
| 419 |
+
as real data loss. `update`'s big lock used to span `_read_strict` + `fn` + the upload, so
|
| 420 |
+
this worker could never snapshot mid-update. With the cache lock narrowed it can:
|
| 421 |
+
async write -> `_dirty=True`, this worker scheduled, sleeps `_FLUSH_DELAY`
|
| 422 |
+
sync update t+0.5 -> takes the dirty branch, runs `fn`, enters `_upload` (no cache
|
| 423 |
+
lock held — that is the whole point of the ticket)
|
| 424 |
+
worker t+2.0 -> `self._lock` is FREE and `_dirty` is still True (the sync commit
|
| 425 |
+
has not happened yet), so it snapshots the PRE-UPDATE cache,
|
| 426 |
+
clears the mark, and uploads it AFTER the sync upload lands
|
| 427 |
+
result -> cache has the new value, the hub has the old one, `_dirty` is
|
| 428 |
+
False, and nothing will ever re-upload. Divergence with no
|
| 429 |
+
symptom until the next process boots and reads the hub.
|
| 430 |
+
⚠ `user_tables` has ~40 sync writers against 2 async ones, so that window is the COMMON
|
| 431 |
+
case, not an exotic one — it is D-177's own shape, re-created one layer down.
|
| 432 |
+
⛔ LOCK ORDER IS KEY-THEN-CACHE, EVERYWHERE. `_update_locked` takes them in that order, so
|
| 433 |
+
this must too; taking `self._lock` first on this path is the deadlock. Both sleeps stay
|
| 434 |
+
OUTSIDE the key lock — a worker that dozed while holding it would block every writer of
|
| 435 |
+
that key for `_FLUSH_MIN_GAP`, which is the freeze this ticket exists to remove.
|
| 436 |
+
"""
|
| 437 |
+
backoff = 2.0
|
| 438 |
+
while True:
|
| 439 |
+
_time.sleep(_FLUSH_DELAY)
|
| 440 |
+
wait, err, snapshot = None, None, None
|
| 441 |
+
with self._key_lock(name):
|
| 442 |
+
with self._lock:
|
| 443 |
+
gap = _time.monotonic() - self._last_flush.get(name, 0.0)
|
| 444 |
+
if gap < _FLUSH_MIN_GAP:
|
| 445 |
+
wait = _FLUSH_MIN_GAP - gap
|
| 446 |
+
elif not self._dirty.get(name):
|
| 447 |
+
self._flushers.pop(name, None)
|
| 448 |
+
return
|
| 449 |
+
else:
|
| 450 |
+
self._dirty[name] = False
|
| 451 |
+
snapshot = json.loads(json.dumps(self._cache.get(name, {})))
|
| 452 |
+
if wait is None:
|
| 453 |
+
try:
|
| 454 |
+
self._upload(name, snapshot)
|
| 455 |
+
with self._lock:
|
| 456 |
+
self._last_flush[name] = _time.monotonic()
|
| 457 |
+
backoff = 2.0
|
| 458 |
+
except Exception as e: # noqa: BLE001
|
| 459 |
+
with self._lock:
|
| 460 |
+
self._dirty[name] = True # keep it queued; cache is still the truth
|
| 461 |
+
err = e
|
| 462 |
+
if wait is not None:
|
| 463 |
+
_time.sleep(wait)
|
| 464 |
+
continue
|
| 465 |
+
if err is not None:
|
| 466 |
+
try:
|
| 467 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 468 |
+
_tel.error(f'store:flush:{name}', err)
|
| 469 |
+
except Exception:
|
| 470 |
+
pass
|
| 471 |
+
_time.sleep(backoff)
|
| 472 |
+
backoff = min(backoff * 2, 60.0)
|
| 473 |
+
|
| 474 |
+
def _schedule_flush(self, name):
|
| 475 |
+
"""Mark dirty + ensure exactly one live worker for the key. Caller holds the lock."""
|
| 476 |
+
self._dirty[name] = True
|
| 477 |
+
t = self._flushers.get(name)
|
| 478 |
+
if t is not None and t.is_alive():
|
| 479 |
+
return
|
| 480 |
+
t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True,
|
| 481 |
+
name=f'store-flush-{self.repo}:{name}')
|
| 482 |
+
self._flushers[name] = t
|
| 483 |
+
t.start()
|
| 484 |
+
|
| 485 |
+
def flush(self, name=None, timeout=30.0):
|
| 486 |
+
"""Block until pending async writes for `name` (or ALL keys) reach the hub — QA gates
|
| 487 |
+
and shutdown hooks use this; the app itself never needs to."""
|
| 488 |
+
deadline = _time.monotonic() + timeout
|
| 489 |
+
names = [name] if name else None
|
| 490 |
+
while _time.monotonic() < deadline:
|
| 491 |
+
with self._lock:
|
| 492 |
+
pending = {k for k, v in self._dirty.items() if v} | {
|
| 493 |
+
k for k, t in self._flushers.items() if t.is_alive()}
|
| 494 |
+
if names is not None:
|
| 495 |
+
pending &= set(names)
|
| 496 |
+
if not pending:
|
| 497 |
+
return True
|
| 498 |
+
_time.sleep(0.2)
|
| 499 |
+
return False
|
| 500 |
+
|
| 501 |
+
def _flush_now_at_exit(self):
|
| 502 |
+
"""Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
|
| 503 |
+
gap — a Space restart must not eat the last seconds of the async window.
|
| 504 |
+
|
| 505 |
+
⚠ W31-T13 — DELIBERATELY NOT KEY-LOCKED, stated so the omission is not read as one. This
|
| 506 |
+
runs at interpreter exit with the process going away: a writer mid-`update` will never
|
| 507 |
+
finish, so waiting for its key lock could only turn a partial save into no save at all.
|
| 508 |
+
Best-effort is the correct contract here, and it is the one case where taking the lock
|
| 509 |
+
would be worse than skipping it.
|
| 510 |
+
"""
|
| 511 |
+
with self._lock:
|
| 512 |
+
dirty = [k for k, v in self._dirty.items() if v]
|
| 513 |
+
snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty}
|
| 514 |
+
for k in dirty:
|
| 515 |
+
self._dirty[k] = False
|
| 516 |
+
for k, d in snaps.items():
|
| 517 |
+
try:
|
| 518 |
+
self._upload(k, d)
|
| 519 |
+
except Exception:
|
| 520 |
+
pass # exiting anyway — nothing left to reschedule
|
| 521 |
+
|
| 522 |
+
def update(self, name, fn, flush='sync'):
|
| 523 |
+
"""Read-modify-write. STRICT first read (raises on a transient error rather than
|
| 524 |
+
starting from an empty dict), apply fn(data)->data, persist.
|
| 525 |
+
|
| 526 |
+
flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the
|
| 527 |
+
table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and
|
| 528 |
+
returns immediately; a coalescing worker commits. The strict read still guards the
|
| 529 |
+
FIRST async update of a key per process.
|
| 530 |
+
|
| 531 |
+
⭐⭐ W31-T13 (D-177(d)) — TWO LOCKS, AND THE SPLIT IS THE WHOLE TICKET.
|
| 532 |
+
`self._key_lock(name)` is held for the entire read-modify-write, so two writers of one key
|
| 533 |
+
still cannot interleave (last-write-wins is the contract; a lost update is not).
|
| 534 |
+
`self._lock` — the one every `get()` takes — is now held only for the microseconds a cache
|
| 535 |
+
read or write needs, never across a download or an upload. Before this, one sync write
|
| 536 |
+
froze every reader in the process for two network round trips, on a store whose warm
|
| 537 |
+
`get()` already costs 703 ms of deep copy on tenant #0.
|
| 538 |
+
⚠ Uploading outside `self._lock` is not a new posture: `_flush_worker` has done exactly
|
| 539 |
+
that since wave 7, and `_repo_ok` racing is benign because `create_repo` is `exist_ok=True`.
|
| 540 |
+
"""
|
| 541 |
+
with self._key_lock(name):
|
| 542 |
+
return self._update_locked(name, fn, flush)
|
| 543 |
+
|
| 544 |
+
def _update_locked(self, name, fn, flush):
|
| 545 |
+
"""`update`'s body, with this key's write lock already held. See its docstring.
|
| 546 |
+
|
| 547 |
+
⚠ THE CACHE LOCK IS TAKEN THREE TIMES AND RELEASED BETWEEN THEM, which is only safe
|
| 548 |
+
because THIS key's write lock is held throughout: nothing else can be mid-write on it, so
|
| 549 |
+
the state read in the first section is still ours in the third. A caller reaching this
|
| 550 |
+
without the key lock would have the old interleaving bug back.
|
| 551 |
+
"""
|
| 552 |
+
# ⚠ A FLAG, NOT A `None` SENTINEL. `None` can be a real cached value, and a sentinel that
|
| 553 |
+
# can equal real data is not a sentinel — this repo has paid for that exact shape before
|
| 554 |
+
# (`routes_nav`'s locked-mode default). The flag says "the cache answered", full stop.
|
| 555 |
+
data, from_cache = None, False
|
| 556 |
+
with self._lock:
|
| 557 |
+
if flush == 'async' and self._owned.get(name) and name in self._cache:
|
| 558 |
+
data, from_cache = json.loads(json.dumps(self._cache[name])), True
|
| 559 |
+
elif self._dirty.get(name) and self._owned.get(name) and name in self._cache:
|
| 560 |
+
# ⛔ A SYNC READ-MODIFY-WRITE MUST NOT DISCARD A PENDING ASYNC WRITE.
|
| 561 |
+
#
|
| 562 |
+
# THE BUG THIS LINE EXISTS FOR (owner report 2026-08-09, MEASURED on staging):
|
| 563 |
+
# `POST /rows` answered `201 {"rid":"100"}` and the row never existed.
|
| 564 |
+
# `user_tables.add_row` commits `flush='async'` — the row lives ONLY in
|
| 565 |
+
# `self._cache` until the coalescing worker uploads it, which is `_FLUSH_DELAY`
|
| 566 |
+
# (2s) away and up to `_FLUSH_MIN_GAP` (20s) if that key committed recently.
|
| 567 |
+
# Inside that window ANY sync writer of the SAME key took the `else` branch
|
| 568 |
+
# below, and `_read_strict` does `self._cache[name] = self._download(name)` —
|
| 569 |
+
# it REPLACES the cache with the hub's copy, erasing the only place the new row
|
| 570 |
+
# existed, and then `put()` uploaded that. Nothing raised. The 201 was truthful
|
| 571 |
+
# about `add_row`'s return value and false about the outcome.
|
| 572 |
+
#
|
| 573 |
+
# ⚠ `user_tables` has ~40 sync writers against 2 async ones, so this was never
|
| 574 |
+
# one bad call site — it is a property of the KEY. Wave 27 only made it fire
|
| 575 |
+
# every time, by putting the relation refresh on the add path.
|
| 576 |
+
#
|
| 577 |
+
# ⛔ AND RE-DOWNLOADING WAS PROTECTING NOTHING. Once a key is dirty our cache
|
| 578 |
+
# has already diverged and the pending upload was going to overwrite the hub
|
| 579 |
+
# anyway — so the fresh read cannot preserve a remote write, it can only destroy
|
| 580 |
+
# a local one we already acknowledged. This is the class's OWN documented
|
| 581 |
+
# contract, restored: see `_owned` above — *"read-your-writes: every reader
|
| 582 |
+
# consults the cache first"* — and `_flush_worker`'s *"the cache stays the truth
|
| 583 |
+
# and the dirty mark survives, so nothing is lost silently."*
|
| 584 |
+
#
|
| 585 |
+
# ⚠ NOT a weakening of `_read_strict`'s purpose. That exists so a FAILED read
|
| 586 |
+
# aborts the RMW instead of merging into `{}`; the cache is populated
|
| 587 |
+
# known-good local state, and this branch is only reachable when it is.
|
| 588 |
+
data, from_cache = json.loads(json.dumps(self._cache[name])), True
|
| 589 |
+
# else: the strict hub read happens BELOW, outside this lock (W31-T13)
|
| 590 |
+
|
| 591 |
+
# ⭐ W31-T13 — THE HUB READ AND THE CALLER'S `fn` RUN WITH NO CACHE LOCK HELD. `fn` is
|
| 592 |
+
# arbitrary caller code — `automation_engine`'s relation pass has run whole-tenant work
|
| 593 |
+
# inside one — and it used to execute with every reader in the process blocked behind it.
|
| 594 |
+
if not from_cache:
|
| 595 |
+
data = self._read_strict(name) # network; re-checks the dirty mark on the way back
|
| 596 |
+
result = fn(data)
|
| 597 |
+
data = result if result is not None else data
|
| 598 |
+
|
| 599 |
+
if flush == 'async':
|
| 600 |
+
if not self.available():
|
| 601 |
+
raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
|
| 602 |
+
with self._lock:
|
| 603 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 604 |
+
self._owned[name] = True
|
| 605 |
+
self._bump(name) # C6 — the async write is ACCEPTED here, not at flush time
|
| 606 |
+
self._schedule_flush(name)
|
| 607 |
+
return data
|
| 608 |
+
|
| 609 |
+
# ⛔ THE SYNC COMMIT, SPELLED OUT HERE RATHER THAN THROUGH `put()`, AND THAT IS A CHOICE.
|
| 610 |
+
# `put()` holds `self._lock` across its own `_upload` — which is exactly the freeze this
|
| 611 |
+
# ticket removes — but it is PUBLIC with seven callers outside this file (`users.py`,
|
| 612 |
+
# `modules/ar.py`, `modules/map.py`, `runtime.py`, `store_backend.py`,
|
| 613 |
+
# `ops/seed_pg_from_hf.py`), and changing its locking changes all of their concurrency
|
| 614 |
+
# semantics at once. Duplicating four assignments with this comment beside them is the
|
| 615 |
+
# smaller blast radius; if `put` is ever split, this collapses back into a call.
|
| 616 |
+
if not self.available():
|
| 617 |
+
raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
|
| 618 |
+
self._upload(name, data) # network, no cache lock held
|
| 619 |
+
with self._lock:
|
| 620 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 621 |
+
self._owned[name] = True
|
| 622 |
+
self._last_flush[name] = _time.monotonic()
|
| 623 |
+
# ⛔ AFTER THE UPLOAD, NEVER BEFORE — `put()`'s own law. `_upload` raises on failure,
|
| 624 |
+
# and a bump on that path would have `revision()` report a change that never landed.
|
| 625 |
+
self._bump(name)
|
| 626 |
+
# The upload above CARRIED whatever was pending, so the key is no longer ahead of
|
| 627 |
+
# the hub. Without this the flush worker wakes on a still-dirty mark and re-uploads
|
| 628 |
+
# identical content — a second commit against the 256-commits/hr budget for nothing.
|
| 629 |
+
# ⚠ AFTER the upload, for the same reason: on a raise the mark MUST survive (the cache
|
| 630 |
+
# is still the truth and the row must still be committed by the worker). Clearing it
|
| 631 |
+
# first would turn a transient upload error into the very silent loss the branch above
|
| 632 |
+
# exists to stop — which is why it sits here and not before `_upload`.
|
| 633 |
+
self._dirty[name] = False
|
| 634 |
+
return data
|
| 635 |
+
|
| 636 |
+
|
| 637 |
+
# ------------------------------------------------------------------- instances + module API
|
| 638 |
+
_INSTANCES = {}
|
| 639 |
+
_INSTANCES_LOCK = threading.RLock()
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def for_repo(repo_id):
|
| 643 |
+
"""The HF Store bound to `repo_id` — one instance per repo per process (each with its own
|
| 644 |
+
cache and flush state).
|
| 645 |
+
|
| 646 |
+
⚠ WAVE 20: this is now the HF-SPECIFIC factory. Callers that want "the right store for this
|
| 647 |
+
tenant, whatever the backend is" call `handle()` below. `for_repo` keeps its exact old
|
| 648 |
+
behaviour because the seed/migration tooling has to be able to name the FILE store
|
| 649 |
+
explicitly while the app runs on Postgres — a migration that could only reach the active
|
| 650 |
+
backend could not copy between them.
|
| 651 |
+
"""
|
| 652 |
+
rid = str(repo_id or '').strip() or REPO
|
| 653 |
+
with _INSTANCES_LOCK:
|
| 654 |
+
inst = _INSTANCES.get(rid)
|
| 655 |
+
if inst is None:
|
| 656 |
+
inst = Store(rid)
|
| 657 |
+
_INSTANCES[rid] = inst
|
| 658 |
+
return inst
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
# =============================================================================================
|
| 662 |
+
# WAVE 20 (owner ruling R1, closes DEBT D-4) — THE BACKEND SEAM.
|
| 663 |
+
#
|
| 664 |
+
# `core/store_backend.py` has carried this warning since EXIT-2b, and it was correct at the time:
|
| 665 |
+
#
|
| 666 |
+
# "Flipping STORE_BACKEND=pg does NOT redirect the ~40 existing callers that say
|
| 667 |
+
# `import core.store as store` — they are bound to the HF module directly. Rewiring them is
|
| 668 |
+
# task C-4 … THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise."
|
| 669 |
+
#
|
| 670 |
+
# **R1 IS C-4.** The owner ruled the cutover on 2026-08-05 (the D-4 trigger that fired: a SECOND
|
| 671 |
+
# server process — `royal-imports/cfo-os` came back as a pinned LIVE environment beside staging,
|
| 672 |
+
# and two containers on one last-write-wins file store is the race B-3 was always about).
|
| 673 |
+
#
|
| 674 |
+
# THE REWIRE IS HERE RATHER THAN IN 28 FILES, and that is a deliberate choice over the obvious
|
| 675 |
+
# alternative of `sed`-ing every `import core.store as store` to `import core.store_backend`:
|
| 676 |
+
# * every one of those callers means "the store for the tenant I am serving", which is exactly
|
| 677 |
+
# what this module has always meant. The BACKEND is not their concern and making it their
|
| 678 |
+
# concern is how one of them gets missed;
|
| 679 |
+
# * a missed caller under a search-and-replace does not fail — it silently keeps writing to the
|
| 680 |
+
# file store while everything else writes to Postgres. That is the split-brain this whole
|
| 681 |
+
# ruling exists to end, reintroduced by the fix for it;
|
| 682 |
+
# * `store_backend.py`'s stated reason for refusing to do it this way — "no dual-read window and
|
| 683 |
+
# no way to compare the two stores' contents first" — is satisfied: `ops/seed_pg_from_hf.py`
|
| 684 |
+
# copies, then `--verify` diffs the two stores key-by-key before anything flips.
|
| 685 |
+
#
|
| 686 |
+
# ⛔ THE IMPORT IS LAZY AND MUST STAY LAZY. `core/store_pg.py` imports psycopg only inside
|
| 687 |
+
# `_pool()`, so an `hf` deployment installs no driver; importing it at module scope here would
|
| 688 |
+
# undo that and make the default backend depend on the optional dependency.
|
| 689 |
+
# =============================================================================================
|
| 690 |
+
|
| 691 |
+
def backend():
|
| 692 |
+
"""`'hf'` | `'pg'` — validated, resolved per call so a test can flip the env var.
|
| 693 |
+
|
| 694 |
+
Mirrors `core.store_backend.name()` deliberately rather than importing it: that module
|
| 695 |
+
imports THIS one, so reaching back would be a cycle. `verify_store_pg` asserts the two agree
|
| 696 |
+
on every value, which is the guard against them drifting apart.
|
| 697 |
+
"""
|
| 698 |
+
raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
|
| 699 |
+
if raw not in ('hf', 'pg'):
|
| 700 |
+
raise RuntimeError(
|
| 701 |
+
f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store) or 'pg' "
|
| 702 |
+
f"(Postgres, needs DATABASE_URL). Refusing to guess: a typo that silently served the "
|
| 703 |
+
f"other store is how data ends up in two places.")
|
| 704 |
+
return raw
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
def handle(repo_id=None, slug=None):
|
| 708 |
+
"""THE tenant-bound store handle for the ACTIVE backend — the one seam every caller crosses.
|
| 709 |
+
|
| 710 |
+
`repo_id` addresses the HF backend (a dataset repo); `slug` addresses Postgres (a schema).
|
| 711 |
+
Both are passed by `harness.runtime.get_runtime`, which knows both facts, so flipping the
|
| 712 |
+
backend never needs a lookup table between them — and neither identifier has to be invented
|
| 713 |
+
for the backend that does not use it.
|
| 714 |
+
"""
|
| 715 |
+
if backend() == 'pg':
|
| 716 |
+
import core.store_pg as _pg # noqa: PLC0415 — lazy: see the banner above
|
| 717 |
+
return _pg.PgStore(slug or os.environ.get('AIOS_TENANT') or 'royal-imports')
|
| 718 |
+
return for_repo(repo_id)
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
#: The HF default instance. Kept as a module global (not a property) because the seed/migration
|
| 722 |
+
#: tooling and `_flush_all_at_exit` both need the FILE store by name even when pg is active.
|
| 723 |
+
_DEFAULT = for_repo(REPO)
|
| 724 |
+
|
| 725 |
+
|
| 726 |
+
def _d():
|
| 727 |
+
"""The default tenant's store on the active backend — what every module function below uses.
|
| 728 |
+
|
| 729 |
+
⚠ Resolved PER CALL, never cached. A cached default would freeze the backend at import time,
|
| 730 |
+
and import order is exactly what nobody controls: `api/main.py` imports half the platform
|
| 731 |
+
before it has read a single environment variable it did not inherit.
|
| 732 |
+
"""
|
| 733 |
+
return handle() if backend() == 'pg' else _DEFAULT
|
| 734 |
+
|
| 735 |
+
|
| 736 |
+
def available():
|
| 737 |
+
return _d().available()
|
| 738 |
+
|
| 739 |
+
|
| 740 |
+
def get(name, fresh=False):
|
| 741 |
+
return _d().get(name, fresh=fresh)
|
| 742 |
+
|
| 743 |
+
|
| 744 |
+
def _read_strict(name):
|
| 745 |
+
return _d()._read_strict(name)
|
| 746 |
+
|
| 747 |
+
|
| 748 |
+
def exists(name):
|
| 749 |
+
return _d().exists(name)
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
def upload_bytes(path_in_repo, data, message=None):
|
| 753 |
+
return _d().upload_bytes(path_in_repo, data, message=message)
|
| 754 |
+
|
| 755 |
+
|
| 756 |
+
def download_bytes(path_in_repo):
|
| 757 |
+
return _d().download_bytes(path_in_repo)
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
def delete_path(path_in_repo):
|
| 761 |
+
return _d().delete_path(path_in_repo)
|
| 762 |
+
|
| 763 |
+
|
| 764 |
+
def put(name, data):
|
| 765 |
+
return _d().put(name, data)
|
| 766 |
+
|
| 767 |
+
|
| 768 |
+
def flush(name=None, timeout=30.0):
|
| 769 |
+
return _d().flush(name=name, timeout=timeout)
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
def update(name, fn, flush='sync'):
|
| 773 |
+
return _d().update(name, fn, flush=flush)
|
| 774 |
+
|
| 775 |
+
|
| 776 |
+
def get_projection(name, drop=()):
|
| 777 |
+
"""A projected read — see `Store.get_projection`. Wave 32, W32-T01 (D-185).
|
| 778 |
+
|
| 779 |
+
⛔ AN ELEVENTH STORE OPERATION, AND IT HAD TO BECOME ONE — the method alone was NOT enough,
|
| 780 |
+
which is the opposite of what it looks like from `revision()`'s precedent. `revision` could
|
| 781 |
+
ship method-only for a few hours because its caller held a Store. This one's caller is
|
| 782 |
+
`harness.runtime.TenantRuntime`, and `TenantRuntime._store()` returns **the MODULE** whenever
|
| 783 |
+
`store_handle is None` — which is precisely tenant #0, the tenant whose 703 ms `/nav` copy this
|
| 784 |
+
exists to remove. A method-only projection would have been unreachable from the only place that
|
| 785 |
+
needed it, while every gate stayed green.
|
| 786 |
+
|
| 787 |
+
⛔ SO `verify_store_pg.IFACE` GAINS IT IN THE SAME CHANGE, and `core/store_pg.py` gains the
|
| 788 |
+
twin below it. That gate's `extra:` check would red on this function otherwise — correctly:
|
| 789 |
+
this addresses a key and reads it, so it is a store operation and both backends must answer.
|
| 790 |
+
`backend`/`handle` are exempt because they SELECT a store; this one reads one.
|
| 791 |
+
"""
|
| 792 |
+
return _d().get_projection(name, drop=drop)
|
| 793 |
+
|
| 794 |
+
|
| 795 |
+
def revision(name):
|
| 796 |
+
"""The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing.
|
| 797 |
+
|
| 798 |
+
⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS
|
| 799 |
+
HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely
|
| 800 |
+
so a capability one backend lacks cannot be reached through the shared door and die at the
|
| 801 |
+
cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change
|
| 802 |
+
and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half
|
| 803 |
+
had no owner: a method is a capability the HF store has, a module function is a promise both
|
| 804 |
+
backends keep.
|
| 805 |
+
"""
|
| 806 |
+
return _d().revision(name)
|
| 807 |
+
|
| 808 |
+
|
| 809 |
+
def _flush_all_at_exit():
|
| 810 |
+
with _INSTANCES_LOCK:
|
| 811 |
+
instances = list(_INSTANCES.values())
|
| 812 |
+
for inst in instances:
|
| 813 |
+
inst._flush_now_at_exit()
|
| 814 |
+
|
| 815 |
+
|
| 816 |
+
import atexit # noqa: E402 (registered after the class it needs)
|
| 817 |
+
atexit.register(_flush_all_at_exit)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
platform/core/table_store.py
CHANGED
|
@@ -1,618 +1,618 @@
|
|
| 1 |
-
"""The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
|
| 2 |
-
|
| 3 |
-
One durable store key holds one table OBJECT's per-user Airtable-style state:
|
| 4 |
-
|
| 5 |
-
{username: {'views': {view_id: SavedView},
|
| 6 |
-
'fields': {field_key: Field}, # notes + custom_ + measure_ strata
|
| 7 |
-
'overlays': {str(pid): {field_key: value}}}}
|
| 8 |
-
|
| 9 |
-
`make(table_key)` returns the six operations a table page's host loop needs, closed over that
|
| 10 |
-
key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
|
| 11 |
-
these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
|
| 12 |
-
pattern to a new object is a registry row + a config, not a copy of the store plumbing
|
| 13 |
-
(owner directive: the table-page factory).
|
| 14 |
-
|
| 15 |
-
A LIST (membership/formula semantics) is deliberately a different store from a VIEW
|
| 16 |
-
(presentation/query state) — see modules/customer_data.py's customer_lists key.
|
| 17 |
-
"""
|
| 18 |
-
import core.store as store
|
| 19 |
-
|
| 20 |
-
#: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
|
| 21 |
-
#: their creator live here instead of in a personal workspace, under a key that cannot collide
|
| 22 |
-
#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
|
| 23 |
-
#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
|
| 24 |
-
#: here, and a view shared is removed from its creator's workspace. Two homes would mean two
|
| 25 |
-
#: divergent copies the moment either was edited.
|
| 26 |
-
SHARED_KEY = '__shared__'
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
def _may_see(view, viewer, is_admin=False):
|
| 30 |
-
"""Visibility for ONE shared view, fail-closed.
|
| 31 |
-
|
| 32 |
-
'collaborative' = everyone who can already open the module (the caller has gated that).
|
| 33 |
-
'users' = the named users, plus the creator, plus admins — an admin who could not
|
| 34 |
-
see a view could not administer it either.
|
| 35 |
-
Anything unrecognised returns False rather than defaulting open: an unreadable permission
|
| 36 |
-
must never widen access ([[aios-permissioning]] — no fail-open defaults).
|
| 37 |
-
"""
|
| 38 |
-
if not isinstance(view, dict):
|
| 39 |
-
return False
|
| 40 |
-
if view.get('createdBy') == viewer or is_admin:
|
| 41 |
-
return True
|
| 42 |
-
perms = view.get('permissions') or {}
|
| 43 |
-
edit = perms.get('edit')
|
| 44 |
-
if edit == 'collaborative':
|
| 45 |
-
return True
|
| 46 |
-
if edit == 'users':
|
| 47 |
-
return viewer in set(perms.get('users') or ())
|
| 48 |
-
return False # 'personal', absent, or junk
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def _may_edit(view, viewer, is_admin=False):
|
| 52 |
-
"""Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
|
| 53 |
-
can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
|
| 54 |
-
separate function so they can diverge (a future read-only share) without hunting callers."""
|
| 55 |
-
return _may_see(view, viewer, is_admin)
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
def _may_administer(view, viewer, is_admin=False):
|
| 59 |
-
"""Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
|
| 60 |
-
|
| 61 |
-
Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
|
| 62 |
-
could grant themselves sole ownership of somebody else's view, or quietly widen a
|
| 63 |
-
users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
|
| 64 |
-
"""
|
| 65 |
-
if not isinstance(view, dict):
|
| 66 |
-
return False
|
| 67 |
-
return bool(is_admin) or view.get('createdBy') == viewer
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
def is_shared(view):
|
| 71 |
-
"""A view belongs in the shared bucket when its permissions reach beyond its creator."""
|
| 72 |
-
return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
def source_override_is_empty(payload):
|
| 76 |
-
"""Does this stored definition of a SOURCE (non-custom) column carry any user state?
|
| 77 |
-
|
| 78 |
-
A cleared note on an immutable source field returns to the canonical schema instead of
|
| 79 |
-
leaving a meaningless override row. Custom fields remain even with an empty note — and so
|
| 80 |
-
does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
|
| 81 |
-
override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
|
| 82 |
-
|
| 83 |
-
⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
|
| 84 |
-
stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
|
| 85 |
-
built an override whose only content was that summary, so this rule threw the whole row away
|
| 86 |
-
on write while the menu went on reading "Summary: Average" from the client's own optimistic
|
| 87 |
-
copy until the next login — a discarded WRITE wearing the face of a failed read
|
| 88 |
-
([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
|
| 89 |
-
its definitions through another door entirely, kept it.
|
| 90 |
-
|
| 91 |
-
⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a
|
| 92 |
-
discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
|
| 93 |
-
⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
|
| 94 |
-
column goes back to whatever the contract declares for it.
|
| 95 |
-
"""
|
| 96 |
-
payload = payload or {}
|
| 97 |
-
if payload.get('custom'):
|
| 98 |
-
return False
|
| 99 |
-
return (not str(payload.get('note') or '').strip()
|
| 100 |
-
and not isinstance(payload.get('measure'), dict)
|
| 101 |
-
and not isinstance(payload.get('format'), dict)
|
| 102 |
-
and not str(payload.get('agg') or '').strip())
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
|
| 106 |
-
"""Allocate one human-facing name inside a store.update transaction.
|
| 107 |
-
|
| 108 |
-
Keys/ids remain structural identity. Names compare case-insensitively after collapsing
|
| 109 |
-
whitespace, because those variants are indistinguishable in the UI. This helper belongs
|
| 110 |
-
in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
|
| 111 |
-
choose the same free name before either write lands.
|
| 112 |
-
"""
|
| 113 |
-
limit = max(1, int(max_len))
|
| 114 |
-
|
| 115 |
-
def _clean(value):
|
| 116 |
-
return ' '.join(str(value or '').split())
|
| 117 |
-
|
| 118 |
-
base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
|
| 119 |
-
taken = {_clean(value).casefold() for value in existing if _clean(value)}
|
| 120 |
-
if base.casefold() not in taken:
|
| 121 |
-
return base
|
| 122 |
-
index = 2
|
| 123 |
-
while True:
|
| 124 |
-
suffix = f' {index}'
|
| 125 |
-
stem = base[:max(0, limit - len(suffix))].rstrip()
|
| 126 |
-
candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
|
| 127 |
-
if candidate.casefold() not in taken:
|
| 128 |
-
return candidate
|
| 129 |
-
index += 1
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
class TableStore:
|
| 133 |
-
"""The six store operations for one table object's workspace, closed over its store key.
|
| 134 |
-
|
| 135 |
-
`st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
|
| 136 |
-
`update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
|
| 137 |
-
zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
|
| 138 |
-
apply the tenant prefix / repo binding — which is what makes a user table created by a
|
| 139 |
-
Nurilab admin land in Nurilab's store instead of Royal's.
|
| 140 |
-
"""
|
| 141 |
-
|
| 142 |
-
def __init__(self, table_key, st=None):
|
| 143 |
-
self.table_key = table_key
|
| 144 |
-
self._st = st if st is not None else store
|
| 145 |
-
|
| 146 |
-
@property
|
| 147 |
-
def st(self):
|
| 148 |
-
"""The bound store handle — for SIBLING registries (core/shares) that must read the
|
| 149 |
-
same tenant's buckets this workspace lives in (wave 21, C1)."""
|
| 150 |
-
return self._st
|
| 151 |
-
|
| 152 |
-
def find_view(self, view_id):
|
| 153 |
-
"""`(owner_username, view)` for a view living in ANY personal stratum, else None.
|
| 154 |
-
|
| 155 |
-
⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
|
| 156 |
-
view means locating the OWNER's record inside this topic's bucket. Personal strata
|
| 157 |
-
only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
|
| 158 |
-
view from two finders is how two copies drift."""
|
| 159 |
-
vid = str(view_id or '').strip()
|
| 160 |
-
if not vid:
|
| 161 |
-
return None
|
| 162 |
-
try:
|
| 163 |
-
data = self._st.get(self.table_key) or {}
|
| 164 |
-
except Exception:
|
| 165 |
-
return None
|
| 166 |
-
for username, ws in data.items():
|
| 167 |
-
if username == SHARED_KEY or not isinstance(ws, dict):
|
| 168 |
-
continue
|
| 169 |
-
v = (ws.get('views') or {}).get(vid)
|
| 170 |
-
if isinstance(v, dict):
|
| 171 |
-
return str(username), dict(v)
|
| 172 |
-
return None
|
| 173 |
-
|
| 174 |
-
def find_folder(self, folder_id):
|
| 175 |
-
"""`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
|
| 176 |
-
personal stratum, else None. `find_view`'s sibling, and here for the same reason.
|
| 177 |
-
|
| 178 |
-
⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
|
| 179 |
-
`folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
|
| 180 |
-
this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
|
| 181 |
-
Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
|
| 182 |
-
owns it, and WHICH views are filed in it. Folder membership lives in the owner's
|
| 183 |
-
`itemFolders` map (item id -> folder id), never on the view record, so the views are
|
| 184 |
-
found by asking that map rather than by reading a list off the folder.
|
| 185 |
-
|
| 186 |
-
Views only (`folders['views']`): the cohort surface has its own store and its own
|
| 187 |
-
sharing question, and answering both here would make one function mean two things.
|
| 188 |
-
"""
|
| 189 |
-
fid = str(folder_id or '').strip()
|
| 190 |
-
if not fid:
|
| 191 |
-
return None
|
| 192 |
-
try:
|
| 193 |
-
data = self._st.get(self.table_key) or {}
|
| 194 |
-
except Exception:
|
| 195 |
-
return None
|
| 196 |
-
for username, ws in data.items():
|
| 197 |
-
if username == SHARED_KEY or not isinstance(ws, dict):
|
| 198 |
-
continue
|
| 199 |
-
rows = (ws.get('folders') or {}).get('views') or []
|
| 200 |
-
hit = next((f for f in rows
|
| 201 |
-
if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
|
| 202 |
-
if not hit:
|
| 203 |
-
continue
|
| 204 |
-
# ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
|
| 205 |
-
# reading item ids off the top level finds the surface names instead and matches
|
| 206 |
-
# nothing, so the projection silently returns an EMPTY folder and the feature looks
|
| 207 |
-
# exactly as broken as it was before the fix. Caught by this change's own gate,
|
| 208 |
-
# which is the entire argument for writing one.
|
| 209 |
-
placed = (ws.get('itemFolders') or {}).get('views') or {}
|
| 210 |
-
views = ws.get('views') or {}
|
| 211 |
-
inside = {str(vid): dict(v) for vid, v in views.items()
|
| 212 |
-
if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
|
| 213 |
-
return str(username), dict(hit), inside
|
| 214 |
-
return None
|
| 215 |
-
|
| 216 |
-
# ---------------------------------------------------------------- read
|
| 217 |
-
def workspace(self, username, consume_corrections=True):
|
| 218 |
-
"""One user's durable workspace: always the full three-strata shape."""
|
| 219 |
-
try:
|
| 220 |
-
data = self._st.get(self.table_key) or {}
|
| 221 |
-
ws = data.get(username, {}) or {}
|
| 222 |
-
# A collision acknowledgement is protocol state, not part of a field definition.
|
| 223 |
-
# Consume it with the first fresh workspace payload after the correcting write,
|
| 224 |
-
# then splice a bounded copy into that payload only. Keeping it out of `fields`
|
| 225 |
-
# prevents an old request id surviving forever and overriding a later rename.
|
| 226 |
-
corrections = {}
|
| 227 |
-
if consume_corrections and ws.get('fieldCorrections'):
|
| 228 |
-
def _take(current):
|
| 229 |
-
current_ws = current.get(username) or {}
|
| 230 |
-
pending = current_ws.get('fieldCorrections') or {}
|
| 231 |
-
corrections.update({
|
| 232 |
-
str(key)[:80]: dict(value)
|
| 233 |
-
for key, value in pending.items()
|
| 234 |
-
if isinstance(value, dict)
|
| 235 |
-
})
|
| 236 |
-
current_ws.pop('fieldCorrections', None)
|
| 237 |
-
return current
|
| 238 |
-
|
| 239 |
-
data = self._st.update(self.table_key, _take, flush='async')
|
| 240 |
-
ws = (data or {}).get(username, {}) or {}
|
| 241 |
-
except Exception:
|
| 242 |
-
ws = {}
|
| 243 |
-
corrections = {}
|
| 244 |
-
fields = {
|
| 245 |
-
key: dict(value) if isinstance(value, dict) else value
|
| 246 |
-
for key, value in (ws.get('fields') or {}).items()
|
| 247 |
-
}
|
| 248 |
-
for key, ack in corrections.items():
|
| 249 |
-
field = fields.get(key)
|
| 250 |
-
accepted_label = str(ack.get('label') or '')[:120]
|
| 251 |
-
requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
|
| 252 |
-
correction_id = str(ack.get('labelCorrectionId') or '')[:180]
|
| 253 |
-
# A newer field write clears/replaces the pending ack in the SAME transaction.
|
| 254 |
-
# The label check is an extra belt against ever attaching a stale ack to a newer
|
| 255 |
-
# definition if a future store implementation weakens that ordering.
|
| 256 |
-
if (isinstance(field, dict) and accepted_label
|
| 257 |
-
and str(field.get('label') or '') == accepted_label
|
| 258 |
-
and requested_label and correction_id):
|
| 259 |
-
field['labelCorrectedFrom'] = requested_label
|
| 260 |
-
field['labelCorrectionId'] = correction_id
|
| 261 |
-
out = {
|
| 262 |
-
'views': dict(ws.get('views') or {}),
|
| 263 |
-
'fields': fields,
|
| 264 |
-
'overlays': dict(ws.get('overlays') or {}),
|
| 265 |
-
# wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
|
| 266 |
-
# stratum rather than a key on each item — see aios_grid.clean_folders for why
|
| 267 |
-
# (a cohort lives in another store, and filing is an organising act, not part of
|
| 268 |
-
# what a view is). Absent for every workspace saved before this wave, which is
|
| 269 |
-
# exactly "no folders yet".
|
| 270 |
-
'folders': dict(ws.get('folders') or {}),
|
| 271 |
-
'itemFolders': dict(ws.get('itemFolders') or {}),
|
| 272 |
-
}
|
| 273 |
-
# 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
|
| 274 |
-
# client's localStorage copy wins when present; this is the server's answer for a
|
| 275 |
-
# fresh profile, which used to fall all the way to the system default view.
|
| 276 |
-
if ws.get('activeViewId'):
|
| 277 |
-
out['activeViewId'] = str(ws['activeViewId'])
|
| 278 |
-
# Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
|
| 279 |
-
# stratum, absent until the user first reorders — exactly "default order".
|
| 280 |
-
if isinstance(ws.get('recordLayout'), dict):
|
| 281 |
-
out['recordLayout'] = dict(ws['recordLayout'])
|
| 282 |
-
return out
|
| 283 |
-
|
| 284 |
-
# ---------------------------------------------------------------- write
|
| 285 |
-
def _update(self, username, change):
|
| 286 |
-
def _up(data):
|
| 287 |
-
ws = data.setdefault(username, {})
|
| 288 |
-
ws.setdefault('views', {})
|
| 289 |
-
ws.setdefault('fields', {})
|
| 290 |
-
ws.setdefault('overlays', {})
|
| 291 |
-
ws.setdefault('folders', {})
|
| 292 |
-
ws.setdefault('itemFolders', {})
|
| 293 |
-
change(ws)
|
| 294 |
-
return data
|
| 295 |
-
# flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
|
| 296 |
-
# column note and typed overlay cell lands here inside the component round-trip, and
|
| 297 |
-
# the historical synchronous hub commit cost seconds per edit. The mutation applies to
|
| 298 |
-
# the in-process cache (read-your-writes for every subsequent render); the hub write
|
| 299 |
-
# coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
|
| 300 |
-
return self._st.update(self.table_key, _up, flush='async')
|
| 301 |
-
|
| 302 |
-
def rename_choice_values(self, username, change):
|
| 303 |
-
"""Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
|
| 304 |
-
|
| 305 |
-
⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
|
| 306 |
-
public "do anything to the workspace" method is an invitation to put write logic in
|
| 307 |
-
callers instead of here, and every OTHER method on this class exists precisely because
|
| 308 |
-
that logic belongs in one place. Renaming a choice is the one operation that must touch
|
| 309 |
-
three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
|
| 310 |
-
filter or colour by the old value — inside a SINGLE transaction, because a rename that
|
| 311 |
-
updated the cells and not the filters would leave a saved view matching nothing.
|
| 312 |
-
|
| 313 |
-
`change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
|
| 314 |
-
"""
|
| 315 |
-
return self._update(username, change)
|
| 316 |
-
|
| 317 |
-
def save_active_view(self, username, view_id):
|
| 318 |
-
"""Remember which view this user last opened (owner item 3, 2026-07-31).
|
| 319 |
-
|
| 320 |
-
Presentation state, not authorisation: the READ side re-validates the id against what
|
| 321 |
-
the caller may actually see, so a stale or foreign id degrades to the default view
|
| 322 |
-
rather than granting anything. Stored per user like every other stratum.
|
| 323 |
-
"""
|
| 324 |
-
vid = str(view_id or '').strip()[:120]
|
| 325 |
-
if not vid or username == SHARED_KEY:
|
| 326 |
-
return
|
| 327 |
-
|
| 328 |
-
def _set(ws):
|
| 329 |
-
ws['activeViewId'] = vid
|
| 330 |
-
self._update(username, _set)
|
| 331 |
-
|
| 332 |
-
def save_record_layout(self, username, order):
|
| 333 |
-
"""The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
|
| 334 |
-
|
| 335 |
-
Presentation state for ONE surface — the record modal. Deliberately not view config:
|
| 336 |
-
the owner's ask is per-user, not per-view, and it must never reorder grid columns.
|
| 337 |
-
The event handler validated keys against the live field set; the wire re-validates at
|
| 338 |
-
serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
|
| 339 |
-
An empty order clears the stratum back to "default order".
|
| 340 |
-
"""
|
| 341 |
-
if username == SHARED_KEY:
|
| 342 |
-
return
|
| 343 |
-
clean, seen = [], set()
|
| 344 |
-
for key in (order or [])[:200]:
|
| 345 |
-
key = str(key or '').strip()[:80]
|
| 346 |
-
if key and key not in seen:
|
| 347 |
-
seen.add(key)
|
| 348 |
-
clean.append(key)
|
| 349 |
-
|
| 350 |
-
def _set(ws):
|
| 351 |
-
if clean:
|
| 352 |
-
ws['recordLayout'] = {'order': clean}
|
| 353 |
-
else:
|
| 354 |
-
ws.pop('recordLayout', None)
|
| 355 |
-
|
| 356 |
-
self._update(username, _set)
|
| 357 |
-
|
| 358 |
-
def save_folders(self, username, folders, item_folders):
|
| 359 |
-
"""Replace the folder stratum wholesale (wave-8 I11).
|
| 360 |
-
|
| 361 |
-
Wholesale rather than per-folder because the caller has ALREADY validated the complete
|
| 362 |
-
picture through aios_grid.clean_folders / clean_item_folders, and those two are
|
| 363 |
-
interdependent: a placement is only legal while its folder exists, so committing them
|
| 364 |
-
separately would leave a window where a reader sees an item filed into a folder that is
|
| 365 |
-
not there yet. One write, one consistent state.
|
| 366 |
-
"""
|
| 367 |
-
def _set(ws):
|
| 368 |
-
ws['folders'] = dict(folders or {})
|
| 369 |
-
ws['itemFolders'] = dict(item_folders or {})
|
| 370 |
-
self._update(username, _set)
|
| 371 |
-
|
| 372 |
-
def save_view_order(self, username, order):
|
| 373 |
-
"""⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
|
| 374 |
-
|
| 375 |
-
Wholesale, like `save_folders` above and for the same reason: the client sends the full
|
| 376 |
-
list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
|
| 377 |
-
view went.
|
| 378 |
-
|
| 379 |
-
⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
|
| 380 |
-
`aios_grid`'s own folder note argues it out for placements and every word applies: an
|
| 381 |
-
arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
|
| 382 |
-
of the view config means duplicating, sharing or exporting a view does not drag one
|
| 383 |
-
person's rail position along with it. It also means a SHARED view can sit in a different
|
| 384 |
-
place for each person who can see it, which is the only coherent answer once two people
|
| 385 |
-
share one view.
|
| 386 |
-
|
| 387 |
-
An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
|
| 388 |
-
"""
|
| 389 |
-
def _set(ws):
|
| 390 |
-
clean = []
|
| 391 |
-
seen = set()
|
| 392 |
-
for vid in (order or []):
|
| 393 |
-
vid = str(vid).strip()[:120]
|
| 394 |
-
if vid and vid not in seen:
|
| 395 |
-
seen.add(vid)
|
| 396 |
-
clean.append(vid)
|
| 397 |
-
if clean:
|
| 398 |
-
ws['viewOrder'] = clean
|
| 399 |
-
else:
|
| 400 |
-
ws.pop('viewOrder', None)
|
| 401 |
-
self._update(username, _set)
|
| 402 |
-
|
| 403 |
-
def shared_views(self, viewer, is_admin=False):
|
| 404 |
-
"""Every SHARED view this viewer may see, by id (wave-9 I17).
|
| 405 |
-
|
| 406 |
-
Read-only and independent of the viewer's own workspace: the caller merges. Returns
|
| 407 |
-
only what `_may_see` allows, so a caller cannot accidentally render somebody else's
|
| 408 |
-
personal view by forgetting to filter.
|
| 409 |
-
"""
|
| 410 |
-
try:
|
| 411 |
-
bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
|
| 412 |
-
except Exception:
|
| 413 |
-
return {}
|
| 414 |
-
return {vid: dict(v) for vid, v in bucket.items()
|
| 415 |
-
if _may_see(v, viewer, is_admin)}
|
| 416 |
-
|
| 417 |
-
def shared_view(self, view_id):
|
| 418 |
-
"""One shared view RAW — no visibility filter. For authorisation decisions only: a
|
| 419 |
-
caller must know a view exists and who owns it before it can decide whether the actor
|
| 420 |
-
may touch it. Never hand the result to a renderer without checking `_may_see`."""
|
| 421 |
-
try:
|
| 422 |
-
return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
|
| 423 |
-
).get('views', {}).get(str(view_id))
|
| 424 |
-
except Exception:
|
| 425 |
-
return None
|
| 426 |
-
|
| 427 |
-
def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
|
| 428 |
-
"""Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
|
| 429 |
-
|
| 430 |
-
`shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
|
| 431 |
-
it lands in, the view is REMOVED from the other, so a view can never exist as two
|
| 432 |
-
copies that diverge on the next edit.
|
| 433 |
-
|
| 434 |
-
⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
|
| 435 |
-
moves data and does not know who is asking. `_cl_handle_one` is the wall.
|
| 436 |
-
"""
|
| 437 |
-
view_id = str((view or {}).get('id') or '').strip()
|
| 438 |
-
if not view_id:
|
| 439 |
-
raise ValueError('view id is required')
|
| 440 |
-
if username == SHARED_KEY:
|
| 441 |
-
raise ValueError('reserved username')
|
| 442 |
-
to_shared = is_shared(view) if shared is None else bool(shared)
|
| 443 |
-
requested = dict(view)
|
| 444 |
-
accepted = {}
|
| 445 |
-
|
| 446 |
-
def _up(data):
|
| 447 |
-
# View names are tenant-global: every personal workspace plus the shared bucket.
|
| 448 |
-
# This deliberately includes views the actor cannot see. The only disclosed fact
|
| 449 |
-
# is that a display name is already taken, while the categorical "no duplicate
|
| 450 |
-
# view names" contract remains true when a personal view is later shared.
|
| 451 |
-
names = list(reserved_names or ())
|
| 452 |
-
for workspace in data.values():
|
| 453 |
-
if not isinstance(workspace, dict):
|
| 454 |
-
continue
|
| 455 |
-
names.extend(
|
| 456 |
-
value.get('name')
|
| 457 |
-
for candidate_id, value in (workspace.get('views') or {}).items()
|
| 458 |
-
if candidate_id != view_id and isinstance(value, dict)
|
| 459 |
-
)
|
| 460 |
-
payload = dict(requested)
|
| 461 |
-
payload['name'] = _unique_name(payload.get('name'), names)
|
| 462 |
-
accepted.clear()
|
| 463 |
-
accepted.update(payload)
|
| 464 |
-
if to_shared:
|
| 465 |
-
bucket = data.setdefault(SHARED_KEY, {})
|
| 466 |
-
bucket.setdefault('views', {})[view_id] = payload
|
| 467 |
-
# it may have lived in the creator's workspace before being shared
|
| 468 |
-
owner = data.get(payload.get('createdBy') or username) or {}
|
| 469 |
-
(owner.get('views') or {}).pop(view_id, None)
|
| 470 |
-
else:
|
| 471 |
-
ws = data.setdefault(username, {})
|
| 472 |
-
ws.setdefault('views', {})[view_id] = payload
|
| 473 |
-
(data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
|
| 474 |
-
return data
|
| 475 |
-
|
| 476 |
-
self._st.update(self.table_key, _up, flush='async')
|
| 477 |
-
return dict(accepted)
|
| 478 |
-
|
| 479 |
-
def delete_view(self, username, view_id):
|
| 480 |
-
"""Delete a custom/list view override. The system all-rows view is guarded by caller.
|
| 481 |
-
|
| 482 |
-
Removes from BOTH homes: the caller has already authorised the delete, and leaving a
|
| 483 |
-
stale copy in the other bucket would resurrect the view on the next read.
|
| 484 |
-
"""
|
| 485 |
-
vid = str(view_id)
|
| 486 |
-
|
| 487 |
-
def _up(data):
|
| 488 |
-
(data.get(username, {}).get('views') or {}).pop(vid, None)
|
| 489 |
-
(data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
|
| 490 |
-
return data
|
| 491 |
-
|
| 492 |
-
self._st.update(self.table_key, _up, flush='async')
|
| 493 |
-
|
| 494 |
-
def save_field(self, username, field, reserved_names=(), correction_id=None):
|
| 495 |
-
"""Persist a column note or a user-created (custom_/measure_) field definition."""
|
| 496 |
-
key = str((field or {}).get('key') or '').strip()
|
| 497 |
-
if not key:
|
| 498 |
-
raise ValueError('field key is required')
|
| 499 |
-
requested = dict(field)
|
| 500 |
-
accepted = {}
|
| 501 |
-
|
| 502 |
-
def _save(ws):
|
| 503 |
-
names = list(reserved_names or ())
|
| 504 |
-
names.extend(
|
| 505 |
-
value.get('label')
|
| 506 |
-
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 507 |
-
if candidate_key != key and isinstance(value, dict)
|
| 508 |
-
)
|
| 509 |
-
payload = dict(requested)
|
| 510 |
-
payload.pop('labelCorrectedFrom', None)
|
| 511 |
-
payload.pop('labelCorrectionId', None)
|
| 512 |
-
requested_label = ' '.join(
|
| 513 |
-
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 514 |
-
payload['label'] = _unique_name(requested_label, names)
|
| 515 |
-
corrections = ws.setdefault('fieldCorrections', {})
|
| 516 |
-
corrections.pop(key, None)
|
| 517 |
-
if payload['label'] != requested_label and correction_id:
|
| 518 |
-
corrections[key] = {
|
| 519 |
-
'label': payload['label'],
|
| 520 |
-
'labelCorrectedFrom': requested_label,
|
| 521 |
-
'labelCorrectionId': str(correction_id)[:180],
|
| 522 |
-
}
|
| 523 |
-
if not corrections:
|
| 524 |
-
ws.pop('fieldCorrections', None)
|
| 525 |
-
accepted.clear()
|
| 526 |
-
accepted.update(payload)
|
| 527 |
-
if source_override_is_empty(payload):
|
| 528 |
-
ws['fields'].pop(key, None)
|
| 529 |
-
else:
|
| 530 |
-
ws['fields'][key] = payload
|
| 531 |
-
|
| 532 |
-
self._update(username, _save)
|
| 533 |
-
return dict(accepted)
|
| 534 |
-
|
| 535 |
-
def duplicate_field(self, username, source_key, new_key, field,
|
| 536 |
-
reserved_names=(), correction_id=None):
|
| 537 |
-
"""Clone a user-created field in ONE store transaction (wave-5 item 1): the new
|
| 538 |
-
definition plus — for `custom_` overlay sources only — every stored cell value under
|
| 539 |
-
the source key. One transaction, because a def without its values (or values without a
|
| 540 |
-
def) is exactly the orphan state delete_field exists to prevent, in reverse.
|
| 541 |
-
The caller validated both keys (same created stratum) and stamped the clone's
|
| 542 |
-
createdBy; this layer only moves data."""
|
| 543 |
-
source_key = str(source_key or '').strip()
|
| 544 |
-
new_key = str(new_key or '').strip()
|
| 545 |
-
if not source_key or not new_key or source_key == new_key:
|
| 546 |
-
raise ValueError('duplicate_field needs two distinct keys')
|
| 547 |
-
requested = dict(field)
|
| 548 |
-
accepted = {}
|
| 549 |
-
|
| 550 |
-
def _dup(ws):
|
| 551 |
-
names = list(reserved_names or ())
|
| 552 |
-
names.extend(
|
| 553 |
-
value.get('label')
|
| 554 |
-
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 555 |
-
if candidate_key != new_key and isinstance(value, dict)
|
| 556 |
-
)
|
| 557 |
-
payload = dict(requested)
|
| 558 |
-
payload.pop('labelCorrectedFrom', None)
|
| 559 |
-
payload.pop('labelCorrectionId', None)
|
| 560 |
-
requested_label = ' '.join(
|
| 561 |
-
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 562 |
-
payload['label'] = _unique_name(requested_label, names)
|
| 563 |
-
corrections = ws.setdefault('fieldCorrections', {})
|
| 564 |
-
corrections.pop(new_key, None)
|
| 565 |
-
if payload['label'] != requested_label and correction_id:
|
| 566 |
-
corrections[new_key] = {
|
| 567 |
-
'label': payload['label'],
|
| 568 |
-
'labelCorrectedFrom': requested_label,
|
| 569 |
-
'labelCorrectionId': str(correction_id)[:180],
|
| 570 |
-
}
|
| 571 |
-
if not corrections:
|
| 572 |
-
ws.pop('fieldCorrections', None)
|
| 573 |
-
accepted.clear()
|
| 574 |
-
accepted.update(payload)
|
| 575 |
-
ws['fields'][new_key] = payload
|
| 576 |
-
if source_key.startswith('custom_'):
|
| 577 |
-
for row in ws['overlays'].values():
|
| 578 |
-
if isinstance(row, dict) and source_key in row:
|
| 579 |
-
row[new_key] = row[source_key]
|
| 580 |
-
|
| 581 |
-
self._update(username, _dup)
|
| 582 |
-
return dict(accepted)
|
| 583 |
-
|
| 584 |
-
def delete_field(self, username, key):
|
| 585 |
-
"""Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
|
| 586 |
-
|
| 587 |
-
Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
|
| 588 |
-
columns — the caller enforces the prefix). The stored overlay VALUES for the key are
|
| 589 |
-
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 590 |
-
silently resurface if the key were ever reused. Views referencing the key self-heal on
|
| 591 |
-
their next autosave (an unknown colId is dropped) — the rule every stale key rides.
|
| 592 |
-
"""
|
| 593 |
-
key = str(key or '').strip()
|
| 594 |
-
if not key:
|
| 595 |
-
return
|
| 596 |
-
|
| 597 |
-
def _drop(ws):
|
| 598 |
-
ws['fields'].pop(key, None)
|
| 599 |
-
for row in ws['overlays'].values():
|
| 600 |
-
if isinstance(row, dict):
|
| 601 |
-
row.pop(key, None)
|
| 602 |
-
|
| 603 |
-
self._update(username, _drop)
|
| 604 |
-
|
| 605 |
-
def patch_overlay(self, username, pid, updates):
|
| 606 |
-
"""Patch only the external editable stratum; never writes to the source system."""
|
| 607 |
-
clean = dict(updates or {})
|
| 608 |
-
if not clean:
|
| 609 |
-
return
|
| 610 |
-
|
| 611 |
-
def _patch(ws):
|
| 612 |
-
ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
|
| 613 |
-
|
| 614 |
-
self._update(username, _patch)
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
def make(table_key, st=None):
|
| 618 |
-
return TableStore(table_key, st=st)
|
|
|
|
| 1 |
+
"""The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
|
| 2 |
+
|
| 3 |
+
One durable store key holds one table OBJECT's per-user Airtable-style state:
|
| 4 |
+
|
| 5 |
+
{username: {'views': {view_id: SavedView},
|
| 6 |
+
'fields': {field_key: Field}, # notes + custom_ + measure_ strata
|
| 7 |
+
'overlays': {str(pid): {field_key: value}}}}
|
| 8 |
+
|
| 9 |
+
`make(table_key)` returns the six operations a table page's host loop needs, closed over that
|
| 10 |
+
key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
|
| 11 |
+
these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
|
| 12 |
+
pattern to a new object is a registry row + a config, not a copy of the store plumbing
|
| 13 |
+
(owner directive: the table-page factory).
|
| 14 |
+
|
| 15 |
+
A LIST (membership/formula semantics) is deliberately a different store from a VIEW
|
| 16 |
+
(presentation/query state) — see modules/customer_data.py's customer_lists key.
|
| 17 |
+
"""
|
| 18 |
+
import core.store as store
|
| 19 |
+
|
| 20 |
+
#: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
|
| 21 |
+
#: their creator live here instead of in a personal workspace, under a key that cannot collide
|
| 22 |
+
#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
|
| 23 |
+
#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
|
| 24 |
+
#: here, and a view shared is removed from its creator's workspace. Two homes would mean two
|
| 25 |
+
#: divergent copies the moment either was edited.
|
| 26 |
+
SHARED_KEY = '__shared__'
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _may_see(view, viewer, is_admin=False):
|
| 30 |
+
"""Visibility for ONE shared view, fail-closed.
|
| 31 |
+
|
| 32 |
+
'collaborative' = everyone who can already open the module (the caller has gated that).
|
| 33 |
+
'users' = the named users, plus the creator, plus admins — an admin who could not
|
| 34 |
+
see a view could not administer it either.
|
| 35 |
+
Anything unrecognised returns False rather than defaulting open: an unreadable permission
|
| 36 |
+
must never widen access ([[aios-permissioning]] — no fail-open defaults).
|
| 37 |
+
"""
|
| 38 |
+
if not isinstance(view, dict):
|
| 39 |
+
return False
|
| 40 |
+
if view.get('createdBy') == viewer or is_admin:
|
| 41 |
+
return True
|
| 42 |
+
perms = view.get('permissions') or {}
|
| 43 |
+
edit = perms.get('edit')
|
| 44 |
+
if edit == 'collaborative':
|
| 45 |
+
return True
|
| 46 |
+
if edit == 'users':
|
| 47 |
+
return viewer in set(perms.get('users') or ())
|
| 48 |
+
return False # 'personal', absent, or junk
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _may_edit(view, viewer, is_admin=False):
|
| 52 |
+
"""Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
|
| 53 |
+
can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
|
| 54 |
+
separate function so they can diverge (a future read-only share) without hunting callers."""
|
| 55 |
+
return _may_see(view, viewer, is_admin)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _may_administer(view, viewer, is_admin=False):
|
| 59 |
+
"""Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
|
| 60 |
+
|
| 61 |
+
Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
|
| 62 |
+
could grant themselves sole ownership of somebody else's view, or quietly widen a
|
| 63 |
+
users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
|
| 64 |
+
"""
|
| 65 |
+
if not isinstance(view, dict):
|
| 66 |
+
return False
|
| 67 |
+
return bool(is_admin) or view.get('createdBy') == viewer
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def is_shared(view):
|
| 71 |
+
"""A view belongs in the shared bucket when its permissions reach beyond its creator."""
|
| 72 |
+
return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def source_override_is_empty(payload):
|
| 76 |
+
"""Does this stored definition of a SOURCE (non-custom) column carry any user state?
|
| 77 |
+
|
| 78 |
+
A cleared note on an immutable source field returns to the canonical schema instead of
|
| 79 |
+
leaving a meaningless override row. Custom fields remain even with an empty note — and so
|
| 80 |
+
does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
|
| 81 |
+
override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
|
| 82 |
+
|
| 83 |
+
⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
|
| 84 |
+
stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
|
| 85 |
+
built an override whose only content was that summary, so this rule threw the whole row away
|
| 86 |
+
on write while the menu went on reading "Summary: Average" from the client's own optimistic
|
| 87 |
+
copy until the next login — a discarded WRITE wearing the face of a failed read
|
| 88 |
+
([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
|
| 89 |
+
its definitions through another door entirely, kept it.
|
| 90 |
+
|
| 91 |
+
⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a
|
| 92 |
+
discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
|
| 93 |
+
⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
|
| 94 |
+
column goes back to whatever the contract declares for it.
|
| 95 |
+
"""
|
| 96 |
+
payload = payload or {}
|
| 97 |
+
if payload.get('custom'):
|
| 98 |
+
return False
|
| 99 |
+
return (not str(payload.get('note') or '').strip()
|
| 100 |
+
and not isinstance(payload.get('measure'), dict)
|
| 101 |
+
and not isinstance(payload.get('format'), dict)
|
| 102 |
+
and not str(payload.get('agg') or '').strip())
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
|
| 106 |
+
"""Allocate one human-facing name inside a store.update transaction.
|
| 107 |
+
|
| 108 |
+
Keys/ids remain structural identity. Names compare case-insensitively after collapsing
|
| 109 |
+
whitespace, because those variants are indistinguishable in the UI. This helper belongs
|
| 110 |
+
in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
|
| 111 |
+
choose the same free name before either write lands.
|
| 112 |
+
"""
|
| 113 |
+
limit = max(1, int(max_len))
|
| 114 |
+
|
| 115 |
+
def _clean(value):
|
| 116 |
+
return ' '.join(str(value or '').split())
|
| 117 |
+
|
| 118 |
+
base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
|
| 119 |
+
taken = {_clean(value).casefold() for value in existing if _clean(value)}
|
| 120 |
+
if base.casefold() not in taken:
|
| 121 |
+
return base
|
| 122 |
+
index = 2
|
| 123 |
+
while True:
|
| 124 |
+
suffix = f' {index}'
|
| 125 |
+
stem = base[:max(0, limit - len(suffix))].rstrip()
|
| 126 |
+
candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
|
| 127 |
+
if candidate.casefold() not in taken:
|
| 128 |
+
return candidate
|
| 129 |
+
index += 1
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class TableStore:
|
| 133 |
+
"""The six store operations for one table object's workspace, closed over its store key.
|
| 134 |
+
|
| 135 |
+
`st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
|
| 136 |
+
`update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
|
| 137 |
+
zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
|
| 138 |
+
apply the tenant prefix / repo binding — which is what makes a user table created by a
|
| 139 |
+
Nurilab admin land in Nurilab's store instead of Royal's.
|
| 140 |
+
"""
|
| 141 |
+
|
| 142 |
+
def __init__(self, table_key, st=None):
|
| 143 |
+
self.table_key = table_key
|
| 144 |
+
self._st = st if st is not None else store
|
| 145 |
+
|
| 146 |
+
@property
|
| 147 |
+
def st(self):
|
| 148 |
+
"""The bound store handle — for SIBLING registries (core/shares) that must read the
|
| 149 |
+
same tenant's buckets this workspace lives in (wave 21, C1)."""
|
| 150 |
+
return self._st
|
| 151 |
+
|
| 152 |
+
def find_view(self, view_id):
|
| 153 |
+
"""`(owner_username, view)` for a view living in ANY personal stratum, else None.
|
| 154 |
+
|
| 155 |
+
⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
|
| 156 |
+
view means locating the OWNER's record inside this topic's bucket. Personal strata
|
| 157 |
+
only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
|
| 158 |
+
view from two finders is how two copies drift."""
|
| 159 |
+
vid = str(view_id or '').strip()
|
| 160 |
+
if not vid:
|
| 161 |
+
return None
|
| 162 |
+
try:
|
| 163 |
+
data = self._st.get(self.table_key) or {}
|
| 164 |
+
except Exception:
|
| 165 |
+
return None
|
| 166 |
+
for username, ws in data.items():
|
| 167 |
+
if username == SHARED_KEY or not isinstance(ws, dict):
|
| 168 |
+
continue
|
| 169 |
+
v = (ws.get('views') or {}).get(vid)
|
| 170 |
+
if isinstance(v, dict):
|
| 171 |
+
return str(username), dict(v)
|
| 172 |
+
return None
|
| 173 |
+
|
| 174 |
+
def find_folder(self, folder_id):
|
| 175 |
+
"""`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
|
| 176 |
+
personal stratum, else None. `find_view`'s sibling, and here for the same reason.
|
| 177 |
+
|
| 178 |
+
⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
|
| 179 |
+
`folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
|
| 180 |
+
this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
|
| 181 |
+
Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
|
| 182 |
+
owns it, and WHICH views are filed in it. Folder membership lives in the owner's
|
| 183 |
+
`itemFolders` map (item id -> folder id), never on the view record, so the views are
|
| 184 |
+
found by asking that map rather than by reading a list off the folder.
|
| 185 |
+
|
| 186 |
+
Views only (`folders['views']`): the cohort surface has its own store and its own
|
| 187 |
+
sharing question, and answering both here would make one function mean two things.
|
| 188 |
+
"""
|
| 189 |
+
fid = str(folder_id or '').strip()
|
| 190 |
+
if not fid:
|
| 191 |
+
return None
|
| 192 |
+
try:
|
| 193 |
+
data = self._st.get(self.table_key) or {}
|
| 194 |
+
except Exception:
|
| 195 |
+
return None
|
| 196 |
+
for username, ws in data.items():
|
| 197 |
+
if username == SHARED_KEY or not isinstance(ws, dict):
|
| 198 |
+
continue
|
| 199 |
+
rows = (ws.get('folders') or {}).get('views') or []
|
| 200 |
+
hit = next((f for f in rows
|
| 201 |
+
if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
|
| 202 |
+
if not hit:
|
| 203 |
+
continue
|
| 204 |
+
# ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
|
| 205 |
+
# reading item ids off the top level finds the surface names instead and matches
|
| 206 |
+
# nothing, so the projection silently returns an EMPTY folder and the feature looks
|
| 207 |
+
# exactly as broken as it was before the fix. Caught by this change's own gate,
|
| 208 |
+
# which is the entire argument for writing one.
|
| 209 |
+
placed = (ws.get('itemFolders') or {}).get('views') or {}
|
| 210 |
+
views = ws.get('views') or {}
|
| 211 |
+
inside = {str(vid): dict(v) for vid, v in views.items()
|
| 212 |
+
if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
|
| 213 |
+
return str(username), dict(hit), inside
|
| 214 |
+
return None
|
| 215 |
+
|
| 216 |
+
# ---------------------------------------------------------------- read
|
| 217 |
+
def workspace(self, username, consume_corrections=True):
|
| 218 |
+
"""One user's durable workspace: always the full three-strata shape."""
|
| 219 |
+
try:
|
| 220 |
+
data = self._st.get(self.table_key) or {}
|
| 221 |
+
ws = data.get(username, {}) or {}
|
| 222 |
+
# A collision acknowledgement is protocol state, not part of a field definition.
|
| 223 |
+
# Consume it with the first fresh workspace payload after the correcting write,
|
| 224 |
+
# then splice a bounded copy into that payload only. Keeping it out of `fields`
|
| 225 |
+
# prevents an old request id surviving forever and overriding a later rename.
|
| 226 |
+
corrections = {}
|
| 227 |
+
if consume_corrections and ws.get('fieldCorrections'):
|
| 228 |
+
def _take(current):
|
| 229 |
+
current_ws = current.get(username) or {}
|
| 230 |
+
pending = current_ws.get('fieldCorrections') or {}
|
| 231 |
+
corrections.update({
|
| 232 |
+
str(key)[:80]: dict(value)
|
| 233 |
+
for key, value in pending.items()
|
| 234 |
+
if isinstance(value, dict)
|
| 235 |
+
})
|
| 236 |
+
current_ws.pop('fieldCorrections', None)
|
| 237 |
+
return current
|
| 238 |
+
|
| 239 |
+
data = self._st.update(self.table_key, _take, flush='async')
|
| 240 |
+
ws = (data or {}).get(username, {}) or {}
|
| 241 |
+
except Exception:
|
| 242 |
+
ws = {}
|
| 243 |
+
corrections = {}
|
| 244 |
+
fields = {
|
| 245 |
+
key: dict(value) if isinstance(value, dict) else value
|
| 246 |
+
for key, value in (ws.get('fields') or {}).items()
|
| 247 |
+
}
|
| 248 |
+
for key, ack in corrections.items():
|
| 249 |
+
field = fields.get(key)
|
| 250 |
+
accepted_label = str(ack.get('label') or '')[:120]
|
| 251 |
+
requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
|
| 252 |
+
correction_id = str(ack.get('labelCorrectionId') or '')[:180]
|
| 253 |
+
# A newer field write clears/replaces the pending ack in the SAME transaction.
|
| 254 |
+
# The label check is an extra belt against ever attaching a stale ack to a newer
|
| 255 |
+
# definition if a future store implementation weakens that ordering.
|
| 256 |
+
if (isinstance(field, dict) and accepted_label
|
| 257 |
+
and str(field.get('label') or '') == accepted_label
|
| 258 |
+
and requested_label and correction_id):
|
| 259 |
+
field['labelCorrectedFrom'] = requested_label
|
| 260 |
+
field['labelCorrectionId'] = correction_id
|
| 261 |
+
out = {
|
| 262 |
+
'views': dict(ws.get('views') or {}),
|
| 263 |
+
'fields': fields,
|
| 264 |
+
'overlays': dict(ws.get('overlays') or {}),
|
| 265 |
+
# wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
|
| 266 |
+
# stratum rather than a key on each item — see aios_grid.clean_folders for why
|
| 267 |
+
# (a cohort lives in another store, and filing is an organising act, not part of
|
| 268 |
+
# what a view is). Absent for every workspace saved before this wave, which is
|
| 269 |
+
# exactly "no folders yet".
|
| 270 |
+
'folders': dict(ws.get('folders') or {}),
|
| 271 |
+
'itemFolders': dict(ws.get('itemFolders') or {}),
|
| 272 |
+
}
|
| 273 |
+
# 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
|
| 274 |
+
# client's localStorage copy wins when present; this is the server's answer for a
|
| 275 |
+
# fresh profile, which used to fall all the way to the system default view.
|
| 276 |
+
if ws.get('activeViewId'):
|
| 277 |
+
out['activeViewId'] = str(ws['activeViewId'])
|
| 278 |
+
# Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
|
| 279 |
+
# stratum, absent until the user first reorders — exactly "default order".
|
| 280 |
+
if isinstance(ws.get('recordLayout'), dict):
|
| 281 |
+
out['recordLayout'] = dict(ws['recordLayout'])
|
| 282 |
+
return out
|
| 283 |
+
|
| 284 |
+
# ---------------------------------------------------------------- write
|
| 285 |
+
def _update(self, username, change):
|
| 286 |
+
def _up(data):
|
| 287 |
+
ws = data.setdefault(username, {})
|
| 288 |
+
ws.setdefault('views', {})
|
| 289 |
+
ws.setdefault('fields', {})
|
| 290 |
+
ws.setdefault('overlays', {})
|
| 291 |
+
ws.setdefault('folders', {})
|
| 292 |
+
ws.setdefault('itemFolders', {})
|
| 293 |
+
change(ws)
|
| 294 |
+
return data
|
| 295 |
+
# flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
|
| 296 |
+
# column note and typed overlay cell lands here inside the component round-trip, and
|
| 297 |
+
# the historical synchronous hub commit cost seconds per edit. The mutation applies to
|
| 298 |
+
# the in-process cache (read-your-writes for every subsequent render); the hub write
|
| 299 |
+
# coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
|
| 300 |
+
return self._st.update(self.table_key, _up, flush='async')
|
| 301 |
+
|
| 302 |
+
def rename_choice_values(self, username, change):
|
| 303 |
+
"""Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
|
| 304 |
+
|
| 305 |
+
⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
|
| 306 |
+
public "do anything to the workspace" method is an invitation to put write logic in
|
| 307 |
+
callers instead of here, and every OTHER method on this class exists precisely because
|
| 308 |
+
that logic belongs in one place. Renaming a choice is the one operation that must touch
|
| 309 |
+
three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
|
| 310 |
+
filter or colour by the old value — inside a SINGLE transaction, because a rename that
|
| 311 |
+
updated the cells and not the filters would leave a saved view matching nothing.
|
| 312 |
+
|
| 313 |
+
`change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
|
| 314 |
+
"""
|
| 315 |
+
return self._update(username, change)
|
| 316 |
+
|
| 317 |
+
def save_active_view(self, username, view_id):
|
| 318 |
+
"""Remember which view this user last opened (owner item 3, 2026-07-31).
|
| 319 |
+
|
| 320 |
+
Presentation state, not authorisation: the READ side re-validates the id against what
|
| 321 |
+
the caller may actually see, so a stale or foreign id degrades to the default view
|
| 322 |
+
rather than granting anything. Stored per user like every other stratum.
|
| 323 |
+
"""
|
| 324 |
+
vid = str(view_id or '').strip()[:120]
|
| 325 |
+
if not vid or username == SHARED_KEY:
|
| 326 |
+
return
|
| 327 |
+
|
| 328 |
+
def _set(ws):
|
| 329 |
+
ws['activeViewId'] = vid
|
| 330 |
+
self._update(username, _set)
|
| 331 |
+
|
| 332 |
+
def save_record_layout(self, username, order):
|
| 333 |
+
"""The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
|
| 334 |
+
|
| 335 |
+
Presentation state for ONE surface — the record modal. Deliberately not view config:
|
| 336 |
+
the owner's ask is per-user, not per-view, and it must never reorder grid columns.
|
| 337 |
+
The event handler validated keys against the live field set; the wire re-validates at
|
| 338 |
+
serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
|
| 339 |
+
An empty order clears the stratum back to "default order".
|
| 340 |
+
"""
|
| 341 |
+
if username == SHARED_KEY:
|
| 342 |
+
return
|
| 343 |
+
clean, seen = [], set()
|
| 344 |
+
for key in (order or [])[:200]:
|
| 345 |
+
key = str(key or '').strip()[:80]
|
| 346 |
+
if key and key not in seen:
|
| 347 |
+
seen.add(key)
|
| 348 |
+
clean.append(key)
|
| 349 |
+
|
| 350 |
+
def _set(ws):
|
| 351 |
+
if clean:
|
| 352 |
+
ws['recordLayout'] = {'order': clean}
|
| 353 |
+
else:
|
| 354 |
+
ws.pop('recordLayout', None)
|
| 355 |
+
|
| 356 |
+
self._update(username, _set)
|
| 357 |
+
|
| 358 |
+
def save_folders(self, username, folders, item_folders):
|
| 359 |
+
"""Replace the folder stratum wholesale (wave-8 I11).
|
| 360 |
+
|
| 361 |
+
Wholesale rather than per-folder because the caller has ALREADY validated the complete
|
| 362 |
+
picture through aios_grid.clean_folders / clean_item_folders, and those two are
|
| 363 |
+
interdependent: a placement is only legal while its folder exists, so committing them
|
| 364 |
+
separately would leave a window where a reader sees an item filed into a folder that is
|
| 365 |
+
not there yet. One write, one consistent state.
|
| 366 |
+
"""
|
| 367 |
+
def _set(ws):
|
| 368 |
+
ws['folders'] = dict(folders or {})
|
| 369 |
+
ws['itemFolders'] = dict(item_folders or {})
|
| 370 |
+
self._update(username, _set)
|
| 371 |
+
|
| 372 |
+
def save_view_order(self, username, order):
|
| 373 |
+
"""⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
|
| 374 |
+
|
| 375 |
+
Wholesale, like `save_folders` above and for the same reason: the client sends the full
|
| 376 |
+
list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
|
| 377 |
+
view went.
|
| 378 |
+
|
| 379 |
+
⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
|
| 380 |
+
`aios_grid`'s own folder note argues it out for placements and every word applies: an
|
| 381 |
+
arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
|
| 382 |
+
of the view config means duplicating, sharing or exporting a view does not drag one
|
| 383 |
+
person's rail position along with it. It also means a SHARED view can sit in a different
|
| 384 |
+
place for each person who can see it, which is the only coherent answer once two people
|
| 385 |
+
share one view.
|
| 386 |
+
|
| 387 |
+
An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
|
| 388 |
+
"""
|
| 389 |
+
def _set(ws):
|
| 390 |
+
clean = []
|
| 391 |
+
seen = set()
|
| 392 |
+
for vid in (order or []):
|
| 393 |
+
vid = str(vid).strip()[:120]
|
| 394 |
+
if vid and vid not in seen:
|
| 395 |
+
seen.add(vid)
|
| 396 |
+
clean.append(vid)
|
| 397 |
+
if clean:
|
| 398 |
+
ws['viewOrder'] = clean
|
| 399 |
+
else:
|
| 400 |
+
ws.pop('viewOrder', None)
|
| 401 |
+
self._update(username, _set)
|
| 402 |
+
|
| 403 |
+
def shared_views(self, viewer, is_admin=False):
|
| 404 |
+
"""Every SHARED view this viewer may see, by id (wave-9 I17).
|
| 405 |
+
|
| 406 |
+
Read-only and independent of the viewer's own workspace: the caller merges. Returns
|
| 407 |
+
only what `_may_see` allows, so a caller cannot accidentally render somebody else's
|
| 408 |
+
personal view by forgetting to filter.
|
| 409 |
+
"""
|
| 410 |
+
try:
|
| 411 |
+
bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
|
| 412 |
+
except Exception:
|
| 413 |
+
return {}
|
| 414 |
+
return {vid: dict(v) for vid, v in bucket.items()
|
| 415 |
+
if _may_see(v, viewer, is_admin)}
|
| 416 |
+
|
| 417 |
+
def shared_view(self, view_id):
|
| 418 |
+
"""One shared view RAW — no visibility filter. For authorisation decisions only: a
|
| 419 |
+
caller must know a view exists and who owns it before it can decide whether the actor
|
| 420 |
+
may touch it. Never hand the result to a renderer without checking `_may_see`."""
|
| 421 |
+
try:
|
| 422 |
+
return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
|
| 423 |
+
).get('views', {}).get(str(view_id))
|
| 424 |
+
except Exception:
|
| 425 |
+
return None
|
| 426 |
+
|
| 427 |
+
def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
|
| 428 |
+
"""Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
|
| 429 |
+
|
| 430 |
+
`shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
|
| 431 |
+
it lands in, the view is REMOVED from the other, so a view can never exist as two
|
| 432 |
+
copies that diverge on the next edit.
|
| 433 |
+
|
| 434 |
+
⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
|
| 435 |
+
moves data and does not know who is asking. `_cl_handle_one` is the wall.
|
| 436 |
+
"""
|
| 437 |
+
view_id = str((view or {}).get('id') or '').strip()
|
| 438 |
+
if not view_id:
|
| 439 |
+
raise ValueError('view id is required')
|
| 440 |
+
if username == SHARED_KEY:
|
| 441 |
+
raise ValueError('reserved username')
|
| 442 |
+
to_shared = is_shared(view) if shared is None else bool(shared)
|
| 443 |
+
requested = dict(view)
|
| 444 |
+
accepted = {}
|
| 445 |
+
|
| 446 |
+
def _up(data):
|
| 447 |
+
# View names are tenant-global: every personal workspace plus the shared bucket.
|
| 448 |
+
# This deliberately includes views the actor cannot see. The only disclosed fact
|
| 449 |
+
# is that a display name is already taken, while the categorical "no duplicate
|
| 450 |
+
# view names" contract remains true when a personal view is later shared.
|
| 451 |
+
names = list(reserved_names or ())
|
| 452 |
+
for workspace in data.values():
|
| 453 |
+
if not isinstance(workspace, dict):
|
| 454 |
+
continue
|
| 455 |
+
names.extend(
|
| 456 |
+
value.get('name')
|
| 457 |
+
for candidate_id, value in (workspace.get('views') or {}).items()
|
| 458 |
+
if candidate_id != view_id and isinstance(value, dict)
|
| 459 |
+
)
|
| 460 |
+
payload = dict(requested)
|
| 461 |
+
payload['name'] = _unique_name(payload.get('name'), names)
|
| 462 |
+
accepted.clear()
|
| 463 |
+
accepted.update(payload)
|
| 464 |
+
if to_shared:
|
| 465 |
+
bucket = data.setdefault(SHARED_KEY, {})
|
| 466 |
+
bucket.setdefault('views', {})[view_id] = payload
|
| 467 |
+
# it may have lived in the creator's workspace before being shared
|
| 468 |
+
owner = data.get(payload.get('createdBy') or username) or {}
|
| 469 |
+
(owner.get('views') or {}).pop(view_id, None)
|
| 470 |
+
else:
|
| 471 |
+
ws = data.setdefault(username, {})
|
| 472 |
+
ws.setdefault('views', {})[view_id] = payload
|
| 473 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
|
| 474 |
+
return data
|
| 475 |
+
|
| 476 |
+
self._st.update(self.table_key, _up, flush='async')
|
| 477 |
+
return dict(accepted)
|
| 478 |
+
|
| 479 |
+
def delete_view(self, username, view_id):
|
| 480 |
+
"""Delete a custom/list view override. The system all-rows view is guarded by caller.
|
| 481 |
+
|
| 482 |
+
Removes from BOTH homes: the caller has already authorised the delete, and leaving a
|
| 483 |
+
stale copy in the other bucket would resurrect the view on the next read.
|
| 484 |
+
"""
|
| 485 |
+
vid = str(view_id)
|
| 486 |
+
|
| 487 |
+
def _up(data):
|
| 488 |
+
(data.get(username, {}).get('views') or {}).pop(vid, None)
|
| 489 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
|
| 490 |
+
return data
|
| 491 |
+
|
| 492 |
+
self._st.update(self.table_key, _up, flush='async')
|
| 493 |
+
|
| 494 |
+
def save_field(self, username, field, reserved_names=(), correction_id=None):
|
| 495 |
+
"""Persist a column note or a user-created (custom_/measure_) field definition."""
|
| 496 |
+
key = str((field or {}).get('key') or '').strip()
|
| 497 |
+
if not key:
|
| 498 |
+
raise ValueError('field key is required')
|
| 499 |
+
requested = dict(field)
|
| 500 |
+
accepted = {}
|
| 501 |
+
|
| 502 |
+
def _save(ws):
|
| 503 |
+
names = list(reserved_names or ())
|
| 504 |
+
names.extend(
|
| 505 |
+
value.get('label')
|
| 506 |
+
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 507 |
+
if candidate_key != key and isinstance(value, dict)
|
| 508 |
+
)
|
| 509 |
+
payload = dict(requested)
|
| 510 |
+
payload.pop('labelCorrectedFrom', None)
|
| 511 |
+
payload.pop('labelCorrectionId', None)
|
| 512 |
+
requested_label = ' '.join(
|
| 513 |
+
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 514 |
+
payload['label'] = _unique_name(requested_label, names)
|
| 515 |
+
corrections = ws.setdefault('fieldCorrections', {})
|
| 516 |
+
corrections.pop(key, None)
|
| 517 |
+
if payload['label'] != requested_label and correction_id:
|
| 518 |
+
corrections[key] = {
|
| 519 |
+
'label': payload['label'],
|
| 520 |
+
'labelCorrectedFrom': requested_label,
|
| 521 |
+
'labelCorrectionId': str(correction_id)[:180],
|
| 522 |
+
}
|
| 523 |
+
if not corrections:
|
| 524 |
+
ws.pop('fieldCorrections', None)
|
| 525 |
+
accepted.clear()
|
| 526 |
+
accepted.update(payload)
|
| 527 |
+
if source_override_is_empty(payload):
|
| 528 |
+
ws['fields'].pop(key, None)
|
| 529 |
+
else:
|
| 530 |
+
ws['fields'][key] = payload
|
| 531 |
+
|
| 532 |
+
self._update(username, _save)
|
| 533 |
+
return dict(accepted)
|
| 534 |
+
|
| 535 |
+
def duplicate_field(self, username, source_key, new_key, field,
|
| 536 |
+
reserved_names=(), correction_id=None):
|
| 537 |
+
"""Clone a user-created field in ONE store transaction (wave-5 item 1): the new
|
| 538 |
+
definition plus — for `custom_` overlay sources only — every stored cell value under
|
| 539 |
+
the source key. One transaction, because a def without its values (or values without a
|
| 540 |
+
def) is exactly the orphan state delete_field exists to prevent, in reverse.
|
| 541 |
+
The caller validated both keys (same created stratum) and stamped the clone's
|
| 542 |
+
createdBy; this layer only moves data."""
|
| 543 |
+
source_key = str(source_key or '').strip()
|
| 544 |
+
new_key = str(new_key or '').strip()
|
| 545 |
+
if not source_key or not new_key or source_key == new_key:
|
| 546 |
+
raise ValueError('duplicate_field needs two distinct keys')
|
| 547 |
+
requested = dict(field)
|
| 548 |
+
accepted = {}
|
| 549 |
+
|
| 550 |
+
def _dup(ws):
|
| 551 |
+
names = list(reserved_names or ())
|
| 552 |
+
names.extend(
|
| 553 |
+
value.get('label')
|
| 554 |
+
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 555 |
+
if candidate_key != new_key and isinstance(value, dict)
|
| 556 |
+
)
|
| 557 |
+
payload = dict(requested)
|
| 558 |
+
payload.pop('labelCorrectedFrom', None)
|
| 559 |
+
payload.pop('labelCorrectionId', None)
|
| 560 |
+
requested_label = ' '.join(
|
| 561 |
+
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 562 |
+
payload['label'] = _unique_name(requested_label, names)
|
| 563 |
+
corrections = ws.setdefault('fieldCorrections', {})
|
| 564 |
+
corrections.pop(new_key, None)
|
| 565 |
+
if payload['label'] != requested_label and correction_id:
|
| 566 |
+
corrections[new_key] = {
|
| 567 |
+
'label': payload['label'],
|
| 568 |
+
'labelCorrectedFrom': requested_label,
|
| 569 |
+
'labelCorrectionId': str(correction_id)[:180],
|
| 570 |
+
}
|
| 571 |
+
if not corrections:
|
| 572 |
+
ws.pop('fieldCorrections', None)
|
| 573 |
+
accepted.clear()
|
| 574 |
+
accepted.update(payload)
|
| 575 |
+
ws['fields'][new_key] = payload
|
| 576 |
+
if source_key.startswith('custom_'):
|
| 577 |
+
for row in ws['overlays'].values():
|
| 578 |
+
if isinstance(row, dict) and source_key in row:
|
| 579 |
+
row[new_key] = row[source_key]
|
| 580 |
+
|
| 581 |
+
self._update(username, _dup)
|
| 582 |
+
return dict(accepted)
|
| 583 |
+
|
| 584 |
+
def delete_field(self, username, key):
|
| 585 |
+
"""Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
|
| 586 |
+
|
| 587 |
+
Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
|
| 588 |
+
columns — the caller enforces the prefix). The stored overlay VALUES for the key are
|
| 589 |
+
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 590 |
+
silently resurface if the key were ever reused. Views referencing the key self-heal on
|
| 591 |
+
their next autosave (an unknown colId is dropped) — the rule every stale key rides.
|
| 592 |
+
"""
|
| 593 |
+
key = str(key or '').strip()
|
| 594 |
+
if not key:
|
| 595 |
+
return
|
| 596 |
+
|
| 597 |
+
def _drop(ws):
|
| 598 |
+
ws['fields'].pop(key, None)
|
| 599 |
+
for row in ws['overlays'].values():
|
| 600 |
+
if isinstance(row, dict):
|
| 601 |
+
row.pop(key, None)
|
| 602 |
+
|
| 603 |
+
self._update(username, _drop)
|
| 604 |
+
|
| 605 |
+
def patch_overlay(self, username, pid, updates):
|
| 606 |
+
"""Patch only the external editable stratum; never writes to the source system."""
|
| 607 |
+
clean = dict(updates or {})
|
| 608 |
+
if not clean:
|
| 609 |
+
return
|
| 610 |
+
|
| 611 |
+
def _patch(ws):
|
| 612 |
+
ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
|
| 613 |
+
|
| 614 |
+
self._update(username, _patch)
|
| 615 |
+
|
| 616 |
+
|
| 617 |
+
def make(table_key, st=None):
|
| 618 |
+
return TableStore(table_key, st=st)
|
platform/core/user_tables.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/core/users.py
CHANGED
|
@@ -445,20 +445,21 @@ def assignable_people(tenant=None):
|
|
| 445 |
|
| 446 |
Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry
|
| 447 |
is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
|
| 448 |
-
cross-tenant name leak.
|
| 449 |
-
|
| 450 |
-
⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and
|
| 451 |
-
return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was
|
| 452 |
-
written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at
|
| 453 |
-
EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform
|
| 454 |
-
roster one forgotten kwarg away, with no error to notice
|
| 455 |
-
([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing).
|
| 456 |
-
⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass
|
| 457 |
-
`session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate
|
| 458 |
-
changes — and a future caller that forgets gets an empty picker somebody notices instead of a
|
| 459 |
-
leak nobody does.
|
| 460 |
"""
|
| 461 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
|
| 463 |
|
| 464 |
def set_avatar(username, data_url):
|
|
@@ -485,41 +486,20 @@ def avatar_map(tenant=None):
|
|
| 485 |
name, so the display name is the only join a renderer has. Two active accounts sharing a
|
| 486 |
display name share one option; the first WITH a photo wins the key rather than a coin
|
| 487 |
flip deciding whether the option has a face. `tenant` scopes it exactly as
|
| 488 |
-
`assignable_people(tenant)` does, and for the same leak
|
| 489 |
-
|
| 490 |
-
|
|
|
|
|
|
|
|
|
|
| 491 |
out = {}
|
| 492 |
-
for username, u in sorted(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
av = u.get('avatar')
|
| 494 |
nm = str(u.get('name') or username)
|
| 495 |
if av and nm not in out:
|
| 496 |
out[nm] = str(av)
|
| 497 |
return out
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
def _tenant_accounts(tenant):
|
| 501 |
-
"""`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED.
|
| 502 |
-
|
| 503 |
-
⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice,
|
| 504 |
-
identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a
|
| 505 |
-
blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of
|
| 506 |
-
them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of
|
| 507 |
-
the same predicate and is lane C's to fold in.
|
| 508 |
-
|
| 509 |
-
⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a
|
| 510 |
-
pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public`
|
| 511 |
-
normalises it the same way. What changed is the WANT side — a blank want is now a refusal
|
| 512 |
-
rather than a wildcard.
|
| 513 |
-
"""
|
| 514 |
-
want = str(tenant or '').strip().lower()
|
| 515 |
-
if not want:
|
| 516 |
-
# Not an error and not everything: nobody. See `assignable_people`'s note — the one
|
| 517 |
-
# caller this exemption was written for (the Streamlit host) was deleted at EXIT-6.
|
| 518 |
-
return []
|
| 519 |
-
try:
|
| 520 |
-
reg = registry() or {}
|
| 521 |
-
except Exception:
|
| 522 |
-
return []
|
| 523 |
-
return [(n, u) for n, u in reg.items()
|
| 524 |
-
if isinstance(u, dict) and u.get('active') is not False
|
| 525 |
-
and str(u.get('tenant') or 'royal-imports').strip().lower() == want]
|
|
|
|
| 445 |
|
| 446 |
Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry
|
| 447 |
is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
|
| 448 |
+
cross-tenant name leak. None = unscoped (the Streamlit host, tenant #0's process).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 449 |
"""
|
| 450 |
+
try:
|
| 451 |
+
reg = registry() or {}
|
| 452 |
+
except Exception:
|
| 453 |
+
return []
|
| 454 |
+
want = str(tenant or '').strip().lower()
|
| 455 |
+
out = []
|
| 456 |
+
for username, u in reg.items():
|
| 457 |
+
if not isinstance(u, dict) or u.get('active') is False:
|
| 458 |
+
continue
|
| 459 |
+
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 460 |
+
continue
|
| 461 |
+
out.append(str(u.get('name') or username))
|
| 462 |
+
return sorted(set(out))
|
| 463 |
|
| 464 |
|
| 465 |
def set_avatar(username, data_url):
|
|
|
|
| 486 |
name, so the display name is the only join a renderer has. Two active accounts sharing a
|
| 487 |
display name share one option; the first WITH a photo wins the key rather than a coin
|
| 488 |
flip deciding whether the option has a face. `tenant` scopes it exactly as
|
| 489 |
+
`assignable_people(tenant)` does, and for the same leak."""
|
| 490 |
+
try:
|
| 491 |
+
reg = registry() or {}
|
| 492 |
+
except Exception:
|
| 493 |
+
return {}
|
| 494 |
+
want = str(tenant or '').strip().lower()
|
| 495 |
out = {}
|
| 496 |
+
for username, u in sorted(reg.items()):
|
| 497 |
+
if not isinstance(u, dict) or u.get('active') is False:
|
| 498 |
+
continue
|
| 499 |
+
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 500 |
+
continue
|
| 501 |
av = u.get('avatar')
|
| 502 |
nm = str(u.get('name') or username)
|
| 503 |
if av and nm not in out:
|
| 504 |
out[nm] = str(av)
|
| 505 |
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
platform/harness/datastore.py
CHANGED
|
@@ -188,35 +188,14 @@ ENTITIES = {
|
|
| 188 |
# backfill below has walked every partner (a partner's `write_date` did not move because
|
| 189 |
# WE changed the schema). Every projection degrades a missing/NULL column to blank rather
|
| 190 |
# than raising, so declaring early is safe — it is just not yet true.
|
| 191 |
-
# ⭐⭐ WAVE 33 (W33-T48, owner item 13) — THE CONTACT/IDENTITY HALF, MEASURED BEFORE IT WAS
|
| 192 |
-
# ADDED. The owner: *"I could have sworn we have more relevant Fields under 'Odoo vendors'
|
| 193 |
-
# etc."* He was right, and the census
|
| 194 |
-
# (`.claude/wiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how
|
| 195 |
-
# much: `res.partner` declares 198 fields, **77 are populated on all 3,621 customers and 76
|
| 196 |
-
# on all 658 vendors**, and this mirror carried THIRTEEN. Every one of the six added here
|
| 197 |
-
# was measured non-empty on the live population first — none is added on the strength of
|
| 198 |
-
# "Odoo probably has that", which is how a schema grows columns nobody can fill.
|
| 199 |
-
# ⚠ THE MIRROR IS THE CEILING, NOT THE GRID. Every reader in `odoo_relational.py` reads
|
| 200 |
-
# THIS file, so a column added to a `*_fields()` contract with no entry here renders as a
|
| 201 |
-
# blank column forever — which is D-165's shape exactly, and the reason the widening starts
|
| 202 |
-
# on this line rather than on the grid.
|
| 203 |
"fields": ["name", "team_id", "city", "state_id", "country_id", "active",
|
| 204 |
"agent", "salesman_as_agent", "customer_rank",
|
| 205 |
-
"street", "street2", "zip",
|
| 206 |
-
"email", "phone", "mobile", "website", "vat", "ref",
|
| 207 |
-
"write_date"],
|
| 208 |
# `backfill_fields` carries it too, so mirrors that already exist pick it up on the next
|
| 209 |
# sync_all() instead of needing a reseed.
|
| 210 |
-
# ⛔ D-165 IS ABOUT THIS LIST, AND THE CENSUS SETTLED WHICH HALF IS WRONG: `street`,
|
| 211 |
-
# `street2` and `zip` are `store=True` on `res.partner` and POPULATED in live Odoo — they
|
| 212 |
-
# are declared here and still NULL in the mirror purely because a backfill pass has not
|
| 213 |
-
# walked every partner (changing OUR schema does not move a partner's `write_date`, so an
|
| 214 |
-
# incremental sync never revisits them). Declaring is not filling; the fix is a run.
|
| 215 |
"backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28
|
| 216 |
"customer_rank", # added 2026-08-11 (W29-T54)
|
| 217 |
-
"street", "street2", "zip",
|
| 218 |
-
"email", "phone", "mobile", # added 2026-08-14 (W33-T48)
|
| 219 |
-
"website", "vat", "ref"],
|
| 220 |
# m2m fields land as <base>_id (the FIRST linked id — the Customers-module convention:
|
| 221 |
# a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
|
| 222 |
# JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
|
|
|
|
| 188 |
# backfill below has walked every partner (a partner's `write_date` did not move because
|
| 189 |
# WE changed the schema). Every projection degrades a missing/NULL column to blank rather
|
| 190 |
# than raising, so declaring early is safe — it is just not yet true.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
"fields": ["name", "team_id", "city", "state_id", "country_id", "active",
|
| 192 |
"agent", "salesman_as_agent", "customer_rank",
|
| 193 |
+
"street", "street2", "zip", "write_date"],
|
|
|
|
|
|
|
| 194 |
# `backfill_fields` carries it too, so mirrors that already exist pick it up on the next
|
| 195 |
# sync_all() instead of needing a reseed.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
"backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28
|
| 197 |
"customer_rank", # added 2026-08-11 (W29-T54)
|
| 198 |
+
"street", "street2", "zip"], # added 2026-08-12 (W30-T33)
|
|
|
|
|
|
|
| 199 |
# m2m fields land as <base>_id (the FIRST linked id — the Customers-module convention:
|
| 200 |
# a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
|
| 201 |
# JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
|
platform/harness/meta_store.py
CHANGED
|
@@ -1,521 +1,521 @@
|
|
| 1 |
-
"""meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
|
| 2 |
-
|
| 3 |
-
python platform/harness/meta_store.py --sync pull into tenant #0's mirror
|
| 4 |
-
python platform/harness/meta_store.py --sync --tenant gtmlab
|
| 5 |
-
python platform/harness/meta_store.py --status what is in the mirror now
|
| 6 |
-
|
| 7 |
-
⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
|
| 8 |
-
hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of
|
| 9 |
-
spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
|
| 10 |
-
per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
|
| 11 |
-
delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
|
| 12 |
-
store with two loaders, not two stores.**
|
| 13 |
-
|
| 14 |
-
⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
|
| 15 |
-
the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
|
| 16 |
-
`XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
|
| 17 |
-
`Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
|
| 18 |
-
different first hop.
|
| 19 |
-
|
| 20 |
-
⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
|
| 21 |
-
carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
|
| 22 |
-
Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57
|
| 23 |
-
⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
|
| 24 |
-
came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
|
| 25 |
-
nothing going red — which is exactly the "do not drop any column" instruction, broken silently.
|
| 26 |
-
The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
|
| 27 |
-
|
| 28 |
-
⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
|
| 29 |
-
answers `403 (#200) Requires business_management permission`, which this token does not carry. One
|
| 30 |
-
column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause.
|
| 31 |
-
|
| 32 |
-
⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
|
| 33 |
-
larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
|
| 34 |
-
(W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
|
| 35 |
-
"""
|
| 36 |
-
import argparse
|
| 37 |
-
import json
|
| 38 |
-
import os
|
| 39 |
-
import sys
|
| 40 |
-
import time
|
| 41 |
-
import urllib.error
|
| 42 |
-
import urllib.parse
|
| 43 |
-
import urllib.request
|
| 44 |
-
from pathlib import Path
|
| 45 |
-
|
| 46 |
-
_HERE = Path(__file__).resolve().parent
|
| 47 |
-
if str(_HERE.parent) not in sys.path:
|
| 48 |
-
sys.path.insert(0, str(_HERE.parent))
|
| 49 |
-
|
| 50 |
-
from harness import datastore # noqa: E402
|
| 51 |
-
|
| 52 |
-
GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
|
| 53 |
-
GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
|
| 54 |
-
|
| 55 |
-
#: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and
|
| 56 |
-
#: including anything Graph returns as a nested object, which is stored as compact JSON text.
|
| 57 |
-
_INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
|
| 58 |
-
"inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
|
| 59 |
-
"account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
|
| 60 |
-
"min_daily_budget", "min_campaign_group_spend_cap"}
|
| 61 |
-
_DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
|
| 62 |
-
"inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
|
| 63 |
-
"daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
|
| 64 |
-
"daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
|
| 65 |
-
"lifetime_spend_cap", "lifetime_imps"}
|
| 66 |
-
|
| 67 |
-
#: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
|
| 68 |
-
#: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
|
| 69 |
-
#: the grid links in `meta_relational`.
|
| 70 |
-
ACCOUNT_FIELDS = (
|
| 71 |
-
"account_id account_status age amount_spent balance business_city business_country_code "
|
| 72 |
-
"business_name business_state business_street business_street2 business_zip capabilities "
|
| 73 |
-
"created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
|
| 74 |
-
"has_migrated_permissions id io_number is_attribution_spec_system_default "
|
| 75 |
-
"is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
|
| 76 |
-
"is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
|
| 77 |
-
"name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
|
| 78 |
-
"timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
|
| 79 |
-
).split()
|
| 80 |
-
|
| 81 |
-
CAMPAIGN_FIELDS = (
|
| 82 |
-
"account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
|
| 83 |
-
"campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
|
| 84 |
-
"created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
|
| 85 |
-
"last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
|
| 86 |
-
"promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
|
| 87 |
-
"special_ad_category special_ad_category_country spend_cap start_time status stop_time "
|
| 88 |
-
"topline_id updated_time"
|
| 89 |
-
).split()
|
| 90 |
-
|
| 91 |
-
ADSET_FIELDS = (
|
| 92 |
-
"account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
|
| 93 |
-
"bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
|
| 94 |
-
"campaign_active_time campaign_attribution campaign_id configured_status created_time "
|
| 95 |
-
"creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
|
| 96 |
-
"effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
|
| 97 |
-
"issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
|
| 98 |
-
"lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
|
| 99 |
-
"optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
|
| 100 |
-
"rf_prediction_id source_adset source_adset_id start_time status targeting "
|
| 101 |
-
"targeting_optimization_types time_based_ad_rotation_id_blocks "
|
| 102 |
-
"time_based_ad_rotation_intervals updated_time use_new_app_click"
|
| 103 |
-
).split()
|
| 104 |
-
|
| 105 |
-
AD_FIELDS = (
|
| 106 |
-
"account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
|
| 107 |
-
"adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
|
| 108 |
-
"conversion_domain created_time creative demolink_hash display_sequence effective_status "
|
| 109 |
-
"engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
|
| 110 |
-
"preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
|
| 111 |
-
"tracking_and_conversion_with_defaults tracking_specs updated_time"
|
| 112 |
-
).split()
|
| 113 |
-
|
| 114 |
-
CREATIVE_FIELDS = (
|
| 115 |
-
"account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
|
| 116 |
-
"branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
|
| 117 |
-
"category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
|
| 118 |
-
"destination_set_id dynamic_ad_voice effective_authorization_category "
|
| 119 |
-
"effective_instagram_media_id effective_object_story_id enable_direct_install "
|
| 120 |
-
"enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
|
| 121 |
-
"instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
|
| 122 |
-
"link_deep_link_url link_destination_display_url link_og_id link_url "
|
| 123 |
-
"messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
|
| 124 |
-
"object_type object_url place_page_set_id platform_customizations playable_asset_id "
|
| 125 |
-
"portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
|
| 126 |
-
"template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
|
| 127 |
-
"use_page_actor_override video_id"
|
| 128 |
-
).split()
|
| 129 |
-
|
| 130 |
-
INSIGHT_FIELDS = (
|
| 131 |
-
"account_currency account_id account_name action_values actions ad_id ad_name adset_id "
|
| 132 |
-
"adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
|
| 133 |
-
"clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
|
| 134 |
-
"cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
|
| 135 |
-
"date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
|
| 136 |
-
"impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
|
| 137 |
-
"optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
|
| 138 |
-
"social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
|
| 139 |
-
"video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
|
| 140 |
-
"video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
|
| 141 |
-
"video_play_actions video_thruplay_watched_actions website_purchase_roas"
|
| 142 |
-
).split()
|
| 143 |
-
|
| 144 |
-
#: table -> (graph edge on the account | None, measured fields, parent column)
|
| 145 |
-
SPECS = {
|
| 146 |
-
"meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
|
| 147 |
-
"meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
|
| 148 |
-
"meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
|
| 149 |
-
"meta_ads": ("ads", AD_FIELDS, "adset_id"),
|
| 150 |
-
"meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
|
| 151 |
-
}
|
| 152 |
-
|
| 153 |
-
#: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights
|
| 154 |
-
#: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
|
| 155 |
-
INSIGHTS_TABLE = "meta_insights_daily"
|
| 156 |
-
INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
|
| 157 |
-
|
| 158 |
-
#: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
|
| 159 |
-
#: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
|
| 160 |
-
#: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the
|
| 161 |
-
#: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
|
| 162 |
-
#: page 25 answer 200. So the window is walked in slices with every column intact:
|
| 163 |
-
#: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows
|
| 164 |
-
#: 57 fields · account level · 7d -> 200, 7 rows
|
| 165 |
-
#: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over —
|
| 166 |
-
#: it drops columns R2 requires, and it does so invisibly.
|
| 167 |
-
INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
|
| 168 |
-
INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
|
| 169 |
-
INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
def _slices(days, size):
|
| 173 |
-
"""[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
|
| 174 |
-
from datetime import date, timedelta
|
| 175 |
-
end = date.today()
|
| 176 |
-
start = end - timedelta(days=max(1, days) - 1)
|
| 177 |
-
out, cur = [], start
|
| 178 |
-
while cur <= end:
|
| 179 |
-
stop = min(cur + timedelta(days=max(1, size) - 1), end)
|
| 180 |
-
out.append((cur.isoformat(), stop.isoformat()))
|
| 181 |
-
cur = stop + timedelta(days=1)
|
| 182 |
-
return out
|
| 183 |
-
|
| 184 |
-
#: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
|
| 185 |
-
#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
|
| 186 |
-
#: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it
|
| 187 |
-
#: is reported as a problem, never absorbed.
|
| 188 |
-
MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
|
| 189 |
-
PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
class MetaError(RuntimeError):
|
| 193 |
-
"""A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
def token():
|
| 197 |
-
"""The Meta token from the environment or gitignored `platform/.env`; "" when absent.
|
| 198 |
-
|
| 199 |
-
⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
|
| 200 |
-
`platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
|
| 201 |
-
(`core` never imports up). Fifteen lines is the price of that boundary.
|
| 202 |
-
"""
|
| 203 |
-
tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
|
| 204 |
-
if tok:
|
| 205 |
-
return tok.strip()
|
| 206 |
-
env = _HERE.parent / ".env"
|
| 207 |
-
if env.exists():
|
| 208 |
-
for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 209 |
-
if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
|
| 210 |
-
_, _, v = line.partition("=")
|
| 211 |
-
return v.strip().strip('"').strip("'")
|
| 212 |
-
return ""
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
def _get(path, tok, **params):
|
| 216 |
-
params["access_token"] = tok
|
| 217 |
-
url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
|
| 218 |
-
req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
|
| 219 |
-
try:
|
| 220 |
-
with urllib.request.urlopen(req, timeout=120) as r:
|
| 221 |
-
return json.loads(r.read().decode("utf-8", "replace"))
|
| 222 |
-
except urllib.error.HTTPError as e:
|
| 223 |
-
try:
|
| 224 |
-
msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
|
| 225 |
-
).get("message") or ""
|
| 226 |
-
except Exception:
|
| 227 |
-
msg = ""
|
| 228 |
-
raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
|
| 229 |
-
except Exception as e:
|
| 230 |
-
raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
#: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
|
| 234 |
-
#: why it cannot be treated as "the server is broken, give up".
|
| 235 |
-
_TOO_MUCH = "reduce the amount of data"
|
| 236 |
-
|
| 237 |
-
#: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
|
| 238 |
-
#: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
|
| 239 |
-
#: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the
|
| 240 |
-
#: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
|
| 241 |
-
#: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
|
| 242 |
-
#: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
|
| 243 |
-
#: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
|
| 244 |
-
_RATE_LIMITED = "too many calls"
|
| 245 |
-
_RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
|
| 246 |
-
_RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
def _walk(path, tok, log, **params):
|
| 250 |
-
"""Every page of an edge, paged by CURSOR under our own parameters.
|
| 251 |
-
|
| 252 |
-
⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
|
| 253 |
-
first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
|
| 254 |
-
you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL
|
| 255 |
-
limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
|
| 256 |
-
Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
|
| 257 |
-
|
| 258 |
-
⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
|
| 259 |
-
silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
|
| 260 |
-
every column per row.
|
| 261 |
-
|
| 262 |
-
-> (rows, hit_page_cap)
|
| 263 |
-
"""
|
| 264 |
-
out, pages = [], 0
|
| 265 |
-
limit = int(params.pop("limit", None) or PAGE)
|
| 266 |
-
after, waited = None, 0
|
| 267 |
-
while True:
|
| 268 |
-
call = dict(params, limit=limit)
|
| 269 |
-
if after:
|
| 270 |
-
call["after"] = after
|
| 271 |
-
try:
|
| 272 |
-
body = _get(path, tok, **call)
|
| 273 |
-
except MetaError as e:
|
| 274 |
-
if _TOO_MUCH in str(e) and limit > 5:
|
| 275 |
-
limit = max(5, limit // 4)
|
| 276 |
-
log(f" page too heavy for /{path} - retrying at limit={limit} "
|
| 277 |
-
f"(a payload ceiling, not a row cap; every column is still asked for)")
|
| 278 |
-
continue
|
| 279 |
-
if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
|
| 280 |
-
waited += 1
|
| 281 |
-
log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
|
| 282 |
-
f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
|
| 283 |
-
time.sleep(_RATE_WAIT)
|
| 284 |
-
continue
|
| 285 |
-
if _RATE_LIMITED in str(e).lower():
|
| 286 |
-
log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
|
| 287 |
-
f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
|
| 288 |
-
f"limit is per ad account and persists for minutes. Fix: re-run "
|
| 289 |
-
f"`--sync --only <table>` later; the upsert is idempotent.")
|
| 290 |
-
return out, True
|
| 291 |
-
raise
|
| 292 |
-
out.extend(body.get("data") or [])
|
| 293 |
-
pages += 1
|
| 294 |
-
after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
|
| 295 |
-
has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
|
| 296 |
-
if not has_next:
|
| 297 |
-
return out, False
|
| 298 |
-
if pages >= MAX_PAGES:
|
| 299 |
-
log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
|
| 300 |
-
f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
|
| 301 |
-
f"window with META_INSIGHTS_DAYS.")
|
| 302 |
-
return out, True
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
def _cell(value):
|
| 306 |
-
"""One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
|
| 307 |
-
being dropped: R2 says every field the API returns, and `targeting` is a field."""
|
| 308 |
-
if value is None or isinstance(value, (str, int, float, bool)):
|
| 309 |
-
return value
|
| 310 |
-
return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
def _coltype(name, rows=None):
|
| 314 |
-
"""The DuckDB type for one column — decided by the DATA when there is data, by the name only
|
| 315 |
-
as a fallback.
|
| 316 |
-
|
| 317 |
-
⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
|
| 318 |
-
`cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
|
| 319 |
-
action-type objects** on the Insights edge:
|
| 320 |
-
[{"action_type":"outbound_click","value":"1.459854"}]
|
| 321 |
-
Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
|
| 322 |
-
'[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked
|
| 323 |
-
like it worked and then blew up on the last table.
|
| 324 |
-
⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
|
| 325 |
-
do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
|
| 326 |
-
that is what `_cell` stores; anything else falls back to the measured name hints.
|
| 327 |
-
"""
|
| 328 |
-
if rows:
|
| 329 |
-
seen, kinds = 0, set()
|
| 330 |
-
for r in rows:
|
| 331 |
-
v = r.get(name)
|
| 332 |
-
if v is None or v == "":
|
| 333 |
-
continue
|
| 334 |
-
kinds.add("json" if isinstance(v, (list, dict)) else
|
| 335 |
-
"bool" if isinstance(v, bool) else
|
| 336 |
-
"int" if isinstance(v, int) else
|
| 337 |
-
"float" if isinstance(v, float) else "str")
|
| 338 |
-
seen += 1
|
| 339 |
-
if seen >= 200:
|
| 340 |
-
break
|
| 341 |
-
if kinds:
|
| 342 |
-
if "json" in kinds or "str" in kinds:
|
| 343 |
-
return "VARCHAR" # a JSON blob, or a numeric STRING
|
| 344 |
-
if kinds <= {"int", "bool"}:
|
| 345 |
-
return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
|
| 346 |
-
return "DOUBLE"
|
| 347 |
-
if name in _INT:
|
| 348 |
-
return "BIGINT"
|
| 349 |
-
if name in _DBL:
|
| 350 |
-
return "DOUBLE"
|
| 351 |
-
return "VARCHAR" # ids included — see the header
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
def _ensure(con, table, fields, rows=None):
|
| 355 |
-
"""Create or widen the table, with every column typed from `rows` when they are available.
|
| 356 |
-
|
| 357 |
-
⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
|
| 358 |
-
column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type
|
| 359 |
-
disagreement drops and recreates rather than limping on with a column that refuses every
|
| 360 |
-
insert. The alternative is a mirror that is permanently unwritable for one bad guess.
|
| 361 |
-
"""
|
| 362 |
-
want = {f: _coltype(f, rows) for f in fields}
|
| 363 |
-
have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
|
| 364 |
-
if have:
|
| 365 |
-
clash = [f for f, ty in want.items() if f in have and have[f] != ty
|
| 366 |
-
and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
|
| 367 |
-
if clash:
|
| 368 |
-
con.execute(f"DROP TABLE {table}")
|
| 369 |
-
have = {}
|
| 370 |
-
if not have:
|
| 371 |
-
cols = ", ".join(f"{f} {want[f]}" for f in fields)
|
| 372 |
-
con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
|
| 373 |
-
if "id" not in fields else
|
| 374 |
-
f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
|
| 375 |
-
return
|
| 376 |
-
for f in fields:
|
| 377 |
-
if f not in have:
|
| 378 |
-
con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
def _upsert(con, table, fields, rows):
|
| 382 |
-
"""Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so
|
| 383 |
-
a re-sync updates in place and can never append a second copy of the same object."""
|
| 384 |
-
if not rows:
|
| 385 |
-
return 0
|
| 386 |
-
cols = list(fields)
|
| 387 |
-
ids = [str(r.get("id") or "") for r in rows]
|
| 388 |
-
q = ",".join("?" for _ in ids)
|
| 389 |
-
con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
|
| 390 |
-
con.executemany(
|
| 391 |
-
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
|
| 392 |
-
[[_cell(r.get(c)) for c in cols] for r in rows])
|
| 393 |
-
return len(rows)
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
|
| 397 |
-
"""Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
|
| 398 |
-
|
| 399 |
-
Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
|
| 400 |
-
exactly as `odoo_relational.refresh` is.
|
| 401 |
-
"""
|
| 402 |
-
tok = tok or token()
|
| 403 |
-
# ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a
|
| 404 |
-
# live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
|
| 405 |
-
# `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
|
| 406 |
-
# imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
|
| 407 |
-
# that trips the per-ad-account rate limit and never finishes. The comment beside that call
|
| 408 |
-
# claimed a short window the code could not deliver.
|
| 409 |
-
# ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
|
| 410 |
-
# makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
|
| 411 |
-
days = int(insights_days or INSIGHTS_DAYS)
|
| 412 |
-
report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
|
| 413 |
-
if not tok:
|
| 414 |
-
report["problems"].append(
|
| 415 |
-
"META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
|
| 416 |
-
"was pulled. This is a missing CREDENTIAL, not a missing capability.")
|
| 417 |
-
return report
|
| 418 |
-
|
| 419 |
-
path = datastore.path_for(tenant_key)
|
| 420 |
-
if Path(datastore.DB_PATH) != Path(path):
|
| 421 |
-
datastore.use_path(path)
|
| 422 |
-
con = datastore.connect()
|
| 423 |
-
log(f" mirror: {Path(path).name}")
|
| 424 |
-
|
| 425 |
-
accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
|
| 426 |
-
report["accounts"] = [a.get("id") for a in accts]
|
| 427 |
-
if not accts:
|
| 428 |
-
report["problems"].append("the token reaches no ad accounts")
|
| 429 |
-
return report
|
| 430 |
-
|
| 431 |
-
# ── the five entity levels ────────────────────────────────────────────────────────────────
|
| 432 |
-
for table, (edge, fields, _parent) in SPECS.items():
|
| 433 |
-
rows, capped = [], False
|
| 434 |
-
for acct in accts:
|
| 435 |
-
aid = acct["id"]
|
| 436 |
-
if edge is None:
|
| 437 |
-
rows.append(_get(aid, tok, fields=",".join(fields)))
|
| 438 |
-
else:
|
| 439 |
-
got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
|
| 440 |
-
rows.extend(got)
|
| 441 |
-
capped = capped or hit
|
| 442 |
-
_ensure(con, table, fields, rows)
|
| 443 |
-
n = _upsert(con, table, fields, rows)
|
| 444 |
-
total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 445 |
-
report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 446 |
-
log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
|
| 447 |
-
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 448 |
-
if capped:
|
| 449 |
-
report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
|
| 450 |
-
|
| 451 |
-
# ── the daily Insights grain ──────────────────────────────────────────────────────────────
|
| 452 |
-
if insights:
|
| 453 |
-
rows, capped = [], False
|
| 454 |
-
for acct in accts:
|
| 455 |
-
for since, until in _slices(days, INSIGHTS_SLICE):
|
| 456 |
-
got, hit = _walk(f"{acct['id']}/insights", tok, log,
|
| 457 |
-
fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
|
| 458 |
-
time_increment="1", limit=INSIGHTS_PAGE,
|
| 459 |
-
time_range=json.dumps({"since": since, "until": until}))
|
| 460 |
-
rows.extend(got)
|
| 461 |
-
capped = capped or hit
|
| 462 |
-
# ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync
|
| 463 |
-
# appends a second copy of the same day — so it is composed from the grain itself.
|
| 464 |
-
for r in rows:
|
| 465 |
-
r["id"] = ":".join(str(r.get(k) or "") for k in
|
| 466 |
-
(f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
|
| 467 |
-
fields = ["id"] + INSIGHT_FIELDS
|
| 468 |
-
_ensure(con, INSIGHTS_TABLE, fields, rows)
|
| 469 |
-
n = _upsert(con, INSIGHTS_TABLE, fields, rows)
|
| 470 |
-
total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
|
| 471 |
-
report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 472 |
-
log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
|
| 473 |
-
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 474 |
-
|
| 475 |
-
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 476 |
-
["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
|
| 477 |
-
sum(t["in_mirror"] for t in report["tables"].values()),
|
| 478 |
-
time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 479 |
-
return report
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
def status(tenant_key="royal-imports"):
|
| 483 |
-
"""What is in the mirror right now, per table. Never fetches."""
|
| 484 |
-
path = datastore.path_for(tenant_key)
|
| 485 |
-
if Path(datastore.DB_PATH) != Path(path):
|
| 486 |
-
datastore.use_path(path)
|
| 487 |
-
out = {}
|
| 488 |
-
con = datastore.connect()
|
| 489 |
-
for table in list(SPECS) + [INSIGHTS_TABLE]:
|
| 490 |
-
try:
|
| 491 |
-
out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 492 |
-
except Exception:
|
| 493 |
-
out[table] = None # table absent = never synced
|
| 494 |
-
return out
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
def main(argv=None):
|
| 498 |
-
ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
|
| 499 |
-
ap.add_argument("--sync", action="store_true")
|
| 500 |
-
ap.add_argument("--status", action="store_true")
|
| 501 |
-
ap.add_argument("--tenant", default="royal-imports")
|
| 502 |
-
ap.add_argument("--no-insights", action="store_true")
|
| 503 |
-
ap.add_argument("--insights-days", type=int, default=None,
|
| 504 |
-
help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
|
| 505 |
-
a = ap.parse_args(argv)
|
| 506 |
-
if a.status:
|
| 507 |
-
for k, v in status(a.tenant).items():
|
| 508 |
-
print(f" {k:<20} {'(never synced)' if v is None else v}")
|
| 509 |
-
return 0
|
| 510 |
-
if not a.sync:
|
| 511 |
-
ap.print_help()
|
| 512 |
-
return 2
|
| 513 |
-
rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
|
| 514 |
-
for p in rep["problems"]:
|
| 515 |
-
print(" PROBLEM:", p)
|
| 516 |
-
print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
|
| 517 |
-
return 1 if rep["problems"] else 0
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
if __name__ == "__main__":
|
| 521 |
-
sys.exit(main())
|
|
|
|
| 1 |
+
"""meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
|
| 2 |
+
|
| 3 |
+
python platform/harness/meta_store.py --sync pull into tenant #0's mirror
|
| 4 |
+
python platform/harness/meta_store.py --sync --tenant gtmlab
|
| 5 |
+
python platform/harness/meta_store.py --status what is in the mirror now
|
| 6 |
+
|
| 7 |
+
⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
|
| 8 |
+
hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of
|
| 9 |
+
spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
|
| 10 |
+
per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
|
| 11 |
+
delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
|
| 12 |
+
store with two loaders, not two stores.**
|
| 13 |
+
|
| 14 |
+
⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
|
| 15 |
+
the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
|
| 16 |
+
`XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
|
| 17 |
+
`Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
|
| 18 |
+
different first hop.
|
| 19 |
+
|
| 20 |
+
⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
|
| 21 |
+
carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
|
| 22 |
+
Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57
|
| 23 |
+
⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
|
| 24 |
+
came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
|
| 25 |
+
nothing going red — which is exactly the "do not drop any column" instruction, broken silently.
|
| 26 |
+
The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
|
| 27 |
+
|
| 28 |
+
⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
|
| 29 |
+
answers `403 (#200) Requires business_management permission`, which this token does not carry. One
|
| 30 |
+
column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause.
|
| 31 |
+
|
| 32 |
+
⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
|
| 33 |
+
larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
|
| 34 |
+
(W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
|
| 35 |
+
"""
|
| 36 |
+
import argparse
|
| 37 |
+
import json
|
| 38 |
+
import os
|
| 39 |
+
import sys
|
| 40 |
+
import time
|
| 41 |
+
import urllib.error
|
| 42 |
+
import urllib.parse
|
| 43 |
+
import urllib.request
|
| 44 |
+
from pathlib import Path
|
| 45 |
+
|
| 46 |
+
_HERE = Path(__file__).resolve().parent
|
| 47 |
+
if str(_HERE.parent) not in sys.path:
|
| 48 |
+
sys.path.insert(0, str(_HERE.parent))
|
| 49 |
+
|
| 50 |
+
from harness import datastore # noqa: E402
|
| 51 |
+
|
| 52 |
+
GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
|
| 53 |
+
GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
|
| 54 |
+
|
| 55 |
+
#: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and
|
| 56 |
+
#: including anything Graph returns as a nested object, which is stored as compact JSON text.
|
| 57 |
+
_INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
|
| 58 |
+
"inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
|
| 59 |
+
"account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
|
| 60 |
+
"min_daily_budget", "min_campaign_group_spend_cap"}
|
| 61 |
+
_DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
|
| 62 |
+
"inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
|
| 63 |
+
"daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
|
| 64 |
+
"daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
|
| 65 |
+
"lifetime_spend_cap", "lifetime_imps"}
|
| 66 |
+
|
| 67 |
+
#: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
|
| 68 |
+
#: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
|
| 69 |
+
#: the grid links in `meta_relational`.
|
| 70 |
+
ACCOUNT_FIELDS = (
|
| 71 |
+
"account_id account_status age amount_spent balance business_city business_country_code "
|
| 72 |
+
"business_name business_state business_street business_street2 business_zip capabilities "
|
| 73 |
+
"created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
|
| 74 |
+
"has_migrated_permissions id io_number is_attribution_spec_system_default "
|
| 75 |
+
"is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
|
| 76 |
+
"is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
|
| 77 |
+
"name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
|
| 78 |
+
"timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
|
| 79 |
+
).split()
|
| 80 |
+
|
| 81 |
+
CAMPAIGN_FIELDS = (
|
| 82 |
+
"account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
|
| 83 |
+
"campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
|
| 84 |
+
"created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
|
| 85 |
+
"last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
|
| 86 |
+
"promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
|
| 87 |
+
"special_ad_category special_ad_category_country spend_cap start_time status stop_time "
|
| 88 |
+
"topline_id updated_time"
|
| 89 |
+
).split()
|
| 90 |
+
|
| 91 |
+
ADSET_FIELDS = (
|
| 92 |
+
"account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
|
| 93 |
+
"bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
|
| 94 |
+
"campaign_active_time campaign_attribution campaign_id configured_status created_time "
|
| 95 |
+
"creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
|
| 96 |
+
"effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
|
| 97 |
+
"issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
|
| 98 |
+
"lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
|
| 99 |
+
"optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
|
| 100 |
+
"rf_prediction_id source_adset source_adset_id start_time status targeting "
|
| 101 |
+
"targeting_optimization_types time_based_ad_rotation_id_blocks "
|
| 102 |
+
"time_based_ad_rotation_intervals updated_time use_new_app_click"
|
| 103 |
+
).split()
|
| 104 |
+
|
| 105 |
+
AD_FIELDS = (
|
| 106 |
+
"account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
|
| 107 |
+
"adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
|
| 108 |
+
"conversion_domain created_time creative demolink_hash display_sequence effective_status "
|
| 109 |
+
"engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
|
| 110 |
+
"preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
|
| 111 |
+
"tracking_and_conversion_with_defaults tracking_specs updated_time"
|
| 112 |
+
).split()
|
| 113 |
+
|
| 114 |
+
CREATIVE_FIELDS = (
|
| 115 |
+
"account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
|
| 116 |
+
"branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
|
| 117 |
+
"category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
|
| 118 |
+
"destination_set_id dynamic_ad_voice effective_authorization_category "
|
| 119 |
+
"effective_instagram_media_id effective_object_story_id enable_direct_install "
|
| 120 |
+
"enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
|
| 121 |
+
"instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
|
| 122 |
+
"link_deep_link_url link_destination_display_url link_og_id link_url "
|
| 123 |
+
"messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
|
| 124 |
+
"object_type object_url place_page_set_id platform_customizations playable_asset_id "
|
| 125 |
+
"portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
|
| 126 |
+
"template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
|
| 127 |
+
"use_page_actor_override video_id"
|
| 128 |
+
).split()
|
| 129 |
+
|
| 130 |
+
INSIGHT_FIELDS = (
|
| 131 |
+
"account_currency account_id account_name action_values actions ad_id ad_name adset_id "
|
| 132 |
+
"adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
|
| 133 |
+
"clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
|
| 134 |
+
"cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
|
| 135 |
+
"date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
|
| 136 |
+
"impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
|
| 137 |
+
"optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
|
| 138 |
+
"social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
|
| 139 |
+
"video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
|
| 140 |
+
"video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
|
| 141 |
+
"video_play_actions video_thruplay_watched_actions website_purchase_roas"
|
| 142 |
+
).split()
|
| 143 |
+
|
| 144 |
+
#: table -> (graph edge on the account | None, measured fields, parent column)
|
| 145 |
+
SPECS = {
|
| 146 |
+
"meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
|
| 147 |
+
"meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
|
| 148 |
+
"meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
|
| 149 |
+
"meta_ads": ("ads", AD_FIELDS, "adset_id"),
|
| 150 |
+
"meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
#: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights
|
| 154 |
+
#: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
|
| 155 |
+
INSIGHTS_TABLE = "meta_insights_daily"
|
| 156 |
+
INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
|
| 157 |
+
|
| 158 |
+
#: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
|
| 159 |
+
#: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
|
| 160 |
+
#: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the
|
| 161 |
+
#: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
|
| 162 |
+
#: page 25 answer 200. So the window is walked in slices with every column intact:
|
| 163 |
+
#: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows
|
| 164 |
+
#: 57 fields · account level · 7d -> 200, 7 rows
|
| 165 |
+
#: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over —
|
| 166 |
+
#: it drops columns R2 requires, and it does so invisibly.
|
| 167 |
+
INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
|
| 168 |
+
INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
|
| 169 |
+
INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _slices(days, size):
|
| 173 |
+
"""[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
|
| 174 |
+
from datetime import date, timedelta
|
| 175 |
+
end = date.today()
|
| 176 |
+
start = end - timedelta(days=max(1, days) - 1)
|
| 177 |
+
out, cur = [], start
|
| 178 |
+
while cur <= end:
|
| 179 |
+
stop = min(cur + timedelta(days=max(1, size) - 1), end)
|
| 180 |
+
out.append((cur.isoformat(), stop.isoformat()))
|
| 181 |
+
cur = stop + timedelta(days=1)
|
| 182 |
+
return out
|
| 183 |
+
|
| 184 |
+
#: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
|
| 185 |
+
#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
|
| 186 |
+
#: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it
|
| 187 |
+
#: is reported as a problem, never absorbed.
|
| 188 |
+
MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
|
| 189 |
+
PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
class MetaError(RuntimeError):
|
| 193 |
+
"""A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def token():
|
| 197 |
+
"""The Meta token from the environment or gitignored `platform/.env`; "" when absent.
|
| 198 |
+
|
| 199 |
+
⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
|
| 200 |
+
`platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
|
| 201 |
+
(`core` never imports up). Fifteen lines is the price of that boundary.
|
| 202 |
+
"""
|
| 203 |
+
tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
|
| 204 |
+
if tok:
|
| 205 |
+
return tok.strip()
|
| 206 |
+
env = _HERE.parent / ".env"
|
| 207 |
+
if env.exists():
|
| 208 |
+
for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 209 |
+
if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
|
| 210 |
+
_, _, v = line.partition("=")
|
| 211 |
+
return v.strip().strip('"').strip("'")
|
| 212 |
+
return ""
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _get(path, tok, **params):
|
| 216 |
+
params["access_token"] = tok
|
| 217 |
+
url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
|
| 218 |
+
req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
|
| 219 |
+
try:
|
| 220 |
+
with urllib.request.urlopen(req, timeout=120) as r:
|
| 221 |
+
return json.loads(r.read().decode("utf-8", "replace"))
|
| 222 |
+
except urllib.error.HTTPError as e:
|
| 223 |
+
try:
|
| 224 |
+
msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
|
| 225 |
+
).get("message") or ""
|
| 226 |
+
except Exception:
|
| 227 |
+
msg = ""
|
| 228 |
+
raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
|
| 229 |
+
except Exception as e:
|
| 230 |
+
raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
#: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
|
| 234 |
+
#: why it cannot be treated as "the server is broken, give up".
|
| 235 |
+
_TOO_MUCH = "reduce the amount of data"
|
| 236 |
+
|
| 237 |
+
#: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
|
| 238 |
+
#: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
|
| 239 |
+
#: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the
|
| 240 |
+
#: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
|
| 241 |
+
#: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
|
| 242 |
+
#: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
|
| 243 |
+
#: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
|
| 244 |
+
_RATE_LIMITED = "too many calls"
|
| 245 |
+
_RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
|
| 246 |
+
_RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _walk(path, tok, log, **params):
|
| 250 |
+
"""Every page of an edge, paged by CURSOR under our own parameters.
|
| 251 |
+
|
| 252 |
+
⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
|
| 253 |
+
first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
|
| 254 |
+
you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL
|
| 255 |
+
limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
|
| 256 |
+
Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
|
| 257 |
+
|
| 258 |
+
⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
|
| 259 |
+
silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
|
| 260 |
+
every column per row.
|
| 261 |
+
|
| 262 |
+
-> (rows, hit_page_cap)
|
| 263 |
+
"""
|
| 264 |
+
out, pages = [], 0
|
| 265 |
+
limit = int(params.pop("limit", None) or PAGE)
|
| 266 |
+
after, waited = None, 0
|
| 267 |
+
while True:
|
| 268 |
+
call = dict(params, limit=limit)
|
| 269 |
+
if after:
|
| 270 |
+
call["after"] = after
|
| 271 |
+
try:
|
| 272 |
+
body = _get(path, tok, **call)
|
| 273 |
+
except MetaError as e:
|
| 274 |
+
if _TOO_MUCH in str(e) and limit > 5:
|
| 275 |
+
limit = max(5, limit // 4)
|
| 276 |
+
log(f" page too heavy for /{path} - retrying at limit={limit} "
|
| 277 |
+
f"(a payload ceiling, not a row cap; every column is still asked for)")
|
| 278 |
+
continue
|
| 279 |
+
if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
|
| 280 |
+
waited += 1
|
| 281 |
+
log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
|
| 282 |
+
f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
|
| 283 |
+
time.sleep(_RATE_WAIT)
|
| 284 |
+
continue
|
| 285 |
+
if _RATE_LIMITED in str(e).lower():
|
| 286 |
+
log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
|
| 287 |
+
f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
|
| 288 |
+
f"limit is per ad account and persists for minutes. Fix: re-run "
|
| 289 |
+
f"`--sync --only <table>` later; the upsert is idempotent.")
|
| 290 |
+
return out, True
|
| 291 |
+
raise
|
| 292 |
+
out.extend(body.get("data") or [])
|
| 293 |
+
pages += 1
|
| 294 |
+
after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
|
| 295 |
+
has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
|
| 296 |
+
if not has_next:
|
| 297 |
+
return out, False
|
| 298 |
+
if pages >= MAX_PAGES:
|
| 299 |
+
log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
|
| 300 |
+
f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
|
| 301 |
+
f"window with META_INSIGHTS_DAYS.")
|
| 302 |
+
return out, True
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def _cell(value):
|
| 306 |
+
"""One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
|
| 307 |
+
being dropped: R2 says every field the API returns, and `targeting` is a field."""
|
| 308 |
+
if value is None or isinstance(value, (str, int, float, bool)):
|
| 309 |
+
return value
|
| 310 |
+
return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _coltype(name, rows=None):
|
| 314 |
+
"""The DuckDB type for one column — decided by the DATA when there is data, by the name only
|
| 315 |
+
as a fallback.
|
| 316 |
+
|
| 317 |
+
⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
|
| 318 |
+
`cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
|
| 319 |
+
action-type objects** on the Insights edge:
|
| 320 |
+
[{"action_type":"outbound_click","value":"1.459854"}]
|
| 321 |
+
Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
|
| 322 |
+
'[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked
|
| 323 |
+
like it worked and then blew up on the last table.
|
| 324 |
+
⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
|
| 325 |
+
do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
|
| 326 |
+
that is what `_cell` stores; anything else falls back to the measured name hints.
|
| 327 |
+
"""
|
| 328 |
+
if rows:
|
| 329 |
+
seen, kinds = 0, set()
|
| 330 |
+
for r in rows:
|
| 331 |
+
v = r.get(name)
|
| 332 |
+
if v is None or v == "":
|
| 333 |
+
continue
|
| 334 |
+
kinds.add("json" if isinstance(v, (list, dict)) else
|
| 335 |
+
"bool" if isinstance(v, bool) else
|
| 336 |
+
"int" if isinstance(v, int) else
|
| 337 |
+
"float" if isinstance(v, float) else "str")
|
| 338 |
+
seen += 1
|
| 339 |
+
if seen >= 200:
|
| 340 |
+
break
|
| 341 |
+
if kinds:
|
| 342 |
+
if "json" in kinds or "str" in kinds:
|
| 343 |
+
return "VARCHAR" # a JSON blob, or a numeric STRING
|
| 344 |
+
if kinds <= {"int", "bool"}:
|
| 345 |
+
return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
|
| 346 |
+
return "DOUBLE"
|
| 347 |
+
if name in _INT:
|
| 348 |
+
return "BIGINT"
|
| 349 |
+
if name in _DBL:
|
| 350 |
+
return "DOUBLE"
|
| 351 |
+
return "VARCHAR" # ids included — see the header
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def _ensure(con, table, fields, rows=None):
|
| 355 |
+
"""Create or widen the table, with every column typed from `rows` when they are available.
|
| 356 |
+
|
| 357 |
+
⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
|
| 358 |
+
column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type
|
| 359 |
+
disagreement drops and recreates rather than limping on with a column that refuses every
|
| 360 |
+
insert. The alternative is a mirror that is permanently unwritable for one bad guess.
|
| 361 |
+
"""
|
| 362 |
+
want = {f: _coltype(f, rows) for f in fields}
|
| 363 |
+
have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
|
| 364 |
+
if have:
|
| 365 |
+
clash = [f for f, ty in want.items() if f in have and have[f] != ty
|
| 366 |
+
and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
|
| 367 |
+
if clash:
|
| 368 |
+
con.execute(f"DROP TABLE {table}")
|
| 369 |
+
have = {}
|
| 370 |
+
if not have:
|
| 371 |
+
cols = ", ".join(f"{f} {want[f]}" for f in fields)
|
| 372 |
+
con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
|
| 373 |
+
if "id" not in fields else
|
| 374 |
+
f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
|
| 375 |
+
return
|
| 376 |
+
for f in fields:
|
| 377 |
+
if f not in have:
|
| 378 |
+
con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def _upsert(con, table, fields, rows):
|
| 382 |
+
"""Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so
|
| 383 |
+
a re-sync updates in place and can never append a second copy of the same object."""
|
| 384 |
+
if not rows:
|
| 385 |
+
return 0
|
| 386 |
+
cols = list(fields)
|
| 387 |
+
ids = [str(r.get("id") or "") for r in rows]
|
| 388 |
+
q = ",".join("?" for _ in ids)
|
| 389 |
+
con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
|
| 390 |
+
con.executemany(
|
| 391 |
+
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
|
| 392 |
+
[[_cell(r.get(c)) for c in cols] for r in rows])
|
| 393 |
+
return len(rows)
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
|
| 397 |
+
"""Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
|
| 398 |
+
|
| 399 |
+
Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
|
| 400 |
+
exactly as `odoo_relational.refresh` is.
|
| 401 |
+
"""
|
| 402 |
+
tok = tok or token()
|
| 403 |
+
# ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a
|
| 404 |
+
# live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
|
| 405 |
+
# `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
|
| 406 |
+
# imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
|
| 407 |
+
# that trips the per-ad-account rate limit and never finishes. The comment beside that call
|
| 408 |
+
# claimed a short window the code could not deliver.
|
| 409 |
+
# ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
|
| 410 |
+
# makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
|
| 411 |
+
days = int(insights_days or INSIGHTS_DAYS)
|
| 412 |
+
report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
|
| 413 |
+
if not tok:
|
| 414 |
+
report["problems"].append(
|
| 415 |
+
"META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
|
| 416 |
+
"was pulled. This is a missing CREDENTIAL, not a missing capability.")
|
| 417 |
+
return report
|
| 418 |
+
|
| 419 |
+
path = datastore.path_for(tenant_key)
|
| 420 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 421 |
+
datastore.use_path(path)
|
| 422 |
+
con = datastore.connect()
|
| 423 |
+
log(f" mirror: {Path(path).name}")
|
| 424 |
+
|
| 425 |
+
accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
|
| 426 |
+
report["accounts"] = [a.get("id") for a in accts]
|
| 427 |
+
if not accts:
|
| 428 |
+
report["problems"].append("the token reaches no ad accounts")
|
| 429 |
+
return report
|
| 430 |
+
|
| 431 |
+
# ── the five entity levels ────────────────────────────────────────────────────────────────
|
| 432 |
+
for table, (edge, fields, _parent) in SPECS.items():
|
| 433 |
+
rows, capped = [], False
|
| 434 |
+
for acct in accts:
|
| 435 |
+
aid = acct["id"]
|
| 436 |
+
if edge is None:
|
| 437 |
+
rows.append(_get(aid, tok, fields=",".join(fields)))
|
| 438 |
+
else:
|
| 439 |
+
got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
|
| 440 |
+
rows.extend(got)
|
| 441 |
+
capped = capped or hit
|
| 442 |
+
_ensure(con, table, fields, rows)
|
| 443 |
+
n = _upsert(con, table, fields, rows)
|
| 444 |
+
total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 445 |
+
report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 446 |
+
log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
|
| 447 |
+
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 448 |
+
if capped:
|
| 449 |
+
report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
|
| 450 |
+
|
| 451 |
+
# ── the daily Insights grain ──────────────────────────────────────────────────────────────
|
| 452 |
+
if insights:
|
| 453 |
+
rows, capped = [], False
|
| 454 |
+
for acct in accts:
|
| 455 |
+
for since, until in _slices(days, INSIGHTS_SLICE):
|
| 456 |
+
got, hit = _walk(f"{acct['id']}/insights", tok, log,
|
| 457 |
+
fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
|
| 458 |
+
time_increment="1", limit=INSIGHTS_PAGE,
|
| 459 |
+
time_range=json.dumps({"since": since, "until": until}))
|
| 460 |
+
rows.extend(got)
|
| 461 |
+
capped = capped or hit
|
| 462 |
+
# ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync
|
| 463 |
+
# appends a second copy of the same day — so it is composed from the grain itself.
|
| 464 |
+
for r in rows:
|
| 465 |
+
r["id"] = ":".join(str(r.get(k) or "") for k in
|
| 466 |
+
(f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
|
| 467 |
+
fields = ["id"] + INSIGHT_FIELDS
|
| 468 |
+
_ensure(con, INSIGHTS_TABLE, fields, rows)
|
| 469 |
+
n = _upsert(con, INSIGHTS_TABLE, fields, rows)
|
| 470 |
+
total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
|
| 471 |
+
report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 472 |
+
log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
|
| 473 |
+
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 474 |
+
|
| 475 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 476 |
+
["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
|
| 477 |
+
sum(t["in_mirror"] for t in report["tables"].values()),
|
| 478 |
+
time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 479 |
+
return report
|
| 480 |
+
|
| 481 |
+
|
| 482 |
+
def status(tenant_key="royal-imports"):
|
| 483 |
+
"""What is in the mirror right now, per table. Never fetches."""
|
| 484 |
+
path = datastore.path_for(tenant_key)
|
| 485 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 486 |
+
datastore.use_path(path)
|
| 487 |
+
out = {}
|
| 488 |
+
con = datastore.connect()
|
| 489 |
+
for table in list(SPECS) + [INSIGHTS_TABLE]:
|
| 490 |
+
try:
|
| 491 |
+
out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 492 |
+
except Exception:
|
| 493 |
+
out[table] = None # table absent = never synced
|
| 494 |
+
return out
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def main(argv=None):
|
| 498 |
+
ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
|
| 499 |
+
ap.add_argument("--sync", action="store_true")
|
| 500 |
+
ap.add_argument("--status", action="store_true")
|
| 501 |
+
ap.add_argument("--tenant", default="royal-imports")
|
| 502 |
+
ap.add_argument("--no-insights", action="store_true")
|
| 503 |
+
ap.add_argument("--insights-days", type=int, default=None,
|
| 504 |
+
help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
|
| 505 |
+
a = ap.parse_args(argv)
|
| 506 |
+
if a.status:
|
| 507 |
+
for k, v in status(a.tenant).items():
|
| 508 |
+
print(f" {k:<20} {'(never synced)' if v is None else v}")
|
| 509 |
+
return 0
|
| 510 |
+
if not a.sync:
|
| 511 |
+
ap.print_help()
|
| 512 |
+
return 2
|
| 513 |
+
rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
|
| 514 |
+
for p in rep["problems"]:
|
| 515 |
+
print(" PROBLEM:", p)
|
| 516 |
+
print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
|
| 517 |
+
return 1 if rep["problems"] else 0
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
if __name__ == "__main__":
|
| 521 |
+
sys.exit(main())
|
platform/harness/tools.py
CHANGED
|
@@ -65,22 +65,13 @@ def _err(msg):
|
|
| 65 |
# ------------------------------------------------------------------ schema tools
|
| 66 |
|
| 67 |
def list_topics():
|
| 68 |
-
"""The 'what data exists' tool.
|
| 69 |
-
|
| 70 |
-
W33-T49 adds `grid` — the store key of the database this topic describes, when it describes
|
| 71 |
-
one. It is what lets the agent connect "the thing the person is looking at" to "the topic I
|
| 72 |
-
can query", which is the whole point of the entity topics: before this, the semantic layer
|
| 73 |
-
could see `sale_order_line` and `account_move` and had never heard of `ut_odoo_customers`.
|
| 74 |
-
"""
|
| 75 |
out = []
|
| 76 |
for k, t in SEM.topics().items():
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
if t.get("grid"):
|
| 82 |
-
row["grid"] = t["grid"]
|
| 83 |
-
out.append(row)
|
| 84 |
return out
|
| 85 |
|
| 86 |
|
|
@@ -92,30 +83,10 @@ def describe_topic(topic):
|
|
| 92 |
mets = {k: {"label": m.get("label"), "description": m.get("description"),
|
| 93 |
"format": m.get("format"), "ai_context": m.get("ai_context")}
|
| 94 |
for k, m in SEM.metrics().items() if m["topic"] == topic}
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
# ⭐⭐ W33-T49 — THE FIELD SEMANTICS, which this tool did not carry AT ALL.
|
| 100 |
-
#
|
| 101 |
-
# ⛔ THE GAP WAS NOT THE TOPIC FILES, IT WAS THIS RETURN. Before this, `describe_topic`
|
| 102 |
-
# answered with scope, grain, dims, metrics and `ai_context` — and **no vocabulary of
|
| 103 |
-
# COLUMNS**. So an agent told to describe a database could name what it MEASURES and never
|
| 104 |
-
# what it HOLDS: the owner's item 13 asks for "ALL populated fields … including its semantic
|
| 105 |
-
# layer", and adding eight topic files without this would have left `describe_topic` returning
|
| 106 |
-
# the same field-free shape it always did, one screen away from the fields.
|
| 107 |
-
#
|
| 108 |
-
# ⚠ `kind` is the part worth reading, and it is the difference between a column an agent can
|
| 109 |
-
# FILTER and one it cannot: `data` is stored on the row; `rollup` is computed from another
|
| 110 |
-
# topic (`from_topic` names it, and that is where a windowed or reconciled number must come
|
| 111 |
-
# from); `link` points at another database. An agent that treats a rollup as a stored column
|
| 112 |
-
# will ask this topic for a number this topic does not hold.
|
| 113 |
-
if t.get("fields"):
|
| 114 |
-
out["fields"] = [dict(f) for f in t["fields"]]
|
| 115 |
-
for k in ("grid", "subject", "entity"):
|
| 116 |
-
if t.get(k):
|
| 117 |
-
out[k] = t[k]
|
| 118 |
-
return out
|
| 119 |
|
| 120 |
|
| 121 |
# ------------------------------------------------------------------ query tools
|
|
|
|
| 65 |
# ------------------------------------------------------------------ schema tools
|
| 66 |
|
| 67 |
def list_topics():
|
| 68 |
+
"""The 'what data exists' tool."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
out = []
|
| 70 |
for k, t in SEM.topics().items():
|
| 71 |
+
out.append({"topic": k, "label": t.get("label"), "entity": t.get("entity"),
|
| 72 |
+
"grain": t.get("grain"),
|
| 73 |
+
"dims": list((t.get("store") or {}).get("dims") or {}),
|
| 74 |
+
"metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]})
|
|
|
|
|
|
|
|
|
|
| 75 |
return out
|
| 76 |
|
| 77 |
|
|
|
|
| 83 |
mets = {k: {"label": m.get("label"), "description": m.get("description"),
|
| 84 |
"format": m.get("format"), "ai_context": m.get("ai_context")}
|
| 85 |
for k, m in SEM.metrics().items() if m["topic"] == topic}
|
| 86 |
+
return {"topic": topic, "label": t.get("label"), "scope": t.get("scope"),
|
| 87 |
+
"grain": t.get("grain"), "ai_context": t.get("ai_context"),
|
| 88 |
+
"dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()},
|
| 89 |
+
"metrics": mets}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
|
| 92 |
# ------------------------------------------------------------------ query tools
|
platform/modules/product_data.py
CHANGED
|
@@ -1,804 +1,797 @@
|
|
| 1 |
-
"""modules/product_data.py — the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
|
| 2 |
-
|
| 3 |
-
The second object on the table-page factory: same grid, same engine, same permission wall — the
|
| 4 |
-
only difference is the field schema and the identity. `modules/customer_data.pool()` is the
|
| 5 |
-
template and this deliberately mirrors its signature and its row shape.
|
| 6 |
-
|
| 7 |
-
⛔ THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
|
| 8 |
-
|
| 9 |
-
1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
|
| 10 |
-
Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
|
| 11 |
-
integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
|
| 12 |
-
same id on every pull and across processes — never an enumeration index, which would reshuffle
|
| 13 |
-
whenever the catalogue changes) and `code`, the real business key, as a visible column.
|
| 14 |
-
`_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
|
| 15 |
-
silently merge in every downstream set operation, and a loud build failure is the only version
|
| 16 |
-
of that anyone would notice.
|
| 17 |
-
|
| 18 |
-
2. **THE SCOPE RULE — inventory columns are CONSOLIDATED and are therefore OMITTED for a
|
| 19 |
-
BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
|
| 20 |
-
is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
|
| 21 |
-
brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
|
| 22 |
-
company-wide stock IN THE SAME ROW — the mixed-scope value defect wave 15's amendment 3 exists
|
| 23 |
-
for, arriving through a different door. The honest options were "omit" or "label the columns
|
| 24 |
-
company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
|
| 25 |
-
whereas a column that is silently company-wide answers one wrongly.
|
| 26 |
-
|
| 27 |
-
3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
|
| 28 |
-
COUNTRY are not in Odoo at all — they live in the inventory WORKBOOK
|
| 29 |
-
([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
|
| 30 |
-
Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
|
| 31 |
-
allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
|
| 32 |
-
does not pretend to cover columns that are absent.
|
| 33 |
-
|
| 34 |
-
4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
|
| 35 |
-
2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the
|
| 36 |
-
union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
|
| 37 |
-
showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
|
| 38 |
-
|
| 39 |
-
· ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
|
| 40 |
-
window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
|
| 41 |
-
that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
|
| 42 |
-
~2,550 active SKUs have never sold in wholesale scope at all.
|
| 43 |
-
· ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
|
| 44 |
-
revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and
|
| 45 |
-
not this one, and that zero is a measurement. A row that appears in NO revenue read
|
| 46 |
-
carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
|
| 47 |
-
client renders as an empty cell. Blank is an admission; zero is a measurement.
|
| 48 |
-
· ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
|
| 49 |
-
that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
|
| 50 |
-
UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
|
| 51 |
-
carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two
|
| 52 |
-
sources are JOINED rather than merged.
|
| 53 |
-
"""
|
| 54 |
-
import json
|
| 55 |
-
import zlib
|
| 56 |
-
from pathlib import Path
|
| 57 |
-
|
| 58 |
-
import core.odoo as O
|
| 59 |
-
import core.periods as P
|
| 60 |
-
import core.shared_overlay as shared_overlay
|
| 61 |
-
import core.table_store as table_store
|
| 62 |
-
import modules.products as products
|
| 63 |
-
|
| 64 |
-
#: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
|
| 65 |
-
#: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
|
| 66 |
-
#: signal built on a stock number the caller cannot see would be a recommendation nobody could
|
| 67 |
-
#: check ([[no-unverifiable-aggregates]]).
|
| 68 |
-
#:
|
| 69 |
-
#: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
|
| 70 |
-
#: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
|
| 71 |
-
#: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field —
|
| 72 |
-
#: computed as `validate()`'s oracle, projected onto no wire — so it belongs here for the first
|
| 73 |
-
#: reader and is inert for the second. Removing it would stop the scope gate proving that a
|
| 74 |
-
#: BU-scoped pull withholds it.
|
| 75 |
-
#:
|
| 76 |
-
#: ⭐⭐ OWNER RULING 2026-08-11 — THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
|
| 77 |
-
#: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
|
| 78 |
-
#: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
|
| 79 |
-
#: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
|
| 80 |
-
#: what a business unit can actually shape:
|
| 81 |
-
#:
|
| 82 |
-
#: * `on_hand` / `unit_cost` / `inv_value` — ONE physical warehouse, no Fisch shelf and no
|
| 83 |
-
#: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
|
| 84 |
-
#: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` — all SALES-derived, so a
|
| 85 |
-
#: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
|
| 86 |
-
#: Sales from Fisch")
|
| 87 |
-
#:
|
| 88 |
-
#: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
|
| 89 |
-
#: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
|
| 90 |
-
#: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
|
| 91 |
-
#: asserts the columns are WITHHELD — it asserts the split above, which is a stronger claim and
|
| 92 |
-
#: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
|
| 93 |
-
CONSOLIDATED_ONLY = ()
|
| 94 |
-
|
| 95 |
-
#: ⛔ POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD — they must never reach a browser.
|
| 96 |
-
#:
|
| 97 |
-
#: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
|
| 98 |
-
#: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
|
| 99 |
-
#: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
|
| 100 |
-
#: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
|
| 101 |
-
#:
|
| 102 |
-
#: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
|
| 103 |
-
#: Field while keeping its computation, and the gate — which was asserting a FIELD property from
|
| 104 |
-
#: a ROW-key list — went red with no way to tell a deliberate divergence from a dropped column.
|
| 105 |
-
UNSHIPPED_ROW_KEYS = ("buy_now",)
|
| 106 |
-
|
| 107 |
-
#: Wave 17 (owner item 13, ruling R3) — the SUPPLIER MASTER, from the curated mastersheet map
|
| 108 |
-
#: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
|
| 109 |
-
#: the owner confirmed this data came from the Inventory System Mastersheet, which is why
|
| 110 |
-
#: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
|
| 111 |
-
#: quietly missing" and needed "a loader this module deliberately does not invent". This is
|
| 112 |
-
#: that loader.
|
| 113 |
-
#:
|
| 114 |
-
#: ⛔ THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
|
| 115 |
-
#: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
|
| 116 |
-
#: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
|
| 117 |
-
#: filtered on a user-created column would, for every OTHER account, name a column that does not
|
| 118 |
-
#: exist — and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
|
| 119 |
-
#: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
|
| 120 |
-
#: its author. Contract columns are identical for every reader, so the view means one thing.
|
| 121 |
-
#:
|
| 122 |
-
#: ⭐⭐ 2026-08-12 (W30-T36) — THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
|
| 123 |
-
#: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
|
| 124 |
-
#: It was parked because a per-user column silently WIDENS a shared view — not because editing was
|
| 125 |
-
#: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
|
| 126 |
-
#: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
|
| 127 |
-
#: still means ONE thing to every reader and a shared view still filters honestly.
|
| 128 |
-
#: ⇒ The four columns below are now `source: "overlay"` + `shared: true` in the canonical
|
| 129 |
-
#: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
|
| 130 |
-
#: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
|
| 131 |
-
_SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
|
| 132 |
-
_SUPPLIER_CACHE = {}
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
def supplier_master():
|
| 136 |
-
"""`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
|
| 137 |
-
|
| 138 |
-
Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
|
| 139 |
-
that will not render because a master map is missing is a worse failure than one with blank
|
| 140 |
-
supplier columns.
|
| 141 |
-
"""
|
| 142 |
-
if _SUPPLIER_CACHE:
|
| 143 |
-
return _SUPPLIER_CACHE
|
| 144 |
-
try:
|
| 145 |
-
raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
|
| 146 |
-
except Exception:
|
| 147 |
-
return {}
|
| 148 |
-
for code, meta in (raw or {}).items():
|
| 149 |
-
if not isinstance(meta, dict):
|
| 150 |
-
continue
|
| 151 |
-
lead = meta.get("lead")
|
| 152 |
-
_SUPPLIER_CACHE[str(code)] = {
|
| 153 |
-
"supplier": (meta.get("vendor") or "") or None,
|
| 154 |
-
"lead_days": int(lead) if isinstance(lead, (int, float)) else None,
|
| 155 |
-
"origin_country": (meta.get("country") or "") or None,
|
| 156 |
-
"first_cost": meta.get("first_cost"),
|
| 157 |
-
}
|
| 158 |
-
return _SUPPLIER_CACHE
|
| 159 |
-
|
| 160 |
-
#: Wave 16 C-TOPIC — the PRODUCT table's OWN workspace bucket. ⛔ Never the customer one:
|
| 161 |
-
#: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
|
| 162 |
-
#: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
|
| 163 |
-
#: customer (or the reverse). Separate store keys make that structurally impossible, which is
|
| 164 |
-
#: the whole reason the table-page factory exists ("a new table object gets its own
|
| 165 |
-
#: table_store.make('<its>_table_workspace')").
|
| 166 |
-
TABLE_KEY = 'product_table_workspace'
|
| 167 |
-
|
| 168 |
-
_GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
|
| 169 |
-
_SHARED_KEYS = None
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
def SHARED_KEYS():
|
| 173 |
-
"""The product columns whose values are TENANT-WIDE — derived from the canonical contract's
|
| 174 |
-
own `shared: true`, never typed out a second time.
|
| 175 |
-
|
| 176 |
-
⛔ IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
|
| 177 |
-
write into the per-user stratum with nothing going wrong anywhere — the widening defect
|
| 178 |
-
reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
|
| 179 |
-
canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
|
| 180 |
-
same file with no guard), so a raise here costs nothing that was still working.
|
| 181 |
-
"""
|
| 182 |
-
global _SHARED_KEYS
|
| 183 |
-
if _SHARED_KEYS is None:
|
| 184 |
-
doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
|
| 185 |
-
_SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
|
| 186 |
-
if f.get('shared'))
|
| 187 |
-
return _SHARED_KEYS
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
class _ProductTableStore(table_store.TableStore):
|
| 191 |
-
"""The product workspace, with the SHARED columns routed to the tenant-wide stratum.
|
| 192 |
-
|
| 193 |
-
⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
|
| 194 |
-
THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** — measured, zero
|
| 195 |
-
call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` → `grid_events.
|
| 196 |
-
handle_one` → `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
|
| 197 |
-
`routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
|
| 198 |
-
doors pass through; intercepting at either route would have left the other one writing a
|
| 199 |
-
per-user value that only its author could see.
|
| 200 |
-
|
| 201 |
-
⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
|
| 202 |
-
handle as the per-user one it sits beside — `_tops`' own comment explains that a split, where
|
| 203 |
-
one side is tenant-scoped and the other is not, is worse than a stated residency error because
|
| 204 |
-
a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
|
| 205 |
-
together the day that singleton gains a tenant handle.
|
| 206 |
-
"""
|
| 207 |
-
|
| 208 |
-
def patch_overlay(self, username, pid, updates):
|
| 209 |
-
clean = dict(updates or {})
|
| 210 |
-
if not clean:
|
| 211 |
-
return
|
| 212 |
-
keys = set(SHARED_KEYS())
|
| 213 |
-
shared = {k: v for k, v in clean.items() if k in keys}
|
| 214 |
-
personal = {k: v for k, v in clean.items() if k not in keys}
|
| 215 |
-
if shared:
|
| 216 |
-
shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
|
| 217 |
-
if personal:
|
| 218 |
-
super().patch_overlay(username, pid, personal)
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
TABLE_OPS = _ProductTableStore(TABLE_KEY)
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
def shared_cells(pids, st=None):
|
| 225 |
-
"""`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
|
| 226 |
-
|
| 227 |
-
⚠ `pids` is required and positional all the way down — `shared_overlay.cells` refuses to serve
|
| 228 |
-
"everything" by design, and the caller here always holds an already-scoped pool.
|
| 229 |
-
"""
|
| 230 |
-
return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
def sku_pid(code):
|
| 234 |
-
"""A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
|
| 235 |
-
always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
|
| 236 |
-
return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
def _assert_no_pid_collision(rows):
|
| 240 |
-
"""Two SKUs sharing a pid would MERGE in every set operation downstream — cohort membership,
|
| 241 |
-
allowed_pids, the measure channel — and nothing would report it. Fail the build instead."""
|
| 242 |
-
seen = {}
|
| 243 |
-
for r in rows:
|
| 244 |
-
prior = seen.get(r["pid"])
|
| 245 |
-
if prior is not None and prior != r["code"]:
|
| 246 |
-
raise ValueError(
|
| 247 |
-
f"product_data: pid collision — {prior!r} and {r['code']!r} both hash to "
|
| 248 |
-
f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
|
| 249 |
-
f"before shipping this catalogue.")
|
| 250 |
-
seen[r["pid"]] = r["code"]
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
def _inventory_by_code(t):
|
| 254 |
-
"""`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
|
| 255 |
-
|
| 256 |
-
Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
|
| 257 |
-
its own slow families: a product table that will not render because inventory is momentarily
|
| 258 |
-
unreachable is a worse failure than one with blank stock columns.
|
| 259 |
-
"""
|
| 260 |
-
try:
|
| 261 |
-
import modules.inventory as inventory
|
| 262 |
-
return inventory.sku_inventory(t=t) or {}
|
| 263 |
-
except Exception:
|
| 264 |
-
return {}
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
def _bu_ltm_share(t, team_id):
|
| 268 |
-
"""`{code: 0.0..1.0}` — this unit's SHARE of the SKU's last-twelve-months units.
|
| 269 |
-
|
| 270 |
-
⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
|
| 271 |
-
inventory block has to be re-shaped by a BU fact, and this is that fact.
|
| 272 |
-
|
| 273 |
-
⛔ A SHARE, NOT THE UNIT COUNT ITSELF — AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
|
| 274 |
-
version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
|
| 275 |
-
leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
|
| 276 |
-
question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
|
| 277 |
-
check found **Fisch's LTM units EXCEEDING the company's** — a subset larger than its superset,
|
| 278 |
-
which no reader could explain and no reconciliation could survive.
|
| 279 |
-
|
| 280 |
-
Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
|
| 281 |
-
construction and the scoped figure can never exceed the consolidated one. It also leaves the
|
| 282 |
-
consolidated column exactly as it was — the Inventory page and the Product grid still agree
|
| 283 |
-
about company-wide units, which is what "leave the rest" asked for.
|
| 284 |
-
|
| 285 |
-
`all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
|
| 286 |
-
"this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
|
| 287 |
-
|
| 288 |
-
Degrades to `{}` on any failure, matching `_inventory_by_code` — and a missing code then takes
|
| 289 |
-
the `_DEFAULT_SHARE` below rather than a silent 0.
|
| 290 |
-
"""
|
| 291 |
-
try:
|
| 292 |
-
f, to = P.ltm(t)
|
| 293 |
-
allq = {code: (r.get('qty') or 0.0)
|
| 294 |
-
for code, r in (products._sku_rev(f, to, None) or {}).items()}
|
| 295 |
-
buq = {code: (r.get('qty') or 0.0)
|
| 296 |
-
for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
|
| 297 |
-
except Exception:
|
| 298 |
-
return {}
|
| 299 |
-
out = {}
|
| 300 |
-
for code, total in allq.items():
|
| 301 |
-
out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
|
| 302 |
-
return out
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
#: What a SKU absent from the LTM sales read is worth to a business unit. ZERO — it did not sell
|
| 306 |
-
#: in anybody's book over the window, so no unit can claim its velocity. Named rather than
|
| 307 |
-
#: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
|
| 308 |
-
#: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
|
| 309 |
-
_DEFAULT_SHARE = 0.0
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
def _rescope_inventory(e, share):
|
| 313 |
-
"""`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
|
| 314 |
-
|
| 315 |
-
The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator — NOT a second
|
| 316 |
-
idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
|
| 317 |
-
copies of a threshold table is how the Product grid and the Inventory page start disagreeing
|
| 318 |
-
about which SKUs are dead.
|
| 319 |
-
|
| 320 |
-
⚠ `on_hand` is whatever the warehouse holds, unscoped — so a Fisch reader's `dos` answers
|
| 321 |
-
"how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
|
| 322 |
-
actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
|
| 323 |
-
and inventing one would put a number on screen that no Odoo query could reproduce.
|
| 324 |
-
"""
|
| 325 |
-
on_hand = e.get("on_hand")
|
| 326 |
-
if on_hand is None:
|
| 327 |
-
return None, None, None # no inventory row for this SKU: blank, never zero
|
| 328 |
-
qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
|
| 329 |
-
daily = qty / 365.0
|
| 330 |
-
if daily > 0:
|
| 331 |
-
dos_raw = on_hand / daily
|
| 332 |
-
else:
|
| 333 |
-
dos_raw = float('inf') if on_hand > 0 else 0.0
|
| 334 |
-
try:
|
| 335 |
-
import modules.inventory as inventory
|
| 336 |
-
bucket = inventory._bucket(dos_raw, on_hand, qty)
|
| 337 |
-
except Exception:
|
| 338 |
-
bucket = None
|
| 339 |
-
return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
def _catalogue_by_code():
|
| 343 |
-
"""`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from.
|
| 344 |
-
|
| 345 |
-
⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
|
| 346 |
-
Inventory degrades to `{}` because a product table with blank stock columns beats one that
|
| 347 |
-
will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that
|
| 348 |
-
failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
|
| 349 |
-
green and nothing on screen saying so, which is the defect this seam exists to end
|
| 350 |
-
([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
|
| 351 |
-
into a 503 that names the cause, so the loud failure has somewhere honest to land.
|
| 352 |
-
|
| 353 |
-
It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
|
| 354 |
-
same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
|
| 355 |
-
to build a pool without Odoo.
|
| 356 |
-
"""
|
| 357 |
-
return products.catalogue()
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
def _pricelist_by_code():
|
| 361 |
-
"""`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`.
|
| 362 |
-
|
| 363 |
-
A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
|
| 364 |
-
(these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
|
| 365 |
-
pool without Odoo. ⛔ Unstubbed there, section H would reach live Odoo through the back door
|
| 366 |
-
and the whole file would stop being runnable offline.
|
| 367 |
-
"""
|
| 368 |
-
try:
|
| 369 |
-
return products.pricelist_by_code()
|
| 370 |
-
except Exception as e:
|
| 371 |
-
return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
def pool(team_id=None, t=None):
|
| 375 |
-
"""One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
|
| 376 |
-
|
| 377 |
-
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
|
| 378 |
-
is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
|
| 379 |
-
that sold in its window and contributes NO rows of its own.
|
| 380 |
-
|
| 381 |
-
`team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
|
| 382 |
-
passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
|
| 383 |
-
driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
|
| 384 |
-
catalogue has no team.
|
| 385 |
-
"""
|
| 386 |
-
t = t or P.today()
|
| 387 |
-
consolidated = team_id is None
|
| 388 |
-
# ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
|
| 389 |
-
# itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
|
| 390 |
-
inv = _inventory_by_code(t)
|
| 391 |
-
bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
|
| 392 |
-
sup = supplier_master()
|
| 393 |
-
prices, _price_report = _pricelist_by_code()
|
| 394 |
-
cat = _catalogue_by_code()
|
| 395 |
-
# The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
|
| 396 |
-
# does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those
|
| 397 |
-
# out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
|
| 398 |
-
# that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
|
| 399 |
-
rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
|
| 400 |
-
|
| 401 |
-
rows = []
|
| 402 |
-
for code, meta in cat.items():
|
| 403 |
-
r = rev.get(code)
|
| 404 |
-
row = {
|
| 405 |
-
"pid": sku_pid(code),
|
| 406 |
-
"code": code,
|
| 407 |
-
# ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's
|
| 408 |
-
# m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
|
| 409 |
-
# record's `display_name` is the current truth, and it is the same string for every
|
| 410 |
-
# SKU that is not re-SKUed. `directory()` derives the category identically.
|
| 411 |
-
"product": meta.get("product") or code,
|
| 412 |
-
"category": meta.get("category") or "(uncategorized)",
|
| 413 |
-
# ⭐
|
| 414 |
-
#
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
"
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
#
|
| 429 |
-
#
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
#
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
"
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
qty_ltm
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
#
|
| 464 |
-
#
|
| 465 |
-
#
|
| 466 |
-
#
|
| 467 |
-
#
|
| 468 |
-
#
|
| 469 |
-
#
|
| 470 |
-
#
|
| 471 |
-
#
|
| 472 |
-
#
|
| 473 |
-
#
|
| 474 |
-
#
|
| 475 |
-
#
|
| 476 |
-
#
|
| 477 |
-
#
|
| 478 |
-
#
|
| 479 |
-
#
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
#:
|
| 506 |
-
#:
|
| 507 |
-
#:
|
| 508 |
-
#:
|
| 509 |
-
#:
|
| 510 |
-
#:
|
| 511 |
-
#:
|
| 512 |
-
#:
|
| 513 |
-
#:
|
| 514 |
-
#:
|
| 515 |
-
#:
|
| 516 |
-
#:
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
#
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
for
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
#
|
| 707 |
-
#
|
| 708 |
-
#
|
| 709 |
-
#
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
#
|
| 753 |
-
#
|
| 754 |
-
#
|
| 755 |
-
#
|
| 756 |
-
#
|
| 757 |
-
# engine
|
| 758 |
-
#
|
| 759 |
-
#
|
| 760 |
-
#
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
|
| 772 |
-
|
| 773 |
-
|
| 774 |
-
|
| 775 |
-
|
| 776 |
-
|
| 777 |
-
#
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
"detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
|
| 799 |
-
"only_in_signal": sorted(signal_rows - view_rows)[:10],
|
| 800 |
-
# Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
|
| 801 |
-
# have silently dropped. 0 today does not make that filter correct.
|
| 802 |
-
"cover_gap_rounds_to_zero": len(rounding_would_miss)},
|
| 803 |
-
})
|
| 804 |
-
return checks
|
|
|
|
| 1 |
+
"""modules/product_data.py — the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
|
| 2 |
+
|
| 3 |
+
The second object on the table-page factory: same grid, same engine, same permission wall — the
|
| 4 |
+
only difference is the field schema and the identity. `modules/customer_data.pool()` is the
|
| 5 |
+
template and this deliberately mirrors its signature and its row shape.
|
| 6 |
+
|
| 7 |
+
⛔ THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
|
| 8 |
+
|
| 9 |
+
1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
|
| 10 |
+
Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
|
| 11 |
+
integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
|
| 12 |
+
same id on every pull and across processes — never an enumeration index, which would reshuffle
|
| 13 |
+
whenever the catalogue changes) and `code`, the real business key, as a visible column.
|
| 14 |
+
`_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
|
| 15 |
+
silently merge in every downstream set operation, and a loud build failure is the only version
|
| 16 |
+
of that anyone would notice.
|
| 17 |
+
|
| 18 |
+
2. **THE SCOPE RULE — inventory columns are CONSOLIDATED and are therefore OMITTED for a
|
| 19 |
+
BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
|
| 20 |
+
is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
|
| 21 |
+
brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
|
| 22 |
+
company-wide stock IN THE SAME ROW — the mixed-scope value defect wave 15's amendment 3 exists
|
| 23 |
+
for, arriving through a different door. The honest options were "omit" or "label the columns
|
| 24 |
+
company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
|
| 25 |
+
whereas a column that is silently company-wide answers one wrongly.
|
| 26 |
+
|
| 27 |
+
3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
|
| 28 |
+
COUNTRY are not in Odoo at all — they live in the inventory WORKBOOK
|
| 29 |
+
([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
|
| 30 |
+
Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
|
| 31 |
+
allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
|
| 32 |
+
does not pretend to cover columns that are absent.
|
| 33 |
+
|
| 34 |
+
4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
|
| 35 |
+
2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the
|
| 36 |
+
union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
|
| 37 |
+
showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
|
| 38 |
+
|
| 39 |
+
· ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
|
| 40 |
+
window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
|
| 41 |
+
that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
|
| 42 |
+
~2,550 active SKUs have never sold in wholesale scope at all.
|
| 43 |
+
· ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
|
| 44 |
+
revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and
|
| 45 |
+
not this one, and that zero is a measurement. A row that appears in NO revenue read
|
| 46 |
+
carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
|
| 47 |
+
client renders as an empty cell. Blank is an admission; zero is a measurement.
|
| 48 |
+
· ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
|
| 49 |
+
that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
|
| 50 |
+
UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
|
| 51 |
+
carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two
|
| 52 |
+
sources are JOINED rather than merged.
|
| 53 |
+
"""
|
| 54 |
+
import json
|
| 55 |
+
import zlib
|
| 56 |
+
from pathlib import Path
|
| 57 |
+
|
| 58 |
+
import core.odoo as O
|
| 59 |
+
import core.periods as P
|
| 60 |
+
import core.shared_overlay as shared_overlay
|
| 61 |
+
import core.table_store as table_store
|
| 62 |
+
import modules.products as products
|
| 63 |
+
|
| 64 |
+
#: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
|
| 65 |
+
#: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
|
| 66 |
+
#: signal built on a stock number the caller cannot see would be a recommendation nobody could
|
| 67 |
+
#: check ([[no-unverifiable-aggregates]]).
|
| 68 |
+
#:
|
| 69 |
+
#: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
|
| 70 |
+
#: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
|
| 71 |
+
#: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field —
|
| 72 |
+
#: computed as `validate()`'s oracle, projected onto no wire — so it belongs here for the first
|
| 73 |
+
#: reader and is inert for the second. Removing it would stop the scope gate proving that a
|
| 74 |
+
#: BU-scoped pull withholds it.
|
| 75 |
+
#:
|
| 76 |
+
#: ⭐⭐ OWNER RULING 2026-08-11 — THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
|
| 77 |
+
#: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
|
| 78 |
+
#: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
|
| 79 |
+
#: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
|
| 80 |
+
#: what a business unit can actually shape:
|
| 81 |
+
#:
|
| 82 |
+
#: * `on_hand` / `unit_cost` / `inv_value` — ONE physical warehouse, no Fisch shelf and no
|
| 83 |
+
#: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
|
| 84 |
+
#: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` — all SALES-derived, so a
|
| 85 |
+
#: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
|
| 86 |
+
#: Sales from Fisch")
|
| 87 |
+
#:
|
| 88 |
+
#: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
|
| 89 |
+
#: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
|
| 90 |
+
#: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
|
| 91 |
+
#: asserts the columns are WITHHELD — it asserts the split above, which is a stronger claim and
|
| 92 |
+
#: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
|
| 93 |
+
CONSOLIDATED_ONLY = ()
|
| 94 |
+
|
| 95 |
+
#: ⛔ POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD — they must never reach a browser.
|
| 96 |
+
#:
|
| 97 |
+
#: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
|
| 98 |
+
#: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
|
| 99 |
+
#: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
|
| 100 |
+
#: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
|
| 101 |
+
#:
|
| 102 |
+
#: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
|
| 103 |
+
#: Field while keeping its computation, and the gate — which was asserting a FIELD property from
|
| 104 |
+
#: a ROW-key list — went red with no way to tell a deliberate divergence from a dropped column.
|
| 105 |
+
UNSHIPPED_ROW_KEYS = ("buy_now",)
|
| 106 |
+
|
| 107 |
+
#: Wave 17 (owner item 13, ruling R3) — the SUPPLIER MASTER, from the curated mastersheet map
|
| 108 |
+
#: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
|
| 109 |
+
#: the owner confirmed this data came from the Inventory System Mastersheet, which is why
|
| 110 |
+
#: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
|
| 111 |
+
#: quietly missing" and needed "a loader this module deliberately does not invent". This is
|
| 112 |
+
#: that loader.
|
| 113 |
+
#:
|
| 114 |
+
#: ⛔ THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
|
| 115 |
+
#: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
|
| 116 |
+
#: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
|
| 117 |
+
#: filtered on a user-created column would, for every OTHER account, name a column that does not
|
| 118 |
+
#: exist — and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
|
| 119 |
+
#: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
|
| 120 |
+
#: its author. Contract columns are identical for every reader, so the view means one thing.
|
| 121 |
+
#:
|
| 122 |
+
#: ⭐⭐ 2026-08-12 (W30-T36) — THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
|
| 123 |
+
#: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
|
| 124 |
+
#: It was parked because a per-user column silently WIDENS a shared view — not because editing was
|
| 125 |
+
#: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
|
| 126 |
+
#: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
|
| 127 |
+
#: still means ONE thing to every reader and a shared view still filters honestly.
|
| 128 |
+
#: ⇒ The four columns below are now `source: "overlay"` + `shared: true` in the canonical
|
| 129 |
+
#: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
|
| 130 |
+
#: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
|
| 131 |
+
_SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
|
| 132 |
+
_SUPPLIER_CACHE = {}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def supplier_master():
|
| 136 |
+
"""`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
|
| 137 |
+
|
| 138 |
+
Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
|
| 139 |
+
that will not render because a master map is missing is a worse failure than one with blank
|
| 140 |
+
supplier columns.
|
| 141 |
+
"""
|
| 142 |
+
if _SUPPLIER_CACHE:
|
| 143 |
+
return _SUPPLIER_CACHE
|
| 144 |
+
try:
|
| 145 |
+
raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
|
| 146 |
+
except Exception:
|
| 147 |
+
return {}
|
| 148 |
+
for code, meta in (raw or {}).items():
|
| 149 |
+
if not isinstance(meta, dict):
|
| 150 |
+
continue
|
| 151 |
+
lead = meta.get("lead")
|
| 152 |
+
_SUPPLIER_CACHE[str(code)] = {
|
| 153 |
+
"supplier": (meta.get("vendor") or "") or None,
|
| 154 |
+
"lead_days": int(lead) if isinstance(lead, (int, float)) else None,
|
| 155 |
+
"origin_country": (meta.get("country") or "") or None,
|
| 156 |
+
"first_cost": meta.get("first_cost"),
|
| 157 |
+
}
|
| 158 |
+
return _SUPPLIER_CACHE
|
| 159 |
+
|
| 160 |
+
#: Wave 16 C-TOPIC — the PRODUCT table's OWN workspace bucket. ⛔ Never the customer one:
|
| 161 |
+
#: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
|
| 162 |
+
#: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
|
| 163 |
+
#: customer (or the reverse). Separate store keys make that structurally impossible, which is
|
| 164 |
+
#: the whole reason the table-page factory exists ("a new table object gets its own
|
| 165 |
+
#: table_store.make('<its>_table_workspace')").
|
| 166 |
+
TABLE_KEY = 'product_table_workspace'
|
| 167 |
+
|
| 168 |
+
_GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
|
| 169 |
+
_SHARED_KEYS = None
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def SHARED_KEYS():
|
| 173 |
+
"""The product columns whose values are TENANT-WIDE — derived from the canonical contract's
|
| 174 |
+
own `shared: true`, never typed out a second time.
|
| 175 |
+
|
| 176 |
+
⛔ IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
|
| 177 |
+
write into the per-user stratum with nothing going wrong anywhere — the widening defect
|
| 178 |
+
reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
|
| 179 |
+
canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
|
| 180 |
+
same file with no guard), so a raise here costs nothing that was still working.
|
| 181 |
+
"""
|
| 182 |
+
global _SHARED_KEYS
|
| 183 |
+
if _SHARED_KEYS is None:
|
| 184 |
+
doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
|
| 185 |
+
_SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
|
| 186 |
+
if f.get('shared'))
|
| 187 |
+
return _SHARED_KEYS
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
class _ProductTableStore(table_store.TableStore):
|
| 191 |
+
"""The product workspace, with the SHARED columns routed to the tenant-wide stratum.
|
| 192 |
+
|
| 193 |
+
⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
|
| 194 |
+
THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** — measured, zero
|
| 195 |
+
call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` → `grid_events.
|
| 196 |
+
handle_one` → `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
|
| 197 |
+
`routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
|
| 198 |
+
doors pass through; intercepting at either route would have left the other one writing a
|
| 199 |
+
per-user value that only its author could see.
|
| 200 |
+
|
| 201 |
+
⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
|
| 202 |
+
handle as the per-user one it sits beside — `_tops`' own comment explains that a split, where
|
| 203 |
+
one side is tenant-scoped and the other is not, is worse than a stated residency error because
|
| 204 |
+
a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
|
| 205 |
+
together the day that singleton gains a tenant handle.
|
| 206 |
+
"""
|
| 207 |
+
|
| 208 |
+
def patch_overlay(self, username, pid, updates):
|
| 209 |
+
clean = dict(updates or {})
|
| 210 |
+
if not clean:
|
| 211 |
+
return
|
| 212 |
+
keys = set(SHARED_KEYS())
|
| 213 |
+
shared = {k: v for k, v in clean.items() if k in keys}
|
| 214 |
+
personal = {k: v for k, v in clean.items() if k not in keys}
|
| 215 |
+
if shared:
|
| 216 |
+
shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
|
| 217 |
+
if personal:
|
| 218 |
+
super().patch_overlay(username, pid, personal)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
TABLE_OPS = _ProductTableStore(TABLE_KEY)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def shared_cells(pids, st=None):
|
| 225 |
+
"""`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
|
| 226 |
+
|
| 227 |
+
⚠ `pids` is required and positional all the way down — `shared_overlay.cells` refuses to serve
|
| 228 |
+
"everything" by design, and the caller here always holds an already-scoped pool.
|
| 229 |
+
"""
|
| 230 |
+
return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def sku_pid(code):
|
| 234 |
+
"""A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
|
| 235 |
+
always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
|
| 236 |
+
return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _assert_no_pid_collision(rows):
|
| 240 |
+
"""Two SKUs sharing a pid would MERGE in every set operation downstream — cohort membership,
|
| 241 |
+
allowed_pids, the measure channel — and nothing would report it. Fail the build instead."""
|
| 242 |
+
seen = {}
|
| 243 |
+
for r in rows:
|
| 244 |
+
prior = seen.get(r["pid"])
|
| 245 |
+
if prior is not None and prior != r["code"]:
|
| 246 |
+
raise ValueError(
|
| 247 |
+
f"product_data: pid collision — {prior!r} and {r['code']!r} both hash to "
|
| 248 |
+
f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
|
| 249 |
+
f"before shipping this catalogue.")
|
| 250 |
+
seen[r["pid"]] = r["code"]
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _inventory_by_code(t):
|
| 254 |
+
"""`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
|
| 255 |
+
|
| 256 |
+
Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
|
| 257 |
+
its own slow families: a product table that will not render because inventory is momentarily
|
| 258 |
+
unreachable is a worse failure than one with blank stock columns.
|
| 259 |
+
"""
|
| 260 |
+
try:
|
| 261 |
+
import modules.inventory as inventory
|
| 262 |
+
return inventory.sku_inventory(t=t) or {}
|
| 263 |
+
except Exception:
|
| 264 |
+
return {}
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _bu_ltm_share(t, team_id):
|
| 268 |
+
"""`{code: 0.0..1.0}` — this unit's SHARE of the SKU's last-twelve-months units.
|
| 269 |
+
|
| 270 |
+
⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
|
| 271 |
+
inventory block has to be re-shaped by a BU fact, and this is that fact.
|
| 272 |
+
|
| 273 |
+
⛔ A SHARE, NOT THE UNIT COUNT ITSELF — AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
|
| 274 |
+
version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
|
| 275 |
+
leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
|
| 276 |
+
question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
|
| 277 |
+
check found **Fisch's LTM units EXCEEDING the company's** — a subset larger than its superset,
|
| 278 |
+
which no reader could explain and no reconciliation could survive.
|
| 279 |
+
|
| 280 |
+
Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
|
| 281 |
+
construction and the scoped figure can never exceed the consolidated one. It also leaves the
|
| 282 |
+
consolidated column exactly as it was — the Inventory page and the Product grid still agree
|
| 283 |
+
about company-wide units, which is what "leave the rest" asked for.
|
| 284 |
+
|
| 285 |
+
`all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
|
| 286 |
+
"this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
|
| 287 |
+
|
| 288 |
+
Degrades to `{}` on any failure, matching `_inventory_by_code` — and a missing code then takes
|
| 289 |
+
the `_DEFAULT_SHARE` below rather than a silent 0.
|
| 290 |
+
"""
|
| 291 |
+
try:
|
| 292 |
+
f, to = P.ltm(t)
|
| 293 |
+
allq = {code: (r.get('qty') or 0.0)
|
| 294 |
+
for code, r in (products._sku_rev(f, to, None) or {}).items()}
|
| 295 |
+
buq = {code: (r.get('qty') or 0.0)
|
| 296 |
+
for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
|
| 297 |
+
except Exception:
|
| 298 |
+
return {}
|
| 299 |
+
out = {}
|
| 300 |
+
for code, total in allq.items():
|
| 301 |
+
out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
|
| 302 |
+
return out
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
#: What a SKU absent from the LTM sales read is worth to a business unit. ZERO — it did not sell
|
| 306 |
+
#: in anybody's book over the window, so no unit can claim its velocity. Named rather than
|
| 307 |
+
#: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
|
| 308 |
+
#: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
|
| 309 |
+
_DEFAULT_SHARE = 0.0
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def _rescope_inventory(e, share):
|
| 313 |
+
"""`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
|
| 314 |
+
|
| 315 |
+
The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator — NOT a second
|
| 316 |
+
idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
|
| 317 |
+
copies of a threshold table is how the Product grid and the Inventory page start disagreeing
|
| 318 |
+
about which SKUs are dead.
|
| 319 |
+
|
| 320 |
+
⚠ `on_hand` is whatever the warehouse holds, unscoped — so a Fisch reader's `dos` answers
|
| 321 |
+
"how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
|
| 322 |
+
actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
|
| 323 |
+
and inventing one would put a number on screen that no Odoo query could reproduce.
|
| 324 |
+
"""
|
| 325 |
+
on_hand = e.get("on_hand")
|
| 326 |
+
if on_hand is None:
|
| 327 |
+
return None, None, None # no inventory row for this SKU: blank, never zero
|
| 328 |
+
qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
|
| 329 |
+
daily = qty / 365.0
|
| 330 |
+
if daily > 0:
|
| 331 |
+
dos_raw = on_hand / daily
|
| 332 |
+
else:
|
| 333 |
+
dos_raw = float('inf') if on_hand > 0 else 0.0
|
| 334 |
+
try:
|
| 335 |
+
import modules.inventory as inventory
|
| 336 |
+
bucket = inventory._bucket(dos_raw, on_hand, qty)
|
| 337 |
+
except Exception:
|
| 338 |
+
bucket = None
|
| 339 |
+
return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def _catalogue_by_code():
|
| 343 |
+
"""`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from.
|
| 344 |
+
|
| 345 |
+
⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
|
| 346 |
+
Inventory degrades to `{}` because a product table with blank stock columns beats one that
|
| 347 |
+
will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that
|
| 348 |
+
failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
|
| 349 |
+
green and nothing on screen saying so, which is the defect this seam exists to end
|
| 350 |
+
([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
|
| 351 |
+
into a 503 that names the cause, so the loud failure has somewhere honest to land.
|
| 352 |
+
|
| 353 |
+
It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
|
| 354 |
+
same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
|
| 355 |
+
to build a pool without Odoo.
|
| 356 |
+
"""
|
| 357 |
+
return products.catalogue()
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def _pricelist_by_code():
|
| 361 |
+
"""`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`.
|
| 362 |
+
|
| 363 |
+
A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
|
| 364 |
+
(these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
|
| 365 |
+
pool without Odoo. ⛔ Unstubbed there, section H would reach live Odoo through the back door
|
| 366 |
+
and the whole file would stop being runnable offline.
|
| 367 |
+
"""
|
| 368 |
+
try:
|
| 369 |
+
return products.pricelist_by_code()
|
| 370 |
+
except Exception as e:
|
| 371 |
+
return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def pool(team_id=None, t=None):
|
| 375 |
+
"""One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
|
| 376 |
+
|
| 377 |
+
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
|
| 378 |
+
is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
|
| 379 |
+
that sold in its window and contributes NO rows of its own.
|
| 380 |
+
|
| 381 |
+
`team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
|
| 382 |
+
passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
|
| 383 |
+
driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
|
| 384 |
+
catalogue has no team.
|
| 385 |
+
"""
|
| 386 |
+
t = t or P.today()
|
| 387 |
+
consolidated = team_id is None
|
| 388 |
+
# ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
|
| 389 |
+
# itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
|
| 390 |
+
inv = _inventory_by_code(t)
|
| 391 |
+
bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
|
| 392 |
+
sup = supplier_master()
|
| 393 |
+
prices, _price_report = _pricelist_by_code()
|
| 394 |
+
cat = _catalogue_by_code()
|
| 395 |
+
# The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
|
| 396 |
+
# does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those
|
| 397 |
+
# out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
|
| 398 |
+
# that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
|
| 399 |
+
rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
|
| 400 |
+
|
| 401 |
+
rows = []
|
| 402 |
+
for code, meta in cat.items():
|
| 403 |
+
r = rev.get(code)
|
| 404 |
+
row = {
|
| 405 |
+
"pid": sku_pid(code),
|
| 406 |
+
"code": code,
|
| 407 |
+
# ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's
|
| 408 |
+
# m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
|
| 409 |
+
# record's `display_name` is the current truth, and it is the same string for every
|
| 410 |
+
# SKU that is not re-SKUed. `directory()` derives the category identically.
|
| 411 |
+
"product": meta.get("product") or code,
|
| 412 |
+
"category": meta.get("category") or "(uncategorized)",
|
| 413 |
+
# ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
|
| 414 |
+
# keeps its measured numbers including a real 0.0 (it sold last year, not this one).
|
| 415 |
+
"rev_ytd": r.get("rev_ytd", 0.0) if r else None,
|
| 416 |
+
"rev_ly": r.get("rev_ly", 0.0) if r else None,
|
| 417 |
+
"yoy_pct": r.get("yoy_pct") if r else None,
|
| 418 |
+
"qty_ytd": r.get("qty_ytd", 0.0) if r else None,
|
| 419 |
+
"orders_ytd": r.get("orders_ytd", 0) if r else None,
|
| 420 |
+
}
|
| 421 |
+
# ⭐ WAVE 30 W30-T34 — the PRICELIST stratum, one column per declared list. Catalogue
|
| 422 |
+
# data like the supplier block, so it rides every pull, scoped or not: a price book is
|
| 423 |
+
# not a thing a business unit owns a slice of, and a Fisch reader who cannot see the
|
| 424 |
+
# Fisch price is exactly who this column is for.
|
| 425 |
+
#
|
| 426 |
+
# ⛔ `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is
|
| 427 |
+
# W29-T52's own negative control: the single `3_global` rule computes over `list_price`,
|
| 428 |
+
# which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer —
|
| 429 |
+
# it is a wrong one wearing a currency sign.
|
| 430 |
+
p = prices.get(code) or {}
|
| 431 |
+
for _col, _name in products.PRICELIST_COLUMNS:
|
| 432 |
+
row[_col] = p.get(_col)
|
| 433 |
+
# Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock).
|
| 434 |
+
s = sup.get(code) or {}
|
| 435 |
+
row.update({
|
| 436 |
+
"supplier": s.get("supplier") or "(none)",
|
| 437 |
+
"lead_days": s.get("lead_days"),
|
| 438 |
+
"origin_country": s.get("origin_country") or "(none)",
|
| 439 |
+
"first_cost": s.get("first_cost"),
|
| 440 |
+
})
|
| 441 |
+
e = inv.get(code) or {}
|
| 442 |
+
if consolidated:
|
| 443 |
+
qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket")
|
| 444 |
+
else:
|
| 445 |
+
qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE))
|
| 446 |
+
row.update({
|
| 447 |
+
# UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf.
|
| 448 |
+
"on_hand": e.get("on_hand"),
|
| 449 |
+
"unit_cost": e.get("unit_cost"),
|
| 450 |
+
"inv_value": e.get("inv_value"),
|
| 451 |
+
# SCOPED: these three are functions of how fast THIS unit sells the SKU.
|
| 452 |
+
"qty_ltm": qty_ltm,
|
| 453 |
+
"dos": dos,
|
| 454 |
+
"stock_bucket": bucket,
|
| 455 |
+
})
|
| 456 |
+
# THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule:
|
| 457 |
+
# buy when the shelf runs out before a reorder could land.
|
| 458 |
+
#
|
| 459 |
+
# ⚠ BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400
|
| 460 |
+
# no lead time; `dos` is null for anything that never sells through. "We don't know"
|
| 461 |
+
# and "you're fine" are different sentences, and only one of them is safe to print
|
| 462 |
+
# beside a purchasing decision.
|
| 463 |
+
#
|
| 464 |
+
# ⛔ `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it
|
| 465 |
+
# is not a preset field — it is a formula over the two columns beside it, and the
|
| 466 |
+
# platform has a formula field type for exactly that. Its Field is gone from
|
| 467 |
+
# `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that
|
| 468 |
+
# contract, so no Field means no cell on the wire. Nothing renders this key.
|
| 469 |
+
#
|
| 470 |
+
# It stays computed because deleting it would delete the PROOF. `validate()` below
|
| 471 |
+
# reconciles the formula's predicate against this one, row for row, which is the only
|
| 472 |
+
# thing that makes "the same exact figures" a claim rather than an assertion — and
|
| 473 |
+
# this repo has the scar already (wave 17: archiving `ar` "would have DELETED the
|
| 474 |
+
# proof"). Two comparisons per SKU is what that costs.
|
| 475 |
+
#
|
| 476 |
+
# ⭐ READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer
|
| 477 |
+
# "does the shelf outlast a reorder AT FISCH'S RATE" — reading the consolidated figure
|
| 478 |
+
# here would print a buy signal computed from both units' velocity beside a days-of-supply
|
| 479 |
+
# computed from one, and the two columns would disagree on the same row.
|
| 480 |
+
lead = s.get("lead_days")
|
| 481 |
+
if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0:
|
| 482 |
+
row["cover_gap_d"] = int(round(dos - lead))
|
| 483 |
+
row["buy_now"] = "Buy now" if dos < lead else "OK"
|
| 484 |
+
else:
|
| 485 |
+
row["cover_gap_d"] = None
|
| 486 |
+
row["buy_now"] = None
|
| 487 |
+
rows.append(row)
|
| 488 |
+
|
| 489 |
+
# `or 0.0` reads a BLANK as zero FOR SORTING ONLY — never for the cell. A never-sold SKU
|
| 490 |
+
# settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
|
| 491 |
+
# value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
|
| 492 |
+
# nothing sold".
|
| 493 |
+
rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
|
| 494 |
+
_assert_no_pid_collision(rows)
|
| 495 |
+
return rows
|
| 496 |
+
|
| 497 |
+
|
| 498 |
+
#: ⭐ THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03).
|
| 499 |
+
#:
|
| 500 |
+
#: The owner's ruling was that "Buy signal" is not a preset field — it is a formula over two
|
| 501 |
+
#: columns the product table already carries, and the platform has a formula field type for it.
|
| 502 |
+
#: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON
|
| 503 |
+
#: contract's `_product_removed_buy_now` note records:
|
| 504 |
+
#:
|
| 505 |
+
#: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
|
| 506 |
+
#:
|
| 507 |
+
#: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a
|
| 508 |
+
#: restatement of the business rule — that is the whole point, because the two could drift and
|
| 509 |
+
#: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly:
|
| 510 |
+
#:
|
| 511 |
+
#: · a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything
|
| 512 |
+
#: that is not a finite number or a string);
|
| 513 |
+
#: · `cmp` returns BLANK unless BOTH sides read as numbers — it never coerces a blank to 0,
|
| 514 |
+
#: which is the difference between this and a filter engine's `toNum(null) === 0`;
|
| 515 |
+
#: · `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison
|
| 516 |
+
#: propagates out as a blank cell rather than taking the false branch.
|
| 517 |
+
BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")'
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
def _buy_signal_formula(row):
|
| 521 |
+
"""Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank)."""
|
| 522 |
+
def num(v):
|
| 523 |
+
# The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite
|
| 524 |
+
# number is null. `isinstance(True, int)` is True in Python, so bool is excluded first.
|
| 525 |
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
| 526 |
+
return None
|
| 527 |
+
return v if v == v and v not in (float('inf'), float('-inf')) else None
|
| 528 |
+
|
| 529 |
+
lead, dos = num(row.get("lead_days")), num(row.get("dos"))
|
| 530 |
+
if lead is None: # `{lead_days} > 0` is blank -> IF(blank, …) is blank
|
| 531 |
+
return ""
|
| 532 |
+
if not lead > 0:
|
| 533 |
+
return "" # the formula's own else-branch
|
| 534 |
+
if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, …) is blank
|
| 535 |
+
return ""
|
| 536 |
+
return "Buy now" if dos < lead else "OK"
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def validate(team_id=None, t=None):
|
| 540 |
+
"""Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number
|
| 541 |
+
which does not tie to Odoo does not ship.
|
| 542 |
+
|
| 543 |
+
⛔⛔ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
|
| 544 |
+
(wave 29, contract C8). It reconciled Σ per-SKU YTD revenue against `products._sku_rev` — the
|
| 545 |
+
same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
|
| 546 |
+
not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
|
| 547 |
+
No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
|
| 548 |
+
That is the leg below, and it is the control that would have caught it
|
| 549 |
+
([[no-unverifiable-aggregates]]).
|
| 550 |
+
|
| 551 |
+
⚠ The revenue leg was NOT deleted with it — it answers a different question ("did the join
|
| 552 |
+
lose money?") and is still the right shape for that one. What changed is that it now has to
|
| 553 |
+
account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
|
| 554 |
+
products out of the grid; the total is decomposed rather than compared loosely, so a growing
|
| 555 |
+
"outside the grid" figure shows up as a number rather than as slack in a tolerance.
|
| 556 |
+
|
| 557 |
+
Still deliberately NOT covered, and said rather than implied: the inventory columns are a
|
| 558 |
+
different module's reconciliation, and they are absent on a scoped pull anyway.
|
| 559 |
+
"""
|
| 560 |
+
t = t or P.today()
|
| 561 |
+
rows = pool(team_id=team_id, t=t)
|
| 562 |
+
|
| 563 |
+
# ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
|
| 564 |
+
# products, asked fresh — not a `len()` over anything this module or `products.catalogue()`
|
| 565 |
+
# built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
|
| 566 |
+
n_active = products.catalogue_count()
|
| 567 |
+
checks = [{
|
| 568 |
+
"check": "the pool holds one row per ACTIVE product (R12) — row count == an INDEPENDENT "
|
| 569 |
+
"Odoo search_count('product.product', active=True)",
|
| 570 |
+
"ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
|
| 571 |
+
}]
|
| 572 |
+
|
| 573 |
+
yf, yt = P.ytd(t)
|
| 574 |
+
sku_rev = products._sku_rev(yf, yt, team_id) or {}
|
| 575 |
+
in_pool = {r["code"] for r in rows}
|
| 576 |
+
outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
|
| 577 |
+
ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
|
| 578 |
+
theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
|
| 579 |
+
dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
|
| 580 |
+
checks.append({
|
| 581 |
+
"check": "Σ per-SKU YTD revenue in the grid + Σ YTD revenue of codes no ACTIVE product "
|
| 582 |
+
"carries == modules.products._sku_rev over the same window",
|
| 583 |
+
"ours": round(ours + dropped, 2), "theirs": theirs,
|
| 584 |
+
"ok": abs((ours + dropped) - theirs) < 0.01,
|
| 585 |
+
"detail": {"in_the_grid": ours, "outside_the_grid": dropped,
|
| 586 |
+
"codes_outside": len(outside)},
|
| 587 |
+
})
|
| 588 |
+
# ⛔ AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
|
| 589 |
+
# trivially true when checked against the dict that did the dropping — that is the
|
| 590 |
+
# self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
|
| 591 |
+
# product carries means `products.catalogue()` missed a row that has revenue, which is the
|
| 592 |
+
# 2,717 defect returning in a smaller costume.
|
| 593 |
+
# ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
|
| 594 |
+
# catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
|
| 595 |
+
coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
|
| 596 |
+
still_active = (O.get_odoo().search_count(
|
| 597 |
+
'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
|
| 598 |
+
if coded_outside else 0)
|
| 599 |
+
checks.append({
|
| 600 |
+
"check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
|
| 601 |
+
"stay out) — asked of Odoo, never of the catalogue that did the dropping",
|
| 602 |
+
"ours": still_active, "theirs": 0, "ok": still_active == 0,
|
| 603 |
+
"detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
|
| 604 |
+
})
|
| 605 |
+
# ── ⭐⭐ WAVE 30 W30-T34 — THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ──
|
| 606 |
+
#
|
| 607 |
+
# Three legs, because the column can fail in three different ways and only one of them is a
|
| 608 |
+
# count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each
|
| 609 |
+
# rule over the products it names, IN PYTHON — that expansion is the fragile part, so every
|
| 610 |
+
# leg below re-asks ODOO instead of re-reading the map.
|
| 611 |
+
price_report = _pricelist_by_code()[1]
|
| 612 |
+
_pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
|
| 613 |
+
_pl_id = {}
|
| 614 |
+
for _p in _pl_rows:
|
| 615 |
+
_pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id'])
|
| 616 |
+
_today = P.today().isoformat()
|
| 617 |
+
|
| 618 |
+
_targets_memo = {}
|
| 619 |
+
|
| 620 |
+
def _rule_targets(list_name):
|
| 621 |
+
"""(variant_ids, template_ids) a pricelist prices today — read FRESH from Odoo.
|
| 622 |
+
|
| 623 |
+
Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and
|
| 624 |
+
without this `validate()` makes six identical round trips instead of three (measured:
|
| 625 |
+
~48s of the run). Deliberately NOT an `lru_cache` — an oracle that survives the process
|
| 626 |
+
is an oracle reading yesterday's Odoo.
|
| 627 |
+
"""
|
| 628 |
+
if list_name in _targets_memo:
|
| 629 |
+
return _targets_memo[list_name]
|
| 630 |
+
var, tmpl = set(), set()
|
| 631 |
+
plid = _pl_id.get(list_name)
|
| 632 |
+
if plid is None:
|
| 633 |
+
return var, tmpl
|
| 634 |
+
for r in O.search_read(
|
| 635 |
+
'product.pricelist.item',
|
| 636 |
+
[('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'),
|
| 637 |
+
('applied_on', 'in', ['0_product_variant', '1_product'])],
|
| 638 |
+
['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
|
| 639 |
+
'date_start', 'date_end']):
|
| 640 |
+
ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10]
|
| 641 |
+
if (ds and ds > _today) or (de and de < _today):
|
| 642 |
+
continue
|
| 643 |
+
fp = r.get('fixed_price')
|
| 644 |
+
if not isinstance(fp, (int, float)) or fp <= 0:
|
| 645 |
+
continue
|
| 646 |
+
if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
|
| 647 |
+
var.add(O.m2o_id(r['product_id']))
|
| 648 |
+
elif r.get('product_tmpl_id'):
|
| 649 |
+
tmpl.add(O.m2o_id(r['product_tmpl_id']))
|
| 650 |
+
_targets_memo[list_name] = (var, tmpl)
|
| 651 |
+
return var, tmpl
|
| 652 |
+
|
| 653 |
+
# LEG 1 — COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of
|
| 654 |
+
# ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS
|
| 655 |
+
# the data) but the EXPANSION is not, and the expansion is what breaks.
|
| 656 |
+
for _col, _name in products.PRICELIST_COLUMNS:
|
| 657 |
+
_var, _tmpl = _rule_targets(_name)
|
| 658 |
+
_dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)),
|
| 659 |
+
('product_tmpl_id', 'in', sorted(_tmpl))]
|
| 660 |
+
theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0
|
| 661 |
+
ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float)))
|
| 662 |
+
checks.append({
|
| 663 |
+
"check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count "
|
| 664 |
+
f"of ACTIVE products its date-valid fixed rules reach",
|
| 665 |
+
"ours": ours, "theirs": theirs, "ok": ours == theirs,
|
| 666 |
+
"detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)},
|
| 667 |
+
})
|
| 668 |
+
|
| 669 |
+
# LEG 2 — ⛔ THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must
|
| 670 |
+
# render BLANK. Asked of ODOO, never of the map that produced the blank — "everything I left
|
| 671 |
+
# empty was genuinely unpriced" is trivially true when checked against the dict that emptied
|
| 672 |
+
# it, which is the self-sealing shape the population leg above exists to end.
|
| 673 |
+
_unpriced = [r for r in rows
|
| 674 |
+
if not any(isinstance(r.get(c), (int, float))
|
| 675 |
+
for c, _n in products.PRICELIST_COLUMNS)]
|
| 676 |
+
_codes = {r["code"] for r in _unpriced}
|
| 677 |
+
_stray = 0
|
| 678 |
+
if _codes:
|
| 679 |
+
# Resolve those codes back to Odoo ids and ask whether ANY declared list prices them.
|
| 680 |
+
_ids, _tmpls = set(), set()
|
| 681 |
+
for _p in O.search_read('product.product',
|
| 682 |
+
[('active', '=', True),
|
| 683 |
+
('default_code', 'in', sorted(c for c in _codes
|
| 684 |
+
if not c.startswith("pid:")))],
|
| 685 |
+
['id', 'product_tmpl_id']):
|
| 686 |
+
_ids.add(_p['id'])
|
| 687 |
+
_tmpls.add(O.m2o_id(_p.get('product_tmpl_id')))
|
| 688 |
+
for _col, _name in products.PRICELIST_COLUMNS:
|
| 689 |
+
_v, _t = _rule_targets(_name)
|
| 690 |
+
_stray += len((_v & _ids) | ({t for t in _tmpls if t in _t}))
|
| 691 |
+
checks.append({
|
| 692 |
+
"check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared "
|
| 693 |
+
"lists (W29-T52's NC: no fallback dressed as a price) — asked of Odoo",
|
| 694 |
+
"ours": _stray, "theirs": 0, "ok": _stray == 0,
|
| 695 |
+
"detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)},
|
| 696 |
+
})
|
| 697 |
+
|
| 698 |
+
# LEG 3 — ⭐ R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you
|
| 699 |
+
# need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is
|
| 700 |
+
# counted here rather than dropped. The ASSERTION is the one thing that must never be true —
|
| 701 |
+
# a price cell that is present and not a positive number, i.e. a "free" SKU — while the
|
| 702 |
+
# rest rides `detail` so it is reported without reddening an honest day's data.
|
| 703 |
+
_bad = [r["code"] for r in rows
|
| 704 |
+
for c, _n in products.PRICELIST_COLUMNS
|
| 705 |
+
if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)]
|
| 706 |
+
# ⛔ MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends
|
| 707 |
+
# every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole
|
| 708 |
+
# dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this
|
| 709 |
+
# column ties to the cent HERE and ships rounded. Reported, never silently enforced.
|
| 710 |
+
_cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
|
| 711 |
+
if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9)
|
| 712 |
+
_to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
|
| 713 |
+
if isinstance(r.get(c), (int, float)) and round(r[c]) == 0)
|
| 714 |
+
checks.append({
|
| 715 |
+
"check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; "
|
| 716 |
+
"an unset rule must be blank), and what the reader cannot see is COUNTED",
|
| 717 |
+
"ours": len(_bad), "theirs": 0, "ok": not _bad,
|
| 718 |
+
"detail": {"bad_cells": _bad[:10], "reader_report": price_report,
|
| 719 |
+
"wire_rounding_loses_cents_on": _cents,
|
| 720 |
+
"wire_rounding_to_zero_dollars": _to_zero},
|
| 721 |
+
})
|
| 722 |
+
|
| 723 |
+
# WAVE 17 R3 — the BUY SIGNAL must be a total, exact partition of the catalogue, and every
|
| 724 |
+
# member of it must be re-derivable from the two columns beside it. A signal somebody buys
|
| 725 |
+
# stock on cannot be "mostly right": the failure that matters is a row that says OK because
|
| 726 |
+
# an input was missing, so the blank leg is checked as hard as the other two.
|
| 727 |
+
if team_id is None: # consolidated only — the inputs exist only there
|
| 728 |
+
buy = [r for r in rows if r.get("buy_now") == "Buy now"]
|
| 729 |
+
ok_rows = [r for r in rows if r.get("buy_now") == "OK"]
|
| 730 |
+
blank = [r for r in rows if r.get("buy_now") is None]
|
| 731 |
+
mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"]))
|
| 732 |
+
mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"]))
|
| 733 |
+
# A blank must be UNKNOWN — never a row we could have answered and quietly did not.
|
| 734 |
+
mis += sum(1 for r in blank
|
| 735 |
+
if isinstance(r.get("dos"), (int, float))
|
| 736 |
+
and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0)
|
| 737 |
+
checks.append({
|
| 738 |
+
"check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none "
|
| 739 |
+
"misclassified)",
|
| 740 |
+
"ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows),
|
| 741 |
+
"ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows),
|
| 742 |
+
"detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank),
|
| 743 |
+
"misclassified": mis},
|
| 744 |
+
})
|
| 745 |
+
# ⭐ OWNER 2026-08-03 — "use the formula fields to come up to the same EXACT figures".
|
| 746 |
+
#
|
| 747 |
+
# This is that sentence, as a check. The retired preset column and the formula that
|
| 748 |
+
# replaces it must agree on every SKU, in all three states, or the replacement is not a
|
| 749 |
+
# replacement. `_buy_signal_formula` is a line-by-line port of the client formula
|
| 750 |
+
# engine's evaluation of
|
| 751 |
+
#
|
| 752 |
+
# IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
|
| 753 |
+
#
|
| 754 |
+
# including the part that is easy to get wrong: a comparison against a BLANK is blank,
|
| 755 |
+
# never a coerced 0. (The client engine's `cmp` returns null unless both sides read as
|
| 756 |
+
# numbers, and `IF` refuses a non-boolean condition — so a missing `dos` yields "" and
|
| 757 |
+
# not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that
|
| 758 |
+
# never sells through; the formula engine does not, and this check is what holds it.)
|
| 759 |
+
#
|
| 760 |
+
# ⚠ It compares SETS OF SKUs, not counts. Two different partitions can share a shape.
|
| 761 |
+
disagree = sorted(r["code"] for r in rows
|
| 762 |
+
if (r.get("buy_now") or "") != _buy_signal_formula(r))
|
| 763 |
+
checks.append({
|
| 764 |
+
"check": 'Buy signal as a FORMULA field == the retired preset column, per SKU '
|
| 765 |
+
'(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))',
|
| 766 |
+
"ours": len(rows) - len(disagree), "theirs": len(rows),
|
| 767 |
+
"ok": not disagree,
|
| 768 |
+
"detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)},
|
| 769 |
+
})
|
| 770 |
+
# The other half of "the same figures": the SHARED Buy list view can no longer filter on
|
| 771 |
+
# the retired column, and its replacement conditions must select the same SKUs. They are
|
| 772 |
+
# `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days`
|
| 773 |
+
# (_seed_wave17.views), so this reproduces exactly that conjunction.
|
| 774 |
+
#
|
| 775 |
+
# ⛔ THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the
|
| 776 |
+
# obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine
|
| 777 |
+
# gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on.
|
| 778 |
+
view_rows = {r["code"] for r in rows
|
| 779 |
+
if isinstance(r.get("dos"), (int, float))
|
| 780 |
+
and isinstance(r.get("lead_days"), (int, float))
|
| 781 |
+
and r["lead_days"] > 0 and r["dos"] < r["lead_days"]}
|
| 782 |
+
signal_rows = {r["code"] for r in buy}
|
| 783 |
+
rounding_would_miss = sorted(
|
| 784 |
+
c for c in signal_rows
|
| 785 |
+
if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0)
|
| 786 |
+
checks.append({
|
| 787 |
+
"check": "Buy list view conditions (dos/lead_days, no retired column) select "
|
| 788 |
+
"exactly the Buy-now SKUs",
|
| 789 |
+
"ours": len(view_rows), "theirs": len(signal_rows),
|
| 790 |
+
"ok": view_rows == signal_rows,
|
| 791 |
+
"detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
|
| 792 |
+
"only_in_signal": sorted(signal_rows - view_rows)[:10],
|
| 793 |
+
# Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
|
| 794 |
+
# have silently dropped. 0 today does not make that filter correct.
|
| 795 |
+
"cover_gap_rounds_to_zero": len(rounding_would_miss)},
|
| 796 |
+
})
|
| 797 |
+
return checks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
platform/modules/products.py
CHANGED
|
@@ -1,737 +1,728 @@
|
|
| 1 |
-
"""Products / SKU module — SKU health: YoY movers (risers & decliners), zombie SKUs
|
| 2 |
-
(catalog rot — sellable, formerly selling, now dead), new winners, coverage collapse
|
| 3 |
-
(SKUs losing customer breadth — the FFS-recovery early-warning signal), and velocity leaders.
|
| 4 |
-
|
| 5 |
-
Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
|
| 6 |
-
Basket / co-purchase (MBA) is intentionally deferred — the existing client app computes it
|
| 7 |
-
runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
|
| 8 |
-
project guardrails keep off the local PC. (See BACKLOG.)
|
| 9 |
-
|
| 10 |
-
All line-level (sale.order.line), RI+FFS scope, excluded accounts removed — reusing sale_line_domain.
|
| 11 |
-
Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
|
| 12 |
-
window), so it's its own function the app calls lazily and caches.
|
| 13 |
-
"""
|
| 14 |
-
import sys
|
| 15 |
-
import datetime as dt
|
| 16 |
-
from pathlib import Path
|
| 17 |
-
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 18 |
-
from functools import lru_cache
|
| 19 |
-
import core.odoo as O
|
| 20 |
-
import core.periods as P
|
| 21 |
-
import modules.sales as sales_mod
|
| 22 |
-
|
| 23 |
-
# SKU-health views are about real products — exclude service/delivery pseudo-SKUs
|
| 24 |
-
# (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
|
| 25 |
-
# on the dot-path does not). Validation reconciles on this same non-service universe.
|
| 26 |
-
_NO_SVC = [('product_id.type', '!=', 'service')]
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
@lru_cache(maxsize=1)
|
| 30 |
-
def _code_map():
|
| 31 |
-
"""product_id → SKU code. Multiple product records can share a default_code (re-SKUing
|
| 32 |
-
/duplicates); keying by code merges them so a re-coded item doesn't read as a fake
|
| 33 |
-
decliner + fake riser. Products without a code fall back to a per-id key. Archived
|
| 34 |
-
products are INCLUDED — re-SKUing typically archives the old record and creates a new
|
| 35 |
-
one under the same code, and the old record still carries last-year sales."""
|
| 36 |
-
prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
|
| 37 |
-
limit=50000)
|
| 38 |
-
return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 39 |
-
for p in prods}
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
def _sku_rev(date_from, date_to, team_id=None):
|
| 43 |
-
"""{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
|
| 44 |
-
duplicate product records merged by SKU code)."""
|
| 45 |
-
g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
|
| 46 |
-
['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
|
| 47 |
-
codes = _code_map()
|
| 48 |
-
out = {}
|
| 49 |
-
for r in g:
|
| 50 |
-
pid = O.m2o_id(r.get('product_id'))
|
| 51 |
-
if not pid:
|
| 52 |
-
continue
|
| 53 |
-
key = codes.get(pid, f"pid:{pid}")
|
| 54 |
-
e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
|
| 55 |
-
'rev': 0.0, 'qty': 0.0, 'orders': 0})
|
| 56 |
-
e['rev'] += r.get('price_subtotal') or 0.0
|
| 57 |
-
e['qty'] += r.get('product_uom_qty') or 0.0
|
| 58 |
-
e['orders'] += r.get('__count') or 0
|
| 59 |
-
return out
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
def yoy_movers(t=None, limit=20, team_id=None):
|
| 63 |
-
"""Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
|
| 64 |
-
t = t or P.today()
|
| 65 |
-
yf, yt = P.ytd(t)
|
| 66 |
-
lf, lt = P.ytd_last_year(t)
|
| 67 |
-
this = _sku_rev(yf, yt, team_id)
|
| 68 |
-
last = _sku_rev(lf, lt, team_id)
|
| 69 |
-
rows = []
|
| 70 |
-
for pid in set(this) | set(last):
|
| 71 |
-
tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
|
| 72 |
-
lr = last.get(pid, {'rev': 0.0})
|
| 73 |
-
name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
|
| 74 |
-
rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 75 |
-
'change': tr['rev'] - lr['rev']})
|
| 76 |
-
risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
|
| 77 |
-
decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
|
| 78 |
-
return {'risers': risers, 'decliners': decliners}
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
|
| 82 |
-
"""Catalog rot: SKUs that sold materially last year but are ~dead this year."""
|
| 83 |
-
t = t or P.today()
|
| 84 |
-
yf, yt = P.ytd(t)
|
| 85 |
-
lf, lt = P.ytd_last_year(t)
|
| 86 |
-
this = _sku_rev(yf, yt, team_id)
|
| 87 |
-
last = _sku_rev(lf, lt, team_id)
|
| 88 |
-
rows = []
|
| 89 |
-
for pid, lr in last.items():
|
| 90 |
-
if lr['rev'] < min_prior:
|
| 91 |
-
continue
|
| 92 |
-
tr = this.get(pid, {'rev': 0.0})
|
| 93 |
-
if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior → not a zombie
|
| 94 |
-
continue
|
| 95 |
-
rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
|
| 96 |
-
'lost': lr['rev'] - tr['rev']})
|
| 97 |
-
rows.sort(key=lambda x: -x['lost'])
|
| 98 |
-
return rows[:limit]
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
|
| 102 |
-
"""SKUs that barely sold last year but are selling well this year."""
|
| 103 |
-
t = t or P.today()
|
| 104 |
-
yf, yt = P.ytd(t)
|
| 105 |
-
lf, lt = P.ytd_last_year(t)
|
| 106 |
-
this = _sku_rev(yf, yt, team_id)
|
| 107 |
-
last = _sku_rev(lf, lt, team_id)
|
| 108 |
-
rows = []
|
| 109 |
-
for pid, tr in this.items():
|
| 110 |
-
if tr['rev'] < min_this:
|
| 111 |
-
continue
|
| 112 |
-
lr = last.get(pid, {'rev': 0.0})
|
| 113 |
-
if lr['rev'] > 0.05 * tr['rev']:
|
| 114 |
-
continue
|
| 115 |
-
rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 116 |
-
'gained': tr['rev'] - lr['rev']})
|
| 117 |
-
rows.sort(key=lambda x: -x['gained'])
|
| 118 |
-
return rows[:limit]
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
def velocity_leaders(t=None, limit=25, team_id=None):
|
| 122 |
-
"""Top SKUs by LTM unit velocity (units/month) and revenue."""
|
| 123 |
-
t = t or P.today()
|
| 124 |
-
lf, lt = P.ltm(t)
|
| 125 |
-
sku = _sku_rev(lf, lt, team_id)
|
| 126 |
-
rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
|
| 127 |
-
'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
|
| 128 |
-
rows.sort(key=lambda x: -x['units_ltm'])
|
| 129 |
-
return rows[:limit]
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
def _code_category():
|
| 133 |
-
"""code -> category name. Built from _code_map (pid->code) + the sales category map
|
| 134 |
-
(pid->category); the first record carrying a code sets that code's category."""
|
| 135 |
-
codes = _code_map()
|
| 136 |
-
cats = sales_mod._product_cat()
|
| 137 |
-
out = {}
|
| 138 |
-
for pid, code in codes.items():
|
| 139 |
-
if code not in out and pid in cats:
|
| 140 |
-
out[code] = cats[pid]
|
| 141 |
-
return out
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
def directory(t=None, team_id=None):
|
| 145 |
-
"""Every SKU that sold this YTD or the same period last year, with the fields the SKU
|
| 146 |
-
drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
|
| 147 |
-
customer directory — the full list (filtering happens in the UI), sorted by YTD revenue."""
|
| 148 |
-
t = t or P.today()
|
| 149 |
-
yf, yt = P.ytd(t)
|
| 150 |
-
lf, lt = P.ytd_last_year(t)
|
| 151 |
-
this = _sku_rev(yf, yt, team_id)
|
| 152 |
-
last = _sku_rev(lf, lt, team_id)
|
| 153 |
-
code_cat = _code_category()
|
| 154 |
-
rows = []
|
| 155 |
-
for code in set(this) | set(last):
|
| 156 |
-
tr = this.get(code, {})
|
| 157 |
-
lr = last.get(code, {})
|
| 158 |
-
rev_ytd = tr.get('rev', 0.0)
|
| 159 |
-
rev_ly = lr.get('rev', 0.0)
|
| 160 |
-
rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
|
| 161 |
-
'category': code_cat.get(code, '(uncategorized)'),
|
| 162 |
-
'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
|
| 163 |
-
'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
|
| 164 |
-
'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
|
| 165 |
-
rows.sort(key=lambda r: -r['rev_ytd'])
|
| 166 |
-
return rows
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
def catalogue():
|
| 170 |
-
"""`{sku_code: {'product', 'category'
|
| 171 |
-
|
| 172 |
-
⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
|
| 173 |
-
row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
|
| 174 |
-
that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`,
|
| 175 |
-
`zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe —
|
| 176 |
-
and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
|
| 177 |
-
5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
|
| 178 |
-
the consumer, never a window removed from `directory()`: removing the window alone lands at
|
| 179 |
-
3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
|
| 180 |
-
|
| 181 |
-
⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
|
| 182 |
-
catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
|
| 183 |
-
merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
|
| 184 |
-
|
| 185 |
-
Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active
|
| 186 |
-
records that carry none — so the join against `directory()` is code-for-code with no
|
| 187 |
-
normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
|
| 188 |
-
off a sale line, so a never-sold row wears the same format as a sold one.
|
| 189 |
-
|
| 190 |
-
⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull
|
| 191 |
-
would put the grid back at a plausible wrong number with every gate green — the exact failure
|
| 192 |
-
this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
|
| 193 |
-
`modules/backorders.py:161` already uses).
|
| 194 |
-
"""
|
| 195 |
-
dom = [('active', '=', True)]
|
| 196 |
-
prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
|
| 197 |
-
limit=50000)
|
| 198 |
-
n = O.get_odoo().search_count('product.product', dom)
|
| 199 |
-
if len(prods) != n:
|
| 200 |
-
raise ValueError(
|
| 201 |
-
f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against "
|
| 202 |
-
f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
|
| 203 |
-
f"nothing reporting it; raise the limit before shipping this.")
|
| 204 |
-
code_cat = _code_category()
|
| 205 |
-
out = {}
|
| 206 |
-
for p in prods:
|
| 207 |
-
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 208 |
-
# First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
|
| 209 |
-
# active products share a code, so this branch is a guard, not a merge policy.
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
#:
|
| 225 |
-
#:
|
| 226 |
-
#:
|
| 227 |
-
#:
|
| 228 |
-
#:
|
| 229 |
-
#:
|
| 230 |
-
#:
|
| 231 |
-
#:
|
| 232 |
-
#:
|
| 233 |
-
#:
|
| 234 |
-
#:
|
| 235 |
-
#:
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
**
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
#
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
#
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
cov
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
def
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
for
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
yf, yt
|
| 492 |
-
lf, lt
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
lf, lt
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
for
|
| 645 |
-
|
| 646 |
-
s =
|
| 647 |
-
s['
|
| 648 |
-
s
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
'
|
| 716 |
-
'
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
|
| 730 |
-
if top_code:
|
| 731 |
-
bb = sku_buyer_bridge(top_code, t, team_id=team_id)
|
| 732 |
-
recon = bb['retained']['rev'] + bb['new']['rev']
|
| 733 |
-
checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
|
| 734 |
-
'a': round(recon, 2), 'b': round(bb['this_total'], 2),
|
| 735 |
-
'gap': round(recon - bb['this_total'], 2),
|
| 736 |
-
'ok': abs(recon - bb['this_total']) <= 1.0})
|
| 737 |
-
return checks
|
|
|
|
| 1 |
+
"""Products / SKU module — SKU health: YoY movers (risers & decliners), zombie SKUs
|
| 2 |
+
(catalog rot — sellable, formerly selling, now dead), new winners, coverage collapse
|
| 3 |
+
(SKUs losing customer breadth — the FFS-recovery early-warning signal), and velocity leaders.
|
| 4 |
+
|
| 5 |
+
Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
|
| 6 |
+
Basket / co-purchase (MBA) is intentionally deferred — the existing client app computes it
|
| 7 |
+
runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
|
| 8 |
+
project guardrails keep off the local PC. (See BACKLOG.)
|
| 9 |
+
|
| 10 |
+
All line-level (sale.order.line), RI+FFS scope, excluded accounts removed — reusing sale_line_domain.
|
| 11 |
+
Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
|
| 12 |
+
window), so it's its own function the app calls lazily and caches.
|
| 13 |
+
"""
|
| 14 |
+
import sys
|
| 15 |
+
import datetime as dt
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 18 |
+
from functools import lru_cache
|
| 19 |
+
import core.odoo as O
|
| 20 |
+
import core.periods as P
|
| 21 |
+
import modules.sales as sales_mod
|
| 22 |
+
|
| 23 |
+
# SKU-health views are about real products — exclude service/delivery pseudo-SKUs
|
| 24 |
+
# (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
|
| 25 |
+
# on the dot-path does not). Validation reconciles on this same non-service universe.
|
| 26 |
+
_NO_SVC = [('product_id.type', '!=', 'service')]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@lru_cache(maxsize=1)
|
| 30 |
+
def _code_map():
|
| 31 |
+
"""product_id → SKU code. Multiple product records can share a default_code (re-SKUing
|
| 32 |
+
/duplicates); keying by code merges them so a re-coded item doesn't read as a fake
|
| 33 |
+
decliner + fake riser. Products without a code fall back to a per-id key. Archived
|
| 34 |
+
products are INCLUDED — re-SKUing typically archives the old record and creates a new
|
| 35 |
+
one under the same code, and the old record still carries last-year sales."""
|
| 36 |
+
prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
|
| 37 |
+
limit=50000)
|
| 38 |
+
return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 39 |
+
for p in prods}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _sku_rev(date_from, date_to, team_id=None):
|
| 43 |
+
"""{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
|
| 44 |
+
duplicate product records merged by SKU code)."""
|
| 45 |
+
g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
|
| 46 |
+
['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
|
| 47 |
+
codes = _code_map()
|
| 48 |
+
out = {}
|
| 49 |
+
for r in g:
|
| 50 |
+
pid = O.m2o_id(r.get('product_id'))
|
| 51 |
+
if not pid:
|
| 52 |
+
continue
|
| 53 |
+
key = codes.get(pid, f"pid:{pid}")
|
| 54 |
+
e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
|
| 55 |
+
'rev': 0.0, 'qty': 0.0, 'orders': 0})
|
| 56 |
+
e['rev'] += r.get('price_subtotal') or 0.0
|
| 57 |
+
e['qty'] += r.get('product_uom_qty') or 0.0
|
| 58 |
+
e['orders'] += r.get('__count') or 0
|
| 59 |
+
return out
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def yoy_movers(t=None, limit=20, team_id=None):
|
| 63 |
+
"""Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
|
| 64 |
+
t = t or P.today()
|
| 65 |
+
yf, yt = P.ytd(t)
|
| 66 |
+
lf, lt = P.ytd_last_year(t)
|
| 67 |
+
this = _sku_rev(yf, yt, team_id)
|
| 68 |
+
last = _sku_rev(lf, lt, team_id)
|
| 69 |
+
rows = []
|
| 70 |
+
for pid in set(this) | set(last):
|
| 71 |
+
tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
|
| 72 |
+
lr = last.get(pid, {'rev': 0.0})
|
| 73 |
+
name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
|
| 74 |
+
rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 75 |
+
'change': tr['rev'] - lr['rev']})
|
| 76 |
+
risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
|
| 77 |
+
decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
|
| 78 |
+
return {'risers': risers, 'decliners': decliners}
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
|
| 82 |
+
"""Catalog rot: SKUs that sold materially last year but are ~dead this year."""
|
| 83 |
+
t = t or P.today()
|
| 84 |
+
yf, yt = P.ytd(t)
|
| 85 |
+
lf, lt = P.ytd_last_year(t)
|
| 86 |
+
this = _sku_rev(yf, yt, team_id)
|
| 87 |
+
last = _sku_rev(lf, lt, team_id)
|
| 88 |
+
rows = []
|
| 89 |
+
for pid, lr in last.items():
|
| 90 |
+
if lr['rev'] < min_prior:
|
| 91 |
+
continue
|
| 92 |
+
tr = this.get(pid, {'rev': 0.0})
|
| 93 |
+
if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior → not a zombie
|
| 94 |
+
continue
|
| 95 |
+
rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
|
| 96 |
+
'lost': lr['rev'] - tr['rev']})
|
| 97 |
+
rows.sort(key=lambda x: -x['lost'])
|
| 98 |
+
return rows[:limit]
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
|
| 102 |
+
"""SKUs that barely sold last year but are selling well this year."""
|
| 103 |
+
t = t or P.today()
|
| 104 |
+
yf, yt = P.ytd(t)
|
| 105 |
+
lf, lt = P.ytd_last_year(t)
|
| 106 |
+
this = _sku_rev(yf, yt, team_id)
|
| 107 |
+
last = _sku_rev(lf, lt, team_id)
|
| 108 |
+
rows = []
|
| 109 |
+
for pid, tr in this.items():
|
| 110 |
+
if tr['rev'] < min_this:
|
| 111 |
+
continue
|
| 112 |
+
lr = last.get(pid, {'rev': 0.0})
|
| 113 |
+
if lr['rev'] > 0.05 * tr['rev']:
|
| 114 |
+
continue
|
| 115 |
+
rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 116 |
+
'gained': tr['rev'] - lr['rev']})
|
| 117 |
+
rows.sort(key=lambda x: -x['gained'])
|
| 118 |
+
return rows[:limit]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def velocity_leaders(t=None, limit=25, team_id=None):
|
| 122 |
+
"""Top SKUs by LTM unit velocity (units/month) and revenue."""
|
| 123 |
+
t = t or P.today()
|
| 124 |
+
lf, lt = P.ltm(t)
|
| 125 |
+
sku = _sku_rev(lf, lt, team_id)
|
| 126 |
+
rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
|
| 127 |
+
'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
|
| 128 |
+
rows.sort(key=lambda x: -x['units_ltm'])
|
| 129 |
+
return rows[:limit]
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _code_category():
|
| 133 |
+
"""code -> category name. Built from _code_map (pid->code) + the sales category map
|
| 134 |
+
(pid->category); the first record carrying a code sets that code's category."""
|
| 135 |
+
codes = _code_map()
|
| 136 |
+
cats = sales_mod._product_cat()
|
| 137 |
+
out = {}
|
| 138 |
+
for pid, code in codes.items():
|
| 139 |
+
if code not in out and pid in cats:
|
| 140 |
+
out[code] = cats[pid]
|
| 141 |
+
return out
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def directory(t=None, team_id=None):
|
| 145 |
+
"""Every SKU that sold this YTD or the same period last year, with the fields the SKU
|
| 146 |
+
drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
|
| 147 |
+
customer directory — the full list (filtering happens in the UI), sorted by YTD revenue."""
|
| 148 |
+
t = t or P.today()
|
| 149 |
+
yf, yt = P.ytd(t)
|
| 150 |
+
lf, lt = P.ytd_last_year(t)
|
| 151 |
+
this = _sku_rev(yf, yt, team_id)
|
| 152 |
+
last = _sku_rev(lf, lt, team_id)
|
| 153 |
+
code_cat = _code_category()
|
| 154 |
+
rows = []
|
| 155 |
+
for code in set(this) | set(last):
|
| 156 |
+
tr = this.get(code, {})
|
| 157 |
+
lr = last.get(code, {})
|
| 158 |
+
rev_ytd = tr.get('rev', 0.0)
|
| 159 |
+
rev_ly = lr.get('rev', 0.0)
|
| 160 |
+
rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
|
| 161 |
+
'category': code_cat.get(code, '(uncategorized)'),
|
| 162 |
+
'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
|
| 163 |
+
'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
|
| 164 |
+
'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
|
| 165 |
+
rows.sort(key=lambda r: -r['rev_ytd'])
|
| 166 |
+
return rows
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def catalogue():
|
| 170 |
+
"""`{sku_code: {'product', 'category'}}` — EVERY ACTIVE product, sold or not.
|
| 171 |
+
|
| 172 |
+
⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
|
| 173 |
+
row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
|
| 174 |
+
that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`,
|
| 175 |
+
`zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe —
|
| 176 |
+
and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
|
| 177 |
+
5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
|
| 178 |
+
the consumer, never a window removed from `directory()`: removing the window alone lands at
|
| 179 |
+
3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
|
| 180 |
+
|
| 181 |
+
⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
|
| 182 |
+
catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
|
| 183 |
+
merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
|
| 184 |
+
|
| 185 |
+
Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active
|
| 186 |
+
records that carry none — so the join against `directory()` is code-for-code with no
|
| 187 |
+
normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
|
| 188 |
+
off a sale line, so a never-sold row wears the same format as a sold one.
|
| 189 |
+
|
| 190 |
+
⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull
|
| 191 |
+
would put the grid back at a plausible wrong number with every gate green — the exact failure
|
| 192 |
+
this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
|
| 193 |
+
`modules/backorders.py:161` already uses).
|
| 194 |
+
"""
|
| 195 |
+
dom = [('active', '=', True)]
|
| 196 |
+
prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
|
| 197 |
+
limit=50000)
|
| 198 |
+
n = O.get_odoo().search_count('product.product', dom)
|
| 199 |
+
if len(prods) != n:
|
| 200 |
+
raise ValueError(
|
| 201 |
+
f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against "
|
| 202 |
+
f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
|
| 203 |
+
f"nothing reporting it; raise the limit before shipping this.")
|
| 204 |
+
code_cat = _code_category()
|
| 205 |
+
out = {}
|
| 206 |
+
for p in prods:
|
| 207 |
+
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 208 |
+
# First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
|
| 209 |
+
# active products share a code, so this branch is a guard, not a merge policy.
|
| 210 |
+
out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
|
| 211 |
+
'category': code_cat.get(code, '(uncategorized)')})
|
| 212 |
+
return out
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
#: ⭐⭐ WAVE 30 / W30-T34 (the carried W29-T52) — THE PRICELIST STRATUM, PER LIST.
|
| 216 |
+
#:
|
| 217 |
+
#: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than
|
| 218 |
+
#: decorating it:
|
| 219 |
+
#:
|
| 220 |
+
#: · **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139,
|
| 221 |
+
#: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that
|
| 222 |
+
#: is Fisch 3,971 · Royal 1 1,920 · Royal 2 1,857 · Public 6 · Giftware 0.
|
| 223 |
+
#: · **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.**
|
| 224 |
+
#: · **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** — median relative
|
| 225 |
+
#: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a
|
| 226 |
+
#: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the
|
| 227 |
+
#: only question the column exists to answer.
|
| 228 |
+
#: · ⛔ **`list_price` is 1.00 on 5,817 of the 5,871** — so the single `3_global` rule
|
| 229 |
+
#: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls
|
| 230 |
+
#: through to it**, and that is the ticket's own negative control: a SKU with no specific
|
| 231 |
+
#: item renders BLANK, never a fallback dressed as a price.
|
| 232 |
+
#:
|
| 233 |
+
#: ⚠ RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go
|
| 234 |
+
#: blank and makes `report["lists_missing"]` name it — a hard-coded id would keep pointing at
|
| 235 |
+
#: whatever inherited it and mislabel every cell silently.
|
| 236 |
+
PRICELIST_COLUMNS = (
|
| 237 |
+
("price_fisch", "Fisch"),
|
| 238 |
+
("price_royal_1", "Royal 1"),
|
| 239 |
+
("price_royal_2", "Royal 2"),
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def pricelist_by_code():
|
| 244 |
+
"""`({code: {column_key: price}}, report)` — the date-valid FIXED base-tier price for each
|
| 245 |
+
declared pricelist, keyed EXACTLY as `catalogue()` keys its rows.
|
| 246 |
+
|
| 247 |
+
⛔ THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a
|
| 248 |
+
`pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other
|
| 249 |
+
way would leave those SKUs permanently blank while an oracle counting active products
|
| 250 |
+
counted them — a red gate on a working build, or worse, a silent hole nobody counts.
|
| 251 |
+
|
| 252 |
+
**The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at
|
| 253 |
+
or below an order's quantity, because it is pricing a LINE that has one. A catalogue column
|
| 254 |
+
has no quantity in hand, so it takes the LOWEST `min_quantity` — the price at qty 1. Rules
|
| 255 |
+
above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only
|
| 256 |
+
rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all).
|
| 257 |
+
|
| 258 |
+
**Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant`
|
| 259 |
+
rule is the more specific statement about this exact SKU.
|
| 260 |
+
|
| 261 |
+
⚠ DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than
|
| 262 |
+
`catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is
|
| 263 |
+
momentarily unreachable is a worse failure than one with blank price columns. The blanks are
|
| 264 |
+
not silent — `product_data.validate()`'s coverage leg reconciles each column against a fresh
|
| 265 |
+
Odoo count and goes red at zero.
|
| 266 |
+
|
| 267 |
+
⭐ **The report exists because R6's second sentence is law** (*"if it can't be done, you need
|
| 268 |
+
to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is
|
| 269 |
+
counted rather than dropped: rules on pricelists the contract does not declare, non-fixed
|
| 270 |
+
(`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a
|
| 271 |
+
quantity break.
|
| 272 |
+
"""
|
| 273 |
+
# ⚠ THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a
|
| 274 |
+
# report and a misleading one. A rule is classified ONCE, by the first reason it is skipped:
|
| 275 |
+
# undeclared list → out of date → not fixed → global → zero price. So `rules_not_fixed: 0`
|
| 276 |
+
# means "no formula rule on a list we declare", NOT "this Odoo has no formula rules"
|
| 277 |
+
# (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the
|
| 278 |
+
# contract does not declare — so it lands in `rules_undeclared_list`).
|
| 279 |
+
report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0,
|
| 280 |
+
"rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0,
|
| 281 |
+
"rules_zero_price": 0, "qty_break_only": 0}
|
| 282 |
+
try:
|
| 283 |
+
pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
|
| 284 |
+
by_name = {}
|
| 285 |
+
for p in pl_rows:
|
| 286 |
+
by_name.setdefault(str(p.get('name') or '').strip(), p['id'])
|
| 287 |
+
wanted = {}
|
| 288 |
+
for col, name in PRICELIST_COLUMNS:
|
| 289 |
+
pid = by_name.get(name)
|
| 290 |
+
if pid is None:
|
| 291 |
+
report["lists_missing"].append(name)
|
| 292 |
+
else:
|
| 293 |
+
wanted[pid] = col
|
| 294 |
+
|
| 295 |
+
# ⭐⭐ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE.
|
| 296 |
+
# Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on
|
| 297 |
+
# this connection; the same rules under a server-side domain with 6 fields cost
|
| 298 |
+
# **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's
|
| 299 |
+
# first-ever build, so that 16.5s is 16.5s of somebody's page load.
|
| 300 |
+
#
|
| 301 |
+
# ⚠ The domain reproduces the Python predicate EXACTLY, and the null legs are the part
|
| 302 |
+
# that is easy to get wrong: an absent `date_start` is `False`, not a past date, so
|
| 303 |
+
# `('date_start','<=',today)` ALONE would drop every open-ended rule — which is almost
|
| 304 |
+
# all of them.
|
| 305 |
+
today = P.today().isoformat()
|
| 306 |
+
_declared = sorted(wanted)
|
| 307 |
+
_kinds = ['0_product_variant', '1_product']
|
| 308 |
+
_live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'),
|
| 309 |
+
('applied_on', 'in', _kinds),
|
| 310 |
+
'|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 311 |
+
'|', ('date_end', '=', False), ('date_end', '>=', today),
|
| 312 |
+
('fixed_price', '>', 0)]
|
| 313 |
+
rules = O.search_read(
|
| 314 |
+
'product.pricelist.item', _live,
|
| 315 |
+
['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
|
| 316 |
+
'min_quantity']) if _declared else []
|
| 317 |
+
|
| 318 |
+
# ⭐ R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six
|
| 319 |
+
# counts cost ~1.2s together; the rules they count would cost 16s to read. What this
|
| 320 |
+
# reader cannot see is still REPORTED — it is just no longer transferred.
|
| 321 |
+
def _n(extra):
|
| 322 |
+
try:
|
| 323 |
+
return O.get_odoo().search_count('product.pricelist.item', extra)
|
| 324 |
+
except Exception:
|
| 325 |
+
return -1 # -1 reads as "not measured", never as zero
|
| 326 |
+
_dated = ['|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 327 |
+
'|', ('date_end', '=', False), ('date_end', '>=', today)]
|
| 328 |
+
report["rules_total"] = _n([])
|
| 329 |
+
report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \
|
| 330 |
+
if _declared else report["rules_total"]
|
| 331 |
+
if _declared:
|
| 332 |
+
_on = [('pricelist_id', 'in', _declared)]
|
| 333 |
+
# `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot
|
| 334 |
+
# price is one the operator is TOLD about, never one that quietly becomes a blank.
|
| 335 |
+
report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')])
|
| 336 |
+
report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'),
|
| 337 |
+
('applied_on', 'not in', _kinds)])
|
| 338 |
+
report["rules_out_of_date"] = (
|
| 339 |
+
_n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)])
|
| 340 |
+
- _n(_on + [('compute_price', '=', 'fixed'),
|
| 341 |
+
('applied_on', 'in', _kinds)] + _dated))
|
| 342 |
+
# A zero price is not "free" — it is an unset rule. Blank says so; 0 does not.
|
| 343 |
+
report["rules_zero_price"] = _n(
|
| 344 |
+
_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]
|
| 345 |
+
+ _dated + [('fixed_price', '<=', 0)])
|
| 346 |
+
|
| 347 |
+
by_var, by_tmpl = {}, {}
|
| 348 |
+
for r in rules:
|
| 349 |
+
col = wanted[O.m2o_id(r.get('pricelist_id'))]
|
| 350 |
+
if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
|
| 351 |
+
by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r)
|
| 352 |
+
elif r.get('product_tmpl_id'):
|
| 353 |
+
by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r)
|
| 354 |
+
|
| 355 |
+
dom = [('active', '=', True)]
|
| 356 |
+
prods = O.search_read('product.product', dom,
|
| 357 |
+
['id', 'default_code', 'product_tmpl_id'], limit=50000)
|
| 358 |
+
n = O.get_odoo().search_count('product.product', dom)
|
| 359 |
+
if len(prods) != n:
|
| 360 |
+
# Same guard, same reason as `catalogue()`: a short pull renders as a plausible
|
| 361 |
+
# smaller set of priced SKUs with nothing reporting it.
|
| 362 |
+
raise ValueError(
|
| 363 |
+
f"products.pricelist_by_code: the product pull is TRUNCATED — read {len(prods)} "
|
| 364 |
+
f"rows against a search_count of {n}.")
|
| 365 |
+
except Exception as e:
|
| 366 |
+
report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 367 |
+
return {}, report
|
| 368 |
+
|
| 369 |
+
out = {}
|
| 370 |
+
for p in prods:
|
| 371 |
+
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 372 |
+
tmpl = O.m2o_id(p.get('product_tmpl_id'))
|
| 373 |
+
cells = {}
|
| 374 |
+
for col, _name in PRICELIST_COLUMNS:
|
| 375 |
+
cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl))
|
| 376 |
+
if not cands:
|
| 377 |
+
continue
|
| 378 |
+
base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
|
| 379 |
+
if (base.get('min_quantity') or 0.0) > 1.0:
|
| 380 |
+
report["qty_break_only"] += 1
|
| 381 |
+
cells[col] = base.get('fixed_price')
|
| 382 |
+
if cells:
|
| 383 |
+
out.setdefault(code, {}).update(cells)
|
| 384 |
+
return out, report
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
def catalogue_count():
|
| 388 |
+
"""The INDEPENDENT population oracle: Odoo's own count of active products.
|
| 389 |
+
|
| 390 |
+
Deliberately a bare `search_count` and not a `len()` over anything this module built — the
|
| 391 |
+
2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
|
| 392 |
+
of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
|
| 393 |
+
"""
|
| 394 |
+
return O.get_odoo().search_count('product.product', [('active', '=', True)])
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def categories(t=None, team_id=None):
|
| 398 |
+
"""Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
|
| 399 |
+
return sorted({r['category'] for r in directory(t, team_id)})
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def _coverage(date_from, date_to, team_id=None):
|
| 403 |
+
"""{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code.
|
| 404 |
+
(A customer buying two records sharing a code can count twice; duplicates are rare, and
|
| 405 |
+
code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)"""
|
| 406 |
+
g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
|
| 407 |
+
['__count'], ['product_id', 'order_partner_id'], lazy=False)
|
| 408 |
+
codes = _code_map()
|
| 409 |
+
cov = {}
|
| 410 |
+
names = {}
|
| 411 |
+
for r in g:
|
| 412 |
+
pid = O.m2o_id(r.get('product_id'))
|
| 413 |
+
if not pid:
|
| 414 |
+
continue
|
| 415 |
+
key = codes.get(pid, f"pid:{pid}")
|
| 416 |
+
cov[key] = cov.get(key, 0) + 1
|
| 417 |
+
names.setdefault(key, O.m2o_name(r.get('product_id')))
|
| 418 |
+
return cov, names
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None):
|
| 422 |
+
"""SKUs that lost the most customer breadth YoY — early warning a SKU is dying even if
|
| 423 |
+
revenue hasn't fully cratered yet."""
|
| 424 |
+
t = t or P.today()
|
| 425 |
+
yf, yt = P.ytd(t)
|
| 426 |
+
lf, lt = P.ytd_last_year(t)
|
| 427 |
+
cov_t, names_t = _coverage(yf, yt, team_id)
|
| 428 |
+
cov_l, names_l = _coverage(lf, lt, team_id)
|
| 429 |
+
rows = []
|
| 430 |
+
for pid, lc in cov_l.items():
|
| 431 |
+
if lc < min_prior_custs:
|
| 432 |
+
continue
|
| 433 |
+
tc = cov_t.get(pid, 0)
|
| 434 |
+
drop = lc - tc
|
| 435 |
+
if drop <= 0:
|
| 436 |
+
continue
|
| 437 |
+
rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '',
|
| 438 |
+
'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop,
|
| 439 |
+
'pct_drop': drop / lc * 100})
|
| 440 |
+
rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop']))
|
| 441 |
+
return rows[:limit]
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
# ====================================================================== SKU DRAWER (mirror of customer drawer)
|
| 445 |
+
def _sku_product_ids(code):
|
| 446 |
+
"""All product.product ids sharing this SKU code (merged variants / archived records)."""
|
| 447 |
+
return [pid for pid, c in _code_map().items() if c == code]
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _sku_dom(pids, date_from, date_to, team_id=None):
|
| 451 |
+
return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC)
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def _sku_name_category(pids):
|
| 455 |
+
rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id'])
|
| 456 |
+
name = rows[0]['name'] if rows else '(unknown)'
|
| 457 |
+
catmap = sales_mod._product_cat()
|
| 458 |
+
main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)')
|
| 459 |
+
return name, main
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
def _sku_buyers(pids, date_from, date_to, team_id=None):
|
| 463 |
+
"""{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window."""
|
| 464 |
+
g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id),
|
| 465 |
+
['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
|
| 466 |
+
out = {}
|
| 467 |
+
for r in g:
|
| 468 |
+
pid = O.m2o_id(r.get('order_partner_id'))
|
| 469 |
+
if pid:
|
| 470 |
+
out[pid] = {'name': O.m2o_name(r.get('order_partner_id')),
|
| 471 |
+
'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0}
|
| 472 |
+
return out
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None):
|
| 476 |
+
"""KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code.
|
| 477 |
+
Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read."""
|
| 478 |
+
t = t or P.today()
|
| 479 |
+
pids = _sku_product_ids(code)
|
| 480 |
+
if not pids:
|
| 481 |
+
return None
|
| 482 |
+
yf, yt = P.ytd(t)
|
| 483 |
+
lf, lt = P.ytd_last_year(t)
|
| 484 |
+
mf, mt = P.ltm(t)
|
| 485 |
+
name, category = _sku_name_category(pids)
|
| 486 |
+
|
| 487 |
+
def s(df, dtt, field='price_subtotal'):
|
| 488 |
+
return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field)
|
| 489 |
+
rev_ytd, rev_ly = s(yf, yt), s(lf, lt)
|
| 490 |
+
qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty')
|
| 491 |
+
buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id))
|
| 492 |
+
buyers_ly = len(_sku_buyers(pids, lf, lt, team_id))
|
| 493 |
+
g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id),
|
| 494 |
+
['price_subtotal:sum', 'margin:sum'], [], lazy=False)
|
| 495 |
+
line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0
|
| 496 |
+
margin = (g[0].get('margin') if g else 0) or 0.0
|
| 497 |
+
|
| 498 |
+
# Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected
|
| 499 |
+
# on sale.order.line), in 2 reads instead of 26 point queries.
|
| 500 |
+
range_start = dt.date(t.year - 2, t.month, 1).isoformat()
|
| 501 |
+
lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id),
|
| 502 |
+
['price_subtotal', 'order_id'])
|
| 503 |
+
oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')})
|
| 504 |
+
odate = {}
|
| 505 |
+
for i in range(0, len(oids), 1000):
|
| 506 |
+
for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']):
|
| 507 |
+
if o.get('date_order'):
|
| 508 |
+
odate[o['id']] = str(o['date_order'])[:7]
|
| 509 |
+
mrev = {}
|
| 510 |
+
for line in lines:
|
| 511 |
+
ym = odate.get(O.m2o_id(line.get('order_id')))
|
| 512 |
+
if ym:
|
| 513 |
+
mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0)
|
| 514 |
+
monthly = []
|
| 515 |
+
for ym, start, end in P.month_starts(n_months, t):
|
| 516 |
+
y, m = int(ym[:4]) - 1, int(ym[5:7])
|
| 517 |
+
monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0),
|
| 518 |
+
'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)})
|
| 519 |
+
|
| 520 |
+
allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id)
|
| 521 |
+
total = sum(v['rev'] for v in allsku.values()) or 1.0
|
| 522 |
+
rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev']))
|
| 523 |
+
if c == code), None)
|
| 524 |
+
return {
|
| 525 |
+
'code': code, 'name': name, 'category': category,
|
| 526 |
+
'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
|
| 527 |
+
'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly),
|
| 528 |
+
'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly,
|
| 529 |
+
'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin,
|
| 530 |
+
'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100,
|
| 531 |
+
'monthly': monthly,
|
| 532 |
+
}
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
def sku_buyer_bridge(code, t=None, team_id=None, top=12):
|
| 536 |
+
"""Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list."""
|
| 537 |
+
t = t or P.today()
|
| 538 |
+
pids = _sku_product_ids(code)
|
| 539 |
+
yf, yt = P.ytd(t)
|
| 540 |
+
lf, lt = P.ytd_last_year(t)
|
| 541 |
+
this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
|
| 542 |
+
tset, lset = set(this), set(last)
|
| 543 |
+
retained, new, churned = tset & lset, tset - lset, lset - tset
|
| 544 |
+
churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']}
|
| 545 |
+
for p in churned], key=lambda x: -x['ly_rev'])[:top]
|
| 546 |
+
return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)},
|
| 547 |
+
'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)},
|
| 548 |
+
'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)},
|
| 549 |
+
'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0,
|
| 550 |
+
'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset),
|
| 551 |
+
'this_total': sum(v['rev'] for v in this.values())}
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
def sku_concentration(code, t=None, team_id=None):
|
| 555 |
+
"""Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM)."""
|
| 556 |
+
t = t or P.today()
|
| 557 |
+
mf, mt = P.ltm(t)
|
| 558 |
+
buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id)
|
| 559 |
+
revs = sorted([v['rev'] for v in buyers.values()], reverse=True)
|
| 560 |
+
total = sum(revs) or 1.0
|
| 561 |
+
hhi = sum((r / total) ** 2 for r in revs)
|
| 562 |
+
return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0,
|
| 563 |
+
'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0,
|
| 564 |
+
'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0}
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def sku_price_dispersion(code, t=None, team_id=None, cap=60):
|
| 568 |
+
"""Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP
|
| 569 |
+
accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever."""
|
| 570 |
+
t = t or P.today()
|
| 571 |
+
mf, mt = P.ltm(t)
|
| 572 |
+
g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id),
|
| 573 |
+
['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
|
| 574 |
+
rows, tot_rev, tot_qty = [], 0.0, 0.0
|
| 575 |
+
for r in g:
|
| 576 |
+
pid = O.m2o_id(r.get('order_partner_id'))
|
| 577 |
+
rev = r.get('price_subtotal') or 0.0
|
| 578 |
+
qty = r.get('product_uom_qty') or 0.0
|
| 579 |
+
if not pid or qty <= 0:
|
| 580 |
+
continue
|
| 581 |
+
rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')),
|
| 582 |
+
'price': rev / qty, 'qty': qty, 'rev': rev})
|
| 583 |
+
tot_rev += rev
|
| 584 |
+
tot_qty += qty
|
| 585 |
+
vwap = (tot_rev / tot_qty) if tot_qty else 0.0
|
| 586 |
+
for r in rows:
|
| 587 |
+
r['recoverable'] = max(0.0, vwap - r['price']) * r['qty']
|
| 588 |
+
rows.sort(key=lambda x: -x['recoverable'])
|
| 589 |
+
return {'vwap': vwap, 'n_buyers': len(rows),
|
| 590 |
+
'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]}
|
| 591 |
+
|
| 592 |
+
|
| 593 |
+
def sku_top_buyers(code, t=None, team_id=None, top=15):
|
| 594 |
+
"""Ranked buyers of this SKU (YTD) with YoY — clickable to open the customer drawer."""
|
| 595 |
+
t = t or P.today()
|
| 596 |
+
pids = _sku_product_ids(code)
|
| 597 |
+
yf, yt = P.ytd(t)
|
| 598 |
+
lf, lt = P.ytd_last_year(t)
|
| 599 |
+
this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
|
| 600 |
+
rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'],
|
| 601 |
+
'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()]
|
| 602 |
+
rows.sort(key=lambda x: -x['rev'])
|
| 603 |
+
return rows[:top]
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
_BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10–25k)', 10000.0),
|
| 607 |
+
('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)]
|
| 608 |
+
|
| 609 |
+
|
| 610 |
+
def sku_customer_analysis(code, t=None, team_id=None, top=15):
|
| 611 |
+
"""WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL
|
| 612 |
+
spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend — who gets hurt
|
| 613 |
+
most if it stocks out / who to protect)."""
|
| 614 |
+
t = t or P.today()
|
| 615 |
+
lf, lt = P.ltm(t)
|
| 616 |
+
pids = _sku_product_ids(code)
|
| 617 |
+
empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0}
|
| 618 |
+
if not pids:
|
| 619 |
+
return empty
|
| 620 |
+
buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}}
|
| 621 |
+
if not buyers:
|
| 622 |
+
return empty
|
| 623 |
+
bpids = list(buyers)
|
| 624 |
+
g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)],
|
| 625 |
+
['amount_untaxed:sum'], ['partner_id'], lazy=False)
|
| 626 |
+
total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')}
|
| 627 |
+
|
| 628 |
+
def tier(rev):
|
| 629 |
+
for nm, lo in _BUYER_TIERS:
|
| 630 |
+
if rev >= lo:
|
| 631 |
+
return nm
|
| 632 |
+
return _BUYER_TIERS[-1][0]
|
| 633 |
+
seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS}
|
| 634 |
+
dep = []
|
| 635 |
+
for p, v in buyers.items():
|
| 636 |
+
ct = total.get(p, v['rev']) or v['rev']
|
| 637 |
+
s = seg[tier(ct)]
|
| 638 |
+
s['buyers'] += 1
|
| 639 |
+
s['sku_rev'] += v['rev']
|
| 640 |
+
dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct,
|
| 641 |
+
'dependency_pct': (v['rev'] / ct * 100) if ct else None})
|
| 642 |
+
tot = sum(s['sku_rev'] for s in seg.values()) or 1.0
|
| 643 |
+
segments = []
|
| 644 |
+
for nm, _ in _BUYER_TIERS:
|
| 645 |
+
s = seg[nm]
|
| 646 |
+
s['rev_share'] = s['sku_rev'] / tot * 100
|
| 647 |
+
s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0
|
| 648 |
+
segments.append(s)
|
| 649 |
+
dep.sort(key=lambda x: -x['sku_rev'])
|
| 650 |
+
deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None]
|
| 651 |
+
return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers),
|
| 652 |
+
'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0}
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def sku_whitespace(code, t=None, team_id=None, top=15):
|
| 656 |
+
"""Customers who buy this SKU's category but NOT this SKU — ranked prospect list."""
|
| 657 |
+
import modules.customers as cust_mod
|
| 658 |
+
t = t or P.today()
|
| 659 |
+
pids = _sku_product_ids(code)
|
| 660 |
+
_name, category = _sku_name_category(pids)
|
| 661 |
+
cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set()
|
| 662 |
+
mf, mt = P.ltm(t)
|
| 663 |
+
prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id)))
|
| 664 |
+
if not prospects:
|
| 665 |
+
return {'category': category, 'rows': []}
|
| 666 |
+
g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)],
|
| 667 |
+
['amount_untaxed:sum'], ['partner_id'], lazy=False)
|
| 668 |
+
rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']),
|
| 669 |
+
'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')]
|
| 670 |
+
rows.sort(key=lambda x: -x['total_spend'])
|
| 671 |
+
return {'category': category, 'rows': rows[:top]}
|
| 672 |
+
|
| 673 |
+
|
| 674 |
+
def sku_drawer_bundle(code, t=None, team_id=None, allsku=None):
|
| 675 |
+
"""The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate),
|
| 676 |
+
then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call)."""
|
| 677 |
+
detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku)
|
| 678 |
+
if detail is None:
|
| 679 |
+
return {'detail': None}
|
| 680 |
+
bridge, conc, price, buyers, white, ca = O.parallel([
|
| 681 |
+
lambda: sku_buyer_bridge(code, t, team_id),
|
| 682 |
+
lambda: sku_concentration(code, t, team_id),
|
| 683 |
+
lambda: sku_price_dispersion(code, t, team_id),
|
| 684 |
+
lambda: sku_top_buyers(code, t, team_id),
|
| 685 |
+
lambda: sku_whitespace(code, t, team_id),
|
| 686 |
+
lambda: sku_customer_analysis(code, t, team_id),
|
| 687 |
+
])
|
| 688 |
+
return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price,
|
| 689 |
+
'buyers': buyers, 'white': white, 'ca': ca}
|
| 690 |
+
|
| 691 |
+
|
| 692 |
+
def validate(t=None, team_id=None):
|
| 693 |
+
"""Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU
|
| 694 |
+
selected) every check runs SCOPED to that BU, so the validation panel never reconciles
|
| 695 |
+
against — or exposes — the other BU's numbers. All checks here are BU-scopeable (no cross-BU
|
| 696 |
+
mirror), so team_id threads straight through."""
|
| 697 |
+
t = t or P.today()
|
| 698 |
+
yf, yt = P.ytd(t)
|
| 699 |
+
checks = []
|
| 700 |
+
sku = _sku_rev(yf, yt, team_id)
|
| 701 |
+
sku_sum = sum(v['rev'] for v in sku.values())
|
| 702 |
+
line_total = O.sum_field('sale.order.line',
|
| 703 |
+
O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal')
|
| 704 |
+
checks.append({'check': 'SKU rev: Σ(per-SKU) == total line revenue, ex-services (YTD)',
|
| 705 |
+
'a': round(sku_sum, 2), 'b': round(line_total, 2),
|
| 706 |
+
'gap': round(sku_sum - line_total, 2),
|
| 707 |
+
'ok': abs(sku_sum - line_total) <= 1.0})
|
| 708 |
+
|
| 709 |
+
m = yoy_movers(t, limit=10**9, team_id=team_id)
|
| 710 |
+
movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners'])
|
| 711 |
+
lf, lt = P.ytd_last_year(t)
|
| 712 |
+
last_total = O.sum_field('sale.order.line',
|
| 713 |
+
O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal')
|
| 714 |
+
checks.append({'check': 'SKU movers: Σ(Δ) == (YTD − LY) total',
|
| 715 |
+
'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2),
|
| 716 |
+
'gap': round(movers_sum - (line_total - last_total), 2),
|
| 717 |
+
'ok': abs(movers_sum - (line_total - last_total)) <= 1.0})
|
| 718 |
+
|
| 719 |
+
# SKU drawer: buyer-bridge reconciles — retained + new buyer revenue == this-period SKU revenue
|
| 720 |
+
top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
|
| 721 |
+
if top_code:
|
| 722 |
+
bb = sku_buyer_bridge(top_code, t, team_id=team_id)
|
| 723 |
+
recon = bb['retained']['rev'] + bb['new']['rev']
|
| 724 |
+
checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
|
| 725 |
+
'a': round(recon, 2), 'b': round(bb['this_total'], 2),
|
| 726 |
+
'gap': round(recon - bb['this_total'], 2),
|
| 727 |
+
'ok': abs(recon - bb['this_total']) <= 1.0})
|
| 728 |
+
return checks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/public/sample_customers.json
CHANGED
|
@@ -1,558 +1,545 @@
|
|
| 1 |
-
{
|
| 2 |
-
"fields": [
|
| 3 |
-
{
|
| 4 |
-
"key": "customer",
|
| 5 |
-
"label": "Customer",
|
| 6 |
-
"type": "text",
|
| 7 |
-
"source": "odoo",
|
| 8 |
-
"pinned": true,
|
| 9 |
-
"default": true,
|
| 10 |
-
"description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 11 |
-
},
|
| 12 |
-
{
|
| 13 |
-
"key": "
|
| 14 |
-
"label": "Odoo
|
| 15 |
-
"type": "
|
| 16 |
-
"source": "odoo",
|
| 17 |
-
"
|
| 18 |
-
"
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
"
|
| 23 |
-
"
|
| 24 |
-
"
|
| 25 |
-
"
|
| 26 |
-
"
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
"
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
"
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
"description": "The
|
| 40 |
-
},
|
| 41 |
-
{
|
| 42 |
-
"key": "
|
| 43 |
-
"label": "
|
| 44 |
-
"type": "
|
| 45 |
-
"source": "odoo",
|
| 46 |
-
"default": false,
|
| 47 |
-
"
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
"
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
"
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
"
|
| 59 |
-
"
|
| 60 |
-
"
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
"
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
"
|
| 67 |
-
"
|
| 68 |
-
"
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
"
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
"
|
| 75 |
-
"
|
| 76 |
-
"
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
"
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
"
|
| 83 |
-
"
|
| 84 |
-
"
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
"
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
"
|
| 91 |
-
"
|
| 92 |
-
"
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
"
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
"
|
| 99 |
-
"
|
| 100 |
-
"
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
"
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
"
|
| 107 |
-
"
|
| 108 |
-
"
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
"
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
"
|
| 115 |
-
"
|
| 116 |
-
"
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
"
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
"
|
| 123 |
-
"
|
| 124 |
-
"
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
"
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
"
|
| 131 |
-
"
|
| 132 |
-
"
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
"
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
"
|
| 139 |
-
"
|
| 140 |
-
"
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
"
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
"
|
| 147 |
-
"
|
| 148 |
-
"
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
"
|
| 152 |
-
"
|
| 153 |
-
"
|
| 154 |
-
"
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
"
|
| 160 |
-
"
|
| 161 |
-
"
|
| 162 |
-
"
|
| 163 |
-
"
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
"
|
| 167 |
-
"
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
"
|
| 171 |
-
"
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
"
|
| 175 |
-
"
|
| 176 |
-
"
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
"
|
| 180 |
-
"
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
"
|
| 184 |
-
"
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
"
|
| 188 |
-
"
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
"
|
| 192 |
-
"
|
| 193 |
-
"
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
"
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
"
|
| 200 |
-
"
|
| 201 |
-
"
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
"
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
"
|
| 208 |
-
"
|
| 209 |
-
"
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
"
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
"
|
| 216 |
-
"
|
| 217 |
-
"
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
"
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
"
|
| 224 |
-
"
|
| 225 |
-
"
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
"
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
"
|
| 232 |
-
"
|
| 233 |
-
"
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
"
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
"
|
| 240 |
-
"
|
| 241 |
-
"
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
"
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
"
|
| 248 |
-
"
|
| 249 |
-
"
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
"
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
"
|
| 256 |
-
"
|
| 257 |
-
"
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
"
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
"
|
| 264 |
-
"
|
| 265 |
-
"
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
"
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
"
|
| 272 |
-
"
|
| 273 |
-
"
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
"
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
"
|
| 280 |
-
"
|
| 281 |
-
"
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
"
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
"
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
"
|
| 293 |
-
"
|
| 294 |
-
"
|
| 295 |
-
"
|
| 296 |
-
"
|
| 297 |
-
"
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
"
|
| 303 |
-
"
|
| 304 |
-
"
|
| 305 |
-
"
|
| 306 |
-
"
|
| 307 |
-
"
|
| 308 |
-
"
|
| 309 |
-
"
|
| 310 |
-
"
|
| 311 |
-
"
|
| 312 |
-
"
|
| 313 |
-
"
|
| 314 |
-
"
|
| 315 |
-
"
|
| 316 |
-
"
|
| 317 |
-
"
|
| 318 |
-
"
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
"
|
| 322 |
-
"
|
| 323 |
-
"
|
| 324 |
-
"
|
| 325 |
-
"
|
| 326 |
-
"
|
| 327 |
-
"
|
| 328 |
-
"
|
| 329 |
-
"
|
| 330 |
-
"
|
| 331 |
-
"
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
"
|
| 335 |
-
"
|
| 336 |
-
"
|
| 337 |
-
"
|
| 338 |
-
"
|
| 339 |
-
"
|
| 340 |
-
"
|
| 341 |
-
"
|
| 342 |
-
"
|
| 343 |
-
"
|
| 344 |
-
"
|
| 345 |
-
"
|
| 346 |
-
"
|
| 347 |
-
"
|
| 348 |
-
"
|
| 349 |
-
"
|
| 350 |
-
"
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
"
|
| 354 |
-
"
|
| 355 |
-
"
|
| 356 |
-
"
|
| 357 |
-
"
|
| 358 |
-
"
|
| 359 |
-
"
|
| 360 |
-
"
|
| 361 |
-
"
|
| 362 |
-
"
|
| 363 |
-
"
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
"
|
| 367 |
-
"
|
| 368 |
-
"
|
| 369 |
-
"
|
| 370 |
-
"
|
| 371 |
-
"
|
| 372 |
-
"
|
| 373 |
-
"
|
| 374 |
-
"
|
| 375 |
-
"
|
| 376 |
-
"
|
| 377 |
-
"
|
| 378 |
-
"
|
| 379 |
-
"
|
| 380 |
-
"
|
| 381 |
-
"
|
| 382 |
-
"
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
"
|
| 386 |
-
"
|
| 387 |
-
"
|
| 388 |
-
"
|
| 389 |
-
"
|
| 390 |
-
"
|
| 391 |
-
"
|
| 392 |
-
"
|
| 393 |
-
"
|
| 394 |
-
"
|
| 395 |
-
"
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
"
|
| 399 |
-
"
|
| 400 |
-
"
|
| 401 |
-
"
|
| 402 |
-
"
|
| 403 |
-
"
|
| 404 |
-
"
|
| 405 |
-
"
|
| 406 |
-
"
|
| 407 |
-
"
|
| 408 |
-
"
|
| 409 |
-
"
|
| 410 |
-
"
|
| 411 |
-
"
|
| 412 |
-
"
|
| 413 |
-
"
|
| 414 |
-
"
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
"
|
| 418 |
-
"
|
| 419 |
-
"
|
| 420 |
-
"
|
| 421 |
-
"
|
| 422 |
-
"
|
| 423 |
-
"
|
| 424 |
-
"
|
| 425 |
-
"
|
| 426 |
-
"
|
| 427 |
-
"
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
"
|
| 431 |
-
"
|
| 432 |
-
"
|
| 433 |
-
"
|
| 434 |
-
"
|
| 435 |
-
"
|
| 436 |
-
"
|
| 437 |
-
"
|
| 438 |
-
"
|
| 439 |
-
"
|
| 440 |
-
"
|
| 441 |
-
"
|
| 442 |
-
"
|
| 443 |
-
"
|
| 444 |
-
"
|
| 445 |
-
"
|
| 446 |
-
"
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
"
|
| 450 |
-
"
|
| 451 |
-
"
|
| 452 |
-
"
|
| 453 |
-
"
|
| 454 |
-
"
|
| 455 |
-
"
|
| 456 |
-
"
|
| 457 |
-
"
|
| 458 |
-
"
|
| 459 |
-
"
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
"
|
| 463 |
-
"
|
| 464 |
-
"
|
| 465 |
-
"
|
| 466 |
-
"
|
| 467 |
-
"
|
| 468 |
-
"
|
| 469 |
-
"
|
| 470 |
-
"
|
| 471 |
-
"
|
| 472 |
-
"
|
| 473 |
-
"
|
| 474 |
-
"
|
| 475 |
-
"
|
| 476 |
-
"
|
| 477 |
-
"
|
| 478 |
-
"
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
"
|
| 482 |
-
"
|
| 483 |
-
"
|
| 484 |
-
"
|
| 485 |
-
"
|
| 486 |
-
"
|
| 487 |
-
"
|
| 488 |
-
"
|
| 489 |
-
"
|
| 490 |
-
"
|
| 491 |
-
"
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
"
|
| 495 |
-
"
|
| 496 |
-
"
|
| 497 |
-
"
|
| 498 |
-
"
|
| 499 |
-
"
|
| 500 |
-
"
|
| 501 |
-
"
|
| 502 |
-
"
|
| 503 |
-
"
|
| 504 |
-
"
|
| 505 |
-
"
|
| 506 |
-
"
|
| 507 |
-
"
|
| 508 |
-
"
|
| 509 |
-
"
|
| 510 |
-
"
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
"
|
| 514 |
-
"
|
| 515 |
-
"
|
| 516 |
-
"
|
| 517 |
-
"
|
| 518 |
-
"
|
| 519 |
-
"
|
| 520 |
-
"
|
| 521 |
-
"
|
| 522 |
-
"
|
| 523 |
-
"
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
"
|
| 527 |
-
"
|
| 528 |
-
"
|
| 529 |
-
"
|
| 530 |
-
"
|
| 531 |
-
"
|
| 532 |
-
"
|
| 533 |
-
"
|
| 534 |
-
"
|
| 535 |
-
"
|
| 536 |
-
"
|
| 537 |
-
"
|
| 538 |
-
"
|
| 539 |
-
"
|
| 540 |
-
"
|
| 541 |
-
"
|
| 542 |
-
"
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
"top_category_pct": 1.0,
|
| 547 |
-
"sku_count": 1,
|
| 548 |
-
"top_sku": "(none)",
|
| 549 |
-
"days_to_pay": 12,
|
| 550 |
-
"_created": "2023-08-15 09:17:00",
|
| 551 |
-
"lat": 33.749,
|
| 552 |
-
"lon": -84.388,
|
| 553 |
-
"odoo_status": "Active",
|
| 554 |
-
"dba": "Royal",
|
| 555 |
-
"ar_outstanding": 9240
|
| 556 |
-
}
|
| 557 |
-
]
|
| 558 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fields": [
|
| 3 |
+
{
|
| 4 |
+
"key": "customer",
|
| 5 |
+
"label": "Customer",
|
| 6 |
+
"type": "text",
|
| 7 |
+
"source": "odoo",
|
| 8 |
+
"pinned": true,
|
| 9 |
+
"default": true,
|
| 10 |
+
"description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"key": "odoo_status",
|
| 14 |
+
"label": "Odoo record",
|
| 15 |
+
"type": "status",
|
| 16 |
+
"source": "odoo",
|
| 17 |
+
"default": false,
|
| 18 |
+
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"key": "agent",
|
| 22 |
+
"label": "Agent",
|
| 23 |
+
"type": "text",
|
| 24 |
+
"source": "odoo",
|
| 25 |
+
"default": true,
|
| 26 |
+
"description": "The sales agent who owns this account."
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"key": "dba",
|
| 30 |
+
"label": "DBA",
|
| 31 |
+
"type": "select",
|
| 32 |
+
"source": "odoo",
|
| 33 |
+
"default": false,
|
| 34 |
+
"options": [
|
| 35 |
+
"Fisch",
|
| 36 |
+
"Royal",
|
| 37 |
+
"Both"
|
| 38 |
+
],
|
| 39 |
+
"description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"key": "salesperson",
|
| 43 |
+
"label": "Salesperson",
|
| 44 |
+
"type": "text",
|
| 45 |
+
"source": "odoo",
|
| 46 |
+
"default": false,
|
| 47 |
+
"description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"key": "street",
|
| 51 |
+
"label": "Street",
|
| 52 |
+
"type": "text",
|
| 53 |
+
"source": "odoo",
|
| 54 |
+
"default": false,
|
| 55 |
+
"description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"key": "street2",
|
| 59 |
+
"label": "Street 2",
|
| 60 |
+
"type": "text",
|
| 61 |
+
"source": "odoo",
|
| 62 |
+
"default": false,
|
| 63 |
+
"description": "Second address line (suite, unit, floor) on the customer's Odoo address."
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"key": "city",
|
| 67 |
+
"label": "City",
|
| 68 |
+
"type": "text",
|
| 69 |
+
"source": "odoo",
|
| 70 |
+
"default": true,
|
| 71 |
+
"description": "City on the customer's Odoo address."
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"key": "state",
|
| 75 |
+
"label": "State",
|
| 76 |
+
"type": "text",
|
| 77 |
+
"source": "odoo",
|
| 78 |
+
"default": true,
|
| 79 |
+
"description": "State or province on the customer's Odoo address."
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"key": "country",
|
| 83 |
+
"label": "Country",
|
| 84 |
+
"type": "text",
|
| 85 |
+
"source": "odoo",
|
| 86 |
+
"default": false,
|
| 87 |
+
"description": "Country on the customer's Odoo address."
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"key": "zip",
|
| 91 |
+
"label": "ZIP",
|
| 92 |
+
"type": "text",
|
| 93 |
+
"source": "odoo",
|
| 94 |
+
"default": false,
|
| 95 |
+
"description": "Postal code on the customer's Odoo address."
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"key": "customer_since",
|
| 99 |
+
"label": "Customer since",
|
| 100 |
+
"type": "date",
|
| 101 |
+
"source": "odoo",
|
| 102 |
+
"default": false,
|
| 103 |
+
"description": "When this customer was first set up in Odoo."
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"key": "tags",
|
| 107 |
+
"label": "Tags",
|
| 108 |
+
"type": "text",
|
| 109 |
+
"source": "odoo",
|
| 110 |
+
"default": false,
|
| 111 |
+
"description": "Odoo labels on this customer, comma-separated."
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"key": "pricelist",
|
| 115 |
+
"label": "Price list",
|
| 116 |
+
"type": "text",
|
| 117 |
+
"source": "odoo",
|
| 118 |
+
"default": false,
|
| 119 |
+
"description": "The price list this customer buys on."
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"key": "payment_terms",
|
| 123 |
+
"label": "Payment terms",
|
| 124 |
+
"type": "text",
|
| 125 |
+
"source": "odoo",
|
| 126 |
+
"default": false,
|
| 127 |
+
"description": "Payment terms on this customer's account — Net 30, for example."
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"key": "last_order",
|
| 131 |
+
"label": "Last order",
|
| 132 |
+
"type": "date",
|
| 133 |
+
"source": "odoo",
|
| 134 |
+
"default": true,
|
| 135 |
+
"description": "Date of the most recent confirmed order."
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"key": "overdue_days",
|
| 139 |
+
"label": "Overdue days",
|
| 140 |
+
"type": "int",
|
| 141 |
+
"source": "odoo",
|
| 142 |
+
"default": true,
|
| 143 |
+
"description": "How many days late this customer is running against their own usual ordering rhythm."
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
|
| 147 |
+
"key": "est_missed",
|
| 148 |
+
"label": "Est. missed $",
|
| 149 |
+
"type": "currency",
|
| 150 |
+
"source": "odoo",
|
| 151 |
+
"default": true,
|
| 152 |
+
"agg": "sum",
|
| 153 |
+
"filterable": false,
|
| 154 |
+
"description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 155 |
+
},
|
| 156 |
+
{
|
| 157 |
+
"_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
|
| 158 |
+
"key": "ar_open",
|
| 159 |
+
"label": "AR current $",
|
| 160 |
+
"type": "currency",
|
| 161 |
+
"source": "odoo",
|
| 162 |
+
"default": false,
|
| 163 |
+
"description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
|
| 164 |
+
},
|
| 165 |
+
{
|
| 166 |
+
"key": "ar_overdue",
|
| 167 |
+
"label": "AR overdue $",
|
| 168 |
+
"type": "currency",
|
| 169 |
+
"source": "odoo",
|
| 170 |
+
"default": false,
|
| 171 |
+
"description": "Invoiced money past due — same basis as the Collections page."
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
|
| 175 |
+
"key": "ar_outstanding",
|
| 176 |
+
"label": "AR outstanding $",
|
| 177 |
+
"type": "currency",
|
| 178 |
+
"source": "odoo",
|
| 179 |
+
"default": false,
|
| 180 |
+
"description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"key": "ar_exposure",
|
| 184 |
+
"label": "Credit exposure $",
|
| 185 |
+
"type": "currency",
|
| 186 |
+
"source": "odoo",
|
| 187 |
+
"default": false,
|
| 188 |
+
"description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"key": "ar_aged_1_30",
|
| 192 |
+
"label": "1-30 days $",
|
| 193 |
+
"type": "currency",
|
| 194 |
+
"source": "odoo",
|
| 195 |
+
"default": false,
|
| 196 |
+
"description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
|
| 197 |
+
},
|
| 198 |
+
{
|
| 199 |
+
"key": "ar_aged_31_60",
|
| 200 |
+
"label": "31-60 days $",
|
| 201 |
+
"type": "currency",
|
| 202 |
+
"source": "odoo",
|
| 203 |
+
"default": false,
|
| 204 |
+
"description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
|
| 205 |
+
},
|
| 206 |
+
{
|
| 207 |
+
"key": "ar_aged_61_90",
|
| 208 |
+
"label": "61-90 days $",
|
| 209 |
+
"type": "currency",
|
| 210 |
+
"source": "odoo",
|
| 211 |
+
"default": false,
|
| 212 |
+
"description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
|
| 213 |
+
},
|
| 214 |
+
{
|
| 215 |
+
"key": "ar_aged_90_plus",
|
| 216 |
+
"label": "90+ days $",
|
| 217 |
+
"type": "currency",
|
| 218 |
+
"source": "odoo",
|
| 219 |
+
"default": false,
|
| 220 |
+
"description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"key": "days_to_pay",
|
| 224 |
+
"label": "Days to pay",
|
| 225 |
+
"type": "int",
|
| 226 |
+
"source": "odoo",
|
| 227 |
+
"default": false,
|
| 228 |
+
"description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"key": "top_category",
|
| 232 |
+
"label": "Top category",
|
| 233 |
+
"type": "text",
|
| 234 |
+
"source": "odoo",
|
| 235 |
+
"default": false,
|
| 236 |
+
"description": "The category this customer spent the most on in the last 12 months."
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"key": "top_category_pct",
|
| 240 |
+
"label": "Top category %",
|
| 241 |
+
"type": "pct",
|
| 242 |
+
"source": "odoo",
|
| 243 |
+
"default": false,
|
| 244 |
+
"description": "Share of last-12-months spend that went to the top category."
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"key": "sku_count",
|
| 248 |
+
"label": "SKUs bought",
|
| 249 |
+
"type": "int",
|
| 250 |
+
"source": "odoo",
|
| 251 |
+
"default": false,
|
| 252 |
+
"description": "Distinct products bought in the last 12 months."
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"key": "top_sku",
|
| 256 |
+
"label": "Top SKU",
|
| 257 |
+
"type": "text",
|
| 258 |
+
"source": "odoo",
|
| 259 |
+
"default": false,
|
| 260 |
+
"description": "The product this customer spent the most on in the last 12 months."
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"key": "days_since",
|
| 264 |
+
"label": "Days since order",
|
| 265 |
+
"type": "int",
|
| 266 |
+
"source": "odoo",
|
| 267 |
+
"default": false,
|
| 268 |
+
"description": "Days since the last confirmed order."
|
| 269 |
+
},
|
| 270 |
+
{
|
| 271 |
+
"key": "typical_gap_days",
|
| 272 |
+
"label": "Typical gap days",
|
| 273 |
+
"type": "int",
|
| 274 |
+
"source": "odoo",
|
| 275 |
+
"default": false,
|
| 276 |
+
"description": "Days this customer usually goes between orders, from their own history."
|
| 277 |
+
},
|
| 278 |
+
{
|
| 279 |
+
"key": "notes",
|
| 280 |
+
"label": "Notes",
|
| 281 |
+
"type": "text",
|
| 282 |
+
"source": "overlay",
|
| 283 |
+
"default": false,
|
| 284 |
+
"description": "Your notes on this customer. Saved in this app only, visible only to you."
|
| 285 |
+
}
|
| 286 |
+
],
|
| 287 |
+
"rows": [
|
| 288 |
+
{
|
| 289 |
+
"pid": 101,
|
| 290 |
+
"customer": "Poppy Flowers",
|
| 291 |
+
"status": "New",
|
| 292 |
+
"agent": "Naomi Linnell Rivera",
|
| 293 |
+
"city": "Charlottesville",
|
| 294 |
+
"state": "Virginia (US)",
|
| 295 |
+
"last_order": "2026-07-21",
|
| 296 |
+
"est_missed": 0,
|
| 297 |
+
"notes": "",
|
| 298 |
+
"country": "United States",
|
| 299 |
+
"zip": "02720",
|
| 300 |
+
"payment_terms": "30 Days",
|
| 301 |
+
"pricelist": "Fisch 1 (USD)",
|
| 302 |
+
"tags": "Royal",
|
| 303 |
+
"customer_since": "2023-01-10",
|
| 304 |
+
"salesperson": "Jessica",
|
| 305 |
+
"ar_open": 0,
|
| 306 |
+
"ar_overdue": 0,
|
| 307 |
+
"ar_exposure": 0,
|
| 308 |
+
"top_category": "Styrofoam",
|
| 309 |
+
"top_category_pct": 0.47,
|
| 310 |
+
"sku_count": 93,
|
| 311 |
+
"top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
|
| 312 |
+
"days_to_pay": 34,
|
| 313 |
+
"_created": "2023-01-15 09:10:00",
|
| 314 |
+
"lat": 25.7617,
|
| 315 |
+
"lon": -80.1918,
|
| 316 |
+
"odoo_status": "Archived",
|
| 317 |
+
"dba": "Royal",
|
| 318 |
+
"ar_outstanding": 0
|
| 319 |
+
},
|
| 320 |
+
{
|
| 321 |
+
"pid": 102,
|
| 322 |
+
"customer": "Meadow & Vine Wholesale",
|
| 323 |
+
"status": "Growing",
|
| 324 |
+
"agent": "Carla Jimenez",
|
| 325 |
+
"city": "Portland",
|
| 326 |
+
"state": "Oregon (US)",
|
| 327 |
+
"last_order": "2026-07-19",
|
| 328 |
+
"est_missed": 0,
|
| 329 |
+
"notes": "Expanding to a second storefront.",
|
| 330 |
+
"country": "United States",
|
| 331 |
+
"zip": "77041",
|
| 332 |
+
"payment_terms": "Immediate Payment",
|
| 333 |
+
"pricelist": "Royal 1 (USD)",
|
| 334 |
+
"tags": "Royal, Key account",
|
| 335 |
+
"customer_since": "2024-02-11",
|
| 336 |
+
"salesperson": "Naomi",
|
| 337 |
+
"ar_open": 1240.5,
|
| 338 |
+
"ar_overdue": 0,
|
| 339 |
+
"ar_exposure": 1740.5,
|
| 340 |
+
"top_category": "Ribbon",
|
| 341 |
+
"top_category_pct": 0.95,
|
| 342 |
+
"sku_count": 4,
|
| 343 |
+
"top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
|
| 344 |
+
"days_to_pay": null,
|
| 345 |
+
"_created": "2023-02-15 09:11:00",
|
| 346 |
+
"lat": 27.9506,
|
| 347 |
+
"lon": -82.4572,
|
| 348 |
+
"odoo_status": "Active",
|
| 349 |
+
"dba": "Fisch",
|
| 350 |
+
"ar_outstanding": 1240.5
|
| 351 |
+
},
|
| 352 |
+
{
|
| 353 |
+
"pid": 103,
|
| 354 |
+
"customer": "Bluestem Floral Supply",
|
| 355 |
+
"status": "Growing",
|
| 356 |
+
"agent": "Naomi Linnell Rivera",
|
| 357 |
+
"city": "Kansas City",
|
| 358 |
+
"state": "Missouri (US)",
|
| 359 |
+
"last_order": "2026-07-17",
|
| 360 |
+
"est_missed": 0,
|
| 361 |
+
"notes": "",
|
| 362 |
+
"country": "United States",
|
| 363 |
+
"zip": "11219",
|
| 364 |
+
"payment_terms": "60 Days",
|
| 365 |
+
"pricelist": "Royal 1 (USD)",
|
| 366 |
+
"tags": "Fisch",
|
| 367 |
+
"customer_since": "2025-03-12",
|
| 368 |
+
"salesperson": "Karen",
|
| 369 |
+
"ar_open": 0,
|
| 370 |
+
"ar_overdue": 5120.25,
|
| 371 |
+
"ar_exposure": 5120.25,
|
| 372 |
+
"top_category": "Foams & Finishes",
|
| 373 |
+
"top_category_pct": 0.31,
|
| 374 |
+
"sku_count": 27,
|
| 375 |
+
"top_sku": "SATIN RIBBON 2IN",
|
| 376 |
+
"days_to_pay": 61,
|
| 377 |
+
"_created": "2023-03-15 09:12:00",
|
| 378 |
+
"lat": 28.5384,
|
| 379 |
+
"lon": -81.3789,
|
| 380 |
+
"odoo_status": "Active",
|
| 381 |
+
"dba": "Both",
|
| 382 |
+
"ar_outstanding": 5120.25
|
| 383 |
+
},
|
| 384 |
+
{
|
| 385 |
+
"pid": 104,
|
| 386 |
+
"customer": "Camellia Row Florist",
|
| 387 |
+
"status": "Declining",
|
| 388 |
+
"agent": "Devon Marsh",
|
| 389 |
+
"city": "Savannah",
|
| 390 |
+
"state": "Georgia (US)",
|
| 391 |
+
"last_order": "2026-05-30",
|
| 392 |
+
"est_missed": 41200,
|
| 393 |
+
"notes": "Switched some volume to a local grower.",
|
| 394 |
+
"country": "United States",
|
| 395 |
+
"zip": "07649",
|
| 396 |
+
"payment_terms": "30 Days",
|
| 397 |
+
"pricelist": "Fisch 1 (USD)",
|
| 398 |
+
"tags": "(none)",
|
| 399 |
+
"customer_since": "2026-04-13",
|
| 400 |
+
"salesperson": "(none)",
|
| 401 |
+
"ar_open": 8300,
|
| 402 |
+
"ar_overdue": 940,
|
| 403 |
+
"ar_exposure": 11440,
|
| 404 |
+
"top_category": "All",
|
| 405 |
+
"top_category_pct": 1.0,
|
| 406 |
+
"sku_count": 1,
|
| 407 |
+
"top_sku": "(none)",
|
| 408 |
+
"days_to_pay": 12,
|
| 409 |
+
"_created": "2023-04-15 09:13:00",
|
| 410 |
+
"lat": 30.3322,
|
| 411 |
+
"lon": -81.6557,
|
| 412 |
+
"odoo_status": "Active",
|
| 413 |
+
"dba": "Royal",
|
| 414 |
+
"ar_outstanding": 9240
|
| 415 |
+
},
|
| 416 |
+
{
|
| 417 |
+
"pid": 105,
|
| 418 |
+
"customer": "Harborlight Wholesale Blooms",
|
| 419 |
+
"status": "Growing",
|
| 420 |
+
"agent": "Carla Jimenez",
|
| 421 |
+
"city": "Seattle",
|
| 422 |
+
"state": "Washington (US)",
|
| 423 |
+
"last_order": "2026-07-22",
|
| 424 |
+
"est_missed": 0,
|
| 425 |
+
"notes": "Top-10 account.",
|
| 426 |
+
"country": "United States",
|
| 427 |
+
"zip": "33125",
|
| 428 |
+
"payment_terms": "Immediate Payment",
|
| 429 |
+
"pricelist": "Royal 1 (USD)",
|
| 430 |
+
"tags": "Royal",
|
| 431 |
+
"customer_since": "2023-05-14",
|
| 432 |
+
"salesperson": "Jessica",
|
| 433 |
+
"ar_open": 0,
|
| 434 |
+
"ar_overdue": 0,
|
| 435 |
+
"ar_exposure": 0,
|
| 436 |
+
"top_category": "Styrofoam",
|
| 437 |
+
"top_category_pct": 0.47,
|
| 438 |
+
"sku_count": 93,
|
| 439 |
+
"top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
|
| 440 |
+
"days_to_pay": 34,
|
| 441 |
+
"_created": "2023-05-15 09:14:00",
|
| 442 |
+
"lat": 26.1224,
|
| 443 |
+
"lon": -80.1373,
|
| 444 |
+
"odoo_status": "Active",
|
| 445 |
+
"dba": "",
|
| 446 |
+
"ar_outstanding": 0
|
| 447 |
+
},
|
| 448 |
+
{
|
| 449 |
+
"pid": 106,
|
| 450 |
+
"customer": "Dogwood & Fern Co.",
|
| 451 |
+
"status": "Dormant",
|
| 452 |
+
"agent": "Devon Marsh",
|
| 453 |
+
"city": "Asheville",
|
| 454 |
+
"state": "North Carolina (US)",
|
| 455 |
+
"last_order": "2026-02-11",
|
| 456 |
+
"est_missed": 52400,
|
| 457 |
+
"notes": "No spring order this year.",
|
| 458 |
+
"country": "United States",
|
| 459 |
+
"zip": "90210",
|
| 460 |
+
"payment_terms": "60 Days",
|
| 461 |
+
"pricelist": "Royal 1 (USD)",
|
| 462 |
+
"tags": "Royal, Key account",
|
| 463 |
+
"customer_since": "2024-06-15",
|
| 464 |
+
"salesperson": "Naomi",
|
| 465 |
+
"ar_open": 1240.5,
|
| 466 |
+
"ar_overdue": 0,
|
| 467 |
+
"ar_exposure": 1740.5,
|
| 468 |
+
"top_category": "Ribbon",
|
| 469 |
+
"top_category_pct": 0.95,
|
| 470 |
+
"sku_count": 4,
|
| 471 |
+
"top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
|
| 472 |
+
"days_to_pay": null,
|
| 473 |
+
"_created": "2023-06-15 09:15:00",
|
| 474 |
+
"lat": 27.3364,
|
| 475 |
+
"lon": -82.5307,
|
| 476 |
+
"odoo_status": "Active",
|
| 477 |
+
"dba": "Fisch",
|
| 478 |
+
"ar_outstanding": 1240.5
|
| 479 |
+
},
|
| 480 |
+
{
|
| 481 |
+
"pid": 107,
|
| 482 |
+
"customer": "Verbena Market Florals",
|
| 483 |
+
"status": "Lost",
|
| 484 |
+
"agent": "Naomi Linnell Rivera",
|
| 485 |
+
"city": "Austin",
|
| 486 |
+
"state": "Texas (US)",
|
| 487 |
+
"last_order": "2025-11-04",
|
| 488 |
+
"est_missed": 78300,
|
| 489 |
+
"notes": "Went with a competitor on freight terms.",
|
| 490 |
+
"country": "United States",
|
| 491 |
+
"zip": "08701",
|
| 492 |
+
"payment_terms": "30 Days",
|
| 493 |
+
"pricelist": "Fisch 1 (USD)",
|
| 494 |
+
"tags": "Fisch",
|
| 495 |
+
"customer_since": "2025-07-16",
|
| 496 |
+
"salesperson": "Karen",
|
| 497 |
+
"ar_open": 0,
|
| 498 |
+
"ar_overdue": 5120.25,
|
| 499 |
+
"ar_exposure": 5120.25,
|
| 500 |
+
"top_category": "Foams & Finishes",
|
| 501 |
+
"top_category_pct": 0.31,
|
| 502 |
+
"sku_count": 27,
|
| 503 |
+
"top_sku": "SATIN RIBBON 2IN",
|
| 504 |
+
"days_to_pay": 61,
|
| 505 |
+
"_created": "2023-07-15 09:16:00",
|
| 506 |
+
"lat": null,
|
| 507 |
+
"lon": null,
|
| 508 |
+
"odoo_status": "Active",
|
| 509 |
+
"dba": "Both",
|
| 510 |
+
"ar_outstanding": 5120.25
|
| 511 |
+
},
|
| 512 |
+
{
|
| 513 |
+
"pid": 108,
|
| 514 |
+
"customer": "Larkspur Lane Supply",
|
| 515 |
+
"status": "New",
|
| 516 |
+
"agent": "Carla Jimenez",
|
| 517 |
+
"city": "Denver",
|
| 518 |
+
"state": "Colorado (US)",
|
| 519 |
+
"last_order": "2026-07-14",
|
| 520 |
+
"est_missed": 0,
|
| 521 |
+
"notes": "First order in April.",
|
| 522 |
+
"country": "United States",
|
| 523 |
+
"zip": "60614",
|
| 524 |
+
"payment_terms": "Immediate Payment",
|
| 525 |
+
"pricelist": "Royal 1 (USD)",
|
| 526 |
+
"tags": "(none)",
|
| 527 |
+
"customer_since": "2026-08-17",
|
| 528 |
+
"salesperson": "(none)",
|
| 529 |
+
"ar_open": 8300,
|
| 530 |
+
"ar_overdue": 940,
|
| 531 |
+
"ar_exposure": 11440,
|
| 532 |
+
"top_category": "All",
|
| 533 |
+
"top_category_pct": 1.0,
|
| 534 |
+
"sku_count": 1,
|
| 535 |
+
"top_sku": "(none)",
|
| 536 |
+
"days_to_pay": 12,
|
| 537 |
+
"_created": "2023-08-15 09:17:00",
|
| 538 |
+
"lat": 33.749,
|
| 539 |
+
"lon": -84.388,
|
| 540 |
+
"odoo_status": "Active",
|
| 541 |
+
"dba": "Royal",
|
| 542 |
+
"ar_outstanding": 9240
|
| 543 |
+
}
|
| 544 |
+
]
|
| 545 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/alerts/alertsModel.ts
CHANGED
|
@@ -1,418 +1,365 @@
|
|
| 1 |
-
// ---------------------------------------------------------------------------
|
| 2 |
-
// alerts/alertsModel.ts — WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
|
| 3 |
-
// half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
|
| 4 |
-
//
|
| 5 |
-
// An alert says "tell me when a record ENTERS this view". The client half is
|
| 6 |
-
// small, and every part of it fails silently when it is wrong:
|
| 7 |
-
//
|
| 8 |
-
// · an unread count taken from `items.length` rather than from the server's
|
| 9 |
-
// own `unread` disagrees with the badge the moment a page is capped or a
|
| 10 |
-
// read lands in another tab — and a badge that says 3 when the list shows 9
|
| 11 |
-
// teaches the reader to ignore the badge;
|
| 12 |
-
// · a notification whose `viewId` no longer resolves must open NOTHING rather
|
| 13 |
-
// than a wrong view — alerts outlive the views they were made from;
|
| 14 |
-
// · `topic` is the surface's scope key ("customer", "product", "ut_…"), and
|
| 15 |
-
// the ROUTE is a registry key ("customer_data") — mapping one to the other
|
| 16 |
-
// by guesswork sends every click to a page that does not exist.
|
| 17 |
-
// ---------------------------------------------------------------------------
|
| 18 |
-
|
| 19 |
-
/**
|
| 20 |
-
* ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — WHERE AN INBOX ITEM OPENS.
|
| 21 |
-
*
|
| 22 |
-
* `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
|
| 23 |
-
* in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
|
| 24 |
-
* run log, or the VIEW ID to select on a database. One optional key, two destinations.
|
| 25 |
-
*
|
| 26 |
-
* ⛔ EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
|
| 27 |
-
* than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
|
| 28 |
-
* a module" into "the client silently drops the row". An unknown module is refused by the
|
| 29 |
-
* FRAME's dispatcher, out loud, which is a different thing from never arriving.
|
| 30 |
-
*/
|
| 31 |
-
export interface NotificationTarget {
|
| 32 |
-
module: string;
|
| 33 |
-
id: string;
|
| 34 |
-
tab?: string;
|
| 35 |
-
}
|
| 36 |
-
|
| 37 |
-
/** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
|
| 38 |
-
* Inbox's branch and the gate's fixtures cannot disagree about the word. */
|
| 39 |
-
export const NOTIF_KIND_ALERT = "alert";
|
| 40 |
-
export const NOTIF_KIND_AUTOMATION = "automation";
|
| 41 |
-
export const NOTIF_KIND_SHARE = "share";
|
| 42 |
-
|
| 43 |
-
/** C3's `target.module` vocabulary, and the automation sub-selection. */
|
| 44 |
-
export const TARGET_MODULE_DATABASE = "database";
|
| 45 |
-
export const TARGET_MODULE_AUTOMATION = "automation";
|
| 46 |
-
export const TARGET_TAB_RUNS = "runs";
|
| 47 |
-
|
| 48 |
-
/** One notification, as `GET /api/v1/notifications` sends it. */
|
| 49 |
-
export interface Notification {
|
| 50 |
-
id: string;
|
| 51 |
-
alertId: string;
|
| 52 |
-
viewId: string;
|
| 53 |
-
topic: string;
|
| 54 |
-
rowId: string;
|
| 55 |
-
/** What entered — the record's own label. */
|
| 56 |
-
label: string;
|
| 57 |
-
/** What the alert is called, so a row reads without opening anything. */
|
| 58 |
-
alertLabel: string;
|
| 59 |
-
/** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
|
| 60 |
-
* would re-introduce the browser-clock drift the offset exists to remove. */
|
| 61 |
-
at: string;
|
| 62 |
-
read: boolean;
|
| 63 |
-
/**
|
| 64 |
-
* WAVE 23 (contract C6) — WHAT KIND of notification this is.
|
| 65 |
-
*
|
| 66 |
-
* Absent (and every notification written before this wave) means the original one: a record
|
| 67 |
-
* ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
|
| 68 |
-
* arrived at a review stage and routes by `autoId` instead — a different destination reached
|
| 69 |
-
* from the same list.
|
| 70 |
-
*
|
| 71 |
-
* ⛔ A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
|
| 72 |
-
* is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
|
| 73 |
-
* silently drops the row". Unknown kinds fall through to the view route, which is exactly what
|
| 74 |
-
* they did before this field existed.
|
| 75 |
-
*/
|
| 76 |
-
kind?: string;
|
| 77 |
-
/** `automation_review` only: the automation whose review stage a card reached. Absent on
|
| 78 |
-
* every other kind — and an `automation_review` row that arrives WITHOUT one opens nothing
|
| 79 |
-
* rather than guessing, the same posture `viewId` gets. */
|
| 80 |
-
autoId?: string;
|
| 81 |
-
/** Advisory. A surface that does not scroll to a stage simply selects the automation. */
|
| 82 |
-
stageId?: string;
|
| 83 |
-
/** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
|
| 84 |
-
* never one per record — so this is the number the row's own text is built from. */
|
| 85 |
-
count?: number;
|
| 86 |
-
/**
|
| 87 |
-
* ⭐ WAVE 32 · C3 — THE HEADER LINE, so the Inbox can be laid out like mail.
|
| 88 |
-
*
|
| 89 |
-
* `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
|
| 90 |
-
* that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
|
| 91 |
-
* They were one field, which is why the pane could only ever render a sentence with no sender.
|
| 92 |
-
* Absent on a server that predates this wave — {@link subjectOf} falls back rather than
|
| 93 |
-
* rendering a blank header.
|
| 94 |
-
*/
|
| 95 |
-
subject?: string;
|
| 96 |
-
/**
|
| 97 |
-
*
|
| 98 |
-
*
|
| 99 |
-
*
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
*
|
| 148 |
-
*
|
| 149 |
-
*
|
| 150 |
-
*
|
| 151 |
-
*
|
| 152 |
-
*
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
*
|
| 169 |
-
*
|
| 170 |
-
*
|
| 171 |
-
*
|
| 172 |
-
*
|
| 173 |
-
*/
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
*
|
| 190 |
-
*
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
return
|
| 199 |
-
}
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
*
|
| 206 |
-
*
|
| 207 |
-
*
|
| 208 |
-
*
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
*
|
| 303 |
-
*
|
| 304 |
-
*
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
*
|
| 324 |
-
*
|
| 325 |
-
*
|
| 326 |
-
*
|
| 327 |
-
*
|
| 328 |
-
*
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
const
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
*/
|
| 367 |
-
export function badgeText(unread: number): string {
|
| 368 |
-
const n = Math.max(0, Math.floor(unread));
|
| 369 |
-
if (n <= 0) return "";
|
| 370 |
-
return n > 99 ? "99+" : String(n);
|
| 371 |
-
}
|
| 372 |
-
|
| 373 |
-
/**
|
| 374 |
-
* Apply a read/unread change LOCALLY, mirroring what the server just stored, and
|
| 375 |
-
* return the new inbox with the count corrected.
|
| 376 |
-
*
|
| 377 |
-
* `ids === null` is "all of them" (the API's own convention for mark-all). The
|
| 378 |
-
* count is derived from the ITEMS here — deliberately, and it is the one place
|
| 379 |
-
* that is right to do so: the server's answer is in flight, and the alternative
|
| 380 |
-
* is a badge that keeps its old number until the refetch lands.
|
| 381 |
-
*/
|
| 382 |
-
export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
|
| 383 |
-
const wanted = ids === null ? null : new Set(ids);
|
| 384 |
-
const items = inbox.items.map((n) =>
|
| 385 |
-
wanted === null || wanted.has(n.id) ? { ...n, read } : n
|
| 386 |
-
);
|
| 387 |
-
const seenUnread = items.filter((n) => !n.read).length;
|
| 388 |
-
// A page can hold fewer notifications than the account has, so a partial read
|
| 389 |
-
// must SUBTRACT from the server's count rather than replace it with this
|
| 390 |
-
// page's tally — except when marking everything, where zero is the answer.
|
| 391 |
-
if (wanted === null) return { unread: read ? 0 : items.length, items };
|
| 392 |
-
const changed = inbox.items.filter(
|
| 393 |
-
(n) => wanted.has(n.id) && n.read !== read
|
| 394 |
-
).length;
|
| 395 |
-
const delta = read ? -changed : changed;
|
| 396 |
-
return { unread: Math.max(seenUnread, inbox.unread + delta), items };
|
| 397 |
-
}
|
| 398 |
-
|
| 399 |
-
/**
|
| 400 |
-
* The shell↔rail channel for "make an alert out of this view" — the view rail
|
| 401 |
-
* raises it, the frame (which knows the current route, and therefore the topic)
|
| 402 |
-
* answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
|
| 403 |
-
* import the shell, and it does not know its own scope key.
|
| 404 |
-
*/
|
| 405 |
-
export const ALERT_CREATE_EVENT = "aios:alert-create";
|
| 406 |
-
|
| 407 |
-
export interface AlertCreateRequest {
|
| 408 |
-
viewId: string;
|
| 409 |
-
label: string;
|
| 410 |
-
}
|
| 411 |
-
|
| 412 |
-
export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
|
| 413 |
-
if (!detail || typeof detail !== "object") return null;
|
| 414 |
-
const d = detail as Record<string, unknown>;
|
| 415 |
-
const viewId = str(d.viewId).trim();
|
| 416 |
-
if (!viewId) return null;
|
| 417 |
-
return { viewId, label: str(d.label).trim() || viewId };
|
| 418 |
-
}
|
|
|
|
| 1 |
+
// ---------------------------------------------------------------------------
|
| 2 |
+
// alerts/alertsModel.ts — WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
|
| 3 |
+
// half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
|
| 4 |
+
//
|
| 5 |
+
// An alert says "tell me when a record ENTERS this view". The client half is
|
| 6 |
+
// small, and every part of it fails silently when it is wrong:
|
| 7 |
+
//
|
| 8 |
+
// · an unread count taken from `items.length` rather than from the server's
|
| 9 |
+
// own `unread` disagrees with the badge the moment a page is capped or a
|
| 10 |
+
// read lands in another tab — and a badge that says 3 when the list shows 9
|
| 11 |
+
// teaches the reader to ignore the badge;
|
| 12 |
+
// · a notification whose `viewId` no longer resolves must open NOTHING rather
|
| 13 |
+
// than a wrong view — alerts outlive the views they were made from;
|
| 14 |
+
// · `topic` is the surface's scope key ("customer", "product", "ut_…"), and
|
| 15 |
+
// the ROUTE is a registry key ("customer_data") — mapping one to the other
|
| 16 |
+
// by guesswork sends every click to a page that does not exist.
|
| 17 |
+
// ---------------------------------------------------------------------------
|
| 18 |
+
|
| 19 |
+
/**
|
| 20 |
+
* ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — WHERE AN INBOX ITEM OPENS.
|
| 21 |
+
*
|
| 22 |
+
* `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
|
| 23 |
+
* in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
|
| 24 |
+
* run log, or the VIEW ID to select on a database. One optional key, two destinations.
|
| 25 |
+
*
|
| 26 |
+
* ⛔ EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
|
| 27 |
+
* than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
|
| 28 |
+
* a module" into "the client silently drops the row". An unknown module is refused by the
|
| 29 |
+
* FRAME's dispatcher, out loud, which is a different thing from never arriving.
|
| 30 |
+
*/
|
| 31 |
+
export interface NotificationTarget {
|
| 32 |
+
module: string;
|
| 33 |
+
id: string;
|
| 34 |
+
tab?: string;
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
/** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
|
| 38 |
+
* Inbox's branch and the gate's fixtures cannot disagree about the word. */
|
| 39 |
+
export const NOTIF_KIND_ALERT = "alert";
|
| 40 |
+
export const NOTIF_KIND_AUTOMATION = "automation";
|
| 41 |
+
export const NOTIF_KIND_SHARE = "share";
|
| 42 |
+
|
| 43 |
+
/** C3's `target.module` vocabulary, and the automation sub-selection. */
|
| 44 |
+
export const TARGET_MODULE_DATABASE = "database";
|
| 45 |
+
export const TARGET_MODULE_AUTOMATION = "automation";
|
| 46 |
+
export const TARGET_TAB_RUNS = "runs";
|
| 47 |
+
|
| 48 |
+
/** One notification, as `GET /api/v1/notifications` sends it. */
|
| 49 |
+
export interface Notification {
|
| 50 |
+
id: string;
|
| 51 |
+
alertId: string;
|
| 52 |
+
viewId: string;
|
| 53 |
+
topic: string;
|
| 54 |
+
rowId: string;
|
| 55 |
+
/** What entered — the record's own label. */
|
| 56 |
+
label: string;
|
| 57 |
+
/** What the alert is called, so a row reads without opening anything. */
|
| 58 |
+
alertLabel: string;
|
| 59 |
+
/** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
|
| 60 |
+
* would re-introduce the browser-clock drift the offset exists to remove. */
|
| 61 |
+
at: string;
|
| 62 |
+
read: boolean;
|
| 63 |
+
/**
|
| 64 |
+
* WAVE 23 (contract C6) — WHAT KIND of notification this is.
|
| 65 |
+
*
|
| 66 |
+
* Absent (and every notification written before this wave) means the original one: a record
|
| 67 |
+
* ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
|
| 68 |
+
* arrived at a review stage and routes by `autoId` instead — a different destination reached
|
| 69 |
+
* from the same list.
|
| 70 |
+
*
|
| 71 |
+
* ⛔ A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
|
| 72 |
+
* is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
|
| 73 |
+
* silently drops the row". Unknown kinds fall through to the view route, which is exactly what
|
| 74 |
+
* they did before this field existed.
|
| 75 |
+
*/
|
| 76 |
+
kind?: string;
|
| 77 |
+
/** `automation_review` only: the automation whose review stage a card reached. Absent on
|
| 78 |
+
* every other kind — and an `automation_review` row that arrives WITHOUT one opens nothing
|
| 79 |
+
* rather than guessing, the same posture `viewId` gets. */
|
| 80 |
+
autoId?: string;
|
| 81 |
+
/** Advisory. A surface that does not scroll to a stage simply selects the automation. */
|
| 82 |
+
stageId?: string;
|
| 83 |
+
/** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
|
| 84 |
+
* never one per record — so this is the number the row's own text is built from. */
|
| 85 |
+
count?: number;
|
| 86 |
+
/**
|
| 87 |
+
* ⭐ WAVE 32 · C3 — THE HEADER LINE, so the Inbox can be laid out like mail.
|
| 88 |
+
*
|
| 89 |
+
* `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
|
| 90 |
+
* that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
|
| 91 |
+
* They were one field, which is why the pane could only ever render a sentence with no sender.
|
| 92 |
+
* Absent on a server that predates this wave — {@link subjectOf} falls back rather than
|
| 93 |
+
* rendering a blank header.
|
| 94 |
+
*/
|
| 95 |
+
subject?: string;
|
| 96 |
+
/** ⭐ WAVE 32 · C3 — where clicking it goes. ABSENT when this product cannot resolve a
|
| 97 |
+
* destination (an alert on a table this account can no longer route to), and that absence is
|
| 98 |
+
* load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click
|
| 99 |
+
* that silently does nothing. */
|
| 100 |
+
target?: NotificationTarget;
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
/** One alert, as `GET /api/v1/alerts` sends it. */
|
| 104 |
+
export interface Alert {
|
| 105 |
+
id: string;
|
| 106 |
+
viewId: string;
|
| 107 |
+
topic: string;
|
| 108 |
+
owner: string;
|
| 109 |
+
label: string;
|
| 110 |
+
createdAt: string;
|
| 111 |
+
/** How many records are in its remembered set right now. */
|
| 112 |
+
matched: number;
|
| 113 |
+
seeded: boolean;
|
| 114 |
+
lastRunAt: string;
|
| 115 |
+
lastError: string;
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
export interface Inbox {
|
| 119 |
+
unread: number;
|
| 120 |
+
items: Notification[];
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
|
| 124 |
+
|
| 125 |
+
/**
|
| 126 |
+
* ⭐⭐ WAVE 31 · T23 (owner item 1) — WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
|
| 127 |
+
*
|
| 128 |
+
* Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
|
| 129 |
+
* remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
|
| 130 |
+
* `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
|
| 131 |
+
* and `GET /notifications` answering — measured at **3,280 ms live** — and for ever after a
|
| 132 |
+
* failed fetch:
|
| 133 |
+
* · `items` was `[]`, so the pane printed **"Nothing new."** — a claim, not a wait;
|
| 134 |
+
* · `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
|
| 135 |
+
* already loaded could never be cleared.
|
| 136 |
+
* A confident sentence about somebody's inbox, and the one control that would fix it, both
|
| 137 |
+
* switched off by the same uninitialised state.
|
| 138 |
+
*
|
| 139 |
+
* ⛔ IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
|
| 140 |
+
* compiles and RUNS this module under node; markup it cannot reach is markup no control can
|
| 141 |
+
* mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
|
| 142 |
+
*/
|
| 143 |
+
export type PaneView = "pending" | "rows" | "empty" | "error";
|
| 144 |
+
|
| 145 |
+
/**
|
| 146 |
+
* `pending` while the first read is in flight · `error` when it failed and we have nothing to
|
| 147 |
+
* show · `rows` when there is something · `empty` ONLY when a successful read returned nothing.
|
| 148 |
+
*
|
| 149 |
+
* ⚠ ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
|
| 150 |
+
* the pane already holds notifications should not blank them — the reader loses real information
|
| 151 |
+
* to a transient. The error still reaches them as the pane's message line.
|
| 152 |
+
*/
|
| 153 |
+
export function paneView(
|
| 154 |
+
phase: "pending" | "ready" | "error",
|
| 155 |
+
rowCount: number,
|
| 156 |
+
seededUnread = 0
|
| 157 |
+
): PaneView {
|
| 158 |
+
if (rowCount > 0) return "rows";
|
| 159 |
+
if (phase === "pending") return "pending";
|
| 160 |
+
if (phase === "error") return "error";
|
| 161 |
+
// ⚠ A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
|
| 162 |
+
// there is something; the page we were given does not contain it. Saying "Nothing new" there
|
| 163 |
+
// is the same false confidence in a different costume.
|
| 164 |
+
return seededUnread > 0 ? "error" : "empty";
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
/**
|
| 168 |
+
* May "Mark all read" be pressed?
|
| 169 |
+
*
|
| 170 |
+
* ⛔ NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state — zero
|
| 171 |
+
* until its fetch lands, zero for ever if the fetch fails — so the control was dead in exactly
|
| 172 |
+
* the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
|
| 173 |
+
* the frame already holds, and a failed read does not take the verb away: `POST /notifications/
|
| 174 |
+
* read` with `ids: null` clears the account's inbox whether or not we managed to list it.
|
| 175 |
+
*/
|
| 176 |
+
export function canMarkAll(unread: number, busy = false): boolean {
|
| 177 |
+
return !busy && Math.max(0, Math.floor(unread)) > 0;
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
const str = (v: unknown): string => (typeof v === "string" ? v : "");
|
| 181 |
+
const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
|
| 182 |
+
|
| 183 |
+
/**
|
| 184 |
+
* ⭐ WAVE 32 · C3 — one wire `target` → a {@link NotificationTarget}, or `null`.
|
| 185 |
+
*
|
| 186 |
+
* ⛔ BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is
|
| 187 |
+
* for: a target with a module and no id dispatches an open request naming NOTHING — a click that
|
| 188 |
+
* appears to work and silently does not, which is this repo's most-repeated shape. `null` is
|
| 189 |
+
* rendered as an unclickable row, which a reader can SEE.
|
| 190 |
+
*/
|
| 191 |
+
export function parseTarget(raw: unknown): NotificationTarget | null {
|
| 192 |
+
if (!raw || typeof raw !== "object") return null;
|
| 193 |
+
const t = raw as Record<string, unknown>;
|
| 194 |
+
const module = str(t.module).trim();
|
| 195 |
+
const id = str(t.id).trim();
|
| 196 |
+
if (!module || !id) return null;
|
| 197 |
+
const tab = str(t.tab).trim();
|
| 198 |
+
return { module, id, ...(tab ? { tab } : {}) };
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
/**
|
| 202 |
+
* `GET /notifications` → the inbox, fail-closed.
|
| 203 |
+
*
|
| 204 |
+
* ⚠ `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two
|
| 205 |
+
* can legitimately differ — the list is what this page holds, the count is what
|
| 206 |
+
* the account has — and recomputing it here would make the badge a function of
|
| 207 |
+
* whatever the last fetch happened to include.
|
| 208 |
+
*/
|
| 209 |
+
export function parseInbox(body: unknown): Inbox {
|
| 210 |
+
const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
|
| 211 |
+
const raw = Array.isArray(b.items) ? b.items : [];
|
| 212 |
+
const items: Notification[] = [];
|
| 213 |
+
for (const item of raw) {
|
| 214 |
+
if (!item || typeof item !== "object") continue;
|
| 215 |
+
const n = item as Record<string, unknown>;
|
| 216 |
+
const id = str(n.id);
|
| 217 |
+
// An id-less notification cannot be marked read, so it would sit unread for
|
| 218 |
+
// ever and hold the badge up. Dropped, not rendered.
|
| 219 |
+
if (!id) continue;
|
| 220 |
+
items.push({
|
| 221 |
+
id,
|
| 222 |
+
alertId: str(n.alertId),
|
| 223 |
+
viewId: str(n.viewId),
|
| 224 |
+
topic: str(n.topic),
|
| 225 |
+
rowId: String(n.rowId ?? ""),
|
| 226 |
+
label: str(n.label) || String(n.rowId ?? ""),
|
| 227 |
+
alertLabel: str(n.alertLabel),
|
| 228 |
+
at: str(n.at),
|
| 229 |
+
read: n.read === true,
|
| 230 |
+
// WAVE 23 C6 — ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload
|
| 231 |
+
// that predates this wave keeps today's shape rather than gaining four `undefined` keys,
|
| 232 |
+
// and an `automation_review` row missing its `autoId` is left WITHOUT one rather than
|
| 233 |
+
// with an empty string that would render as a real destination.
|
| 234 |
+
...(str(n.kind) ? { kind: str(n.kind) } : {}),
|
| 235 |
+
...(str(n.autoId) ? { autoId: str(n.autoId) } : {}),
|
| 236 |
+
...(str(n.stageId) ? { stageId: str(n.stageId) } : {}),
|
| 237 |
+
...(num(n.count) > 0 ? { count: num(n.count) } : {}),
|
| 238 |
+
// ⭐ WAVE 32 · C3 — additive and spread-conditional, exactly like the four above.
|
| 239 |
+
...(str(n.subject) ? { subject: str(n.subject) } : {}),
|
| 240 |
+
...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}),
|
| 241 |
+
});
|
| 242 |
+
}
|
| 243 |
+
return { unread: Math.max(0, num(b.unread)), items };
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
/** `GET /alerts` → the alert list, fail-closed. */
|
| 247 |
+
export function parseAlerts(body: unknown): Alert[] {
|
| 248 |
+
const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
|
| 249 |
+
const raw = Array.isArray(b.alerts) ? b.alerts : [];
|
| 250 |
+
const out: Alert[] = [];
|
| 251 |
+
for (const item of raw) {
|
| 252 |
+
if (!item || typeof item !== "object") continue;
|
| 253 |
+
const a = item as Record<string, unknown>;
|
| 254 |
+
const id = str(a.id);
|
| 255 |
+
if (!id) continue;
|
| 256 |
+
out.push({
|
| 257 |
+
id,
|
| 258 |
+
viewId: str(a.viewId),
|
| 259 |
+
topic: str(a.topic),
|
| 260 |
+
owner: str(a.owner),
|
| 261 |
+
label: str(a.label) || "Untitled alert",
|
| 262 |
+
createdAt: str(a.createdAt),
|
| 263 |
+
matched: num(a.matched),
|
| 264 |
+
seeded: a.seeded === true,
|
| 265 |
+
lastRunAt: str(a.lastRunAt),
|
| 266 |
+
lastError: str(a.lastError),
|
| 267 |
+
});
|
| 268 |
+
}
|
| 269 |
+
return out;
|
| 270 |
+
}
|
| 271 |
+
|
| 272 |
+
/**
|
| 273 |
+
* A topic (the grid's scope key) → the hash route that renders it.
|
| 274 |
+
*
|
| 275 |
+
* The two built-ins are the only pair that differ, and they differ because the
|
| 276 |
+
* REGISTRY names the surface while the GRID names the scope; a user table is its
|
| 277 |
+
* own key in both. `null` for anything else: a notification for a topic this
|
| 278 |
+
* client cannot route to must do nothing, not navigate somewhere plausible.
|
| 279 |
+
*/
|
| 280 |
+
export function routeForTopic(topic: string): string | null {
|
| 281 |
+
const t = str(topic).trim();
|
| 282 |
+
if (t === "customer") return "customer_data";
|
| 283 |
+
if (t === "product") return "product_data";
|
| 284 |
+
if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t;
|
| 285 |
+
return null;
|
| 286 |
+
}
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
/**
|
| 294 |
+
* The stamp, made readable WITHOUT touching a clock.
|
| 295 |
+
*
|
| 296 |
+
* ⛔ NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The
|
| 297 |
+
* server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the
|
| 298 |
+
* same instant; parsing it into a browser Date and formatting it back would
|
| 299 |
+
* re-introduce the drift the offset exists to remove — a tenant a day ahead
|
| 300 |
+
* being told an event happened tomorrow ([[date-window-vocabulary]]). This is
|
| 301 |
+
* STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`.
|
| 302 |
+
* Anything that does not look like an ISO stamp passes through untouched, so a
|
| 303 |
+
* format this function has never seen is shown as sent rather than mangled.
|
| 304 |
+
*/
|
| 305 |
+
export function stampText(at: string): string {
|
| 306 |
+
const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at));
|
| 307 |
+
return m ? `${m[1]} ${m[2]}` : str(at);
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
/**
|
| 311 |
+
* The badge's text. Never the raw number past 99: a nav row is 236px wide and a
|
| 312 |
+
* four-digit badge pushes the label out of it.
|
| 313 |
+
*/
|
| 314 |
+
export function badgeText(unread: number): string {
|
| 315 |
+
const n = Math.max(0, Math.floor(unread));
|
| 316 |
+
if (n <= 0) return "";
|
| 317 |
+
return n > 99 ? "99+" : String(n);
|
| 318 |
+
}
|
| 319 |
+
|
| 320 |
+
/**
|
| 321 |
+
* Apply a read/unread change LOCALLY, mirroring what the server just stored, and
|
| 322 |
+
* return the new inbox with the count corrected.
|
| 323 |
+
*
|
| 324 |
+
* `ids === null` is "all of them" (the API's own convention for mark-all). The
|
| 325 |
+
* count is derived from the ITEMS here — deliberately, and it is the one place
|
| 326 |
+
* that is right to do so: the server's answer is in flight, and the alternative
|
| 327 |
+
* is a badge that keeps its old number until the refetch lands.
|
| 328 |
+
*/
|
| 329 |
+
export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
|
| 330 |
+
const wanted = ids === null ? null : new Set(ids);
|
| 331 |
+
const items = inbox.items.map((n) =>
|
| 332 |
+
wanted === null || wanted.has(n.id) ? { ...n, read } : n
|
| 333 |
+
);
|
| 334 |
+
const seenUnread = items.filter((n) => !n.read).length;
|
| 335 |
+
// A page can hold fewer notifications than the account has, so a partial read
|
| 336 |
+
// must SUBTRACT from the server's count rather than replace it with this
|
| 337 |
+
// page's tally — except when marking everything, where zero is the answer.
|
| 338 |
+
if (wanted === null) return { unread: read ? 0 : items.length, items };
|
| 339 |
+
const changed = inbox.items.filter(
|
| 340 |
+
(n) => wanted.has(n.id) && n.read !== read
|
| 341 |
+
).length;
|
| 342 |
+
const delta = read ? -changed : changed;
|
| 343 |
+
return { unread: Math.max(seenUnread, inbox.unread + delta), items };
|
| 344 |
+
}
|
| 345 |
+
|
| 346 |
+
/**
|
| 347 |
+
* The shell↔rail channel for "make an alert out of this view" — the view rail
|
| 348 |
+
* raises it, the frame (which knows the current route, and therefore the topic)
|
| 349 |
+
* answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
|
| 350 |
+
* import the shell, and it does not know its own scope key.
|
| 351 |
+
*/
|
| 352 |
+
export const ALERT_CREATE_EVENT = "aios:alert-create";
|
| 353 |
+
|
| 354 |
+
export interface AlertCreateRequest {
|
| 355 |
+
viewId: string;
|
| 356 |
+
label: string;
|
| 357 |
+
}
|
| 358 |
+
|
| 359 |
+
export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
|
| 360 |
+
if (!detail || typeof detail !== "object") return null;
|
| 361 |
+
const d = detail as Record<string, unknown>;
|
| 362 |
+
const viewId = str(d.viewId).trim();
|
| 363 |
+
if (!viewId) return null;
|
| 364 |
+
return { viewId, label: str(d.label).trim() || viewId };
|
| 365 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/automation/AutomationBuilder.tsx
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
web/src/automation/AutomationDetail.tsx
CHANGED
|
@@ -112,17 +112,6 @@ interface Props {
|
|
| 112 |
* forever.
|
| 113 |
*/
|
| 114 |
liveStep: string;
|
| 115 |
-
/**
|
| 116 |
-
* ⭐ W33 (C-18) — WHICH PANE THE INBOX ASKED FOR: `"runs"`, `"properties"`, or `""`.
|
| 117 |
-
*
|
| 118 |
-
* ⛔ REQUIRED, not optional, and that is deliberate. An optional pane hint defaulting to `""`
|
| 119 |
-
* reads exactly like "no hint was sent" — which is the state the field was already in for the
|
| 120 |
-
* whole of its life, declared on the wire and consumed by nobody. Making it required means the
|
| 121 |
-
* one caller has to decide, and a second caller cannot forget.
|
| 122 |
-
* ⚠ Consumed ONCE, as `aside`'s initial value; see its note. A notification that kept forcing
|
| 123 |
-
* the pane would take the switch away from the person using it.
|
| 124 |
-
*/
|
| 125 |
-
openTab: "properties" | "runs" | "";
|
| 126 |
/**
|
| 127 |
* ⭐ W31-T31 — `fresh` is the definition the WRITE JUST RETURNED, when the caller has one.
|
| 128 |
* Given it, the surface corrects that one row; without it, it falls back to a full reload. Every
|
|
@@ -183,7 +172,6 @@ export default function AutomationDetail({
|
|
| 183 |
vocab,
|
| 184 |
tick,
|
| 185 |
liveStep,
|
| 186 |
-
openTab,
|
| 187 |
onSaved,
|
| 188 |
onDeleted,
|
| 189 |
}: Props) {
|
|
@@ -317,20 +305,7 @@ export default function AutomationDetail({
|
|
| 317 |
* the rarer errand; opening it to change something is the common one, and the panel that
|
| 318 |
* appears should be the one a click on a step is about to fill.
|
| 319 |
*/
|
| 320 |
-
|
| 321 |
-
* ⭐ W33 (C-18) — AN INBOX ITEM MAY ASK FOR THE RUNS PANE, and until now it asked into the void.
|
| 322 |
-
*
|
| 323 |
-
* ⛔ AN INITIAL VALUE, NOT A CONTROLLED ONE, and the difference is the whole of why this is safe.
|
| 324 |
-
* `openTab` is consumed exactly once, when this component mounts for an automation (the parent
|
| 325 |
-
* passes `key={active.id}`, so opening a different automation is a fresh mount and a fresh
|
| 326 |
-
* read). Making `aside` follow the prop on every render would take the switch away from the
|
| 327 |
-
* person the instant after they used it — the notification would keep dragging them back to a
|
| 328 |
-
* pane they had just left.
|
| 329 |
-
* ⚠ AND THE DEFAULT IS UNCHANGED. With no `openTab` this is `"properties"`, for the reason the
|
| 330 |
-
* note above gives: opening an automation to change something is the common errand.
|
| 331 |
-
*/
|
| 332 |
-
const [aside, setAside] = useState<"properties" | "runs">(
|
| 333 |
-
openTab === "runs" ? "runs" : "properties");
|
| 334 |
/**
|
| 335 |
* The switch, drawn in BOTH panel heads so it does not move when the panel does.
|
| 336 |
*
|
|
|
|
| 112 |
* forever.
|
| 113 |
*/
|
| 114 |
liveStep: string;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
/**
|
| 116 |
* ⭐ W31-T31 — `fresh` is the definition the WRITE JUST RETURNED, when the caller has one.
|
| 117 |
* Given it, the surface corrects that one row; without it, it falls back to a full reload. Every
|
|
|
|
| 172 |
vocab,
|
| 173 |
tick,
|
| 174 |
liveStep,
|
|
|
|
| 175 |
onSaved,
|
| 176 |
onDeleted,
|
| 177 |
}: Props) {
|
|
|
|
| 305 |
* the rarer errand; opening it to change something is the common one, and the panel that
|
| 306 |
* appears should be the one a click on a step is about to fill.
|
| 307 |
*/
|
| 308 |
+
const [aside, setAside] = useState<"properties" | "runs">("properties");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
/**
|
| 310 |
* The switch, drawn in BOTH panel heads so it does not move when the panel does.
|
| 311 |
*
|
web/src/automation/AutomationSurface.tsx
CHANGED
|
@@ -1,806 +1,782 @@
|
|
| 1 |
-
// ---------------------------------------------------------------------------
|
| 2 |
-
// automation/AutomationSurface.tsx — the Automation surface (contract C-AUTONAV).
|
| 3 |
-
//
|
| 4 |
-
// ⛔ THE CONTRACT THIS FILE GUARANTEES, verbatim: this module path, a DEFAULT
|
| 5 |
-
// export, and NO PROPS. The shell mounts `<AutomationSurface />` and hands it
|
| 6 |
-
// nothing; it fetches its own data from `/api/v1/automations`. That is the whole
|
| 7 |
-
// interface, and it is stated here because the mount line lives in a file this
|
| 8 |
-
// session does not own — a named export would cost a cross-session round trip.
|
| 9 |
-
//
|
| 10 |
-
// "Automation replaces the Views rail" (owner) is implemented the way the wave
|
| 11 |
-
// scout established: the secondary rail is NOT a shell concept — each surface
|
| 12 |
-
// owns its own. So this renders a sibling of `.cg-views` built from the same
|
| 13 |
-
// `--lp-rail-w` tokens and the same fold behaviour, and the Views rail is
|
| 14 |
-
// untouched.
|
| 15 |
-
// ---------------------------------------------------------------------------
|
| 16 |
-
import type { KeyboardEvent as ReactKeyboardEvent } from "react";
|
| 17 |
-
import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
|
| 18 |
-
|
| 19 |
-
import type { AutomationOpenDetail } from "../apiContract";
|
| 20 |
-
import { AUTOMATION_OPEN_EVENT } from "../apiContract";
|
| 21 |
-
// ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
|
| 22 |
-
// The owner's reason is the mark's own geometry: *"these automations are basically loops."*
|
| 23 |
-
// C7 says consume the existing icon component and do not inline a new SVG path, and
|
| 24 |
-
// `shell/Brand.tsx`'s header says why in more detail than a rule could: the mark's one source of
|
| 25 |
-
// truth is generated, it has already been hand-redrawn once, and the copy silently painted LAST
|
| 26 |
-
// WAVE'S BRAND while a comment asserted parity. So this imports the component that POINTS at the
|
| 27 |
-
// generated artifact. [[loopable-nav-logo-toggle]] — one element paints the mark.
|
| 28 |
-
import { Mark } from "../shell/Brand";
|
| 29 |
-
import type { Automation, AutomationList } from "./automationApi";
|
| 30 |
-
import {
|
| 31 |
-
AutomationError,
|
| 32 |
-
cachedAutomations,
|
| 33 |
-
createAutomation,
|
| 34 |
-
listAutomations,
|
| 35 |
-
liveStepOf,
|
| 36 |
-
patchAutomation,
|
| 37 |
-
rememberAutomation,
|
| 38 |
-
} from "./automationApi";
|
| 39 |
-
|
| 40 |
-
/**
|
| 41 |
-
* ⭐⭐ WAVE 30 · T21 (owner items 4 + 5) — THE EDITOR STOPS SHIPPING INSIDE THE LIST.
|
| 42 |
-
*
|
| 43 |
-
* ⛔ THIS `lazy()` IS THE TICKET. `AutomationDetail` was a STATIC import, and it statically pulls
|
| 44 |
-
* `AutomationBuilder` (which in turn pulls `CondBuilder`, `PresetPlan` and `TriggerPicker`),
|
| 45 |
-
* `AutomationFind` and `AutomationTrigger` — so the whole editor was linked into the one chunk a
|
| 46 |
-
* person must download before the RAIL can paint a list of names. MEASURED at 88,771 B, against
|
| 47 |
-
* `HomePage` at 4,814 and `ConnectorsPage` at 3,551. That is the "Automation takes forever to
|
| 48 |
-
* load" complaint, and it had survived two waves because nothing asserted a number.
|
| 49 |
-
*
|
| 50 |
-
* ⛔ AND IT NEEDS ITS OWN `<Suspense>`, NOT `Shell.tsx`'s. `Lazily` already wraps this whole
|
| 51 |
-
* surface, but a Suspense boundary catches ANY suspending descendant — so without an inner
|
| 52 |
-
* boundary, opening an automation would suspend the boundary that owns the RAIL and blank the
|
| 53 |
-
* list you just clicked in. The inner one keeps the rail painted while the editor streams in.
|
| 54 |
-
* `fallback={null}` is deliberate and is `Lazily`'s own choice, not a shortcut: the work area is
|
| 55 |
-
* simply not there for the moment the chunk is in flight, which is what every other route in this
|
| 56 |
-
* product already does. ⚠ A skeleton here would be a NEW loading screen in the same wave that
|
| 57 |
-
* exists to remove one (T22).
|
| 58 |
-
* ⚠ Default export, checked: `Shell.tsx:53` records that `lazy()` over a NAMED export fails at
|
| 59 |
-
* runtime, on click, and only for that one surface — `AutomationDetail` exports default.
|
| 60 |
-
*/
|
| 61 |
-
const AutomationDetail = lazy(() => import("./AutomationDetail"));
|
| 62 |
-
|
| 63 |
-
/**
|
| 64 |
-
* The run poll's cadence (item 6). Named constants because they are a MEASURED trade-off, not a
|
| 65 |
-
* taste: `POLL_MAX_MS` is the longest a finished run can still look live, and it is the only
|
| 66 |
-
* cost the backoff has.
|
| 67 |
-
*/
|
| 68 |
-
const POLL_MIN_MS = 2500;
|
| 69 |
-
const POLL_MAX_MS = 8000;
|
| 70 |
-
|
| 71 |
-
/*
|
| 72 |
-
* ⭐ WAVE 26 · ITEM 18 / R14 — `stateTitle` STOOD HERE AND IS DELETED WITH THE DOT IT DESCRIBED.
|
| 73 |
-
*
|
| 74 |
-
* It composed the dot's tooltip ("Running now — <step>", "Last run failed — <summary>"), so it had
|
| 75 |
-
* exactly one reader and no reason to outlive it. Deleting the render and keeping the composer is
|
| 76 |
-
* how a file accumulates functions that look live and are not — and `noUnusedLocals` would have
|
| 77 |
-
* caught this one, which is not a reason to lean on it: the NEXT such function might still have a
|
| 78 |
-
* second caller and compile fine.
|
| 79 |
-
*
|
| 80 |
-
* ⛔ WHY THE TOOLTIP WAS NOT MOVED ONTO THE MARK INSTEAD. R14 is "same mark on every automation;
|
| 81 |
-
* no status colour". Hanging "Last run failed" off a mark that is deliberately status-free just
|
| 82 |
-
* moves the status channel into `title`/`aria-label`, where it is worse: invisible to the eye,
|
| 83 |
-
* announced to a screen reader, and contradicting the visual. The run state has a home one click
|
| 84 |
-
* away — the Builder's own "Last run succeeded / failed" lines, which R13 explicitly keeps.
|
| 85 |
-
*
|
| 86 |
-
* ⚠ `stateOf` ITSELF IS UNTOUCHED in `automationApi.ts`. `Shell.tsx:64/1095` imports it and
|
| 87 |
-
* `liveStepOf` calls it — deleting the export to tidy up this file would break a fence I do not own.
|
| 88 |
-
*/
|
| 89 |
-
|
| 90 |
-
/**
|
| 91 |
-
* The rail's second line — ONE fact, not two.
|
| 92 |
-
*
|
| 93 |
-
* ⚠ It carried "Next 2026-08-04 06:00 · last 2026-08-04 00:20", and the rail is `--lp-rail-w`
|
| 94 |
-
* wide, so it rendered as "Next 2026-08-04 06:00 · l…" — a second line whose only complete word
|
| 95 |
-
* was "Next". Two facts that both truncate are worth less than one that fits. The forward-looking
|
| 96 |
-
* one wins when a schedule exists (it is the question the rail answers: does this run itself?),
|
| 97 |
-
* the last run otherwise. The full picture is one click away in the editor's run history, and the
|
| 98 |
-
* row's `title` already says both.
|
| 99 |
-
*
|
| 100 |
-
* Caught by READING the screenshot: the DOM was correct and the CSS ellipsis was doing exactly
|
| 101 |
-
* its job. [[ui-invisible-to-assertions]] — judge the pixels even when everything is green.
|
| 102 |
-
*/
|
| 103 |
-
function railSubtitle(a: Automation): string {
|
| 104 |
-
/*
|
| 105 |
-
* ⭐ WAVE 24 (owner item 6, "Run once now is laggy / looks stuck") — WHILE IT IS RUNNING,
|
| 106 |
-
* THE ONE FACT WORTH THE LINE IS WHAT IT IS DOING.
|
| 107 |
-
*
|
| 108 |
-
* MEASURED, not guessed (`scratchpad/perf_automations.py`, mailbox D-5): the engine has been
|
| 109 |
-
* publishing a live step the whole time (`status.step`, `routes_automation.py:50-51`) and NO
|
| 110 |
-
* SURFACE HAS EVER RENDERED IT. So a `discover_instagram` run parked in its legitimate
|
| 111 |
-
* `BD_FILTER_WAIT = 120 s` vendor wait and a genuinely hung thread were pixel-identical: a
|
| 112 |
-
* pulsing blue dot and a subtitle still reciting the schedule. "Looks stuck" was not a
|
| 113 |
-
* performance problem — the poll costs 23 ms — it was the product declining to say.
|
| 114 |
-
*
|
| 115 |
-
* It takes the line rather than joining it, for the reason this function's note already
|
| 116 |
-
* gives: the rail is `--lp-rail-w` wide and two facts that both truncate are worth less than
|
| 117 |
-
* one that fits. The schedule is still one click away and the dot's `title` carries both.
|
| 118 |
-
*/
|
| 119 |
-
const step = liveStepOf(a);
|
| 120 |
-
if (step) return step;
|
| 121 |
-
if (a.schedule?.enabled) return a.nextRunAt ? `Next ${a.nextRunAt}` : "Scheduled";
|
| 122 |
-
/*
|
| 123 |
-
* ⭐ WAVE 26 · ITEM 19 / R13 — `Last run 2026-08-06 13:58` IS DELETED FROM THIS LINE.
|
| 124 |
-
*
|
| 125 |
-
* ⛔ SCOPE, because R13 draws a line that is easy to over-read: the ruling names THIS secondary
|
| 126 |
-
* line under the automation's name. The Builder's "Last run succeeded / failed" RESULT lines
|
| 127 |
-
* (`AutomationBuilder.tsx:1676`, `:1679`) are a different surface and are explicitly NOT in
|
| 128 |
-
* scope — they are the answer to "Run once now", so deleting them would leave a button with no
|
| 129 |
-
* outcome.
|
| 130 |
-
*
|
| 131 |
-
* "Manual only" STAYS, and the distinction is the ruling's own: a dated run record is a
|
| 132 |
-
* changing FACT ABOUT THE PAST, while "this one has no schedule" is a stable property of the
|
| 133 |
-
* automation — the same question `Next …`/`Scheduled` answers for its siblings. Dropping it too
|
| 134 |
-
* would leave a manual automation with a blank second line and nothing saying why.
|
| 135 |
-
*/
|
| 136 |
-
return "Manual only";
|
| 137 |
-
}
|
| 138 |
-
|
| 139 |
-
/*
|
| 140 |
-
* ⛔ W23-W5's LISTENER IS AT MODULE SCOPE, AND THAT IS THE POINT — not a stylistic choice.
|
| 141 |
-
*
|
| 142 |
-
* The frame's click-through sets `window.location.hash = "#/automation"` and signals on the
|
| 143 |
-
* VERY NEXT LINE (Shell.tsx:1647-1650, and its comment is right that the order matters). But a
|
| 144 |
-
* hash write does not mount anything synchronously: `hashchange` is delivered as a task, the
|
| 145 |
-
* router state then updates, React renders, and only THEN does a component effect subscribe.
|
| 146 |
-
* A listener registered in `useEffect` therefore misses every click that arrives from another
|
| 147 |
-
* page — which is every click, since a reader looking at Alerts is by definition not already
|
| 148 |
-
* on this surface. The event would dispatch into nothing and every gate would stay green:
|
| 149 |
-
* exactly the wave-20 item-25 failure the contract's own note describes, reproduced one layer
|
| 150 |
-
* down.
|
| 151 |
-
*
|
| 152 |
-
* This module is imported statically by the shell, so this listener exists from app start. It
|
| 153 |
-
* LATCHES the request; the component consumes the latch when it mounts and hears live events
|
| 154 |
-
* while it is mounted. A request nobody claims is dropped on the next one — the latch is a
|
| 155 |
-
* one-slot mailbox, never a queue.
|
| 156 |
-
*/
|
| 157 |
-
let pendingOpen: AutomationOpenDetail | null = null;
|
| 158 |
-
const openSubscribers = new Set<(detail: AutomationOpenDetail) => void>();
|
| 159 |
-
|
| 160 |
-
if (typeof window !== "undefined") {
|
| 161 |
-
window.addEventListener(AUTOMATION_OPEN_EVENT, (event) => {
|
| 162 |
-
const detail = (event as CustomEvent<AutomationOpenDetail>).detail;
|
| 163 |
-
if (!detail?.autoId) return;
|
| 164 |
-
if (openSubscribers.size) {
|
| 165 |
-
for (const notify of openSubscribers) notify(detail);
|
| 166 |
-
return;
|
| 167 |
-
}
|
| 168 |
-
pendingOpen = detail;
|
| 169 |
-
});
|
| 170 |
-
}
|
| 171 |
-
|
| 172 |
-
/*
|
| 173 |
-
* ⛔ THE `AUTOMATION_CREATE_EVENT` LISTENER STOOD HERE AND IS DELETED WITH ITS SIGNALLER
|
| 174 |
-
* (wave 25 item 5a, ruling R8) — latch, subscriber set and all.
|
| 175 |
-
*
|
| 176 |
-
* Wave 24 built it to close the opposite defect: the event was declared, signalled from
|
| 177 |
-
* `Shell.tsx`, and consumed NOWHERE, so all three "Automated database" doors navigated to
|
| 178 |
-
* `#/automation` and then did nothing, in production, with every gate green. R8 now deletes those
|
| 179 |
-
* three doors — creating a database and pointing an automation at it are two acts — and they were
|
| 180 |
-
* the event's only signaller.
|
| 181 |
-
*
|
| 182 |
-
* ⛔ SO THIS SIDE GOES TOO, IN THE SAME CHANGE. A listener with no signaller is the same defect
|
| 183 |
-
* read from the other end: it compiles, it costs nothing at runtime, and it reads to the next
|
| 184 |
-
* person as a live channel. The rule that catches both is now DERIVED rather than remembered —
|
| 185 |
-
* `verify_automation_ui.py` enumerates every `AUTOMATION_*_EVENT` in `apiContract.ts` and demands
|
| 186 |
-
* a signal site AND a listener site for each.
|
| 187 |
-
*
|
| 188 |
-
* ⚠ NOTHING ABOUT CREATING AN AUTOMATION IS LOST. `createAndOpen` is untouched and still has two
|
| 189 |
-
* doors, both on this surface: the rail's button and the empty state's. What is gone is the claim
|
| 190 |
-
* that a database can be created BY asking for an automation.
|
| 191 |
-
*/
|
| 192 |
-
|
| 193 |
-
/**
|
| 194 |
-
* A default name for a brand-new automation, and it has to be one the server ACCEPTS.
|
| 195 |
-
*
|
| 196 |
-
* MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create
|
| 197 |
-
* with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name
|
| 198 |
-
* cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that
|
| 199 |
-
* sorts by name with nothing to tell the rows apart. Numbering from the existing list is a
|
| 200 |
-
* client-side convenience over a server that does not care: if two tabs race, the loser gets a
|
| 201 |
-
* duplicate name, which is legal, visible and renameable — never an error the user has to read.
|
| 202 |
-
*/
|
| 203 |
-
function nextAutomationName(existing: Automation[]): string {
|
| 204 |
-
const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase()));
|
| 205 |
-
for (let n = existing.length + 1; ; n += 1) {
|
| 206 |
-
const candidate = `Automation ${n}`;
|
| 207 |
-
if (!taken.has(candidate.toLowerCase())) return candidate;
|
| 208 |
-
}
|
| 209 |
-
}
|
| 210 |
-
|
| 211 |
-
export default function AutomationSurface() {
|
| 212 |
-
/* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
|
| 213 |
-
through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
|
| 214 |
-
once, on mount, BEFORE the first paint — `useState(cachedAutomations())` would call it on
|
| 215 |
-
every render instead, which is the same value at needless cost. A cold start still returns
|
| 216 |
-
null and still gets the skeleton: it is made rare, not removed (see `automationApi.ts`). */
|
| 217 |
-
const [data, setData] = useState<AutomationList | null>(() => cachedAutomations());
|
| 218 |
-
const [error, setError] = useState("");
|
| 219 |
-
const [activeId, setActiveId] = useState<string>("");
|
| 220 |
-
const [railShut, setRailShut] = useState(false);
|
| 221 |
-
const [busy, setBusy] = useState("");
|
| 222 |
-
/**
|
| 223 |
-
* ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to
|
| 224 |
-
* say it is busy. Without this a second click while the POST is in flight makes a second
|
| 225 |
-
* automation, and the server allows duplicate names, so the user gets two rows that look
|
| 226 |
-
* identical and no error to explain either of them.
|
| 227 |
-
*/
|
| 228 |
-
const [creatingNow, setCreatingNow] = useState(false);
|
| 229 |
-
/** An open request waiting for the list to arrive (see the resolver below). */
|
| 230 |
-
const [openRequest, setOpenRequest] = useState<AutomationOpenDetail | null>(null);
|
| 231 |
-
/* ⛔ `createRequest` LEFT WITH THE EVENT THAT SET IT (wave 25, R8) — see the tombstone above. */
|
| 232 |
-
const listRef = useRef<HTMLDivElement | null>(null);
|
| 233 |
-
|
| 234 |
-
/**
|
| 235 |
-
* ⛔ C14 LEG 2 — THE ID THE USER LAST ASKED FOR, and it is a ref because it has to be
|
| 236 |
-
* written SYNCHRONOUSLY, inside the click, before any promise that was already in flight
|
| 237 |
-
* can resolve. State would not do: a resolution racing React's commit would read the
|
| 238 |
-
* previous value, which is the exact window this guard exists to close.
|
| 239 |
-
*/
|
| 240 |
-
const wantedId = useRef("");
|
| 241 |
-
|
| 242 |
-
/**
|
| 243 |
-
* ⭐ THE ONLY PLACE THE RAIL'S SELECTION MOVES ON PURPOSE. Every deliberate change of
|
| 244 |
-
* automation — a rail click, a create, a delete — goes through here, so "what the user
|
| 245 |
-
* asked for" and "what is on screen" are written in one statement and cannot drift apart.
|
| 246 |
-
* A resolution that wants to steer the rail compares itself against `wantedId` instead.
|
| 247 |
-
*/
|
| 248 |
-
const select = useCallback((id: string, _stage = "") => {
|
| 249 |
-
wantedId.current = id;
|
| 250 |
-
setActiveId(id);
|
| 251 |
-
}, []);
|
| 252 |
-
|
| 253 |
-
/**
|
| 254 |
-
*
|
| 255 |
-
*
|
| 256 |
-
*
|
| 257 |
-
*
|
| 258 |
-
*
|
| 259 |
-
*
|
| 260 |
-
*
|
| 261 |
-
* ⚠
|
| 262 |
-
*
|
| 263 |
-
*
|
| 264 |
-
*
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
*
|
| 317 |
-
*
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
//
|
| 373 |
-
//
|
| 374 |
-
//
|
| 375 |
-
//
|
| 376 |
-
//
|
| 377 |
-
//
|
| 378 |
-
//
|
| 379 |
-
//
|
| 380 |
-
//
|
| 381 |
-
//
|
| 382 |
-
//
|
| 383 |
-
//
|
| 384 |
-
//
|
| 385 |
-
//
|
| 386 |
-
//
|
| 387 |
-
//
|
| 388 |
-
//
|
| 389 |
-
//
|
| 390 |
-
//
|
| 391 |
-
//
|
| 392 |
-
//
|
| 393 |
-
//
|
| 394 |
-
//
|
| 395 |
-
//
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
useEffect(() => {
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
*
|
| 461 |
-
*
|
| 462 |
-
*
|
| 463 |
-
*
|
| 464 |
-
*
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
*
|
| 482 |
-
*
|
| 483 |
-
*
|
| 484 |
-
*
|
| 485 |
-
* `automations`
|
| 486 |
-
*
|
| 487 |
-
*
|
| 488 |
-
*
|
| 489 |
-
*
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
<
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
{
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
<
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
*
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
|
| 760 |
-
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
|
| 772 |
-
|
| 773 |
-
|
| 774 |
-
|
| 775 |
-
|
| 776 |
-
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
also had the header's editable 16px/700 input, against every database page's
|
| 784 |
-
single 16px/650 `shell-db-name`. The shell now wraps this branch in the same
|
| 785 |
-
`shell-db-frame` + `DbHead` a database gets (wiring W23-W1), so the page's name
|
| 786 |
-
is drawn once, by the one component that draws every other page's name. A
|
| 787 |
-
stand-in restyled to match would have been a second copy of the same fact,
|
| 788 |
-
free to drift the day the header moves.
|
| 789 |
-
*/}
|
| 790 |
-
<p className="autob-empty-line">
|
| 791 |
-
A job this workspace runs for you, on demand or on a schedule.
|
| 792 |
-
</p>
|
| 793 |
-
<button
|
| 794 |
-
type="button"
|
| 795 |
-
className="auto-btn is-primary"
|
| 796 |
-
disabled={creatingNow}
|
| 797 |
-
onClick={() => void createAndOpen()}
|
| 798 |
-
>
|
| 799 |
-
New automation
|
| 800 |
-
</button>
|
| 801 |
-
</div>
|
| 802 |
-
)}
|
| 803 |
-
</section>
|
| 804 |
-
</div>
|
| 805 |
-
);
|
| 806 |
-
}
|
|
|
|
| 1 |
+
// ---------------------------------------------------------------------------
|
| 2 |
+
// automation/AutomationSurface.tsx — the Automation surface (contract C-AUTONAV).
|
| 3 |
+
//
|
| 4 |
+
// ⛔ THE CONTRACT THIS FILE GUARANTEES, verbatim: this module path, a DEFAULT
|
| 5 |
+
// export, and NO PROPS. The shell mounts `<AutomationSurface />` and hands it
|
| 6 |
+
// nothing; it fetches its own data from `/api/v1/automations`. That is the whole
|
| 7 |
+
// interface, and it is stated here because the mount line lives in a file this
|
| 8 |
+
// session does not own — a named export would cost a cross-session round trip.
|
| 9 |
+
//
|
| 10 |
+
// "Automation replaces the Views rail" (owner) is implemented the way the wave
|
| 11 |
+
// scout established: the secondary rail is NOT a shell concept — each surface
|
| 12 |
+
// owns its own. So this renders a sibling of `.cg-views` built from the same
|
| 13 |
+
// `--lp-rail-w` tokens and the same fold behaviour, and the Views rail is
|
| 14 |
+
// untouched.
|
| 15 |
+
// ---------------------------------------------------------------------------
|
| 16 |
+
import type { KeyboardEvent as ReactKeyboardEvent } from "react";
|
| 17 |
+
import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
|
| 18 |
+
|
| 19 |
+
import type { AutomationOpenDetail } from "../apiContract";
|
| 20 |
+
import { AUTOMATION_OPEN_EVENT } from "../apiContract";
|
| 21 |
+
// ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
|
| 22 |
+
// The owner's reason is the mark's own geometry: *"these automations are basically loops."*
|
| 23 |
+
// C7 says consume the existing icon component and do not inline a new SVG path, and
|
| 24 |
+
// `shell/Brand.tsx`'s header says why in more detail than a rule could: the mark's one source of
|
| 25 |
+
// truth is generated, it has already been hand-redrawn once, and the copy silently painted LAST
|
| 26 |
+
// WAVE'S BRAND while a comment asserted parity. So this imports the component that POINTS at the
|
| 27 |
+
// generated artifact. [[loopable-nav-logo-toggle]] — one element paints the mark.
|
| 28 |
+
import { Mark } from "../shell/Brand";
|
| 29 |
+
import type { Automation, AutomationList } from "./automationApi";
|
| 30 |
+
import {
|
| 31 |
+
AutomationError,
|
| 32 |
+
cachedAutomations,
|
| 33 |
+
createAutomation,
|
| 34 |
+
listAutomations,
|
| 35 |
+
liveStepOf,
|
| 36 |
+
patchAutomation,
|
| 37 |
+
rememberAutomation,
|
| 38 |
+
} from "./automationApi";
|
| 39 |
+
|
| 40 |
+
/**
|
| 41 |
+
* ⭐⭐ WAVE 30 · T21 (owner items 4 + 5) — THE EDITOR STOPS SHIPPING INSIDE THE LIST.
|
| 42 |
+
*
|
| 43 |
+
* ⛔ THIS `lazy()` IS THE TICKET. `AutomationDetail` was a STATIC import, and it statically pulls
|
| 44 |
+
* `AutomationBuilder` (which in turn pulls `CondBuilder`, `PresetPlan` and `TriggerPicker`),
|
| 45 |
+
* `AutomationFind` and `AutomationTrigger` — so the whole editor was linked into the one chunk a
|
| 46 |
+
* person must download before the RAIL can paint a list of names. MEASURED at 88,771 B, against
|
| 47 |
+
* `HomePage` at 4,814 and `ConnectorsPage` at 3,551. That is the "Automation takes forever to
|
| 48 |
+
* load" complaint, and it had survived two waves because nothing asserted a number.
|
| 49 |
+
*
|
| 50 |
+
* ⛔ AND IT NEEDS ITS OWN `<Suspense>`, NOT `Shell.tsx`'s. `Lazily` already wraps this whole
|
| 51 |
+
* surface, but a Suspense boundary catches ANY suspending descendant — so without an inner
|
| 52 |
+
* boundary, opening an automation would suspend the boundary that owns the RAIL and blank the
|
| 53 |
+
* list you just clicked in. The inner one keeps the rail painted while the editor streams in.
|
| 54 |
+
* `fallback={null}` is deliberate and is `Lazily`'s own choice, not a shortcut: the work area is
|
| 55 |
+
* simply not there for the moment the chunk is in flight, which is what every other route in this
|
| 56 |
+
* product already does. ⚠ A skeleton here would be a NEW loading screen in the same wave that
|
| 57 |
+
* exists to remove one (T22).
|
| 58 |
+
* ⚠ Default export, checked: `Shell.tsx:53` records that `lazy()` over a NAMED export fails at
|
| 59 |
+
* runtime, on click, and only for that one surface — `AutomationDetail` exports default.
|
| 60 |
+
*/
|
| 61 |
+
const AutomationDetail = lazy(() => import("./AutomationDetail"));
|
| 62 |
+
|
| 63 |
+
/**
|
| 64 |
+
* The run poll's cadence (item 6). Named constants because they are a MEASURED trade-off, not a
|
| 65 |
+
* taste: `POLL_MAX_MS` is the longest a finished run can still look live, and it is the only
|
| 66 |
+
* cost the backoff has.
|
| 67 |
+
*/
|
| 68 |
+
const POLL_MIN_MS = 2500;
|
| 69 |
+
const POLL_MAX_MS = 8000;
|
| 70 |
+
|
| 71 |
+
/*
|
| 72 |
+
* ⭐ WAVE 26 · ITEM 18 / R14 — `stateTitle` STOOD HERE AND IS DELETED WITH THE DOT IT DESCRIBED.
|
| 73 |
+
*
|
| 74 |
+
* It composed the dot's tooltip ("Running now — <step>", "Last run failed — <summary>"), so it had
|
| 75 |
+
* exactly one reader and no reason to outlive it. Deleting the render and keeping the composer is
|
| 76 |
+
* how a file accumulates functions that look live and are not — and `noUnusedLocals` would have
|
| 77 |
+
* caught this one, which is not a reason to lean on it: the NEXT such function might still have a
|
| 78 |
+
* second caller and compile fine.
|
| 79 |
+
*
|
| 80 |
+
* ⛔ WHY THE TOOLTIP WAS NOT MOVED ONTO THE MARK INSTEAD. R14 is "same mark on every automation;
|
| 81 |
+
* no status colour". Hanging "Last run failed" off a mark that is deliberately status-free just
|
| 82 |
+
* moves the status channel into `title`/`aria-label`, where it is worse: invisible to the eye,
|
| 83 |
+
* announced to a screen reader, and contradicting the visual. The run state has a home one click
|
| 84 |
+
* away — the Builder's own "Last run succeeded / failed" lines, which R13 explicitly keeps.
|
| 85 |
+
*
|
| 86 |
+
* ⚠ `stateOf` ITSELF IS UNTOUCHED in `automationApi.ts`. `Shell.tsx:64/1095` imports it and
|
| 87 |
+
* `liveStepOf` calls it — deleting the export to tidy up this file would break a fence I do not own.
|
| 88 |
+
*/
|
| 89 |
+
|
| 90 |
+
/**
|
| 91 |
+
* The rail's second line — ONE fact, not two.
|
| 92 |
+
*
|
| 93 |
+
* ⚠ It carried "Next 2026-08-04 06:00 · last 2026-08-04 00:20", and the rail is `--lp-rail-w`
|
| 94 |
+
* wide, so it rendered as "Next 2026-08-04 06:00 · l…" — a second line whose only complete word
|
| 95 |
+
* was "Next". Two facts that both truncate are worth less than one that fits. The forward-looking
|
| 96 |
+
* one wins when a schedule exists (it is the question the rail answers: does this run itself?),
|
| 97 |
+
* the last run otherwise. The full picture is one click away in the editor's run history, and the
|
| 98 |
+
* row's `title` already says both.
|
| 99 |
+
*
|
| 100 |
+
* Caught by READING the screenshot: the DOM was correct and the CSS ellipsis was doing exactly
|
| 101 |
+
* its job. [[ui-invisible-to-assertions]] — judge the pixels even when everything is green.
|
| 102 |
+
*/
|
| 103 |
+
function railSubtitle(a: Automation): string {
|
| 104 |
+
/*
|
| 105 |
+
* ⭐ WAVE 24 (owner item 6, "Run once now is laggy / looks stuck") — WHILE IT IS RUNNING,
|
| 106 |
+
* THE ONE FACT WORTH THE LINE IS WHAT IT IS DOING.
|
| 107 |
+
*
|
| 108 |
+
* MEASURED, not guessed (`scratchpad/perf_automations.py`, mailbox D-5): the engine has been
|
| 109 |
+
* publishing a live step the whole time (`status.step`, `routes_automation.py:50-51`) and NO
|
| 110 |
+
* SURFACE HAS EVER RENDERED IT. So a `discover_instagram` run parked in its legitimate
|
| 111 |
+
* `BD_FILTER_WAIT = 120 s` vendor wait and a genuinely hung thread were pixel-identical: a
|
| 112 |
+
* pulsing blue dot and a subtitle still reciting the schedule. "Looks stuck" was not a
|
| 113 |
+
* performance problem — the poll costs 23 ms — it was the product declining to say.
|
| 114 |
+
*
|
| 115 |
+
* It takes the line rather than joining it, for the reason this function's note already
|
| 116 |
+
* gives: the rail is `--lp-rail-w` wide and two facts that both truncate are worth less than
|
| 117 |
+
* one that fits. The schedule is still one click away and the dot's `title` carries both.
|
| 118 |
+
*/
|
| 119 |
+
const step = liveStepOf(a);
|
| 120 |
+
if (step) return step;
|
| 121 |
+
if (a.schedule?.enabled) return a.nextRunAt ? `Next ${a.nextRunAt}` : "Scheduled";
|
| 122 |
+
/*
|
| 123 |
+
* ⭐ WAVE 26 · ITEM 19 / R13 — `Last run 2026-08-06 13:58` IS DELETED FROM THIS LINE.
|
| 124 |
+
*
|
| 125 |
+
* ⛔ SCOPE, because R13 draws a line that is easy to over-read: the ruling names THIS secondary
|
| 126 |
+
* line under the automation's name. The Builder's "Last run succeeded / failed" RESULT lines
|
| 127 |
+
* (`AutomationBuilder.tsx:1676`, `:1679`) are a different surface and are explicitly NOT in
|
| 128 |
+
* scope — they are the answer to "Run once now", so deleting them would leave a button with no
|
| 129 |
+
* outcome.
|
| 130 |
+
*
|
| 131 |
+
* "Manual only" STAYS, and the distinction is the ruling's own: a dated run record is a
|
| 132 |
+
* changing FACT ABOUT THE PAST, while "this one has no schedule" is a stable property of the
|
| 133 |
+
* automation — the same question `Next …`/`Scheduled` answers for its siblings. Dropping it too
|
| 134 |
+
* would leave a manual automation with a blank second line and nothing saying why.
|
| 135 |
+
*/
|
| 136 |
+
return "Manual only";
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
/*
|
| 140 |
+
* ⛔ W23-W5's LISTENER IS AT MODULE SCOPE, AND THAT IS THE POINT — not a stylistic choice.
|
| 141 |
+
*
|
| 142 |
+
* The frame's click-through sets `window.location.hash = "#/automation"` and signals on the
|
| 143 |
+
* VERY NEXT LINE (Shell.tsx:1647-1650, and its comment is right that the order matters). But a
|
| 144 |
+
* hash write does not mount anything synchronously: `hashchange` is delivered as a task, the
|
| 145 |
+
* router state then updates, React renders, and only THEN does a component effect subscribe.
|
| 146 |
+
* A listener registered in `useEffect` therefore misses every click that arrives from another
|
| 147 |
+
* page — which is every click, since a reader looking at Alerts is by definition not already
|
| 148 |
+
* on this surface. The event would dispatch into nothing and every gate would stay green:
|
| 149 |
+
* exactly the wave-20 item-25 failure the contract's own note describes, reproduced one layer
|
| 150 |
+
* down.
|
| 151 |
+
*
|
| 152 |
+
* This module is imported statically by the shell, so this listener exists from app start. It
|
| 153 |
+
* LATCHES the request; the component consumes the latch when it mounts and hears live events
|
| 154 |
+
* while it is mounted. A request nobody claims is dropped on the next one — the latch is a
|
| 155 |
+
* one-slot mailbox, never a queue.
|
| 156 |
+
*/
|
| 157 |
+
let pendingOpen: AutomationOpenDetail | null = null;
|
| 158 |
+
const openSubscribers = new Set<(detail: AutomationOpenDetail) => void>();
|
| 159 |
+
|
| 160 |
+
if (typeof window !== "undefined") {
|
| 161 |
+
window.addEventListener(AUTOMATION_OPEN_EVENT, (event) => {
|
| 162 |
+
const detail = (event as CustomEvent<AutomationOpenDetail>).detail;
|
| 163 |
+
if (!detail?.autoId) return;
|
| 164 |
+
if (openSubscribers.size) {
|
| 165 |
+
for (const notify of openSubscribers) notify(detail);
|
| 166 |
+
return;
|
| 167 |
+
}
|
| 168 |
+
pendingOpen = detail;
|
| 169 |
+
});
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
/*
|
| 173 |
+
* ⛔ THE `AUTOMATION_CREATE_EVENT` LISTENER STOOD HERE AND IS DELETED WITH ITS SIGNALLER
|
| 174 |
+
* (wave 25 item 5a, ruling R8) — latch, subscriber set and all.
|
| 175 |
+
*
|
| 176 |
+
* Wave 24 built it to close the opposite defect: the event was declared, signalled from
|
| 177 |
+
* `Shell.tsx`, and consumed NOWHERE, so all three "Automated database" doors navigated to
|
| 178 |
+
* `#/automation` and then did nothing, in production, with every gate green. R8 now deletes those
|
| 179 |
+
* three doors — creating a database and pointing an automation at it are two acts — and they were
|
| 180 |
+
* the event's only signaller.
|
| 181 |
+
*
|
| 182 |
+
* ⛔ SO THIS SIDE GOES TOO, IN THE SAME CHANGE. A listener with no signaller is the same defect
|
| 183 |
+
* read from the other end: it compiles, it costs nothing at runtime, and it reads to the next
|
| 184 |
+
* person as a live channel. The rule that catches both is now DERIVED rather than remembered —
|
| 185 |
+
* `verify_automation_ui.py` enumerates every `AUTOMATION_*_EVENT` in `apiContract.ts` and demands
|
| 186 |
+
* a signal site AND a listener site for each.
|
| 187 |
+
*
|
| 188 |
+
* ⚠ NOTHING ABOUT CREATING AN AUTOMATION IS LOST. `createAndOpen` is untouched and still has two
|
| 189 |
+
* doors, both on this surface: the rail's button and the empty state's. What is gone is the claim
|
| 190 |
+
* that a database can be created BY asking for an automation.
|
| 191 |
+
*/
|
| 192 |
+
|
| 193 |
+
/**
|
| 194 |
+
* A default name for a brand-new automation, and it has to be one the server ACCEPTS.
|
| 195 |
+
*
|
| 196 |
+
* MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create
|
| 197 |
+
* with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name
|
| 198 |
+
* cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that
|
| 199 |
+
* sorts by name with nothing to tell the rows apart. Numbering from the existing list is a
|
| 200 |
+
* client-side convenience over a server that does not care: if two tabs race, the loser gets a
|
| 201 |
+
* duplicate name, which is legal, visible and renameable — never an error the user has to read.
|
| 202 |
+
*/
|
| 203 |
+
function nextAutomationName(existing: Automation[]): string {
|
| 204 |
+
const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase()));
|
| 205 |
+
for (let n = existing.length + 1; ; n += 1) {
|
| 206 |
+
const candidate = `Automation ${n}`;
|
| 207 |
+
if (!taken.has(candidate.toLowerCase())) return candidate;
|
| 208 |
+
}
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
export default function AutomationSurface() {
|
| 212 |
+
/* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
|
| 213 |
+
through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
|
| 214 |
+
once, on mount, BEFORE the first paint — `useState(cachedAutomations())` would call it on
|
| 215 |
+
every render instead, which is the same value at needless cost. A cold start still returns
|
| 216 |
+
null and still gets the skeleton: it is made rare, not removed (see `automationApi.ts`). */
|
| 217 |
+
const [data, setData] = useState<AutomationList | null>(() => cachedAutomations());
|
| 218 |
+
const [error, setError] = useState("");
|
| 219 |
+
const [activeId, setActiveId] = useState<string>("");
|
| 220 |
+
const [railShut, setRailShut] = useState(false);
|
| 221 |
+
const [busy, setBusy] = useState("");
|
| 222 |
+
/**
|
| 223 |
+
* ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to
|
| 224 |
+
* say it is busy. Without this a second click while the POST is in flight makes a second
|
| 225 |
+
* automation, and the server allows duplicate names, so the user gets two rows that look
|
| 226 |
+
* identical and no error to explain either of them.
|
| 227 |
+
*/
|
| 228 |
+
const [creatingNow, setCreatingNow] = useState(false);
|
| 229 |
+
/** An open request waiting for the list to arrive (see the resolver below). */
|
| 230 |
+
const [openRequest, setOpenRequest] = useState<AutomationOpenDetail | null>(null);
|
| 231 |
+
/* ⛔ `createRequest` LEFT WITH THE EVENT THAT SET IT (wave 25, R8) — see the tombstone above. */
|
| 232 |
+
const listRef = useRef<HTMLDivElement | null>(null);
|
| 233 |
+
|
| 234 |
+
/**
|
| 235 |
+
* ⛔ C14 LEG 2 — THE ID THE USER LAST ASKED FOR, and it is a ref because it has to be
|
| 236 |
+
* written SYNCHRONOUSLY, inside the click, before any promise that was already in flight
|
| 237 |
+
* can resolve. State would not do: a resolution racing React's commit would read the
|
| 238 |
+
* previous value, which is the exact window this guard exists to close.
|
| 239 |
+
*/
|
| 240 |
+
const wantedId = useRef("");
|
| 241 |
+
|
| 242 |
+
/**
|
| 243 |
+
* ⭐ THE ONLY PLACE THE RAIL'S SELECTION MOVES ON PURPOSE. Every deliberate change of
|
| 244 |
+
* automation — a rail click, a create, a delete — goes through here, so "what the user
|
| 245 |
+
* asked for" and "what is on screen" are written in one statement and cannot drift apart.
|
| 246 |
+
* A resolution that wants to steer the rail compares itself against `wantedId` instead.
|
| 247 |
+
*/
|
| 248 |
+
const select = useCallback((id: string, _stage = "") => {
|
| 249 |
+
wantedId.current = id;
|
| 250 |
+
setActiveId(id);
|
| 251 |
+
}, []);
|
| 252 |
+
|
| 253 |
+
/**
|
| 254 |
+
* ⛔ C14 LEG 3 (the paint half) — A STALE READ MAY NOT REPAINT.
|
| 255 |
+
*
|
| 256 |
+
* Every call takes the next generation; only the newest one is allowed to `setData`. The
|
| 257 |
+
* defect this closes is not cosmetic: a poll issued BEFORE a delete resolves AFTER it, and
|
| 258 |
+
* a list that still contains the deleted automation puts a row back in the rail that the
|
| 259 |
+
* user has just watched disappear. Same for a toggle, and same for a create.
|
| 260 |
+
*
|
| 261 |
+
* ⚠ THE RETURN VALUE IS NOT GUARDED, deliberately. The caller asked a question and gets
|
| 262 |
+
* its own answer — coupling the two would mean a create whose `load()` was overtaken by a
|
| 263 |
+
* poll silently failed to select the automation it had just made.
|
| 264 |
+
*/
|
| 265 |
+
const gen = useRef(0);
|
| 266 |
+
|
| 267 |
+
const load = useCallback(async (abort?: AbortSignal) => {
|
| 268 |
+
const mine = ++gen.current;
|
| 269 |
+
try {
|
| 270 |
+
const next = await listAutomations(abort);
|
| 271 |
+
if (mine === gen.current) {
|
| 272 |
+
setData(next);
|
| 273 |
+
setError("");
|
| 274 |
+
}
|
| 275 |
+
return next;
|
| 276 |
+
} catch (e) {
|
| 277 |
+
if ((e as Error)?.name === "AbortError") return null;
|
| 278 |
+
if (mine === gen.current)
|
| 279 |
+
setError(
|
| 280 |
+
e instanceof AutomationError
|
| 281 |
+
? e.message
|
| 282 |
+
: "The automation service did not answer."
|
| 283 |
+
);
|
| 284 |
+
return null;
|
| 285 |
+
}
|
| 286 |
+
}, []);
|
| 287 |
+
|
| 288 |
+
/**
|
| 289 |
+
* ⭐⭐ WAVE 31 · T31 — one freshly-written automation replaces its own row, with no round trip.
|
| 290 |
+
*
|
| 291 |
+
* ⚠ IT TAKES THE GENERATION, and that is not decoration. `load`'s guard exists so a stale READ
|
| 292 |
+
* cannot repaint; a WRITE response is by definition newer than any read already in flight, so it
|
| 293 |
+
* must win the same race rather than sit outside it. Without the bump, a `GET /automations`
|
| 294 |
+
* issued before the save could resolve after the merge and paint the pre-save row back — the
|
| 295 |
+
* exact ghost `gen` was introduced for, arriving through the new door.
|
| 296 |
+
*
|
| 297 |
+
* ⚠ Returns FALSE when there is nothing to merge into (a cold surface has no memo), and the
|
| 298 |
+
* caller then does a real load. Painting a one-row list would be worse than a round trip.
|
| 299 |
+
*/
|
| 300 |
+
const mergeSaved = useCallback((fresh: Automation) => {
|
| 301 |
+
const next = rememberAutomation(fresh);
|
| 302 |
+
if (!next) return false;
|
| 303 |
+
gen.current += 1;
|
| 304 |
+
setData(next);
|
| 305 |
+
setError("");
|
| 306 |
+
return true;
|
| 307 |
+
}, []);
|
| 308 |
+
|
| 309 |
+
useEffect(() => {
|
| 310 |
+
const ac = new AbortController();
|
| 311 |
+
void load(ac.signal);
|
| 312 |
+
return () => ac.abort();
|
| 313 |
+
}, [load]);
|
| 314 |
+
|
| 315 |
+
/**
|
| 316 |
+
* ⭐⭐ WAVE 24 / C-CREATE (a) — CREATE AND OPEN. This REPLACED the three-step wizard.
|
| 317 |
+
*
|
| 318 |
+
* R6 is what makes it possible: `plain` is a real kind with no machine graph nodes, and it is
|
| 319 |
+
* what `POST /automations` stores when the body names none. So "new automation" stopped being
|
| 320 |
+
* a question ("which of three kinds?" — two of which can no longer be created at all) and
|
| 321 |
+
* became what it says: a new automation, open, on its trigger picker. Choosing a SOURCE is now
|
| 322 |
+
* picking the `ig_profile_match` trigger, which is where that choice belongs.
|
| 323 |
+
*
|
| 324 |
+
* ⛔ THE GUARD IS A REF, not the `creatingNow` state, and this file already carries the scar
|
| 325 |
+
* that explains why (C14 leg 2): a state read inside a click closure is the value from the
|
| 326 |
+
* last render, so two fast clicks both see `false` and both POST. Duplicate names are legal
|
| 327 |
+
* server-side, so the user would get two identical rows and no error. The ref is written
|
| 328 |
+
* synchronously, inside the click, before anything can await.
|
| 329 |
+
*/
|
| 330 |
+
const creatingRef = useRef(false);
|
| 331 |
+
const createAndOpen = useCallback(async () => {
|
| 332 |
+
if (creatingRef.current) return;
|
| 333 |
+
creatingRef.current = true;
|
| 334 |
+
setCreatingNow(true);
|
| 335 |
+
try {
|
| 336 |
+
const res = await createAutomation({
|
| 337 |
+
name: nextAutomationName(data?.automations || []),
|
| 338 |
+
});
|
| 339 |
+
const id = res?.automation?.id || "";
|
| 340 |
+
// The create flow is the one caller allowed to name a different id (C14 leg 2) — the
|
| 341 |
+
// automation did not exist when the click happened, so there is nothing to race with.
|
| 342 |
+
const next = await load();
|
| 343 |
+
if (id && next) select(id);
|
| 344 |
+
} catch (e) {
|
| 345 |
+
setError(
|
| 346 |
+
e instanceof AutomationError ? e.message : "The automation was not created."
|
| 347 |
+
);
|
| 348 |
+
} finally {
|
| 349 |
+
creatingRef.current = false;
|
| 350 |
+
setCreatingNow(false);
|
| 351 |
+
}
|
| 352 |
+
}, [data, load, select]);
|
| 353 |
+
|
| 354 |
+
// A run is a background thread on the server, so the surface has to ASK whether it
|
| 355 |
+
// finished. Polling only while something is actually running keeps an idle surface
|
| 356 |
+
// silent — a fixed interval would be a request every few seconds forever, for a page
|
| 357 |
+
// whose contents change a handful of times a day.
|
| 358 |
+
//
|
| 359 |
+
// ⛔ C14 LEG 3 (the abort half). It used to call `load()` bare — no signal, nothing to
|
| 360 |
+
// cancel — so a request the interval had already issued kept going after the effect that
|
| 361 |
+
// owned it was gone. One controller per effect run, aborted with the interval, means a poll
|
| 362 |
+
// cannot outlive the condition that justified it. The generation guard inside `load` covers
|
| 363 |
+
// the rest: a response that survives the abort still cannot repaint over a newer one.
|
| 364 |
+
//
|
| 365 |
+
// ⭐ WAVE 24 (item 6, C-PERF) — THE INTERVAL BECAME A BACKOFF, and the measurement is why it is
|
| 366 |
+
// a SMALL change rather than the big one the contract's hypothesis 2 asked for.
|
| 367 |
+
//
|
| 368 |
+
// MEASURED (`scratchpad/perf_automations.py`, mailbox D-5): this poll costs ~23 ms and 24 KB
|
| 369 |
+
// for a ten-automation tenant, and the engine work the hypothesis blamed — a `graph()` rebuild
|
| 370 |
+
// per automation — is **41 µs each**, i.e. under 4% of the request. The payload was never the
|
| 371 |
+
// problem, so nothing here gets a cheaper endpoint.
|
| 372 |
+
//
|
| 373 |
+
// What the numbers DO indict is the aggregate: a discovery run legitimately blocks up to
|
| 374 |
+
// `BD_FILTER_WAIT` = 120 s, and two independent 2.5 s polls across this file and the detail
|
| 375 |
+
// spend ~96 requests and ~2.3 MB over that window — in the same single process the run itself
|
| 376 |
+
// is a thread in. So the delay grows 2.5 → 5 → 8 s and stops there.
|
| 377 |
+
//
|
| 378 |
+
// ⚠ NOT "to learn nothing", and the distinction is the whole justification. That WAS true when
|
| 379 |
+
// this file rendered no live step: the poll's only observable effect was a dot that had already
|
| 380 |
+
// been pulsing for two minutes. It stopped being true in this same change — `liveStepOf` now
|
| 381 |
+
// paints `status.step`, so a poll carries the one fact worth having. The backoff is therefore
|
| 382 |
+
// NOT "stop asking a pointless question"; it is FEWER ROUND TRIPS FOR THE SAME INFORMATION, on
|
| 383 |
+
// a step text that changes every few seconds at most, not every 2.5.
|
| 384 |
+
// The cap is deliberately low: it bounds how long a FINISHED run can still look live, which is
|
| 385 |
+
// the only thing a backoff can make worse, and 8 s of that is worth ~⅔ fewer requests.
|
| 386 |
+
//
|
| 387 |
+
// ⛔ THE KEY IS THE RUNNING SET, NOT A BOOLEAN, and that is what makes the reset correct: the
|
| 388 |
+
// effect re-runs — and the delay drops back to 2.5 s — the moment a run starts or finishes, so
|
| 389 |
+
// a user who clicks Run now gets the fast cadence again instead of inheriting the tail of the
|
| 390 |
+
// previous run's backoff. A bare `anyRunning` boolean cannot see the second run start.
|
| 391 |
+
// A step text changing does NOT re-key it, so the interval never thrashes.
|
| 392 |
+
//
|
| 393 |
+
// C14 LEG 3 (the abort half) is unchanged and still load-bearing: one controller per effect
|
| 394 |
+
// run, aborted with the timer, so a poll cannot outlive the condition that justified it. The
|
| 395 |
+
// generation guard inside `load` covers the rest.
|
| 396 |
+
const runningKey = (data?.automations || [])
|
| 397 |
+
.filter((a) => a.running)
|
| 398 |
+
.map((a) => a.id)
|
| 399 |
+
.sort()
|
| 400 |
+
.join(",");
|
| 401 |
+
useEffect(() => {
|
| 402 |
+
if (!runningKey) return undefined;
|
| 403 |
+
const ac = new AbortController();
|
| 404 |
+
let delay = POLL_MIN_MS;
|
| 405 |
+
let timer = 0;
|
| 406 |
+
const tick = () => {
|
| 407 |
+
void load(ac.signal);
|
| 408 |
+
delay = Math.min(delay * 2, POLL_MAX_MS);
|
| 409 |
+
timer = window.setTimeout(tick, delay);
|
| 410 |
+
};
|
| 411 |
+
timer = window.setTimeout(tick, delay);
|
| 412 |
+
return () => {
|
| 413 |
+
window.clearTimeout(timer);
|
| 414 |
+
ac.abort();
|
| 415 |
+
};
|
| 416 |
+
}, [runningKey, load]);
|
| 417 |
+
|
| 418 |
+
// ── W23-W5, the surface's half: hear the request, then answer it when we CAN ──────────
|
| 419 |
+
useEffect(() => {
|
| 420 |
+
const notify = (detail: AutomationOpenDetail) => setOpenRequest(detail);
|
| 421 |
+
openSubscribers.add(notify);
|
| 422 |
+
if (pendingOpen) {
|
| 423 |
+
const latched = pendingOpen;
|
| 424 |
+
pendingOpen = null;
|
| 425 |
+
setOpenRequest(latched);
|
| 426 |
+
}
|
| 427 |
+
return () => {
|
| 428 |
+
openSubscribers.delete(notify);
|
| 429 |
+
};
|
| 430 |
+
}, []);
|
| 431 |
+
|
| 432 |
+
/* ⛔ THE C-CREATE(b) SUBSCRIBE EFFECT AND ITS RESOLVER STOOD HERE (wave 25, R8). They heard the
|
| 433 |
+
create event and, once the list had arrived, called `createAndOpen` — the "wait for the list
|
| 434 |
+
or every automation is named Automation 1" note lives on in `nextAutomationName`, which is
|
| 435 |
+
still numbered off `existing`. With no signaller there is nothing to hear. */
|
| 436 |
+
|
| 437 |
+
/**
|
| 438 |
+
* ⚠ THE REQUEST OUTLIVES THE FETCH, and it has to. The reader clicks a notification from
|
| 439 |
+
* another page, so this surface is mounting WITH AN EMPTY LIST — "select it if it is in the
|
| 440 |
+
* list" would drop every real click and keep only the one case where the reader was already
|
| 441 |
+
* here. So the request is held until `data` exists, and only then answered.
|
| 442 |
+
*
|
| 443 |
+
* An automation the reader can no longer open does NOTHING (the contract's own words): the
|
| 444 |
+
* request is cleared either way, so a stale id cannot sit here re-firing against every
|
| 445 |
+
* subsequent list.
|
| 446 |
+
*/
|
| 447 |
+
useEffect(() => {
|
| 448 |
+
if (!openRequest || !data) return;
|
| 449 |
+
const found = (data.automations || []).some((a) => a.id === openRequest.autoId);
|
| 450 |
+
if (found) {
|
| 451 |
+
select(openRequest.autoId, openRequest.stageId || "");
|
| 452 |
+
}
|
| 453 |
+
setOpenRequest(null);
|
| 454 |
+
}, [openRequest, data, select]);
|
| 455 |
+
|
| 456 |
+
const items = data?.automations || [];
|
| 457 |
+
const active = items.find((a) => a.id === activeId) || null;
|
| 458 |
+
|
| 459 |
+
/**
|
| 460 |
+
* ⭐⭐ WAVE 32 · T43 (owner item 2) — TURNING AN AUTOMATION ON OR OFF STOPS TAKING FOREVER.
|
| 461 |
+
*
|
| 462 |
+
* Owner: turning an automation on or off "takes forever". ⛔ THE TICKET'S HYPOTHESIS WAS THE
|
| 463 |
+
* SERVER AND IT IS NOT — MEASURED. `engine.patch` for a schedule-only body takes ONE
|
| 464 |
+
* `automations` bucket read, ZERO `user_tables` reads and ~1 ms against a 2.8 MB fixture, and
|
| 465 |
+
* `GET /automations` is one read too since W30-T12. The cost was never the document shape
|
| 466 |
+
* (D-179/D-175's family): it was that this handler paid **two sequential round trips** for one
|
| 467 |
+
* click, and painted nothing until BOTH had landed. The second one re-downloaded the entire
|
| 468 |
+
* rail payload — 88,771 B measured, see the note at the top of this file — to learn one boolean
|
| 469 |
+
* the first response had already returned.
|
| 470 |
+
*
|
| 471 |
+
* ⛔ AND THE FIX ALREADY EXISTED IN THIS FILE, POINTED AT A DIFFERENT BUTTON. `mergeSaved` is
|
| 472 |
+
* W31-T31, built for the detail pane's saves, with its own note about the write response being
|
| 473 |
+
* newer than any read in flight. The rail's own switch never got it — a fix applied to one of
|
| 474 |
+
* two twins, which is this wave's recurring shape.
|
| 475 |
+
*
|
| 476 |
+
* Three states, in order, and the middle one is what the owner actually asked for:
|
| 477 |
+
* 1. FLIP IT NOW. The switch is the user's instruction, not a question; painting it after a
|
| 478 |
+
* network round trip is what "takes forever" describes even when the round trip is fast.
|
| 479 |
+
* 2. adopt the server's own answer when it lands, which carries the derived fields an
|
| 480 |
+
* optimistic row cannot invent (`nextRunAt`, the status line).
|
| 481 |
+
* 3. put the OLD row back if the write failed, beside the error — an optimistic switch that
|
| 482 |
+
* stays on after a refusal is a lie, and a worse one than a slow switch.
|
| 483 |
+
*
|
| 484 |
+
* ⚠ `mergeSaved` TAKES THE GENERATION, which is what stops [[refetch-eats-its-own-write]] here:
|
| 485 |
+
* a `GET /automations` issued before the click can still resolve after it, and without the bump
|
| 486 |
+
* it would paint the pre-toggle row back over both the optimistic flip and the confirmation.
|
| 487 |
+
* ⚠ `busy` is no longer what makes the switch look right — it stays only to keep a second click
|
| 488 |
+
* from racing the first.
|
| 489 |
+
*/
|
| 490 |
+
const toggleEnabled = async (a: Automation) => {
|
| 491 |
+
const flipped: Automation = {
|
| 492 |
+
...a,
|
| 493 |
+
schedule: { ...a.schedule, enabled: !a.schedule.enabled },
|
| 494 |
+
};
|
| 495 |
+
setBusy(a.id);
|
| 496 |
+
const painted = mergeSaved(flipped);
|
| 497 |
+
try {
|
| 498 |
+
const { automation: fresh } = await patchAutomation(a.id, {
|
| 499 |
+
schedule: { cron: a.schedule.cron, enabled: !a.schedule.enabled },
|
| 500 |
+
});
|
| 501 |
+
// A cold surface has no memo to merge into (`mergeSaved` answers false), and a one-row
|
| 502 |
+
// repaint would be worse than a round trip — the same rule `onSaved` follows.
|
| 503 |
+
if (!fresh || !mergeSaved(fresh)) await load();
|
| 504 |
+
} catch (e) {
|
| 505 |
+
if (painted) mergeSaved(a);
|
| 506 |
+
setError(e instanceof AutomationError ? e.message : "That change was not saved.");
|
| 507 |
+
} finally {
|
| 508 |
+
setBusy("");
|
| 509 |
+
}
|
| 510 |
+
};
|
| 511 |
+
|
| 512 |
+
const onRailKey = (event: ReactKeyboardEvent<HTMLDivElement>) => {
|
| 513 |
+
if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return;
|
| 514 |
+
const rows = Array.from(
|
| 515 |
+
listRef.current?.querySelectorAll<HTMLButtonElement>(".auto-row-main") || []
|
| 516 |
+
);
|
| 517 |
+
if (!rows.length) return;
|
| 518 |
+
event.preventDefault();
|
| 519 |
+
const at = rows.indexOf(document.activeElement as HTMLButtonElement);
|
| 520 |
+
const next = event.key === "ArrowDown" ? (at + 1) % rows.length
|
| 521 |
+
: (at - 1 + rows.length) % rows.length;
|
| 522 |
+
rows[next < 0 ? 0 : next]?.focus();
|
| 523 |
+
};
|
| 524 |
+
|
| 525 |
+
return (
|
| 526 |
+
<div className="auto-surface">
|
| 527 |
+
<aside
|
| 528 |
+
className={"auto-rail" + (railShut ? " is-collapsed" : "")}
|
| 529 |
+
aria-label="Automations"
|
| 530 |
+
>
|
| 531 |
+
{/* The same three-bars fold control the Views rail carries — the two rails are
|
| 532 |
+
one idea, so they must not fold with two different affordances. */}
|
| 533 |
+
<div className="auto-rail-top">
|
| 534 |
+
<button
|
| 535 |
+
type="button"
|
| 536 |
+
className="cg-rail-toggle"
|
| 537 |
+
aria-label={railShut ? "Expand automations" : "Minimize automations"}
|
| 538 |
+
aria-expanded={!railShut}
|
| 539 |
+
title={railShut ? "Expand automations" : "Minimize automations"}
|
| 540 |
+
onClick={() => setRailShut((v) => !v)}
|
| 541 |
+
>
|
| 542 |
+
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
| 543 |
+
<path
|
| 544 |
+
d="M2.5 4.4h11M2.5 8h11M2.5 11.6h11"
|
| 545 |
+
stroke="currentColor"
|
| 546 |
+
strokeWidth="1.35"
|
| 547 |
+
strokeLinecap="round"
|
| 548 |
+
/>
|
| 549 |
+
</svg>
|
| 550 |
+
</button>
|
| 551 |
+
</div>
|
| 552 |
+
|
| 553 |
+
<div className="auto-create">
|
| 554 |
+
<button
|
| 555 |
+
type="button"
|
| 556 |
+
className="auto-create-btn"
|
| 557 |
+
disabled={creatingNow}
|
| 558 |
+
onClick={() => void createAndOpen()}
|
| 559 |
+
>
|
| 560 |
+
<svg width="14" height="14" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
| 561 |
+
<path
|
| 562 |
+
d="M8 3.4v9.2M3.4 8h9.2"
|
| 563 |
+
stroke="currentColor"
|
| 564 |
+
strokeWidth="1.5"
|
| 565 |
+
strokeLinecap="round"
|
| 566 |
+
/>
|
| 567 |
+
</svg>
|
| 568 |
+
New automation
|
| 569 |
+
</button>
|
| 570 |
+
</div>
|
| 571 |
+
|
| 572 |
+
<div className="auto-list" ref={listRef} onKeyDown={onRailKey}>
|
| 573 |
+
{items.map((a) => {
|
| 574 |
+
return (
|
| 575 |
+
<div
|
| 576 |
+
key={a.id}
|
| 577 |
+
className={"auto-row" + (a.id === activeId ? " is-active" : "")}
|
| 578 |
+
>
|
| 579 |
+
<button
|
| 580 |
+
type="button"
|
| 581 |
+
className="auto-row-main"
|
| 582 |
+
onClick={() => select(a.id)}
|
| 583 |
+
>
|
| 584 |
+
{/* ⭐ ITEM 18 / R14 — THE LOOP MARK, WHERE THE STATUS DOT WAS. Same mark on
|
| 585 |
+
every row, no status colour: the owner's point is that an automation IS a
|
| 586 |
+
loop, not that this one is green. The `.auto-loopmark` wrapper is what sizes
|
| 587 |
+
it — `.lp-mark` is shell-owned CSS and outside this session's region, so the
|
| 588 |
+
box is mine and the mark is theirs. */}
|
| 589 |
+
<span className="auto-loopmark">
|
| 590 |
+
<Mark size={15} />
|
| 591 |
+
</span>
|
| 592 |
+
<span className="auto-row-text">
|
| 593 |
+
<span className="auto-row-name">{a.name}</span>
|
| 594 |
+
<span className="auto-row-desc">{railSubtitle(a)}</span>
|
| 595 |
+
</span>
|
| 596 |
+
</button>
|
| 597 |
+
<button
|
| 598 |
+
type="button"
|
| 599 |
+
className={"auto-row-switch" + (a.schedule?.enabled ? " is-on" : "")}
|
| 600 |
+
disabled={busy === a.id}
|
| 601 |
+
aria-pressed={!!a.schedule?.enabled}
|
| 602 |
+
title={
|
| 603 |
+
a.schedule?.enabled
|
| 604 |
+
? `Scheduled: ${a.schedule.cron}. Click to pause.`
|
| 605 |
+
: "Not scheduled. Click to enable."
|
| 606 |
+
}
|
| 607 |
+
onClick={() => void toggleEnabled(a)}
|
| 608 |
+
>
|
| 609 |
+
<span className="auto-row-switch-knob" />
|
| 610 |
+
</button>
|
| 611 |
+
</div>
|
| 612 |
+
);
|
| 613 |
+
})}
|
| 614 |
+
{/* ONE LINE (R13). It used to describe the kinds — "one reads a public web page…
|
| 615 |
+
the other fills an automation column…" — which was two sentences, wrong by the
|
| 616 |
+
time a third kind shipped, and printed inside a `--lp-rail-w` column. What the
|
| 617 |
+
kinds are belongs to the create form, which is one click away and lists all of
|
| 618 |
+
them from the server. */}
|
| 619 |
+
{/* ⭐ WAVE 29 (W29-T01) — A LOADING RAIL IS NOT AN EMPTY ONE. `data` is null until the
|
| 620 |
+
first `listAutomations()` resolves, and this branch read only `items.length`, so
|
| 621 |
+
every visit to Automation asserted "No automations yet." to a person who has
|
| 622 |
+
several — then replaced it with their list. Half of owner item 5's "takes a while
|
| 623 |
+
to appear" is that sentence: the wait is real, but being told you own nothing is
|
| 624 |
+
what makes it read as broken rather than slow. `data === null` is the one state
|
| 625 |
+
that means NOBODY HAS LOOKED, and it gets skeleton rows, not a claim. */}
|
| 626 |
+
{data === null && !error ? (
|
| 627 |
+
<div className="auto-rail-loading" aria-hidden="true">
|
| 628 |
+
<span className="auto-rail-skel" />
|
| 629 |
+
<span className="auto-rail-skel" />
|
| 630 |
+
<span className="auto-rail-skel" />
|
| 631 |
+
</div>
|
| 632 |
+
) : null}
|
| 633 |
+
{data !== null && !items.length && !error ? (
|
| 634 |
+
<p className="auto-rail-empty">No automations yet.</p>
|
| 635 |
+
) : null}
|
| 636 |
+
</div>
|
| 637 |
+
</aside>
|
| 638 |
+
|
| 639 |
+
<section className="auto-main">
|
| 640 |
+
{error ? (
|
| 641 |
+
<div className="auto-banner is-error" role="alert">
|
| 642 |
+
{error}
|
| 643 |
+
</div>
|
| 644 |
+
) : null}
|
| 645 |
+
{data && !data.storeAvailable ? (
|
| 646 |
+
<div className="auto-banner is-warn" role="status">
|
| 647 |
+
The tenant store is unavailable, so nothing can be saved right now.
|
| 648 |
+
</div>
|
| 649 |
+
) : null}
|
| 650 |
+
|
| 651 |
+
{/*
|
| 652 |
+
⛔ THE `creating` BRANCH IS GONE (wave 24, C-CREATE a / owner items 3 + 4).
|
| 653 |
+
`AutomationCreate` used to render here — a three-step wizard whose first step asked
|
| 654 |
+
which DATABASE to write into (an existing one, a new one, or one the automation itself
|
| 655 |
+
would create) and whose second asked which of three KINDS.
|
| 656 |
+
R6 retires that question at the root: two of the three kinds can no longer be created
|
| 657 |
+
at all, and the third is now reached by picking a trigger. A new automation is created
|
| 658 |
+
the moment it is asked for (`createAndOpen`) and opens on its own Builder, so there is
|
| 659 |
+
no intermediate face left to render and no `creating` state to hold.
|
| 660 |
+
⭐ WAVE 25 (R8) retires the LAST of that vocabulary: the third answer was the
|
| 661 |
+
"automated database", and it is gone from every door in the product. This button is now
|
| 662 |
+
one of exactly two ways to make an automation, and both are on this surface.
|
| 663 |
+
The import went with it — it is what would break `npx tsc -b` for the WHOLE client the
|
| 664 |
+
moment session B deletes the file, which is why this deletion is sequenced first.
|
| 665 |
+
*/}
|
| 666 |
+
{/* T21: the editor's own Suspense boundary — see the `lazy()` above for why it cannot be
|
| 667 |
+
`Shell.tsx`'s. `fallback={null}`, matching `Lazily`. */}
|
| 668 |
+
{active ? (
|
| 669 |
+
<Suspense fallback={null}>
|
| 670 |
+
<AutomationDetail
|
| 671 |
+
key={active.id}
|
| 672 |
+
automation={active}
|
| 673 |
+
/* ⛔ `kinds={data?.kinds || []}` LEFT HERE WITH THE PROP IT FED (wave 25, D-57) —
|
| 674 |
+
both halves in one change, because either alone is a `tsc` error. */
|
| 675 |
+
cronPresets={data?.cronPresets || []}
|
| 676 |
+
paidReady={!!data?.paidReady}
|
| 677 |
+
discover={data?.discover}
|
| 678 |
+
// The trigger vocabulary (C3). Forwarded as-is — absent stays absent, so the
|
| 679 |
+
// trigger face can tell "the server offered nothing" from "the server offered
|
| 680 |
+
// an empty list" rather than collapsing both into a picker with no options.
|
| 681 |
+
triggers={data?.triggers}
|
| 682 |
+
// C4's action menu + the builder's ceilings. Forwarded as-is for the same reason
|
| 683 |
+
// `triggers` is: absent must stay absent, so the builder can tell "the server
|
| 684 |
+
// offered nothing" from "the server offered an empty list".
|
| 685 |
+
catalog={data?.actionsCatalog}
|
| 686 |
+
vocab={data?.flow}
|
| 687 |
+
// `?? null` and never `|| {enabled:false}`: an absent tick bit is "the server did
|
| 688 |
+
// not say", which Step 1 prints as its own sentence. Defaulting it here would
|
| 689 |
+
// turn a missing field into a claim about production (C6 amendment #1).
|
| 690 |
+
tick={data?.tick ?? null}
|
| 691 |
+
/*
|
| 692 |
+
* ⭐ W24-W1 (item 6) — WHAT THIS RUN IS DOING, on the one surface that survives in
|
| 693 |
+
* BOTH views. REQUIRED on the far side deliberately: an optional prop that nobody
|
| 694 |
+
* passes degrades to "the feature does not exist", which is indistinguishable from
|
| 695 |
+
* "it was never built" — and this whole item exists because a live step text rode
|
| 696 |
+
* the wire for three waves with nothing rendering it.
|
| 697 |
+
* `liveStepOf` returns the step ONLY while the automation is running; a stale step
|
| 698 |
+
* from a finished run is a worse answer than none.
|
| 699 |
+
*/
|
| 700 |
+
liveStep={liveStepOf(active)}
|
| 701 |
+
/*
|
| 702 |
+
* ⛔ C14 LEG 2 — THE STALE-ID WRITE-BACK, and this is the ghost's second cause.
|
| 703 |
+
*
|
| 704 |
+
* The detail calls `onSaved(automation.id)` from six places (save, the trigger
|
| 705 |
+
* picker, a schedule change, a node switch, a card move, a run). Each closure
|
| 706 |
+
* captures the automation it was mounted for, so a save that resolves AFTER the
|
| 707 |
+
* user has clicked a different row used to call `setActiveId(the OLD id)` — the
|
| 708 |
+
* rail jumped back, the detail remounted, and what the user saw was the previous
|
| 709 |
+
* automation reappearing over the one they had just opened. It looked like a
|
| 710 |
+
* rendering bug; it was a resolution steering the selection.
|
| 711 |
+
*
|
| 712 |
+
* A resolution may no longer steer anything. It reloads the list — that part was
|
| 713 |
+
* always right — and it re-asserts the selection ONLY when its id is still the one
|
| 714 |
+
* the user asked for, which makes the write a no-op in the good case and nothing
|
| 715 |
+
* at all in the bad one.
|
| 716 |
+
*/
|
| 717 |
+
onSaved={async (id, fresh) => {
|
| 718 |
+
// ⭐⭐ WAVE 31 · T31 — CORRECT THE ONE ROW WE WERE JUST TOLD ABOUT.
|
| 719 |
+
// `patchAutomation` and `toggleNode` return the updated definition; re-reading the
|
| 720 |
+
// whole list to learn it is the second of the two round trips "Saving…" used to
|
| 721 |
+
// span. `mergeSaved` keeps the memo and the component in step (see
|
| 722 |
+
// `automationApi.rememberAutomation`) and takes the generation, because a write
|
| 723 |
+
// response is newer than any read still in flight — the same rule `load` follows,
|
| 724 |
+
// pointed the other way.
|
| 725 |
+
if (fresh && mergeSaved(fresh)) {
|
| 726 |
+
if (id && id === wantedId.current) select(id);
|
| 727 |
+
return;
|
| 728 |
+
}
|
| 729 |
+
const next = await load();
|
| 730 |
+
if (id && next && id === wantedId.current) select(id);
|
| 731 |
+
}}
|
| 732 |
+
onDeleted={async () => {
|
| 733 |
+
select("");
|
| 734 |
+
await load();
|
| 735 |
+
}}
|
| 736 |
+
/>
|
| 737 |
+
</Suspense>
|
| 738 |
+
) : (
|
| 739 |
+
/*
|
| 740 |
+
* THE EMPTY STATE IS ONE LINE AND A BUTTON (owner ruling R13 — "never
|
| 741 |
+
* over-explain", now a DESIGN.md law).
|
| 742 |
+
*
|
| 743 |
+
* It was a heading, a three-sentence paragraph and a 130-word bulleted list
|
| 744 |
+
* describing all three kinds. Every word of it was true and none of it was READ:
|
| 745 |
+
* an empty state is passed through, not studied, and the person looking at it has
|
| 746 |
+
* already decided to make an automation. The kinds are described where the choice
|
| 747 |
+
* is actually made — the create form lists them FROM THE SERVER, so that copy also
|
| 748 |
+
* cannot go stale the way this list had (it described two kinds after a third
|
| 749 |
+
* shipped).
|
| 750 |
+
*
|
| 751 |
+
* The button is here rather than only in the rail because this pane is where the
|
| 752 |
+
* eye is; a create affordance the user has to go find is the same defect as the
|
| 753 |
+
* paragraph, spent differently.
|
| 754 |
+
*/
|
| 755 |
+
<div className="autob-empty">
|
| 756 |
+
{/*
|
| 757 |
+
⛔ NO TITLE HERE ANY MORE (C13, owner item 2). This pane carried an `h1`
|
| 758 |
+
reading "Automations" at 20px/600 — a THIRD title treatment on a page that
|
| 759 |
+
also had the header's editable 16px/700 input, against every database page's
|
| 760 |
+
single 16px/650 `shell-db-name`. The shell now wraps this branch in the same
|
| 761 |
+
`shell-db-frame` + `DbHead` a database gets (wiring W23-W1), so the page's name
|
| 762 |
+
is drawn once, by the one component that draws every other page's name. A
|
| 763 |
+
stand-in restyled to match would have been a second copy of the same fact,
|
| 764 |
+
free to drift the day the header moves.
|
| 765 |
+
*/}
|
| 766 |
+
<p className="autob-empty-line">
|
| 767 |
+
A job this workspace runs for you, on demand or on a schedule.
|
| 768 |
+
</p>
|
| 769 |
+
<button
|
| 770 |
+
type="button"
|
| 771 |
+
className="auto-btn is-primary"
|
| 772 |
+
disabled={creatingNow}
|
| 773 |
+
onClick={() => void createAndOpen()}
|
| 774 |
+
>
|
| 775 |
+
New automation
|
| 776 |
+
</button>
|
| 777 |
+
</div>
|
| 778 |
+
)}
|
| 779 |
+
</section>
|
| 780 |
+
</div>
|
| 781 |
+
);
|
| 782 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/automation/automationApi.ts
CHANGED
|
@@ -518,17 +518,6 @@ export interface FlowVocab {
|
|
| 518 |
/** The ceiling `clean_post_groups` judges a group's limit against (the action's own maxPosts
|
| 519 |
* is the real bound; this is the absolute one, so a hint can be honest about both). */
|
| 520 |
maxPostsPerPull?: number;
|
| 521 |
-
/**
|
| 522 |
-
* ⭐⭐ W33-T52 (owner item 16) — THE CONFIG KEYS EACH ACTION KIND REQUIRES, straight off the
|
| 523 |
-
* server's `ACTION_REQUIRED`. `phrase` is the human wording the runner's own refusal sentence is
|
| 524 |
-
* built from ("a column to write into"); `key` is the config key it names.
|
| 525 |
-
* ⛔ The panel reads THIS to decide which controls carry a required mark. A client-side copy of
|
| 526 |
-
* the same table would agree on the day it was written and diverge the first time a kind gained
|
| 527 |
-
* a key — leaving the one surface that exists to say "this step is not finished" saying it is.
|
| 528 |
-
* ⚠ OPTIONAL, like every other vocabulary here: absent means a server older than this client,
|
| 529 |
-
* and the panel marks nothing rather than inventing requirements the save door does not have.
|
| 530 |
-
*/
|
| 531 |
-
actionRequired?: Record<string, { phrase: string; key: string }[]>;
|
| 532 |
/** Legacy read compatibility only; the server no longer sends Board review vocabulary. */
|
| 533 |
reviewDeciders?: string[];
|
| 534 |
/** Legacy read compatibility only; the server no longer sends Board review readiness. */
|
|
|
|
| 518 |
/** The ceiling `clean_post_groups` judges a group's limit against (the action's own maxPosts
|
| 519 |
* is the real bound; this is the absolute one, so a hint can be honest about both). */
|
| 520 |
maxPostsPerPull?: number;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 521 |
/** Legacy read compatibility only; the server no longer sends Board review vocabulary. */
|
| 522 |
reviewDeciders?: string[];
|
| 523 |
/** Legacy read compatibility only; the server no longer sends Board review readiness. */
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -622,20 +622,6 @@ export function ConnectorCard({
|
|
| 622 |
{connected ? "Manage keys" : "Add keys"}
|
| 623 |
</button>
|
| 624 |
) : null}
|
| 625 |
-
{/* ⭐⭐ WAVE 33 · D-204 — THE MEMBER'S OWN DOOR. Same room, different promise: this
|
| 626 |
-
card connects an account that is YOURS, which is why the word is not "Manage
|
| 627 |
-
keys". It appears only for connectors whose credential may be personal — the
|
| 628 |
-
server decides that from `TENANT_WIDE_TYPES`, so a member never sees it on Odoo
|
| 629 |
-
or Meta Ads, where storing a personal key is refused.
|
| 630 |
-
⚠ `connected` on the row is per-reader (`_entry_types` reads THIS user's
|
| 631 |
-
entries), so a member whose colleague connected Stripe business-wide still reads
|
| 632 |
-
the card as connected and is offered nothing to fix — which is correct: there is
|
| 633 |
-
nothing for them to do. */}
|
| 634 |
-
{!planned && row.manage === "keychain_personal" ? (
|
| 635 |
-
<button type="button" className="conn-card-go" onClick={onKeychain}>
|
| 636 |
-
{connected ? "Your connection" : "Connect your own"}
|
| 637 |
-
</button>
|
| 638 |
-
) : null}
|
| 639 |
{!planned && row.manage === "automation" ? (
|
| 640 |
<a className="conn-card-go" href="#/automation">
|
| 641 |
Open automations
|
|
|
|
| 622 |
{connected ? "Manage keys" : "Add keys"}
|
| 623 |
</button>
|
| 624 |
) : null}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
{!planned && row.manage === "automation" ? (
|
| 626 |
<a className="conn-card-go" href="#/automation">
|
| 627 |
Open automations
|
web/src/connectors/brandMarks.tsx
CHANGED
|
@@ -1,411 +1,338 @@
|
|
| 1 |
-
// ---------------------------------------------------------------------------
|
| 2 |
-
// connectors/brandMarks.tsx — WAVE 30, owner item 7 (rulings R3 + R4):
|
| 3 |
-
// THE ONE PLACE A CONNECTOR'S IDENTITY IS DRAWN.
|
| 4 |
-
//
|
| 5 |
-
// The owner, verbatim and for the third time: *"their Instagram and Tiktok logo
|
| 6 |
-
// to be fucking displayed, same with all logos under the Connectors module."*
|
| 7 |
-
//
|
| 8 |
-
// ⛔ THIS SUPERSEDES A RULE THAT WAS STILL SHIPPING. Wave-23 R8 said connector
|
| 9 |
-
// identity is a two-letter monogram in the chip family, and it was enforced —
|
| 10 |
-
// `verify_home.py:section_connectors` asserted a real mark could not appear.
|
| 11 |
-
// R4 replaces it FOR CONNECTOR IDENTITY ONLY. Everything else the design
|
| 12 |
-
// constitution says is untouched: no emojis anywhere, and no decorative
|
| 13 |
-
// third-party imagery outside this file's purpose, which is telling a person
|
| 14 |
-
// which company a row is about.
|
| 15 |
-
//
|
| 16 |
-
// ⛔
|
| 17 |
-
//
|
| 18 |
-
//
|
| 19 |
-
//
|
| 20 |
-
//
|
| 21 |
-
//
|
| 22 |
-
//
|
| 23 |
-
//
|
| 24 |
-
//
|
| 25 |
-
//
|
| 26 |
-
//
|
| 27 |
-
//
|
| 28 |
-
//
|
| 29 |
-
//
|
| 30 |
-
//
|
| 31 |
-
//
|
| 32 |
-
//
|
| 33 |
-
//
|
| 34 |
-
//
|
| 35 |
-
//
|
| 36 |
-
//
|
| 37 |
-
//
|
| 38 |
-
//
|
| 39 |
-
//
|
| 40 |
-
//
|
| 41 |
-
//
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
/
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
/
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
*
|
| 73 |
-
*
|
| 74 |
-
*
|
| 75 |
-
*
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
*
|
| 92 |
-
*
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
*
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
/>
|
| 260 |
-
<
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
<path d="
|
| 289 |
-
<path d="
|
| 290 |
-
<path d="
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
if (name === "shopify")
|
| 340 |
-
return (
|
| 341 |
-
<svg {...f}>
|
| 342 |
-
{/* The bag, in Shopify's two greens: the darker one is the handle, which
|
| 343 |
-
is the part that reads as a bag rather than as a box. */}
|
| 344 |
-
<path d="M3.5 5.2h9l-.75 8.4a.9.9 0 0 1-.9.8H5.15a.9.9 0 0 1-.9-.8z" fill="#95BF47" />
|
| 345 |
-
<path
|
| 346 |
-
d="M6.1 6.4V4.5a1.9 1.9 0 0 1 3.8 0v1.9"
|
| 347 |
-
fill="none" stroke="#5E8E3E" strokeWidth="1.4" strokeLinecap="round"
|
| 348 |
-
/>
|
| 349 |
-
</svg>
|
| 350 |
-
);
|
| 351 |
-
/*
|
| 352 |
-
⚠ THE LAST TWO ARE OURS, NOT BRANDS — `currentColor`, house stroke weight,
|
| 353 |
-
so they sit in the chip family beside the coloured logos rather than
|
| 354 |
-
pretending to be a company. Keeping them here is what lets every caller ask
|
| 355 |
-
one question instead of two.
|
| 356 |
-
*/
|
| 357 |
-
if (name === "webhooks")
|
| 358 |
-
return (
|
| 359 |
-
<svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
|
| 360 |
-
strokeLinecap="round" strokeLinejoin="round">
|
| 361 |
-
<path d="M6.6 9.4 4.9 11a2.6 2.6 0 1 1-1.5-4.4" />
|
| 362 |
-
<path d="M9.4 6.6 11.1 5a2.6 2.6 0 1 1 1.5 4.4" />
|
| 363 |
-
<path d="M6.2 8h3.6" />
|
| 364 |
-
</svg>
|
| 365 |
-
);
|
| 366 |
-
if (name === "scraper")
|
| 367 |
-
return (
|
| 368 |
-
<svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
|
| 369 |
-
strokeLinecap="round" strokeLinejoin="round">
|
| 370 |
-
{/* A magnifier over a grid of rows: it reads a catalogue of profiles and
|
| 371 |
-
makes records out of what matches — which is what the bucket does,
|
| 372 |
-
and it is deliberately NOT either platform's mark, because the whole
|
| 373 |
-
point of R3 is that the bucket is neither one of them. */}
|
| 374 |
-
<path d="M2.6 3.4h10.8M2.6 6.2h5.2M2.6 9h3.4" />
|
| 375 |
-
<circle cx="10.4" cy="10.2" r="3" />
|
| 376 |
-
<path d="m12.6 12.4 1.6 1.6" />
|
| 377 |
-
</svg>
|
| 378 |
-
);
|
| 379 |
-
return null;
|
| 380 |
-
}
|
| 381 |
-
|
| 382 |
-
/**
|
| 383 |
-
* Does this kind wear a LOGO rather than a house glyph?
|
| 384 |
-
*
|
| 385 |
-
* ⛔ IT EXISTS BECAUSE THE TILE HAS TO CHANGE TOO, and forgetting that would have shipped the
|
| 386 |
-
* defect quietly: `.autox-card-mark` paints a blue tint and sets `color: var(--lp-blue-deep)`,
|
| 387 |
-
* so TikTok's note — which is `currentColor` by design, so it survives dark mode — would have
|
| 388 |
-
* rendered BLUE, and Instagram's gradient would have sat on a coloured chip. The caller adds
|
| 389 |
-
* `is-brand`, which neutralises both. One predicate, so the mark and its tile cannot disagree
|
| 390 |
-
* about whether a row is branded.
|
| 391 |
-
*/
|
| 392 |
-
export function hasBrandKind(kind: string): boolean {
|
| 393 |
-
return Boolean(BRAND_OF_KIND[kind]);
|
| 394 |
-
}
|
| 395 |
-
|
| 396 |
-
/** The connector-key twin of `hasBrandKind`, for the directory's tiles. */
|
| 397 |
-
export function hasBrandConnector(key: string): boolean {
|
| 398 |
-
return Boolean(BRAND_OF_CONNECTOR[key]);
|
| 399 |
-
}
|
| 400 |
-
|
| 401 |
-
/** The mark for a trigger or action KIND, or null when it is not a branded one. */
|
| 402 |
-
export function brandForKind(kind: string, size = 15): ReactElement | null {
|
| 403 |
-
const name = BRAND_OF_KIND[kind];
|
| 404 |
-
return name ? <BrandMark name={name} size={size} /> : null;
|
| 405 |
-
}
|
| 406 |
-
|
| 407 |
-
/** The mark for a connector/provider KEY, or null when we have no logo for it. */
|
| 408 |
-
export function brandForConnector(key: string, size = 15): ReactElement | null {
|
| 409 |
-
const name = BRAND_OF_CONNECTOR[key];
|
| 410 |
-
return name ? <BrandMark name={name} size={size} /> : null;
|
| 411 |
-
}
|
|
|
|
| 1 |
+
// ---------------------------------------------------------------------------
|
| 2 |
+
// connectors/brandMarks.tsx — WAVE 30, owner item 7 (rulings R3 + R4):
|
| 3 |
+
// THE ONE PLACE A CONNECTOR'S IDENTITY IS DRAWN.
|
| 4 |
+
//
|
| 5 |
+
// The owner, verbatim and for the third time: *"their Instagram and Tiktok logo
|
| 6 |
+
// to be fucking displayed, same with all logos under the Connectors module."*
|
| 7 |
+
//
|
| 8 |
+
// ⛔ THIS SUPERSEDES A RULE THAT WAS STILL SHIPPING. Wave-23 R8 said connector
|
| 9 |
+
// identity is a two-letter monogram in the chip family, and it was enforced —
|
| 10 |
+
// `verify_home.py:section_connectors` asserted a real mark could not appear.
|
| 11 |
+
// R4 replaces it FOR CONNECTOR IDENTITY ONLY. Everything else the design
|
| 12 |
+
// constitution says is untouched: no emojis anywhere, and no decorative
|
| 13 |
+
// third-party imagery outside this file's purpose, which is telling a person
|
| 14 |
+
// which company a row is about.
|
| 15 |
+
//
|
| 16 |
+
// ⛔ INLINE SVG, NEVER A FILE AND NEVER A REMOTE URL. There is no `assets/` or
|
| 17 |
+
// `icons/` directory in this app and there is not going to be one: every icon
|
| 18 |
+
// in the product is a React component, `index.css` inlines even its font as a
|
| 19 |
+
// `data:` URI, and `customer-grid/types.ts` carries the standing sentence
|
| 20 |
+
// *"Never an external URL: those are neither same-origin nor CSP-safe here."*
|
| 21 |
+
// A remote logo would work today (no CSP header is set) and would be the one
|
| 22 |
+
// thing on the page that can be taken away by somebody else's outage.
|
| 23 |
+
//
|
| 24 |
+
// ⛔ ONE MODULE FOR THREE SURFACES, and that is the whole design. The trigger
|
| 25 |
+
// picker, the action menu and the Connectors directory all answer "which
|
| 26 |
+
// company is this?" — three copies of a logo set is three chances to add a
|
| 27 |
+
// platform in two places and forget the third, which is precisely the defect
|
| 28 |
+
// wave 30 exists to fix on the server side of the same question.
|
| 29 |
+
//
|
| 30 |
+
// ⚠ WHAT THESE ARE, HONESTLY: recognisable SIMPLIFICATIONS drawn from each
|
| 31 |
+
// brand's published silhouette and brand colour, at 16px, not the trademarked
|
| 32 |
+
// artwork. That is the right call at this size — the real Google `G` is a
|
| 33 |
+
// 500-byte path that renders as mush in a 15px box — and it is stated here so
|
| 34 |
+
// nobody "upgrades" them by pasting a vendor's asset file into a repo that has
|
| 35 |
+
// no place to put one.
|
| 36 |
+
//
|
| 37 |
+
// ⚠ THEME. Marks are brand-coloured and must survive BOTH themes, so no mark
|
| 38 |
+
// relies on pure black or pure white for its silhouette: TikTok's note is
|
| 39 |
+
// `currentColor` with its two brand offsets behind it, which is also how the
|
| 40 |
+
// real mark is constructed.
|
| 41 |
+
// ---------------------------------------------------------------------------
|
| 42 |
+
import type { ReactElement } from "react";
|
| 43 |
+
import { useId } from "react";
|
| 44 |
+
|
| 45 |
+
/**
|
| 46 |
+
* The canonical brand names this module can draw. Deliberately NOT the same
|
| 47 |
+
* strings as the connector keys or the trigger kinds: three vocabularies point
|
| 48 |
+
* AT this one, and folding them together would make a rename in any of the
|
| 49 |
+
* three a silent loss of an icon.
|
| 50 |
+
*/
|
| 51 |
+
export type BrandName =
|
| 52 |
+
| "instagram"
|
| 53 |
+
| "tiktok"
|
| 54 |
+
| "odoo"
|
| 55 |
+
| "google"
|
| 56 |
+
| "gmail"
|
| 57 |
+
| "stripe"
|
| 58 |
+
| "shopify"
|
| 59 |
+
| "webhooks"
|
| 60 |
+
| "scraper"
|
| 61 |
+
| "slack"
|
| 62 |
+
| "meta";
|
| 63 |
+
|
| 64 |
+
/**
|
| 65 |
+
* Trigger and action KIND -> brand.
|
| 66 |
+
*
|
| 67 |
+
* ⛔ KEYED ON THE KIND AND NOT ON THE CONNECTOR, and the reason is R3 itself:
|
| 68 |
+
* after this wave Instagram and TikTok share `connector.key === "scraper"`, so
|
| 69 |
+
* the connector can no longer tell them apart. The kind is the only thing that
|
| 70 |
+
* still distinguishes the two products, which is exactly why the picker's own
|
| 71 |
+
* `TriggerMark` has always been keyed this way.
|
| 72 |
+
*
|
| 73 |
+
* ⚠ An unmapped kind returns null and the caller draws its existing glyph — a
|
| 74 |
+
* trigger the engine adds tomorrow renders plainly rather than not at all.
|
| 75 |
+
*/
|
| 76 |
+
export const BRAND_OF_KIND: Record<string, BrandName> = {
|
| 77 |
+
ig_profile_match: "instagram",
|
| 78 |
+
tiktok_profile_match: "tiktok",
|
| 79 |
+
enrich_instagram: "instagram",
|
| 80 |
+
enrich_tiktok: "tiktok",
|
| 81 |
+
field_instagram: "instagram",
|
| 82 |
+
};
|
| 83 |
+
|
| 84 |
+
/**
|
| 85 |
+
* Connector/provider KEY -> brand, for the Connectors directory and for the
|
| 86 |
+
* collapsed connector nests in both automation menus.
|
| 87 |
+
*
|
| 88 |
+
* ⚠ `scraper` and `webhooks` are OURS, not brands — they get house glyphs in
|
| 89 |
+
* `currentColor` so they sit in the chip family the way they always have. They
|
| 90 |
+
* are in this map rather than special-cased at the call site so that "what mark
|
| 91 |
+
* does this row wear" has ONE answer, whoever is asking.
|
| 92 |
+
*/
|
| 93 |
+
export const BRAND_OF_CONNECTOR: Record<string, BrandName> = {
|
| 94 |
+
odoo: "odoo",
|
| 95 |
+
// ⛔⛔ WAVE 32 · OWNER ITEM 12 / R6 — THESE TWO MARKS ARE CURRENTLY UNREACHABLE, AND KEPT.
|
| 96 |
+
// `oauth_connect.offered("google")` is False, so the directory serves no `google` row and this
|
| 97 |
+
// map is never asked for either of these. They stay for the reason the ruling is worded as
|
| 98 |
+
// *"HIDDEN until they are real"* rather than "removed": the blocker is money, not code (D-45 —
|
| 99 |
+
// Google's CASA verification, ~$540–1,800/yr, for the `gmail.readonly` scope). The day that is
|
| 100 |
+
// paid for, ONE key flips server-side and the card comes back wearing its own logo. Deleting
|
| 101 |
+
// the art would turn a one-key restore into a re-draw, and would leave the next reader
|
| 102 |
+
// wondering where Gmail's mark went as well as where Gmail went.
|
| 103 |
+
google: "google",
|
| 104 |
+
gmail: "gmail",
|
| 105 |
+
stripe: "stripe",
|
| 106 |
+
shopify: "shopify",
|
| 107 |
+
scraper: "scraper",
|
| 108 |
+
webhooks: "webhooks",
|
| 109 |
+
instagram: "instagram",
|
| 110 |
+
tiktok: "tiktok",
|
| 111 |
+
// ⭐ WAVE 31 · R7 — the Nango breadth play's first two marks. `meta_ads` is the KEYCHAIN
|
| 112 |
+
// connector (a token an admin pastes today); `meta` would be the delegated-OAuth door the same
|
| 113 |
+
// company owns, so BOTH keys point at one mark rather than the map growing a near-duplicate.
|
| 114 |
+
slack: "slack",
|
| 115 |
+
meta: "meta",
|
| 116 |
+
meta_ads: "meta",
|
| 117 |
+
};
|
| 118 |
+
|
| 119 |
+
/** The house frame every mark is drawn in. 16×16 user units, sized by the caller. */
|
| 120 |
+
function frame(size: number) {
|
| 121 |
+
return { width: size, height: size, viewBox: "0 0 16 16", "aria-hidden": true } as const;
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
/**
|
| 125 |
+
* ⚠ TIKTOK'S NOTE, DRAWN THREE TIMES. The real mark is one glyph offset in
|
| 126 |
+
* cyan and magenta behind a solid copy; drawing only the solid copy loses the
|
| 127 |
+
* brand entirely at this size, and drawing the offsets in black loses it in
|
| 128 |
+
* dark mode. The solid copy is `currentColor` so it inherits the row.
|
| 129 |
+
*/
|
| 130 |
+
const TT_NOTE =
|
| 131 |
+
"M9.05 1.9h2.02c.13 1.2.87 2.13 2.03 2.3v2.03c-.78 0-1.53-.2-2.18-.58v3.98"
|
| 132 |
+
+ "a3.62 3.62 0 1 1-3.62-3.62c.16 0 .32.01.48.03v2.06a1.6 1.6 0 1 0 1.27 1.56z";
|
| 133 |
+
|
| 134 |
+
export function BrandMark({
|
| 135 |
+
name,
|
| 136 |
+
size = 15,
|
| 137 |
+
}: {
|
| 138 |
+
name: BrandName;
|
| 139 |
+
size?: number;
|
| 140 |
+
}): ReactElement | null {
|
| 141 |
+
const f = frame(size);
|
| 142 |
+
// Per-instance, for the reason spelled out at the `<defs>` below. Hooks run for every
|
| 143 |
+
// name, not only Instagram, because a hook cannot sit behind a conditional return.
|
| 144 |
+
const gradId = `aios-brand-ig-${useId()}`;
|
| 145 |
+
if (name === "instagram")
|
| 146 |
+
return (
|
| 147 |
+
<svg {...f} fill="none">
|
| 148 |
+
{/*
|
| 149 |
+
⛔⛔ THE GRADIENT ID IS PER-INSTANCE, AND THE FIXED ONE WAS A REAL BUG — FOUND BY A
|
| 150 |
+
PICTURE, NOT BY A GATE. It shipped for an hour as the constant `aios-brand-ig`, on the
|
| 151 |
+
reasoning that duplicate ids resolve to the FIRST definition and every definition is
|
| 152 |
+
byte-identical, so it cannot matter. MEASURED in the connectors shot: the Scraper card
|
| 153 |
+
holds an Instagram mark inside a COLLAPSED `<details>`, a second card below holds a
|
| 154 |
+
visible one — and the visible one painted NOTHING. A closed `<details>` is
|
| 155 |
+
`content-visibility: hidden`, its subtree is not rendered, and a paint server defined
|
| 156 |
+
inside a skipped subtree does not resolve for a user outside it. "The first definition
|
| 157 |
+
wins" IS the failure: whichever copy happens to be hidden decides for all of them.
|
| 158 |
+
⚠ Nothing about `<details>` is special — `display:none` does the same. Any page holding
|
| 159 |
+
two of this mark with the first one hidden loses the second.
|
| 160 |
+
⚠ `useId` rather than a module counter, so the markup stays DETERMINISTIC for a given
|
| 161 |
+
tree — the render gates could not assert against a random id — and stable across a
|
| 162 |
+
server render and its hydration.
|
| 163 |
+
⚠ Every check that looks for this mark matches the PREFIX `url(#aios-brand-ig`.
|
| 164 |
+
*/}
|
| 165 |
+
<defs>
|
| 166 |
+
{/* ⚠ `userSpaceOnUse` IS LOAD-BEARING AND ITS ABSENCE LOOKED FINE. The default is
|
| 167 |
+
`objectBoundingBox`, where the coordinates are FRACTIONS of the shape's box — so
|
| 168 |
+
`1,15 → 15,1` put the whole gradient vector far outside every path and each one
|
| 169 |
+
painted a single clamped stop. The mark still rendered, still read as Instagram,
|
| 170 |
+
and was simply flat pink: a defect only a picture catches, which is why one was
|
| 171 |
+
taken ([[ui-invisible-to-assertions]]). In user space the numbers mean viewBox
|
| 172 |
+
units, which is what they were written as. */}
|
| 173 |
+
<linearGradient
|
| 174 |
+
id={gradId} gradientUnits="userSpaceOnUse"
|
| 175 |
+
x1="2" y1="14" x2="14" y2="2"
|
| 176 |
+
>
|
| 177 |
+
<stop offset="0" stopColor="#FEDA75" />
|
| 178 |
+
<stop offset="0.32" stopColor="#FA7E1E" />
|
| 179 |
+
<stop offset="0.62" stopColor="#D62976" />
|
| 180 |
+
<stop offset="1" stopColor="#962FBF" />
|
| 181 |
+
</linearGradient>
|
| 182 |
+
</defs>
|
| 183 |
+
<rect
|
| 184 |
+
x="1.7" y="1.7" width="12.6" height="12.6" rx="3.7"
|
| 185 |
+
stroke={`url(#${gradId})`} strokeWidth="1.5"
|
| 186 |
+
/>
|
| 187 |
+
<circle cx="8" cy="8" r="3" stroke={`url(#${gradId})`} strokeWidth="1.5" />
|
| 188 |
+
<circle cx="11.6" cy="4.4" r="0.95" fill={`url(#${gradId})`} />
|
| 189 |
+
</svg>
|
| 190 |
+
);
|
| 191 |
+
if (name === "tiktok")
|
| 192 |
+
return (
|
| 193 |
+
<svg {...f}>
|
| 194 |
+
<path d={TT_NOTE} fill="#25F4EE" transform="translate(-0.9 -0.5)" />
|
| 195 |
+
<path d={TT_NOTE} fill="#FE2C55" transform="translate(0.9 0.5)" />
|
| 196 |
+
<path d={TT_NOTE} fill="currentColor" />
|
| 197 |
+
</svg>
|
| 198 |
+
);
|
| 199 |
+
if (name === "odoo")
|
| 200 |
+
return (
|
| 201 |
+
<svg {...f}>
|
| 202 |
+
{/* Odoo's brand purple, and its wordmark's twin `o`s — the part of that
|
| 203 |
+
wordmark that survives being shrunk to 15px. */}
|
| 204 |
+
<rect x="1" y="1" width="14" height="14" rx="3.4" fill="#714B67" />
|
| 205 |
+
<circle cx="5.7" cy="8.2" r="2.1" fill="none" stroke="#FFFFFF" strokeWidth="1.5" />
|
| 206 |
+
<circle cx="10.4" cy="8.2" r="2.1" fill="none" stroke="#FFFFFF" strokeWidth="1.5" />
|
| 207 |
+
</svg>
|
| 208 |
+
);
|
| 209 |
+
if (name === "google")
|
| 210 |
+
return (
|
| 211 |
+
<svg {...f} fill="none" strokeWidth="2.3">
|
| 212 |
+
{/* The four-arc G. Each arc is one brand colour, and the bar into the
|
| 213 |
+
centre is what makes it a G rather than a ring. */}
|
| 214 |
+
<path d="M4.46 4.46A5 5 0 0 1 11.54 4.46" stroke="#EA4335" strokeLinecap="round" />
|
| 215 |
+
<path d="M4.46 4.46A5 5 0 0 0 4.46 11.54" stroke="#FBBC05" strokeLinecap="round" />
|
| 216 |
+
<path d="M4.46 11.54A5 5 0 0 0 11.54 11.54" stroke="#34A853" strokeLinecap="round" />
|
| 217 |
+
<path d="M11.54 11.54A5 5 0 0 0 13 8" stroke="#4285F4" strokeLinecap="round" />
|
| 218 |
+
<path d="M13 8H8.4" stroke="#4285F4" />
|
| 219 |
+
</svg>
|
| 220 |
+
);
|
| 221 |
+
if (name === "gmail")
|
| 222 |
+
return (
|
| 223 |
+
<svg {...f} fill="none" stroke="#EA4335" strokeWidth="1.5" strokeLinejoin="round">
|
| 224 |
+
<rect x="1.6" y="3.6" width="12.8" height="9" rx="1.6" />
|
| 225 |
+
<path d="m2 4.4 6 4.6 6-4.6" />
|
| 226 |
+
</svg>
|
| 227 |
+
);
|
| 228 |
+
// ⭐ WAVE 31 · R7. Slack's mark is FOUR bars in four brand colours, drawn as rounded rects
|
| 229 |
+
// rather than the true interlocking glyph: at 15px the real lozenges collapse into mud, and a
|
| 230 |
+
// mark nobody can identify is worse than a two-letter tile. Colours are Slack's own.
|
| 231 |
+
if (name === "slack")
|
| 232 |
+
return (
|
| 233 |
+
<svg {...f}>
|
| 234 |
+
<rect x="2" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#36C5F0" />
|
| 235 |
+
<rect x="8.6" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#2EB67D" />
|
| 236 |
+
<rect x="6.6" y="2" width="2.8" height="5.4" rx="1.4" fill="#ECB22E" />
|
| 237 |
+
<rect x="6.6" y="8.6" width="2.8" height="5.4" rx="1.4" fill="#E01E5A" />
|
| 238 |
+
</svg>
|
| 239 |
+
);
|
| 240 |
+
// Meta's infinity loop, as two overlapping strokes in the brand blue. ⚠ `fill="none"` with a
|
| 241 |
+
// round cap: filled, the two lobes merge into a blob at this size.
|
| 242 |
+
if (name === "meta")
|
| 243 |
+
return (
|
| 244 |
+
<svg {...f} fill="none">
|
| 245 |
+
<path
|
| 246 |
+
d="M2.2 10.4c0-3 1.5-5.2 3.3-5.2 1.2 0 2 .8 3 2.4l1 1.6c.9 1.4 1.5 2.1 2.2 2.1.8 0
|
| 247 |
+
1.3-.8 1.3-2.1 0-1.6-.7-3.4-2-3.4-.9 0-1.7.7-2.6 2"
|
| 248 |
+
stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
|
| 249 |
+
/>
|
| 250 |
+
<path
|
| 251 |
+
d="M2.2 10.4c0 1.3.6 2 1.5 2 .8 0 1.4-.5 2.5-2.2l3-4.6"
|
| 252 |
+
stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
|
| 253 |
+
/>
|
| 254 |
+
</svg>
|
| 255 |
+
);
|
| 256 |
+
if (name === "stripe")
|
| 257 |
+
return (
|
| 258 |
+
<svg {...f}>
|
| 259 |
+
<rect x="1" y="1" width="14" height="14" rx="3.4" fill="#635BFF" />
|
| 260 |
+
<path
|
| 261 |
+
d="M10.5 5.6a2.3 2.3 0 0 0-3.6 1.2c0 2.1 4.1 1.5 4.1 3.5a2.4 2.4 0 0 1-3.7 1.3"
|
| 262 |
+
fill="none" stroke="#FFFFFF" strokeWidth="1.5" strokeLinecap="round"
|
| 263 |
+
/>
|
| 264 |
+
</svg>
|
| 265 |
+
);
|
| 266 |
+
if (name === "shopify")
|
| 267 |
+
return (
|
| 268 |
+
<svg {...f}>
|
| 269 |
+
{/* The bag, in Shopify's two greens: the darker one is the handle, which
|
| 270 |
+
is the part that reads as a bag rather than as a box. */}
|
| 271 |
+
<path d="M3.5 5.2h9l-.75 8.4a.9.9 0 0 1-.9.8H5.15a.9.9 0 0 1-.9-.8z" fill="#95BF47" />
|
| 272 |
+
<path
|
| 273 |
+
d="M6.1 6.4V4.5a1.9 1.9 0 0 1 3.8 0v1.9"
|
| 274 |
+
fill="none" stroke="#5E8E3E" strokeWidth="1.4" strokeLinecap="round"
|
| 275 |
+
/>
|
| 276 |
+
</svg>
|
| 277 |
+
);
|
| 278 |
+
/*
|
| 279 |
+
⚠ THE LAST TWO ARE OURS, NOT BRANDS — `currentColor`, house stroke weight,
|
| 280 |
+
so they sit in the chip family beside the coloured logos rather than
|
| 281 |
+
pretending to be a company. Keeping them here is what lets every caller ask
|
| 282 |
+
one question instead of two.
|
| 283 |
+
*/
|
| 284 |
+
if (name === "webhooks")
|
| 285 |
+
return (
|
| 286 |
+
<svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
|
| 287 |
+
strokeLinecap="round" strokeLinejoin="round">
|
| 288 |
+
<path d="M6.6 9.4 4.9 11a2.6 2.6 0 1 1-1.5-4.4" />
|
| 289 |
+
<path d="M9.4 6.6 11.1 5a2.6 2.6 0 1 1 1.5 4.4" />
|
| 290 |
+
<path d="M6.2 8h3.6" />
|
| 291 |
+
</svg>
|
| 292 |
+
);
|
| 293 |
+
if (name === "scraper")
|
| 294 |
+
return (
|
| 295 |
+
<svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
|
| 296 |
+
strokeLinecap="round" strokeLinejoin="round">
|
| 297 |
+
{/* A magnifier over a grid of rows: it reads a catalogue of profiles and
|
| 298 |
+
makes records out of what matches — which is what the bucket does,
|
| 299 |
+
and it is deliberately NOT either platform's mark, because the whole
|
| 300 |
+
point of R3 is that the bucket is neither one of them. */}
|
| 301 |
+
<path d="M2.6 3.4h10.8M2.6 6.2h5.2M2.6 9h3.4" />
|
| 302 |
+
<circle cx="10.4" cy="10.2" r="3" />
|
| 303 |
+
<path d="m12.6 12.4 1.6 1.6" />
|
| 304 |
+
</svg>
|
| 305 |
+
);
|
| 306 |
+
return null;
|
| 307 |
+
}
|
| 308 |
+
|
| 309 |
+
/**
|
| 310 |
+
* Does this kind wear a LOGO rather than a house glyph?
|
| 311 |
+
*
|
| 312 |
+
* ⛔ IT EXISTS BECAUSE THE TILE HAS TO CHANGE TOO, and forgetting that would have shipped the
|
| 313 |
+
* defect quietly: `.autox-card-mark` paints a blue tint and sets `color: var(--lp-blue-deep)`,
|
| 314 |
+
* so TikTok's note — which is `currentColor` by design, so it survives dark mode — would have
|
| 315 |
+
* rendered BLUE, and Instagram's gradient would have sat on a coloured chip. The caller adds
|
| 316 |
+
* `is-brand`, which neutralises both. One predicate, so the mark and its tile cannot disagree
|
| 317 |
+
* about whether a row is branded.
|
| 318 |
+
*/
|
| 319 |
+
export function hasBrandKind(kind: string): boolean {
|
| 320 |
+
return Boolean(BRAND_OF_KIND[kind]);
|
| 321 |
+
}
|
| 322 |
+
|
| 323 |
+
/** The connector-key twin of `hasBrandKind`, for the directory's tiles. */
|
| 324 |
+
export function hasBrandConnector(key: string): boolean {
|
| 325 |
+
return Boolean(BRAND_OF_CONNECTOR[key]);
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
/** The mark for a trigger or action KIND, or null when it is not a branded one. */
|
| 329 |
+
export function brandForKind(kind: string, size = 15): ReactElement | null {
|
| 330 |
+
const name = BRAND_OF_KIND[kind];
|
| 331 |
+
return name ? <BrandMark name={name} size={size} /> : null;
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
+
/** The mark for a connector/provider KEY, or null when we have no logo for it. */
|
| 335 |
+
export function brandForConnector(key: string, size = 15): ReactElement | null {
|
| 336 |
+
const name = BRAND_OF_CONNECTOR[key];
|
| 337 |
+
return name ? <BrandMark name={name} size={size} /> : null;
|
| 338 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/customer-grid/ColumnMenu.tsx
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
web/src/customer-grid/CustomerGrid.tsx
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
web/src/customer-grid/ImportDialog.css
CHANGED
|
@@ -1,154 +1,154 @@
|
|
| 1 |
-
/* customer-grid / ImportDialog.css — WAVE-29 T25 (owner item 6).
|
| 2 |
-
Tokens only (`--lp-*`): this file may not invent a colour, and `index.css` is another
|
| 3 |
-
session's fence, so the dialog's own styles live beside its component. */
|
| 4 |
-
|
| 5 |
-
.cg-import-scrim {
|
| 6 |
-
position: fixed;
|
| 7 |
-
inset: 0;
|
| 8 |
-
z-index: 60;
|
| 9 |
-
display: flex;
|
| 10 |
-
align-items: center;
|
| 11 |
-
justify-content: center;
|
| 12 |
-
background: rgba(32, 36, 51, 0.28);
|
| 13 |
-
padding: 24px;
|
| 14 |
-
}
|
| 15 |
-
|
| 16 |
-
.cg-import {
|
| 17 |
-
display: flex;
|
| 18 |
-
flex-direction: column;
|
| 19 |
-
width: min(560px, 100%);
|
| 20 |
-
max-height: min(80vh, 720px);
|
| 21 |
-
background: var(--lp-bg);
|
| 22 |
-
border: 1px solid var(--lp-line);
|
| 23 |
-
border-radius: var(--lp-r-lg);
|
| 24 |
-
box-shadow: 0 12px 32px rgba(32, 36, 51, 0.18);
|
| 25 |
-
overflow: hidden;
|
| 26 |
-
}
|
| 27 |
-
|
| 28 |
-
.cg-import-head {
|
| 29 |
-
display: flex;
|
| 30 |
-
align-items: center;
|
| 31 |
-
justify-content: space-between;
|
| 32 |
-
gap: 12px;
|
| 33 |
-
padding: 14px 16px;
|
| 34 |
-
border-bottom: 1px solid var(--lp-line);
|
| 35 |
-
}
|
| 36 |
-
|
| 37 |
-
.cg-import-head h2 {
|
| 38 |
-
margin: 0;
|
| 39 |
-
font-size: var(--lp-fs-md);
|
| 40 |
-
font-weight: 600;
|
| 41 |
-
color: var(--lp-ink);
|
| 42 |
-
}
|
| 43 |
-
|
| 44 |
-
.cg-import-x {
|
| 45 |
-
border: 0;
|
| 46 |
-
background: none;
|
| 47 |
-
color: var(--lp-muted);
|
| 48 |
-
font-size: var(--lp-fs-lg);
|
| 49 |
-
line-height: 1;
|
| 50 |
-
cursor: pointer;
|
| 51 |
-
padding: 2px 6px;
|
| 52 |
-
}
|
| 53 |
-
|
| 54 |
-
.cg-import-x:hover {
|
| 55 |
-
color: var(--lp-ink);
|
| 56 |
-
}
|
| 57 |
-
|
| 58 |
-
.cg-import-body {
|
| 59 |
-
padding: 14px 16px;
|
| 60 |
-
overflow: auto;
|
| 61 |
-
}
|
| 62 |
-
|
| 63 |
-
.cg-import-lede {
|
| 64 |
-
margin: 0 0 8px;
|
| 65 |
-
font-size: var(--lp-fs-sm);
|
| 66 |
-
color: var(--lp-ink);
|
| 67 |
-
}
|
| 68 |
-
|
| 69 |
-
.cg-import-note {
|
| 70 |
-
margin: 0 0 12px;
|
| 71 |
-
font-size: var(--lp-fs-xs);
|
| 72 |
-
color: var(--lp-muted);
|
| 73 |
-
}
|
| 74 |
-
|
| 75 |
-
.cg-import-file {
|
| 76 |
-
font-size: var(--lp-fs-sm);
|
| 77 |
-
}
|
| 78 |
-
|
| 79 |
-
/* The mapping list: the spreadsheet's column on the left, this database's field on the right,
|
| 80 |
-
so the eye reads it as "this becomes that" rather than as a form. */
|
| 81 |
-
.cg-import-map {
|
| 82 |
-
display: flex;
|
| 83 |
-
flex-direction: column;
|
| 84 |
-
gap: 6px;
|
| 85 |
-
}
|
| 86 |
-
|
| 87 |
-
.cg-import-row {
|
| 88 |
-
display: grid;
|
| 89 |
-
grid-template-columns: 1fr 200px;
|
| 90 |
-
align-items: center;
|
| 91 |
-
gap: 10px;
|
| 92 |
-
}
|
| 93 |
-
|
| 94 |
-
.cg-import-col {
|
| 95 |
-
font-size: var(--lp-fs-sm);
|
| 96 |
-
color: var(--lp-ink);
|
| 97 |
-
overflow: hidden;
|
| 98 |
-
text-overflow: ellipsis;
|
| 99 |
-
white-space: nowrap;
|
| 100 |
-
}
|
| 101 |
-
|
| 102 |
-
.cg-import-row select {
|
| 103 |
-
font-family: inherit;
|
| 104 |
-
font-size: var(--lp-fs-xs);
|
| 105 |
-
padding: 4px 6px;
|
| 106 |
-
border: 1px solid var(--lp-line);
|
| 107 |
-
border-radius: var(--lp-r-sm);
|
| 108 |
-
background: var(--lp-bg);
|
| 109 |
-
color: var(--lp-ink);
|
| 110 |
-
}
|
| 111 |
-
|
| 112 |
-
/* A refusal is not an error state to be styled red-alert; it is a list of things to fix, so it
|
| 113 |
-
takes the yellow tint the rest of the product uses for "look at this before continuing". */
|
| 114 |
-
.cg-import-problems {
|
| 115 |
-
margin-top: 12px;
|
| 116 |
-
padding: 10px 12px;
|
| 117 |
-
border: 1px solid var(--lp-yellow);
|
| 118 |
-
border-radius: var(--lp-r-sm);
|
| 119 |
-
background: var(--lp-yellow-tint);
|
| 120 |
-
color: var(--lp-yellow-deep);
|
| 121 |
-
font-size: var(--lp-fs-xs);
|
| 122 |
-
}
|
| 123 |
-
|
| 124 |
-
.cg-import-problems ul {
|
| 125 |
-
margin: 6px 0;
|
| 126 |
-
padding-left: 18px;
|
| 127 |
-
}
|
| 128 |
-
|
| 129 |
-
.cg-import-problems p {
|
| 130 |
-
margin: 6px 0 0;
|
| 131 |
-
}
|
| 132 |
-
|
| 133 |
-
.cg-import-error {
|
| 134 |
-
margin: 0;
|
| 135 |
-
padding: 10px 16px;
|
| 136 |
-
background: var(--lp-red-tint);
|
| 137 |
-
color: var(--lp-red-deep);
|
| 138 |
-
font-size: var(--lp-fs-xs);
|
| 139 |
-
}
|
| 140 |
-
|
| 141 |
-
.cg-import-foot {
|
| 142 |
-
display: flex;
|
| 143 |
-
align-items: center;
|
| 144 |
-
justify-content: flex-end;
|
| 145 |
-
gap: 8px;
|
| 146 |
-
padding: 12px 16px;
|
| 147 |
-
border-top: 1px solid var(--lp-line);
|
| 148 |
-
}
|
| 149 |
-
|
| 150 |
-
.cg-import-count {
|
| 151 |
-
margin-right: auto;
|
| 152 |
-
font-size: var(--lp-fs-xs);
|
| 153 |
-
color: var(--lp-muted);
|
| 154 |
-
}
|
|
|
|
| 1 |
+
/* customer-grid / ImportDialog.css — WAVE-29 T25 (owner item 6).
|
| 2 |
+
Tokens only (`--lp-*`): this file may not invent a colour, and `index.css` is another
|
| 3 |
+
session's fence, so the dialog's own styles live beside its component. */
|
| 4 |
+
|
| 5 |
+
.cg-import-scrim {
|
| 6 |
+
position: fixed;
|
| 7 |
+
inset: 0;
|
| 8 |
+
z-index: 60;
|
| 9 |
+
display: flex;
|
| 10 |
+
align-items: center;
|
| 11 |
+
justify-content: center;
|
| 12 |
+
background: rgba(32, 36, 51, 0.28);
|
| 13 |
+
padding: 24px;
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
.cg-import {
|
| 17 |
+
display: flex;
|
| 18 |
+
flex-direction: column;
|
| 19 |
+
width: min(560px, 100%);
|
| 20 |
+
max-height: min(80vh, 720px);
|
| 21 |
+
background: var(--lp-bg);
|
| 22 |
+
border: 1px solid var(--lp-line);
|
| 23 |
+
border-radius: var(--lp-r-lg);
|
| 24 |
+
box-shadow: 0 12px 32px rgba(32, 36, 51, 0.18);
|
| 25 |
+
overflow: hidden;
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
.cg-import-head {
|
| 29 |
+
display: flex;
|
| 30 |
+
align-items: center;
|
| 31 |
+
justify-content: space-between;
|
| 32 |
+
gap: 12px;
|
| 33 |
+
padding: 14px 16px;
|
| 34 |
+
border-bottom: 1px solid var(--lp-line);
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
.cg-import-head h2 {
|
| 38 |
+
margin: 0;
|
| 39 |
+
font-size: var(--lp-fs-md);
|
| 40 |
+
font-weight: 600;
|
| 41 |
+
color: var(--lp-ink);
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
.cg-import-x {
|
| 45 |
+
border: 0;
|
| 46 |
+
background: none;
|
| 47 |
+
color: var(--lp-muted);
|
| 48 |
+
font-size: var(--lp-fs-lg);
|
| 49 |
+
line-height: 1;
|
| 50 |
+
cursor: pointer;
|
| 51 |
+
padding: 2px 6px;
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
.cg-import-x:hover {
|
| 55 |
+
color: var(--lp-ink);
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
.cg-import-body {
|
| 59 |
+
padding: 14px 16px;
|
| 60 |
+
overflow: auto;
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
.cg-import-lede {
|
| 64 |
+
margin: 0 0 8px;
|
| 65 |
+
font-size: var(--lp-fs-sm);
|
| 66 |
+
color: var(--lp-ink);
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
.cg-import-note {
|
| 70 |
+
margin: 0 0 12px;
|
| 71 |
+
font-size: var(--lp-fs-xs);
|
| 72 |
+
color: var(--lp-muted);
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
.cg-import-file {
|
| 76 |
+
font-size: var(--lp-fs-sm);
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
/* The mapping list: the spreadsheet's column on the left, this database's field on the right,
|
| 80 |
+
so the eye reads it as "this becomes that" rather than as a form. */
|
| 81 |
+
.cg-import-map {
|
| 82 |
+
display: flex;
|
| 83 |
+
flex-direction: column;
|
| 84 |
+
gap: 6px;
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
.cg-import-row {
|
| 88 |
+
display: grid;
|
| 89 |
+
grid-template-columns: 1fr 200px;
|
| 90 |
+
align-items: center;
|
| 91 |
+
gap: 10px;
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
.cg-import-col {
|
| 95 |
+
font-size: var(--lp-fs-sm);
|
| 96 |
+
color: var(--lp-ink);
|
| 97 |
+
overflow: hidden;
|
| 98 |
+
text-overflow: ellipsis;
|
| 99 |
+
white-space: nowrap;
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
.cg-import-row select {
|
| 103 |
+
font-family: inherit;
|
| 104 |
+
font-size: var(--lp-fs-xs);
|
| 105 |
+
padding: 4px 6px;
|
| 106 |
+
border: 1px solid var(--lp-line);
|
| 107 |
+
border-radius: var(--lp-r-sm);
|
| 108 |
+
background: var(--lp-bg);
|
| 109 |
+
color: var(--lp-ink);
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
/* A refusal is not an error state to be styled red-alert; it is a list of things to fix, so it
|
| 113 |
+
takes the yellow tint the rest of the product uses for "look at this before continuing". */
|
| 114 |
+
.cg-import-problems {
|
| 115 |
+
margin-top: 12px;
|
| 116 |
+
padding: 10px 12px;
|
| 117 |
+
border: 1px solid var(--lp-yellow);
|
| 118 |
+
border-radius: var(--lp-r-sm);
|
| 119 |
+
background: var(--lp-yellow-tint);
|
| 120 |
+
color: var(--lp-yellow-deep);
|
| 121 |
+
font-size: var(--lp-fs-xs);
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
.cg-import-problems ul {
|
| 125 |
+
margin: 6px 0;
|
| 126 |
+
padding-left: 18px;
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
.cg-import-problems p {
|
| 130 |
+
margin: 6px 0 0;
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
.cg-import-error {
|
| 134 |
+
margin: 0;
|
| 135 |
+
padding: 10px 16px;
|
| 136 |
+
background: var(--lp-red-tint);
|
| 137 |
+
color: var(--lp-red-deep);
|
| 138 |
+
font-size: var(--lp-fs-xs);
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
.cg-import-foot {
|
| 142 |
+
display: flex;
|
| 143 |
+
align-items: center;
|
| 144 |
+
justify-content: flex-end;
|
| 145 |
+
gap: 8px;
|
| 146 |
+
padding: 12px 16px;
|
| 147 |
+
border-top: 1px solid var(--lp-line);
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
.cg-import-count {
|
| 151 |
+
margin-right: auto;
|
| 152 |
+
font-size: var(--lp-fs-xs);
|
| 153 |
+
color: var(--lp-muted);
|
| 154 |
+
}
|
web/src/customer-grid/ImportDialog.tsx
CHANGED
|
@@ -1,299 +1,299 @@
|
|
| 1 |
-
// ---------------------------------------------------------------------------
|
| 2 |
-
// customer-grid / ImportDialog.tsx
|
| 3 |
-
//
|
| 4 |
-
// ⭐⭐ WAVE-29 T25 (owner item 6) — IMPORT A SPREADSHEET INTO A USER DATABASE.
|
| 5 |
-
//
|
| 6 |
-
// Three steps in one window, because each answers the question the last one raised:
|
| 7 |
-
// 1. pick a file -> the server parses it and says which columns it found
|
| 8 |
-
// 2. map its columns -> to fields, or to nothing; unmapped is the default
|
| 9 |
-
// 3. check + import -> every value coerced FIRST, the problems named BEFORE anything writes
|
| 10 |
-
//
|
| 11 |
-
// ⛔ APPEND-ONLY IN V1, AND THE DIALOG SAYS SO. Nothing is matched, nothing is overwritten — every
|
| 12 |
-
// row becomes a new record. The only bulk-merge primitive in the product is upsert-BY-KEY shaped,
|
| 13 |
-
// and choosing a key on the user's behalf is how an import quietly rewrites rows somebody edited.
|
| 14 |
-
// The sentence is on screen rather than in a doc, because that is where the decision is made.
|
| 15 |
-
//
|
| 16 |
-
// ⛔ REUSE, NOT A SECOND COERCER. `clipboard.coerceClipboardValue` is THE client answer to "can
|
| 17 |
-
// this text be this field's value" (it is what a paste already goes through), and
|
| 18 |
-
// `POST /api/v1/uploads/tabular` is the parser that already exists — 5 MB / 20,000 rows, xlsx AND
|
| 19 |
-
// csv, and it never writes a row. A second copy of either is a second set of rules for one
|
| 20 |
-
// question ([[one-evaluator-per-question]]).
|
| 21 |
-
//
|
| 22 |
-
// ⚠ THE SERVER IS THE WALL, not this dialog. It refuses computed columns BY NAME and refuses the
|
| 23 |
-
// whole batch if it would pass the row cap. What happens here is the part a person needs before
|
| 24 |
-
// they press the button: which column goes where, and what will not survive the trip.
|
| 25 |
-
// ---------------------------------------------------------------------------
|
| 26 |
-
|
| 27 |
-
import { useMemo, useRef, useState } from "react";
|
| 28 |
-
// ⚠ From `apiContract`, which is where these two are DECLARED — not re-exported from
|
| 29 |
-
// `apiBridge`, which does not export them and is another session's fence this wave.
|
| 30 |
-
import { API_V1, CREDENTIALS } from "../apiContract";
|
| 31 |
-
import { isImportTarget } from "./types";
|
| 32 |
-
import type { Field } from "./types";
|
| 33 |
-
// ⭐ W29-T76 — the plan lives in a pure module so a test can EXECUTE it against the server's real
|
| 34 |
-
// payload. This file holds the WINDOW; `importPlan.ts` holds what would be written.
|
| 35 |
-
import { buildImportPlan } from "./importPlan";
|
| 36 |
-
import type { ImportMapping, ParsedUpload } from "./importPlan";
|
| 37 |
-
import "./ImportDialog.css";
|
| 38 |
-
|
| 39 |
-
/** The one header a JSON POST needs. `apiBridge` keeps its own module-local copy; this is a
|
| 40 |
-
* literal rather than an import because that constant is not exported and that file is not
|
| 41 |
-
* this session's to change. */
|
| 42 |
-
const JSON_HEADERS = { "Content-Type": "application/json" };
|
| 43 |
-
|
| 44 |
-
/** The upload door's answer (`routes_uploads.tabular`) — declared in `importPlan.ts`, beside the
|
| 45 |
-
* only code that reads its shape. ⛔ It is COLUMN-MAJOR; the copy that used to live here said
|
| 46 |
-
* `string[][]` and that one wrong word is why nothing ever imported. */
|
| 47 |
-
type Parsed = ParsedUpload;
|
| 48 |
-
|
| 49 |
-
export interface ImportDialogProps {
|
| 50 |
-
/** The target database's fields — the mapping targets, before this file filters them. */
|
| 51 |
-
fields: Field[];
|
| 52 |
-
/** `ut_*` table key; the import door is per-table. */
|
| 53 |
-
scope: string;
|
| 54 |
-
/** How the database calls itself, for the sentence that says where rows are going. */
|
| 55 |
-
tableName: string;
|
| 56 |
-
/** Fired after a successful import so the host can re-read its rows. */
|
| 57 |
-
onImported: (count: number) => void;
|
| 58 |
-
onClose: () => void;
|
| 59 |
-
}
|
| 60 |
-
|
| 61 |
-
const NONE = "";
|
| 62 |
-
|
| 63 |
-
export default function ImportDialog({
|
| 64 |
-
fields,
|
| 65 |
-
scope,
|
| 66 |
-
tableName,
|
| 67 |
-
onImported,
|
| 68 |
-
onClose,
|
| 69 |
-
}: ImportDialogProps): React.ReactElement {
|
| 70 |
-
const [parsed, setParsed] = useState<Parsed | null>(null);
|
| 71 |
-
const [mapping, setMapping] = useState<ImportMapping>({});
|
| 72 |
-
const [busy, setBusy] = useState(false);
|
| 73 |
-
const [error, setError] = useState("");
|
| 74 |
-
const fileRef = useRef<HTMLInputElement | null>(null);
|
| 75 |
-
|
| 76 |
-
const targets = useMemo(() => fields.filter(isImportTarget), [fields]);
|
| 77 |
-
|
| 78 |
-
/**
|
| 79 |
-
* The parse. `FormData` and NO `Content-Type` header: the browser has to set the multipart
|
| 80 |
-
* boundary itself, and spelling the header by hand is the classic way to make a working upload
|
| 81 |
-
* 400 with a message about a missing part.
|
| 82 |
-
*/
|
| 83 |
-
async function pick(file: File): Promise<void> {
|
| 84 |
-
setBusy(true);
|
| 85 |
-
setError("");
|
| 86 |
-
try {
|
| 87 |
-
const form = new FormData();
|
| 88 |
-
form.append("file", file);
|
| 89 |
-
const res = await fetch(`${API_V1}/uploads/tabular`, {
|
| 90 |
-
method: "POST",
|
| 91 |
-
credentials: CREDENTIALS,
|
| 92 |
-
body: form,
|
| 93 |
-
});
|
| 94 |
-
const body = (await res.json().catch(() => null)) as
|
| 95 |
-
| (Parsed & { error?: { message?: string } })
|
| 96 |
-
| null;
|
| 97 |
-
if (!res.ok || !body) {
|
| 98 |
-
setError(
|
| 99 |
-
body?.error?.message ??
|
| 100 |
-
`That file could not be read (the server answered ${res.status}).`
|
| 101 |
-
);
|
| 102 |
-
return;
|
| 103 |
-
}
|
| 104 |
-
setParsed(body);
|
| 105 |
-
// ⭐ AUTO-MAP BY NAME, case- and space-insensitively, because the overwhelmingly common
|
| 106 |
-
// case is a sheet exported from this very grid. Every guess is VISIBLE in a picker the
|
| 107 |
-
// user can change — a silent auto-map is only dangerous when it is also invisible.
|
| 108 |
-
const byName = new Map(
|
| 109 |
-
targets.map((f) => [f.label.trim().toLowerCase(), f.key] as const)
|
| 110 |
-
);
|
| 111 |
-
const guess: ImportMapping = {};
|
| 112 |
-
(body.columns ?? []).forEach((name, i) => {
|
| 113 |
-
const hit = byName.get(String(name).trim().toLowerCase());
|
| 114 |
-
if (hit) guess[i] = hit;
|
| 115 |
-
});
|
| 116 |
-
setMapping(guess);
|
| 117 |
-
} catch {
|
| 118 |
-
setError("That file could not be read.");
|
| 119 |
-
} finally {
|
| 120 |
-
setBusy(false);
|
| 121 |
-
}
|
| 122 |
-
}
|
| 123 |
-
|
| 124 |
-
/**
|
| 125 |
-
* What would be written, and what cannot be. The work is `importPlan.buildImportPlan` — a pure
|
| 126 |
-
* module, so the node suite executes it against the server's real payload rather than grepping
|
| 127 |
-
* this file for the shape of a loop.
|
| 128 |
-
*/
|
| 129 |
-
const plan = useMemo(
|
| 130 |
-
() => buildImportPlan(parsed, mapping, targets),
|
| 131 |
-
[parsed, mapping, targets]
|
| 132 |
-
);
|
| 133 |
-
|
| 134 |
-
async function run(): Promise<void> {
|
| 135 |
-
setBusy(true);
|
| 136 |
-
setError("");
|
| 137 |
-
try {
|
| 138 |
-
const res = await fetch(
|
| 139 |
-
`${API_V1}/tables/${encodeURIComponent(scope)}/rows/import`,
|
| 140 |
-
{
|
| 141 |
-
method: "POST",
|
| 142 |
-
credentials: CREDENTIALS,
|
| 143 |
-
headers: JSON_HEADERS,
|
| 144 |
-
body: JSON.stringify({ rows: plan.rows }),
|
| 145 |
-
}
|
| 146 |
-
);
|
| 147 |
-
const body = (await res.json().catch(() => null)) as
|
| 148 |
-
| { imported?: number; error?: { message?: string } }
|
| 149 |
-
| null;
|
| 150 |
-
if (!res.ok) {
|
| 151 |
-
// The server's sentence beats anything invented here: it names the column, or the cap.
|
| 152 |
-
setError(body?.error?.message ?? `The server answered ${res.status}.`);
|
| 153 |
-
return;
|
| 154 |
-
}
|
| 155 |
-
onImported(body?.imported ?? plan.rows.length);
|
| 156 |
-
onClose();
|
| 157 |
-
} catch {
|
| 158 |
-
setError("Nothing was imported — the request did not complete.");
|
| 159 |
-
} finally {
|
| 160 |
-
setBusy(false);
|
| 161 |
-
}
|
| 162 |
-
}
|
| 163 |
-
|
| 164 |
-
const mappedCount = Object.values(mapping).filter((k) => k !== NONE).length;
|
| 165 |
-
|
| 166 |
-
return (
|
| 167 |
-
<div className="cg-import-scrim" role="presentation" onClick={onClose}>
|
| 168 |
-
<div
|
| 169 |
-
className="cg-import"
|
| 170 |
-
role="dialog"
|
| 171 |
-
aria-modal="true"
|
| 172 |
-
aria-label={`Import records into ${tableName}`}
|
| 173 |
-
onClick={(event) => event.stopPropagation()}
|
| 174 |
-
>
|
| 175 |
-
<div className="cg-import-head">
|
| 176 |
-
<h2>Import records</h2>
|
| 177 |
-
<button type="button" className="cg-import-x" aria-label="Close" onClick={onClose}>
|
| 178 |
-
×
|
| 179 |
-
</button>
|
| 180 |
-
</div>
|
| 181 |
-
|
| 182 |
-
{!parsed ? (
|
| 183 |
-
<div className="cg-import-body">
|
| 184 |
-
<p className="cg-import-lede">
|
| 185 |
-
Add records to <strong>{tableName}</strong> from a spreadsheet. Excel
|
| 186 |
-
(<code>.xlsx</code>) or <code>.csv</code>, up to 5 MB.
|
| 187 |
-
</p>
|
| 188 |
-
<p className="cg-import-note">
|
| 189 |
-
Every row becomes a <strong>new record</strong>. Nothing already in this database is
|
| 190 |
-
matched or overwritten.
|
| 191 |
-
</p>
|
| 192 |
-
<input
|
| 193 |
-
ref={fileRef}
|
| 194 |
-
type="file"
|
| 195 |
-
accept=".xlsx,.csv"
|
| 196 |
-
className="cg-import-file"
|
| 197 |
-
onChange={(event) => {
|
| 198 |
-
const file = event.currentTarget.files?.[0];
|
| 199 |
-
if (file) void pick(file);
|
| 200 |
-
}}
|
| 201 |
-
/>
|
| 202 |
-
</div>
|
| 203 |
-
) : (
|
| 204 |
-
<div className="cg-import-body">
|
| 205 |
-
<p className="cg-import-lede">
|
| 206 |
-
{parsed.rows.toLocaleString()} row{parsed.rows === 1 ? "" : "s"} read
|
| 207 |
-
{parsed.truncated ? " (the first 20,000 — the rest were not read)" : ""}. Choose
|
| 208 |
-
where each column goes.
|
| 209 |
-
</p>
|
| 210 |
-
<div className="cg-import-map" role="list">
|
| 211 |
-
{parsed.columns.map((name, i) => (
|
| 212 |
-
<label className="cg-import-row" role="listitem" key={`${name}:${i}`}>
|
| 213 |
-
<span className="cg-import-col" title={name}>
|
| 214 |
-
{name || `Column ${i + 1}`}
|
| 215 |
-
</span>
|
| 216 |
-
<select
|
| 217 |
-
value={mapping[i] ?? NONE}
|
| 218 |
-
onChange={(event) => {
|
| 219 |
-
const next = event.target.value;
|
| 220 |
-
setMapping((cur) => ({ ...cur, [i]: next }));
|
| 221 |
-
}}
|
| 222 |
-
>
|
| 223 |
-
<option value={NONE}>Don’t import</option>
|
| 224 |
-
{targets.map((f) => (
|
| 225 |
-
<option key={f.key} value={f.key}>
|
| 226 |
-
{f.label}
|
| 227 |
-
</option>
|
| 228 |
-
))}
|
| 229 |
-
</select>
|
| 230 |
-
</label>
|
| 231 |
-
))}
|
| 232 |
-
</div>
|
| 233 |
-
{plan.problems.length > 0 && (
|
| 234 |
-
<div className="cg-import-problems" role="alert">
|
| 235 |
-
<strong>
|
| 236 |
-
{plan.problems.length.toLocaleString()} value
|
| 237 |
-
{plan.problems.length === 1 ? "" : "s"} cannot go in the column
|
| 238 |
-
{plan.problems.length === 1 ? "" : "s"} chosen for them
|
| 239 |
-
</strong>
|
| 240 |
-
<ul>
|
| 241 |
-
{plan.problems.slice(0, 5).map((p, i) => (
|
| 242 |
-
<li key={i}>
|
| 243 |
-
Row {p.row}, {p.column}: {p.value ? `“${p.value}”` : "(blank)"} is not a
|
| 244 |
-
valid {p.field}
|
| 245 |
-
</li>
|
| 246 |
-
))}
|
| 247 |
-
</ul>
|
| 248 |
-
{plan.problems.length > 5 && (
|
| 249 |
-
<p>…and {(plan.problems.length - 5).toLocaleString()} more.</p>
|
| 250 |
-
)}
|
| 251 |
-
<p>
|
| 252 |
-
Fix them in the file, or point that column somewhere else — nothing has been
|
| 253 |
-
imported yet.
|
| 254 |
-
</p>
|
| 255 |
-
</div>
|
| 256 |
-
)}
|
| 257 |
-
</div>
|
| 258 |
-
)}
|
| 259 |
-
|
| 260 |
-
{error && (
|
| 261 |
-
<p className="cg-import-error" role="alert">
|
| 262 |
-
{error}
|
| 263 |
-
</p>
|
| 264 |
-
)}
|
| 265 |
-
|
| 266 |
-
<div className="cg-import-foot">
|
| 267 |
-
{parsed && (
|
| 268 |
-
<span className="cg-import-count">
|
| 269 |
-
{mappedCount === 0
|
| 270 |
-
? "No columns chosen yet"
|
| 271 |
-
: `${plan.rows.length.toLocaleString()} record${
|
| 272 |
-
plan.rows.length === 1 ? "" : "s"
|
| 273 |
-
} from ${mappedCount} column${mappedCount === 1 ? "" : "s"}`}
|
| 274 |
-
</span>
|
| 275 |
-
)}
|
| 276 |
-
<button type="button" className="cg-btn" onClick={onClose}>
|
| 277 |
-
Cancel
|
| 278 |
-
</button>
|
| 279 |
-
{parsed && (
|
| 280 |
-
<button
|
| 281 |
-
type="button"
|
| 282 |
-
className="cg-btn cg-btn--primary"
|
| 283 |
-
/* ⛔ DISABLED WHILE ANY VALUE IS REFUSED. The alternative — import the good rows and
|
| 284 |
-
report the rest — leaves somebody reconciling a spreadsheet against a table by
|
| 285 |
-
hand, and neither existing write door demonstrates a partial policy. Refusing
|
| 286 |
-
whole is recoverable in one edit; a half-import is not. */
|
| 287 |
-
disabled={busy || plan.rows.length === 0 || plan.problems.length > 0}
|
| 288 |
-
onClick={() => void run()}
|
| 289 |
-
>
|
| 290 |
-
{busy ? "Importing…" : `Import ${plan.rows.length.toLocaleString()} record${
|
| 291 |
-
plan.rows.length === 1 ? "" : "s"
|
| 292 |
-
}`}
|
| 293 |
-
</button>
|
| 294 |
-
)}
|
| 295 |
-
</div>
|
| 296 |
-
</div>
|
| 297 |
-
</div>
|
| 298 |
-
);
|
| 299 |
-
}
|
|
|
|
| 1 |
+
// ---------------------------------------------------------------------------
|
| 2 |
+
// customer-grid / ImportDialog.tsx
|
| 3 |
+
//
|
| 4 |
+
// ⭐⭐ WAVE-29 T25 (owner item 6) — IMPORT A SPREADSHEET INTO A USER DATABASE.
|
| 5 |
+
//
|
| 6 |
+
// Three steps in one window, because each answers the question the last one raised:
|
| 7 |
+
// 1. pick a file -> the server parses it and says which columns it found
|
| 8 |
+
// 2. map its columns -> to fields, or to nothing; unmapped is the default
|
| 9 |
+
// 3. check + import -> every value coerced FIRST, the problems named BEFORE anything writes
|
| 10 |
+
//
|
| 11 |
+
// ⛔ APPEND-ONLY IN V1, AND THE DIALOG SAYS SO. Nothing is matched, nothing is overwritten — every
|
| 12 |
+
// row becomes a new record. The only bulk-merge primitive in the product is upsert-BY-KEY shaped,
|
| 13 |
+
// and choosing a key on the user's behalf is how an import quietly rewrites rows somebody edited.
|
| 14 |
+
// The sentence is on screen rather than in a doc, because that is where the decision is made.
|
| 15 |
+
//
|
| 16 |
+
// ⛔ REUSE, NOT A SECOND COERCER. `clipboard.coerceClipboardValue` is THE client answer to "can
|
| 17 |
+
// this text be this field's value" (it is what a paste already goes through), and
|
| 18 |
+
// `POST /api/v1/uploads/tabular` is the parser that already exists — 5 MB / 20,000 rows, xlsx AND
|
| 19 |
+
// csv, and it never writes a row. A second copy of either is a second set of rules for one
|
| 20 |
+
// question ([[one-evaluator-per-question]]).
|
| 21 |
+
//
|
| 22 |
+
// ⚠ THE SERVER IS THE WALL, not this dialog. It refuses computed columns BY NAME and refuses the
|
| 23 |
+
// whole batch if it would pass the row cap. What happens here is the part a person needs before
|
| 24 |
+
// they press the button: which column goes where, and what will not survive the trip.
|
| 25 |
+
// ---------------------------------------------------------------------------
|
| 26 |
+
|
| 27 |
+
import { useMemo, useRef, useState } from "react";
|
| 28 |
+
// ⚠ From `apiContract`, which is where these two are DECLARED — not re-exported from
|
| 29 |
+
// `apiBridge`, which does not export them and is another session's fence this wave.
|
| 30 |
+
import { API_V1, CREDENTIALS } from "../apiContract";
|
| 31 |
+
import { isImportTarget } from "./types";
|
| 32 |
+
import type { Field } from "./types";
|
| 33 |
+
// ⭐ W29-T76 — the plan lives in a pure module so a test can EXECUTE it against the server's real
|
| 34 |
+
// payload. This file holds the WINDOW; `importPlan.ts` holds what would be written.
|
| 35 |
+
import { buildImportPlan } from "./importPlan";
|
| 36 |
+
import type { ImportMapping, ParsedUpload } from "./importPlan";
|
| 37 |
+
import "./ImportDialog.css";
|
| 38 |
+
|
| 39 |
+
/** The one header a JSON POST needs. `apiBridge` keeps its own module-local copy; this is a
|
| 40 |
+
* literal rather than an import because that constant is not exported and that file is not
|
| 41 |
+
* this session's to change. */
|
| 42 |
+
const JSON_HEADERS = { "Content-Type": "application/json" };
|
| 43 |
+
|
| 44 |
+
/** The upload door's answer (`routes_uploads.tabular`) — declared in `importPlan.ts`, beside the
|
| 45 |
+
* only code that reads its shape. ⛔ It is COLUMN-MAJOR; the copy that used to live here said
|
| 46 |
+
* `string[][]` and that one wrong word is why nothing ever imported. */
|
| 47 |
+
type Parsed = ParsedUpload;
|
| 48 |
+
|
| 49 |
+
export interface ImportDialogProps {
|
| 50 |
+
/** The target database's fields — the mapping targets, before this file filters them. */
|
| 51 |
+
fields: Field[];
|
| 52 |
+
/** `ut_*` table key; the import door is per-table. */
|
| 53 |
+
scope: string;
|
| 54 |
+
/** How the database calls itself, for the sentence that says where rows are going. */
|
| 55 |
+
tableName: string;
|
| 56 |
+
/** Fired after a successful import so the host can re-read its rows. */
|
| 57 |
+
onImported: (count: number) => void;
|
| 58 |
+
onClose: () => void;
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
const NONE = "";
|
| 62 |
+
|
| 63 |
+
export default function ImportDialog({
|
| 64 |
+
fields,
|
| 65 |
+
scope,
|
| 66 |
+
tableName,
|
| 67 |
+
onImported,
|
| 68 |
+
onClose,
|
| 69 |
+
}: ImportDialogProps): React.ReactElement {
|
| 70 |
+
const [parsed, setParsed] = useState<Parsed | null>(null);
|
| 71 |
+
const [mapping, setMapping] = useState<ImportMapping>({});
|
| 72 |
+
const [busy, setBusy] = useState(false);
|
| 73 |
+
const [error, setError] = useState("");
|
| 74 |
+
const fileRef = useRef<HTMLInputElement | null>(null);
|
| 75 |
+
|
| 76 |
+
const targets = useMemo(() => fields.filter(isImportTarget), [fields]);
|
| 77 |
+
|
| 78 |
+
/**
|
| 79 |
+
* The parse. `FormData` and NO `Content-Type` header: the browser has to set the multipart
|
| 80 |
+
* boundary itself, and spelling the header by hand is the classic way to make a working upload
|
| 81 |
+
* 400 with a message about a missing part.
|
| 82 |
+
*/
|
| 83 |
+
async function pick(file: File): Promise<void> {
|
| 84 |
+
setBusy(true);
|
| 85 |
+
setError("");
|
| 86 |
+
try {
|
| 87 |
+
const form = new FormData();
|
| 88 |
+
form.append("file", file);
|
| 89 |
+
const res = await fetch(`${API_V1}/uploads/tabular`, {
|
| 90 |
+
method: "POST",
|
| 91 |
+
credentials: CREDENTIALS,
|
| 92 |
+
body: form,
|
| 93 |
+
});
|
| 94 |
+
const body = (await res.json().catch(() => null)) as
|
| 95 |
+
| (Parsed & { error?: { message?: string } })
|
| 96 |
+
| null;
|
| 97 |
+
if (!res.ok || !body) {
|
| 98 |
+
setError(
|
| 99 |
+
body?.error?.message ??
|
| 100 |
+
`That file could not be read (the server answered ${res.status}).`
|
| 101 |
+
);
|
| 102 |
+
return;
|
| 103 |
+
}
|
| 104 |
+
setParsed(body);
|
| 105 |
+
// ⭐ AUTO-MAP BY NAME, case- and space-insensitively, because the overwhelmingly common
|
| 106 |
+
// case is a sheet exported from this very grid. Every guess is VISIBLE in a picker the
|
| 107 |
+
// user can change — a silent auto-map is only dangerous when it is also invisible.
|
| 108 |
+
const byName = new Map(
|
| 109 |
+
targets.map((f) => [f.label.trim().toLowerCase(), f.key] as const)
|
| 110 |
+
);
|
| 111 |
+
const guess: ImportMapping = {};
|
| 112 |
+
(body.columns ?? []).forEach((name, i) => {
|
| 113 |
+
const hit = byName.get(String(name).trim().toLowerCase());
|
| 114 |
+
if (hit) guess[i] = hit;
|
| 115 |
+
});
|
| 116 |
+
setMapping(guess);
|
| 117 |
+
} catch {
|
| 118 |
+
setError("That file could not be read.");
|
| 119 |
+
} finally {
|
| 120 |
+
setBusy(false);
|
| 121 |
+
}
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
/**
|
| 125 |
+
* What would be written, and what cannot be. The work is `importPlan.buildImportPlan` — a pure
|
| 126 |
+
* module, so the node suite executes it against the server's real payload rather than grepping
|
| 127 |
+
* this file for the shape of a loop.
|
| 128 |
+
*/
|
| 129 |
+
const plan = useMemo(
|
| 130 |
+
() => buildImportPlan(parsed, mapping, targets),
|
| 131 |
+
[parsed, mapping, targets]
|
| 132 |
+
);
|
| 133 |
+
|
| 134 |
+
async function run(): Promise<void> {
|
| 135 |
+
setBusy(true);
|
| 136 |
+
setError("");
|
| 137 |
+
try {
|
| 138 |
+
const res = await fetch(
|
| 139 |
+
`${API_V1}/tables/${encodeURIComponent(scope)}/rows/import`,
|
| 140 |
+
{
|
| 141 |
+
method: "POST",
|
| 142 |
+
credentials: CREDENTIALS,
|
| 143 |
+
headers: JSON_HEADERS,
|
| 144 |
+
body: JSON.stringify({ rows: plan.rows }),
|
| 145 |
+
}
|
| 146 |
+
);
|
| 147 |
+
const body = (await res.json().catch(() => null)) as
|
| 148 |
+
| { imported?: number; error?: { message?: string } }
|
| 149 |
+
| null;
|
| 150 |
+
if (!res.ok) {
|
| 151 |
+
// The server's sentence beats anything invented here: it names the column, or the cap.
|
| 152 |
+
setError(body?.error?.message ?? `The server answered ${res.status}.`);
|
| 153 |
+
return;
|
| 154 |
+
}
|
| 155 |
+
onImported(body?.imported ?? plan.rows.length);
|
| 156 |
+
onClose();
|
| 157 |
+
} catch {
|
| 158 |
+
setError("Nothing was imported — the request did not complete.");
|
| 159 |
+
} finally {
|
| 160 |
+
setBusy(false);
|
| 161 |
+
}
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
const mappedCount = Object.values(mapping).filter((k) => k !== NONE).length;
|
| 165 |
+
|
| 166 |
+
return (
|
| 167 |
+
<div className="cg-import-scrim" role="presentation" onClick={onClose}>
|
| 168 |
+
<div
|
| 169 |
+
className="cg-import"
|
| 170 |
+
role="dialog"
|
| 171 |
+
aria-modal="true"
|
| 172 |
+
aria-label={`Import records into ${tableName}`}
|
| 173 |
+
onClick={(event) => event.stopPropagation()}
|
| 174 |
+
>
|
| 175 |
+
<div className="cg-import-head">
|
| 176 |
+
<h2>Import records</h2>
|
| 177 |
+
<button type="button" className="cg-import-x" aria-label="Close" onClick={onClose}>
|
| 178 |
+
×
|
| 179 |
+
</button>
|
| 180 |
+
</div>
|
| 181 |
+
|
| 182 |
+
{!parsed ? (
|
| 183 |
+
<div className="cg-import-body">
|
| 184 |
+
<p className="cg-import-lede">
|
| 185 |
+
Add records to <strong>{tableName}</strong> from a spreadsheet. Excel
|
| 186 |
+
(<code>.xlsx</code>) or <code>.csv</code>, up to 5 MB.
|
| 187 |
+
</p>
|
| 188 |
+
<p className="cg-import-note">
|
| 189 |
+
Every row becomes a <strong>new record</strong>. Nothing already in this database is
|
| 190 |
+
matched or overwritten.
|
| 191 |
+
</p>
|
| 192 |
+
<input
|
| 193 |
+
ref={fileRef}
|
| 194 |
+
type="file"
|
| 195 |
+
accept=".xlsx,.csv"
|
| 196 |
+
className="cg-import-file"
|
| 197 |
+
onChange={(event) => {
|
| 198 |
+
const file = event.currentTarget.files?.[0];
|
| 199 |
+
if (file) void pick(file);
|
| 200 |
+
}}
|
| 201 |
+
/>
|
| 202 |
+
</div>
|
| 203 |
+
) : (
|
| 204 |
+
<div className="cg-import-body">
|
| 205 |
+
<p className="cg-import-lede">
|
| 206 |
+
{parsed.rows.toLocaleString()} row{parsed.rows === 1 ? "" : "s"} read
|
| 207 |
+
{parsed.truncated ? " (the first 20,000 — the rest were not read)" : ""}. Choose
|
| 208 |
+
where each column goes.
|
| 209 |
+
</p>
|
| 210 |
+
<div className="cg-import-map" role="list">
|
| 211 |
+
{parsed.columns.map((name, i) => (
|
| 212 |
+
<label className="cg-import-row" role="listitem" key={`${name}:${i}`}>
|
| 213 |
+
<span className="cg-import-col" title={name}>
|
| 214 |
+
{name || `Column ${i + 1}`}
|
| 215 |
+
</span>
|
| 216 |
+
<select
|
| 217 |
+
value={mapping[i] ?? NONE}
|
| 218 |
+
onChange={(event) => {
|
| 219 |
+
const next = event.target.value;
|
| 220 |
+
setMapping((cur) => ({ ...cur, [i]: next }));
|
| 221 |
+
}}
|
| 222 |
+
>
|
| 223 |
+
<option value={NONE}>Don’t import</option>
|
| 224 |
+
{targets.map((f) => (
|
| 225 |
+
<option key={f.key} value={f.key}>
|
| 226 |
+
{f.label}
|
| 227 |
+
</option>
|
| 228 |
+
))}
|
| 229 |
+
</select>
|
| 230 |
+
</label>
|
| 231 |
+
))}
|
| 232 |
+
</div>
|
| 233 |
+
{plan.problems.length > 0 && (
|
| 234 |
+
<div className="cg-import-problems" role="alert">
|
| 235 |
+
<strong>
|
| 236 |
+
{plan.problems.length.toLocaleString()} value
|
| 237 |
+
{plan.problems.length === 1 ? "" : "s"} cannot go in the column
|
| 238 |
+
{plan.problems.length === 1 ? "" : "s"} chosen for them
|
| 239 |
+
</strong>
|
| 240 |
+
<ul>
|
| 241 |
+
{plan.problems.slice(0, 5).map((p, i) => (
|
| 242 |
+
<li key={i}>
|
| 243 |
+
Row {p.row}, {p.column}: {p.value ? `“${p.value}”` : "(blank)"} is not a
|
| 244 |
+
valid {p.field}
|
| 245 |
+
</li>
|
| 246 |
+
))}
|
| 247 |
+
</ul>
|
| 248 |
+
{plan.problems.length > 5 && (
|
| 249 |
+
<p>…and {(plan.problems.length - 5).toLocaleString()} more.</p>
|
| 250 |
+
)}
|
| 251 |
+
<p>
|
| 252 |
+
Fix them in the file, or point that column somewhere else — nothing has been
|
| 253 |
+
imported yet.
|
| 254 |
+
</p>
|
| 255 |
+
</div>
|
| 256 |
+
)}
|
| 257 |
+
</div>
|
| 258 |
+
)}
|
| 259 |
+
|
| 260 |
+
{error && (
|
| 261 |
+
<p className="cg-import-error" role="alert">
|
| 262 |
+
{error}
|
| 263 |
+
</p>
|
| 264 |
+
)}
|
| 265 |
+
|
| 266 |
+
<div className="cg-import-foot">
|
| 267 |
+
{parsed && (
|
| 268 |
+
<span className="cg-import-count">
|
| 269 |
+
{mappedCount === 0
|
| 270 |
+
? "No columns chosen yet"
|
| 271 |
+
: `${plan.rows.length.toLocaleString()} record${
|
| 272 |
+
plan.rows.length === 1 ? "" : "s"
|
| 273 |
+
} from ${mappedCount} column${mappedCount === 1 ? "" : "s"}`}
|
| 274 |
+
</span>
|
| 275 |
+
)}
|
| 276 |
+
<button type="button" className="cg-btn" onClick={onClose}>
|
| 277 |
+
Cancel
|
| 278 |
+
</button>
|
| 279 |
+
{parsed && (
|
| 280 |
+
<button
|
| 281 |
+
type="button"
|
| 282 |
+
className="cg-btn cg-btn--primary"
|
| 283 |
+
/* ⛔ DISABLED WHILE ANY VALUE IS REFUSED. The alternative — import the good rows and
|
| 284 |
+
report the rest — leaves somebody reconciling a spreadsheet against a table by
|
| 285 |
+
hand, and neither existing write door demonstrates a partial policy. Refusing
|
| 286 |
+
whole is recoverable in one edit; a half-import is not. */
|
| 287 |
+
disabled={busy || plan.rows.length === 0 || plan.problems.length > 0}
|
| 288 |
+
onClick={() => void run()}
|
| 289 |
+
>
|
| 290 |
+
{busy ? "Importing…" : `Import ${plan.rows.length.toLocaleString()} record${
|
| 291 |
+
plan.rows.length === 1 ? "" : "s"
|
| 292 |
+
}`}
|
| 293 |
+
</button>
|
| 294 |
+
)}
|
| 295 |
+
</div>
|
| 296 |
+
</div>
|
| 297 |
+
</div>
|
| 298 |
+
);
|
| 299 |
+
}
|