Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- Dockerfile +8 -0
- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +662 -58
- api/connectors_meta.py +508 -0
- api/main.py +150 -0
- api/meta_relational.py +296 -0
- api/oauth_connect.py +107 -0
- api/odoo_relational.py +11 -0
- api/routes_alerts.py +59 -14
- api/routes_automation.py +26 -35
- api/routes_connected_tables.py +63 -0
- api/routes_connectors.py +323 -70
- api/routes_grid.py +34 -1
- api/routes_nav.py +58 -13
- api/routes_odoo_tables.py +241 -14
- api/routes_tables.py +133 -22
- api/routes_web_agent.py +83 -0
- api/web_agent.py +552 -0
- jobs/requirements-web-agent.txt +30 -0
- jobs/web_agent_job.py +736 -0
- platform/core/keychain.py +90 -3
- platform/core/store.py +682 -561
- platform/core/user_tables.py +123 -7
- platform/harness/meta_store.py +461 -0
- platform/harness/runtime.py +113 -5
- web/src/alerts/AlertsPane.tsx +64 -8
- web/src/alerts/alertsModel.ts +55 -0
- web/src/automation/AutomationBuilder.tsx +22 -0
- web/src/automation/AutomationDetail.tsx +50 -12
- web/src/automation/AutomationSurface.tsx +34 -1
- web/src/automation/automationApi.ts +66 -0
- web/src/connectors/ConnectorsPage.tsx +89 -2
- web/src/connectors/brandMarks.tsx +37 -1
- web/src/customer-grid/CustomerGrid.tsx +10 -1
- web/src/customer-grid/apiBridge.ts +49 -2
- web/src/customer-grid/display.ts +34 -0
- web/src/customer-grid/types.ts +15 -0
- web/src/customer-grid/useCustomerData.ts +22 -1
- web/src/index.css +56 -0
- web/src/shell/Shell.tsx +95 -5
- web/src/shell/nav.ts +90 -4
Dockerfile
CHANGED
|
@@ -39,6 +39,14 @@ COPY api/ ./api/
|
|
| 39 |
# died with `failed to calculate checksum ... "/the": not found`. Only RUN tolerates it, and
|
| 40 |
# only because the shell eats the `#`. Gated in ops/verify_portability.py.
|
| 41 |
COPY platform/ ./platform/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
COPY --from=web /web/dist ./web/dist
|
| 43 |
|
| 44 |
# Run unprivileged β the C2 security baseline, and matched to ops/Dockerfile.api so the two images
|
|
|
|
| 39 |
# died with `failed to calculate checksum ... "/the": not found`. Only RUN tolerates it, and
|
| 40 |
# only because the shell eats the `#`. Gated in ops/verify_portability.py.
|
| 41 |
COPY platform/ ./platform/
|
| 42 |
+
# the ephemeral-HF-Job scripts the API submits BY PATH (wave 31, R10/C5)
|
| 43 |
+
# β WITHOUT THIS LINE THE DEPLOY UPLOADS jobs/ AND THE IMAGE STILL DOES NOT HAVE IT. Found at
|
| 44 |
+
# wave 31 close: `deploy_web.py::_jobs_files()` was fixed to SHIP the two files, `--check` proved
|
| 45 |
+
# `jobs=2`, and the container would still have answered "the browser job's script is missing from
|
| 46 |
+
# this deployment" β because reaching the Space REPO and reaching the IMAGE are two different
|
| 47 |
+
# things, and only this instruction does the second. Same shape as the manifest defect one layer
|
| 48 |
+
# up: a file the product opens BY PATH is invisible to every import-following guard we own.
|
| 49 |
+
COPY jobs/ ./jobs/
|
| 50 |
COPY --from=web /web/dist ./web/dist
|
| 51 |
|
| 52 |
# Run unprivileged β the C2 security baseline, and matched to ops/Dockerfile.api so the two images
|
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "e4041c9",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
e4041c9
|
api/automation_engine.py
CHANGED
|
@@ -874,7 +874,7 @@ def disable_for_table(rt, table_key, note="target database deleted"):
|
|
| 874 |
return cur
|
| 875 |
|
| 876 |
if key:
|
| 877 |
-
|
| 878 |
return touched
|
| 879 |
|
| 880 |
|
|
@@ -1010,11 +1010,26 @@ MACHINE_OWNERS = ("automation", "scheduler")
|
|
| 1010 |
|
| 1011 |
|
| 1012 |
def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="",
|
| 1013 |
-
record_mode="", lock_fields=False):
|
| 1014 |
"""Create the table if it is missing; return its key. Idempotent β a re-run of an automation
|
| 1015 |
that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given)
|
| 1016 |
and an existing table with that key is adopted, not duplicated.
|
| 1017 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1018 |
Fields are MERGED, never replaced: a user who added a column to an automation's table keeps
|
| 1019 |
it, and a new source column joins on the next run.
|
| 1020 |
|
|
@@ -1068,7 +1083,7 @@ def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="",
|
|
| 1068 |
# β SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store
|
| 1069 |
# commit re-writing an identical definition β against a 20 s flush floor and a 256/hr repo
|
| 1070 |
# budget, an idempotent helper that always writes is the same defect as a per-row insert.
|
| 1071 |
-
have = ut_get(rt, key)
|
| 1072 |
have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])}
|
| 1073 |
missing = [f for f in wanted if f.get("key") not in have_fields]
|
| 1074 |
missing_locks = [f for f in wanted
|
|
@@ -2226,7 +2241,12 @@ def set_state(rt, auto_id, patch_state):
|
|
| 2226 |
d["state"] = st
|
| 2227 |
return cur
|
| 2228 |
|
| 2229 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2230 |
|
| 2231 |
|
| 2232 |
def _without_retired_board(definition):
|
|
@@ -2328,16 +2348,81 @@ def retire_automation_board_state(rt, tables=None):
|
|
| 2328 |
cur[aid] = cleaned
|
| 2329 |
return cur
|
| 2330 |
|
| 2331 |
-
|
| 2332 |
return {"definitions": len(plan), **fields}
|
| 2333 |
|
| 2334 |
|
| 2335 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2336 |
try:
|
| 2337 |
raw = dict(rt.get(STORE_KEY) or {})
|
| 2338 |
except Exception:
|
| 2339 |
return {}
|
| 2340 |
-
|
|
|
|
|
|
|
|
|
|
| 2341 |
|
| 2342 |
|
| 2343 |
def _new_id(existing):
|
|
@@ -2465,7 +2550,7 @@ def create(rt, raw, username=""):
|
|
| 2465 |
# β NO PRESET GUARD HERE, deliberately: `patch` may skip the spawn because it can compare
|
| 2466 |
# against a PREVIOUS definition, and a create has none β R10's columns must exist before a run,
|
| 2467 |
# so this call stays unconditional.
|
| 2468 |
-
|
| 2469 |
_presets_after_write(rt, defn, username) # R10: the columns exist before a run
|
| 2470 |
if defn.get("trigger"):
|
| 2471 |
_seed_event_state(rt, defn)
|
|
@@ -2510,7 +2595,7 @@ def patch(rt, auto_id, raw, username=""):
|
|
| 2510 |
# The reasoning, the read-your-writes contract and the `user_tables.add_row` precedent are
|
| 2511 |
# written out in full at `remove` above (same wave, same owner report); this is that ruling
|
| 2512 |
# applied to the save path rather than a second argument for it.
|
| 2513 |
-
|
| 2514 |
# β THE PRESET SPAWN RUNS ONLY WHEN ITS OWN INPUTS MOVED β the other half of the same report.
|
| 2515 |
# MEASURED as `nurilab-admin` on `4258a93`: **7,912 ms** for one rename against a **288 ms**
|
| 2516 |
# warm `GET /automations` on the same connection, i.e. 27x the read this wave just made fast
|
|
@@ -2528,8 +2613,16 @@ def patch(rt, auto_id, raw, username=""):
|
|
| 2528 |
# run-time `ut_ensure` R10 deliberately kept is still the backstop if the TABLE drifted
|
| 2529 |
# underneath β which is not a thing a rename should be repairing. Shaped after the trigger
|
| 2530 |
# comparison directly below, which already compares `prev` against the cleaned definition.
|
| 2531 |
-
|
| 2532 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2533 |
_presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS
|
| 2534 |
# A2(1): a NEW or CHANGED condition trigger starts with everything currently matching
|
| 2535 |
# DISARMED β enabling never fires for records already in the state.
|
|
@@ -2563,11 +2656,99 @@ def _presets_after_write(rt, defn, username):
|
|
| 2563 |
arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the
|
| 2564 |
honest failure still has exactly one home.
|
| 2565 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2566 |
cfg = defn.get("config") or {}
|
| 2567 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2568 |
if not target:
|
| 2569 |
return
|
|
|
|
|
|
|
|
|
|
| 2570 |
try:
|
|
|
|
|
|
|
| 2571 |
# ββ WAVE 30 Β· T06 β BOTH DISCOVERY KINDS SPAWN ON SAVE, and TikTok's absence here was
|
| 2572 |
# R10 simply not holding for the second platform: `TT_PROFILE_FIELDS` reached `ut_ensure`
|
| 2573 |
# at exactly ONE site β inside `run_discover_tiktok` β so the database did not exist until
|
|
@@ -2583,40 +2764,55 @@ def _presets_after_write(rt, defn, username):
|
|
| 2583 |
# caught only because that check happened to exist.
|
| 2584 |
# β R10 is a rule about what somebody PICKED IN THE PICKER, so it reads the picker's own
|
| 2585 |
# answer. `DISCOVERY_TRIGGER_KIND` is that map, and a gate asserts it agrees with law 1.
|
| 2586 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2587 |
if _dkind:
|
| 2588 |
_, _, _spawn_label, _spawn_fields = discovery_facts(_dkind)
|
| 2589 |
-
ut_ensure(rt,
|
| 2590 |
-
key=target, flow_tag=
|
| 2591 |
-
return
|
| 2592 |
-
|
| 2593 |
-
|
| 2594 |
-
|
| 2595 |
-
|
| 2596 |
-
|
| 2597 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2598 |
bound = next((str(f.get("key") or "") for f in table.get("fields") or []
|
| 2599 |
if isinstance(f.get("profile"), dict)), "") or named
|
| 2600 |
if enrich_actions and bound:
|
| 2601 |
ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username,
|
| 2602 |
-
key=target, flow_tag=
|
|
|
|
| 2603 |
# β WAVE 30 Β· T08 β the TikTok half of W25/R10: the columns an `enrich_tiktok` step will
|
| 2604 |
# fill appear when the automation is SAVED, not when money is first spent. Resolved
|
| 2605 |
# through `profile_field_key(..., source=PROFILE_SOURCE_TT)` so it cannot adopt an
|
| 2606 |
# Instagram binding, and through the SAME `_tt_profile_schema_for` the runner's own top-up
|
| 2607 |
# uses β R10's rule is that the column set does not change between Save and the first run,
|
| 2608 |
# and one shared resolver is what makes that structural instead of a convention.
|
| 2609 |
-
tt_actions = _actions_of_kind(((defn.get("flow") or {}).get("actions") or []),
|
| 2610 |
-
"enrich_tiktok")
|
| 2611 |
if tt_actions:
|
| 2612 |
-
tt_named = next((
|
| 2613 |
-
for a in tt_actions
|
| 2614 |
-
if str((a.get("config") or {}).get("profileField") or "").strip()), "")
|
| 2615 |
tt_bound = profile_field_key(table, tt_named, source=PROFILE_SOURCE_TT)
|
| 2616 |
if tt_bound:
|
| 2617 |
ut_ensure(rt, table.get("label") or target, _tt_profile_schema_for(tt_bound),
|
| 2618 |
-
username, key=target, flow_tag=
|
| 2619 |
-
|
| 2620 |
# β WAVE 30 Β· T10 β and the CHILD databases the step's own switches promise, at
|
| 2621 |
# SAVE, before any money. R10's rule is "the columns a step will fill appear when
|
| 2622 |
# you press Save"; a `postMetrics` toggle whose database only exists after a paid
|
|
@@ -2624,14 +2820,14 @@ def _presets_after_write(rt, defn, username):
|
|
| 2624 |
# β GATED ON THE SWITCHES, never spawned unconditionally: a database that exists
|
| 2625 |
# because somebody looked at a toggle, and then never fills, is the "SECOND, empty
|
| 2626 |
# database" the discovery default was rewritten to stop producing.
|
| 2627 |
-
tt_cfgs = [(a.get("config") or {}) for a in tt_actions]
|
| 2628 |
for _flag, _child in (("postMetrics", TT_POSTS_TABLE),
|
| 2629 |
("postMetrics", TT_POST_SNAPSHOTS_TABLE),
|
| 2630 |
("commentMetrics", TT_COMMENTS_TABLE)):
|
| 2631 |
-
if any(
|
|
|
|
| 2632 |
ut_ensure(rt, TT_TABLE_LABELS[_child], TT_TABLE_FIELDS[_child], username,
|
| 2633 |
-
key=_child, flow_tag=
|
| 2634 |
-
|
| 2635 |
except Exception as e: # noqa: BLE001
|
| 2636 |
# β NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so
|
| 2637 |
# raising here would answer 500 for an automation that IS stored β the caller would retry
|
|
@@ -2664,7 +2860,7 @@ def remove(rt, auto_id):
|
|
| 2664 |
# stricter than a row insert. [[sync-write-eats-pending-async-write]] is the hazard in the
|
| 2665 |
# OTHER direction (a sync writer discarding a pending async write) and `Store.update`'s dirty
|
| 2666 |
# branch already handles it.
|
| 2667 |
-
|
| 2668 |
# C8 β the fields this flow tagged are DISABLED with the reason on them, never orphaned
|
| 2669 |
# silently: the column keeps its values and its config, and says why it stopped. Scanned
|
| 2670 |
# first so a delete with no tagged fields costs no store commit.
|
|
@@ -2673,19 +2869,34 @@ def remove(rt, auto_id):
|
|
| 2673 |
# update below reads it a second time. Removing that needs a tagged-field index, which is a
|
| 2674 |
# feature and not a wave-tail edit β booked rather than improvised. What it no longer costs
|
| 2675 |
# is the part the owner could feel: the blocking network commits.
|
| 2676 |
-
|
| 2677 |
-
|
| 2678 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2679 |
return
|
| 2680 |
|
| 2681 |
def _disable(cur):
|
| 2682 |
cur = cur if isinstance(cur, dict) else {}
|
| 2683 |
-
|
| 2684 |
-
|
|
|
|
| 2685 |
a = f.get("automation")
|
| 2686 |
-
if
|
| 2687 |
a["disabled"] = True
|
| 2688 |
-
a["statusNote"] =
|
| 2689 |
return cur
|
| 2690 |
# The expensive half of the owner's complaint: a whole-document rewrite, committed while the
|
| 2691 |
# person waits. Async for the reason given above β the disable is visible to the next read
|
|
@@ -2787,7 +2998,7 @@ def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=No
|
|
| 2787 |
f"{entry['summary'][:120]}")
|
| 2788 |
return cur
|
| 2789 |
|
| 2790 |
-
|
| 2791 |
_notify_outcome(rt, auto_id, state, entry)
|
| 2792 |
return entry
|
| 2793 |
|
|
@@ -3332,6 +3543,40 @@ TT_TABLE_LABELS = {
|
|
| 3332 |
TT_POST_SNAPSHOTS_TABLE: "TikTok post measurements",
|
| 3333 |
TT_COMMENTS_TABLE: "TikTok comments",
|
| 3334 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3335 |
|
| 3336 |
|
| 3337 |
# ---------------------------------------------------------------------------------------------
|
|
@@ -4944,6 +5189,37 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 4944 |
# exist, or that is not an Instagram profile table, has nothing to migrate.
|
| 4945 |
detected = set(_ig_profile_tables(rt))
|
| 4946 |
targets = ([only] if only in detected else []) if only else sorted(detected)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4947 |
for table_key in targets:
|
| 4948 |
tbl = ut_get(rt, table_key) or {}
|
| 4949 |
fields = [dict(f) for f in tbl.get("fields") or []]
|
|
@@ -6879,9 +7155,56 @@ def _predicate_sentence(preds, operator="and"):
|
|
| 6879 |
#: than a second, smaller number nobody could explain: a plain automation's records ARE its
|
| 6880 |
#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a
|
| 6881 |
#: flow could be pointed at and which would not finish.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6882 |
MAX_FLOW_RECORDS = MAX_UT_ROWS
|
| 6883 |
|
| 6884 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6885 |
def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None):
|
| 6886 |
"""β WAVE 24 / R6 β the runner for an automation with NO machine step: its flow IS the whole
|
| 6887 |
automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special
|
|
@@ -6947,14 +7270,32 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No
|
|
| 6947 |
ids = selected
|
| 6948 |
step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}")
|
| 6949 |
counts = {"records": len(ids)}
|
| 6950 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6951 |
# The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names
|
| 6952 |
-
# both numbers so "it only processed some of them" is readable rather than deducible
|
|
|
|
| 6953 |
total = len(ids)
|
| 6954 |
-
ids = ids[:
|
| 6955 |
return ("partial",
|
| 6956 |
-
f"{total} records in {table} β this run walked the first "
|
| 6957 |
-
|
| 6958 |
n = len(ids)
|
| 6959 |
# ββ 2026-08-07 (owner report) β SAY IT IN WORDS A PERSON CAN CHECK.
|
| 6960 |
# Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)?
|
|
@@ -7366,6 +7707,30 @@ def toggle_node(rt, auto_id, node_id):
|
|
| 7366 |
# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we
|
| 7367 |
# had added. `migrate_ig_tables` now drops those columns wherever an automation created them.
|
| 7368 |
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7369 |
# β WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE:
|
| 7370 |
# * `lane_match` + `LANE_OPS` + the condition trees below β never board-only. They are the
|
| 7371 |
# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was
|
|
@@ -7542,6 +7907,26 @@ MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flow
|
|
| 7542 |
MAX_BRANCHES = 6 # per If / then
|
| 7543 |
MAX_ACTION_VALUES = 20 # cells one update/create action may write
|
| 7544 |
FIND_LIMIT_MAX = 100
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7545 |
#: β WAVE 24 Β· C-ACT β THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder`
|
| 7546 |
#: and never by a literal list of group names: a client-side ordering is a second copy of this
|
| 7547 |
#: table, and the way it fails is that a group added here renders last, or not at all, with
|
|
@@ -7565,8 +7950,16 @@ ACTION_CATALOG = [
|
|
| 7565 |
"detail": "Needs the recorder that captures a selector instead of a coordinate"},
|
| 7566 |
{"kind": "web_click", "label": "Click something", "group": "Web action", "ready": False,
|
| 7567 |
"detail": "Needs the recorder that captures a selector instead of a coordinate"},
|
| 7568 |
-
|
| 7569 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7570 |
{"kind": "web_repair", "label": "Repair a broken step", "group": "Web action",
|
| 7571 |
"ready": False,
|
| 7572 |
"detail": "Self-healing selectors arrive with the browser engine (Scrapling)"},
|
|
@@ -7942,6 +8335,55 @@ def _clean_action_config(kind, cfg, depth, seen, count):
|
|
| 7942 |
if limit < 1 or limit > FIND_LIMIT_MAX:
|
| 7943 |
return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}"
|
| 7944 |
return {"table": table, "cond": cond, "limit": limit}, None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7945 |
return {}, None
|
| 7946 |
|
| 7947 |
|
|
@@ -8498,6 +8940,30 @@ def _actions_of_kind(actions, kind):
|
|
| 8498 |
return out
|
| 8499 |
|
| 8500 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8501 |
def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print,
|
| 8502 |
step=_no_step):
|
| 8503 |
"""Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts.
|
|
@@ -8525,7 +8991,15 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 8525 |
# server still refuses to let it be removed.
|
| 8526 |
if ig_action_pinned(defn, 0) and actions:
|
| 8527 |
actions = actions[1:]
|
| 8528 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8529 |
# β C4 β THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected
|
| 8530 |
# across the WHOLE walk and written once at the end, for the reason the module header states
|
| 8531 |
# and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
|
|
@@ -8702,6 +9176,82 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 8702 |
# the table would silently apply one action's uniqueness rule to the other's rows.
|
| 8703 |
creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals)
|
| 8704 |
counts["created"] += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8705 |
elif kind == "enrich_instagram":
|
| 8706 |
# β C4 (R3/R4). Reuses `pull_profile` and `capture_rows` β the SAME functions
|
| 8707 |
# `run_field_instagram` calls, not a second implementation of either.
|
|
@@ -8748,6 +9298,10 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 8748 |
# the row it belongs to: the collector writes preset cells back onto THAT record,
|
| 8749 |
# and a run-wide list would have no way to say which handle each snapshot was for.
|
| 8750 |
pend_prof = []
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8751 |
res = pull_profile(handle_raw,
|
| 8752 |
max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
|
| 8753 |
log=log,
|
|
@@ -8792,6 +9346,32 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 8792 |
# this same flow sees the enriched values β the rule `update_record` already
|
| 8793 |
# follows, and without it "enrich, then route by follower count" would judge
|
| 8794 |
# the record on the values it had before the pull.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8795 |
cells = preset_cells(res, pulled)
|
| 8796 |
if cfg.get("dryRun"):
|
| 8797 |
enrich["dry"] = True
|
|
@@ -9230,8 +9810,17 @@ def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=prin
|
|
| 9230 |
for table_key, rows_in, key_field in plan:
|
| 9231 |
if not rows_in:
|
| 9232 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9233 |
real_key = ut_ensure(rt, TT_TABLE_LABELS[table_key], TT_TABLE_FIELDS[table_key], username,
|
| 9234 |
-
key=table_key, flow_tag=tag, lock_fields=True
|
|
|
|
| 9235 |
missing.extend(ut_missing(rt, real_key))
|
| 9236 |
merged, counts_ = upsert_rows(dict((ut_get(rt, real_key) or {}).get("rows") or {}),
|
| 9237 |
rows_in, key_field, cap=row_cap(real_key))
|
|
@@ -9419,7 +10008,7 @@ def queue_pending_metric_snapshots(rt, auto_id, pending):
|
|
| 9419 |
state["pendingMetricSnapshots"] = existing
|
| 9420 |
return cur
|
| 9421 |
|
| 9422 |
-
|
| 9423 |
return added[0]
|
| 9424 |
|
| 9425 |
|
|
@@ -9517,7 +10106,7 @@ def queue_pending_profile_snapshots(rt, auto_id, pending):
|
|
| 9517 |
state["pendingProfileSnapshots"] = existing
|
| 9518 |
return cur
|
| 9519 |
|
| 9520 |
-
|
| 9521 |
return added[0]
|
| 9522 |
|
| 9523 |
|
|
@@ -10112,7 +10701,7 @@ def review_audit(rt, auto_id, row_id, from_label, to_label, username,
|
|
| 10112 |
d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS]
|
| 10113 |
return cur
|
| 10114 |
|
| 10115 |
-
|
| 10116 |
|
| 10117 |
|
| 10118 |
# ---------------------------------------------------------------------------------------------
|
|
@@ -11384,7 +11973,7 @@ def _pause_trigger(rt, auto_id, note):
|
|
| 11384 |
trg["paused"] = True
|
| 11385 |
d["statusNote"] = _s(note, 200)
|
| 11386 |
return cur
|
| 11387 |
-
|
| 11388 |
|
| 11389 |
|
| 11390 |
def trigger_fire(rt, tenant, auto_id, log=print, rows=None):
|
|
@@ -11576,7 +12165,12 @@ def grid_hook(evt):
|
|
| 11576 |
tenant = str(getattr(st, "key", "") or "royal-imports")
|
| 11577 |
table = str(evt.get("table") or "")
|
| 11578 |
kind = str(evt.get("type") or "")
|
| 11579 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11580 |
field = str(evt.get("field") or "")
|
| 11581 |
rid = str(evt.get("rowId") or "")
|
| 11582 |
for aid, d in defs.items():
|
|
@@ -11593,6 +12187,16 @@ def grid_hook(evt):
|
|
| 11593 |
if ridn is None or ridn <= hw:
|
| 11594 |
continue # once per record EVER (A2(4)) β undo-proof
|
| 11595 |
set_state(st, aid, {"rcHighwater": ridn})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11596 |
trigger_fire(st, tenant, aid, rows=[rid])
|
| 11597 |
elif kind == "event_field" and key == "event_field":
|
| 11598 |
# β WAVE 24 Β· law 4 β the per-FIELD narrowing is gone with the stored key. Every
|
|
@@ -11708,7 +12312,7 @@ def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None):
|
|
| 11708 |
if dd is not None:
|
| 11709 |
dd["statusNote"] = _s(err, 200)
|
| 11710 |
return cur
|
| 11711 |
-
|
| 11712 |
return None
|
| 11713 |
lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n))
|
| 11714 |
reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid))
|
|
|
|
| 874 |
return cur
|
| 875 |
|
| 876 |
if key:
|
| 877 |
+
_store_update(rt, _up, flush="sync")
|
| 878 |
return touched
|
| 879 |
|
| 880 |
|
|
|
|
| 1010 |
|
| 1011 |
|
| 1012 |
def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="",
|
| 1013 |
+
record_mode="", lock_fields=False, tables=None):
|
| 1014 |
"""Create the table if it is missing; return its key. Idempotent β a re-run of an automation
|
| 1015 |
that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given)
|
| 1016 |
and an existing table with that key is adopted, not duplicated.
|
| 1017 |
|
| 1018 |
+
ββ WAVE 31 Β· T30 β `tables` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING, and it is
|
| 1019 |
+
the SKIP TEST below that it pays for. `rt.get` deep-copies the whole tenant document on every
|
| 1020 |
+
call (documented ceiling 35.8 MB / ~1.4 s), so a caller that ensures FOUR tables in one pass
|
| 1021 |
+
paid four copies to answer four questions about one document. Owner item 7, verbatim: *"It still
|
| 1022 |
+
takes forever to change the Config for automation as well. It just say Saving..."* β measured
|
| 1023 |
+
**7,912 ms** for one save on `4258a93`.
|
| 1024 |
+
|
| 1025 |
+
β THE LEND IS READ-ONLY AND SAFE BY INSPECTION, not by hope β the same argument
|
| 1026 |
+
`retire_automation_stage_fields` (this module, same parameter name, same contract) already
|
| 1027 |
+
makes: only the `have` lookup below reads it, and the WRITE still goes through `rt.update`,
|
| 1028 |
+
whose `_up` re-reads the live document under the store lock. **A lent snapshot can therefore
|
| 1029 |
+
never be the thing that gets written back**, and the worst a stale one can do is spend a commit
|
| 1030 |
+
that would have been skipped β never write a wrong value. Callers that ensure two tables
|
| 1031 |
+
under the SAME key in one pass pass `tables=None` for the second (see `_spawn_presets`).
|
| 1032 |
+
|
| 1033 |
Fields are MERGED, never replaced: a user who added a column to an automation's table keeps
|
| 1034 |
it, and a new source column joins on the next run.
|
| 1035 |
|
|
|
|
| 1083 |
# β SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store
|
| 1084 |
# commit re-writing an identical definition β against a 20 s flush floor and a 256/hr repo
|
| 1085 |
# budget, an idempotent helper that always writes is the same defect as a per-row insert.
|
| 1086 |
+
have = ut_get(rt, key) if tables is None else tables.get(str(key))
|
| 1087 |
have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])}
|
| 1088 |
missing = [f for f in wanted if f.get("key") not in have_fields]
|
| 1089 |
missing_locks = [f for f in wanted
|
|
|
|
| 2241 |
d["state"] = st
|
| 2242 |
return cur
|
| 2243 |
|
| 2244 |
+
# β W31-T35 β THE ONE `keeps_defs=True` IN THIS MODULE. `set_state` writes `state` and never a
|
| 2245 |
+
# trigger, and `grid_hook` calls it once per CREATED RECORD; clearing the definitions memo here
|
| 2246 |
+
# would re-read the whole bucket one row later than before and undo D-134 entirely. Safe
|
| 2247 |
+
# because `grid_hook` reads exactly one state key (`rcHighwater`) and mirrors its own write
|
| 2248 |
+
# into the memo in the same statement. Read `_store_update`'s note before adding a second.
|
| 2249 |
+
_store_update(rt, _up, flush="sync", keeps_defs=True)
|
| 2250 |
|
| 2251 |
|
| 2252 |
def _without_retired_board(definition):
|
|
|
|
| 2348 |
cur[aid] = cleaned
|
| 2349 |
return cur
|
| 2350 |
|
| 2351 |
+
_store_update(rt, _up, flush="sync")
|
| 2352 |
return {"definitions": len(plan), **fields}
|
| 2353 |
|
| 2354 |
|
| 2355 |
+
#: ββ WAVE 31 Β· T35 (D-134) β the definitions memo, and the two things that make it safe.
|
| 2356 |
+
#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which
|
| 2357 |
+
#: exactly one caller uses (`grid_hook`).
|
| 2358 |
+
_DEFS_MEMO = {}
|
| 2359 |
+
#: Deliberately SHORT. This exists to collapse one burst of row events into one read, not to be a
|
| 2360 |
+
#: cache β an import of 20,000 rows arrives in far less than this, and a stale trigger for two
|
| 2361 |
+
#: seconds is bounded and recoverable where a stale one for a minute is a mystery.
|
| 2362 |
+
_DEFS_TTL = 2.0
|
| 2363 |
+
|
| 2364 |
+
|
| 2365 |
+
def _store_update(rt, fn, flush="sync", keeps_defs=False):
|
| 2366 |
+
"""THE one door every write to the automations bucket goes through β and the ONLY reason it
|
| 2367 |
+
exists is that the memo's invalidation must be DERIVED rather than remembered.
|
| 2368 |
+
|
| 2369 |
+
β `keeps_defs=True` IS AN EXPLICIT, SINGLE-SITE OPT-OUT, and it is here because the safe
|
| 2370 |
+
default would otherwise make the memo useless on the exact path it was built for. `set_state`
|
| 2371 |
+
writes `state` and NEVER a trigger, and it is called by `grid_hook` itself once per created
|
| 2372 |
+
record β so clearing on it would mean a 20,000-row import re-read the bucket 20,000 times
|
| 2373 |
+
anyway, one row later than before. The opt-out is safe because `grid_hook` reads exactly one
|
| 2374 |
+
state key (`rcHighwater`) and mirrors its own write into the memo in the same statement.
|
| 2375 |
+
β The DEFAULT is the safe one, so a thirteenth writer added next wave gets invalidation
|
| 2376 |
+
without knowing this exists; only a caller that has read this paragraph can opt out.
|
| 2377 |
+
|
| 2378 |
+
β THE ALTERNATIVE WAS TWELVE CALL SITES. `rt.update(STORE_KEY, β¦)` appeared twelve times in
|
| 2379 |
+
this module; hanging an invalidation on each would be a hand-maintained list, and the way that
|
| 2380 |
+
fails is silent: a thirteenth writer added next wave leaves `grid_hook` firing triggers off a
|
| 2381 |
+
definition set that no longer exists, with nothing red. Same defect class as `patch`'s
|
| 2382 |
+
hand-maintained patchable-key list, which this module already documents as *"the silent-drop
|
| 2383 |
+
seat"* [[a-constant-two-features-share]].
|
| 2384 |
+
|
| 2385 |
+
β IT CLEARS THE WHOLE MEMO, not this tenant's row, on purpose: identifying the tenant here
|
| 2386 |
+
would mean trusting `getattr(rt, 'key')` at a WRITE, and a wrong answer there is a stale
|
| 2387 |
+
trigger set for another tenant. The memo holds at most a handful of entries and the correct,
|
| 2388 |
+
boring thing costs nothing.
|
| 2389 |
+
"""
|
| 2390 |
+
if not keeps_defs:
|
| 2391 |
+
_DEFS_MEMO.clear()
|
| 2392 |
+
return rt.update(STORE_KEY, fn, flush=flush)
|
| 2393 |
+
|
| 2394 |
+
|
| 2395 |
+
def all_definitions(rt, cached=False):
|
| 2396 |
+
"""Every automation definition for this tenant.
|
| 2397 |
+
|
| 2398 |
+
ββ `cached=True` IS D-134's FIX, AND IT IS OPT-IN FOR A REASON. `grid_hook` runs ONCE PER ROW
|
| 2399 |
+
EVENT and called this unconditionally, so a 20,000-row import performed 20,000 whole-document
|
| 2400 |
+
deep copies of the automations bucket β `Store.get` re-serialises on every call, hit or miss,
|
| 2401 |
+
under the store lock. Every other caller is a request handler that reads it once, so widening
|
| 2402 |
+
the memo to all of them would trade a real guarantee (a request sees the current store) for
|
| 2403 |
+
nothing measurable.
|
| 2404 |
+
|
| 2405 |
+
β THE MEMO IS THE SAME OBJECT ON A HIT, and `grid_hook` MUTATES it deliberately β see the
|
| 2406 |
+
`rcHighwater` write there. That is not a leak of an implementation detail, it is the fix to the
|
| 2407 |
+
hazard a naive memo creates: `grid_hook` both READS `state.rcHighwater` and WRITES it through
|
| 2408 |
+
`set_state`, so a memo that went stale against its own write would re-fire `record_created`
|
| 2409 |
+
for records it had already handled, breaking A2(4)'s *"once per record EVER β undo-proof"*.
|
| 2410 |
+
Writes through `_store_update` drop the memo; the one write `grid_hook` makes to its OWN copy
|
| 2411 |
+
is mirrored into it in the same statement.
|
| 2412 |
+
"""
|
| 2413 |
+
if cached:
|
| 2414 |
+
key = str(getattr(rt, "key", "") or "")
|
| 2415 |
+
hit = _DEFS_MEMO.get(key)
|
| 2416 |
+
if hit is not None and (time.monotonic() - hit[0]) < _DEFS_TTL:
|
| 2417 |
+
return hit[1]
|
| 2418 |
try:
|
| 2419 |
raw = dict(rt.get(STORE_KEY) or {})
|
| 2420 |
except Exception:
|
| 2421 |
return {}
|
| 2422 |
+
out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()}
|
| 2423 |
+
if cached:
|
| 2424 |
+
_DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out)
|
| 2425 |
+
return out
|
| 2426 |
|
| 2427 |
|
| 2428 |
def _new_id(existing):
|
|
|
|
| 2550 |
# β NO PRESET GUARD HERE, deliberately: `patch` may skip the spawn because it can compare
|
| 2551 |
# against a PREVIOUS definition, and a create has none β R10's columns must exist before a run,
|
| 2552 |
# so this call stays unconditional.
|
| 2553 |
+
_store_update(rt, _up, flush="async")
|
| 2554 |
_presets_after_write(rt, defn, username) # R10: the columns exist before a run
|
| 2555 |
if defn.get("trigger"):
|
| 2556 |
_seed_event_state(rt, defn)
|
|
|
|
| 2595 |
# The reasoning, the read-your-writes contract and the `user_tables.add_row` precedent are
|
| 2596 |
# written out in full at `remove` above (same wave, same owner report); this is that ruling
|
| 2597 |
# applied to the save path rather than a second argument for it.
|
| 2598 |
+
_store_update(rt, _up, flush="async")
|
| 2599 |
# β THE PRESET SPAWN RUNS ONLY WHEN ITS OWN INPUTS MOVED β the other half of the same report.
|
| 2600 |
# MEASURED as `nurilab-admin` on `4258a93`: **7,912 ms** for one rename against a **288 ms**
|
| 2601 |
# warm `GET /automations` on the same connection, i.e. 27x the read this wave just made fast
|
|
|
|
| 2613 |
# run-time `ut_ensure` R10 deliberately kept is still the backstop if the TABLE drifted
|
| 2614 |
# underneath β which is not a thing a rename should be repairing. Shaped after the trigger
|
| 2615 |
# comparison directly below, which already compares `prev` against the cleaned definition.
|
| 2616 |
+
# ββ WAVE 31 Β· T30 β AND THE PREDICATE IS NOW THE SPAWN'S OWN ARGUMENT, which is the fix.
|
| 2617 |
+
# The four-whole-objects test above was right in principle and inert in practice: `persist`
|
| 2618 |
+
# sends a REBUILT `config`, so `prev["config"] != defn["config"]` on essentially every save and
|
| 2619 |
+
# the guard fired every time β sparing only *rename*, the one edit the owner stopped
|
| 2620 |
+
# complaining about after wave 30. `preset_inputs` is the canonical projection of exactly what
|
| 2621 |
+
# `_spawn_presets` reads, and `_spawn_presets` is handed nothing else, so this comparison
|
| 2622 |
+
# cannot go stale behind a future input the way a hand-listed key set would. The full argument,
|
| 2623 |
+
# including why this is STRONGER than what it replaces rather than a relaxation, is in
|
| 2624 |
+
# `preset_inputs`' own docstring β read that before widening or narrowing this line.
|
| 2625 |
+
if preset_inputs(prev) != preset_inputs(defn):
|
| 2626 |
_presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS
|
| 2627 |
# A2(1): a NEW or CHANGED condition trigger starts with everything currently matching
|
| 2628 |
# DISARMED β enabling never fires for records already in the state.
|
|
|
|
| 2656 |
arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the
|
| 2657 |
honest failure still has exactly one home.
|
| 2658 |
"""
|
| 2659 |
+
_spawn_presets(rt, preset_inputs(defn), username)
|
| 2660 |
+
|
| 2661 |
+
|
| 2662 |
+
#: The enrich actions whose OWN config decides which preset columns and child databases a save
|
| 2663 |
+
#: spawns, and the exact keys read off each. DECLARED here rather than spelled inside
|
| 2664 |
+
#: `preset_inputs`, so `_spawn_presets` and the save-path guard cannot drift apart by one key.
|
| 2665 |
+
PRESET_ACTION_INPUTS = {
|
| 2666 |
+
"enrich_instagram": ("profileField",),
|
| 2667 |
+
"enrich_tiktok": ("profileField", "postMetrics", "commentMetrics"),
|
| 2668 |
+
}
|
| 2669 |
+
|
| 2670 |
+
|
| 2671 |
+
def preset_inputs(defn):
|
| 2672 |
+
"""ββ WAVE 31 Β· T30 β EXACTLY the values `_spawn_presets` consumes off a definition, canonical.
|
| 2673 |
+
|
| 2674 |
+
β THIS IS THE WHOLE POINT OF THE TICKET, so it is worth being precise about what changed and
|
| 2675 |
+
why it is SAFER than what it replaces, not weaker. Owner item 7, verbatim: *"It still takes
|
| 2676 |
+
forever to change the Config for automation as well. It just say Saving... and takes a long time
|
| 2677 |
+
for me to change options and configs etc."*
|
| 2678 |
+
|
| 2679 |
+
Wave 30 guarded the spawn on `any(prev[k] != defn[k] for k in ("config","flow","trigger",
|
| 2680 |
+
"kind"))` and reasoned β correctly β that *"the predicate is the INPUT SET of the function being
|
| 2681 |
+
skipped, not a hand-listed set of cheap edits"*. The flaw was not the reasoning; it was that the
|
| 2682 |
+
input set was stated as four WHOLE OBJECTS while the spawn reads a handful of leaves out of
|
| 2683 |
+
them. `AutomationDetail.tsx::persist` sends `config: buildConfig()` β a value RECONSTRUCTED from
|
| 2684 |
+
React state, not the stored object β so the comparison is against something rebuilt on every
|
| 2685 |
+
save and the guard never fires on the one edit the owner is complaining about. W30's fix spared
|
| 2686 |
+
*rename* (the client omits `flow`/`trigger` entirely, and `kind`/`name` are unchanged), which is
|
| 2687 |
+
exactly the edit the owner stopped complaining about after wave 30.
|
| 2688 |
+
|
| 2689 |
+
β WHY NARROWING IS SAFE HERE AND WAS NOT SAFE THEN: this projection is not a hand-listed
|
| 2690 |
+
allow-list that a future wave can silently outgrow. `_presets_after_write` hands this dict to
|
| 2691 |
+
`_spawn_presets` and `_spawn_presets` **never receives `defn` at all** β so it is structurally
|
| 2692 |
+
incapable of reading a fifth key that this function does not carry. Teaching the spawn to read
|
| 2693 |
+
something new REQUIRES adding it here, and the guard then follows for free. That is a stronger
|
| 2694 |
+
guarantee than wave 30's, which held only as long as somebody remembered the comment.
|
| 2695 |
+
|
| 2696 |
+
β CANONICAL, because the input is a rebuild: each leaf is normalised exactly the way the spawn
|
| 2697 |
+
itself normalises it (`str(...).strip()` for a field name, `bool(...)` for a switch). Without
|
| 2698 |
+
that, `postMetrics: false` and a missing `postMetrics` would compare unequal while the spawn
|
| 2699 |
+
treats them identically β a guard that fires on a difference its consumer cannot see is the
|
| 2700 |
+
same defect in a smaller font.
|
| 2701 |
+
|
| 2702 |
+
β IT DELIBERATELY COVERS THE DISCOVERY BRANCH **AND** THE ENRICH BRANCH IN ONE PROJECTION, even
|
| 2703 |
+
though today's control flow returns before the second can run for a discovery automation. That
|
| 2704 |
+
early return is D-178 (W31-T34) and it is going away; a projection built around today's branch
|
| 2705 |
+
would silently narrow the guard the moment it does.
|
| 2706 |
+
"""
|
| 2707 |
+
defn = defn or {}
|
| 2708 |
cfg = defn.get("config") or {}
|
| 2709 |
+
actions = ((defn.get("flow") or {}).get("actions") or [])
|
| 2710 |
+
return {
|
| 2711 |
+
"target": str(_flow_table(defn) or cfg.get("targetTable") or ""),
|
| 2712 |
+
"label": str(cfg.get("targetLabel") or ""),
|
| 2713 |
+
"flowId": str(defn.get("id") or ""),
|
| 2714 |
+
"discoveryKind": DISCOVERY_TRIGGER_KIND.get(
|
| 2715 |
+
(defn.get("trigger") or {}).get("key") or ""),
|
| 2716 |
+
"actions": {
|
| 2717 |
+
kind: [{k: (_norm_switch(a, k) if k != "profileField"
|
| 2718 |
+
else str((a.get("config") or {}).get(k) or "").strip())
|
| 2719 |
+
for k in keys}
|
| 2720 |
+
for a in _actions_of_kind(actions, kind)]
|
| 2721 |
+
for kind, keys in PRESET_ACTION_INPUTS.items()
|
| 2722 |
+
},
|
| 2723 |
+
}
|
| 2724 |
+
|
| 2725 |
+
|
| 2726 |
+
def _norm_switch(action, key):
|
| 2727 |
+
"""A capture switch as the spawn reads it β `bool`, so absent and False are ONE value."""
|
| 2728 |
+
return bool((action.get("config") or {}).get(key))
|
| 2729 |
+
|
| 2730 |
+
|
| 2731 |
+
def _spawn_presets(rt, inputs, username):
|
| 2732 |
+
"""The spawn itself. Takes `preset_inputs(defn)` β never the definition β see that docstring.
|
| 2733 |
+
|
| 2734 |
+
ββ WAVE 31 Β· T30, THE SECOND HALF: **ONE `user_tables` read for the whole pass, not five.**
|
| 2735 |
+
Every `ut_ensure` below used to take its own copy of the tenant document through
|
| 2736 |
+
`ut_get` β `ut_all` β `rt.get`, which deep-copies unconditionally (ceiling 35.8 MB / ~1.4 s), and
|
| 2737 |
+
the TikTok arm reaches FOUR of them plus the `ut_get` above β measured **7,912 ms** on
|
| 2738 |
+
`4258a93`. Lending one snapshot is the same fix W30-T13 made on the automation READ path
|
| 2739 |
+
(`routes_automation.py::_LentTables`); this is deliberately NOT a third copy of that class, but
|
| 2740 |
+
the `tables=` parameter `retire_automation_stage_fields` in this very module already
|
| 2741 |
+
established, because here every consumer is a function we own and pass to directly.
|
| 2742 |
+
"""
|
| 2743 |
+
target = inputs["target"]
|
| 2744 |
if not target:
|
| 2745 |
return
|
| 2746 |
+
tag = inputs["flowId"]
|
| 2747 |
+
ig_actions = inputs["actions"]["enrich_instagram"]
|
| 2748 |
+
tt_actions = inputs["actions"]["enrich_tiktok"]
|
| 2749 |
try:
|
| 2750 |
+
# β THE ONE READ. Everything below answers out of this snapshot.
|
| 2751 |
+
tables = ut_all(rt)
|
| 2752 |
# ββ WAVE 30 Β· T06 β BOTH DISCOVERY KINDS SPAWN ON SAVE, and TikTok's absence here was
|
| 2753 |
# R10 simply not holding for the second platform: `TT_PROFILE_FIELDS` reached `ut_ensure`
|
| 2754 |
# at exactly ONE site β inside `run_discover_tiktok` β so the database did not exist until
|
|
|
|
| 2764 |
# caught only because that check happened to exist.
|
| 2765 |
# β R10 is a rule about what somebody PICKED IN THE PICKER, so it reads the picker's own
|
| 2766 |
# answer. `DISCOVERY_TRIGGER_KIND` is that map, and a gate asserts it agrees with law 1.
|
| 2767 |
+
# β TRACKED because it is the ONE key several `ut_ensure` calls in this pass can share: the
|
| 2768 |
+
# discovery arm and both enrich arms all ensure `target`. A later one must not answer out of
|
| 2769 |
+
# a snapshot an earlier one invalidated β see the `tables=` arguments below.
|
| 2770 |
+
wrote_target = False
|
| 2771 |
+
_dkind = inputs["discoveryKind"]
|
| 2772 |
if _dkind:
|
| 2773 |
_, _, _spawn_label, _spawn_fields = discovery_facts(_dkind)
|
| 2774 |
+
ut_ensure(rt, inputs["label"] or _spawn_label, _spawn_fields, username,
|
| 2775 |
+
key=target, flow_tag=tag, lock_fields=True, tables=tables)
|
| 2776 |
+
# ββ WAVE 31 Β· T34 (D-178) β AND THE `return` THAT USED TO BE HERE IS GONE.
|
| 2777 |
+
#
|
| 2778 |
+
# β WHAT IT COST: a DISCOVERY automation could never spawn its child databases. The
|
| 2779 |
+
# capture switches below are the ones that promise `ut_tt_posts` / `ut_tt_comments` /
|
| 2780 |
+
# `ut_tt_post_snapshots` at SAVE, before any money is spent (W30-T10) β and only a
|
| 2781 |
+
# `plain` flow ever reached them, because a discovery flow left this function three
|
| 2782 |
+
# lines earlier. So the exact automation the owner would build for TikTok discovery β
|
| 2783 |
+
# find profiles, then capture their posts β silently got the profile table and nothing
|
| 2784 |
+
# else, and the toggles it showed were promises the save path never kept.
|
| 2785 |
+
# β IT IS A TRAP, NOT TODAY'S ONLY SYMPTOM: nurilab's child tables are absent for a
|
| 2786 |
+
# CONFIGURATION reason as well, so fixing this alone will not make them appear there.
|
| 2787 |
+
#
|
| 2788 |
+
# β THE SNAPSHOT IS NOW STALE FOR `target`, and the arms below resolve their profile
|
| 2789 |
+
# binding out of that very table (`bound` reads its `fields`). One re-read, paid only on
|
| 2790 |
+
# a save that actually reached the discovery spawn β correctness before the read count,
|
| 2791 |
+
# and it is one read against the 41 this function used to take. `wrote_target` stays
|
| 2792 |
+
# False precisely BECAUSE we refreshed: it tracks staleness, not "did somebody write".
|
| 2793 |
+
tables = ut_all(rt)
|
| 2794 |
+
enrich_actions = ig_actions
|
| 2795 |
+
table = tables.get(target) or {}
|
| 2796 |
+
named = next((a["profileField"] for a in enrich_actions if a["profileField"]), "")
|
| 2797 |
bound = next((str(f.get("key") or "") for f in table.get("fields") or []
|
| 2798 |
if isinstance(f.get("profile"), dict)), "") or named
|
| 2799 |
if enrich_actions and bound:
|
| 2800 |
ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username,
|
| 2801 |
+
key=target, flow_tag=tag, lock_fields=True, tables=tables)
|
| 2802 |
+
wrote_target = True
|
| 2803 |
# β WAVE 30 Β· T08 β the TikTok half of W25/R10: the columns an `enrich_tiktok` step will
|
| 2804 |
# fill appear when the automation is SAVED, not when money is first spent. Resolved
|
| 2805 |
# through `profile_field_key(..., source=PROFILE_SOURCE_TT)` so it cannot adopt an
|
| 2806 |
# Instagram binding, and through the SAME `_tt_profile_schema_for` the runner's own top-up
|
| 2807 |
# uses β R10's rule is that the column set does not change between Save and the first run,
|
| 2808 |
# and one shared resolver is what makes that structural instead of a convention.
|
|
|
|
|
|
|
| 2809 |
if tt_actions:
|
| 2810 |
+
tt_named = next((a["profileField"] for a in tt_actions if a["profileField"]), "")
|
|
|
|
|
|
|
| 2811 |
tt_bound = profile_field_key(table, tt_named, source=PROFILE_SOURCE_TT)
|
| 2812 |
if tt_bound:
|
| 2813 |
ut_ensure(rt, table.get("label") or target, _tt_profile_schema_for(tt_bound),
|
| 2814 |
+
username, key=target, flow_tag=tag, lock_fields=True,
|
| 2815 |
+
tables=None if wrote_target else tables)
|
| 2816 |
# β WAVE 30 Β· T10 β and the CHILD databases the step's own switches promise, at
|
| 2817 |
# SAVE, before any money. R10's rule is "the columns a step will fill appear when
|
| 2818 |
# you press Save"; a `postMetrics` toggle whose database only exists after a paid
|
|
|
|
| 2820 |
# β GATED ON THE SWITCHES, never spawned unconditionally: a database that exists
|
| 2821 |
# because somebody looked at a toggle, and then never fills, is the "SECOND, empty
|
| 2822 |
# database" the discovery default was rewritten to stop producing.
|
|
|
|
| 2823 |
for _flag, _child in (("postMetrics", TT_POSTS_TABLE),
|
| 2824 |
("postMetrics", TT_POST_SNAPSHOTS_TABLE),
|
| 2825 |
("commentMetrics", TT_COMMENTS_TABLE)):
|
| 2826 |
+
if any(a[_flag] for a in tt_actions):
|
| 2827 |
+
# R9 (W31-T32): the child arrives LOCKED, from its own declaration.
|
| 2828 |
ut_ensure(rt, TT_TABLE_LABELS[_child], TT_TABLE_FIELDS[_child], username,
|
| 2829 |
+
key=_child, flow_tag=tag, lock_fields=True, tables=tables,
|
| 2830 |
+
record_mode=tt_record_mode(_child))
|
| 2831 |
except Exception as e: # noqa: BLE001
|
| 2832 |
# β NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so
|
| 2833 |
# raising here would answer 500 for an automation that IS stored β the caller would retry
|
|
|
|
| 2860 |
# stricter than a row insert. [[sync-write-eats-pending-async-write]] is the hazard in the
|
| 2861 |
# OTHER direction (a sync writer discarding a pending async write) and `Store.update`'s dirty
|
| 2862 |
# branch already handles it.
|
| 2863 |
+
_store_update(rt, _up, flush="async")
|
| 2864 |
# C8 β the fields this flow tagged are DISABLED with the reason on them, never orphaned
|
| 2865 |
# silently: the column keeps its values and its config, and says why it stopped. Scanned
|
| 2866 |
# first so a delete with no tagged fields costs no store commit.
|
|
|
|
| 2869 |
# update below reads it a second time. Removing that needs a tagged-field index, which is a
|
| 2870 |
# feature and not a wave-tail edit β booked rather than improvised. What it no longer costs
|
| 2871 |
# is the part the owner could feel: the blocking network commits.
|
| 2872 |
+
# β WAVE 31 Β· T39(a) β D-177(a): ONE WALK, AND THE PLAN CARRIES WHAT IT FOUND.
|
| 2873 |
+
# This used to scan the whole document to answer a boolean (`hit`), then walk the WHOLE
|
| 2874 |
+
# document again inside `_disable` to re-derive the same matches. The scan now records the
|
| 2875 |
+
# `(table, field)` pairs it found and the mutation applies exactly those β the shape
|
| 2876 |
+
# `bind_unbound_fields` in this module already uses, so a delete costs one walk instead of two
|
| 2877 |
+
# and the write touches only the fields it named.
|
| 2878 |
+
# β HONEST RESIDUAL, STATED RATHER THAN SMOOTHED (R6's second sentence): this still costs ONE
|
| 2879 |
+
# full read of the tenant's `user_tables` document, and `rt.update` below takes its own strict
|
| 2880 |
+
# read. **That second read cannot be removed here** β it belongs to the write, and answering
|
| 2881 |
+
# "which fields did this flow tag?" without a scan needs a tagged-field INDEX, which is a
|
| 2882 |
+
# feature and not a wave-tail edit. It is booked as D-179; what changed is the second WALK,
|
| 2883 |
+
# not the read count.
|
| 2884 |
+
plan = [(tk, str(f.get("key") or ""))
|
| 2885 |
+
for tk, t in ut_all(rt).items()
|
| 2886 |
+
for f in ((t or {}).get("fields") or [])
|
| 2887 |
+
if str((f.get("automation") or {}).get("flowId") or "") == aid]
|
| 2888 |
+
if not plan:
|
| 2889 |
return
|
| 2890 |
|
| 2891 |
def _disable(cur):
|
| 2892 |
cur = cur if isinstance(cur, dict) else {}
|
| 2893 |
+
note = f"its automation was deleted {_stamp()}"
|
| 2894 |
+
for tk, fk in plan:
|
| 2895 |
+
for f in ((cur.get(tk) or {}).get("fields") or []):
|
| 2896 |
a = f.get("automation")
|
| 2897 |
+
if f.get("key") == fk and isinstance(a, dict):
|
| 2898 |
a["disabled"] = True
|
| 2899 |
+
a["statusNote"] = note
|
| 2900 |
return cur
|
| 2901 |
# The expensive half of the owner's complaint: a whole-document rewrite, committed while the
|
| 2902 |
# person waits. Async for the reason given above β the disable is visible to the next read
|
|
|
|
| 2998 |
f"{entry['summary'][:120]}")
|
| 2999 |
return cur
|
| 3000 |
|
| 3001 |
+
_store_update(rt, _up, flush="sync")
|
| 3002 |
_notify_outcome(rt, auto_id, state, entry)
|
| 3003 |
return entry
|
| 3004 |
|
|
|
|
| 3543 |
TT_POST_SNAPSHOTS_TABLE: "TikTok post measurements",
|
| 3544 |
TT_COMMENTS_TABLE: "TikTok comments",
|
| 3545 |
}
|
| 3546 |
+
#: ββ WAVE 31 Β· OWNER RULING R9 β the `ut_tt_*` children a person may not type records into.
|
| 3547 |
+
#: Owner, verbatim: *"Tiktok database for post and comments and their snapshots should also be a
|
| 3548 |
+
#: locked database with the lock icon, exactly like how instagram"*.
|
| 3549 |
+
#:
|
| 3550 |
+
#: β DECLARED ONCE BECAUSE THE DEFECT WAS TWO CALL SITES DISAGREEING WITH A THIRD. `ensure_ig_graph`
|
| 3551 |
+
#: passes `record_mode=AUTOMATION_RECORD_MODE` for all four Instagram children; TikTok's two spawn
|
| 3552 |
+
#: sites β `_spawn_presets`' child loop (SAVE) and `_tt_write_tables` (RUN) β each passed
|
| 3553 |
+
#: `lock_fields=True` and no `record_mode` at all. That is the whole bug: not a missing feature, one
|
| 3554 |
+
#: argument missing at two places, which is exactly the shape `_tt_write_tables`' own docstring warns
|
| 3555 |
+
#: about (*"one function, two callers, so the inline and deferred paths cannot answer differently"*).
|
| 3556 |
+
#: A SET plus a resolver means a sixth `ut_tt_*` table joins by declaration, not by remembering.
|
| 3557 |
+
#:
|
| 3558 |
+
#: β FOUR, NOT THE THREE R9 ENUMERATES, and the fourth is argued rather than assumed.
|
| 3559 |
+
#: `ut_tt_snapshots` is the profile measurement series β the exact twin of `ut_ig_snapshots`, which
|
| 3560 |
+
#: IS locked. R9's own comparison is *"exactly like how instagram"*, and the alternative is to leave
|
| 3561 |
+
#: a machine-append time series that a person can type into, one table away from three that they
|
| 3562 |
+
#: cannot. That is the fix-applied-to-one-platform-and-not-its-twin shape this very wave is closing
|
| 3563 |
+
#: elsewhere (D-177(c)). `ut_tt_profile` is deliberately ABSENT: it is the database a human adds
|
| 3564 |
+
#: handles to, and its Instagram counterpart is not locked either.
|
| 3565 |
+
#:
|
| 3566 |
+
#: β LOCKED DATABASE, NOT READ-ONLY (DESIGN.md Β§4): `records_mutable` goes False, so records cannot
|
| 3567 |
+
#: be added/edited/deleted β **adding a FIELD must still work**, and a UI hiding add-field here is a
|
| 3568 |
+
#: defect, not the intent.
|
| 3569 |
+
TT_LOCKED_TABLES = frozenset({TT_SNAPSHOTS_TABLE, TT_POSTS_TABLE, TT_POST_SNAPSHOTS_TABLE,
|
| 3570 |
+
TT_COMMENTS_TABLE})
|
| 3571 |
+
|
| 3572 |
+
|
| 3573 |
+
def tt_record_mode(table_key):
|
| 3574 |
+
"""R9's answer for one `ut_tt_*` key β the `record_mode` its `ut_ensure` must carry.
|
| 3575 |
+
|
| 3576 |
+
`""` for anything outside the locked set, which is what `ut_ensure` already treats as "say
|
| 3577 |
+
nothing about record mode", so an unlocked table is untouched rather than explicitly opened.
|
| 3578 |
+
"""
|
| 3579 |
+
return AUTOMATION_RECORD_MODE if str(table_key) in TT_LOCKED_TABLES else ""
|
| 3580 |
|
| 3581 |
|
| 3582 |
# ---------------------------------------------------------------------------------------------
|
|
|
|
| 5189 |
# exist, or that is not an Instagram profile table, has nothing to migrate.
|
| 5190 |
detected = set(_ig_profile_tables(rt))
|
| 5191 |
targets = ([only] if only in detected else []) if only else sorted(detected)
|
| 5192 |
+
# ββ WAVE 31 Β· T30 β `only` NOW NARROWS THE WHOLE FUNCTION, AND IT DID NOT.
|
| 5193 |
+
#
|
| 5194 |
+
# β THE BUG WAS THAT `only` NARROWED THE LOOP ABOVE AND NOTHING ELSE. The tenant-wide passes
|
| 5195 |
+
# at the bottom β two full `_ig_schema_contract` + `_reconcile_ig_graph_fields` rounds and two
|
| 5196 |
+
# backfills β ignore `only` entirely, so `migrate_ig_tables(only=X)` paid a whole-tenant
|
| 5197 |
+
# Instagram reconciliation no matter what X was. And `ut_ensure` calls it that way on EVERY
|
| 5198 |
+
# ensure whose field set intersects `PRESET_PROFILE_KEYS` β which TikTok's profile schema does,
|
| 5199 |
+
# because the two networks share column names.
|
| 5200 |
+
# MEASURED on the wave-30 fixture, `only="ut_tt_profile"`: `_ig_profile_tables` returns `[]`,
|
| 5201 |
+
# `stats` comes back completely empty and ZERO store writes happen β at a cost of **10
|
| 5202 |
+
# whole-document reads**, each a `Store.get` deep copy (documented ceiling 35.8 MB / ~1.4 s).
|
| 5203 |
+
# Across one save's four `ut_ensure` calls that was **40 of the 41** reads behind owner item 7's
|
| 5204 |
+
# *"it just say Saving... and takes a long time"*. The migration was not slow; it was running at
|
| 5205 |
+
# all.
|
| 5206 |
+
# β THE COMMENT ONE SCREEN DOWN IS WHY THIS WENT UNSEEN FOR THREE WAVES: *"Both are guarded on
|
| 5207 |
+
# 'is there anything to do' and cost a dict scan in the steady state, which is what makes them
|
| 5208 |
+
# safe on a function that rides every `ut_ensure`."* That guard is on the WRITE. The read was
|
| 5209 |
+
# never guarded, and a scan of a freshly deep-copied 35 MB document is not a dict scan
|
| 5210 |
+
# [[read-a-gates-predicate-for-what-it-excludes]].
|
| 5211 |
+
# β SCOPED TO THE PROVEN-EMPTY CASE ONLY, deliberately. When `only` IS a detected profile table
|
| 5212 |
+
# the function is unchanged, tenant-wide passes included β that is the Instagram path every
|
| 5213 |
+
# migration check in `verify_automation` exercises. And a caller passing no `only` still gets
|
| 5214 |
+
# the full sweep. The claim being made here is narrow and checkable: *a table that is not in the
|
| 5215 |
+
# Instagram graph has no Instagram migration*, which is the same thing the `detected` test above
|
| 5216 |
+
# already decided one line earlier and then threw away.
|
| 5217 |
+
# β AND IT IS NOT A LEND. Lending one snapshot through the rest of this function would be the
|
| 5218 |
+
# obvious next fix and it is WRONG: the second contract pass exists precisely because it must
|
| 5219 |
+
# read what the first pass WROTE (see its own note below), so a read-write-read sequence cannot
|
| 5220 |
+
# answer out of one snapshot without silently un-fixing that idempotency.
|
| 5221 |
+
if only and not targets:
|
| 5222 |
+
return stats
|
| 5223 |
for table_key in targets:
|
| 5224 |
tbl = ut_get(rt, table_key) or {}
|
| 5225 |
fields = [dict(f) for f in tbl.get("fields") or []]
|
|
|
|
| 7155 |
#: than a second, smaller number nobody could explain: a plain automation's records ARE its
|
| 7156 |
#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a
|
| 7157 |
#: flow could be pointed at and which would not finish.
|
| 7158 |
+
#: β W31-T36 (D-143 / R6) β THIS IS NOW THE FALLBACK ONLY: what a flow may walk when the table's
|
| 7159 |
+
#: own limit cannot be resolved (no runtime, or a key `core.user_tables` does not know). The real
|
| 7160 |
+
#: answer is per TABLE and comes from `core.user_tables.row_limit`; see `_flow_record_cap`.
|
| 7161 |
+
#: β Do not re-point a caller at this constant β a private copy of "is this database bounded" is
|
| 7162 |
+
#: exactly the second evaluator D-143 was.
|
| 7163 |
MAX_FLOW_RECORDS = MAX_UT_ROWS
|
| 7164 |
|
| 7165 |
|
| 7166 |
+
def _flow_record_cap(table_key, rt=None):
|
| 7167 |
+
"""How many records may a flow walk in this database? `None` means NO CAP (R6).
|
| 7168 |
+
|
| 7169 |
+
β DELEGATES, NEVER DECIDES. `core.user_tables.row_limit` is the one evaluator for "is this
|
| 7170 |
+
database bounded", and it distinguishes three states this module must not re-derive: `0`
|
| 7171 |
+
(read-through β its rows are not in the document at all), `None` (connected and UNCAPPED, which
|
| 7172 |
+
is R6's whole point), and `MAX_ROWS` (the editable substrate, genuinely bounded by the shared
|
| 7173 |
+
document). A second copy here is exactly the defect D-143 IS.
|
| 7174 |
+
|
| 7175 |
+
β `0` MEANS "NO ROWS LIVE HERE", NOT "WALK NOTHING". A read-through grid's rows arrive through
|
| 7176 |
+
the mirror route, so a flow that reached this function already holds ids from somewhere else;
|
| 7177 |
+
capping it at zero would refuse a walk whose records are in hand. Treated as uncapped, and the
|
| 7178 |
+
honest place to fix a read-through walk is the walker.
|
| 7179 |
+
"""
|
| 7180 |
+
try:
|
| 7181 |
+
import core.user_tables as _ut_cap
|
| 7182 |
+
cap = _ut_cap.row_limit(str(table_key or ""), st=rt)
|
| 7183 |
+
except Exception: # noqa: BLE001
|
| 7184 |
+
return MAX_FLOW_RECORDS # the fallback, and it is the conservative direction
|
| 7185 |
+
return None if not cap else int(cap)
|
| 7186 |
+
|
| 7187 |
+
|
| 7188 |
+
def _flow_cap_reason(table_key, rt=None):
|
| 7189 |
+
"""R6's SECOND sentence, appended to the run's own note: the cause and the recommended fix.
|
| 7190 |
+
|
| 7191 |
+
β R6 IS TWO SENTENCES AND THE SECOND IS THE ONE THAT GETS DROPPED β *"if there is lag or it
|
| 7192 |
+
can't be done, you need to explicitly tell me why and recommend a fix."* A run that says only
|
| 7193 |
+
*"walked the first 5000"* has disclosed the number and hidden everything a person could act on.
|
| 7194 |
+
`core.user_tables.limit_report` already carries both, so this READS them rather than writing a
|
| 7195 |
+
second wording that would drift from the one the grid shows.
|
| 7196 |
+
"""
|
| 7197 |
+
try:
|
| 7198 |
+
import core.user_tables as _ut_cap
|
| 7199 |
+
rep = _ut_cap.limit_report(str(table_key or ""), st=rt) or {}
|
| 7200 |
+
except Exception: # noqa: BLE001
|
| 7201 |
+
return ""
|
| 7202 |
+
cause, fix = str(rep.get("cause") or ""), str(rep.get("recommendation") or "")
|
| 7203 |
+
if not cause and not fix:
|
| 7204 |
+
return ""
|
| 7205 |
+
return f" β {cause}." + (f" To walk them all: {fix}." if fix else "")
|
| 7206 |
+
|
| 7207 |
+
|
| 7208 |
def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None):
|
| 7209 |
"""β WAVE 24 / R6 β the runner for an automation with NO machine step: its flow IS the whole
|
| 7210 |
automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special
|
|
|
|
| 7270 |
ids = selected
|
| 7271 |
step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}")
|
| 7272 |
counts = {"records": len(ids)}
|
| 7273 |
+
# ββ WAVE 31 Β· T36 (D-143, owner ruling R6) β THE CAP IS THE TABLE'S, NOT THIS MODULE'S.
|
| 7274 |
+
#
|
| 7275 |
+
# β WHAT WAS WRONG, and it was a SECOND EVALUATOR rather than a wrong number. This line read
|
| 7276 |
+
# `MAX_FLOW_RECORDS = MAX_UT_ROWS = 5000` β a constant mirrored from the EDITABLE substrate's
|
| 7277 |
+
# ceiling β and applied it to every database alike. `ut_odoo_orders` is a CONNECTED source
|
| 7278 |
+
# (32,826 rows live), and R6 is explicit that connected-source data has NO CAP: *"I thought we
|
| 7279 |
+
# decided there is no cap in how many data from the API source β¦ can be pulled into the app."*
|
| 7280 |
+
# So a flow over it silently walked 5,000 of 32,826 β 15% of the records β and every total it
|
| 7281 |
+
# produced understated the book while looking exactly like a complete run.
|
| 7282 |
+
#
|
| 7283 |
+
# β `core.user_tables.row_limit` IS THE ONE EVALUATOR for this question and it already answers
|
| 7284 |
+
# the three cases (0 = read-through, None = connected and UNCAPPED, MAX_ROWS = editable). Using
|
| 7285 |
+
# it here retires this module's private copy instead of correcting it β one question, one
|
| 7286 |
+
# normaliser [[one-question-two-normalizers]]. `limit_report` is R6's SECOND sentence already
|
| 7287 |
+
# expressed as data, so the sentence a person reads carries the cause and the recommended fix
|
| 7288 |
+
# rather than just a number.
|
| 7289 |
+
_cap = _flow_record_cap(table, rt)
|
| 7290 |
+
if _cap is not None and len(ids) > _cap:
|
| 7291 |
# The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names
|
| 7292 |
+
# both numbers so "it only processed some of them" is readable rather than deducible β and
|
| 7293 |
+
# since T36, WHY it applies and what to do about it.
|
| 7294 |
total = len(ids)
|
| 7295 |
+
ids = ids[:_cap]
|
| 7296 |
return ("partial",
|
| 7297 |
+
f"{total} records in {table} β this run walked the first {_cap}"
|
| 7298 |
+
+ _flow_cap_reason(table, rt), counts, ids, {})
|
| 7299 |
n = len(ids)
|
| 7300 |
# ββ 2026-08-07 (owner report) β SAY IT IN WORDS A PERSON CAN CHECK.
|
| 7301 |
# Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)?
|
|
|
|
| 7707 |
# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we
|
| 7708 |
# had added. `migrate_ig_tables` now drops those columns wherever an automation created them.
|
| 7709 |
#
|
| 7710 |
+
# ββ `notify_review` IS GONE FROM HERE AND ITS CONSUMER IS STILL ALIVE β DEBT D-101, and this
|
| 7711 |
+
# paragraph is its TOMBSTONE, placed where the next reader of the review machinery will look.
|
| 7712 |
+
#
|
| 7713 |
+
# `notify_review` was the producer of the `automation_review` notification: a card reaching a
|
| 7714 |
+
# review stage told a human to come and look. It had ZERO callers even before R3 (wave-27 session
|
| 7715 |
+
# C found it and correctly left it alone as another lane's subject), and R3's deletion took the
|
| 7716 |
+
# function with the rest of the board. `verify_automation`'s W23-B section asserts its absence by
|
| 7717 |
+
# name, so it cannot come back by accident.
|
| 7718 |
+
#
|
| 7719 |
+
# β THE SWEEP FOR ITS READERS, RUN 2026-08-12 (W31-T37) AND REPORTED RATHER THAN ASSUMED EMPTY:
|
| 7720 |
+
# Β· SERVER β `notify_review` appears in exactly ONE file, `verify_automation.py`, inside the
|
| 7721 |
+
# list of names asserted GONE. **No `.py` file anywhere produces an `automation_review`
|
| 7722 |
+
# notification**; `core.alerts.notify` defaults `topic='automation'`, which is a different,
|
| 7723 |
+
# live shape (the run-outcome alert).
|
| 7724 |
+
# Β· CLIENT β the branch is FULLY ALIVE and is NOT in this fence:
|
| 7725 |
+
# `web/src/alerts/alertsModel.ts` (`AUTOMATION_REVIEW_KIND`, `isAutomationReview`, the
|
| 7726 |
+
# `autoId` guard) and `web/src/alerts/AlertsPane.tsx` (two call sites), plus legs in
|
| 7727 |
+
# `web/verify_alerts.py`. All four are session B's files.
|
| 7728 |
+
# β **The producer is deleted; the consumer guards an event that can never arrive.** D-101's own
|
| 7729 |
+
# exit condition says these are one defect seen from two ends and must be closed in ONE change β
|
| 7730 |
+
# so this half is the tombstone plus the sweep, and the client half is routed to B rather than
|
| 7731 |
+
# reached across a fence. If a future wave gives review stages a real producer, THIS is the note
|
| 7732 |
+
# that says the client already knows how to render it.
|
| 7733 |
+
#
|
| 7734 |
# β WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE:
|
| 7735 |
# * `lane_match` + `LANE_OPS` + the condition trees below β never board-only. They are the
|
| 7736 |
# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was
|
|
|
|
| 7907 |
MAX_BRANCHES = 6 # per If / then
|
| 7908 |
MAX_ACTION_VALUES = 20 # cells one update/create action may write
|
| 7909 |
FIND_LIMIT_MAX = 100
|
| 7910 |
+
#: ββ W31-T38 / E-4 β HOW MANY BROWSER JOBS ONE RUN MAY SUBMIT, and it belongs HERE because this
|
| 7911 |
+
#: is the only layer that knows a record WALK is happening. `web_agent.MAX_STEPS` bounds steps per
|
| 7912 |
+
#: JOB; nothing bounded jobs per RUN, and the web arm submits one job PER RECORD. A `web_read` on a
|
| 7913 |
+
#: flow over the Customer grid is therefore thousands of ~10 s submissions against HF's
|
| 7914 |
+
#: 6-concurrent cap β a run whose wall-clock is `records x 10 s` and a burst the `/hf-jobs` rules
|
| 7915 |
+
#: exist to prevent. Session E raised it and could not fix it: the seam sees one step and cannot
|
| 7916 |
+
#: know a walk is happening.
|
| 7917 |
+
#: β 50 IS A STATED CHOICE, NOT A MEASUREMENT: at E's measured ~9-32 s per job that is roughly
|
| 7918 |
+
#: 8-27 minutes of serial browsing, which is a long automation run and not a runaway one. It is
|
| 7919 |
+
#: DISCLOSED when it binds (R6's second sentence β see the web arm), never silently applied.
|
| 7920 |
+
#: β THE REAL FIX IS BATCHING, NOT A BIGGER NUMBER: `web_agent.run_plan(steps, ctx)` already takes
|
| 7921 |
+
#: a list and the cost is per JOB, so a flow's web steps collected into ONE call would make this
|
| 7922 |
+
#: ceiling nearly unreachable. That is E's own PENDING row; this bound is what stops the damage
|
| 7923 |
+
#: until it lands.
|
| 7924 |
+
MAX_WEB_JOBS_PER_RUN = 50
|
| 7925 |
+
#: β W31-T38 (C5) β the web step's own clamps. `20000` matches the default E's seam documents;
|
| 7926 |
+
#: the ceiling exists because this blocks the record walk (~9-32 s of cold start ALREADY), and an
|
| 7927 |
+
#: automation that can be configured to wait ten minutes per record is a stalled run, not a slow one.
|
| 7928 |
+
WEB_READ_TIMEOUT_MS = 20000
|
| 7929 |
+
WEB_READ_TIMEOUT_MAX_MS = 120000
|
| 7930 |
#: β WAVE 24 Β· C-ACT β THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder`
|
| 7931 |
#: and never by a literal list of group names: a client-side ordering is a second copy of this
|
| 7932 |
#: table, and the way it fails is that a group added here renders last, or not at all, with
|
|
|
|
| 7950 |
"detail": "Needs the recorder that captures a selector instead of a coordinate"},
|
| 7951 |
{"kind": "web_click", "label": "Click something", "group": "Web action", "ready": False,
|
| 7952 |
"detail": "Needs the recorder that captures a selector instead of a coordinate"},
|
| 7953 |
+
# ββ WAVE 31 Β· T38 (C5, owner ruling R10) β THE ONE ROW THAT FLIPS. `web_read` is the
|
| 7954 |
+
# unattended READ: it fetches a real page in an ephemeral HF Job and writes one value into one
|
| 7955 |
+
# column. Its four siblings stay `ready:False` because each WRITES to a third party, and D-51/R5
|
| 7956 |
+
# defaults any such action to needs-approval β a gate this wave does not build. The runner
|
| 7957 |
+
# refuses them by kind as well, so flipping one here would ship an action that is addable and
|
| 7958 |
+
# inert, which is worse than a faded row that says why.
|
| 7959 |
+
# β THE FADED ROW IS A WALL, NOT A STYLE: `clean_actions` refuses an unready kind with a
|
| 7960 |
+
# sentence, so this flag is what makes the action storable at all.
|
| 7961 |
+
{"kind": "web_read", "label": "Read from the page", "group": "Web action", "ready": True,
|
| 7962 |
+
"detail": "Reads one value off a live page in a browser job β expect ~10-30 s per step"},
|
| 7963 |
{"kind": "web_repair", "label": "Repair a broken step", "group": "Web action",
|
| 7964 |
"ready": False,
|
| 7965 |
"detail": "Self-healing selectors arrive with the browser engine (Scrapling)"},
|
|
|
|
| 8335 |
if limit < 1 or limit > FIND_LIMIT_MAX:
|
| 8336 |
return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}"
|
| 8337 |
return {"table": table, "cond": cond, "limit": limit}, None
|
| 8338 |
+
if kind == "web_read":
|
| 8339 |
+
# ββ WAVE 31 Β· T38 (C5 / R10) β AND THIS ARM IS THE HALF THAT WOULD HAVE SHIPPED MISSING.
|
| 8340 |
+
#
|
| 8341 |
+
# β FOUND BY THE GATE, NOT BY REVIEW, AND IT IS THE TICKET'S OWN FAILURE MODE. Flipping
|
| 8342 |
+
# `ready:True` and mounting the dispatch arm is not enough: with no branch here the
|
| 8343 |
+
# function falls through to `return {}, None` below, so a stored `web_read` action keeps
|
| 8344 |
+
# its KIND and loses its `url`, `selector` and `field` **silently, with no error anywhere**.
|
| 8345 |
+
# The action would be addable, storable, runnable β and would fetch `""` forever. That is
|
| 8346 |
+
# precisely the *"addable and inert"* outcome E's hand-off warned about, arriving through a
|
| 8347 |
+
# different door than the one being watched, and the same silent-drop class as `patch`'s
|
| 8348 |
+
# hand-maintained key list one screen up.
|
| 8349 |
+
#
|
| 8350 |
+
# β THE URL IS NOT `guard`ed HERE, on purpose. `guard` does DNS, and this is a validator
|
| 8351 |
+
# that runs on every save of every automation β a save must not block on name resolution,
|
| 8352 |
+
# and a template like `https://{Domain}/p` is not resolvable until a record supplies the
|
| 8353 |
+
# value. The rail is enforced where the fetch happens (the job, and `web_agent`'s own
|
| 8354 |
+
# refusal), which is the only place the FINAL url exists. What is enforced here is the
|
| 8355 |
+
# shape: http(s) or an interpolation that could become one.
|
| 8356 |
+
# ββ AN EMPTY CONFIG IS STORED, NOT REFUSED β THE A3 STORED-INERT PATTERN, and the first
|
| 8357 |
+
# draft of this arm got it wrong in a way only a gate could see. Requiring `url` /
|
| 8358 |
+
# `selector` / `field` here makes the action UNADDABLE: `AutomationBuilder::seedFor` mints
|
| 8359 |
+
# a new step from a seed and persists it immediately, so a validator that refuses an empty
|
| 8360 |
+
# seed yields a red banner and no card β the wave-23 illegal-default-seed defect, caught by
|
| 8361 |
+
# `verify_automation_ui`'s catalog-derived seed check the moment `web_read` became ready.
|
| 8362 |
+
# β The same posture the enrich arm already takes with `profileField`: a step may be added
|
| 8363 |
+
# to a flow before it is configured, and it FAILS CLOSED AT RUN with a sentence naming what
|
| 8364 |
+
# is missing. What is refused here is a value somebody actually TYPED and got wrong β never
|
| 8365 |
+
# the absence of one they have not typed yet.
|
| 8366 |
+
url = _s(cfg.get("url"), 500).strip()
|
| 8367 |
+
# β `{{` IS THE TEMPLATE FORM β `interpolate`'s syntax is `{{field_key}}`, NOT `{field}`.
|
| 8368 |
+
# A url that STARTS with a placeholder is legitimate (the record supplies the host), so the
|
| 8369 |
+
# shape test admits it; anything else must be an absolute http(s) address.
|
| 8370 |
+
if url and not (url.startswith("http://") or url.startswith("https://")
|
| 8371 |
+
or url.startswith("{{")):
|
| 8372 |
+
return None, "a web address must start with http:// or https://"
|
| 8373 |
+
selector = _s(cfg.get("selector"), 200).strip()
|
| 8374 |
+
field = _s(cfg.get("field"), 80).strip()
|
| 8375 |
+
try:
|
| 8376 |
+
timeout = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS)
|
| 8377 |
+
except (TypeError, ValueError):
|
| 8378 |
+
return None, "the web step timeout must be a whole number of milliseconds"
|
| 8379 |
+
timeout = max(1000, min(timeout, WEB_READ_TIMEOUT_MAX_MS))
|
| 8380 |
+
out = {"url": url, "selector": selector, "field": field,
|
| 8381 |
+
"attr": _s(cfg.get("attr"), 40).strip() or "text",
|
| 8382 |
+
"all": bool(cfg.get("all")), "timeoutMs": timeout}
|
| 8383 |
+
wait = _s(cfg.get("waitFor"), 200).strip()
|
| 8384 |
+
if wait:
|
| 8385 |
+
out["waitFor"] = wait
|
| 8386 |
+
return out, None
|
| 8387 |
return {}, None
|
| 8388 |
|
| 8389 |
|
|
|
|
| 8940 |
return out
|
| 8941 |
|
| 8942 |
|
| 8943 |
+
def _web_agent():
|
| 8944 |
+
"""C5's seam, resolved LAZILY β `web_agent.run_step(step, ctx) -> (dict|None, str)`.
|
| 8945 |
+
|
| 8946 |
+
β IMPORTED INSIDE THE CALL, like every `connectors_tt` site in this module, and here it also
|
| 8947 |
+
buys a failure mode worth having: if `web_agent.py` is ever missing from a deployment, the
|
| 8948 |
+
engine still imports and every OTHER action still runs β the web step alone reports a sentence.
|
| 8949 |
+
A module-level import would turn one absent file into a dead automation module.
|
| 8950 |
+
|
| 8951 |
+
β AND THE ABSENCE IS REPORTED, NEVER SWALLOWED (R6's second sentence). The shim below answers
|
| 8952 |
+
the same `(None, sentence)` contract the real seam does, so the caller's `if why:` branch is
|
| 8953 |
+
the only branch there has ever been.
|
| 8954 |
+
"""
|
| 8955 |
+
try:
|
| 8956 |
+
import web_agent
|
| 8957 |
+
return web_agent
|
| 8958 |
+
except Exception as exc: # noqa: BLE001
|
| 8959 |
+
class _Absent:
|
| 8960 |
+
@staticmethod
|
| 8961 |
+
def run_step(_step, _ctx=None):
|
| 8962 |
+
return None, ("The web-browsing agent is not available in this deployment "
|
| 8963 |
+
f"({type(exc).__name__}). Nothing was read.")
|
| 8964 |
+
return _Absent
|
| 8965 |
+
|
| 8966 |
+
|
| 8967 |
def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print,
|
| 8968 |
step=_no_step):
|
| 8969 |
"""Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts.
|
|
|
|
| 8991 |
# server still refuses to let it be removed.
|
| 8992 |
if ig_action_pinned(defn, 0) and actions:
|
| 8993 |
actions = actions[1:]
|
| 8994 |
+
# β `webRead` / `webBlocked` JOIN HERE (W31-T38, C5): declared up front like every other
|
| 8995 |
+
# counter, so a run that read nothing and a run with no web step are DIFFERENT numbers. R6's
|
| 8996 |
+
# second sentence is a reporting rule, and a counter that only exists once it is non-zero
|
| 8997 |
+
# cannot report a refusal.
|
| 8998 |
+
counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0,
|
| 8999 |
+
"webRead": 0, "webBlocked": 0}
|
| 9000 |
+
#: W31-T38 β the once-per-run markers the web arm uses, so an unconfigured step says so ONCE
|
| 9001 |
+
#: rather than once per record. Same shape as the enrich accumulators' `notes`.
|
| 9002 |
+
web_notes = []
|
| 9003 |
# β C4 β THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected
|
| 9004 |
# across the WHOLE walk and written once at the end, for the reason the module header states
|
| 9005 |
# and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
|
|
|
|
| 9176 |
# the table would silently apply one action's uniqueness rule to the other's rows.
|
| 9177 |
creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals)
|
| 9178 |
counts["created"] += 1
|
| 9179 |
+
elif kind == "web_read":
|
| 9180 |
+
# ββ WAVE 31 Β· T38 (C5, owner ruling R10) β THE WEB-BROWSING AGENT'S ONE LIVE ARM.
|
| 9181 |
+
#
|
| 9182 |
+
# β WHY THIS ARM EXISTS SEPARATELY FROM THE RUNNER: session E built
|
| 9183 |
+
# `web_agent.run_step` and **could not verify its own mounting**. An unmounted
|
| 9184 |
+
# runner is a whole, correct, unreachable feature β the exact class five wave-29
|
| 9185 |
+
# features shipped as β so the mount and its `verify_wiring` row are C's, in one
|
| 9186 |
+
# change. `web_read` is the ONLY kind flipped to `ready:True`;
|
| 9187 |
+
# `web_goto`/`web_fill`/`web_click`/`web_repair` WRITE to a third party and D-51/R5
|
| 9188 |
+
# defaults those to needs-approval, which this wave does not build. The runner also
|
| 9189 |
+
# refuses them by kind, so flipping one would ship an addable, inert action.
|
| 9190 |
+
#
|
| 9191 |
+
# β IT BLOCKS FOR ~9-32 s (E measured it; `proto/web-agent-job.md` Β§4). That is
|
| 9192 |
+
# tolerable HERE and nowhere else: this is the automation RUNNER, already a
|
| 9193 |
+
# background walk. It must never be called from a route a person is waiting on.
|
| 9194 |
+
#
|
| 9195 |
+
# β `interpolate` ON THE CALLER'S SIDE, exactly as the update/create arms do it, so
|
| 9196 |
+
# `{Field}` works in a url or a selector. E's seam takes a plain dict and does not
|
| 9197 |
+
# know about rows.
|
| 9198 |
+
# β FAIL CLOSED, ONCE, WITH THE REASON β the counterpart to the validator storing
|
| 9199 |
+
# an unconfigured step (see `_clean_action_config`'s `web_read` arm). Reported once
|
| 9200 |
+
# per RUN and not per record: the missing config is a property of the flow, so a
|
| 9201 |
+
# 100-record walk would otherwise print the same sentence a hundred times and bury
|
| 9202 |
+
# everything else. The same shape the enrich arm's `unbound` note uses.
|
| 9203 |
+
_missing = [n for n, v in (("a URL", cfg.get("url")),
|
| 9204 |
+
("a CSS selector", cfg.get("selector")),
|
| 9205 |
+
("a column to write into", cfg.get("field")))
|
| 9206 |
+
if not str(v or "").strip()]
|
| 9207 |
+
if _missing:
|
| 9208 |
+
if "web_read_unconfigured" not in web_notes:
|
| 9209 |
+
web_notes.append("web_read_unconfigured")
|
| 9210 |
+
log("[aios-auto] web_read: this step still needs "
|
| 9211 |
+
+ ", ".join(_missing) + " β nothing was read")
|
| 9212 |
+
counts["webBlocked"] += 1
|
| 9213 |
+
continue
|
| 9214 |
+
# ββ THE PER-RUN JOB CEILING (E-4). One browser job per RECORD, ~9-32 s each,
|
| 9215 |
+
# against HF's 6-concurrent cap β a flow over a few thousand rows would submit a few
|
| 9216 |
+
# thousand jobs and run for days. Reported ONCE with its cause AND the fix, which is
|
| 9217 |
+
# R6's second sentence: a limit that cannot be removed today must say why and what
|
| 9218 |
+
# would remove it. `MAX_WEB_JOBS_PER_RUN` carries the reasoning.
|
| 9219 |
+
if counts["webRead"] >= MAX_WEB_JOBS_PER_RUN:
|
| 9220 |
+
if "web_cap" not in web_notes:
|
| 9221 |
+
web_notes.append("web_cap")
|
| 9222 |
+
log(f"[aios-auto] web_read: this run stopped after "
|
| 9223 |
+
f"{MAX_WEB_JOBS_PER_RUN} pages β each one is a separate browser job "
|
| 9224 |
+
f"of about 10-30 seconds, and they do not run in parallel. To read "
|
| 9225 |
+
f"more, narrow the flow's records, or batch the flow's web steps into "
|
| 9226 |
+
f"one job (web_agent.run_plan).")
|
| 9227 |
+
counts["webBlocked"] += 1
|
| 9228 |
+
continue
|
| 9229 |
+
_url = interpolate(str(cfg.get("url") or ""), row)
|
| 9230 |
+
_sel = interpolate(str(cfg.get("selector") or ""), row)
|
| 9231 |
+
_wait = cfg.get("waitFor")
|
| 9232 |
+
result, why = _web_agent().run_step(
|
| 9233 |
+
{"kind": "web_read", "id": str(act.get("id") or ""),
|
| 9234 |
+
"url": _url, "selector": _sel,
|
| 9235 |
+
"attr": str(cfg.get("attr") or "text"),
|
| 9236 |
+
"all": bool(cfg.get("all")),
|
| 9237 |
+
"waitFor": interpolate(str(_wait), row) if _wait else None,
|
| 9238 |
+
"timeoutMs": int(cfg.get("timeoutMs") or 20000)},
|
| 9239 |
+
{"tenant": str(defn.get("tenant") or ""),
|
| 9240 |
+
"automationId": str(defn.get("id") or ""),
|
| 9241 |
+
"runId": str(defn.get("id") or ""), "log": log})
|
| 9242 |
+
# β EXACTLY ONE OF THE TWO IS TRUTHY β E's contract, and `if why:` is the COMPLETE
|
| 9243 |
+
# client-side branch. There is no exception path to catch: `run_step` turns
|
| 9244 |
+
# everything into a sentence, which is R6's second sentence built into the seam.
|
| 9245 |
+
if why:
|
| 9246 |
+
log(f"[aios-auto] web_read: {why}")
|
| 9247 |
+
counts["webBlocked"] += 1
|
| 9248 |
+
continue # β NOTHING IS WRITTEN on a refusal.
|
| 9249 |
+
_target = str(cfg.get("field") or "")
|
| 9250 |
+
if _target:
|
| 9251 |
+
vals = {_target: (result or {}).get("value")}
|
| 9252 |
+
row.update(vals) # later actions see the write, as they must
|
| 9253 |
+
_act_row_patch(patches, table, rid, vals)
|
| 9254 |
+
counts["webRead"] += 1
|
| 9255 |
elif kind == "enrich_instagram":
|
| 9256 |
# β C4 (R3/R4). Reuses `pull_profile` and `capture_rows` β the SAME functions
|
| 9257 |
# `run_field_instagram` calls, not a second implementation of either.
|
|
|
|
| 9298 |
# the row it belongs to: the collector writes preset cells back onto THAT record,
|
| 9299 |
# and a run-wide list would have no way to say which handle each snapshot was for.
|
| 9300 |
pend_prof = []
|
| 9301 |
+
# β W31-T39(c) β the DEFERRAL WATERMARK, taken before the pull. See the `partial`
|
| 9302 |
+
# report below: a capability that was deferred is not a capability that failed,
|
| 9303 |
+
# and `pull_profile` appends into these two sinks from inside the call.
|
| 9304 |
+
_pend_before = len(enrich["pending"])
|
| 9305 |
res = pull_profile(handle_raw,
|
| 9306 |
max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
|
| 9307 |
log=log,
|
|
|
|
| 9346 |
# this same flow sees the enriched values β the rule `update_record` already
|
| 9347 |
# follows, and without it "enrich, then route by follower count" would judge
|
| 9348 |
# the record on the values it had before the pull.
|
| 9349 |
+
# ββ WAVE 31 Β· T39(c) β D-177(c): THE FIX APPLIED TO ONE PLATFORM AND NOT ITS
|
| 9350 |
+
# TWIN, and that is the shape worth naming rather than the line.
|
| 9351 |
+
#
|
| 9352 |
+
# W30-T11 found this on TikTok the expensive way: a paid run enriched two
|
| 9353 |
+
# profiles with `postMetrics` ON, wrote no posts, and said NOTHING β because
|
| 9354 |
+
# `partial` takes THIS branch and only the `else` ever appended a note, so the
|
| 9355 |
+
# reason `pull_profile` carries in `res["note"]` was discarded. Instagram has
|
| 9356 |
+
# the identical branch and never got the fix; the sibling arm below has carried
|
| 9357 |
+
# it since wave 30. One defect, two networks, one of them repaired β which is
|
| 9358 |
+
# exactly how the last four TikTok/Instagram divergences were found.
|
| 9359 |
+
#
|
| 9360 |
+
# β ONLY WHEN IT WAS ASKED FOR, and β NOT WHEN THE BATCH DEFERRED β both
|
| 9361 |
+
# conditions copied deliberately from the TikTok arm rather than re-reasoned.
|
| 9362 |
+
# With `postMetrics` off, a `partial` is the normal answer for every profile
|
| 9363 |
+
# and reporting it would put one useless line per record in the run entry; a
|
| 9364 |
+
# DEFERRAL is a paid snapshot already filed for collection, and reporting it as
|
| 9365 |
+
# a miss teaches a person to re-run and buy it twice.
|
| 9366 |
+
# β Instagram's deferral sinks are `enrich["pending"]` (post metrics) and
|
| 9367 |
+
# `pend_prof` (the profile snapshot) β the two lists `pull_profile` appends
|
| 9368 |
+
# into β where TikTok reads `res["deferredMedia"]`. Different signal, same
|
| 9369 |
+
# question: did this record's work get filed rather than lost?
|
| 9370 |
+
if (cfg.get("postMetrics") and not (res.get("posts") or [])
|
| 9371 |
+
and len(enrich["pending"]) == _pend_before and not pend_prof):
|
| 9372 |
+
_why = _s(res.get("note"), 300)
|
| 9373 |
+
if _why:
|
| 9374 |
+
enrich["notes"].append(f"@{handle_raw}: {_why}")
|
| 9375 |
cells = preset_cells(res, pulled)
|
| 9376 |
if cfg.get("dryRun"):
|
| 9377 |
enrich["dry"] = True
|
|
|
|
| 9810 |
for table_key, rows_in, key_field in plan:
|
| 9811 |
if not rows_in:
|
| 9812 |
continue
|
| 9813 |
+
# β R9 (W31-T32) β THE RUN PATH IS THE SITE THAT ACTUALLY CREATED TODAY'S TABLES, and it
|
| 9814 |
+
# is the one the ticket's `how:` does not name. `waves/wave30/proof/tiktok-e2e-ut_tt_posts
|
| 9815 |
+
# .png` shows `ut_tt_posts` with 8 real rows offering "+ New record" β those rows arrived
|
| 9816 |
+
# HERE, not through the save path (which, until W31-T34, returned before the child spawn for
|
| 9817 |
+
# a discovery automation). Stamping only the save site would have left every table that
|
| 9818 |
+
# already exists unlocked. β And an EXISTING table does come forward on the next call:
|
| 9819 |
+
# `ut_ensure`'s skip test carries `not (record_mode and have.get("recordMode") !=
|
| 9820 |
+
# record_mode)`, so no separate migration is needed.
|
| 9821 |
real_key = ut_ensure(rt, TT_TABLE_LABELS[table_key], TT_TABLE_FIELDS[table_key], username,
|
| 9822 |
+
key=table_key, flow_tag=tag, lock_fields=True,
|
| 9823 |
+
record_mode=tt_record_mode(table_key))
|
| 9824 |
missing.extend(ut_missing(rt, real_key))
|
| 9825 |
merged, counts_ = upsert_rows(dict((ut_get(rt, real_key) or {}).get("rows") or {}),
|
| 9826 |
rows_in, key_field, cap=row_cap(real_key))
|
|
|
|
| 10008 |
state["pendingMetricSnapshots"] = existing
|
| 10009 |
return cur
|
| 10010 |
|
| 10011 |
+
_store_update(rt, _up, flush="sync")
|
| 10012 |
return added[0]
|
| 10013 |
|
| 10014 |
|
|
|
|
| 10106 |
state["pendingProfileSnapshots"] = existing
|
| 10107 |
return cur
|
| 10108 |
|
| 10109 |
+
_store_update(rt, _up, flush="sync")
|
| 10110 |
return added[0]
|
| 10111 |
|
| 10112 |
|
|
|
|
| 10701 |
d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS]
|
| 10702 |
return cur
|
| 10703 |
|
| 10704 |
+
_store_update(rt, _up, flush="sync")
|
| 10705 |
|
| 10706 |
|
| 10707 |
# ---------------------------------------------------------------------------------------------
|
|
|
|
| 11973 |
trg["paused"] = True
|
| 11974 |
d["statusNote"] = _s(note, 200)
|
| 11975 |
return cur
|
| 11976 |
+
_store_update(rt, _up, flush="sync")
|
| 11977 |
|
| 11978 |
|
| 11979 |
def trigger_fire(rt, tenant, auto_id, log=print, rows=None):
|
|
|
|
| 12165 |
tenant = str(getattr(st, "key", "") or "royal-imports")
|
| 12166 |
table = str(evt.get("table") or "")
|
| 12167 |
kind = str(evt.get("type") or "")
|
| 12168 |
+
# ββ WAVE 31 Β· T35 (D-134) β `cached=True`, and this is the ONLY caller that passes it.
|
| 12169 |
+
# This function runs once per ROW EVENT, so a 20,000-row import used to perform 20,000
|
| 12170 |
+
# whole-document deep copies of the automations bucket, under the store lock, to re-read a
|
| 12171 |
+
# trigger set that had not changed. See `all_definitions` for why the memo is safe here
|
| 12172 |
+
# and nowhere else.
|
| 12173 |
+
defs = all_definitions(st, cached=True)
|
| 12174 |
field = str(evt.get("field") or "")
|
| 12175 |
rid = str(evt.get("rowId") or "")
|
| 12176 |
for aid, d in defs.items():
|
|
|
|
| 12187 |
if ridn is None or ridn <= hw:
|
| 12188 |
continue # once per record EVER (A2(4)) β undo-proof
|
| 12189 |
set_state(st, aid, {"rcHighwater": ridn})
|
| 12190 |
+
# ββ W31-T35 β MIRROR THE WRITE INTO THE MEMO'S OWN COPY, IN THE SAME STATEMENT.
|
| 12191 |
+
# This is the hazard a definitions memo creates and the reason D-134 is not a
|
| 12192 |
+
# one-line change: this branch READS `rcHighwater` and WRITES it, so within one
|
| 12193 |
+
# import burst the second row would compare against the highwater the FIRST row
|
| 12194 |
+
# set β and read the pre-write value out of the memo, fire again, and break
|
| 12195 |
+
# A2(4)'s *"once per record EVER β undo-proof"*. `set_state` goes through
|
| 12196 |
+
# `_store_update`, which drops the memo, but `defs` is the object already in hand
|
| 12197 |
+
# for the rest of THIS event; keeping the two in step is what makes the memo safe
|
| 12198 |
+
# rather than merely fast [[read-path-cannot-witness-write-path]].
|
| 12199 |
+
(d.setdefault("state", {}))["rcHighwater"] = ridn
|
| 12200 |
trigger_fire(st, tenant, aid, rows=[rid])
|
| 12201 |
elif kind == "event_field" and key == "event_field":
|
| 12202 |
# β WAVE 24 Β· law 4 β the per-FIELD narrowing is gone with the stored key. Every
|
|
|
|
| 12312 |
if dd is not None:
|
| 12313 |
dd["statusNote"] = _s(err, 200)
|
| 12314 |
return cur
|
| 12315 |
+
_store_update(rt, _note, flush="sync")
|
| 12316 |
return None
|
| 12317 |
lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n))
|
| 12318 |
reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid))
|
api/connectors_meta.py
ADDED
|
@@ -0,0 +1,508 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""connectors_meta.py β the Meta Ads (Marketing API) connector: identity, credentials, and the
|
| 2 |
+
schema PROBE that W31-T40 measures with.
|
| 3 |
+
|
| 4 |
+
β WHY A PROBE AND NOT A FIELD LIST. Owner ruling R8: the token exists to *"measure the schema now,
|
| 5 |
+
with nobody logging in via Facebook yet"*, and R2 requires the spawned grids to carry **every field
|
| 6 |
+
the API returns** β with T40's measured list as T47's only oracle (absent 0 / extra 0). Meta's own
|
| 7 |
+
Insights documentation declines to enumerate its fields, so a docs-derived list is not an answer to
|
| 8 |
+
either ruling. This file asks the API what it has.
|
| 9 |
+
|
| 10 |
+
β THE MEASUREMENT IS `?metadata=1`, WHICH IS GRAPH'S OWN INTROSPECTION, and that is the whole
|
| 11 |
+
design. Requesting a node without `fields=` returns Meta's DEFAULT subset β a fraction of the
|
| 12 |
+
schema, and a list that changes when they change their defaults. `metadata=1` returns the node's
|
| 13 |
+
FIELD TABLE (name + type + description) whether or not this account populates it, so the answer is
|
| 14 |
+
a property of the API rather than of our ad account's data. Insights has no such table, so its
|
| 15 |
+
catalog is measured the only other honest way: request a candidate set and record which names the
|
| 16 |
+
API ACCEPTS and which it REJECTS BY NAME. Both halves are written to the asset, so a reader can see
|
| 17 |
+
the question as well as the answer.
|
| 18 |
+
|
| 19 |
+
β SECRETS. `META_ADS_ACCESS_TOKEN` lives in gitignored `platform/.env`. It is read here, sent as a
|
| 20 |
+
POST-shaped query parameter to Graph over TLS, and **never printed, never logged, never written to
|
| 21 |
+
the asset** β the asset records the URL PATH and the field names, not the query string. Every call
|
| 22 |
+
is a read-only GET.
|
| 23 |
+
|
| 24 |
+
python aios-web/api/connectors_meta.py --probe # writes the wave asset
|
| 25 |
+
python aios-web/api/connectors_meta.py --check # credentials only, no schema walk
|
| 26 |
+
"""
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
import json
|
| 30 |
+
import os
|
| 31 |
+
import sys
|
| 32 |
+
import time
|
| 33 |
+
import urllib.error
|
| 34 |
+
import urllib.parse
|
| 35 |
+
import urllib.request
|
| 36 |
+
from pathlib import Path
|
| 37 |
+
|
| 38 |
+
#: The connector's identity in the platform's registries. β `ut_meta_` is declared ONCE, in
|
| 39 |
+
#: `routes_connectors.CONNECTED_PREFIXES`, and read from there β a second spelling here is the
|
| 40 |
+
#: [[one-question-two-normalizers]] shape and `verify_meta` asserts it does not exist.
|
| 41 |
+
KEY = "meta_ads"
|
| 42 |
+
LABEL = "Meta Ads"
|
| 43 |
+
|
| 44 |
+
#: The keychain entry type an admin stores a token under. β `core.keychain.ENTRY_TYPES` must learn
|
| 45 |
+
#: this or `add_entry` 400s β that is W31-T48's half, not this file's.
|
| 46 |
+
ENTRY_TYPE = "meta_ads"
|
| 47 |
+
|
| 48 |
+
#: β VERSIONED, AND THE VERSION IS REPORTED RATHER THAN ASSUMED. Graph deprecates versions on a
|
| 49 |
+
#: schedule; a pinned literal that has aged out fails with an error that reads like a bad token.
|
| 50 |
+
#: Overridable so a probe can be re-run against a newer version without an edit.
|
| 51 |
+
GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
|
| 52 |
+
GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
|
| 53 |
+
|
| 54 |
+
#: The five entity levels R2 names, in the order a person reads them. `edge` is the connection on
|
| 55 |
+
#: its parent; `node` is what `metadata=1` is asked about.
|
| 56 |
+
LEVELS = [
|
| 57 |
+
("adaccount", "Ad Account", None),
|
| 58 |
+
("campaign", "Campaign", "campaigns"),
|
| 59 |
+
("adset", "Ad Set", "adsets"),
|
| 60 |
+
("ad", "Ad", "ads"),
|
| 61 |
+
("adcreative", "Creative", "adcreatives"),
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
#: β INPUTS, NOT ANSWERS β the names OFFERED to the API per entity level. The measurement is
|
| 65 |
+
#: which of them Graph ACCEPTS (see `node_fields`); a name missing from a list here is "not
|
| 66 |
+
#: asked", never "not available", and the asset says so on its own face.
|
| 67 |
+
#: β THESE EXIST BECAUSE `?metadata=1` RETURNS NOTHING ON v21.0 β measured, not assumed. Graph
|
| 68 |
+
#: withdrew the introspection table that used to make a candidate list unnecessary.
|
| 69 |
+
ENTITY_CANDIDATES = {
|
| 70 |
+
"adaccount": [
|
| 71 |
+
"account_id", "account_status", "age", "amount_spent", "balance", "business_city",
|
| 72 |
+
"business_country_code", "business_name", "business_state", "business_street",
|
| 73 |
+
"business_street2", "business_zip", "capabilities", "created_time", "currency",
|
| 74 |
+
"disable_reason", "end_advertiser", "end_advertiser_name", "funding_source",
|
| 75 |
+
"has_migrated_permissions", "id", "io_number", "is_attribution_spec_system_default",
|
| 76 |
+
"is_direct_deals_enabled", "is_notifications_enabled", "is_personal",
|
| 77 |
+
"is_prepay_account", "is_tax_id_required", "line_numbers", "media_agency",
|
| 78 |
+
"min_campaign_group_spend_cap", "min_daily_budget", "name",
|
| 79 |
+
"offsite_pixels_tos_accepted", "owner", "partner", "spend_cap", "tax_id",
|
| 80 |
+
"tax_id_status", "tax_id_type", "timezone_id", "timezone_name",
|
| 81 |
+
"timezone_offset_hours_utc", "tos_accepted", "user_tasks", "user_tos_accepted"],
|
| 82 |
+
"campaign": [
|
| 83 |
+
"account_id", "bid_strategy", "boosted_object_id", "budget_rebalance_flag",
|
| 84 |
+
"budget_remaining", "buying_type", "campaign_group_active_time",
|
| 85 |
+
"can_create_brand_lift_study", "can_use_spend_cap", "configured_status", "created_time",
|
| 86 |
+
"daily_budget", "effective_status", "id", "is_skadnetwork_attribution", "issues_info",
|
| 87 |
+
"last_budget_toggling_time", "lifetime_budget", "name", "objective", "pacing_type",
|
| 88 |
+
"primary_attribution", "promoted_object", "smart_promotion_type", "source_campaign",
|
| 89 |
+
"source_campaign_id", "special_ad_categories", "special_ad_category",
|
| 90 |
+
"special_ad_category_country", "spend_cap", "start_time", "status", "stop_time",
|
| 91 |
+
"topline_id", "updated_time"],
|
| 92 |
+
"adset": [
|
| 93 |
+
"account_id", "adlabels", "adset_schedule", "asset_feed_id", "attribution_spec",
|
| 94 |
+
"bid_adjustments", "bid_amount", "bid_constraints", "bid_info", "bid_strategy",
|
| 95 |
+
"billing_event", "budget_remaining", "campaign", "campaign_active_time",
|
| 96 |
+
"campaign_attribution", "campaign_id", "configured_status", "created_time",
|
| 97 |
+
"creative_sequence", "daily_budget", "daily_min_spend_target", "daily_spend_cap",
|
| 98 |
+
"destination_type", "effective_status", "end_time", "frequency_control_specs", "id",
|
| 99 |
+
"instagram_actor_id", "is_dynamic_creative", "issues_info", "learning_stage_info",
|
| 100 |
+
"lifetime_budget", "lifetime_imps", "lifetime_min_spend_target", "lifetime_spend_cap",
|
| 101 |
+
"multi_optimization_goal_weight", "name", "optimization_goal", "optimization_sub_event",
|
| 102 |
+
"pacing_type", "promoted_object", "recurring_budget_semantics", "review_feedback",
|
| 103 |
+
"rf_prediction_id", "source_adset", "source_adset_id", "start_time", "status",
|
| 104 |
+
"targeting", "targeting_optimization_types", "time_based_ad_rotation_id_blocks",
|
| 105 |
+
"time_based_ad_rotation_intervals", "updated_time", "use_new_app_click"],
|
| 106 |
+
"ad": [
|
| 107 |
+
"account_id", "ad_active_time", "ad_review_feedback", "ad_schedule_end_time",
|
| 108 |
+
"ad_schedule_start_time", "adlabels", "adset", "adset_id", "bid_amount", "bid_info",
|
| 109 |
+
"bid_type", "campaign", "campaign_id", "configured_status", "conversion_domain",
|
| 110 |
+
"created_time", "creative", "demolink_hash", "display_sequence", "effective_status",
|
| 111 |
+
"engagement_audience", "failed_delivery_checks", "id", "issues_info",
|
| 112 |
+
"last_updated_by_app_id", "name", "preview_shareable_link", "priority",
|
| 113 |
+
"recommendations", "source_ad", "source_ad_id", "status", "targeting",
|
| 114 |
+
"tracking_and_conversion_with_defaults", "tracking_specs", "updated_time"],
|
| 115 |
+
"adcreative": [
|
| 116 |
+
"account_id", "actor_id", "adlabels", "applink_treatment", "asset_feed_spec",
|
| 117 |
+
"authorization_category", "body", "branded_content_sponsor_page_id", "bundle_folder_id",
|
| 118 |
+
"call_to_action_type", "categorization_criteria", "category_media_source",
|
| 119 |
+
"collaborative_ads_lsb_image_bank_id", "degrees_of_freedom_spec", "destination_set_id",
|
| 120 |
+
"dynamic_ad_voice", "effective_authorization_category", "effective_instagram_media_id",
|
| 121 |
+
"effective_object_story_id", "enable_direct_install", "enable_launch_instant_app", "id",
|
| 122 |
+
"image_crops", "image_hash", "image_url", "instagram_actor_id",
|
| 123 |
+
"instagram_permalink_url", "instagram_story_id", "instagram_user_id",
|
| 124 |
+
"interactive_components_spec", "link_deep_link_url", "link_destination_display_url",
|
| 125 |
+
"link_og_id", "link_url", "messenger_sponsored_message", "name", "object_id",
|
| 126 |
+
"object_store_url", "object_story_id", "object_story_spec", "object_type", "object_url",
|
| 127 |
+
"page_link", "page_message", "place_page_set_id", "platform_customizations",
|
| 128 |
+
"playable_asset_id", "portrait_customizations", "product_set_id", "recommender_settings",
|
| 129 |
+
"source_instagram_media_id", "status", "template_url", "template_url_spec",
|
| 130 |
+
"thumbnail_id", "thumbnail_url", "title", "url_tags", "use_page_actor_override",
|
| 131 |
+
"video_id"],
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
#: β AN INPUT, NOT AN ANSWER β and labelled as one in the asset. Insights has no `metadata=1`
|
| 135 |
+
#: table, so the catalog is measured by ACCEPTANCE: every name here is offered to the API and the
|
| 136 |
+
#: probe records which it takes and which it names in an error. A name missing from this list is
|
| 137 |
+
#: therefore "not asked", never "not available", and the asset says so.
|
| 138 |
+
INSIGHTS_CANDIDATES = [
|
| 139 |
+
"account_id", "account_name", "account_currency", "campaign_id", "campaign_name",
|
| 140 |
+
"adset_id", "adset_name", "ad_id", "ad_name", "date_start", "date_stop",
|
| 141 |
+
"impressions", "reach", "frequency", "clicks", "unique_clicks", "ctr", "unique_ctr",
|
| 142 |
+
"cpc", "cpm", "cpp", "spend", "social_spend", "actions", "action_values",
|
| 143 |
+
"conversions", "conversion_values", "cost_per_action_type", "cost_per_unique_click",
|
| 144 |
+
"cost_per_inline_link_click", "cost_per_thruplay", "purchase_roas", "website_purchase_roas",
|
| 145 |
+
"inline_link_clicks", "inline_link_click_ctr", "inline_post_engagement",
|
| 146 |
+
"outbound_clicks", "outbound_clicks_ctr", "unique_outbound_clicks",
|
| 147 |
+
"video_play_actions", "video_p25_watched_actions", "video_p50_watched_actions",
|
| 148 |
+
"video_p75_watched_actions", "video_p95_watched_actions", "video_p100_watched_actions",
|
| 149 |
+
"video_avg_time_watched_actions", "video_thruplay_watched_actions",
|
| 150 |
+
"quality_ranking", "engagement_rate_ranking", "conversion_rate_ranking",
|
| 151 |
+
"objective", "optimization_goal", "buying_type", "attribution_setting",
|
| 152 |
+
"canvas_avg_view_time", "estimated_ad_recallers", "full_view_impressions",
|
| 153 |
+
]
|
| 154 |
+
|
| 155 |
+
#: Every probe call, for the mailbox log the PRD's shared-resource table demands ("log every probe
|
| 156 |
+
#: in mailbox/D.md; the rate limit is shared across the token"). Paths only β never a query string,
|
| 157 |
+
#: because the token rides in one.
|
| 158 |
+
CALLS: list = []
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class MetaError(RuntimeError):
|
| 162 |
+
"""A Graph refusal, carrying Meta's own message. β Its `str()` is safe to print: the token is
|
| 163 |
+
stripped from every URL before it reaches here."""
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _token():
|
| 167 |
+
"""The Meta token from the environment, or from gitignored `platform/.env`. Returns "" when
|
| 168 |
+
absent β the caller REPORTS that rather than crashing, because "no token" is a real state of
|
| 169 |
+
this deployment and it is the state as of 2026-08-12."""
|
| 170 |
+
tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
|
| 171 |
+
if tok:
|
| 172 |
+
return tok.strip()
|
| 173 |
+
env = Path(__file__).resolve().parents[2] / "platform" / ".env"
|
| 174 |
+
if env.exists():
|
| 175 |
+
for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 176 |
+
if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
|
| 177 |
+
_, _, v = line.partition("=")
|
| 178 |
+
return v.strip().strip('"').strip("'")
|
| 179 |
+
return ""
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _safe(url):
|
| 183 |
+
"""`url` with every query value redacted β the only form allowed near a log or an asset."""
|
| 184 |
+
parts = urllib.parse.urlsplit(url)
|
| 185 |
+
q = urllib.parse.parse_qsl(parts.query, keep_blank_values=True)
|
| 186 |
+
shown = "&".join(f"{k}=<redacted>" if k == "access_token" else f"{k}={v}" for k, v in q)
|
| 187 |
+
return urllib.parse.urlunsplit((parts.scheme, parts.netloc, parts.path, shown, ""))
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def get(path, token, **params):
|
| 191 |
+
"""One read-only Graph GET. Returns the decoded body, or raises `MetaError` with Meta's reason.
|
| 192 |
+
|
| 193 |
+
β A 400 FROM GRAPH IS AN ANSWER, NOT A CRASH β an unknown field name comes back as a 400 whose
|
| 194 |
+
message NAMES the field, and that is precisely how the Insights catalog is measured below.
|
| 195 |
+
"""
|
| 196 |
+
params["access_token"] = token
|
| 197 |
+
url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
|
| 198 |
+
CALLS.append({"path": path.lstrip("/"), "params": sorted(k for k in params
|
| 199 |
+
if k != "access_token")})
|
| 200 |
+
req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-probe/1"})
|
| 201 |
+
try:
|
| 202 |
+
with urllib.request.urlopen(req, timeout=60) as r:
|
| 203 |
+
return json.loads(r.read().decode("utf-8", "replace"))
|
| 204 |
+
except urllib.error.HTTPError as e:
|
| 205 |
+
body = e.read().decode("utf-8", "replace")
|
| 206 |
+
try:
|
| 207 |
+
err = (json.loads(body).get("error") or {})
|
| 208 |
+
msg = err.get("message") or body[:300]
|
| 209 |
+
code = err.get("code")
|
| 210 |
+
except Exception: # noqa: BLE001
|
| 211 |
+
msg, code = body[:300], None
|
| 212 |
+
raise MetaError(f"HTTP {e.code} (code {code}) on {_safe(url)}: {msg}") from None
|
| 213 |
+
except Exception as e: # noqa: BLE001
|
| 214 |
+
raise MetaError(f"{type(e).__name__} on {_safe(url)}: {e}") from None
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def node_fields(node_id, token, candidates, batch=60):
|
| 218 |
+
"""`(accepted, rejected, gated)` for ONE node, measured by ACCEPTANCE + BISECT.
|
| 219 |
+
|
| 220 |
+
ββ `?metadata=1` IS DEAD AND THAT WAS MEASURED, NOT ASSUMED. This function used to ask Graph
|
| 221 |
+
for its published introspection table, which is the documented way to enumerate a node's
|
| 222 |
+
fields. On **v21.0 it returns nothing**: `GET /act_β¦?metadata=1` answers with keys
|
| 223 |
+
`['account_id','id']` and `GET /<campaign_id>?metadata=1` with `['id']` β `metadata.fields` is
|
| 224 |
+
EMPTY in both. The probe reported "0 fields" for all five levels and looked like a broken
|
| 225 |
+
probe; it was a withdrawn API. So the entity levels are measured exactly as Insights already
|
| 226 |
+
was: offer names, keep what the API takes.
|
| 227 |
+
|
| 228 |
+
ββ **ACCEPTED IS NOT RETURNED, AND THIS IS THE ONE THAT BREAKS R2 SILENTLY.** A 20-field batch
|
| 229 |
+
on a live campaign answers **HTTP 200 with 14 keys** β Graph omits nulls from the response. A
|
| 230 |
+
catalog built from the keys that came BACK loses roughly a third of the schema, which is
|
| 231 |
+
R2's *"do not drop any column"* violated with no gate anywhere going red. The catalog is
|
| 232 |
+
therefore what was **ASKED and ACCEPTED**, and this function never inspects the payload's keys.
|
| 233 |
+
|
| 234 |
+
β **A 400 NAMES THE OFFENDER; A 403 NAMES NOTHING.** `(#100) Tried accessing nonexisting
|
| 235 |
+
field (x)` tells us exactly which name to drop, so a batch converges in one retry. But the Ad
|
| 236 |
+
Account level answers `(#200) Requires business_management permission` β no field named β and
|
| 237 |
+
dropping the batch there reports "Ad Account: 0 fields", which is the honest-looking wrong
|
| 238 |
+
answer. An unnamed refusal is BISECTED instead: 11 calls classified all 46, finding exactly one
|
| 239 |
+
gated column (`owner`).
|
| 240 |
+
"""
|
| 241 |
+
accepted, rejected, gated = [], {}, {}
|
| 242 |
+
|
| 243 |
+
def ask(names):
|
| 244 |
+
"""`(ok, err)` β one batch. `ok` means every name in it is a real, readable field."""
|
| 245 |
+
try:
|
| 246 |
+
get(str(node_id), token, fields=",".join(names))
|
| 247 |
+
return True, ""
|
| 248 |
+
except MetaError as e:
|
| 249 |
+
return False, str(e)
|
| 250 |
+
|
| 251 |
+
def resolve(names, depth=0):
|
| 252 |
+
if not names:
|
| 253 |
+
return
|
| 254 |
+
ok, err = ask(names)
|
| 255 |
+
if ok:
|
| 256 |
+
accepted.extend(names)
|
| 257 |
+
return
|
| 258 |
+
named = [f for f in names if f"({f})" in err or f"field ({f})" in err]
|
| 259 |
+
if named and len(named) < len(names):
|
| 260 |
+
for f in named:
|
| 261 |
+
rejected[f] = err[:200]
|
| 262 |
+
resolve([f for f in names if f not in named], depth + 1)
|
| 263 |
+
return
|
| 264 |
+
if len(names) == 1:
|
| 265 |
+
# a single name that still refuses: classify it by WHAT the refusal was.
|
| 266 |
+
(rejected if "(#100)" in err else gated)[names[0]] = err[:200]
|
| 267 |
+
return
|
| 268 |
+
half = len(names) // 2 # unnamed refusal β bisect, never drop
|
| 269 |
+
resolve(names[:half], depth + 1)
|
| 270 |
+
resolve(names[half:], depth + 1)
|
| 271 |
+
time.sleep(0.15) # the rate limit is shared across this token
|
| 272 |
+
|
| 273 |
+
for start in range(0, len(candidates), batch):
|
| 274 |
+
resolve(list(candidates[start:start + batch]))
|
| 275 |
+
return sorted(set(accepted)), rejected, gated
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def insights_catalog(act_id, token, candidates=None):
|
| 279 |
+
"""`(accepted, rejected)` for the daily Insights field catalog, measured by ACCEPTANCE.
|
| 280 |
+
|
| 281 |
+
Asks in batches; on a 400 that names a field, drops that name and retries. What survives is
|
| 282 |
+
what the API takes at `time_increment=1` β which is the only claim R2's Insights grids need.
|
| 283 |
+
β `rejected` is reported too: "we asked and Meta refused" and "we never asked" are different
|
| 284 |
+
facts, and only the asset can keep them apart.
|
| 285 |
+
"""
|
| 286 |
+
remaining = list(candidates if candidates is not None else INSIGHTS_CANDIDATES)
|
| 287 |
+
rejected, accepted = {}, []
|
| 288 |
+
for start in range(0, len(remaining), 25):
|
| 289 |
+
batch = remaining[start:start + 25]
|
| 290 |
+
while batch:
|
| 291 |
+
try:
|
| 292 |
+
get(f"act_{act_id}/insights", token, fields=",".join(batch),
|
| 293 |
+
time_increment="1", date_preset="last_7d", limit="1")
|
| 294 |
+
accepted.extend(batch)
|
| 295 |
+
break
|
| 296 |
+
except MetaError as e:
|
| 297 |
+
bad = [f for f in batch if f in str(e)]
|
| 298 |
+
if not bad: # a refusal that names no field: report, do not loop
|
| 299 |
+
rejected["(batch)"] = str(e)
|
| 300 |
+
break
|
| 301 |
+
for f in bad:
|
| 302 |
+
rejected[f] = str(e)[:200]
|
| 303 |
+
batch = [f for f in batch if f not in bad]
|
| 304 |
+
time.sleep(0.2) # the rate limit is shared across this token
|
| 305 |
+
return sorted(set(accepted)), rejected
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def probe():
|
| 309 |
+
"""Walk the whole schema and return the report dict. Never raises on a Meta refusal β a
|
| 310 |
+
refusal IS the finding, and a probe that dies on one produces no asset at all."""
|
| 311 |
+
token = _token()
|
| 312 |
+
report = {"graphVersion": GRAPH_VERSION, "token": bool(token), "accounts": [],
|
| 313 |
+
"levels": {}, "insights": {}, "errors": []}
|
| 314 |
+
if not token:
|
| 315 |
+
report["errors"].append(
|
| 316 |
+
"META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env. "
|
| 317 |
+
"Nothing was measured. This is the state of this box as of 2026-08-12: the PRD "
|
| 318 |
+
"records the token as 'recorded in platform/.env' and it is not there.")
|
| 319 |
+
return report
|
| 320 |
+
try:
|
| 321 |
+
me = get("me/adaccounts", token,
|
| 322 |
+
fields="id,account_id,name,account_status,currency,timezone_name,business_name")
|
| 323 |
+
report["accounts"] = me.get("data") or []
|
| 324 |
+
except MetaError as e:
|
| 325 |
+
report["errors"].append(f"reaching /me/adaccounts: {e}")
|
| 326 |
+
return report
|
| 327 |
+
if not report["accounts"]:
|
| 328 |
+
report["errors"].append("the token authenticates but reaches ZERO ad accounts β a "
|
| 329 |
+
"permissions/scope answer, not an empty business")
|
| 330 |
+
return report
|
| 331 |
+
act = report["accounts"][0]
|
| 332 |
+
act_id = str(act.get("account_id") or str(act.get("id") or "").replace("act_", ""))
|
| 333 |
+
node_of = {"adaccount": f"act_{act_id}"}
|
| 334 |
+
for key, _label, edge in LEVELS:
|
| 335 |
+
if edge is None:
|
| 336 |
+
continue
|
| 337 |
+
try:
|
| 338 |
+
got = get(f"act_{act_id}/{edge}", token, fields="id", limit="1")
|
| 339 |
+
rows = got.get("data") or []
|
| 340 |
+
if rows:
|
| 341 |
+
node_of[key] = rows[0]["id"]
|
| 342 |
+
else:
|
| 343 |
+
report["errors"].append(f"{key}: this account has no {edge}, so no real row could "
|
| 344 |
+
f"be read (the field TABLE is still measured below)")
|
| 345 |
+
except MetaError as e:
|
| 346 |
+
report["errors"].append(f"listing {edge}: {e}")
|
| 347 |
+
for key, label, _edge in LEVELS:
|
| 348 |
+
nid = node_of.get(key)
|
| 349 |
+
cands = ENTITY_CANDIDATES.get(key) or []
|
| 350 |
+
if not nid:
|
| 351 |
+
report["levels"][key] = {"label": label, "accepted": [], "rejected": {}, "gated": {},
|
| 352 |
+
"asked": len(cands), "node": "",
|
| 353 |
+
"note": "no node of this level was reachable, so nothing "
|
| 354 |
+
"could be offered to the API"}
|
| 355 |
+
continue
|
| 356 |
+
try:
|
| 357 |
+
accepted, rejected, gated = node_fields(nid, token, cands)
|
| 358 |
+
report["levels"][key] = {"label": label, "node": str(nid), "asked": len(cands),
|
| 359 |
+
"accepted": accepted, "rejected": rejected, "gated": gated}
|
| 360 |
+
except MetaError as e:
|
| 361 |
+
report["levels"][key] = {"label": label, "node": str(nid), "asked": len(cands),
|
| 362 |
+
"accepted": [], "rejected": {}, "gated": {},
|
| 363 |
+
"note": str(e)}
|
| 364 |
+
try:
|
| 365 |
+
accepted, rejected = insights_catalog(act_id, token)
|
| 366 |
+
report["insights"] = {"accepted": accepted, "rejected": rejected,
|
| 367 |
+
"candidatesAsked": len(INSIGHTS_CANDIDATES)}
|
| 368 |
+
except MetaError as e:
|
| 369 |
+
report["errors"].append(f"insights catalog: {e}")
|
| 370 |
+
return report
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
def as_markdown(report):
|
| 374 |
+
"""The wave asset. β Field NAMES and TYPES only β no row values, no account identifiers beyond
|
| 375 |
+
the ad account name the owner already knows, and no query strings."""
|
| 376 |
+
L = ["# W31-T40 β the Meta Ads schema, MEASURED", "",
|
| 377 |
+
f"Graph version: `{report['graphVersion']}` Β· token present: "
|
| 378 |
+
f"**{'yes' if report['token'] else 'NO'}** Β· probe calls: **{len(CALLS)}**", ""]
|
| 379 |
+
if report["errors"]:
|
| 380 |
+
L += ["## What could not be measured", ""]
|
| 381 |
+
L += [f"- {e}" for e in report["errors"]] + [""]
|
| 382 |
+
if not report["token"]:
|
| 383 |
+
L += ["β **NOTHING BELOW WAS MEASURED.** This asset exists so the absence is a recorded "
|
| 384 |
+
"fact rather than a missing file: the probe is built, runs, and reports honestly "
|
| 385 |
+
"that it has no credential. Re-run `python aios-web/api/connectors_meta.py --probe` "
|
| 386 |
+
"the moment a token lands and this file fills itself in.", ""]
|
| 387 |
+
return "\n".join(L)
|
| 388 |
+
L += ["## Reachable ad accounts", "", "| account_id | name | status | currency | timezone |",
|
| 389 |
+
"|---|---|---|---|---|"]
|
| 390 |
+
for a in report["accounts"]:
|
| 391 |
+
L.append(f"| `{a.get('account_id','')}` | {a.get('name','')} | "
|
| 392 |
+
f"{a.get('account_status','')} | {a.get('currency','')} | "
|
| 393 |
+
f"{a.get('timezone_name','')} |")
|
| 394 |
+
L.append("")
|
| 395 |
+
L += ["## Method β and why it is not `?metadata=1`", "",
|
| 396 |
+
"β **Graph v21.0 publishes no introspection table.** `GET /act_β¦?metadata=1` returns "
|
| 397 |
+
"keys `['account_id','id']` and `GET /<campaign_id>?metadata=1` returns `['id']` β "
|
| 398 |
+
"`metadata.fields` is EMPTY. The first run of this probe reported *0 fields* for all "
|
| 399 |
+
"five levels and looked like a broken probe; it was a withdrawn API.", "",
|
| 400 |
+
"β So every level below is measured by **ACCEPTANCE**: the names in "
|
| 401 |
+
"`connectors_meta.ENTITY_CANDIDATES` are offered, and what Graph takes is the catalog. "
|
| 402 |
+
"A `(#100)` refusal NAMES the offending field, so a batch converges in one retry; a "
|
| 403 |
+
"`(#200)` permission refusal names nothing, so the batch is **bisected** rather than "
|
| 404 |
+
"dropped β otherwise the Ad Account level reports a confident, wrong `0 fields`.", "",
|
| 405 |
+
"ββ **ACCEPTED IS NOT RETURNED.** A 20-field batch on a live campaign answers HTTP 200 "
|
| 406 |
+
"with 14 keys, because Graph omits nulls. Every list below is what was **asked and "
|
| 407 |
+
"accepted** β the response's own keys are never inspected. Building columns from "
|
| 408 |
+
"returned keys drops roughly a third of the schema, which is R2's *do not drop any "
|
| 409 |
+
"column* broken with nothing going red.", ""]
|
| 410 |
+
for key, label, _edge in LEVELS:
|
| 411 |
+
lv = report["levels"].get(key) or {}
|
| 412 |
+
fields = lv.get("accepted") or []
|
| 413 |
+
L += [f"## {label} β **{len(fields)} accepted** of {lv.get('asked', 0)} asked", ""]
|
| 414 |
+
if lv.get("note"):
|
| 415 |
+
L += [f"β {lv['note']}", ""]
|
| 416 |
+
if fields:
|
| 417 |
+
L += ["```", ", ".join(fields), "```", ""]
|
| 418 |
+
if lv.get("rejected"):
|
| 419 |
+
L += ["**Not a field on this account** (refused BY NAME, recorded rather than "
|
| 420 |
+
"guessed at):", ""]
|
| 421 |
+
L += [f"- `{k}`" for k in sorted(lv["rejected"])] + [""]
|
| 422 |
+
if lv.get("gated"):
|
| 423 |
+
L += ["β **Permission-gated β a REPORTED limit, not a silent omission** (R6's second "
|
| 424 |
+
"sentence):", ""]
|
| 425 |
+
L += [f"- `{k}` β {v}" for k, v in sorted(lv["gated"].items())] + [""]
|
| 426 |
+
ins = report.get("insights") or {}
|
| 427 |
+
if ins:
|
| 428 |
+
L += [f"## Insights at `time_increment=1` β {len(ins.get('accepted') or [])} accepted of "
|
| 429 |
+
f"{ins.get('candidatesAsked', 0)} asked", "",
|
| 430 |
+
"β Insights publishes no `metadata=1` field table, so this catalog is measured by "
|
| 431 |
+
"ACCEPTANCE: every name below was offered to the API and taken. A name absent from "
|
| 432 |
+
"this file was **not asked**, which is not the same as unavailable.", "",
|
| 433 |
+
"```", ", ".join(ins.get("accepted") or []), "```", ""]
|
| 434 |
+
if ins.get("rejected"):
|
| 435 |
+
L += ["### Refused by name", ""]
|
| 436 |
+
L += [f"- `{k}` β {v}" for k, v in sorted((ins.get("rejected") or {}).items())] + [""]
|
| 437 |
+
return "\n".join(L)
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
def probe_credential(fields, timeout=8):
|
| 441 |
+
"""`{ok, message}` for a stored `meta_ads` credential β the keychain's liveness probe.
|
| 442 |
+
|
| 443 |
+
β IT LIVES HERE, IN THE API LAYER, AND IS REGISTERED INTO `core.keychain` RATHER THAN
|
| 444 |
+
IMPORTED BY IT. `core` never imports up (ARCHITECTURE.md), and the first draft of this arm
|
| 445 |
+
did `sys.path.insert` + `import connectors_meta` from inside `core/keychain.py` β an
|
| 446 |
+
inversion with no gate on it. `keychain.register_prober` is the slot, borrowed from
|
| 447 |
+
`datastore.set_paused_probe`, which exists for exactly this reason one layer down.
|
| 448 |
+
|
| 449 |
+
β ZERO reachable ad accounts is reported as NOT ok, deliberately: a token that authenticates
|
| 450 |
+
but sees no account is a SCOPE answer β the single likeliest failure for a Marketing API
|
| 451 |
+
credential β and `ok: true` would hide the one thing the admin needs to fix.
|
| 452 |
+
"""
|
| 453 |
+
tok = str((fields or {}).get("access_token") or (fields or {}).get("token")
|
| 454 |
+
or (fields or {}).get("api_key") or "")
|
| 455 |
+
if not tok:
|
| 456 |
+
return {"ok": False,
|
| 457 |
+
"message": "store the token under `access_token` (or `token` / `api_key`)"}
|
| 458 |
+
try:
|
| 459 |
+
got = get("me/adaccounts", tok, fields="id", limit="1")
|
| 460 |
+
except MetaError as e:
|
| 461 |
+
return {"ok": False, "message": str(e)[:200]}
|
| 462 |
+
n = len(got.get("data") or [])
|
| 463 |
+
return {"ok": n > 0,
|
| 464 |
+
"message": (f"Meta answered β {n} ad account(s) reachable" if n else
|
| 465 |
+
"the token authenticates but reaches NO ad account: that is a "
|
| 466 |
+
"scope/permission answer, not an empty business")}
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
def register():
|
| 470 |
+
"""Fill the keychain's probe slot for this connector. Called at import by the API layer's
|
| 471 |
+
connector registry (`routes_connectors`), so any process that can serve the directory has
|
| 472 |
+
also registered the prober. Never raises: a probe slot that could not be filled costs a
|
| 473 |
+
liveness CHECK, never the ability to store a credential."""
|
| 474 |
+
try:
|
| 475 |
+
import core.keychain as kc
|
| 476 |
+
return kc.register_prober(ENTRY_TYPE, probe_credential)
|
| 477 |
+
except Exception: # noqa: BLE001
|
| 478 |
+
return False
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def main(argv):
|
| 482 |
+
check_only = "--check" in argv
|
| 483 |
+
token = _token()
|
| 484 |
+
if check_only:
|
| 485 |
+
print(f"META token present: {'yes' if token else 'NO'} "
|
| 486 |
+
f"(never printed; length is not disclosed)")
|
| 487 |
+
if not token:
|
| 488 |
+
return 2
|
| 489 |
+
try:
|
| 490 |
+
me = get("me/adaccounts", token, fields="id,name", limit="5")
|
| 491 |
+
print(f"authenticated; reachable ad accounts: {len(me.get('data') or [])}")
|
| 492 |
+
return 0
|
| 493 |
+
except MetaError as e:
|
| 494 |
+
print(f"REFUSED: {e}")
|
| 495 |
+
return 1
|
| 496 |
+
report = probe()
|
| 497 |
+
out = (Path(__file__).resolve().parents[2] / ".claude" / "wiki" / "research" / "waves"
|
| 498 |
+
/ "wave31" / "proto" / "meta-ads-schema.md")
|
| 499 |
+
out.parent.mkdir(parents=True, exist_ok=True)
|
| 500 |
+
out.write_text(as_markdown(report), encoding="utf-8")
|
| 501 |
+
print(f"wrote {out}")
|
| 502 |
+
for c in CALLS:
|
| 503 |
+
print(f" probe: GET /{c['path']} [{', '.join(c['params'])}]")
|
| 504 |
+
return 0 if report["token"] and not report["errors"] else 2
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
if __name__ == "__main__":
|
| 508 |
+
sys.exit(main(sys.argv[1:]))
|
api/main.py
CHANGED
|
@@ -76,6 +76,8 @@ import routes_connectors # noqa: E402 (wave 23 C11 β the connectors director
|
|
| 76 |
import routes_forms # noqa: E402 (wave 23 C9 β the PUBLIC form door; SESSION D's router)
|
| 77 |
import routes_templates # noqa: E402 (wave 23 C12 β template apply doors; SESSION E's router)
|
| 78 |
import routes_odoo_tables # noqa: E402 (wave 27 item 17 β Odoo relational; SESSION E's router)
|
|
|
|
|
|
|
| 79 |
from core import grid_events # noqa: E402
|
| 80 |
from deps import Session, module_gate # noqa: E402
|
| 81 |
|
|
@@ -230,6 +232,14 @@ app.include_router(routes_forms.router) # item 8 / C9 β the PUBLIC form
|
|
| 230 |
# line β complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks
|
| 231 |
# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED.
|
| 232 |
app.include_router(routes_odoo_tables.router)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
# ββ WAVE 29, item 20 / R11 / contract C6 β F's CHANGE TOKEN. `GET /api/v1/changes?scope=<scope>`
|
| 234 |
# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep
|
| 235 |
# copies), which is what lets a filtered view pick up a row created in another tab, by an automation
|
|
@@ -242,6 +252,12 @@ app.include_router(routes_odoo_tables.router)
|
|
| 242 |
# β Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this
|
| 243 |
# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500).
|
| 244 |
app.include_router(routes_changes.router) # item 20 / C6 β F's router, A's line
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
|
| 246 |
|
| 247 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
@@ -404,11 +420,73 @@ def _seed_and_sync_store():
|
|
| 404 |
# is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild
|
| 405 |
# refuses β correctly, but for a reason that reads like a data loss scare.
|
| 406 |
# β Same thread on purpose: it is already a daemon and nothing serves requests behind it.
|
|
|
|
| 407 |
_rebuild_odoo_relational("boot")
|
| 408 |
except Exception as e: # noqa: BLE001
|
| 409 |
print(f"[aios-api] datastore seed/sync skipped: {e}")
|
| 410 |
|
| 411 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 412 |
def _rebuild_odoo_relational(why):
|
| 413 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 414 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
|
@@ -462,6 +540,7 @@ def _rebuild_odoo_relational(why):
|
|
| 462 |
print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
|
| 463 |
print(f"[aios-api] odoo relational rebuild done ({why}): "
|
| 464 |
+ ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
|
|
|
|
| 465 |
except Exception as e: # noqa: BLE001
|
| 466 |
# β The TYPE is named here as it is in the two inner handlers. The original printed only
|
| 467 |
# `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
|
|
@@ -548,6 +627,77 @@ def _prewarm():
|
|
| 548 |
print(f"[aios-api] prewarm skipped: {e}")
|
| 549 |
|
| 550 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 551 |
if os.environ.get("AIOS_PREWARM") == "1":
|
| 552 |
import threading as _threading
|
| 553 |
_threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
|
|
|
|
| 76 |
import routes_forms # noqa: E402 (wave 23 C9 β the PUBLIC form door; SESSION D's router)
|
| 77 |
import routes_templates # noqa: E402 (wave 23 C12 β template apply doors; SESSION E's router)
|
| 78 |
import routes_odoo_tables # noqa: E402 (wave 27 item 17 β Odoo relational; SESSION E's router)
|
| 79 |
+
import routes_connected_tables # noqa: E402 (wave 31 T49/C4 β the source-neutral grid door; D's)
|
| 80 |
+
import routes_web_agent # noqa: E402 (wave 31 R10/C5 β the web-browsing agent; E's router, A's line)
|
| 81 |
from core import grid_events # noqa: E402
|
| 82 |
from deps import Session, module_gate # noqa: E402
|
| 83 |
|
|
|
|
| 232 |
# line β complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks
|
| 233 |
# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED.
|
| 234 |
app.include_router(routes_odoo_tables.router)
|
| 235 |
+
# ββ WAVE 31 Β· T49 / C4 β THE SOURCE-NEUTRAL DOOR TO THE SAME CAPABILITY.
|
| 236 |
+
# `/api/v1/connected-tables/{key}/rows` is an ALIAS: every request lands in
|
| 237 |
+
# `routes_odoo_tables.odoo_table_rows`, so "the Odoo path behaves byte-identically" holds by
|
| 238 |
+
# CONSTRUCTION rather than by two implementations that agree on the day they were written. R2 puts
|
| 239 |
+
# Meta Ads on the same mirror, and a Meta campaign served from a URL with `odoo` in it is a name
|
| 240 |
+
# that lies to every network tab, log line and bug report.
|
| 241 |
+
# β MOUNTED IN THE SAME CHANGE AS THE MODULE, per the line above and for the same wave-23 scar.
|
| 242 |
+
app.include_router(routes_connected_tables.router)
|
| 243 |
# ββ WAVE 29, item 20 / R11 / contract C6 β F's CHANGE TOKEN. `GET /api/v1/changes?scope=<scope>`
|
| 244 |
# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep
|
| 245 |
# copies), which is what lets a filtered view pick up a row created in another tab, by an automation
|
|
|
|
| 252 |
# β Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this
|
| 253 |
# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500).
|
| 254 |
app.include_router(routes_changes.router) # item 20 / C6 β F's router, A's line
|
| 255 |
+
# β WAVE 31 (R10 / C5) β E's router, taken by the INTEGRATOR under W31-T08's own done-when
|
| 256 |
+
# ("no router ships unmounted") because `main.py` is D's fence and D's queue did not reach it.
|
| 257 |
+
# It was written, gated and 404-dead: `verify_web_agent.py` asserts THIS LINE and was red at
|
| 258 |
+
# 60/61 for it. Four waves of the same defect β three routers in wave 23, four features in
|
| 259 |
+
# wave 29 β is why the assertion exists and why the line is not left for later.
|
| 260 |
+
app.include_router(routes_web_agent.router) # R10 / C5 β E's router, A's line
|
| 261 |
|
| 262 |
|
| 263 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
|
|
| 420 |
# is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild
|
| 421 |
# refuses β correctly, but for a reason that reads like a data loss scare.
|
| 422 |
# β Same thread on purpose: it is already a daemon and nothing serves requests behind it.
|
| 423 |
+
_pull_meta("boot")
|
| 424 |
_rebuild_odoo_relational("boot")
|
| 425 |
except Exception as e: # noqa: BLE001
|
| 426 |
print(f"[aios-api] datastore seed/sync skipped: {e}")
|
| 427 |
|
| 428 |
|
| 429 |
+
def _pull_meta(why):
|
| 430 |
+
"""Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it.
|
| 431 |
+
|
| 432 |
+
β WHY IT HAS TO HAPPEN HERE AND NOT ON A LAPTOP. The mirror is a FILE that lives beside the
|
| 433 |
+
process; the Space's copy is seeded from the HF dataset and knows nothing about a DuckDB on a
|
| 434 |
+
developer's box. Running `meta_store --sync` locally populates the local mirror and the LIVE
|
| 435 |
+
product stays empty β which is the whole difference between "the loader works" and "the
|
| 436 |
+
product has the data". Odoo is already arranged this way (`sync_all` runs in the container);
|
| 437 |
+
this is the same arrangement for the second connector.
|
| 438 |
+
|
| 439 |
+
β FAIL-QUIET AND SILENT WHEN THERE IS NOTHING TO DO. No token => no Meta => one line, no
|
| 440 |
+
error: a tenant that has not connected Meta is a normal state, and this runs on every boot.
|
| 441 |
+
β The window is deliberately SHORT here (`META_INSIGHTS_DAYS`, default 7 at boot) because boot
|
| 442 |
+
is not the place for a 90-day backfill β the resync pass widens it.
|
| 443 |
+
"""
|
| 444 |
+
try:
|
| 445 |
+
from harness import meta_store as _meta
|
| 446 |
+
if not _meta.token():
|
| 447 |
+
print(f"[aios-api] meta sync skipped ({why}): no META_ADS_ACCESS_TOKEN in this "
|
| 448 |
+
f"deployment - the connector is idle, not broken")
|
| 449 |
+
return
|
| 450 |
+
os.environ.setdefault("META_INSIGHTS_DAYS", "7")
|
| 451 |
+
rep = _meta.sync("royal-imports", log=lambda *_a: None)
|
| 452 |
+
for p in rep.get("problems") or []:
|
| 453 |
+
print(f"[aios-api] meta sync PROBLEM ({why}): {p}")
|
| 454 |
+
print(f"[aios-api] meta sync done ({why}): "
|
| 455 |
+
+ ", ".join(f"{k}={v['in_mirror']}" for k, v in sorted(rep["tables"].items())))
|
| 456 |
+
except Exception as e: # noqa: BLE001
|
| 457 |
+
print(f"[aios-api] meta sync FAILED ({why}): {type(e).__name__}: {e}")
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
def _rebuild_meta_relational(why, rt):
|
| 461 |
+
"""Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used.
|
| 462 |
+
|
| 463 |
+
β CALLED FROM INSIDE `_rebuild_odoo_relational`, ON PURPOSE, and the reason is D-29 rather than
|
| 464 |
+
tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and that caller has just
|
| 465 |
+
established which tenant's file this process holds open. Spawning Meta here inherits that
|
| 466 |
+
binding instead of rebinding it under live readers β which is the documented way to serve one
|
| 467 |
+
tenant's rows to another with nothing raised.
|
| 468 |
+
|
| 469 |
+
β SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Meta has no `meta_*` tables
|
| 470 |
+
in its mirror; `refresh` returns `{}` and says so once. That is a normal state, not a failure,
|
| 471 |
+
and it must not print an error every 30 minutes for every tenant that does not use Meta.
|
| 472 |
+
β Its own try/except for the reason the two passes above have theirs: a Meta refusal must not
|
| 473 |
+
cancel an Odoo rebuild that already succeeded.
|
| 474 |
+
"""
|
| 475 |
+
try:
|
| 476 |
+
import meta_relational as _meta
|
| 477 |
+
counts = _meta.refresh(rt, why)
|
| 478 |
+
if counts:
|
| 479 |
+
print(f"[aios-api] meta relational rebuild done ({why}): "
|
| 480 |
+
+ ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
| 481 |
+
try:
|
| 482 |
+
import automation_engine as _engine
|
| 483 |
+
_engine.refresh_relations(rt, log=lambda *_a: None)
|
| 484 |
+
except Exception as e: # noqa: BLE001
|
| 485 |
+
print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}")
|
| 486 |
+
except Exception as e: # noqa: BLE001
|
| 487 |
+
print(f"[aios-api] meta relational rebuild FAILED ({why}): {type(e).__name__}: {e}")
|
| 488 |
+
|
| 489 |
+
|
| 490 |
def _rebuild_odoo_relational(why):
|
| 491 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 492 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
|
|
|
| 540 |
print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
|
| 541 |
print(f"[aios-api] odoo relational rebuild done ({why}): "
|
| 542 |
+ ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
|
| 543 |
+
_rebuild_meta_relational(why, _rt)
|
| 544 |
except Exception as e: # noqa: BLE001
|
| 545 |
# β The TYPE is named here as it is in the two inner handlers. The original printed only
|
| 546 |
# `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
|
|
|
|
| 627 |
print(f"[aios-api] prewarm skipped: {e}")
|
| 628 |
|
| 629 |
|
| 630 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 631 |
+
# ββ W31-T46 / D-160 β THE MIRROR IS SEEDED WHETHER OR NOT `AIOS_PREWARM` IS SET.
|
| 632 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 633 |
+
#
|
| 634 |
+
# THE DEFECT, and it has cost a release once already (owner item 12, wave 20). `_prewarm()` was
|
| 635 |
+
# the ONLY caller of `_seed_and_sync_store()`, which is the ONLY caller of
|
| 636 |
+
# `datastore.ensure_seed()`. So on a fresh Space disk with `AIOS_PREWARM` anything but `1`:
|
| 637 |
+
# no `royal.duckdb` β `datastore.ready()` False forever β `ro_con()` refuses β every measure
|
| 638 |
+
# column blank AND β since wave 30 put the Odoo grids on the mirror β **two user-facing grids
|
| 639 |
+
# serve nothing**, with the app RUNNING, the deploy green and the tag correct. Last time the
|
| 640 |
+
# symptom was read as a pinned tag and a store problem for days.
|
| 641 |
+
#
|
| 642 |
+
# β AND THE FLAG IS EASIER TO LOSE THAN IT LOOKS: `deploy_web.py` pushes `AIOS_PREWARM=1`
|
| 643 |
+
# EXPLICITLY (to overwrite a stale `0`, because a Space secret survives a redeploy) β but that
|
| 644 |
+
# push sits under `if TARGET:`, so an ordinary bare `python deploy_web.py` skips it. A guard
|
| 645 |
+
# written for the dangerous case that the ordinary case walks straight past.
|
| 646 |
+
#
|
| 647 |
+
# β SO THE SEED IS SPLIT OFF AND MADE UNCONDITIONAL, which is T46's first branch rather than its
|
| 648 |
+
# fallback. It is the right half to move because it is the CHEAP, SAFE one: `ensure_seed()`
|
| 649 |
+
# touches no Odoo (it is one `hf_hub_download` of a snapshot), returns immediately when the file
|
| 650 |
+
# is already there, and returns immediately without `HF_TOKEN`. The EXPENSIVE, live half β
|
| 651 |
+
# `sync_all()`'s XML-RPC passes, the pool warm, the resync loop β stays exactly where it was,
|
| 652 |
+
# because the env gate's stated reason is still true: importing `api.main` in a gate must never
|
| 653 |
+
# fire a live Odoo pull.
|
| 654 |
+
#
|
| 655 |
+
# β THE `DB_PATH.exists()` PRE-CHECK IS WHAT KEEPS THIS FREE. It is a stat, and it is False only
|
| 656 |
+
# on a genuinely fresh disk β so on every developer box and in every gate run the thread is never
|
| 657 |
+
# started at all, and on a fresh Space it does exactly the thing whose absence blanks the grids.
|
| 658 |
+
#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a
|
| 659 |
+
#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause.
|
| 660 |
+
MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""}
|
| 661 |
+
|
| 662 |
+
|
| 663 |
+
def _seed_mirror_if_absent():
|
| 664 |
+
"""Hydrate the analytical mirror when this container has none β INDEPENDENT of `AIOS_PREWARM`.
|
| 665 |
+
|
| 666 |
+
Fail-quiet by design, and it records WHY rather than only whether: "there is no mirror and no
|
| 667 |
+
HF_TOKEN to fetch one" and "there is no mirror and the fetch failed" are different operator
|
| 668 |
+
actions, and a blank grid cannot tell them apart.
|
| 669 |
+
"""
|
| 670 |
+
from harness import datastore as _ds
|
| 671 |
+
MIRROR_SEED["attempted"] = True
|
| 672 |
+
try:
|
| 673 |
+
if _ds.ensure_seed():
|
| 674 |
+
MIRROR_SEED["seeded"] = True
|
| 675 |
+
print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
|
| 676 |
+
return True
|
| 677 |
+
if not _ds.DB_PATH.exists():
|
| 678 |
+
MIRROR_SEED["cause"] = (
|
| 679 |
+
"this container has no analytical mirror and the seed snapshot could not be "
|
| 680 |
+
"fetched (no HF_TOKEN, or the dataset was unreachable)")
|
| 681 |
+
MIRROR_SEED["recommendation"] = (
|
| 682 |
+
"set HF_TOKEN on the deployment; until then every connected grid and every "
|
| 683 |
+
"measure column served from the mirror is empty")
|
| 684 |
+
print(f"[aios-api] NO ANALYTICAL MIRROR: {MIRROR_SEED['cause']}")
|
| 685 |
+
except Exception as e: # noqa: BLE001 β boot must not die
|
| 686 |
+
MIRROR_SEED["cause"] = f"the boot seed raised {type(e).__name__}: {e}"
|
| 687 |
+
MIRROR_SEED["recommendation"] = "check HF_TOKEN and the seed dataset's availability"
|
| 688 |
+
print(f"[aios-api] mirror seed skipped: {e}")
|
| 689 |
+
return False
|
| 690 |
+
|
| 691 |
+
|
| 692 |
+
try:
|
| 693 |
+
from harness import datastore as _ds_boot
|
| 694 |
+
if not _ds_boot.DB_PATH.exists():
|
| 695 |
+
import threading as _threading_seed
|
| 696 |
+
_threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True,
|
| 697 |
+
name="mirror-seed").start()
|
| 698 |
+
except Exception as e: # noqa: BLE001
|
| 699 |
+
print(f"[aios-api] mirror seed not scheduled: {e}")
|
| 700 |
+
|
| 701 |
if os.environ.get("AIOS_PREWARM") == "1":
|
| 702 |
import threading as _threading
|
| 703 |
_threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
|
api/meta_relational.py
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""meta_relational.py β the Meta Ads mirror becomes LOCKED DATABASES (W31-T47, ruling R2).
|
| 2 |
+
|
| 3 |
+
The second half of the owner's instruction: *"we just need to pull the data into our template
|
| 4 |
+
database for Meta"*, *"much like how we have done so for Odoo"*. `harness/meta_store.py` pulls
|
| 5 |
+
Graph into the tenant's DuckDB mirror; this reads that mirror and spawns the `ut_meta_*` grids a
|
| 6 |
+
person opens. It is `odoo_relational.py`'s shape with one source swapped.
|
| 7 |
+
|
| 8 |
+
plan(cur) the rows that WOULD be written β no store write at all
|
| 9 |
+
apply_plan(rt, built) create-or-merge every table and its rows, in ONE sync write
|
| 10 |
+
refresh(rt, why) both, with the mirror opened for this tenant
|
| 11 |
+
|
| 12 |
+
ββ THE COLUMNS ARE DERIVED FROM THE MIRROR, NOT TYPED OUT, AND THAT IS THE R2 REQUIREMENT MADE
|
| 13 |
+
STRUCTURAL. The owner's words were *"understand FULLY the schema, do not drop any column etc."* A
|
| 14 |
+
hand-written field list is a second place for the schema to live and the first place for it to
|
| 15 |
+
rot β so `_fields_for` reads `PRAGMA table_info` off the table `meta_store` just wrote and emits
|
| 16 |
+
one preset field per column. Add a column to the loader's measured list and it appears in the grid
|
| 17 |
+
with no edit here. **285 columns across five levels plus the daily Insights grain.**
|
| 18 |
+
|
| 19 |
+
β WHY IT REUSES `odoo_relational._ensure_table_inplace` RATHER THAN COPYING IT. That function is
|
| 20 |
+
where a `ut_*` table becomes a LOCKED DATABASE β `recordMode = AUTOMATION_RECORD_MODE`, the preset
|
| 21 |
+
stamp, the MAX_TABLES refusal, the row reconcile. Two copies of that would be two answers to "is
|
| 22 |
+
this database locked", and the copies would diverge on the first ruling that touches one of them.
|
| 23 |
+
One implementation, two callers, exactly as `_LentTables` became `user_tables.lend` this wave.
|
| 24 |
+
|
| 25 |
+
β MATERIALISED, NOT READ-THROUGH, AND THE REASON IS THE NUMBERS. Odoo's `ut_odoo_gl_lines` reads
|
| 26 |
+
THROUGH the mirror because it is 963,783 rows against a `MAX_ROWS` of 60,000. Meta at this tenant's
|
| 27 |
+
scale is 1 account Β· 55 campaigns Β· 115 ad sets Β· 227 ads Β· creatives Β· ~90 days of daily insights
|
| 28 |
+
β every table two to four orders of magnitude inside the cap, so they are ordinary materialised
|
| 29 |
+
`ut_*` tables like `ut_odoo_agents`. β If a bigger account ever crosses the cap, `plan()` says so
|
| 30 |
+
out loud rather than truncating: R6's second sentence, and the check is at the bottom of `plan`.
|
| 31 |
+
"""
|
| 32 |
+
import os
|
| 33 |
+
import sys
|
| 34 |
+
from pathlib import Path
|
| 35 |
+
|
| 36 |
+
_API = Path(__file__).resolve().parent
|
| 37 |
+
if str(_API.parent.parent / "platform") not in sys.path:
|
| 38 |
+
sys.path.insert(0, str(_API.parent.parent / "platform"))
|
| 39 |
+
|
| 40 |
+
import odoo_relational as _odoo # noqa: E402
|
| 41 |
+
from harness import datastore, meta_store # noqa: E402
|
| 42 |
+
|
| 43 |
+
Refused = _odoo.Refused
|
| 44 |
+
|
| 45 |
+
ACCOUNTS_KEY = "ut_meta_ad_accounts"
|
| 46 |
+
CAMPAIGNS_KEY = "ut_meta_campaigns"
|
| 47 |
+
ADSETS_KEY = "ut_meta_adsets"
|
| 48 |
+
ADS_KEY = "ut_meta_ads"
|
| 49 |
+
CREATIVES_KEY = "ut_meta_creatives"
|
| 50 |
+
INSIGHTS_KEY = "ut_meta_insights"
|
| 51 |
+
|
| 52 |
+
#: ut_* key -> the mirror table `meta_store` writes.
|
| 53 |
+
SOURCE = {
|
| 54 |
+
ACCOUNTS_KEY: "meta_ad_accounts",
|
| 55 |
+
CAMPAIGNS_KEY: "meta_campaigns",
|
| 56 |
+
ADSETS_KEY: "meta_adsets",
|
| 57 |
+
ADS_KEY: "meta_ads",
|
| 58 |
+
CREATIVES_KEY: "meta_creatives",
|
| 59 |
+
INSIGHTS_KEY: meta_store.INSIGHTS_TABLE,
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
LABELS = {
|
| 63 |
+
ACCOUNTS_KEY: "Meta ad accounts", CAMPAIGNS_KEY: "Meta campaigns",
|
| 64 |
+
ADSETS_KEY: "Meta ad sets", ADS_KEY: "Meta ads",
|
| 65 |
+
CREATIVES_KEY: "Meta creatives", INSIGHTS_KEY: "Meta insights (daily)",
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
#: The column each grid shows first. Everything else keeps the API's own order, which is the order
|
| 69 |
+
#: the schema was measured in β a reader comparing the grid to Graph sees the same sequence.
|
| 70 |
+
PINNED = {ACCOUNTS_KEY: "name", CAMPAIGNS_KEY: "name", ADSETS_KEY: "name",
|
| 71 |
+
ADS_KEY: "name", CREATIVES_KEY: "name", INSIGHTS_KEY: "date_start"}
|
| 72 |
+
|
| 73 |
+
#: (this table's column) -> (target ut_ key, the column it matches there). Derived links, so a
|
| 74 |
+
#: person can walk account -> campaign -> ad set -> ad in the product the way they do in Ads
|
| 75 |
+
#: Manager. β Only emitted when BOTH sides exist in the mirror.
|
| 76 |
+
LINKS = {
|
| 77 |
+
CAMPAIGNS_KEY: ("account_id", ACCOUNTS_KEY, "account_id"),
|
| 78 |
+
ADSETS_KEY: ("campaign_id", CAMPAIGNS_KEY, "id"),
|
| 79 |
+
ADS_KEY: ("adset_id", ADSETS_KEY, "id"),
|
| 80 |
+
INSIGHTS_KEY: ("ad_id", ADS_KEY, "id"),
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
#: DuckDB type -> `core.user_tables.UT_FIELD_TYPES` member. β `_clean_field` returns None for an
|
| 84 |
+
#: unknown type, which DELETES the column on the next read rather than erroring β so this map only
|
| 85 |
+
#: ever emits members of that set, and anything unrecognised falls back to `text`.
|
| 86 |
+
_TYPE = {"BIGINT": "int", "INTEGER": "int", "HUGEINT": "int",
|
| 87 |
+
"DOUBLE": "currency", "FLOAT": "currency", "DECIMAL": "currency"}
|
| 88 |
+
|
| 89 |
+
#: Columns whose NAME makes them a date regardless of storage. Graph returns ISO-ish strings.
|
| 90 |
+
_DATEISH = ("_time", "date_start", "date_stop", "created_time", "updated_time",
|
| 91 |
+
"start_time", "stop_time", "end_time")
|
| 92 |
+
|
| 93 |
+
#: β A MONEY COLUMN IS `currency`, NOT `int`, and the distinction is not cosmetic: `spend` is
|
| 94 |
+
#: fractional in every currency Meta reports and an int column would round every row. The account
|
| 95 |
+
#: reports HKD; the grid does not convert, it shows what the API said.
|
| 96 |
+
_MONEY = {"spend", "social_spend", "cpc", "cpm", "cpp", "amount_spent", "balance", "spend_cap",
|
| 97 |
+
"daily_budget", "lifetime_budget", "budget_remaining", "bid_amount",
|
| 98 |
+
"cost_per_inline_link_click", "cost_per_unique_click", "cost_per_thruplay"}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _label(col):
|
| 102 |
+
return col.replace("_", " ").strip().capitalize()
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _ftype(col, duck_type):
|
| 106 |
+
if any(col.endswith(s) or col == s for s in _DATEISH):
|
| 107 |
+
return "date"
|
| 108 |
+
if col in _MONEY:
|
| 109 |
+
return "currency"
|
| 110 |
+
base = str(duck_type or "").upper().split("(")[0]
|
| 111 |
+
if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"):
|
| 112 |
+
return _TYPE.get(base, "int")
|
| 113 |
+
if base in ("DOUBLE", "FLOAT"):
|
| 114 |
+
return "currency"
|
| 115 |
+
return "text"
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def mirror_columns(cur, table):
|
| 119 |
+
"""[(name, type)] for a mirror table, or [] when it has never been synced."""
|
| 120 |
+
try:
|
| 121 |
+
return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
|
| 122 |
+
except Exception:
|
| 123 |
+
return []
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _fields_for(cur, key):
|
| 127 |
+
"""One preset field per mirror column β the whole schema, derived (see the header)."""
|
| 128 |
+
cols = mirror_columns(cur, SOURCE[key])
|
| 129 |
+
if not cols:
|
| 130 |
+
return []
|
| 131 |
+
pinned = PINNED.get(key)
|
| 132 |
+
link = LINKS.get(key)
|
| 133 |
+
out = []
|
| 134 |
+
for name, dtype in cols:
|
| 135 |
+
f = {"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
| 136 |
+
"source": "overlay", "default": name in (pinned, "id", "name", "status", "spend",
|
| 137 |
+
"impressions", "clicks", "date_start"),
|
| 138 |
+
"pinned": name == pinned}
|
| 139 |
+
out.append(_odoo._preset(f, flow="meta_relational"))
|
| 140 |
+
if link:
|
| 141 |
+
col, target, on = link
|
| 142 |
+
if any(c[0] == col for c in cols):
|
| 143 |
+
out.append(_odoo._preset(
|
| 144 |
+
{"key": f"{target}_link", "label": LABELS[target], "type": "link",
|
| 145 |
+
"source": "overlay", "default": True,
|
| 146 |
+
"link": {"table": target, "on": on, "from": col}}, flow="meta_relational"))
|
| 147 |
+
out.append(_odoo._preset(_odoo._refreshed_field(), flow="meta_relational"))
|
| 148 |
+
return out
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _read(cur, key):
|
| 152 |
+
"""[(row dict)] straight off the mirror. `_id` is the object's own Meta id β so a re-sync
|
| 153 |
+
updates in place and 'every id Graph returned is in the table' is checkable, not hopeful."""
|
| 154 |
+
cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
|
| 155 |
+
if not cols:
|
| 156 |
+
return []
|
| 157 |
+
rows = cur.execute(f"SELECT {', '.join(cols)} FROM {SOURCE[key]}").fetchall()
|
| 158 |
+
out = []
|
| 159 |
+
for r in rows:
|
| 160 |
+
row = {"_id": str(r[cols.index("id")]) if "id" in cols else str(len(out))}
|
| 161 |
+
for name, val in zip(cols, r):
|
| 162 |
+
row[name] = "" if val is None else (val if isinstance(val, (int, float)) else str(val))
|
| 163 |
+
out.append(row)
|
| 164 |
+
return out
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def available(tenant_key="royal-imports"):
|
| 168 |
+
"""Has this tenant's mirror ever been fed by `meta_store`? Cheap; never fetches."""
|
| 169 |
+
try:
|
| 170 |
+
counts = meta_store.status(tenant_key)
|
| 171 |
+
except Exception:
|
| 172 |
+
return False
|
| 173 |
+
return any(bool(v) for v in counts.values())
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def plan(cur, rt=None):
|
| 177 |
+
"""{bucket_key: rows} plus `problems`. No store write. Every cap checked BEFORE anything is
|
| 178 |
+
committed β the same order `odoo_relational.plan` keeps and for the same reason."""
|
| 179 |
+
built = {"problems": []}
|
| 180 |
+
ut = _odoo._ut()
|
| 181 |
+
for key in SOURCE:
|
| 182 |
+
cap = getattr(ut, "MAX_ROWS", 60_000)
|
| 183 |
+
# β COUNT BEFORE READ, not after. `_read` materialises every row as a dict β the whole
|
| 184 |
+
# table twice over, once from DuckDB and once as python β and checking the cap afterwards
|
| 185 |
+
# means the one table too big to keep is the one we build in memory first. A `count(*)` is
|
| 186 |
+
# free and answers the same question.
|
| 187 |
+
try:
|
| 188 |
+
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 189 |
+
except Exception:
|
| 190 |
+
continue # table absent = never synced
|
| 191 |
+
if n > cap:
|
| 192 |
+
built["problems"].append(
|
| 193 |
+
f"{key}: the mirror holds {n:,} rows against MAX_ROWS={cap:,}. Nothing was "
|
| 194 |
+
f"truncated and nothing was written for this table. Cause: this account is larger "
|
| 195 |
+
f"than a materialised ut_* table can hold. Fix: bind it read-through off the "
|
| 196 |
+
f"mirror as `ut_odoo_gl_lines` is (odoo_relational.READ_THROUGH_KEYS), which is "
|
| 197 |
+
f"what `routes_connected_tables` already serves.")
|
| 198 |
+
continue
|
| 199 |
+
rows = _read(cur, key)
|
| 200 |
+
if not rows:
|
| 201 |
+
continue
|
| 202 |
+
if len(rows) > cap:
|
| 203 |
+
# β REPORTED, NEVER TRUNCATED (R6's second sentence). The fix is named so the reader
|
| 204 |
+
# does not have to derive it: this table becomes read-through like the Odoo line
|
| 205 |
+
# grains, which is a build, not a bigger constant.
|
| 206 |
+
built["problems"].append(
|
| 207 |
+
f"{key}: the mirror holds {len(rows):,} rows against MAX_ROWS={cap:,}. Nothing was "
|
| 208 |
+
f"truncated and nothing was written for this table. Cause: this account is larger "
|
| 209 |
+
f"than a materialised ut_* table can hold. Fix: bind it read-through off the "
|
| 210 |
+
f"mirror as `ut_odoo_gl_lines` is (odoo_relational.READ_THROUGH_KEYS).")
|
| 211 |
+
continue
|
| 212 |
+
built[key] = rows
|
| 213 |
+
return built
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def apply_plan(rt, built, username="meta", today=None):
|
| 217 |
+
"""Create-or-merge every table in `built`, in ONE `flush="sync"` write.
|
| 218 |
+
|
| 219 |
+
β ONE write for all six tables, not six β `Store.update` is a full download plus a full upload
|
| 220 |
+
of the tenant document, so six would be six round trips and six commits every resync. It is
|
| 221 |
+
also ATOMIC: a `Refused` at MAX_TABLES aborts before anything persists, instead of leaving the
|
| 222 |
+
half-spawn `plan()` exists to prevent.
|
| 223 |
+
"""
|
| 224 |
+
if built.get("problems"):
|
| 225 |
+
raise Refused("; ".join(built["problems"]))
|
| 226 |
+
stamp = today or _odoo._iso_today()
|
| 227 |
+
written = {}
|
| 228 |
+
cur_mirror = datastore.ro_con()
|
| 229 |
+
plans = [(key, LABELS[key], _fields_for(cur_mirror, key), built[key])
|
| 230 |
+
for key in SOURCE if key in built]
|
| 231 |
+
|
| 232 |
+
def _apply_all(doc):
|
| 233 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 234 |
+
for key, label, fields, rows in plans:
|
| 235 |
+
written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
|
| 236 |
+
username, stamp)
|
| 237 |
+
return doc
|
| 238 |
+
|
| 239 |
+
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
| 240 |
+
return written
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def refresh(rt, why="boot", tenant_key=None, log=print):
|
| 244 |
+
"""The one call a caller wants: open this tenant's mirror, plan, apply. -> written counts.
|
| 245 |
+
|
| 246 |
+
Returns `{}` and says why when the mirror has no Meta tables β a tenant that never connected
|
| 247 |
+
Meta is a normal state, not a failure.
|
| 248 |
+
"""
|
| 249 |
+
tenant = tenant_key or getattr(getattr(rt, "tenant", None), "key", None) or "royal-imports"
|
| 250 |
+
path = datastore.path_for(tenant)
|
| 251 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 252 |
+
datastore.use_path(path)
|
| 253 |
+
cur = datastore.ro_con()
|
| 254 |
+
if not any(mirror_columns(cur, t) for t in SOURCE.values()):
|
| 255 |
+
log(f"[meta_relational] {why}: no Meta tables in {tenant}'s mirror β nothing to spawn "
|
| 256 |
+
f"(run `python platform/harness/meta_store.py --sync --tenant {tenant}` first)")
|
| 257 |
+
return {}
|
| 258 |
+
built = plan(cur, rt)
|
| 259 |
+
for p in built.get("problems") or []:
|
| 260 |
+
log(f"[meta_relational] PROBLEM {p}")
|
| 261 |
+
if built.get("problems"):
|
| 262 |
+
built = {k: v for k, v in built.items() if k != "problems"}
|
| 263 |
+
written = apply_plan(rt, built)
|
| 264 |
+
for key, counts in written.items():
|
| 265 |
+
log(f"[meta_relational] {why}: {key:<24} {counts}")
|
| 266 |
+
return written
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def main(argv=None):
|
| 270 |
+
"""`python aios-web/api/meta_relational.py --refresh [--tenant X]` β the manual door."""
|
| 271 |
+
import argparse
|
| 272 |
+
ap = argparse.ArgumentParser()
|
| 273 |
+
ap.add_argument("--refresh", action="store_true")
|
| 274 |
+
ap.add_argument("--plan", action="store_true")
|
| 275 |
+
ap.add_argument("--tenant", default="royal-imports")
|
| 276 |
+
a = ap.parse_args(argv)
|
| 277 |
+
os.environ.setdefault("AIOS_PREWARM", "0")
|
| 278 |
+
from harness import runtime as _rt
|
| 279 |
+
rt = _rt.get_runtime(a.tenant)
|
| 280 |
+
if a.plan:
|
| 281 |
+
path = datastore.path_for(a.tenant)
|
| 282 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 283 |
+
datastore.use_path(path)
|
| 284 |
+
built = plan(datastore.ro_con(), rt)
|
| 285 |
+
for k, v in built.items():
|
| 286 |
+
print(f" {k:<26} {len(v) if isinstance(v, list) else v}")
|
| 287 |
+
return 0
|
| 288 |
+
if a.refresh:
|
| 289 |
+
refresh(rt, why="cli", tenant_key=a.tenant)
|
| 290 |
+
return 0
|
| 291 |
+
ap.print_help()
|
| 292 |
+
return 2
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
if __name__ == "__main__":
|
| 296 |
+
sys.exit(main())
|
api/oauth_connect.py
CHANGED
|
@@ -380,3 +380,110 @@ def gmail_message(access_token, message_id):
|
|
| 380 |
#: `re` is imported for the slot-name law in core.keychain (see USER_SECRET_PROVIDERS there);
|
| 381 |
#: keeping the reference here so a linter cannot 'clean up' what the registry contract uses.
|
| 382 |
_SLOT_SHAPE = re.compile(r"^oauth_[a-z0-9_]{1,24}$")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 380 |
#: `re` is imported for the slot-name law in core.keychain (see USER_SECRET_PROVIDERS there);
|
| 381 |
#: keeping the reference here so a linter cannot 'clean up' what the registry contract uses.
|
| 382 |
_SLOT_SHAPE = re.compile(r"^oauth_[a-z0-9_]{1,24}$")
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 386 |
+
# β W31-T41 β THE NANGO PROBE. Inert on import; reached only by `python oauth_connect.py --nango`.
|
| 387 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 388 |
+
#
|
| 389 |
+
# R5 makes Nango a production dependency ("Meta connects by clicking Connect Facebook; Nango owns
|
| 390 |
+
# that flow"), and T41 is its GO/NO-GO: does the key authenticate, does the Facebook integration
|
| 391 |
+
# carry Marketing API scopes (`ads_read`), and what does the free tier allow against our connection
|
| 392 |
+
# count. β It lives HERE because `::creds` is the seam every poller reads and therefore the seam
|
| 393 |
+
# Nango slots into β the probe belongs beside the thing it would replace, not in a scratch file
|
| 394 |
+
# that dies with the session.
|
| 395 |
+
#
|
| 396 |
+
# β NOTHING BELOW IS WIRED INTO THE FLOW. `PROVIDERS`, `start`, `callback` and `creds` are
|
| 397 |
+
# untouched by this block: a measurement that changes the live auth path before its own verdict is
|
| 398 |
+
# in is the shape R5 was careful to avoid.
|
| 399 |
+
NANGO_API = os.environ.get("NANGO_API_BASE") or "https://api.nango.dev"
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def nango_key():
|
| 403 |
+
"""`NANGO_SECRET_KEY` from the environment or gitignored `platform/.env`; "" when absent."""
|
| 404 |
+
from pathlib import Path as _Path
|
| 405 |
+
tok = os.environ.get("NANGO_SECRET_KEY") or ""
|
| 406 |
+
if tok:
|
| 407 |
+
return tok.strip()
|
| 408 |
+
env = _Path(__file__).resolve().parents[2] / "platform" / ".env"
|
| 409 |
+
if env.exists():
|
| 410 |
+
for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 411 |
+
if line.strip().startswith("NANGO_SECRET_KEY"):
|
| 412 |
+
_, _, v = line.partition("=")
|
| 413 |
+
return v.strip().strip('"').strip("'")
|
| 414 |
+
return ""
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def nango_probe():
|
| 418 |
+
"""`{authenticated, integrations, connections, errors}` β read-only GETs against Nango Cloud.
|
| 419 |
+
|
| 420 |
+
β NEVER PRINTS THE KEY. It rides an Authorization header and the report carries integration
|
| 421 |
+
ids and scope strings only.
|
| 422 |
+
β The scope question is the GO/NO-GO: an integration row's own config states which scopes it
|
| 423 |
+
requests, and `ads_read` is what R2's Insights reads need. If it is absent, the fallback is the
|
| 424 |
+
keychain token path β which `connectors_meta.py` has already built β and that has to be said
|
| 425 |
+
BEFORE any OAuth code is written, per the ticket.
|
| 426 |
+
"""
|
| 427 |
+
import base64
|
| 428 |
+
import json as _json
|
| 429 |
+
import urllib.error
|
| 430 |
+
import urllib.request
|
| 431 |
+
key = nango_key()
|
| 432 |
+
out = {"base": NANGO_API, "authenticated": False, "integrations": [], "connections": [],
|
| 433 |
+
"errors": []}
|
| 434 |
+
if not key:
|
| 435 |
+
out["errors"].append(
|
| 436 |
+
"NANGO_SECRET_KEY is not set in the environment or in platform/.env. Nothing was "
|
| 437 |
+
"measured. The PRD records it as 'recorded in platform/.env' and it is not there.")
|
| 438 |
+
return out
|
| 439 |
+
# β BEARER, NOT BASIC β measured 2026-08-12 by A, against the owner's real key, once it landed.
|
| 440 |
+
# Basic (`base64(key + ":")`, the OLD Nango style) answers **401 `invalid_secret_key_format`:
|
| 441 |
+
# "The provided secret key is not a UUID v4."** β which reads exactly like a bad credential and
|
| 442 |
+
# is not: Nango takes the whole base64 blob as the key and correctly observes it is not a UUID.
|
| 443 |
+
# The SAME key over `Bearer` answers **200 with 1 integration**. β The lesson is the error
|
| 444 |
+
# message: it described the VALUE when the fault was in the ENVELOPE, so the obvious next step
|
| 445 |
+
# ("ask the owner for a valid key") would have been wrong and would have read as their mistake.
|
| 446 |
+
auth = f"Bearer {key}"
|
| 447 |
+
|
| 448 |
+
def _get(path):
|
| 449 |
+
req = urllib.request.Request(f"{NANGO_API}{path}",
|
| 450 |
+
headers={"Authorization": auth,
|
| 451 |
+
"User-Agent": "aios-nango-probe/1"})
|
| 452 |
+
with urllib.request.urlopen(req, timeout=45) as r:
|
| 453 |
+
return _json.loads(r.read().decode("utf-8", "replace"))
|
| 454 |
+
|
| 455 |
+
for label, path, sink in (("integrations", "/integrations", "integrations"),
|
| 456 |
+
("connections", "/connection", "connections")):
|
| 457 |
+
try:
|
| 458 |
+
body = _get(path)
|
| 459 |
+
rows = body if isinstance(body, list) else (body.get("configs")
|
| 460 |
+
or body.get("connections")
|
| 461 |
+
or body.get("data") or [])
|
| 462 |
+
out[sink] = rows
|
| 463 |
+
out["authenticated"] = True
|
| 464 |
+
except urllib.error.HTTPError as e:
|
| 465 |
+
out["errors"].append(f"{label}: HTTP {e.code} β "
|
| 466 |
+
f"{e.read().decode('utf-8', 'replace')[:200]}")
|
| 467 |
+
except Exception as e: # noqa: BLE001
|
| 468 |
+
out["errors"].append(f"{label}: {type(e).__name__}: {e}")
|
| 469 |
+
return out
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
if __name__ == "__main__": # pragma: no cover
|
| 473 |
+
import json as _j
|
| 474 |
+
import sys as _sys
|
| 475 |
+
if "--nango" in _sys.argv:
|
| 476 |
+
rep = nango_probe()
|
| 477 |
+
print(f"Nango key present: {'yes' if nango_key() else 'NO'} (never printed)")
|
| 478 |
+
print(f"authenticated: {rep['authenticated']} Β· integrations: "
|
| 479 |
+
f"{len(rep['integrations'])} Β· connections: {len(rep['connections'])}")
|
| 480 |
+
for e in rep["errors"]:
|
| 481 |
+
print(f" ERROR {e}")
|
| 482 |
+
for i in rep["integrations"]:
|
| 483 |
+
if isinstance(i, dict):
|
| 484 |
+
print(" integration:", _j.dumps({k: v for k, v in i.items()
|
| 485 |
+
if k in ("unique_key", "provider", "scopes",
|
| 486 |
+
"oauth_scopes")}))
|
| 487 |
+
_sys.exit(0 if rep["authenticated"] else 2)
|
| 488 |
+
print("usage: python oauth_connect.py --nango")
|
| 489 |
+
_sys.exit(2)
|
api/odoo_relational.py
CHANGED
|
@@ -1479,6 +1479,17 @@ def refresh(rt, tenant, username="automation", cur=None, today=None):
|
|
| 1479 |
if not is_royal(tenant):
|
| 1480 |
raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal "
|
| 1481 |
f"Imports only); refusing to spawn empty locked databases")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1482 |
if cur is None:
|
| 1483 |
from harness import datastore
|
| 1484 |
cur = datastore.ro_con()
|
|
|
|
| 1479 |
if not is_royal(tenant):
|
| 1480 |
raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal "
|
| 1481 |
f"Imports only); refusing to spawn empty locked databases")
|
| 1482 |
+
# ββ W31-T45 / D-169 β THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS,
|
| 1483 |
+
# and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is
|
| 1484 |
+
# this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process
|
| 1485 |
+
# has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a
|
| 1486 |
+
# provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked
|
| 1487 |
+
# databases from another customer's rows β a spawn, not a read, so the wrong numbers become
|
| 1488 |
+
# durable. Entitlement is not residency.
|
| 1489 |
+
# β It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot
|
| 1490 |
+
# rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks.
|
| 1491 |
+
if rt is not None:
|
| 1492 |
+
rt.assert_datastore_matches()
|
| 1493 |
if cur is None:
|
| 1494 |
from harness import datastore
|
| 1495 |
cur = datastore.ro_con()
|
api/routes_alerts.py
CHANGED
|
@@ -66,8 +66,21 @@ def _owner_session(session: Session, owner: str):
|
|
| 66 |
claims=session.claims, runtime=session.runtime)
|
| 67 |
|
| 68 |
|
| 69 |
-
def _evaluate(session: Session, rec: dict):
|
| 70 |
-
"""Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
import aios_grid
|
| 72 |
from harness import filter_eval
|
| 73 |
|
|
@@ -75,16 +88,28 @@ def _evaluate(session: Session, rec: dict):
|
|
| 75 |
if owner_sess is None:
|
| 76 |
return {"skipped": "owner_unavailable"}
|
| 77 |
topic = str(rec.get("topic") or "")
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
view = (g.get("views") or {}).get(str(rec.get("viewId")))
|
| 90 |
if not isinstance(view, dict):
|
|
@@ -220,10 +245,22 @@ def notifications(session: Session = Depends(require_session)):
|
|
| 220 |
the row landed, so the `at` stamp is detection time, not arrival time.
|
| 221 |
|
| 222 |
`after_write` stays exported for the day the engine can hand over a real identity.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
"""
|
|
|
|
| 224 |
for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
|
| 225 |
try:
|
| 226 |
-
_evaluate(session, rec)
|
| 227 |
except Exception: # noqa: BLE001
|
| 228 |
continue # one bad alert must not empty the pane
|
| 229 |
return alerts.inbox(session.uname, st=session.runtime)
|
|
@@ -246,9 +283,17 @@ def after_write(session: Session, topic_key: str):
|
|
| 246 |
Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
|
| 247 |
upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
|
| 248 |
edit that triggered it.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
"""
|
| 250 |
try:
|
|
|
|
| 251 |
return alerts.after_write(topic_key, st=session.runtime,
|
| 252 |
-
runner=lambda rec: _evaluate(session, rec
|
|
|
|
| 253 |
except Exception: # noqa: BLE001
|
| 254 |
return {"evaluated": 0}
|
|
|
|
| 66 |
claims=session.claims, runtime=session.runtime)
|
| 67 |
|
| 68 |
|
| 69 |
+
def _evaluate(session: Session, rec: dict, assemblies=None):
|
| 70 |
+
"""Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
|
| 71 |
+
|
| 72 |
+
ββ W31-T24 β `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
|
| 73 |
+
of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
|
| 74 |
+
one built a FULL assembly β the pool, the workspace, `rows_from_pool` over every row. Two
|
| 75 |
+
alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
|
| 76 |
+
because each `_evaluate` was a closed call.
|
| 77 |
+
β `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
|
| 78 |
+
note β evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
|
| 79 |
+
wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
|
| 80 |
+
must never share an entry. Getting that key wrong is the one way this optimisation could leak.
|
| 81 |
+
β Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
|
| 82 |
+
they evaluate ONE alert and a memo for a single call is pure overhead.
|
| 83 |
+
"""
|
| 84 |
import aios_grid
|
| 85 |
from harness import filter_eval
|
| 86 |
|
|
|
|
| 88 |
if owner_sess is None:
|
| 89 |
return {"skipped": "owner_unavailable"}
|
| 90 |
topic = str(rec.get("topic") or "")
|
| 91 |
+
memo_key = (topic, str(owner_sess.uname))
|
| 92 |
+
g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
|
| 93 |
+
if g is None:
|
| 94 |
+
try:
|
| 95 |
+
if topic.startswith("ut_"):
|
| 96 |
+
from routes_tables import ut_assembly
|
| 97 |
+
# β `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
|
| 98 |
+
# `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
|
| 99 |
+
# one-shot field-name correction acks for every `ut_` topic that has an alert β
|
| 100 |
+
# taking them from the `/workspace` refresh that exists to show them to the person
|
| 101 |
+
# who made the edit. The customer branch below has always passed False; this one
|
| 102 |
+
# inherited a default nobody re-read. An inbox poll must never consume a one-shot.
|
| 103 |
+
g = ut_assembly(owner_sess, topic,
|
| 104 |
+
storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
|
| 105 |
+
consume_corrections=False)
|
| 106 |
+
else:
|
| 107 |
+
from routes_customers import grid_assembly
|
| 108 |
+
g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
|
| 109 |
+
except Exception as e: # noqa: BLE001
|
| 110 |
+
return {"skipped": "unavailable", "detail": type(e).__name__}
|
| 111 |
+
if isinstance(assemblies, dict):
|
| 112 |
+
assemblies[memo_key] = g
|
| 113 |
|
| 114 |
view = (g.get("views") or {}).get(str(rec.get("viewId")))
|
| 115 |
if not isinstance(view, dict):
|
|
|
|
| 245 |
the row landed, so the `at` stamp is detection time, not arrival time.
|
| 246 |
|
| 247 |
`after_write` stays exported for the day the engine can hand over a real identity.
|
| 248 |
+
|
| 249 |
+
ββ W31-T24 β ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
|
| 250 |
+
β MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
|
| 251 |
+
**20 ms in-process and 3,280 ms live** on tenant #0 β but tenant #0 has **ZERO alerts**
|
| 252 |
+
(censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
|
| 253 |
+
re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
|
| 254 |
+
body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
|
| 255 |
+
grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic Γ owner), which
|
| 256 |
+
is the difference between "fine" and "three seconds per alert" the day somebody uses the
|
| 257 |
+
feature. β Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
|
| 258 |
+
this does: every alert is still evaluated, against the same rows, in the same order.
|
| 259 |
"""
|
| 260 |
+
assemblies = {}
|
| 261 |
for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
|
| 262 |
try:
|
| 263 |
+
_evaluate(session, rec, assemblies=assemblies)
|
| 264 |
except Exception: # noqa: BLE001
|
| 265 |
continue # one bad alert must not empty the pane
|
| 266 |
return alerts.inbox(session.uname, st=session.runtime)
|
|
|
|
| 283 |
Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
|
| 284 |
upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
|
| 285 |
edit that triggered it.
|
| 286 |
+
|
| 287 |
+
β W31-T24 β it shares `/notifications`' memo shape for the same reason: a write that changes
|
| 288 |
+
one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
|
| 289 |
+
β STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
|
| 290 |
+
push hook was resolved the other way). Booked rather than wired: minting a session inside the
|
| 291 |
+
engine's worker thread is the ad-hoc identity construction this file exists to avoid.
|
| 292 |
"""
|
| 293 |
try:
|
| 294 |
+
assemblies = {}
|
| 295 |
return alerts.after_write(topic_key, st=session.runtime,
|
| 296 |
+
runner=lambda rec: _evaluate(session, rec,
|
| 297 |
+
assemblies=assemblies))
|
| 298 |
except Exception: # noqa: BLE001
|
| 299 |
return {"evaluated": 0}
|
api/routes_automation.py
CHANGED
|
@@ -257,44 +257,35 @@ _DISCOVER_DEFAULT_TTL = float(os.environ.get("AIOS_AUTOMATION_DEFAULT_TTL") or 3
|
|
| 257 |
_UT_STORE_KEY = engine.UT_STORE_KEY
|
| 258 |
|
| 259 |
|
| 260 |
-
|
| 261 |
-
"""ββ WAVE 30 Β· T13 β a read-only `st` that serves ONE already-read `user_tables`
|
| 262 |
-
and passes every other key straight through to the real runtime.
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
resolver, unmodified and still called per table; it is simply no longer charged for a document
|
| 277 |
the caller is already holding.
|
| 278 |
-
β AND IT IS A CALLER-SIDE FIX ON PURPOSE: adding a `tables=` parameter to `may_open` would mean
|
| 279 |
-
editing `platform/core/user_tables.py`, which belongs to another lane this wave.
|
| 280 |
-
|
| 281 |
-
β READ-ONLY LENDING, and it is safe by inspection rather than by hope: `all_tables` does
|
| 282 |
-
`dict(st.get(STORE_KEY) or {})`, i.e. it shallow-copies the top level before anybody touches
|
| 283 |
-
it, and `may_open` only ever READS `t.get('createdBy')`. Nothing on this path mutates the
|
| 284 |
-
lent document. Any other store key β the shares registry `role_for` consults, for instance β
|
| 285 |
-
is not intercepted and reaches the real runtime unchanged.
|
| 286 |
"""
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
def get(self, name, fresh=False):
|
| 292 |
-
if name == _UT_STORE_KEY and not fresh:
|
| 293 |
-
return self._tables
|
| 294 |
-
return self._runtime.get(name, fresh)
|
| 295 |
-
|
| 296 |
-
def __getattr__(self, name):
|
| 297 |
-
return getattr(self._runtime, name)
|
| 298 |
|
| 299 |
|
| 300 |
def _discover_default(session, tables):
|
|
|
|
| 257 |
_UT_STORE_KEY = engine.UT_STORE_KEY
|
| 258 |
|
| 259 |
|
| 260 |
+
def _LentTables(runtime, tables):
|
| 261 |
+
"""ββ WAVE 30 Β· T13, NOW W31-C1 β a read-only `st` that serves ONE already-read `user_tables`
|
| 262 |
+
document and passes every other key straight through to the real runtime.
|
| 263 |
+
|
| 264 |
+
ββ WAVE 31 Β· C1 β THE BODY IS NOW `core.user_tables.lend`, AND THE CLASS THAT USED TO BE HERE
|
| 265 |
+
IS GONE. W30's version carried its own note that the general fix was unavailable at the time:
|
| 266 |
+
*"adding a `tables=` parameter to `may_open` would mean editing platform/core/user_tables.py,
|
| 267 |
+
which belongs to another lane this wave."* It is session B's lane THIS wave, they built the
|
| 268 |
+
generalisation (`user_tables.lend` + the `_LENDABLE` allow-list) for W31-T10, and this is the
|
| 269 |
+
second caller adopting it rather than becoming a third copy of the shape.
|
| 270 |
+
|
| 271 |
+
β AND THE SWAP FIXES SOMETHING THIS FUNCTION NEVER COVERED. The old class intercepted exactly
|
| 272 |
+
one key, so `may_open`'s LAST branch β `shares.may_see` β `role_for` β `st.get('object_shares')`
|
| 273 |
+
β still took a fresh read PER TABLE. `_LENDABLE` covers both buckets, so a shared database no
|
| 274 |
+
longer costs a second whole-document read per row of the picker. The name is kept because
|
| 275 |
+
`verify_automation`'s NC66 pins this symbol as the thing it replaces to restore the `1 + N`
|
| 276 |
+
state; keeping it a function means that control still has exactly one seam to swap.
|
| 277 |
+
|
| 278 |
+
β THE OBVIOUS FIX IS STILL THE FORBIDDEN ONE. Inlining the creator-or-admin test here would
|
| 279 |
+
remove the N reads and re-create the exact defect wave 20 fixed: this route USED to carry its
|
| 280 |
+
own wider rule (`createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a
|
| 281 |
+
database in the picker and was refused the moment they opened it. `may_open` stays THE one
|
| 282 |
resolver, unmodified and still called per table; it is simply no longer charged for a document
|
| 283 |
the caller is already holding.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
"""
|
| 285 |
+
# β IMPORTED HERE, not at module scope β `core` is deliberately kept out of this module's
|
| 286 |
+
# import-time graph (the same reason `automation_tables` does it locally 300 lines down).
|
| 287 |
+
import core.user_tables as _ut
|
| 288 |
+
return _ut.lend(runtime, **{_UT_STORE_KEY: tables})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
|
| 290 |
|
| 291 |
def _discover_default(session, tables):
|
api/routes_connected_tables.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_connected_tables.py β the SOURCE-NEUTRAL door to a connected grid (W31-T49, contract C4).
|
| 2 |
+
|
| 3 |
+
`GET /api/v1/connected-tables/{table_key}/rows` β one window over any grid whose rows come from a
|
| 4 |
+
connector and are read THROUGH the per-tenant DuckDB mirror rather than copied into the tenant
|
| 5 |
+
document. Odoo's four locked databases and its two line grains today; Meta Ads' entity and Insights
|
| 6 |
+
grids when `meta_relational` registers its specs (R2).
|
| 7 |
+
|
| 8 |
+
β WHY THIS FILE EXISTS. The read-through window shipped in wave 30 inside `routes_odoo_tables.py`,
|
| 9 |
+
Odoo-named down to the URL β `/api/v1/odoo-tables/{key}/rows`. R2 puts a SECOND connector on the
|
| 10 |
+
same mirror, and a Meta campaign served from a URL with `odoo` in it is a name that lies to every
|
| 11 |
+
reader of a network tab, a log line and a bug report. So the capability gets a name that describes
|
| 12 |
+
what it IS.
|
| 13 |
+
|
| 14 |
+
ββ IT IS AN ALIAS, NOT A REIMPLEMENTATION, AND THAT IS THE WHOLE DESIGN. T49's `done-when` is
|
| 15 |
+
*"the Odoo path behaves byte-identically, proven by an NC"*. There are two ways to make that true:
|
| 16 |
+
write a second handler and test that the two agree, or have ONE handler behind two URLs. Only the
|
| 17 |
+
second is true BY CONSTRUCTION β the first is true on the day it is written and drifts the first
|
| 18 |
+
time somebody fixes a bug in one of them. Every request here lands in
|
| 19 |
+
`routes_odoo_tables.odoo_table_rows`, unchanged, so the two doors cannot disagree without a line
|
| 20 |
+
being deleted. [[one-question-two-normalizers]]
|
| 21 |
+
|
| 22 |
+
β THE OLD URL STAYS, and that is not politeness to a client we control. `CustomerGrid`'s bridge,
|
| 23 |
+
saved views and anything an operator has in a browser tab all name it; retiring it in the same wave
|
| 24 |
+
that introduces the new one would turn a rename into an outage for a feature that is already
|
| 25 |
+
half-proven (D-174). The Odoo-named route is kept as a deliberate alias with a stated retirement
|
| 26 |
+
condition rather than as a leftover: it goes when nothing in `web/src` names it.
|
| 27 |
+
|
| 28 |
+
β THE MOUNT IS THE POINT AND IT HAS BEEN MISSED BEFORE β three finished routers shipped 404-dead
|
| 29 |
+
behind green gates in wave 23, and wave 28 proved a MOUNTED router can still be uncallable
|
| 30 |
+
(`session.is_admin`, an attribute that does not exist, raising above the route's own `try`). So
|
| 31 |
+
`verify_meta` both enumerates `main.app.routes` for this path AND calls it.
|
| 32 |
+
"""
|
| 33 |
+
from fastapi import APIRouter, Depends, Query
|
| 34 |
+
|
| 35 |
+
from deps import Session, require_session
|
| 36 |
+
|
| 37 |
+
router = APIRouter(prefix="/api/v1")
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@router.get("/connected-tables/{table_key}/rows")
|
| 41 |
+
def connected_table_rows(table_key: str,
|
| 42 |
+
offset: int = Query(default=0, ge=0),
|
| 43 |
+
limit: int = Query(default=0),
|
| 44 |
+
filters: str = Query(default=None),
|
| 45 |
+
filterConj: str = Query(default="and"),
|
| 46 |
+
sorts: str = Query(default=None),
|
| 47 |
+
search: str = Query(default=None),
|
| 48 |
+
session: Session = Depends(require_session)):
|
| 49 |
+
"""ONE WINDOW over a connected grid β `{fields, rows, total, totalUnfiltered, offset, limit,
|
| 50 |
+
limits, identity, recordsMutable}`, whichever connector the rows come from.
|
| 51 |
+
|
| 52 |
+
β EVERY GUARANTEE IS INHERITED, NOT RESTATED, because the body below is one call. `total` is a
|
| 53 |
+
`SELECT count(*)` and never `len(rows)`; the filter, sort and search resolve in SQL over the
|
| 54 |
+
whole table; anything that cannot be answered is reported in `limits` with a cause and a
|
| 55 |
+
recommendation (R6's second sentence); the permission wall is `_defn_or_refuse`; and the
|
| 56 |
+
cross-tenant mirror guard (D-169) refuses with `409 cross_tenant_store` before a cursor is
|
| 57 |
+
handed out. Documenting them again HERE would be a second statement of a contract that has one
|
| 58 |
+
home β and the day the two drifted, this file would be the one that lied.
|
| 59 |
+
"""
|
| 60 |
+
import routes_odoo_tables
|
| 61 |
+
return routes_odoo_tables.odoo_table_rows(
|
| 62 |
+
table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
|
| 63 |
+
sorts=sorts, search=search, session=session)
|
api/routes_connectors.py
CHANGED
|
@@ -44,16 +44,11 @@ router = APIRouter(prefix="/api/v1")
|
|
| 44 |
#: likely to come next first. `queue` mirrors the standing provider queue in wave22-split C5-A2
|
| 45 |
#: so the two cannot drift into disagreeing about what is coming.
|
| 46 |
PLANNED = [
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
"desc": "The Outlook half of the email trigger."},
|
| 53 |
-
{"key": "quickbooks", "label": "QuickBooks", "kind": "oauth",
|
| 54 |
-
"desc": "Accounting actuals beside the Odoo sales data."},
|
| 55 |
-
{"key": "xero", "label": "Xero", "kind": "oauth",
|
| 56 |
-
"desc": "Accounting actuals for tenants who do not use QuickBooks."},
|
| 57 |
{"key": "google_sheets", "label": "Google Sheets", "kind": "oauth",
|
| 58 |
"desc": "Read a sheet as a database; the Google connection already exists."},
|
| 59 |
{"key": "google_drive", "label": "Google Drive", "kind": "oauth",
|
|
@@ -96,73 +91,333 @@ SCRAPER_PLATFORMS = [
|
|
| 96 |
TOKEN_CONNECTORS = [
|
| 97 |
{"key": "stripe", "label": "Stripe", "entry": "stripe",
|
| 98 |
"desc": "Payments and payouts against your customer records.",
|
| 99 |
-
"hint": "Create a restricted key with read access in the Stripe dashboard."
|
|
|
|
| 100 |
{"key": "shopify", "label": "Shopify", "entry": "shopify",
|
| 101 |
"desc": "Orders and products from a Shopify storefront.",
|
| 102 |
-
"hint": "Create a custom app in your store admin and copy its Admin API token."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
]
|
| 104 |
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
def _entry_types(rt):
|
| 107 |
-
"""
|
| 108 |
-
|
| 109 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
try:
|
| 111 |
import core.keychain as kc
|
| 112 |
-
return {str(e.get("type") or "") for e in (kc.list_entries(rt) or [])}
|
| 113 |
-
except Exception:
|
| 114 |
-
return set()
|
| 115 |
|
| 116 |
|
| 117 |
-
def directory(rt, uname, is_admin=False):
|
| 118 |
-
"""The composed row set. Pure over the session, so the gate drives it without a request.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
rows = []
|
| 120 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
|
| 122 |
# --- Odoo: the reference connector. Env credentials are tenant #0's only (the R3 rule), so
|
| 123 |
# a stored keychain entry is what makes it connected for anybody else.
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
|
| 131 |
# --- OAuth providers, straight off the registry (so a new entry appears here for free).
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
rows.append({
|
| 136 |
-
"key":
|
| 137 |
-
"desc": "
|
| 138 |
-
|
| 139 |
-
# THREE distinct states, and the distinction is the point: a provider nobody has
|
| 140 |
-
# configured on this deployment is not the same as one this user has not connected,
|
| 141 |
-
# and until now both rendered identically (the C11 status() gap).
|
| 142 |
-
"state": ("connected" if row_st.get("connected") and not row_st.get("reconnect")
|
| 143 |
-
else "reconnect" if row_st.get("reconnect")
|
| 144 |
-
else "available" if row_st.get("configured")
|
| 145 |
-
else "unconfigured"),
|
| 146 |
-
"connectedAs": row_st.get("email") or "",
|
| 147 |
-
"startUrl": f"/api/v1/oauth/{slug}/start",
|
| 148 |
-
"manage": "oauth",
|
| 149 |
-
"needs": "" if row_st.get("configured") else "the owner registers the client",
|
| 150 |
})
|
| 151 |
|
| 152 |
-
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
rows.append({
|
| 155 |
-
"key":
|
| 156 |
-
"
|
| 157 |
-
"state": "
|
| 158 |
-
"
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
|
|
|
|
|
|
| 163 |
})
|
|
|
|
| 164 |
|
| 165 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
import automation_engine as _eng
|
| 167 |
# β WAVE 25 Β· C2 / owner ruling R9 β THE ROW IS "SCRAPER", NOT THE VENDOR'S NAME.
|
| 168 |
#
|
|
@@ -177,30 +432,21 @@ def directory(rt, uname, is_admin=False):
|
|
| 177 |
# constant are the supplier and keep their names β this renames what a CUSTOMER sees, because
|
| 178 |
# which supplier answers a scrape is our problem and not something to make them learn. The
|
| 179 |
# `needs` string still names the env var, deliberately: it is read by whoever has to set it.
|
| 180 |
-
|
|
|
|
| 181 |
"key": "scraper", "label": "Scraper", "kind": "builtin",
|
| 182 |
# β WAVE 30 Β· R4 β the sentence names the CAPABILITY; the platforms are the drill below.
|
| 183 |
# It used to say "Instagram profile and post capture", which stopped being true the day
|
| 184 |
# TikTok discovery shipped and would have to be re-edited for every platform after it.
|
| 185 |
"desc": "Profile, post and comment capture for the discovery automations.",
|
| 186 |
-
"state": "connected" if
|
| 187 |
-
"needs": "" if
|
| 188 |
"manage": "",
|
| 189 |
# β ONE CREDENTIAL, TWO PLATFORMS β which is why this is a field on the row and not two
|
| 190 |
# rows. `bd_ready()` above answers for both; a per-platform state would be a second
|
| 191 |
# answer to a question that has one.
|
| 192 |
"platforms": [dict(p) for p in SCRAPER_PLATFORMS],
|
| 193 |
-
}
|
| 194 |
-
rows.append({
|
| 195 |
-
"key": "webhooks", "label": "Webhooks", "kind": "builtin",
|
| 196 |
-
"desc": "Let another system start an automation by calling a URL.",
|
| 197 |
-
"state": "connected", "manage": "automation",
|
| 198 |
-
})
|
| 199 |
-
|
| 200 |
-
for p in PLANNED:
|
| 201 |
-
rows.append({**p, "state": "planned",
|
| 202 |
-
"needs": "not built yet", "manage": ""})
|
| 203 |
-
return rows
|
| 204 |
|
| 205 |
|
| 206 |
@router.get("/connectors/directory")
|
|
@@ -215,8 +461,15 @@ def connectors_directory(session: Session = Depends(require_session)):
|
|
| 215 |
# β `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
|
| 216 |
# careful and is silently False for every admin on earth β the manage actions would simply
|
| 217 |
# never appear, with nothing to catch it. Attribute name checked against `deps.Session`.
|
| 218 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
return {"connectors": rows,
|
|
|
|
| 220 |
"counts": {"connected": sum(1 for r in rows if r["state"] == "connected"),
|
| 221 |
"available": sum(1 for r in rows if r["state"] == "available"),
|
| 222 |
"planned": sum(1 for r in rows if r["state"] == "planned")}}
|
|
|
|
| 44 |
#: likely to come next first. `queue` mirrors the standing provider queue in wave22-split C5-A2
|
| 45 |
#: so the two cannot drift into disagreeing about what is coming.
|
| 46 |
PLANNED = [
|
| 47 |
+
# ββ WAVE 31 Β· R7 β SIX ROWS LEFT THIS LIST FOR `NANGO_PROVIDERS` BELOW, and that move is the
|
| 48 |
+
# ticket. `slack`, `airtable`, `outlook`, `quickbooks`, `xero` and `hubspot` are providers
|
| 49 |
+
# Nango can AUTHENTICATE today, so leaving them here as "not built yet" is the same
|
| 50 |
+
# misinformation this module's header warns about β one wave later and about a different
|
| 51 |
+
# mechanism. What they still cannot do is INGEST, and that is what their own cards now say.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
{"key": "google_sheets", "label": "Google Sheets", "kind": "oauth",
|
| 53 |
"desc": "Read a sheet as a database; the Google connection already exists."},
|
| 54 |
{"key": "google_drive", "label": "Google Drive", "kind": "oauth",
|
|
|
|
| 91 |
TOKEN_CONNECTORS = [
|
| 92 |
{"key": "stripe", "label": "Stripe", "entry": "stripe",
|
| 93 |
"desc": "Payments and payouts against your customer records.",
|
| 94 |
+
"hint": "Create a restricted key with read access in the Stripe dashboard.",
|
| 95 |
+
"note": "Storing the key connects the account; reading its data lands in a later release."},
|
| 96 |
{"key": "shopify", "label": "Shopify", "entry": "shopify",
|
| 97 |
"desc": "Orders and products from a Shopify storefront.",
|
| 98 |
+
"hint": "Create a custom app in your store admin and copy its Admin API token.",
|
| 99 |
+
"note": "Storing the key connects the account; reading its data lands in a later release."},
|
| 100 |
+
# ββ WAVE 31 Β· R2/R8 β META ADS IS A TOKEN CONNECTOR, AND IT IS THE ONE NEW CARD THAT IS
|
| 101 |
+
# ACTIONABLE TODAY. R8 put the schema work on a pasted token deliberately β *"so that you can
|
| 102 |
+
# also explore the Schema, without any user logging into it with Facebook yet"* β and T41
|
| 103 |
+
# MEASURED that the delegated-OAuth door is not available anyway: the Nango account holds one
|
| 104 |
+
# integration and it is a GitHub sample, so there is no Facebook integration to request
|
| 105 |
+
# `ads_read` from. This row is therefore the real door and the Nango one is honest about not
|
| 106 |
+
# being ready. β Its `entry` must match `keychain.ENTRY_TYPES`, or storing 400s.
|
| 107 |
+
{"key": "meta_ads", "label": "Meta Ads", "entry": "meta_ads",
|
| 108 |
+
"desc": "Campaigns, ad sets, ads, creatives and daily performance from your Meta ad account.",
|
| 109 |
+
"hint": "Paste a Marketing API access token with `ads_read`; the connector verifies it "
|
| 110 |
+
"against your ad accounts before storing it.",
|
| 111 |
+
"note": "The token connects the account and is verified live. The spawned databases "
|
| 112 |
+
"(285 measured columns) land with the Meta grids."},
|
| 113 |
+
]
|
| 114 |
+
|
| 115 |
+
#: ββ WAVE 31 Β· R7 β THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
|
| 116 |
+
#: other potential connection as possible, such as slack etc."*
|
| 117 |
+
#:
|
| 118 |
+
#: β ADDING A PROVIDER HERE IS THE WHOLE FEATURE β one row, no code β which is the claim T43 has
|
| 119 |
+
#: to make demonstrable rather than assert. The row carries only what a CARD needs; the auth flow
|
| 120 |
+
#: is Nango's and is identical for every one of them.
|
| 121 |
+
#:
|
| 122 |
+
#: ββ R7's SECOND SENTENCE IS THE HARD PART AND IT IS ENFORCED IN `_nango_rows` BELOW: *"Nango
|
| 123 |
+
#: buys CONNECT, not INGEST β every lit provider states on its own card what it can do today, or
|
| 124 |
+
#: we ship a directory of buttons that lead nowhere."* MEASURED for T42: the owner's Nango account
|
| 125 |
+
#: holds **1 integration (`github-getting-started`) and 0 connections**, so there is no sync on it
|
| 126 |
+
#: to inspect and NO provider here has a data layer. Every card says so, in its own words, and
|
| 127 |
+
#: none of them offers a button.
|
| 128 |
+
NANGO_PROVIDERS = [
|
| 129 |
+
{"key": "slack", "label": "Slack", "provider": "slack",
|
| 130 |
+
"desc": "Send messages into a channel when an automation reaches a step."},
|
| 131 |
+
{"key": "hubspot", "label": "HubSpot", "provider": "hubspot",
|
| 132 |
+
"desc": "Deals and contacts beside the Odoo customer book."},
|
| 133 |
+
{"key": "airtable", "label": "Airtable", "provider": "airtable",
|
| 134 |
+
"desc": "Import an existing base as a database."},
|
| 135 |
+
{"key": "quickbooks", "label": "QuickBooks", "provider": "quickbooks",
|
| 136 |
+
"desc": "Accounting actuals beside the Odoo sales data."},
|
| 137 |
+
{"key": "xero", "label": "Xero", "provider": "xero",
|
| 138 |
+
"desc": "Accounting actuals for tenants who do not use QuickBooks."},
|
| 139 |
+
{"key": "outlook", "label": "Microsoft Outlook", "provider": "outlook",
|
| 140 |
+
"desc": "The Outlook half of the email trigger."},
|
| 141 |
]
|
| 142 |
|
| 143 |
|
| 144 |
+
#: ββ W31-T44 / OWNER ITEM 6b β WHICH `ut_*` PREFIXES BELONG TO A CONNECTOR (contract C2).
|
| 145 |
+
#:
|
| 146 |
+
#: B's `user_tables.register_connected_prefix` (W31-T25) replaced a hardcoded `ut_odoo_` literal
|
| 147 |
+
#: with a registered SET, and this is where the connector layer declares its half. It lives beside
|
| 148 |
+
#: the other connector registries deliberately: "which databases does this connector own" is the
|
| 149 |
+
#: same kind of fact as "what is this connector called", and putting it anywhere else would make
|
| 150 |
+
#: the answer to one question live in two files.
|
| 151 |
+
#:
|
| 152 |
+
#: β REGISTERING A PREFIX DOES NOT MAKE A TABLE CONNECTED, and B posted this as the trap: the
|
| 153 |
+
#: third leg of `is_connected` still demands `source == AUTOMATION_SOURCE` **and**
|
| 154 |
+
#: `recordMode == AUTOMATION_RECORD_MODE`. A person who names a database "Meta ads spend" gets a
|
| 155 |
+
#: `ut_meta_β¦` key too, and exempting it from `MAX_ROWS` on the strength of its NAME is the silent
|
| 156 |
+
#: hole the stamp closes. The spawn stamps; this only says which family may qualify.
|
| 157 |
+
#: β `ut_odoo_` is SEEDED in `user_tables` rather than registered here, also deliberately β the
|
| 158 |
+
#: convention leg has to answer in a worker process where nothing registered anything.
|
| 159 |
+
CONNECTED_PREFIXES = {
|
| 160 |
+
"odoo": "ut_odoo_",
|
| 161 |
+
"meta": "ut_meta_",
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _register_prefixes():
|
| 166 |
+
"""Declare `CONNECTED_PREFIXES` to the store layer. Called at import β this module is mounted
|
| 167 |
+
by `main.py`, so any process that can serve the directory has also made the declaration.
|
| 168 |
+
|
| 169 |
+
Never raises: a store layer that cannot take the declaration must not stop the API booting,
|
| 170 |
+
and the consequence of a missed registration is a row CAP (R6's subject), not a wrong answer.
|
| 171 |
+
"""
|
| 172 |
+
try:
|
| 173 |
+
import core.user_tables as ut
|
| 174 |
+
return ut.register_connected_prefix(*sorted(CONNECTED_PREFIXES.values()))
|
| 175 |
+
except Exception: # noqa: BLE001
|
| 176 |
+
return frozenset()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
_register_prefixes()
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _register_probers():
|
| 183 |
+
"""Let each connector answer `keychain.test_entry` for its OWN type β the SLOT, filled from
|
| 184 |
+
the layer that knows how to ask.
|
| 185 |
+
|
| 186 |
+
β THIS DIRECTION IS THE POINT. `core.keychain` must not import a connector module: `core`
|
| 187 |
+
never imports up (ARCHITECTURE.md / CLAUDE.md), and the tempting shortcut β a `sys.path`
|
| 188 |
+
insert and an `import connectors_meta` from inside `core` β is an architecture inversion that
|
| 189 |
+
no gate was watching for. Same idiom as `datastore.set_paused_probe`, one layer down, for the
|
| 190 |
+
same reason.
|
| 191 |
+
|
| 192 |
+
Import-time and fail-quiet: an unfillable slot costs a liveness CHECK, never the ability to
|
| 193 |
+
store a credential and never the API's ability to boot.
|
| 194 |
+
"""
|
| 195 |
+
try:
|
| 196 |
+
import connectors_meta
|
| 197 |
+
return connectors_meta.register()
|
| 198 |
+
except Exception: # noqa: BLE001
|
| 199 |
+
return False
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
_register_probers()
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 206 |
+
# ββ W31-T44 β EVERY WAY THIS DIRECTORY CAN COME BACK SHORT, ENUMERATED AND MADE VISIBLE.
|
| 207 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 208 |
+
#
|
| 209 |
+
# The owner named Connectors FIRST in item 6 ("Connectors and Automation module still disappears").
|
| 210 |
+
# `.shell-nav-connectors` is a static `<a>` and cannot be dropped from the rail, so if Connectors
|
| 211 |
+
# "disappears" the subject is THIS payload. Enumerated by reading the composer rather than by
|
| 212 |
+
# guessing, and the answer was sharper than expected:
|
| 213 |
+
#
|
| 214 |
+
# β THERE WAS EXACTLY ONE FAILURE MODE AND IT DROPPED **EVERY** ROW. `directory()` had no
|
| 215 |
+
# `try` anywhere. `oauth_connect.status()`, `import automation_engine` and `bd_ready()` are
|
| 216 |
+
# three unguarded calls into three other subsystems, and a raise in ANY of them turned the
|
| 217 |
+
# whole endpoint into a 500 β at which point `ConnectorsPage` renders one sentence and no
|
| 218 |
+
# cards. A page whose entire job is to be a map of the workspace showed nothing because one
|
| 219 |
+
# registry was unhappy.
|
| 220 |
+
# β AND ONE SILENT ONE, WHICH IS WORSE THAN THE LOUD ONE. `_entry_types` swallowed every
|
| 221 |
+
# exception into an empty set, so an unreadable or locked keychain rendered Odoo, Stripe and
|
| 222 |
+
# Shopify as **"available"** β i.e. *"you have not connected this"* about a connector the
|
| 223 |
+
# tenant HAS connected. Not short: wrong, and confidently.
|
| 224 |
+
#
|
| 225 |
+
# β SO THE COMPOSER IS A LIST OF CONTRIBUTIONS AND EACH ONE FAILS ALONE. A contribution that
|
| 226 |
+
# raises costs its own rows and NOTHING else, and says so in `incomplete` with a cause and a
|
| 227 |
+
# recommendation β R6's second sentence, applied to a page instead of to a row cap. Silence is the
|
| 228 |
+
# violation, not the limit.
|
| 229 |
+
#: The contributions, by name. β THIS LIST IS THE ENUMERATION the ticket asks for, and
|
| 230 |
+
#: `verify_meta` breaks each one IN TURN and requires (a) the other rows still render and (b) this
|
| 231 |
+
#: name appears in `incomplete` β so a contribution added later without a probe is a red, not a
|
| 232 |
+
#: quietly unprotected path.
|
| 233 |
+
CONTRIBUTORS = ("keychain", "odoo", "oauth", "token", "nango", "scraper", "builtin", "planned")
|
| 234 |
+
|
| 235 |
+
|
| 236 |
def _entry_types(rt):
|
| 237 |
+
"""`(types, failure)` β which keychain entry types this tenant has stored, and what stopped us.
|
| 238 |
+
|
| 239 |
+
β IT USED TO RETURN A BARE SET AND SWALLOW EVERYTHING, and that is the silent half of item 6b:
|
| 240 |
+
a locked keychain became an EMPTY set, which is indistinguishable from "this tenant has
|
| 241 |
+
connected nothing" β so the page said `available` about a live Odoo connection. Degrading to
|
| 242 |
+
"nothing connected" is only honest if the reader is told that is what happened.
|
| 243 |
+
"""
|
| 244 |
try:
|
| 245 |
import core.keychain as kc
|
| 246 |
+
return {str(e.get("type") or "") for e in (kc.list_entries(rt) or [])}, None
|
| 247 |
+
except Exception as e: # noqa: BLE001
|
| 248 |
+
return set(), f"{type(e).__name__}: {e}"
|
| 249 |
|
| 250 |
|
| 251 |
+
def directory(rt, uname, is_admin=False, incomplete=None):
|
| 252 |
+
"""The composed row set. Pure over the session, so the gate drives it without a request.
|
| 253 |
+
|
| 254 |
+
`incomplete` is an optional list the caller lends to collect contribution failures; the route
|
| 255 |
+
passes one and puts it on the wire. Callers that do not care pass nothing and simply get the
|
| 256 |
+
rows that composed.
|
| 257 |
+
"""
|
| 258 |
rows = []
|
| 259 |
+
short = incomplete if incomplete is not None else []
|
| 260 |
+
|
| 261 |
+
def contribute(source, fn, why, fix):
|
| 262 |
+
"""Run one contribution. A failure costs ITS rows and records why β never the page."""
|
| 263 |
+
try:
|
| 264 |
+
fn()
|
| 265 |
+
except Exception as e: # noqa: BLE001
|
| 266 |
+
short.append({"source": source, "cause": f"{why} ({type(e).__name__}: {e})",
|
| 267 |
+
"recommendation": fix})
|
| 268 |
+
|
| 269 |
+
have, kc_failed = _entry_types(rt)
|
| 270 |
+
if kc_failed:
|
| 271 |
+
short.append({
|
| 272 |
+
"source": "keychain",
|
| 273 |
+
"cause": f"this tenant's keychain could not be read ({kc_failed}), so a connector "
|
| 274 |
+
f"that IS connected through a stored credential is shown as available",
|
| 275 |
+
"recommendation": "unlock the keychain (AIOS_KEYCHAIN_KEY) and reload; the states "
|
| 276 |
+
"below are otherwise correct"})
|
| 277 |
|
| 278 |
# --- Odoo: the reference connector. Env credentials are tenant #0's only (the R3 rule), so
|
| 279 |
# a stored keychain entry is what makes it connected for anybody else.
|
| 280 |
+
def _odoo():
|
| 281 |
+
rows.append({
|
| 282 |
+
"key": "odoo", "label": "Odoo", "kind": "builtin",
|
| 283 |
+
"desc": "Sales, invoices, products and customers from your Odoo ERP.",
|
| 284 |
+
"state": ("connected" if ("odoo" in have or os.environ.get("ODOO_URL"))
|
| 285 |
+
else "available"),
|
| 286 |
+
"manage": "keychain" if is_admin else "",
|
| 287 |
+
})
|
| 288 |
+
|
| 289 |
+
# β GUARDED EVEN THOUGH IT CANNOT RAISE TODAY β `os.environ.get` and a set membership are the
|
| 290 |
+
# whole of it. The guard is here because the enumeration is only worth something if it is
|
| 291 |
+
# EXHAUSTIVE: an unguarded block is an un-enumerated drop path, and "this one is obviously
|
| 292 |
+
# safe" is how the other five got written in the first place.
|
| 293 |
+
contribute("odoo", _odoo, "the Odoo connector row could not be composed",
|
| 294 |
+
"reload; every other row is unaffected")
|
| 295 |
|
| 296 |
# --- OAuth providers, straight off the registry (so a new entry appears here for free).
|
| 297 |
+
# β `status()` READS EVERY PER-USER CREDENTIAL SLOT, which means it touches the keychain and
|
| 298 |
+
# the store β the two things most likely to be unavailable. Unguarded, it took the whole page
|
| 299 |
+
# with it.
|
| 300 |
+
def _oauth():
|
| 301 |
+
st = oauth_connect.status(rt, uname)
|
| 302 |
+
for slug, meta in oauth_connect.PROVIDERS.items():
|
| 303 |
+
rows.append(_oauth_row(slug, meta, st.get(slug) or {}))
|
| 304 |
+
|
| 305 |
+
contribute("oauth", _oauth,
|
| 306 |
+
"the OAuth provider registry could not be read, so no OAuth connector is listed",
|
| 307 |
+
"the rest of the directory is complete; reload to retry the OAuth states")
|
| 308 |
+
|
| 309 |
+
# --- Token connectors (R8): connectable today, and honest that data flows arrive later.
|
| 310 |
+
def _tokens():
|
| 311 |
+
for t in TOKEN_CONNECTORS:
|
| 312 |
+
rows.append({
|
| 313 |
+
"key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"],
|
| 314 |
+
"hint": t["hint"],
|
| 315 |
+
"state": "connected" if t["entry"] in have else "available",
|
| 316 |
+
"manage": "keychain" if is_admin else "",
|
| 317 |
+
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
|
| 318 |
+
# key is real and useful, and it is not the same as the data being on screen.
|
| 319 |
+
"note": "Storing the key connects the account; reading its data lands in a later "
|
| 320 |
+
"release.",
|
| 321 |
+
})
|
| 322 |
+
|
| 323 |
+
contribute("token", _tokens,
|
| 324 |
+
"the token-connector registry could not be composed",
|
| 325 |
+
"Stripe and Shopify are missing from this list only; nothing else is affected")
|
| 326 |
+
|
| 327 |
+
def _nango():
|
| 328 |
+
rows.extend(_nango_rows())
|
| 329 |
+
|
| 330 |
+
contribute("nango", _nango,
|
| 331 |
+
"the Nango provider registry could not be composed",
|
| 332 |
+
"the providers that connect through Nango are missing from this list; every other "
|
| 333 |
+
"row is unaffected")
|
| 334 |
+
|
| 335 |
+
def _scraper():
|
| 336 |
+
rows.append(_scraper_row())
|
| 337 |
+
|
| 338 |
+
contribute("scraper", _scraper,
|
| 339 |
+
"the Scraper connector's readiness could not be determined "
|
| 340 |
+
"(automation_engine did not answer)",
|
| 341 |
+
"the Scraper card is missing from this list; automations are unaffected")
|
| 342 |
+
|
| 343 |
+
def _builtin():
|
| 344 |
rows.append({
|
| 345 |
+
"key": "webhooks", "label": "Webhooks", "kind": "builtin",
|
| 346 |
+
"desc": "Let another system start an automation by calling a URL.",
|
| 347 |
+
"state": "connected", "manage": "automation",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 348 |
})
|
| 349 |
|
| 350 |
+
contribute("builtin", _builtin, "a built-in connector row could not be composed",
|
| 351 |
+
"reload; the remaining rows are correct")
|
| 352 |
+
|
| 353 |
+
def _planned():
|
| 354 |
+
for p in PLANNED:
|
| 355 |
+
rows.append({**p, "state": "planned", "needs": "not built yet", "manage": ""})
|
| 356 |
+
|
| 357 |
+
contribute("planned", _planned, "the planned-connector list could not be composed",
|
| 358 |
+
"the to-do list is missing; every CONNECTED row above is still correct")
|
| 359 |
+
return rows
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _oauth_row(slug, meta, row_st):
|
| 363 |
+
"""One OAuth provider's card. Lifted out of `directory` whole so the composer reads as a list
|
| 364 |
+
of contributions; behaviour is byte-identical."""
|
| 365 |
+
return {
|
| 366 |
+
"key": slug, "label": meta.get("label") or slug.title(), "kind": "oauth",
|
| 367 |
+
"desc": "Gmail for the email trigger; the same connection unlocks Sheets, Drive "
|
| 368 |
+
"and Calendar later." if slug == "google" else "",
|
| 369 |
+
# THREE distinct states, and the distinction is the point: a provider nobody has
|
| 370 |
+
# configured on this deployment is not the same as one this user has not connected,
|
| 371 |
+
# and until now both rendered identically (the C11 status() gap).
|
| 372 |
+
"state": ("connected" if row_st.get("connected") and not row_st.get("reconnect")
|
| 373 |
+
else "reconnect" if row_st.get("reconnect")
|
| 374 |
+
else "available" if row_st.get("configured")
|
| 375 |
+
else "unconfigured"),
|
| 376 |
+
"connectedAs": row_st.get("email") or "",
|
| 377 |
+
"startUrl": f"/api/v1/oauth/{slug}/start",
|
| 378 |
+
"manage": "oauth",
|
| 379 |
+
"needs": "" if row_st.get("configured") else "the owner registers the client",
|
| 380 |
+
}
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def _nango_rows():
|
| 384 |
+
"""One card per `NANGO_PROVIDERS` row, each stating what it can do TODAY (R7).
|
| 385 |
+
|
| 386 |
+
β NO NETWORK CALL. The obvious "honest" implementation asks Nango which integrations exist
|
| 387 |
+
and lights the ones that do β and that would put a third-party HTTP round trip on a page this
|
| 388 |
+
very wave is making faster, on a route that already had exactly one failure mode. The
|
| 389 |
+
deployment fact this page can answer CHEAPLY is whether the platform holds a Nango key at all,
|
| 390 |
+
which is the same class of check `bd_ready()` and `oauth_connect.configured()` already make.
|
| 391 |
+
β Whether a SPECIFIC integration is configured in the Nango dashboard is a different fact,
|
| 392 |
+
measured once in `proto/nango-verdict.md` (1 integration, and it is a GitHub sample), and it
|
| 393 |
+
is what `needs` names rather than pretends to have checked.
|
| 394 |
+
|
| 395 |
+
ββ EVERY ROW IS `available` AT BEST AND CARRIES NO `manage` DOOR, which is R7's second
|
| 396 |
+
sentence made structural: *a card whose button leads nowhere without saying so* fails the
|
| 397 |
+
ruling, so these cards have no button at all and say why in `needs` and `note`.
|
| 398 |
+
"""
|
| 399 |
+
have_key = bool((os.environ.get("NANGO_SECRET_KEY") or "").strip())
|
| 400 |
+
rows = []
|
| 401 |
+
for p in NANGO_PROVIDERS:
|
| 402 |
rows.append({
|
| 403 |
+
"key": p["key"], "label": p["label"], "kind": "oauth", "desc": p["desc"],
|
| 404 |
+
"provider": p["provider"],
|
| 405 |
+
"state": "available" if have_key else "unconfigured",
|
| 406 |
+
"needs": ("this provider's integration must be created in the Nango dashboard first"
|
| 407 |
+
if have_key else
|
| 408 |
+
"the owner registers NANGO_SECRET_KEY on this deployment"),
|
| 409 |
+
# β THE SENTENCE R7 DEMANDS, on the row rather than in a paragraph somewhere (R13).
|
| 410 |
+
"note": "Connects through Nango. Reading this provider's data into a database is not "
|
| 411 |
+
"built yet β measured: no sync is configured on this account.",
|
| 412 |
+
"manage": "",
|
| 413 |
})
|
| 414 |
+
return rows
|
| 415 |
|
| 416 |
+
|
| 417 |
+
def _scraper_row():
|
| 418 |
+
"""The Scraper card. Its own function because `bd_ready()` reaches into `automation_engine` β
|
| 419 |
+
an import and a call into another subsystem, i.e. the single most likely thing in this file to
|
| 420 |
+
raise, and until W31-T44 it could take the whole page with it."""
|
| 421 |
import automation_engine as _eng
|
| 422 |
# β WAVE 25 Β· C2 / owner ruling R9 β THE ROW IS "SCRAPER", NOT THE VENDOR'S NAME.
|
| 423 |
#
|
|
|
|
| 432 |
# constant are the supplier and keep their names β this renames what a CUSTOMER sees, because
|
| 433 |
# which supplier answers a scrape is our problem and not something to make them learn. The
|
| 434 |
# `needs` string still names the env var, deliberately: it is read by whoever has to set it.
|
| 435 |
+
ready = _eng.bd_ready()
|
| 436 |
+
return {
|
| 437 |
"key": "scraper", "label": "Scraper", "kind": "builtin",
|
| 438 |
# β WAVE 30 Β· R4 β the sentence names the CAPABILITY; the platforms are the drill below.
|
| 439 |
# It used to say "Instagram profile and post capture", which stopped being true the day
|
| 440 |
# TikTok discovery shipped and would have to be re-edited for every platform after it.
|
| 441 |
"desc": "Profile, post and comment capture for the discovery automations.",
|
| 442 |
+
"state": "connected" if ready else "unconfigured",
|
| 443 |
+
"needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
|
| 444 |
"manage": "",
|
| 445 |
# β ONE CREDENTIAL, TWO PLATFORMS β which is why this is a field on the row and not two
|
| 446 |
# rows. `bd_ready()` above answers for both; a per-platform state would be a second
|
| 447 |
# answer to a question that has one.
|
| 448 |
"platforms": [dict(p) for p in SCRAPER_PLATFORMS],
|
| 449 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
|
| 451 |
|
| 452 |
@router.get("/connectors/directory")
|
|
|
|
| 461 |
# β `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
|
| 462 |
# careful and is silently False for every admin on earth β the manage actions would simply
|
| 463 |
# never appear, with nothing to catch it. Attribute name checked against `deps.Session`.
|
| 464 |
+
# β W31-T44 β `incomplete` IS PART OF THE ANSWER, not an error channel. The page's job is to
|
| 465 |
+
# be a map of the workspace; a map missing a region must say which region, or the reader
|
| 466 |
+
# concludes the region does not exist. That is the owner's item 6b in one sentence, and it is
|
| 467 |
+
# R6's second sentence applied to a payload: a limit that cannot be removed is REPORTED with
|
| 468 |
+
# its cause and a recommended fix. `[]` on a healthy read, so the client can test it directly.
|
| 469 |
+
incomplete = []
|
| 470 |
+
rows = directory(session.runtime, session.uname, bool(session.admin), incomplete=incomplete)
|
| 471 |
return {"connectors": rows,
|
| 472 |
+
"incomplete": incomplete,
|
| 473 |
"counts": {"connected": sum(1 for r in rows if r["state"] == "connected"),
|
| 474 |
"available": sum(1 for r in rows if r["state"] == "available"),
|
| 475 |
"planned": sum(1 for r in rows if r["state"] == "planned")}}
|
api/routes_grid.py
CHANGED
|
@@ -150,7 +150,13 @@ def workspace(scope: str = "customer",
|
|
| 150 |
|
| 151 |
storage_key = f"{session.tenant}:{scope}:{session.uname}"
|
| 152 |
try:
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
except grid_events.StoreUnavailable:
|
| 155 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 156 |
workspace = g["workspace"]
|
|
@@ -178,6 +184,12 @@ def workspace(scope: str = "customer",
|
|
| 178 |
workspace["userOptions"] = []
|
| 179 |
workspace["userAvatars"] = {}
|
| 180 |
workspace["scopeKey"] = scope
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
return {"workspace": workspace}
|
| 182 |
|
| 183 |
# ββ Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field
|
|
@@ -768,6 +780,27 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 768 |
from routes_tables import ut_write_ctx
|
| 769 |
|
| 770 |
g = ut_write_ctx(session, scope)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 771 |
else:
|
| 772 |
g = grid_assembly(session, scope=scope, consume_corrections=False)
|
| 773 |
# β THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`,
|
|
|
|
| 150 |
|
| 151 |
storage_key = f"{session.tenant}:{scope}:{session.uname}"
|
| 152 |
try:
|
| 153 |
+
# ββ W31-T20 / D-174 β `with_rows=False`. THIS ROUTE RENDERS NO ROW: the grid fetches
|
| 154 |
+
# them from `/tables/{key}/rows` or `/odoo-tables/{key}/rows` in the same paint, and
|
| 155 |
+
# `useCustomerData.load` fires both calls in one `Promise.all`. Building the table here
|
| 156 |
+
# bought nothing and cost everything β it is why `?scope=ut_odoo_gl_lines` answered
|
| 157 |
+
# `409 window_required` six times out of six on the live deploy while the ROWS door for
|
| 158 |
+
# the same table served a page in ~150 ms, i.e. a grid that could not be opened at all.
|
| 159 |
+
g = ut_assembly(session, scope, storage_key=storage_key, with_rows=False)
|
| 160 |
except grid_events.StoreUnavailable:
|
| 161 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 162 |
workspace = g["workspace"]
|
|
|
|
| 184 |
workspace["userOptions"] = []
|
| 185 |
workspace["userAvatars"] = {}
|
| 186 |
workspace["scopeKey"] = scope
|
| 187 |
+
# β W31-T20 β R6's SECOND SENTENCE REACHES THE BROWSER. On a read-through grid too large
|
| 188 |
+
# for one window the pid set is empty, so cohort membership and a shared view's
|
| 189 |
+
# `memberPids` are UNRESOLVED here (both fail closed, and `workspace_wire` already stamps
|
| 190 |
+
# `missing` on a shortened cohort). An unannounced empty scope is the silent limit R6
|
| 191 |
+
# forbids; this is the announcement, and W31-T22 is the ticket that renders it.
|
| 192 |
+
workspace["limits"] = g.get("limits") or []
|
| 193 |
return {"workspace": workspace}
|
| 194 |
|
| 195 |
# ββ Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field
|
|
|
|
| 780 |
from routes_tables import ut_write_ctx
|
| 781 |
|
| 782 |
g = ut_write_ctx(session, scope)
|
| 783 |
+
# ββ W31-T20 / D-174 β A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the
|
| 784 |
+
# half that is easy to skip because the code already "fails closed" without it. When a
|
| 785 |
+
# read-through grid's population exceeds one window the pid set is EMPTY, and every
|
| 786 |
+
# pid-bearing handler in `grid_events` then answers `False` β `overlay_patch` and
|
| 787 |
+
# `add_to_list` both `return False` for a pid not in `allowed_pids`. On the wire that is
|
| 788 |
+
# `rerender: false` and HTTP 200: a write the user watched succeed, that did nothing
|
| 789 |
+
# ([[lost-write-looks-like-failed-read]]). Schema-only events β hide a field, save a view,
|
| 790 |
+
# rename a column β name no pid and are untouched, which is D-170.
|
| 791 |
+
# β THE TEST IS THE EVENT'S OWN KEYS, not a kind list: a new pid-bearing event type would
|
| 792 |
+
# otherwise inherit the silent no-op the day it is added.
|
| 793 |
+
if g.get("limits"):
|
| 794 |
+
named = [e for e in events
|
| 795 |
+
if isinstance(e, dict) and (e.get("pid") is not None or e.get("pids"))]
|
| 796 |
+
if named:
|
| 797 |
+
lim = g["limits"][0]
|
| 798 |
+
raise err(409, "pid_scope_unresolved",
|
| 799 |
+
f"this database is served through the connector mirror and its rows "
|
| 800 |
+
f"cannot be listed in one window, so a change addressed to particular "
|
| 801 |
+
f"records ({len(named)} of {len(events)} here) cannot be admitted β "
|
| 802 |
+
f"{lim.get('cause') or 'the row set is unresolved'}. "
|
| 803 |
+
f"{lim.get('recommendation') or ''}".strip())
|
| 804 |
else:
|
| 805 |
g = grid_assembly(session, scope=scope, consume_corrections=False)
|
| 806 |
# β THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`,
|
api/routes_nav.py
CHANGED
|
@@ -153,6 +153,11 @@ def _placeable_top_keys(session):
|
|
| 153 |
# The SAME listing `nav()` renders from β `may_open`-filtered, so a placement can never
|
| 154 |
# name a database this session cannot see, and the nav and this door agree by sharing
|
| 155 |
# one predicate rather than by two lists being kept in step.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin,
|
| 157 |
st=session.runtime):
|
| 158 |
keys.add(str(e.get("key") or ""))
|
|
@@ -259,21 +264,60 @@ def nav(session: Session = Depends(require_session)):
|
|
| 259 |
# its own databases, which is what "different set of databases at later waves" means.
|
| 260 |
tcfg = getattr(session.runtime.tenant, "config", None) or {}
|
| 261 |
tmods = tcfg.get("modules", "all")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
if tmods != "all":
|
| 263 |
allowed = {str(k) for k in (tmods or [])}
|
|
|
|
|
|
|
|
|
|
| 264 |
pages = [p for p in pages
|
| 265 |
if p.get("key") in allowed or (p.get("parent") or "") in allowed]
|
| 266 |
# Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows β
|
| 267 |
# the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so
|
| 268 |
# the nav cannot offer a row the table routes would refuse.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
try:
|
| 270 |
import core.user_tables as user_tables
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin,
|
| 272 |
-
st=
|
| 273 |
pages.append({"key": e["key"], "label": e["label"],
|
| 274 |
"source": e.get("source") or "Blank", "chrome": "main"})
|
| 275 |
except Exception:
|
| 276 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
if not pages:
|
| 278 |
if tmods != "all":
|
| 279 |
# A provisioned tenant with no modules and no databases YET is a legitimate empty
|
|
@@ -298,16 +342,10 @@ def nav(session: Session = Depends(require_session)):
|
|
| 298 |
meta = _read_nav_meta(session.runtime)
|
| 299 |
# Wave 21 (C3): the definitions, once β `manage`/`canDelete` below answer from `createdBy`.
|
| 300 |
# WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read.
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
# `.get("recordMode")` is None too β and `None == None` would mark EVERY database locked
|
| 306 |
-
# the moment this import failed. A sentinel that can equal real data is not a sentinel.
|
| 307 |
-
_ut_locked_mode = str(_ut_mod.AUTOMATION_RECORD_MODE)
|
| 308 |
-
except Exception:
|
| 309 |
-
_ut_defs = {}
|
| 310 |
-
_ut_locked_mode = ""
|
| 311 |
for p in pages:
|
| 312 |
# `manage` (R14): may THIS session change this row's icon/name? Answered HERE because
|
| 313 |
# the server is the only end that knows β the client cannot see who created a user
|
|
@@ -376,7 +414,14 @@ def nav(session: Session = Depends(require_session)):
|
|
| 376 |
# answer sitting in a local variable.
|
| 377 |
recents = _read_recents(session.runtime, session.uname,
|
| 378 |
{str(p.get("key", "")) for p in pages})
|
| 379 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 380 |
|
| 381 |
|
| 382 |
@router.post("/nav/opened")
|
|
|
|
| 153 |
# The SAME listing `nav()` renders from β `may_open`-filtered, so a placement can never
|
| 154 |
# name a database this session cannot see, and the nav and this door agree by sharing
|
| 155 |
# one predicate rather than by two lists being kept in step.
|
| 156 |
+
# β W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is
|
| 157 |
+
# ONE document copy rather than `1 + N`. It is not a footnote here β `Shell.tsx` fires
|
| 158 |
+
# `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured
|
| 159 |
+
# **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the
|
| 160 |
+
# owner reports as "the Automation row arrives ten seconds late".
|
| 161 |
for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin,
|
| 162 |
st=session.runtime):
|
| 163 |
keys.add(str(e.get("key") or ""))
|
|
|
|
| 264 |
# its own databases, which is what "different set of databases at later waves" means.
|
| 265 |
tcfg = getattr(session.runtime.tenant, "config", None) or {}
|
| 266 |
tmods = tcfg.get("modules", "all")
|
| 267 |
+
# ββ W31-T11 (owner item 6b) β WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD.
|
| 268 |
+
#
|
| 269 |
+
# Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all
|
| 270 |
+
# `['analyst','automation']` β census in `proto/nav-omission-census.md`), so this filter runs
|
| 271 |
+
# on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a
|
| 272 |
+
# row it removes and a row a store failure dropped are the SAME absence on the wire, and the
|
| 273 |
+
# client renders `null` for both β pixel-identical to still-loading. Naming the removals is
|
| 274 |
+
# what lets the shell tell "not part of this workspace" from "we could not read it".
|
| 275 |
+
_omitted = []
|
| 276 |
if tmods != "all":
|
| 277 |
allowed = {str(k) for k in (tmods or [])}
|
| 278 |
+
_omitted = sorted({str(p.get("key")) for p in pages
|
| 279 |
+
if p.get("key") not in allowed
|
| 280 |
+
and (p.get("parent") or "") not in allowed})
|
| 281 |
pages = [p for p in pages
|
| 282 |
if p.get("key") in allowed or (p.get("parent") or "") in allowed]
|
| 283 |
# Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows β
|
| 284 |
# the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so
|
| 285 |
# the nav cannot offer a row the table routes would refuse.
|
| 286 |
+
# ββ W31-T10 (contract C1, D-175) β THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST.
|
| 287 |
+
#
|
| 288 |
+
# This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one
|
| 289 |
+
# and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below
|
| 290 |
+
# took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms
|
| 291 |
+
# in-process** β the second figure is the one that matters, because with the network gone and
|
| 292 |
+
# the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every
|
| 293 |
+
# other collapses to 20β161 ms. That is per-call CPU, and this is it.
|
| 294 |
+
# β `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/
|
| 295 |
+
# manage loop below consumes this same dict.
|
| 296 |
+
_ut_defs, _ut_locked_mode, _degraded = {}, "", []
|
| 297 |
try:
|
| 298 |
import core.user_tables as user_tables
|
| 299 |
+
_ut_defs = user_tables.all_tables(st=session.runtime) or {}
|
| 300 |
+
# β NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its
|
| 301 |
+
# `.get("recordMode")` is None too β and `None == None` would mark EVERY database locked
|
| 302 |
+
# the moment this import failed. A sentinel that can equal real data is not a sentinel.
|
| 303 |
+
_ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE)
|
| 304 |
+
_lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs})
|
| 305 |
for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin,
|
| 306 |
+
st=_lent):
|
| 307 |
pages.append({"key": e["key"], "label": e["label"],
|
| 308 |
"source": e.get("source") or "Blank", "chrome": "main"})
|
| 309 |
except Exception:
|
| 310 |
+
# ββ W31-T11 (owner item 6b) β STILL SWALLOWED, NO LONGER SILENT.
|
| 311 |
+
#
|
| 312 |
+
# A store blip must not take the whole nav down with it, and that half stands. What
|
| 313 |
+
# changed is that it used to answer **200 OK with every database missing** and say
|
| 314 |
+
# nothing β so a client cannot tell "this tenant has no databases" from "we could not
|
| 315 |
+
# read them", and the rail renders the same complete-looking thing either way. That is
|
| 316 |
+
# the owner's *"Connectors and Automation module still disappears"* class of report:
|
| 317 |
+
# the payload is a claim about what exists, and a claim it could not verify has to be
|
| 318 |
+
# marked as such. `degraded` is that mark; the shell renders it in the affected slot.
|
| 319 |
+
_degraded.append("databases")
|
| 320 |
+
pass
|
| 321 |
if not pages:
|
| 322 |
if tmods != "all":
|
| 323 |
# A provisioned tenant with no modules and no databases YET is a legitimate empty
|
|
|
|
| 342 |
meta = _read_nav_meta(session.runtime)
|
| 343 |
# Wave 21 (C3): the definitions, once β `manage`/`canDelete` below answer from `createdBy`.
|
| 344 |
# WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read.
|
| 345 |
+
# β W31-T10: that read is now the SAME one the merge above did β it used to be a second,
|
| 346 |
+
# independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`.
|
| 347 |
+
# β The fail-closed defaults still hold: both are initialised before the try above, so an
|
| 348 |
+
# import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 349 |
for p in pages:
|
| 350 |
# `manage` (R14): may THIS session change this row's icon/name? Answered HERE because
|
| 351 |
# the server is the only end that knows β the client cannot see who created a user
|
|
|
|
| 414 |
# answer sitting in a local variable.
|
| 415 |
recents = _read_recents(session.runtime, session.uname,
|
| 416 |
{str(p.get("key", "")) for p in pages})
|
| 417 |
+
# β W31-T11 β TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT.
|
| 418 |
+
# `omitted` β this workspace's catalogue does not include these modules. Deliberate.
|
| 419 |
+
# `degraded` β a part of this payload could not be read. NOT deliberate, and the shell says
|
| 420 |
+
# so in the affected slot instead of rendering a confident nothing.
|
| 421 |
+
# β A key a consumer has to test for is a key a consumer forgets to test for; both ship as
|
| 422 |
+
# `[]` rather than being omitted when empty, which is the same rule `limits` follows.
|
| 423 |
+
return {"pages": pages, "landing": landing, "recents": recents,
|
| 424 |
+
"omitted": _omitted, "degraded": _degraded}
|
| 425 |
|
| 426 |
|
| 427 |
@router.post("/nav/opened")
|
api/routes_odoo_tables.py
CHANGED
|
@@ -33,6 +33,59 @@ def _rel():
|
|
| 33 |
return odoo_relational
|
| 34 |
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
@router.post("/odoo-tables/refresh")
|
| 37 |
def refresh_odoo_tables(session: Session = Depends(require_session)):
|
| 38 |
"""Rebuild all four locked Odoo databases from the tenant's mirror β customers, products,
|
|
@@ -112,12 +165,22 @@ def odoo_tables_status(session: Session = Depends(require_session)):
|
|
| 112 |
# β W30-T31 β THE ELIGIBILITY PASS RUNS HERE TOO, and the mirror cursor is taken best-effort:
|
| 113 |
# the freshness surface must answer on a box with no mirror (that is what it is FOR), so a
|
| 114 |
# store that is not ready costs the size half of the question, never the whole endpoint.
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
try:
|
| 117 |
-
|
| 118 |
-
cur = datastore.ro_con()
|
| 119 |
except Exception: # noqa: BLE001
|
| 120 |
-
pass
|
| 121 |
eligible, why_not = sync_read_through(cur)
|
| 122 |
# β ITERATES `rel.TABLES`, NEVER A LITERAL PAIR. It read `(INVOICES_KEY, CUSTOMERS_KEY)`
|
| 123 |
# while those were the only two; the 2026-08-09 widening added products and orders, and a
|
|
@@ -152,7 +215,10 @@ def odoo_tables_status(session: Session = Depends(require_session)):
|
|
| 152 |
out[key] = {
|
| 153 |
"exists": bool(table),
|
| 154 |
"rows": (len(rows) if materialised else n),
|
| 155 |
-
|
|
|
|
|
|
|
|
|
|
| 156 |
"materialised": materialised,
|
| 157 |
# why this grid still keeps its rows in the tenant document, when it does not have to
|
| 158 |
"materialisedBecause": (why_not.get(key, "") if materialised else ""),
|
|
@@ -177,7 +243,43 @@ def odoo_tables_status(session: Session = Depends(require_session)):
|
|
| 177 |
"and the window are already live"}
|
| 178 |
for k in _sources() if k not in out}
|
| 179 |
return {"ok": True, "applicable": rel.is_royal(session.tenant), "tables": out,
|
| 180 |
-
"bound_not_declared": pending
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
|
| 182 |
|
| 183 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -231,20 +333,76 @@ def odoo_tables_status(session: Session = Depends(require_session)):
|
|
| 231 |
#: filled by the grid at render time exactly as they are on the materialised path.
|
| 232 |
GRID_SOURCES = {}
|
| 233 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
|
| 235 |
def _sources():
|
| 236 |
-
"""Build `GRID_SOURCES` lazily
|
| 237 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
"""
|
| 239 |
if GRID_SOURCES:
|
| 240 |
return GRID_SOURCES
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
rel = _rel()
|
| 242 |
# β R6 / W30-T29 β TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops
|
| 243 |
# being a fact about them. `core` never imports up, so the declaration goes this way round.
|
| 244 |
-
# β ALL EIGHT, not just the read-through-bound
|
| 245 |
-
# which is what R6 is about
|
| 246 |
-
# not a property of the data.
|
|
|
|
| 247 |
_ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES])
|
|
|
|
| 248 |
_s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)),
|
| 249 |
(lambda v: float(v or 0.0)))
|
| 250 |
GRID_SOURCES[rel.ORDERS_KEY] = {
|
|
@@ -386,6 +544,12 @@ def _sources():
|
|
| 386 |
return GRID_SOURCES
|
| 387 |
|
| 388 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 389 |
def sync_read_through(cur=None):
|
| 390 |
"""Register every grid that may stop storing rows, and REPORT why the rest may not.
|
| 391 |
|
|
@@ -686,11 +850,26 @@ def odoo_table_rows(table_key: str,
|
|
| 686 |
if fields is None:
|
| 687 |
raise err(404, "not_connected", "that database is not a connected Odoo grid")
|
| 688 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 689 |
try:
|
| 690 |
-
cur =
|
| 691 |
except RuntimeError as e:
|
| 692 |
-
#
|
|
|
|
| 693 |
raise err(503, "store_not_ready", str(e))
|
|
|
|
|
|
|
| 694 |
|
| 695 |
spec = _source_for(cur, table_key)
|
| 696 |
if spec is None:
|
|
@@ -857,7 +1036,40 @@ class TooBigToMaterialise(Exception):
|
|
| 857 |
"""A read-through table asked for WHOLE exceeds one window β refuse, never truncate."""
|
| 858 |
|
| 859 |
|
| 860 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 861 |
"""Every row of a read-through grid, in `scoped_pool`'s shape β or a refusal.
|
| 862 |
|
| 863 |
β THIS EXISTS FOR THE CLIENT WE HAVE, NOT THE ONE WE WANT. The grid still fetches whole
|
|
@@ -866,8 +1078,23 @@ def whole_pool(table_key, cur=None):
|
|
| 866 |
small one is "read all of it from the mirror". `_unmaterialisable` only ever registers a table
|
| 867 |
that fits, so the refusal below is a guard against the population GROWING past the window
|
| 868 |
later β at which point R6 requires a sentence, not a quietly shorter grid.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 869 |
"""
|
| 870 |
from harness import datastore
|
|
|
|
|
|
|
|
|
|
| 871 |
cur = cur if cur is not None else datastore.ro_con()
|
| 872 |
spec = _source_for(cur, table_key)
|
| 873 |
if spec is None:
|
|
|
|
| 33 |
return odoo_relational
|
| 34 |
|
| 35 |
|
| 36 |
+
def _rt():
|
| 37 |
+
from harness import runtime
|
| 38 |
+
return runtime
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 42 |
+
# ββ W31-T45 / D-169 β THE CROSS-TENANT MIRROR GUARD, AND WHY EVERY DOOR BELOW GOES THROUGH IT.
|
| 43 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 44 |
+
#
|
| 45 |
+
# `harness/datastore` holds ONE process-wide DuckDB connection over a process-global `DB_PATH`,
|
| 46 |
+
# and ONE Space process serves EVERY tenant. `TenantRuntime.assert_datastore_matches` was written
|
| 47 |
+
# for exactly the failure that arrangement produces β its own docstring names it: *"an analytical
|
| 48 |
+
# read served from another tenant's file returns real, plausible, wrong rows β¦ and the number
|
| 49 |
+
# reconciles against the wrong book"* β and until this wave its only caller anywhere was
|
| 50 |
+
# `verify_api.py`. The guard was correct, tested, and governed nothing.
|
| 51 |
+
#
|
| 52 |
+
# β WHAT ACTUALLY HELD THE BOUNDARY WAS THE CUSTOMER LIST, NOT THE CODE. Three of the four doors
|
| 53 |
+
# below are reachable only by a tenant whose own `user_tables` document declares the table
|
| 54 |
+
# (`_defn_or_refuse`) or whose slug passes `is_royal` β so no second tenant could reach the mirror
|
| 55 |
+
# because no second tenant had mirror databases. R2 puts Meta Ads on this same mirror for GTM Lab,
|
| 56 |
+
# which ends that fact. A boundary that holds because of who our customers are is not a boundary.
|
| 57 |
+
#
|
| 58 |
+
# β AND ONE DOOR WAS ALREADY THROUGH IT TODAY β measured, not theorised: `/odoo-tables/status`
|
| 59 |
+
# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable`
|
| 60 |
+
# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row
|
| 61 |
+
# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes.
|
| 62 |
+
def _mirror_cur(session):
|
| 63 |
+
"""A mirror cursor for THIS session's tenant, or a refusal that says which failure it is.
|
| 64 |
+
|
| 65 |
+
Returns `(cursor, mismatch_sentence)` β exactly one of the two is None, so a caller cannot
|
| 66 |
+
accidentally treat "refused" as "empty".
|
| 67 |
+
|
| 68 |
+
β IT CATCHES ONLY THE MISMATCH AND LETS THE OTHER `RuntimeError` THROUGH, because the two mean
|
| 69 |
+
opposite actions and every caller already has a policy for the second one:
|
| 70 |
+
|
| 71 |
+
* `DatastoreMismatch` β this process has ANOTHER tenant's file open. It never fixes itself by
|
| 72 |
+
waiting; the deployment is mis-pinned. Returned as a SENTENCE so each door can choose its
|
| 73 |
+
own status (the row doors raise, the status door reports β R6's second sentence).
|
| 74 |
+
* a plain `RuntimeError` β the mirror is completing its first sync, and retrying works. It
|
| 75 |
+
PROPAGATES, so `odoo_table_rows` keeps answering `503 store_not_ready` and the status door
|
| 76 |
+
keeps degrading to "no size half", exactly as each did before this guard existed.
|
| 77 |
+
|
| 78 |
+
β Collapsing them was the tempting simplification and it is the defect: `DatastoreMismatch`
|
| 79 |
+
SUBCLASSES `RuntimeError`, so a single `except RuntimeError` here would render every
|
| 80 |
+
cross-tenant refusal as a retry banner. Catching the subclass first is a correctness rule.
|
| 81 |
+
"""
|
| 82 |
+
rtm = _rt()
|
| 83 |
+
try:
|
| 84 |
+
return rtm.mirror_cursor(session.runtime), None
|
| 85 |
+
except rtm.DatastoreMismatch as e:
|
| 86 |
+
return None, str(e)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
@router.post("/odoo-tables/refresh")
|
| 90 |
def refresh_odoo_tables(session: Session = Depends(require_session)):
|
| 91 |
"""Rebuild all four locked Odoo databases from the tenant's mirror β customers, products,
|
|
|
|
| 165 |
# β W30-T31 β THE ELIGIBILITY PASS RUNS HERE TOO, and the mirror cursor is taken best-effort:
|
| 166 |
# the freshness surface must answer on a box with no mirror (that is what it is FOR), so a
|
| 167 |
# store that is not ready costs the size half of the question, never the whole endpoint.
|
| 168 |
+
# ββ W31-T45 / D-169 β THIS IS THE DOOR THAT WAS ALREADY LEAKING, and it leaked as COUNTS
|
| 169 |
+
# rather than as rows, which is why nothing caught it. There is no tenant gate above this
|
| 170 |
+
# line: `rel.is_royal(session.tenant)` appears once, as an `applicable` FIELD in the response
|
| 171 |
+
# 60 lines below. So a nurilab or gtmlab session took a cursor on tenant #0's `royal.duckdb`,
|
| 172 |
+
# `_source_for` read its columns and `_ds.window` counted its rows β real, plausible numbers
|
| 173 |
+
# belonging to another customer, returned 200 OK.
|
| 174 |
+
# β IT REPORTS RATHER THAN RAISING, deliberately, and the distinction is R6's second sentence:
|
| 175 |
+
# "which Odoo databases exist for me" is a legitimate question for any tenant and its honest
|
| 176 |
+
# answer here is *none* β a 409 would be refusing the question instead of the leak. So the
|
| 177 |
+
# mirror half is refused, `rowsFrom` says so per table, and the refusal rides the payload with
|
| 178 |
+
# its cause. A count that silently became `-1` would be the silent truncation R6 bans.
|
| 179 |
+
cur, mirror_refused = None, None
|
| 180 |
try:
|
| 181 |
+
cur, mirror_refused = _mirror_cur(session)
|
|
|
|
| 182 |
except Exception: # noqa: BLE001
|
| 183 |
+
pass # mid-first-sync (or no mirror at all) β the document half answers
|
| 184 |
eligible, why_not = sync_read_through(cur)
|
| 185 |
# β ITERATES `rel.TABLES`, NEVER A LITERAL PAIR. It read `(INVOICES_KEY, CUSTOMERS_KEY)`
|
| 186 |
# while those were the only two; the 2026-08-09 widening added products and orders, and a
|
|
|
|
| 215 |
out[key] = {
|
| 216 |
"exists": bool(table),
|
| 217 |
"rows": (len(rows) if materialised else n),
|
| 218 |
+
# β W31-T45: a refused mirror says `refused`, never `mirror` β a size that came from
|
| 219 |
+
# nowhere must not be labelled with the source it did not come from.
|
| 220 |
+
"rowsFrom": ("document" if materialised
|
| 221 |
+
else ("refused" if mirror_refused else "mirror")),
|
| 222 |
"materialised": materialised,
|
| 223 |
# why this grid still keeps its rows in the tenant document, when it does not have to
|
| 224 |
"materialisedBecause": (why_not.get(key, "") if materialised else ""),
|
|
|
|
| 243 |
"and the window are already live"}
|
| 244 |
for k in _sources() if k not in out}
|
| 245 |
return {"ok": True, "applicable": rel.is_royal(session.tenant), "tables": out,
|
| 246 |
+
"bound_not_declared": pending,
|
| 247 |
+
# β W31-T45 / D-169 β R6's SECOND SENTENCE, ON A GUARD RATHER THAN ON A ROW CAP. A
|
| 248 |
+
# limit that genuinely cannot be removed must be REPORTED with its cause and a
|
| 249 |
+
# recommended fix; silence is the violation. `null` when the mirror answered.
|
| 250 |
+
"mirrorRefused": (
|
| 251 |
+
{"cause": mirror_refused,
|
| 252 |
+
"effect": "sizes_unavailable",
|
| 253 |
+
"recommendation": "this deployment has another tenant's analytical store open; "
|
| 254 |
+
"sizes are withheld rather than read from it. Pin the process "
|
| 255 |
+
"to this tenant (AIOS_DUCKDB_PATH / datastore.use_path) to "
|
| 256 |
+
"restore them."}
|
| 257 |
+
if mirror_refused else None),
|
| 258 |
+
# β W31-T46 / D-160 β WHY THERE IS NO MIRROR, when there is none. The last time this
|
| 259 |
+
# was silent it read as a pinned tag and a store problem for days. `null` when the
|
| 260 |
+
# container has one; a sentence with a cause and a fix when it does not.
|
| 261 |
+
"mirrorSeed": _seed_state()}
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def _seed_state():
|
| 265 |
+
"""`main.MIRROR_SEED` as a reportable block, or None when the mirror is present.
|
| 266 |
+
|
| 267 |
+
β IMPORTED LAZILY AND FAIL-QUIET: `main` imports this router, so a module-level import here
|
| 268 |
+
would be a cycle, and a freshness surface must never 500 because a diagnostic was unavailable.
|
| 269 |
+
"""
|
| 270 |
+
try:
|
| 271 |
+
import main as _main
|
| 272 |
+
from harness import datastore as _ds
|
| 273 |
+
if _ds.DB_PATH.exists():
|
| 274 |
+
return None
|
| 275 |
+
state = dict(_main.MIRROR_SEED)
|
| 276 |
+
if not state.get("cause"):
|
| 277 |
+
state["cause"] = "this container has no analytical mirror yet"
|
| 278 |
+
state["recommendation"] = ("the boot seed runs independently of AIOS_PREWARM; if this "
|
| 279 |
+
"persists, check HF_TOKEN on the deployment")
|
| 280 |
+
return state
|
| 281 |
+
except Exception: # noqa: BLE001
|
| 282 |
+
return None
|
| 283 |
|
| 284 |
|
| 285 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 333 |
#: filled by the grid at render time exactly as they are on the materialised path.
|
| 334 |
GRID_SOURCES = {}
|
| 335 |
|
| 336 |
+
#: ββ W31-T49 / C4 β THE SOURCE PROVIDERS. Odoo builds its specs below; ANY OTHER connector adds
|
| 337 |
+
#: its own by registering a builder here, and `_sources()` folds them into the one registry every
|
| 338 |
+
#: door already reads.
|
| 339 |
+
#:
|
| 340 |
+
#: β THIS IS THE SEAM THAT KEEPS "ONE REGISTRY" TRUE WHILE THE FILE STAYS ODOO-NAMED. The
|
| 341 |
+
#: alternative β a second `META_GRID_SOURCES` consulted beside this one β would give the platform
|
| 342 |
+
#: two answers to *"how is this connected grid read?"*, and every consumer (`_source_for`,
|
| 343 |
+
#: `whole_pool`, `population`, `odoo_tables_status`, `routes_tables.scoped_pids`) would have to
|
| 344 |
+
#: learn both or silently serve one. That is [[one-question-two-normalizers]] on the read path.
|
| 345 |
+
#: β A provider registers a CALLABLE, not a dict, so its module is not imported until the first
|
| 346 |
+
#: read β the same lazy rule `_sources` already follows for `odoo_relational`.
|
| 347 |
+
_SOURCE_PROVIDERS = []
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def register_source_provider(fn):
|
| 351 |
+
"""Add a `() -> {table_key: spec}` builder to the connected-grid registry.
|
| 352 |
+
|
| 353 |
+
Additive and idempotent by identity, so a re-import cannot double-register. Returns the number
|
| 354 |
+
of providers now known β a caller that wants to assert its registration took has a number.
|
| 355 |
+
"""
|
| 356 |
+
if callable(fn) and fn not in _SOURCE_PROVIDERS:
|
| 357 |
+
_SOURCE_PROVIDERS.append(fn)
|
| 358 |
+
return len(_SOURCE_PROVIDERS)
|
| 359 |
+
|
| 360 |
|
| 361 |
def _sources():
|
| 362 |
+
"""Build `GRID_SOURCES` lazily by asking EVERY registered provider β Odoo is one of them.
|
| 363 |
+
|
| 364 |
+
ββ ODOO GOES THROUGH THE SEAM TOO, AND THAT IS THE POINT OF W31-T49 RATHER THAN A FLOURISH.
|
| 365 |
+
The first version built Odoo's specs inline here and folded other providers in afterwards,
|
| 366 |
+
which quietly says "Odoo is the registry and everyone else is an addendum" β two mechanisms
|
| 367 |
+
for one question, with the second one exercised by nobody until a connector arrives. It also
|
| 368 |
+
left `register_source_provider` with no production caller at all, which `verify_reachability`
|
| 369 |
+
correctly reddened as a capability behind no door ([[artifact-with-no-importer]]). Registering
|
| 370 |
+
the incumbent through its own seam makes the seam load-bearing from the first request.
|
| 371 |
+
|
| 372 |
+
β STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and
|
| 373 |
+
importing that module at file-import time would drag the Odoo layer into every process that
|
| 374 |
+
mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
|
| 375 |
"""
|
| 376 |
if GRID_SOURCES:
|
| 377 |
return GRID_SOURCES
|
| 378 |
+
for build in list(_SOURCE_PROVIDERS):
|
| 379 |
+
try:
|
| 380 |
+
extra = build() or {}
|
| 381 |
+
except Exception: # noqa: BLE001
|
| 382 |
+
# β ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must
|
| 383 |
+
# not go dark because a newer connector's module is unhappy β and vice versa.
|
| 384 |
+
continue
|
| 385 |
+
if extra:
|
| 386 |
+
GRID_SOURCES.update(extra)
|
| 387 |
+
_ut().register_connected(*extra)
|
| 388 |
+
return GRID_SOURCES
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
def _odoo_sources():
|
| 392 |
+
"""Odoo's read-through bindings β `{table_key: spec}`. Registered as a provider below.
|
| 393 |
+
|
| 394 |
+
β IT RETURNS a dict rather than mutating the module global: `_sources` owns the merge, so a
|
| 395 |
+
provider that half-built its specs and raised cannot leave a partial registry behind.
|
| 396 |
+
"""
|
| 397 |
rel = _rel()
|
| 398 |
# β R6 / W30-T29 β TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops
|
| 399 |
# being a fact about them. `core` never imports up, so the declaration goes this way round.
|
| 400 |
+
# β ALL EIGHT, not just the read-through-bound ones β every row in these tables comes from
|
| 401 |
+
# Odoo, which is what R6 is about; being served from the stored copy today is our conversion
|
| 402 |
+
# state, not a property of the data. `_sources`' own `register_connected` covers only the keys
|
| 403 |
+
# a provider RETURNS, so this call is not redundant with it and must not be folded into it.
|
| 404 |
_ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES])
|
| 405 |
+
GRID_SOURCES = {} # the LOCAL registry this builder fills and returns
|
| 406 |
_s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)),
|
| 407 |
(lambda v: float(v or 0.0)))
|
| 408 |
GRID_SOURCES[rel.ORDERS_KEY] = {
|
|
|
|
| 544 |
return GRID_SOURCES
|
| 545 |
|
| 546 |
|
| 547 |
+
#: Odoo registers itself, at import, exactly as any other connector does. β The order providers
|
| 548 |
+
#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
|
| 549 |
+
#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
|
| 550 |
+
register_source_provider(_odoo_sources)
|
| 551 |
+
|
| 552 |
+
|
| 553 |
def sync_read_through(cur=None):
|
| 554 |
"""Register every grid that may stop storing rows, and REPORT why the rest may not.
|
| 555 |
|
|
|
|
| 850 |
if fields is None:
|
| 851 |
raise err(404, "not_connected", "that database is not a connected Odoo grid")
|
| 852 |
|
| 853 |
+
# ββ W31-T45 / D-169 β THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot.
|
| 854 |
+
# That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process
|
| 855 |
+
# has open the one this TENANT's rows live in". A definition wall is satisfied the moment a
|
| 856 |
+
# tenant's own document declares a connected table β which is exactly what R2's Meta Ads spawn
|
| 857 |
+
# gives GTM Lab β and it would then serve that session a window over whatever DuckDB file the
|
| 858 |
+
# process happens to be pinned to. Rows, not counts. So this door RAISES.
|
| 859 |
+
# β 409, NOT 503, and the two were one line from being confused: `DatastoreMismatch` subclasses
|
| 860 |
+
# `RuntimeError`, and the handler directly below turns a `RuntimeError` into
|
| 861 |
+
# `503 store_not_ready` β "the store is completing its first sync, retry in a few minutes". A
|
| 862 |
+
# mis-pinned process never becomes un-mis-pinned by waiting, so relaying it as a retry would
|
| 863 |
+
# turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the
|
| 864 |
+
# try, not inside it.
|
| 865 |
try:
|
| 866 |
+
cur, mismatch = _mirror_cur(session)
|
| 867 |
except RuntimeError as e:
|
| 868 |
+
# β Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
|
| 869 |
+
# mismatch subclass, which is what makes this broad clause safe to keep here.
|
| 870 |
raise err(503, "store_not_ready", str(e))
|
| 871 |
+
if mismatch:
|
| 872 |
+
raise err(409, "cross_tenant_store", mismatch)
|
| 873 |
|
| 874 |
spec = _source_for(cur, table_key)
|
| 875 |
if spec is None:
|
|
|
|
| 1036 |
"""A read-through table asked for WHOLE exceeds one window β refuse, never truncate."""
|
| 1037 |
|
| 1038 |
|
| 1039 |
+
def population(table_key, cur=None, rt=None):
|
| 1040 |
+
"""How many rows a read-through grid HAS, without fetching one β or None if it is not bound.
|
| 1041 |
+
|
| 1042 |
+
ββ W31 / B's ask (mailbox/B.md B-2). `routes_tables.scoped_pids` learned "this grid is too big
|
| 1043 |
+
to list" the only way that existed: call `whole_pool()` and catch `TooBigToMaterialise`. That
|
| 1044 |
+
pulls a full 5,000-row window out of DuckDB and throws it away on every `/workspace` for
|
| 1045 |
+
`ut_odoo_gl_lines` and `ut_odoo_order_lines` β MEASURED by B at ~1,600 ms of a ~3,460 ms
|
| 1046 |
+
in-proc envelope. This asks the SIZE instead: one `SELECT count(*)` over the same predicate.
|
| 1047 |
+
|
| 1048 |
+
β IT ANSWERS A DIFFERENT QUESTION, NOT THE SAME ONE MORE CHEAPLY, and that distinction is what
|
| 1049 |
+
keeps it safe. B's constraint is that the pid set must stay IDENTICAL to `scoped_pool`'s BY
|
| 1050 |
+
CONSTRUCTION rather than by two queries that agree today ([[one-question-two-normalizers]]) β
|
| 1051 |
+
so this returns a COUNT and never a pid. A caller uses it to decide whether to ask for pids at
|
| 1052 |
+
all; when it does ask, the pids still come from the one `whole_pool` fetch they always did.
|
| 1053 |
+
|
| 1054 |
+
β `None` means "no read-through binding on this deployment", which is NOT `0`. A caller that
|
| 1055 |
+
collapses them reports an empty grid where it should report an unbound one β the exact
|
| 1056 |
+
distinction `odoo_tables_status` already reports as `bound_not_declared`.
|
| 1057 |
+
β `total` is the count over the spec's OWN `where`, i.e. the same population `whole_pool`
|
| 1058 |
+
would return β the fixed scope is not dropped for being cheaper.
|
| 1059 |
+
"""
|
| 1060 |
+
from harness import datastore
|
| 1061 |
+
if rt is not None:
|
| 1062 |
+
rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry
|
| 1063 |
+
cur = cur if cur is not None else datastore.ro_con()
|
| 1064 |
+
spec = _source_for(cur, table_key)
|
| 1065 |
+
if spec is None:
|
| 1066 |
+
return None
|
| 1067 |
+
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 1068 |
+
return datastore.window(select="1", where=spec.get("where") or "", order_by=spec["id"],
|
| 1069 |
+
limit=1, cur=cur, **frm)["total"]
|
| 1070 |
+
|
| 1071 |
+
|
| 1072 |
+
def whole_pool(table_key, cur=None, rt=None):
|
| 1073 |
"""Every row of a read-through grid, in `scoped_pool`'s shape β or a refusal.
|
| 1074 |
|
| 1075 |
β THIS EXISTS FOR THE CLIENT WE HAVE, NOT THE ONE WE WANT. The grid still fetches whole
|
|
|
|
| 1078 |
small one is "read all of it from the mirror". `_unmaterialisable` only ever registers a table
|
| 1079 |
that fits, so the refusal below is a guard against the population GROWING past the window
|
| 1080 |
later β at which point R6 requires a sentence, not a quietly shorter grid.
|
| 1081 |
+
|
| 1082 |
+
ββ W31-T45 / D-169 β `rt` IS THE TENANT RUNTIME, AND IT IS OPTIONAL FOR A STATED REASON
|
| 1083 |
+
RATHER THAN A LAZY ONE. This function serves ROWS, so it is the door where a cross-tenant read
|
| 1084 |
+
would be worst β but its only production caller is `routes_tables.py::_read_through_rows`,
|
| 1085 |
+
which is in another lane's fence THIS WAVE (B's W31-T20 is rewriting that exact path) and does
|
| 1086 |
+
not thread a session down to here. Making `rt` required would break that caller on import; a
|
| 1087 |
+
second predicate invented locally would be one question with two normalizers, a recorded
|
| 1088 |
+
defect class here. So the guard fires when a caller passes the runtime it already holds, and
|
| 1089 |
+
the ask to thread `session.runtime` through `_read_through_rows` is posted to B in
|
| 1090 |
+
`mailbox/D.md`. β Until that lands this door is guarded only by `_defn_or_refuse` upstream β
|
| 1091 |
+
stated here rather than left for a reader to discover, because an unguarded row door is
|
| 1092 |
+
precisely what D-169 is about.
|
| 1093 |
"""
|
| 1094 |
from harness import datastore
|
| 1095 |
+
if rt is not None:
|
| 1096 |
+
cur = cur if cur is not None else _rt().mirror_cursor(rt)
|
| 1097 |
+
rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent
|
| 1098 |
cur = cur if cur is not None else datastore.ro_con()
|
| 1099 |
spec = _source_for(cur, table_key)
|
| 1100 |
if spec is None:
|
api/routes_tables.py
CHANGED
|
@@ -155,6 +155,63 @@ def _records_or_refuse(session, table_key):
|
|
| 155 |
#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY.
|
| 156 |
|
| 157 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
def scoped_pool(session: Session, table_key: str):
|
| 159 |
"""`(pids, rows_src, fields_base, defn)` β THE USER-TABLE WALL, on its own.
|
| 160 |
|
|
@@ -175,17 +232,17 @@ def scoped_pool(session: Session, table_key: str):
|
|
| 175 |
# an EMPTY GRID β correct-looking, wrong, and silent.
|
| 176 |
# β The wall above has already run. This adds no scope of its own and takes none away.
|
| 177 |
if not _ut().materialises(table_key, st=session.runtime, defn=defn):
|
| 178 |
-
import routes_odoo_tables
|
| 179 |
try:
|
| 180 |
-
rows_src =
|
| 181 |
-
except
|
| 182 |
# R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
raise err(409, "window_required", str(e))
|
| 184 |
except RuntimeError as e:
|
| 185 |
raise err(503, "store_not_ready", str(e))
|
| 186 |
-
rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"}
|
| 187 |
-
for r in rows_src]
|
| 188 |
-
rows_src.sort(key=lambda r: r["pid"])
|
| 189 |
return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn
|
| 190 |
rows_src = []
|
| 191 |
for rid, row in (defn.get("rows") or {}).items():
|
|
@@ -279,7 +336,7 @@ def delete_shared_field(table_key: str, field_key: str,
|
|
| 279 |
"fields": list(shared_overlay.fields(table_key, st=session.runtime))}
|
| 280 |
|
| 281 |
|
| 282 |
-
def scoped_pids(session: Session, table_key: str):
|
| 283 |
"""`(pids, fields_base, defn)` β the SAME wall and the SAME row set as `scoped_pool`, without
|
| 284 |
building a row.
|
| 285 |
|
|
@@ -298,17 +355,36 @@ def scoped_pids(session: Session, table_key: str):
|
|
| 298 |
|
| 299 |
β It does NOT make the write cheap on its own: `_defn_or_refuse` still costs two whole-document
|
| 300 |
deep copies (`ut.get` then `may_open`), which is D-87 and W30-T31. This removes the row pass.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 301 |
"""
|
| 302 |
defn = _defn_or_refuse(session, table_key)
|
| 303 |
fields_base = [dict(f) for f in (defn.get("fields") or [])]
|
| 304 |
# β W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the
|
| 305 |
# comprehension below would answer an EMPTY pid set β and the promise this function makes is
|
| 306 |
-
# that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It
|
| 307 |
-
# rather than growing a second idea of the row set;
|
| 308 |
-
# every materialised table, which is all of the big ones.
|
| 309 |
if not _ut().materialises(table_key, st=session.runtime, defn=defn):
|
| 310 |
-
|
| 311 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 312 |
pids = frozenset(int(rid) for rid in (defn.get("rows") or {}) if str(rid).isdigit())
|
| 313 |
return pids, fields_base, defn
|
| 314 |
|
|
@@ -323,7 +399,8 @@ def ut_write_ctx(session: Session, table_key: str):
|
|
| 323 |
import aios_grid
|
| 324 |
from core import grid_events
|
| 325 |
|
| 326 |
-
|
|
|
|
| 327 |
ctx = grid_events.EventCtx(
|
| 328 |
uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(),
|
| 329 |
admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime,
|
|
@@ -336,11 +413,13 @@ def ut_write_ctx(session: Session, table_key: str):
|
|
| 336 |
"fields": fields, "views": views, "lists": lists,
|
| 337 |
"derived": aios_grid.cohort_cells(lists),
|
| 338 |
"measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
|
| 339 |
-
|
|
|
|
|
|
|
| 340 |
|
| 341 |
|
| 342 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 343 |
-
consume_corrections: bool = True):
|
| 344 |
"""The user-table mirror of `grid_assembly` / `product_assembly` β SAME g-dict keys, so
|
| 345 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 346 |
|
|
@@ -350,11 +429,29 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 350 |
β WAVE 19 / R9 β `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets
|
| 351 |
cohorts like every other database, out of its OWN bucket (`ut_<slug>_cohorts`), holding its
|
| 352 |
own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and
|
| 353 |
-
wrong the moment the store learned about topics.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
import aios_grid
|
| 355 |
from core import grid_events
|
| 356 |
|
| 357 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
|
| 359 |
ops = _ops(session, table_key)
|
| 360 |
ctx = grid_events.EventCtx(
|
|
@@ -375,7 +472,9 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 375 |
# R9: the Cohorts column's cells from this table's own lists.
|
| 376 |
"derived": aios_grid.cohort_cells(lists),
|
| 377 |
"measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
|
| 378 |
-
|
|
|
|
|
|
|
| 379 |
|
| 380 |
|
| 381 |
def ut_label(defn, key, meta=None):
|
|
@@ -405,17 +504,29 @@ def nav_meta(session):
|
|
| 405 |
|
| 406 |
@router.get("/tables")
|
| 407 |
def list_tables(session: Session = Depends(require_session)):
|
| 408 |
-
"""This session's user tables β the list the '+ New database' surface renders.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
ut = _ut()
|
| 410 |
meta = nav_meta(session)
|
| 411 |
out = []
|
| 412 |
-
|
|
|
|
|
|
|
| 413 |
key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()):
|
| 414 |
-
if not ut.may_open(key, session.uname, session.admin, st=
|
| 415 |
continue
|
| 416 |
out.append({"key": key, "label": ut_label(t, key, meta),
|
| 417 |
"source": t.get("source") or "Blank",
|
| 418 |
-
"recordsMutable": ut.records_mutable(key, st=
|
| 419 |
"createdBy": t.get("createdBy") or "",
|
| 420 |
"created": t.get("created") or "",
|
| 421 |
"fields": [dict(f) for f in (t.get("fields") or [])],
|
|
|
|
| 155 |
#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY.
|
| 156 |
|
| 157 |
|
| 158 |
+
def _too_big():
|
| 159 |
+
"""`routes_odoo_tables.TooBigToMaterialise`, imported lazily β one name, two policies below."""
|
| 160 |
+
import routes_odoo_tables
|
| 161 |
+
return routes_odoo_tables.TooBigToMaterialise
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _read_through_rows(table_key, field_keys, rt=None):
|
| 165 |
+
"""The mirror's rows for one read-through grid, projected to this table's declared columns.
|
| 166 |
+
|
| 167 |
+
ββ W31-T45 / D-169 β `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask,
|
| 168 |
+
`mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*;
|
| 169 |
+
the guard `whole_pool` fires on `rt` answers a DIFFERENT question β *"is the DuckDB file this
|
| 170 |
+
process has open THIS TENANT's"* β and R2 gives GTM Lab connected tables in its own document,
|
| 171 |
+
which is exactly the shape that satisfies the first wall while failing the second.
|
| 172 |
+
`datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every
|
| 173 |
+
tenant, so the two walls are not substitutes for each other.
|
| 174 |
+
|
| 175 |
+
β ONE FETCH, TWO POLICIES β and the split is the whole of W31-T20. Both `scoped_pool` (which
|
| 176 |
+
owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror
|
| 177 |
+
through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's
|
| 178 |
+
by construction rather than by a second query that agrees today. A pid-only `SELECT` would be
|
| 179 |
+
cheaper and would also be a SECOND statement of what a row of this table is
|
| 180 |
+
([[one-question-two-normalizers]]) β the two callers differ in what they do with the REFUSAL,
|
| 181 |
+
never in how they ask.
|
| 182 |
+
|
| 183 |
+
Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides
|
| 184 |
+
whether that is a 409 or an unresolved pid scope.
|
| 185 |
+
"""
|
| 186 |
+
import routes_odoo_tables
|
| 187 |
+
|
| 188 |
+
rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"}
|
| 189 |
+
for r in routes_odoo_tables.whole_pool(table_key, rt=rt)]
|
| 190 |
+
rows_src.sort(key=lambda r: r["pid"])
|
| 191 |
+
return rows_src
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
#: ββ W31-T20 / D-174 β R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED.
|
| 195 |
+
#:
|
| 196 |
+
#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope=
|
| 197 |
+
#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the
|
| 198 |
+
#: grid painted an ERROR PAGE with a Retry button β a whole shipped feature nobody could open.
|
| 199 |
+
#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a
|
| 200 |
+
#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`).
|
| 201 |
+
#:
|
| 202 |
+
#: β SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" β it is
|
| 203 |
+
#: "membership is unresolved on this grid", which is a different claim and has to be made out
|
| 204 |
+
#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are
|
| 205 |
+
#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire`
|
| 206 |
+
#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`.
|
| 207 |
+
_PID_SCOPE_LIMIT = {
|
| 208 |
+
"subject": "pids", "effect": "unresolved",
|
| 209 |
+
"recommendation": "cohorts and shared-view membership are resolved per page on this grid; "
|
| 210 |
+
"filter and read it a window at a time (`/odoo-tables/{key}/rows`), where "
|
| 211 |
+
"every total is a SQL count over the whole table",
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
|
| 215 |
def scoped_pool(session: Session, table_key: str):
|
| 216 |
"""`(pids, rows_src, fields_base, defn)` β THE USER-TABLE WALL, on its own.
|
| 217 |
|
|
|
|
| 232 |
# an EMPTY GRID β correct-looking, wrong, and silent.
|
| 233 |
# β The wall above has already run. This adds no scope of its own and takes none away.
|
| 234 |
if not _ut().materialises(table_key, st=session.runtime, defn=defn):
|
|
|
|
| 235 |
try:
|
| 236 |
+
rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime)
|
| 237 |
+
except _too_big() as e:
|
| 238 |
# R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid.
|
| 239 |
+
# β THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not
|
| 240 |
+
# this): a caller that asked for every row of a 963,783-row grid cannot be served a
|
| 241 |
+
# short one. `scoped_pids` below takes the same refusal and answers a different
|
| 242 |
+
# question with it, because an envelope needs no row.
|
| 243 |
raise err(409, "window_required", str(e))
|
| 244 |
except RuntimeError as e:
|
| 245 |
raise err(503, "store_not_ready", str(e))
|
|
|
|
|
|
|
|
|
|
| 246 |
return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn
|
| 247 |
rows_src = []
|
| 248 |
for rid, row in (defn.get("rows") or {}).items():
|
|
|
|
| 336 |
"fields": list(shared_overlay.fields(table_key, st=session.runtime))}
|
| 337 |
|
| 338 |
|
| 339 |
+
def scoped_pids(session: Session, table_key: str, limits=None):
|
| 340 |
"""`(pids, fields_base, defn)` β the SAME wall and the SAME row set as `scoped_pool`, without
|
| 341 |
building a row.
|
| 342 |
|
|
|
|
| 355 |
|
| 356 |
β It does NOT make the write cheap on its own: `_defn_or_refuse` still costs two whole-document
|
| 357 |
deep copies (`ut.get` then `may_open`), which is D-87 and W30-T31. This removes the row pass.
|
| 358 |
+
|
| 359 |
+
ββ W31-T20 / D-174 β `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a
|
| 360 |
+
list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a
|
| 361 |
+
read-through grid whose population exceeds one window). The pid set is then EMPTY, and every
|
| 362 |
+
consumer of an empty pid set is fail-closed β but "fail-closed and unannounced" is exactly the
|
| 363 |
+
silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a
|
| 364 |
+
write is expected to carry the sentence through. Omitting the list means the caller accepts an
|
| 365 |
+
unannounced empty scope, which is only ever right for a caller that does not use the pids.
|
| 366 |
"""
|
| 367 |
defn = _defn_or_refuse(session, table_key)
|
| 368 |
fields_base = [dict(f) for f in (defn.get("fields") or [])]
|
| 369 |
# β W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the
|
| 370 |
# comprehension below would answer an EMPTY pid set β and the promise this function makes is
|
| 371 |
+
# that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror
|
| 372 |
+
# through the SAME fetch that function uses rather than growing a second idea of the row set;
|
| 373 |
+
# the saving W30-T30 bought stays on every materialised table, which is all of the big ones.
|
| 374 |
if not _ut().materialises(table_key, st=session.runtime, defn=defn):
|
| 375 |
+
try:
|
| 376 |
+
rows = _read_through_rows(table_key, {f["key"] for f in fields_base},
|
| 377 |
+
rt=session.runtime)
|
| 378 |
+
return frozenset(r["pid"] for r in rows), fields_base, defn
|
| 379 |
+
except _too_big() as e:
|
| 380 |
+
# β THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning
|
| 381 |
+
# this into a 409 is what made both line grids unopenable: the envelope was refused
|
| 382 |
+
# over rows it never renders. The scope is empty and SAID to be empty.
|
| 383 |
+
if limits is not None:
|
| 384 |
+
limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)})
|
| 385 |
+
return frozenset(), fields_base, defn
|
| 386 |
+
except RuntimeError as e:
|
| 387 |
+
raise err(503, "store_not_ready", str(e))
|
| 388 |
pids = frozenset(int(rid) for rid in (defn.get("rows") or {}) if str(rid).isdigit())
|
| 389 |
return pids, fields_base, defn
|
| 390 |
|
|
|
|
| 399 |
import aios_grid
|
| 400 |
from core import grid_events
|
| 401 |
|
| 402 |
+
limits = []
|
| 403 |
+
pids, fields_base, defn = scoped_pids(session, table_key, limits=limits)
|
| 404 |
ctx = grid_events.EventCtx(
|
| 405 |
uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(),
|
| 406 |
admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime,
|
|
|
|
| 413 |
"fields": fields, "views": views, "lists": lists,
|
| 414 |
"derived": aios_grid.cohort_cells(lists),
|
| 415 |
"measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
|
| 416 |
+
# β W31-T20 β the write door reads this to refuse a PID-BEARING event loudly rather
|
| 417 |
+
# than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch.
|
| 418 |
+
"limits": limits, "defn": defn}
|
| 419 |
|
| 420 |
|
| 421 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 422 |
+
consume_corrections: bool = True, with_rows: bool = True):
|
| 423 |
"""The user-table mirror of `grid_assembly` / `product_assembly` β SAME g-dict keys, so
|
| 424 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 425 |
|
|
|
|
| 429 |
β WAVE 19 / R9 β `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets
|
| 430 |
cohorts like every other database, out of its OWN bucket (`ut_<slug>_cohorts`), holding its
|
| 431 |
own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and
|
| 432 |
+
wrong the moment the store learned about topics.
|
| 433 |
+
|
| 434 |
+
ββ W31-T20 / D-174 β `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the
|
| 435 |
+
caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from
|
| 436 |
+
`/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow:
|
| 437 |
+
* the read-through line grains become OPENABLE β `scoped_pool` refused their envelope over
|
| 438 |
+
963,783 rows nobody was going to look at, which is D-174 in one sentence;
|
| 439 |
+
* every materialised `ut_*` database stops allocating a dict per row and sorting them on a
|
| 440 |
+
route whose payload has no rows in it β `ut_odoo_orders` was rebuilding 32,826 of them per
|
| 441 |
+
database switch (owner item 7).
|
| 442 |
+
β NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers
|
| 443 |
+
the identical pid set; the flag removes work, never a check β the shape W30-T30 already proved
|
| 444 |
+
on the write door.
|
| 445 |
+
"""
|
| 446 |
import aios_grid
|
| 447 |
from core import grid_events
|
| 448 |
|
| 449 |
+
limits = []
|
| 450 |
+
if with_rows:
|
| 451 |
+
pids, rows_src, fields_base, defn = scoped_pool(session, table_key)
|
| 452 |
+
else:
|
| 453 |
+
pids, fields_base, defn = scoped_pids(session, table_key, limits=limits)
|
| 454 |
+
rows_src = []
|
| 455 |
|
| 456 |
ops = _ops(session, table_key)
|
| 457 |
ctx = grid_events.EventCtx(
|
|
|
|
| 472 |
# R9: the Cohorts column's cells from this table's own lists.
|
| 473 |
"derived": aios_grid.cohort_cells(lists),
|
| 474 |
"measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
|
| 475 |
+
# β ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY β a key a consumer has to test
|
| 476 |
+
# for is a key a consumer forgets to test for, and this one carries a refusal.
|
| 477 |
+
"limits": limits, "defn": defn}
|
| 478 |
|
| 479 |
|
| 480 |
def ut_label(defn, key, meta=None):
|
|
|
|
| 504 |
|
| 505 |
@router.get("/tables")
|
| 506 |
def list_tables(session: Session = Depends(require_session)):
|
| 507 |
+
"""This session's user tables β the list the '+ New database' surface renders.
|
| 508 |
+
|
| 509 |
+
ββ W31-T12 (contract C1, D-175's third instance) β ONE DOCUMENT READ, NOT `1 + 2N`. This
|
| 510 |
+
route was never ticketed and has the same shape `/nav` and `/automations` were fixed for:
|
| 511 |
+
`all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on
|
| 512 |
+
tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that β so a tenant
|
| 513 |
+
with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked
|
| 514 |
+
about every table; it is handed the document this function already holds. See
|
| 515 |
+
`user_tables.lend`'s own note for why inlining the predicate is the one fix that is not
|
| 516 |
+
available.
|
| 517 |
+
"""
|
| 518 |
ut = _ut()
|
| 519 |
meta = nav_meta(session)
|
| 520 |
out = []
|
| 521 |
+
tables = ut.all_tables(st=session.runtime)
|
| 522 |
+
lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables})
|
| 523 |
+
for key, t in sorted(tables.items(),
|
| 524 |
key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()):
|
| 525 |
+
if not ut.may_open(key, session.uname, session.admin, st=lent):
|
| 526 |
continue
|
| 527 |
out.append({"key": key, "label": ut_label(t, key, meta),
|
| 528 |
"source": t.get("source") or "Blank",
|
| 529 |
+
"recordsMutable": ut.records_mutable(key, st=lent),
|
| 530 |
"createdBy": t.get("createdBy") or "",
|
| 531 |
"created": t.get("created") or "",
|
| 532 |
"fields": [dict(f) for f in (t.get("fields") or [])],
|
api/routes_web_agent.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_web_agent.py β the door that lets a person TEST the web agent (wave 31, R10 / D-51).
|
| 2 |
+
|
| 3 |
+
The owner's words are the whole reason this file exists: *"we already laid the foundation of this
|
| 4 |
+
but never test anything."* The capability is otherwise reachable only from inside an automation
|
| 5 |
+
run, which means the first person to discover it is broken is a customer at 3am.
|
| 6 |
+
|
| 7 |
+
GET /api/v1/web-agent/capability any session β can this deployment run a web step, and
|
| 8 |
+
if not, the SENTENCE saying why
|
| 9 |
+
POST /api/v1/web-agent/test ADMIN β run one real `web_read` and show what
|
| 10 |
+
came back, or the sentence
|
| 11 |
+
|
| 12 |
+
β NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract
|
| 13 |
+
C5); these are an operator surface over the same function, so a green test here and a red run
|
| 14 |
+
there cannot disagree about anything except the input.
|
| 15 |
+
|
| 16 |
+
β `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`,
|
| 17 |
+
so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody.
|
| 18 |
+
|
| 19 |
+
β ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's
|
| 20 |
+
network, never from this server, so this is not a door into our own infrastructure. It is still
|
| 21 |
+
admin-gated, because it spends money and because D-51 Β§5's authorisation posture ("only systems
|
| 22 |
+
the tenant is authorised to use, at their instruction") is not something an ordinary member
|
| 23 |
+
should be able to commit the tenant to.
|
| 24 |
+
|
| 25 |
+
β THIS ROUTER IS NOT MOUNTED YET. `main.py` belongs to another lane this wave, so the one
|
| 26 |
+
`app.include_router(routes_web_agent.router)` line is a cross-fence ask β and
|
| 27 |
+
`verify_web_agent.py` FAILS until it lands, deliberately: three finished routers once shipped
|
| 28 |
+
404-dead behind entirely green gates, and a gate that tolerates it is how that happens twice.
|
| 29 |
+
"""
|
| 30 |
+
from fastapi import APIRouter, Body, Depends
|
| 31 |
+
|
| 32 |
+
import web_agent
|
| 33 |
+
from deps import Session, require_session, err
|
| 34 |
+
from routes_admin import admin_gate
|
| 35 |
+
|
| 36 |
+
router = APIRouter(prefix="/api/v1")
|
| 37 |
+
|
| 38 |
+
MAX_URL = 2000
|
| 39 |
+
MAX_SELECTOR = 400
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@router.get("/web-agent/capability")
|
| 43 |
+
def web_agent_capability(session: Session = Depends(require_session)):
|
| 44 |
+
"""Can a web step run here at all? Configuration, not liveness β see `web_agent.capability`.
|
| 45 |
+
|
| 46 |
+
Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say
|
| 47 |
+
if not") and withholds the deployment detail (namespace, which token key, the image) that
|
| 48 |
+
only an operator has any use for. Nothing here is ever a credential.
|
| 49 |
+
"""
|
| 50 |
+
cap = web_agent.capability()
|
| 51 |
+
return {"ready": bool(cap["ready"]), "reason": cap["reason"],
|
| 52 |
+
"runnableKinds": cap["runnableKinds"], "profile": cap["profile"]}
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@router.post("/web-agent/test")
|
| 56 |
+
def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)):
|
| 57 |
+
"""Run ONE real `web_read` and report exactly what the seam returned.
|
| 58 |
+
|
| 59 |
+
The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`,
|
| 60 |
+
or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something
|
| 61 |
+
went wrong" would hide the one thing it exists to show.
|
| 62 |
+
"""
|
| 63 |
+
url = str(body.get("url") or "").strip()
|
| 64 |
+
selector = str(body.get("selector") or "").strip()
|
| 65 |
+
if len(url) > MAX_URL or len(selector) > MAX_SELECTOR:
|
| 66 |
+
raise err(400, "too_long", "the URL or selector is longer than this door accepts")
|
| 67 |
+
|
| 68 |
+
step = {"kind": "web_read", "id": "test", "url": url, "selector": selector,
|
| 69 |
+
"attr": (body.get("attr") or "text"), "all": bool(body.get("all")),
|
| 70 |
+
"timeoutMs": int(body.get("timeoutMs") or 20000)}
|
| 71 |
+
if body.get("waitFor"):
|
| 72 |
+
step["waitFor"] = str(body["waitFor"])
|
| 73 |
+
|
| 74 |
+
notes = []
|
| 75 |
+
result, error = web_agent.run_step(
|
| 76 |
+
step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}",
|
| 77 |
+
"log": notes.append})
|
| 78 |
+
if error:
|
| 79 |
+
# 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the
|
| 80 |
+
# web step did not succeed. A 500 here would make an ordinary "the selector matched
|
| 81 |
+
# nothing" look like a server fault.
|
| 82 |
+
return {"ok": False, "error": error, "log": notes[-6:]}
|
| 83 |
+
return {"ok": True, "result": result, "log": notes[-6:]}
|
api/web_agent.py
ADDED
|
@@ -0,0 +1,552 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""web_agent.py β THE SEAM between an automation step and a browser that runs somewhere else.
|
| 2 |
+
|
| 3 |
+
CONTRACT C5 (wave 31, ruling R10 / D-51). E ships this; **C mounts it** in
|
| 4 |
+
`automation_engine.py`'s action dispatch for `web_read` and adds the `verify_wiring` row β
|
| 5 |
+
β E cannot verify its own mounting, and an unmounted runner is a whole, correct, UNREACHABLE
|
| 6 |
+
feature, which is exactly how five wave-29 features shipped behind green gates.
|
| 7 |
+
|
| 8 |
+
result, error = web_agent.run_step(step, ctx) # -> (dict|None, str)
|
| 9 |
+
|
| 10 |
+
β **EXACTLY ONE OF THE TWO IS TRUTHY, ALWAYS.** Success is `(result, "")`; failure is
|
| 11 |
+
`(None, "<one sentence naming the cause>")`. Never `(None, "")` β a failure that writes nothing
|
| 12 |
+
and says nothing is a recorded defect class here, and it is the shape that makes an automation
|
| 13 |
+
loop forever writing blanks while every gate stays green. This module NEVER raises: every
|
| 14 |
+
exception is caught at the boundary and turned into a sentence, because the caller is a record
|
| 15 |
+
walk that must not die on one bad URL.
|
| 16 |
+
|
| 17 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 18 |
+
WHY A JOB AND NOT A BROWSER HERE
|
| 19 |
+
|
| 20 |
+
Two laws, neither negotiable: the browser runs on cloud compute and never on a developer's
|
| 21 |
+
laptop (W27/R6 + the `/browse` law), and it lives in an EPHEMERAL job so the Space image does not
|
| 22 |
+
grow and β¬0 holds (D-51/R2). So this module submits `jobs/web_agent_job.py` to HF Jobs, polls
|
| 23 |
+
**past `SCHEDULING`** to a forward-progress marker the job's own script printed, reads the result
|
| 24 |
+
out of the log, and returns it. `aios-web/api/verify_web_agent.py` fails if any shipped module
|
| 25 |
+
could launch a browser in-process, this one included.
|
| 26 |
+
|
| 27 |
+
β IT BLOCKS. Measured cold start is **9.3 s warm / ~24 s on a cold node** on the fast profile and
|
| 28 |
+
~28-32 s on the portable one (`waves/wave31/proto/web-agent-job.md` Β§4). That is fine for a
|
| 29 |
+
foreground step of an automation RUN and far too slow for a route a person is waiting on. The
|
| 30 |
+
cost is per JOB, not per step β `run_plan` sends a whole flow's steps in ONE job for that reason.
|
| 31 |
+
|
| 32 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 33 |
+
β THREE THINGS ARE TRUE OF PRODUCTION TODAY AND ARE NOT DEFECTS IN THIS FILE. `capability()`
|
| 34 |
+
reports each as a sentence, and `verify_web_agent.py` holds each to a check, so none of them can
|
| 35 |
+
be discovered by a customer instead of by us:
|
| 36 |
+
|
| 37 |
+
1. THE SPACE'S TOKEN CANNOT PAY FOR A JOB. `deploy_web.py` pushes `HF_TOKEN`, which is scoped to
|
| 38 |
+
the org `royal-imports`; HF answers **402 `Pre-paid credit balance is insufficient`** there
|
| 39 |
+
and 0 jobs have ever run under it. `AIOS_HF_TOKEN` (user `fsanyoto`, 100 jobs) works and is
|
| 40 |
+
NOT pushed. Measured 2026-08-12.
|
| 41 |
+
2. THE SPACE DOES NOT SHIP THE JOB SCRIPT. `deploy_web.py` uploads `api/*.py` and
|
| 42 |
+
`platform/{core,modules,harness}`; `jobs/` is in no manifest. And `check_upload()` walks
|
| 43 |
+
IMPORTS, so it structurally cannot see a file opened by PATH β its own comments say so, about
|
| 44 |
+
`aios_grid_fields.json`, which crashed a Space for exactly this reason.
|
| 45 |
+
3. `routes_web_agent.py` IS NOT MOUNTED. `main.py` is another lane's file this wave.
|
| 46 |
+
|
| 47 |
+
Each needs one line somebody else owns. Until then this module answers with a sentence rather
|
| 48 |
+
than a silence, which is the whole of R6's second half.
|
| 49 |
+
"""
|
| 50 |
+
import base64
|
| 51 |
+
import hashlib
|
| 52 |
+
import json
|
| 53 |
+
import os
|
| 54 |
+
import pathlib
|
| 55 |
+
import time
|
| 56 |
+
|
| 57 |
+
#: β ONE PAIR OF CONSTANTS, AND THEY MUST NAME THE SAME VERSION. The fast profile runs on a
|
| 58 |
+
#: Microsoft image that ships the BROWSERS for one Playwright version and NOT the Python package
|
| 59 |
+
#: (measured β `proto/web-agent-job.md` Β§6), so the image TAG and the pip PIN are a single
|
| 60 |
+
#: decision wearing two spellings. `verify_web_agent.py` asserts they agree, and also that
|
| 61 |
+
#: `jobs/requirements-web-agent.txt` and the job's own PEP-723 block agree with both: a constant
|
| 62 |
+
#: two features share is a defect class this repo has already paid for.
|
| 63 |
+
PLAYWRIGHT_VERSION = "1.62.0"
|
| 64 |
+
|
| 65 |
+
#: A profile is (image, how to reach an interpreter that HAS playwright, what to install in-job).
|
| 66 |
+
#: Two exist for one reason: the fast one depends on a THIRD-PARTY REGISTRY, and a capability with
|
| 67 |
+
#: no second route dies the day that registry blinks.
|
| 68 |
+
#: β `pip` IS PER-PROFILE AND IT IS NOT COSMETIC. The prebuilt image carries no Python packages
|
| 69 |
+
#: at all, so whatever the job imports must be listed here β and the CHECKPOINT import is
|
| 70 |
+
#: conditional, so the list is too. This was a real defect: `huggingface_hub` was absent, the
|
| 71 |
+
#: checkpoint upload raised `ModuleNotFoundError`, and the run still reported a correct result
|
| 72 |
+
#: because the boundary caught it β i.e. the resume half was silently off while the read half
|
| 73 |
+
#: looked perfect. `uv` needs no list: it reads the job's own PEP-723 block.
|
| 74 |
+
PROFILES = {
|
| 75 |
+
# 9.3 s warm to a parsed value. Browsers preinstalled; only the driver is pip-installed.
|
| 76 |
+
"prebuilt": {
|
| 77 |
+
"image": f"mcr.microsoft.com/playwright/python:v{PLAYWRIGHT_VERSION}-noble",
|
| 78 |
+
"pip": [f"playwright=={PLAYWRIGHT_VERSION}"],
|
| 79 |
+
"pip_checkpoint": ["huggingface_hub"],
|
| 80 |
+
"python": "python3 -u",
|
| 81 |
+
"install_deps": "0", "install_browser": "0",
|
| 82 |
+
},
|
| 83 |
+
# ~28-32 s. No external registry and no version coupling: uv reads the job's own PEP-723
|
| 84 |
+
# block, then the job apt-installs the libraries and downloads the browser itself.
|
| 85 |
+
"portable": {
|
| 86 |
+
"image": "ghcr.io/astral-sh/uv:python3.12-bookworm",
|
| 87 |
+
"pip": [], "pip_checkpoint": [],
|
| 88 |
+
"python": "uv run",
|
| 89 |
+
"install_deps": "1", "install_browser": "1",
|
| 90 |
+
},
|
| 91 |
+
}
|
| 92 |
+
DEFAULT_PROFILE = "prebuilt"
|
| 93 |
+
|
| 94 |
+
#: The log protocol. `jobs/web_agent_job.py` spells these too; they are a cross-file contract
|
| 95 |
+
#: between two processes that never import each other, so the gate asserts both sides agree.
|
| 96 |
+
MARK = "[web-agent]"
|
| 97 |
+
BOOT_MARK = f"{MARK} BOOT"
|
| 98 |
+
RESULT_BEGIN = f"{MARK} RESULT-BEGIN"
|
| 99 |
+
RESULT_END = f"{MARK} RESULT-END"
|
| 100 |
+
|
| 101 |
+
#: ββ ALL FIVE WEB ACTIONS RUN β owner ruling 2026-08-12, superseding D-51/R5.
|
| 102 |
+
#: R5 made any web action that WRITES to a third party default to "needs approval". The owner has
|
| 103 |
+
#: revoked it, verbatim: *"make sure we unblock all web actions, we don't need approval step
|
| 104 |
+
#: first wtf, I never ask for that."* There is no approval gate and no per-kind ban.
|
| 105 |
+
#: β The runner enforces the same set independently, so neither end has to trust the other.
|
| 106 |
+
RUNNABLE_KINDS = ("web_read", "web_goto", "web_fill", "web_click", "web_repair")
|
| 107 |
+
|
| 108 |
+
#: What each kind cannot run without. Checked HERE as well as in the job because a 10-second
|
| 109 |
+
#: round trip to be told "you gave me no selector" is a round trip nobody should pay for.
|
| 110 |
+
REQUIRED_FIELDS = {
|
| 111 |
+
"web_goto": ("url",),
|
| 112 |
+
"web_read": ("selector",),
|
| 113 |
+
"web_fill": ("selector", "value"),
|
| 114 |
+
"web_click": ("selector",),
|
| 115 |
+
"web_repair": ("selector",),
|
| 116 |
+
}
|
| 117 |
+
#: Kinds that CHANGE something on somebody else's site. Not a wall β nothing here refuses them
|
| 118 |
+
#: (the owner's ruling above). It exists so the run log, the dry-run affordance and any future
|
| 119 |
+
#: audit surface can all ask one question in one place instead of re-listing three strings.
|
| 120 |
+
WRITES_TO_THIRD_PARTY = ("web_fill", "web_click")
|
| 121 |
+
|
| 122 |
+
#: β THE SCRIPT LIVES AT A DIFFERENT DEPTH IN THE REPO AND IN THE CONTAINER, so a single
|
| 123 |
+
#: `parents[N]` is right in exactly one of the two places this module runs. Measured at wave 31
|
| 124 |
+
#: close, by A, before deploy #2:
|
| 125 |
+
#: repo `<repo>/aios-web/api/web_agent.py` -> parents[2] == `<repo>` -> `<repo>/jobs` β
|
| 126 |
+
#: Space `/app/api/web_agent.py` -> parents[2] == `/` -> `/jobs` β
|
| 127 |
+
#: because `aios-web/Dockerfile` flattens `api/` to `/app/api/`. The original constant was correct
|
| 128 |
+
#: locally and silently wrong in production β `capability()` would have reported "the browser job's
|
| 129 |
+
#: script is missing from this deployment": honest, and useless, which is exactly the outcome this
|
| 130 |
+
#: module's own header warns about. Both candidates are tried, nearest first, so neither layout is
|
| 131 |
+
#: privileged and a future move breaks loudly rather than quietly.
|
| 132 |
+
_HERE = pathlib.Path(__file__).resolve()
|
| 133 |
+
_CANDIDATES = (_HERE.parents[1] / "jobs", # container: /app/api -> /app/jobs
|
| 134 |
+
_HERE.parents[2] / "jobs") # repo: aios-web/api -> <repo>/jobs
|
| 135 |
+
JOB_SCRIPT = next((c / "web_agent_job.py" for c in _CANDIDATES
|
| 136 |
+
if (c / "web_agent_job.py").is_file()),
|
| 137 |
+
_CANDIDATES[-1] / "web_agent_job.py")
|
| 138 |
+
|
| 139 |
+
#: Env knobs, all optional. Names are spelled once and read through `_cfg` so the gate can find
|
| 140 |
+
#: them and `capability()` can report them without a second list.
|
| 141 |
+
ENV_TOKEN = ("AIOS_WEB_AGENT_TOKEN", "AIOS_HF_TOKEN", "HF_TOKEN")
|
| 142 |
+
ENV_NAMESPACE = "AIOS_WEB_AGENT_NAMESPACE"
|
| 143 |
+
ENV_PROFILE = "AIOS_WEB_AGENT_PROFILE"
|
| 144 |
+
ENV_FLAVOR = "AIOS_WEB_AGENT_FLAVOR"
|
| 145 |
+
ENV_REPO = "AIOS_WEB_AGENT_REPO" # optional checkpoint store; absent is fine and stated
|
| 146 |
+
|
| 147 |
+
START_TIMEOUT = float(os.environ.get("AIOS_WEB_AGENT_START_TIMEOUT") or 180)
|
| 148 |
+
RUN_TIMEOUT = float(os.environ.get("AIOS_WEB_AGENT_RUN_TIMEOUT") or 300)
|
| 149 |
+
JOB_TIMEOUT = os.environ.get("AIOS_WEB_AGENT_JOB_TIMEOUT") or "10m"
|
| 150 |
+
DEFAULT_FLAVOR = "cpu-basic" # measured no slower than cpu-upgrade, and 3x cheaper
|
| 151 |
+
POLL_SECONDS = 2.0
|
| 152 |
+
MAX_STEPS = 20 # matches the runner's own ceiling and MAX_ACTIONS
|
| 153 |
+
|
| 154 |
+
_ns_cache = {}
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _token():
|
| 158 |
+
for key in ENV_TOKEN:
|
| 159 |
+
val = (os.environ.get(key) or "").strip()
|
| 160 |
+
if val:
|
| 161 |
+
return val, key
|
| 162 |
+
return "", ""
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _profile():
|
| 166 |
+
name = (os.environ.get(ENV_PROFILE) or DEFAULT_PROFILE).strip()
|
| 167 |
+
return (name if name in PROFILES else DEFAULT_PROFILE), PROFILES.get(name) or \
|
| 168 |
+
PROFILES[DEFAULT_PROFILE]
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _api(token):
|
| 172 |
+
from huggingface_hub import HfApi # already a dependency: the store is HF-backed
|
| 173 |
+
return HfApi(token=token)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _namespace(token):
|
| 177 |
+
"""Which HF namespace the job is billed to. Explicit env wins; otherwise the token's own
|
| 178 |
+
owner, resolved ONCE per process β `whoami` is rate-limited and this answer never changes."""
|
| 179 |
+
fixed = (os.environ.get(ENV_NAMESPACE) or "").strip()
|
| 180 |
+
if fixed:
|
| 181 |
+
return fixed, ""
|
| 182 |
+
if token in _ns_cache:
|
| 183 |
+
return _ns_cache[token], ""
|
| 184 |
+
try:
|
| 185 |
+
name = str(_api(token).whoami().get("name") or "")
|
| 186 |
+
except Exception as exc: # noqa: BLE001 β a boundary
|
| 187 |
+
return "", (f"Hugging Face would not say who this deployment's token belongs to "
|
| 188 |
+
f"({type(exc).__name__}), so there is no namespace to bill a browser job to.")
|
| 189 |
+
if not name:
|
| 190 |
+
return "", "Hugging Face returned no account name for this deployment's token."
|
| 191 |
+
_ns_cache[token] = name
|
| 192 |
+
return name, ""
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def _status_of(exc):
|
| 196 |
+
"""The HTTP status of an HF error, from the RESPONSE β never by grepping the message.
|
| 197 |
+
|
| 198 |
+
β THIS WAS A REAL DEFECT AND IT LIED IN THE MOST EXPENSIVE DIRECTION. The first version asked
|
| 199 |
+
`if "402" in text`, and an `HfHubHTTPError` message carries the URL and a random hex Request
|
| 200 |
+
ID β so a genuine **403 "Make sure your token has the correct permissions"** was reported to
|
| 201 |
+
the operator as *"the account has no prepaid Jobs credit"*, sending them to buy credit for a
|
| 202 |
+
problem credit cannot fix. Measured 2026-08-12, both directions, on live HF. A status code is
|
| 203 |
+
a NUMBER the response already carries; matching it as a substring of free text is matching it
|
| 204 |
+
against every request id in the world. [[one-question-two-normalizers]]
|
| 205 |
+
"""
|
| 206 |
+
resp = getattr(exc, "response", None)
|
| 207 |
+
code = getattr(resp, "status_code", None)
|
| 208 |
+
return int(code) if isinstance(code, int) else None
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _sentence_for(exc, namespace):
|
| 212 |
+
"""Turn an HF exception into ONE sentence a person can act on.
|
| 213 |
+
|
| 214 |
+
402 and 403 get their own arms because neither is a bug in our code and both read like one β
|
| 215 |
+
and they need OPPOSITE fixes, which is exactly why they must not be confused: 402 is money on
|
| 216 |
+
an account we are allowed to bill, 403 is a token that may not bill this account at all.
|
| 217 |
+
"""
|
| 218 |
+
status, text = _status_of(exc), f"{exc}"
|
| 219 |
+
where = namespace or "the configured account"
|
| 220 |
+
if status == 402:
|
| 221 |
+
return (f"Hugging Face refused the browser job: the account it would be billed to "
|
| 222 |
+
f"({where}) has no prepaid Jobs credit. Either add credit to that account, or "
|
| 223 |
+
f"point this deployment at one that has it ({ENV_NAMESPACE} / {ENV_TOKEN[0]}). "
|
| 224 |
+
f"Nothing was read.")
|
| 225 |
+
if status in (401, 403):
|
| 226 |
+
return (f"Hugging Face refused the browser job as unauthorised for {where} β this "
|
| 227 |
+
f"deployment's token does not carry `job.write` on that account. Point it at an "
|
| 228 |
+
f"account the token owns ({ENV_NAMESPACE}), or configure a token that owns this "
|
| 229 |
+
f"one ({ENV_TOKEN[0]}). Nothing was read.")
|
| 230 |
+
return (f"The browser job could not be submitted "
|
| 231 |
+
f"({type(exc).__name__}{f' {status}' if status else ''}: "
|
| 232 |
+
f"{text.splitlines()[0][:200]}). Nothing was read.")
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def capability():
|
| 236 |
+
"""Can this deployment run a web step AT ALL, and if not, WHY β WITHOUT calling HF.
|
| 237 |
+
|
| 238 |
+
This exists so the answer is visible BEFORE a customer's automation fails at 3am. It reports
|
| 239 |
+
configuration, not liveness: a green `ready` means nothing is missing here, not that HF will
|
| 240 |
+
accept the next submission (only `run_step` can learn that, and it reports it as a sentence).
|
| 241 |
+
"""
|
| 242 |
+
token, token_key = _token()
|
| 243 |
+
name, prof = _profile()
|
| 244 |
+
reasons = []
|
| 245 |
+
if not token:
|
| 246 |
+
reasons.append(f"no Hugging Face token is configured ({' / '.join(ENV_TOKEN)})")
|
| 247 |
+
if not JOB_SCRIPT.is_file():
|
| 248 |
+
reasons.append(f"the browser job's script is missing from this deployment "
|
| 249 |
+
f"(expected {JOB_SCRIPT.name} under jobs/)")
|
| 250 |
+
return {
|
| 251 |
+
"ready": not reasons,
|
| 252 |
+
"reason": ("; ".join(reasons) or "configured"),
|
| 253 |
+
"tokenKey": token_key,
|
| 254 |
+
"namespace": (os.environ.get(ENV_NAMESPACE) or "").strip() or "(the token's own account)",
|
| 255 |
+
"profile": name,
|
| 256 |
+
"image": prof["image"],
|
| 257 |
+
"flavor": (os.environ.get(ENV_FLAVOR) or DEFAULT_FLAVOR),
|
| 258 |
+
"scriptPresent": JOB_SCRIPT.is_file(),
|
| 259 |
+
"checkpointRepo": (os.environ.get(ENV_REPO) or "").strip(),
|
| 260 |
+
"runnableKinds": list(RUNNABLE_KINDS),
|
| 261 |
+
"playwrightVersion": PLAYWRIGHT_VERSION,
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _clean_step(step, index, first=True):
|
| 266 |
+
"""A step as the runner will see it, or a sentence. Validated HERE as well as in the job:
|
| 267 |
+
a 10-second round trip to be told a URL is empty is a round trip nobody should pay for.
|
| 268 |
+
|
| 269 |
+
`first` says whether this step opens the journey. Only the FIRST step must carry a `url` β
|
| 270 |
+
every later one may continue on the page its predecessor left behind, which is the whole
|
| 271 |
+
point of a journey (open, fill, click, read).
|
| 272 |
+
"""
|
| 273 |
+
kind = str((step or {}).get("kind") or "")
|
| 274 |
+
if kind not in RUNNABLE_KINDS:
|
| 275 |
+
return None, (f"This step asks for `{kind or '(nothing)'}`, which is not a web action. "
|
| 276 |
+
f"The ones that run are {', '.join(RUNNABLE_KINDS)}.")
|
| 277 |
+
|
| 278 |
+
url = str(step.get("url") or "").strip()
|
| 279 |
+
if url and not url.lower().startswith(("http://", "https://")):
|
| 280 |
+
return None, (f"Step {index} has a web address that is not http or https ({url!r}), so "
|
| 281 |
+
f"there is nothing to open.")
|
| 282 |
+
if first and not url:
|
| 283 |
+
return None, (f"Step {index} is the first step of this journey, so it needs a web "
|
| 284 |
+
f"address to open. Later steps continue on the same page.")
|
| 285 |
+
|
| 286 |
+
for field in REQUIRED_FIELDS.get(kind, ()):
|
| 287 |
+
if field == "url":
|
| 288 |
+
continue # handled above, with a better sentence
|
| 289 |
+
if field == "value":
|
| 290 |
+
if step.get("value") is None:
|
| 291 |
+
return None, (f"Step {index} fills a field but carries no value to type into "
|
| 292 |
+
f"it.")
|
| 293 |
+
continue
|
| 294 |
+
if not str(step.get(field) or "").strip():
|
| 295 |
+
return None, (f"Step {index} ({kind}) names no element ({field}). Give it a CSS "
|
| 296 |
+
f"selector, such as `#username` or `button[type=submit]`.")
|
| 297 |
+
|
| 298 |
+
out = {"id": str(step.get("id") or f"s{index}"), "kind": kind,
|
| 299 |
+
"timeoutMs": int(step.get("timeoutMs") or 20000)}
|
| 300 |
+
if url:
|
| 301 |
+
out["url"] = url
|
| 302 |
+
for key in ("selector", "attr", "waitFor", "hint"):
|
| 303 |
+
if step.get(key):
|
| 304 |
+
out[key] = str(step[key])
|
| 305 |
+
if kind == "web_fill":
|
| 306 |
+
out["value"] = str(step.get("value"))
|
| 307 |
+
# An explicit `secret` wins; otherwise the runner infers it from the field name. Carried
|
| 308 |
+
# through so the decision is made once, by whoever knows, and not re-guessed downstream.
|
| 309 |
+
if step.get("secret") is not None:
|
| 310 |
+
out["secret"] = bool(step["secret"])
|
| 311 |
+
if step.get("all"):
|
| 312 |
+
out["all"] = True
|
| 313 |
+
if step.get("dryRun"):
|
| 314 |
+
out["dryRun"] = True
|
| 315 |
+
return out, ""
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def _shell_command(plan, prof):
|
| 319 |
+
"""The job's whole command: the script, base64-embedded, then the profile's runner.
|
| 320 |
+
|
| 321 |
+
β EMBEDDED RATHER THAN UPLOADED, DELIBERATELY. `run_uv_job`'s local-file path uploads to an
|
| 322 |
+
HF bucket and mounts it, which adds a storage dependency and a second thing to go stale; the
|
| 323 |
+
base64 route is self-contained, costs no repo commit, and cannot ship a version of the script
|
| 324 |
+
that differs from the one on disk.
|
| 325 |
+
"""
|
| 326 |
+
src = JOB_SCRIPT.read_bytes()
|
| 327 |
+
b64 = base64.b64encode(src).decode("ascii")
|
| 328 |
+
plan_b64 = base64.b64encode(json.dumps(plan).encode("utf-8")).decode("ascii")
|
| 329 |
+
repo_configured = bool((os.environ.get(ENV_REPO) or "").strip())
|
| 330 |
+
pkgs = list(prof["pip"]) + (list(prof["pip_checkpoint"]) if repo_configured else [])
|
| 331 |
+
prelude = (f"pip install --break-system-packages -q {' '.join(pkgs)} && " if pkgs else "")
|
| 332 |
+
shell = (f"printf '%s' '{b64}' | base64 -d > /tmp/web_agent_job.py && "
|
| 333 |
+
f"{prelude}{prof['python']} /tmp/web_agent_job.py")
|
| 334 |
+
env = {"WEB_AGENT_PLAN": plan_b64,
|
| 335 |
+
"WEB_AGENT_INSTALL_DEPS": prof["install_deps"],
|
| 336 |
+
"WEB_AGENT_INSTALL_BROWSER": prof["install_browser"]}
|
| 337 |
+
#: β THE JOB GETS A CREDENTIAL ONLY WHEN IT HAS SOMEWHERE TO WRITE. With no checkpoint repo
|
| 338 |
+
#: configured, no token is handed to the browser container at all β which is the better
|
| 339 |
+
#: posture and the default. `secrets` takes actual VALUES (never a key name, never an array:
|
| 340 |
+
#: `run_uv_job`'s array form 400s), and HF delivers them as env inside the container.
|
| 341 |
+
secrets = {}
|
| 342 |
+
repo = (os.environ.get(ENV_REPO) or "").strip()
|
| 343 |
+
if repo:
|
| 344 |
+
env["WEB_AGENT_REPO"] = repo
|
| 345 |
+
token, _ = _token()
|
| 346 |
+
if token:
|
| 347 |
+
secrets["HF_TOKEN"] = token
|
| 348 |
+
return ["/bin/sh", "-c", shell], env, secrets
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def run_plan(steps, ctx=None):
|
| 352 |
+
"""Run a whole flow's web steps in ONE job. `run_step` is the one-step case.
|
| 353 |
+
|
| 354 |
+
Returns `(results, "")` where results is a list aligned to `steps`, or `(None, sentence)`.
|
| 355 |
+
"""
|
| 356 |
+
ctx = ctx or {}
|
| 357 |
+
log = ctx.get("log") or (lambda _m: None)
|
| 358 |
+
|
| 359 |
+
if not steps:
|
| 360 |
+
return None, "No web step was given, so nothing was read."
|
| 361 |
+
if len(steps) > MAX_STEPS:
|
| 362 |
+
return None, (f"A single browser job carries at most {MAX_STEPS} web steps and this flow "
|
| 363 |
+
f"has {len(steps)}. Split the flow.")
|
| 364 |
+
|
| 365 |
+
cleaned = []
|
| 366 |
+
for idx, step in enumerate(steps, 1):
|
| 367 |
+
one, why = _clean_step(step, idx, first=(idx == 1))
|
| 368 |
+
if why:
|
| 369 |
+
return None, why
|
| 370 |
+
cleaned.append(one)
|
| 371 |
+
|
| 372 |
+
cap = capability()
|
| 373 |
+
if not cap["ready"]:
|
| 374 |
+
return None, (f"This deployment cannot run a web step: {cap['reason']}. Nothing was read.")
|
| 375 |
+
|
| 376 |
+
token, _ = _token()
|
| 377 |
+
namespace, why = _namespace(token)
|
| 378 |
+
if why:
|
| 379 |
+
return None, why
|
| 380 |
+
prof_name, prof = _profile()
|
| 381 |
+
#: ββ THE CHECKPOINT KEY IS (caller's runId + THE WORK), NEVER THE runId ALONE, and getting
|
| 382 |
+
#: this wrong returns one page's content for a different page's request β silently, and only
|
| 383 |
+
#: once a checkpoint store is configured. The job resumes on a runId hit by returning the
|
| 384 |
+
#: prior envelope UNCHANGED, so any caller that reuses an id across different work gets the
|
| 385 |
+
#: first answer forever. Two real callers would have done exactly that: this module's own
|
| 386 |
+
#: `/web-agent/test` route keys on `test-<tenant>-<user>` (constant per person β test URL A,
|
| 387 |
+
#: then URL B, get A back), and the automation engine walks records with ONE run id, so
|
| 388 |
+
#: records 2..N would all receive record 1's scraped value. Hashing the cleaned steps in makes
|
| 389 |
+
#: "resume" mean *this exact work already completed*, which is the only thing it may mean.
|
| 390 |
+
#: β `cleaned` is used, not the caller's raw dict: it is normalised and ordered, so two
|
| 391 |
+
#: spellings of one step share a key and a genuine difference cannot hide in a default.
|
| 392 |
+
run_id = str(ctx.get("runId") or "").strip() or f"auto-{int(time.time())}"
|
| 393 |
+
work = hashlib.sha256(
|
| 394 |
+
json.dumps(cleaned, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
| 395 |
+
plan = {"runId": f"{run_id}-{work[:12]}", "steps": cleaned, "callerRunId": run_id}
|
| 396 |
+
|
| 397 |
+
try:
|
| 398 |
+
command, env, secrets = _shell_command(plan, prof)
|
| 399 |
+
except OSError as exc:
|
| 400 |
+
return None, (f"The browser job's script could not be read from this deployment "
|
| 401 |
+
f"({type(exc).__name__}). Nothing was read.")
|
| 402 |
+
|
| 403 |
+
api = _api(token)
|
| 404 |
+
started = time.monotonic()
|
| 405 |
+
try:
|
| 406 |
+
job = api.run_job(image=prof["image"], command=command, env=env, secrets=secrets,
|
| 407 |
+
flavor=(os.environ.get(ENV_FLAVOR) or DEFAULT_FLAVOR),
|
| 408 |
+
timeout=JOB_TIMEOUT, namespace=namespace,
|
| 409 |
+
labels={"aios": "web-agent", "profile": prof_name})
|
| 410 |
+
except Exception as exc: # noqa: BLE001 β a boundary
|
| 411 |
+
return None, _sentence_for(exc, namespace)
|
| 412 |
+
|
| 413 |
+
job_id = str(getattr(job, "id", "") or "")
|
| 414 |
+
log(f"[web-agent] submitted {job_id} ({prof_name}, {namespace}) for {len(cleaned)} step(s)")
|
| 415 |
+
|
| 416 |
+
result, why = _await(api, job_id, namespace, started, log)
|
| 417 |
+
if why:
|
| 418 |
+
return None, why
|
| 419 |
+
rows = result.get("steps", [])
|
| 420 |
+
for row in rows:
|
| 421 |
+
row["jobId"] = job_id
|
| 422 |
+
|
| 423 |
+
# β A JOURNEY THAT STOPPED HALFWAY IS A FAILURE, and returning its rows with an empty error
|
| 424 |
+
# is how a caller "succeeds" on a login that never happened. Measured: a 3-step plan whose
|
| 425 |
+
# click matched nothing came back `(rows, "")` because the envelope carried per-step errors
|
| 426 |
+
# and no top-level one β every row was inspectable and nothing forced anybody to inspect
|
| 427 |
+
# them. The contract this module publishes is exactly-one-truthy, so it has to hold for a
|
| 428 |
+
# plan too. The sentence names the failing step AND what completed before it, because that
|
| 429 |
+
# is the whole diagnostic value the rows would have carried.
|
| 430 |
+
failed = [r for r in rows if not r.get("ok") and r.get("attempted") is not False]
|
| 431 |
+
if failed:
|
| 432 |
+
first = failed[0]
|
| 433 |
+
reason = str(first.get("error") or "no reason given")
|
| 434 |
+
# β ONE step is not a "journey", and wrapping its sentence in journey prose would make
|
| 435 |
+
# the single-step case β the one the automation engine actually calls β read worse than
|
| 436 |
+
# before. The wrapper earns its place only when there were other steps to lose.
|
| 437 |
+
if len(rows) == 1:
|
| 438 |
+
return None, reason
|
| 439 |
+
done = [str(r.get("id") or r.get("kind")) for r in rows if r.get("ok")]
|
| 440 |
+
skipped = [r for r in rows if r.get("attempted") is False]
|
| 441 |
+
return None, (f"Step {rows.index(first) + 1} of {len(rows)} ({first.get('kind')}) failed "
|
| 442 |
+
f"and the journey stopped: {reason}"
|
| 443 |
+
+ (f" Completed first: {', '.join(done)}." if done else "")
|
| 444 |
+
+ (f" Not attempted: {len(skipped)} later step(s)." if skipped else ""))
|
| 445 |
+
return rows, ""
|
| 446 |
+
|
| 447 |
+
|
| 448 |
+
def run_step(step, ctx=None):
|
| 449 |
+
"""C5's seam: ONE declarative web step -> `(result, error_sentence)`.
|
| 450 |
+
|
| 451 |
+
`step` {"kind": "web_read", "url", "selector", "attr"?, "all"?, "waitFor"?, "timeoutMs"?}
|
| 452 |
+
`ctx` optional {"tenant", "automationId", "runId", "log"}
|
| 453 |
+
"""
|
| 454 |
+
try:
|
| 455 |
+
rows, why = run_plan([step], ctx)
|
| 456 |
+
except Exception as exc: # noqa: BLE001 β the LAST boundary
|
| 457 |
+
# Nothing below may reach the caller as an exception: this runs inside a record walk.
|
| 458 |
+
return None, (f"The web step failed unexpectedly ({type(exc).__name__}: "
|
| 459 |
+
f"{str(exc).splitlines()[0][:200]}). Nothing was read.")
|
| 460 |
+
if why:
|
| 461 |
+
return None, why
|
| 462 |
+
if not rows:
|
| 463 |
+
# Belt and braces for the contract's own edge: a job that returned ok with no rows.
|
| 464 |
+
return None, ("The browser job finished without returning anything for this step. "
|
| 465 |
+
"Nothing was read.")
|
| 466 |
+
row = rows[0]
|
| 467 |
+
if not row.get("ok"):
|
| 468 |
+
return None, str(row.get("error") or
|
| 469 |
+
"The web step did not succeed and the job gave no reason.")
|
| 470 |
+
return row, ""
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def _await(api, job_id, namespace, started, log):
|
| 474 |
+
"""Poll past `SCHEDULING`, tail to the job's OWN marker, return its result envelope.
|
| 475 |
+
|
| 476 |
+
β NEVER 'RUNNING' ON SUBMISSION ALONE β the HF-jobs rule, and the reason this function is
|
| 477 |
+
longer than a sleep. Two deadlines, because the two failures are different and a person needs
|
| 478 |
+
to be told which one happened: a job that never STARTS is a platform queue problem, and a job
|
| 479 |
+
that starts and never SPEAKS is ours.
|
| 480 |
+
"""
|
| 481 |
+
seen, state, booted = 0, "", False
|
| 482 |
+
envelope = None
|
| 483 |
+
|
| 484 |
+
while True:
|
| 485 |
+
waited = time.monotonic() - started
|
| 486 |
+
try:
|
| 487 |
+
info = api.inspect_job(job_id=job_id, namespace=namespace)
|
| 488 |
+
state = str(getattr(info.status, "stage", info.status) or "")
|
| 489 |
+
except Exception as exc: # noqa: BLE001
|
| 490 |
+
return None, (f"The browser job {job_id} was submitted but its status could not be "
|
| 491 |
+
f"read ({type(exc).__name__}). Nothing was read.")
|
| 492 |
+
|
| 493 |
+
if state in ("RUNNING", "COMPLETED", "ERROR", "CANCELED"):
|
| 494 |
+
# β `fetch_job_logs` REPLAYS THE WHOLE LOG on every call, so the index is the dedupe.
|
| 495 |
+
# A content-based one would swallow legitimately repeated lines.
|
| 496 |
+
try:
|
| 497 |
+
lines = list(api.fetch_job_logs(job_id=job_id, namespace=namespace))
|
| 498 |
+
except Exception: # noqa: BLE001 β logs lag a fresh container
|
| 499 |
+
lines = []
|
| 500 |
+
fresh, seen = lines[seen:], max(seen, len(lines))
|
| 501 |
+
buf, capture = [], False
|
| 502 |
+
for raw in fresh:
|
| 503 |
+
line = (raw or "").rstrip()
|
| 504 |
+
if not line:
|
| 505 |
+
continue
|
| 506 |
+
if not booted and BOOT_MARK in line:
|
| 507 |
+
booted = True
|
| 508 |
+
log(f"[web-agent] job {job_id} is alive at "
|
| 509 |
+
f"+{time.monotonic() - started:.1f}s")
|
| 510 |
+
if RESULT_BEGIN in line:
|
| 511 |
+
capture, buf = True, []
|
| 512 |
+
continue
|
| 513 |
+
if RESULT_END in line:
|
| 514 |
+
capture = False
|
| 515 |
+
try:
|
| 516 |
+
envelope = json.loads("".join(buf))
|
| 517 |
+
except Exception: # noqa: BLE001
|
| 518 |
+
envelope = None
|
| 519 |
+
continue
|
| 520 |
+
if capture:
|
| 521 |
+
buf.append(line)
|
| 522 |
+
|
| 523 |
+
if envelope is not None:
|
| 524 |
+
break
|
| 525 |
+
if state in ("COMPLETED", "ERROR", "CANCELED"):
|
| 526 |
+
return None, (f"The browser job {job_id} ended {state} without returning a result"
|
| 527 |
+
f"{' β it never started printing' if not booted else ''}. "
|
| 528 |
+
f"Nothing was read.")
|
| 529 |
+
if not booted and waited > START_TIMEOUT:
|
| 530 |
+
# The submitted-but-never-started case, named as itself.
|
| 531 |
+
try:
|
| 532 |
+
api.cancel_job(job_id=job_id, namespace=namespace)
|
| 533 |
+
except Exception: # noqa: BLE001 β best effort; it may be gone
|
| 534 |
+
pass
|
| 535 |
+
return None, (f"The browser job {job_id} was accepted but had not started after "
|
| 536 |
+
f"{int(START_TIMEOUT)}s (it stayed {state or 'queued'}), so it was "
|
| 537 |
+
f"cancelled. Nothing was read β try again, or the platform is busy.")
|
| 538 |
+
if waited > RUN_TIMEOUT:
|
| 539 |
+
try:
|
| 540 |
+
api.cancel_job(job_id=job_id, namespace=namespace)
|
| 541 |
+
except Exception: # noqa: BLE001
|
| 542 |
+
pass
|
| 543 |
+
return None, (f"The browser job {job_id} ran for more than {int(RUN_TIMEOUT)}s "
|
| 544 |
+
f"without finishing and was cancelled. The page may be too slow or may "
|
| 545 |
+
f"never load. Nothing was read.")
|
| 546 |
+
time.sleep(POLL_SECONDS)
|
| 547 |
+
|
| 548 |
+
if not envelope.get("ok") and envelope.get("error"):
|
| 549 |
+
# The runner's OWN sentence, passed through untouched. Rewriting it here would put two
|
| 550 |
+
# vocabularies on one failure.
|
| 551 |
+
return None, str(envelope["error"])
|
| 552 |
+
return envelope, ""
|
jobs/requirements-web-agent.txt
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# requirements-web-agent.txt β what the browser job installs, spelled where a human reads it.
|
| 2 |
+
#
|
| 3 |
+
# β THIS FILE IS NOT THE INSTALLER. `uv` reads the PEP-723 block at the top of
|
| 4 |
+
# `jobs/web_agent_job.py`, and that block is what actually runs. This file exists because the
|
| 5 |
+
# block is buried in a script that only ever executes on someone else's machine, and a
|
| 6 |
+
# dependency surface nobody can see is one nobody audits.
|
| 7 |
+
#
|
| 8 |
+
# β SO THE TWO CAN DIVERGE, AND THAT IS A GATED DEFECT, NOT A STYLE POINT: two halves of one
|
| 9 |
+
# question in two vocabularies is a recorded defect class here. `aios-web/api/verify_web_agent.py`
|
| 10 |
+
# parses BOTH and fails if the requirement sets disagree, so this file is checked, not decorative.
|
| 11 |
+
#
|
| 12 |
+
# β NOTHING HERE IS EVER INSTALLED INTO THE SPACE. The API image must not grow a browser
|
| 13 |
+
# (D-51/R2 β that is how β¬0 holds), and `verify_web_agent.py` also asserts that
|
| 14 |
+
# `aios-web/api/requirements.txt` names none of these.
|
| 15 |
+
|
| 16 |
+
# The browser driver. `playwright install chromium` (run inside the job) fetches the matching
|
| 17 |
+
# Chromium build; `playwright install-deps chromium` apt-installs the shared libraries it links
|
| 18 |
+
# against. Neither happens here.
|
| 19 |
+
#
|
| 20 |
+
# β THE PIN IS LOAD-BEARING AND IT IS SHARED WITH A DOCKER IMAGE TAG. The fast profile runs on
|
| 21 |
+
# `mcr.microsoft.com/playwright/python:v1.62.0-noble`, which ships the BROWSERS for exactly that
|
| 22 |
+
# version and NOT the Python package (measured β see proto/web-agent-job.md Β§6). Bump this pin
|
| 23 |
+
# without bumping the tag and Playwright refuses the browser build it finds; bump the tag alone and
|
| 24 |
+
# the driver is older than the browsers. `verify_web_agent.py` asserts the two name the same version,
|
| 25 |
+
# because a constant two features share is a recorded defect class here.
|
| 26 |
+
playwright==1.62.0
|
| 27 |
+
|
| 28 |
+
# Only for the OPTIONAL result checkpoint (`WEB_AGENT_REPO`) β the primary transport is the
|
| 29 |
+
# framed block on stdout, which costs no repo commit and no upload budget.
|
| 30 |
+
huggingface_hub>=0.34
|
jobs/web_agent_job.py
ADDED
|
@@ -0,0 +1,736 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
+
# requires-python = ">=3.11"
|
| 3 |
+
# dependencies = [
|
| 4 |
+
# "playwright==1.62.0",
|
| 5 |
+
# "huggingface_hub>=0.34",
|
| 6 |
+
# ]
|
| 7 |
+
# ///
|
| 8 |
+
"""web_agent_job.py β THE BROWSER, and it runs HERE and nowhere else (D-51/R2, W31 R10).
|
| 9 |
+
|
| 10 |
+
β THIS FILE IS THE ONLY PLACE IN THE PRODUCT WHERE A BROWSER IS ALLOWED TO EXIST.
|
| 11 |
+
It never ships inside the Space image and it is never imported by the API. It is base64-embedded
|
| 12 |
+
into an ephemeral HF Job's `command` by `aios-web/api/web_agent.py`, runs once, prints its result,
|
| 13 |
+
and dies. Two laws force that shape and they are not ours to relax:
|
| 14 |
+
|
| 15 |
+
LAW (W27/R6 + /browse) the browser runs on cloud compute, never this laptop.
|
| 16 |
+
LAW (D-51/R2) the browser lives in an EPHEMERAL JOB so the Space image does not
|
| 17 |
+
grow and the β¬0 free tier holds.
|
| 18 |
+
|
| 19 |
+
`.claude/skills/browse/verify_browse_law.py` enforces both, and its product-runtime half
|
| 20 |
+
(`aios-web/api/verify_web_agent.py`) goes RED if any SHIPPED module can launch a browser
|
| 21 |
+
in-process. This file is the single declared exception, by path.
|
| 22 |
+
|
| 23 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 24 |
+
THE CONTRACT WITH THE CALLER
|
| 25 |
+
|
| 26 |
+
INPUT env `WEB_AGENT_PLAN` β base64(json) of:
|
| 27 |
+
{"runId": "<opaque>", "steps": [ {step}, ... ]}
|
| 28 |
+
A step is DECLARATIVE β a selector and a URL, never a coordinate (D-51/R3 wants the
|
| 29 |
+
recorder to emit exactly this shape later):
|
| 30 |
+
{"id": "s1", "kind": "web_read", "url": "https://β¦", "selector": "css",
|
| 31 |
+
"attr": null|"href"|"text", "all": false, "waitFor": null|"css",
|
| 32 |
+
"timeoutMs": 20000}
|
| 33 |
+
Only `web_read` is implemented. Four sibling kinds (`web_goto`/`web_fill`/`web_click`/
|
| 34 |
+
`web_repair`) are declared in the API's ACTION_CATALOG as ready:False and this runner
|
| 35 |
+
REFUSES them with a sentence rather than pretending β β a kind that writes to a third
|
| 36 |
+
party does not run at all until D-51/R5's approval gate exists.
|
| 37 |
+
|
| 38 |
+
OUTPUT a framed block on stdout, which IS the transport the caller parses:
|
| 39 |
+
[web-agent] RESULT-BEGIN
|
| 40 |
+
{"ok": true, "runId": β¦, "steps": [...], "phases": {...}}
|
| 41 |
+
[web-agent] RESULT-END
|
| 42 |
+
plus, when `WEB_AGENT_REPO` is set, the same JSON uploaded as a checkpoint so a RETRY
|
| 43 |
+
RESUMES rather than restarts (HF-jobs rule 2).
|
| 44 |
+
|
| 45 |
+
FAILURE a failed step carries a SENTENCE naming the cause, never a silent empty value
|
| 46 |
+
(W31 R6's second sentence). `ok:false` is a real answer; a missing RESULT block is what
|
| 47 |
+
the caller reports as "the job never got there".
|
| 48 |
+
|
| 49 |
+
β EVERY PHASE IS TIMED AND PRINTED. The cold-start number is not a nice-to-have here: it is
|
| 50 |
+
what decides whether a web step in an automation can ever be synchronous, so the log is the
|
| 51 |
+
measurement instrument as well as the transport.
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
import base64
|
| 55 |
+
import json
|
| 56 |
+
import os
|
| 57 |
+
import signal
|
| 58 |
+
import subprocess
|
| 59 |
+
import sys
|
| 60 |
+
import time
|
| 61 |
+
from datetime import datetime, timezone
|
| 62 |
+
|
| 63 |
+
T0 = time.monotonic()
|
| 64 |
+
PHASES: dict[str, float] = {}
|
| 65 |
+
#: Every marker the caller greps for is spelled ONCE, here. `web_agent.py` imports nothing from
|
| 66 |
+
#: this file (it never runs in the same process), so these strings are a cross-file contract and
|
| 67 |
+
#: `verify_web_agent.py` asserts both sides still agree on them.
|
| 68 |
+
MARK = "[web-agent]"
|
| 69 |
+
BOOT = f"{MARK} BOOT"
|
| 70 |
+
RESULT_BEGIN = f"{MARK} RESULT-BEGIN"
|
| 71 |
+
RESULT_END = f"{MARK} RESULT-END"
|
| 72 |
+
|
| 73 |
+
_shutdown = {"asked": False}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _elapsed() -> float:
|
| 77 |
+
return round(time.monotonic() - T0, 2)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def say(msg: str) -> None:
|
| 81 |
+
"""One log line. Unbuffered, because a buffered forward-progress marker is not one."""
|
| 82 |
+
print(f"{MARK} +{_elapsed():>7.2f}s {msg}", flush=True)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def phase(name: str, started: float) -> None:
|
| 86 |
+
PHASES[name] = round(time.monotonic() - started, 2)
|
| 87 |
+
say(f"PHASE {name} took {PHASES[name]}s")
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _on_term(signum, frame):
|
| 91 |
+
# A job killed at its timeout still gets to say so. Without this the caller sees a terminal
|
| 92 |
+
# state and no result and cannot tell a timeout from a crash.
|
| 93 |
+
_shutdown["asked"] = True
|
| 94 |
+
say(f"SIGNAL {signum} β the platform is stopping this job")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
signal.signal(signal.SIGTERM, _on_term)
|
| 98 |
+
signal.signal(signal.SIGINT, _on_term)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ββ boot ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 102 |
+
# β FIRST, BEFORE ANY IMPORT THAT COULD BLOCK. The HF-jobs rule is that "running" is only true
|
| 103 |
+
# once a marker THIS SCRIPT printed appears in the log; a marker printed after a 90-second
|
| 104 |
+
# browser install would make the rule unsatisfiable for 90 seconds.
|
| 105 |
+
print(f"{BOOT} at {datetime.now(timezone.utc).isoformat()} python={sys.version.split()[0]}",
|
| 106 |
+
flush=True)
|
| 107 |
+
|
| 108 |
+
PLAN_B64 = os.environ.get("WEB_AGENT_PLAN", "")
|
| 109 |
+
CHECKPOINT_REPO = os.environ.get("WEB_AGENT_REPO", "").strip()
|
| 110 |
+
HF_TOKEN = os.environ.get("HF_TOKEN", "") or os.environ.get("AIOS_HF_TOKEN", "")
|
| 111 |
+
#: `--with-deps` shells out to apt. Skippable ONLY on an image that already carries the libs.
|
| 112 |
+
INSTALL_DEPS = os.environ.get("WEB_AGENT_INSTALL_DEPS", "1") != "0"
|
| 113 |
+
INSTALL_BROWSER = os.environ.get("WEB_AGENT_INSTALL_BROWSER", "1") != "0"
|
| 114 |
+
MAX_VALUE_CHARS = 8000 # one cell's worth; a page is not a payload
|
| 115 |
+
MAX_STEPS = 20 # matches the engine's MAX_ACTIONS β one flow, one job
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _fail(sentence: str, *, kind: str = "refused", steps=None) -> None:
|
| 119 |
+
"""Print the result envelope with ok:false and a SENTENCE, then exit 0.
|
| 120 |
+
|
| 121 |
+
β EXIT 0 ON A HANDLED FAILURE IS DELIBERATE. A non-zero exit makes HF mark the job ERROR,
|
| 122 |
+
and the caller then reports "the job crashed" over a perfectly well-understood cause like
|
| 123 |
+
"the selector matched nothing". The envelope carries the verdict; the exit code carries only
|
| 124 |
+
whether this script itself survived.
|
| 125 |
+
"""
|
| 126 |
+
_emit({"ok": False, "error": sentence, "errorKind": kind,
|
| 127 |
+
"runId": PLAN.get("runId") if isinstance(PLAN, dict) else None,
|
| 128 |
+
"steps": steps or [], "phases": PHASES})
|
| 129 |
+
sys.exit(0)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _emit(payload: dict) -> None:
|
| 133 |
+
payload.setdefault("finishedAt", datetime.now(timezone.utc).isoformat())
|
| 134 |
+
payload.setdefault("totalSeconds", _elapsed())
|
| 135 |
+
blob = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
| 136 |
+
print(RESULT_BEGIN, flush=True)
|
| 137 |
+
print(blob, flush=True)
|
| 138 |
+
print(RESULT_END, flush=True)
|
| 139 |
+
_checkpoint(blob, payload)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _checkpoint(blob: str, payload: dict) -> None:
|
| 143 |
+
"""Durable copy, so a RETRY RESUMES rather than restarts (HF-jobs rule 2).
|
| 144 |
+
|
| 145 |
+
β IT IS OPTIONAL AND SAYS SO OUT LOUD. With no repo configured the stdout frame is the only
|
| 146 |
+
transport β which is true, survivable and worth knowing; what is NOT acceptable is a caller
|
| 147 |
+
believing a checkpoint exists because the code contains an upload. The line below is what a
|
| 148 |
+
reader of the log has to go on, so it states which of the two happened.
|
| 149 |
+
"""
|
| 150 |
+
run_id = str(payload.get("runId") or "unknown")
|
| 151 |
+
if not CHECKPOINT_REPO:
|
| 152 |
+
say("CHECKPOINT none β WEB_AGENT_REPO unset; the stdout frame is the only transport")
|
| 153 |
+
return
|
| 154 |
+
if not HF_TOKEN:
|
| 155 |
+
say(f"CHECKPOINT skipped β WEB_AGENT_REPO={CHECKPOINT_REPO} but no HF token in this job")
|
| 156 |
+
return
|
| 157 |
+
try:
|
| 158 |
+
from huggingface_hub import HfApi
|
| 159 |
+
HfApi(token=HF_TOKEN).upload_file(
|
| 160 |
+
path_or_fileobj=blob.encode("utf-8"),
|
| 161 |
+
path_in_repo=f"runs/{run_id}.json",
|
| 162 |
+
repo_id=CHECKPOINT_REPO, repo_type="dataset",
|
| 163 |
+
commit_message=f"web-agent run {run_id}")
|
| 164 |
+
say(f"CHECKPOINT wrote runs/{run_id}.json to {CHECKPOINT_REPO}")
|
| 165 |
+
except Exception as exc: # noqa: BLE001 β a boundary
|
| 166 |
+
# β CATCH EVERYTHING. This is a report-don't-raise boundary and the ways an upload can
|
| 167 |
+
# fail are not enumerable (429 commit budget, 403 scope, DNS, a repo that does not
|
| 168 |
+
# exist). A crash here would destroy a result that has ALREADY been obtained and paid
|
| 169 |
+
# for. [[gate-must-go-red-not-crash]]
|
| 170 |
+
say(f"CHECKPOINT failed ({type(exc).__name__}: {str(exc)[:160]}) β "
|
| 171 |
+
f"the stdout frame still carries the result")
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _resume(run_id: str):
|
| 175 |
+
"""A prior run's result, if this exact run already finished. None otherwise."""
|
| 176 |
+
if not (CHECKPOINT_REPO and HF_TOKEN and run_id):
|
| 177 |
+
return None
|
| 178 |
+
try:
|
| 179 |
+
from huggingface_hub import hf_hub_download
|
| 180 |
+
path = hf_hub_download(repo_id=CHECKPOINT_REPO, repo_type="dataset",
|
| 181 |
+
filename=f"runs/{run_id}.json", token=HF_TOKEN,
|
| 182 |
+
force_download=True)
|
| 183 |
+
with open(path, encoding="utf-8") as fh:
|
| 184 |
+
return json.load(fh)
|
| 185 |
+
except Exception: # noqa: BLE001
|
| 186 |
+
return None # absent is the normal case
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
# ββ the plan ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 190 |
+
PLAN: dict = {}
|
| 191 |
+
if not PLAN_B64:
|
| 192 |
+
_fail("This job was started without a plan: env WEB_AGENT_PLAN is empty, so there is "
|
| 193 |
+
"nothing to browse. The caller must set it to base64(json).", kind="no_plan")
|
| 194 |
+
try:
|
| 195 |
+
PLAN = json.loads(base64.b64decode(PLAN_B64).decode("utf-8"))
|
| 196 |
+
except Exception as exc: # noqa: BLE001
|
| 197 |
+
_fail(f"This job's plan could not be decoded ({type(exc).__name__}); WEB_AGENT_PLAN must be "
|
| 198 |
+
f"base64 of a JSON object.", kind="bad_plan")
|
| 199 |
+
|
| 200 |
+
STEPS = PLAN.get("steps") or []
|
| 201 |
+
RUN_ID = str(PLAN.get("runId") or "")
|
| 202 |
+
if not isinstance(STEPS, list) or not STEPS:
|
| 203 |
+
_fail("This job's plan carries no steps, so there is nothing to browse.", kind="bad_plan")
|
| 204 |
+
if len(STEPS) > MAX_STEPS:
|
| 205 |
+
_fail(f"This job's plan carries {len(STEPS)} steps and the ceiling is {MAX_STEPS}. "
|
| 206 |
+
f"Split the flow, or raise MAX_STEPS in jobs/web_agent_job.py.", kind="too_many_steps")
|
| 207 |
+
say(f"PLAN runId={RUN_ID or '(none)'} steps={len(STEPS)} "
|
| 208 |
+
f"kinds={sorted({str(s.get('kind')) for s in STEPS})}")
|
| 209 |
+
|
| 210 |
+
prior = _resume(RUN_ID)
|
| 211 |
+
if prior:
|
| 212 |
+
# RESUME, not restart. The page was already fetched and the result already paid for.
|
| 213 |
+
say(f"RESUME runId={RUN_ID} β a completed checkpoint exists; returning it unchanged")
|
| 214 |
+
prior["resumedFromCheckpoint"] = True
|
| 215 |
+
print(RESULT_BEGIN, flush=True)
|
| 216 |
+
print(json.dumps(prior, ensure_ascii=False, separators=(",", ":")), flush=True)
|
| 217 |
+
print(RESULT_END, flush=True)
|
| 218 |
+
sys.exit(0)
|
| 219 |
+
|
| 220 |
+
#: ββ ALL FIVE WEB ACTIONS RUN β owner ruling 2026-08-12, superseding D-51/R5.
|
| 221 |
+
#: R5 (recorded at the 2026-08-06 grill) made any web action that WRITES to a third party default
|
| 222 |
+
#: to "needs approval". The owner has revoked it, verbatim: *"make sure we unblock all web
|
| 223 |
+
#: actions, we don't need approval step first wtf, I never ask for that."* So there is no approval
|
| 224 |
+
#: gate and no per-kind ban; a journey may open, fill, click, read and repair.
|
| 225 |
+
#: β WHAT REPLACED THE GATE IS NOT NOTHING, and none of it asks anybody for permission: a journey
|
| 226 |
+
#: HALTS at its first failed step rather than clicking on into an unknown page; `dryRun` rehearses
|
| 227 |
+
#: a write against a real page without performing it; `web_repair` PROPOSES a selector instead of
|
| 228 |
+
#: silently rebinding; and secret-looking values never reach a log, a result or a checkpoint.
|
| 229 |
+
RUNNABLE_KINDS = {"web_read", "web_goto", "web_fill", "web_click", "web_repair"}
|
| 230 |
+
bad = sorted({str(s.get("kind")) for s in STEPS} - RUNNABLE_KINDS)
|
| 231 |
+
if bad:
|
| 232 |
+
_fail(f"This runner implements {sorted(RUNNABLE_KINDS)} and the plan asks for {bad}.",
|
| 233 |
+
kind="unready_kind")
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
# ββ the browser βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 237 |
+
def _run(cmd: list[str], label: str) -> int:
|
| 238 |
+
started = time.monotonic()
|
| 239 |
+
say(f"RUN {label}: {' '.join(cmd[:4])}β¦")
|
| 240 |
+
proc = subprocess.run(cmd, capture_output=True, text=True)
|
| 241 |
+
if proc.returncode != 0:
|
| 242 |
+
tail = (proc.stderr or proc.stdout or "").strip().splitlines()[-3:]
|
| 243 |
+
say(f"RUN {label} exited {proc.returncode}: {' | '.join(tail)[:400]}")
|
| 244 |
+
phase(label, started)
|
| 245 |
+
return proc.returncode
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
_deps_started = time.monotonic()
|
| 249 |
+
try:
|
| 250 |
+
import playwright # noqa: F401
|
| 251 |
+
from playwright.sync_api import Error as PWError
|
| 252 |
+
from playwright.sync_api import TimeoutError as PWTimeout
|
| 253 |
+
from playwright.sync_api import sync_playwright
|
| 254 |
+
except Exception as exc: # noqa: BLE001
|
| 255 |
+
_fail(f"Playwright is not importable inside the job ({type(exc).__name__}: "
|
| 256 |
+
f"{str(exc)[:160]}). The dependency block at the top of this file is what installs it.",
|
| 257 |
+
kind="no_playwright")
|
| 258 |
+
phase("import_playwright", _deps_started)
|
| 259 |
+
|
| 260 |
+
if INSTALL_DEPS:
|
| 261 |
+
# This is the apt half, and it is the ATLAS pattern's job: `install-deps` shells out to
|
| 262 |
+
# apt-get for the shared libraries a headless Chromium links against.
|
| 263 |
+
_run([sys.executable, "-m", "playwright", "install-deps", "chromium"], "apt_install_deps")
|
| 264 |
+
if INSTALL_BROWSER:
|
| 265 |
+
rc = _run([sys.executable, "-m", "playwright", "install", "chromium"], "browser_download")
|
| 266 |
+
if rc != 0:
|
| 267 |
+
_fail("The browser could not be installed inside this job, so nothing was browsed. "
|
| 268 |
+
"See the RUN browser_download line above for the installer's own message.",
|
| 269 |
+
kind="no_browser")
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
#: Fields whose VALUE never reaches a log or a result. A journey that logs into a site carries a
|
| 273 |
+
#: password in `value`; echoing it back would put it in the run log, the checkpoint file and the
|
| 274 |
+
#: HF job's stdout, all of which outlive the run.
|
| 275 |
+
SECRET_HINTS = ("password", "passwd", "pwd", "secret", "token", "otp", "cvv", "pin")
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _is_secret(step: dict) -> bool:
|
| 279 |
+
if step.get("secret") is not None:
|
| 280 |
+
return bool(step["secret"])
|
| 281 |
+
blob = f"{step.get('selector', '')} {step.get('field', '')}".lower()
|
| 282 |
+
return any(h in blob for h in SECRET_HINTS)
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def _mask(value: str, secret: bool) -> str:
|
| 286 |
+
return ("β’" * min(len(value), 8)) if secret else value[:200]
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _goto(page, url: str, timeout_ms: int, out: dict) -> bool:
|
| 290 |
+
"""Navigate, recording status/title/finalUrl on `out`. False β `out['error']` is set."""
|
| 291 |
+
try:
|
| 292 |
+
resp = page.goto(url, timeout=timeout_ms, wait_until="domcontentloaded")
|
| 293 |
+
except PWTimeout:
|
| 294 |
+
out["error"] = (f"The page {url} did not load within {timeout_ms} ms. Raise the step's "
|
| 295 |
+
f"timeout, or the site is refusing us.")
|
| 296 |
+
return False
|
| 297 |
+
except PWError as exc:
|
| 298 |
+
out["error"] = f"The page {url} could not be opened: {str(exc).splitlines()[0][:200]}"
|
| 299 |
+
return False
|
| 300 |
+
out["status"] = getattr(resp, "status", None)
|
| 301 |
+
out["finalUrl"] = page.url
|
| 302 |
+
try:
|
| 303 |
+
out["title"] = page.title()
|
| 304 |
+
except PWError:
|
| 305 |
+
out["title"] = None
|
| 306 |
+
return True
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def _settle(page, step: dict, out: dict, timeout_ms: int) -> bool:
|
| 310 |
+
"""Optional `waitFor` after an action. False β `out['error']` is set."""
|
| 311 |
+
wait_for = str(step.get("waitFor") or "").strip()
|
| 312 |
+
if not wait_for:
|
| 313 |
+
return True
|
| 314 |
+
try:
|
| 315 |
+
page.wait_for_selector(wait_for, timeout=timeout_ms)
|
| 316 |
+
return True
|
| 317 |
+
except PWTimeout:
|
| 318 |
+
out["error"] = (f"The page never showed {wait_for!r} within {timeout_ms} ms after this "
|
| 319 |
+
f"step, so the journey stopped here rather than guessing.")
|
| 320 |
+
return False
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def _need(page, step: dict, out: dict, timeout_ms: int, verb: str):
|
| 324 |
+
"""The element a write step acts on, or None with `out['error']` set.
|
| 325 |
+
|
| 326 |
+
β `wait_for_selector` rather than `query_selector`: a write step usually follows a click or a
|
| 327 |
+
navigation, and the element it wants may be a few hundred ms away. Querying once turns an
|
| 328 |
+
ordinary render delay into "the selector is wrong", which sends somebody to fix a selector
|
| 329 |
+
that was always correct.
|
| 330 |
+
"""
|
| 331 |
+
selector = str(step.get("selector") or "").strip()
|
| 332 |
+
if not selector:
|
| 333 |
+
out["error"] = f"This step says to {verb} but names no element (no selector)."
|
| 334 |
+
return None
|
| 335 |
+
try:
|
| 336 |
+
handle = page.wait_for_selector(selector, timeout=timeout_ms, state="visible")
|
| 337 |
+
except PWTimeout:
|
| 338 |
+
out["error"] = (f"Nothing on {page.url} matched {selector!r} within {timeout_ms} ms, so "
|
| 339 |
+
f"there was nothing to {verb}. The page is loaded (title "
|
| 340 |
+
f"{out.get('title') or _title(page)!r}) β the selector is the thing to "
|
| 341 |
+
f"correct, and `web_repair` can propose one.")
|
| 342 |
+
return None
|
| 343 |
+
except PWError as exc:
|
| 344 |
+
out["error"] = (f"The selector {selector!r} is not usable here: "
|
| 345 |
+
f"{str(exc).splitlines()[0][:200]}")
|
| 346 |
+
return None
|
| 347 |
+
return handle
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def _title(page):
|
| 351 |
+
try:
|
| 352 |
+
return page.title()
|
| 353 |
+
except PWError:
|
| 354 |
+
return None
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def _do_read(page, step, out, timeout_ms) -> dict:
|
| 358 |
+
"""`web_read` β pull a value off the page. The only kind that returns page CONTENT."""
|
| 359 |
+
selector = str(step.get("selector") or "").strip()
|
| 360 |
+
attr, want_all = (step.get("attr") or "text"), bool(step.get("all"))
|
| 361 |
+
if not selector:
|
| 362 |
+
out["error"] = "This step says to read but names no element (no selector)."
|
| 363 |
+
return out
|
| 364 |
+
try:
|
| 365 |
+
handles = page.query_selector_all(selector) if want_all else (
|
| 366 |
+
[page.query_selector(selector)] if page.query_selector(selector) else [])
|
| 367 |
+
except PWError as exc:
|
| 368 |
+
out["error"] = (f"The selector {selector!r} is not usable on {page.url}: "
|
| 369 |
+
f"{str(exc).splitlines()[0][:200]}")
|
| 370 |
+
return out
|
| 371 |
+
if not handles:
|
| 372 |
+
# β NOT AN EMPTY SUCCESS. "The selector matched nothing" is the single most likely real
|
| 373 |
+
# outcome and it has to arrive as a sentence, or a broken automation writes blanks
|
| 374 |
+
# forever and looks healthy.
|
| 375 |
+
out["matched"] = 0
|
| 376 |
+
out["error"] = (f"Nothing on {page.url} matched the selector {selector!r}. The page "
|
| 377 |
+
f"loaded (status {out.get('status')}, title "
|
| 378 |
+
f"{out.get('title') or _title(page)!r}), so the selector is the thing to "
|
| 379 |
+
f"correct β `web_repair` can propose one.")
|
| 380 |
+
return out
|
| 381 |
+
values = []
|
| 382 |
+
for handle in handles:
|
| 383 |
+
try:
|
| 384 |
+
values.append(handle.inner_text() if attr == "text"
|
| 385 |
+
else (handle.get_attribute(str(attr)) or ""))
|
| 386 |
+
except PWError:
|
| 387 |
+
values.append("")
|
| 388 |
+
values = [" ".join(str(v).split()) for v in values]
|
| 389 |
+
joined = "\n".join(values) if want_all else (values[0] if values else "")
|
| 390 |
+
out["matched"] = len(handles)
|
| 391 |
+
out["truncated"] = len(joined) > MAX_VALUE_CHARS
|
| 392 |
+
out["value"] = joined[:MAX_VALUE_CHARS]
|
| 393 |
+
out["ok"] = True
|
| 394 |
+
return out
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def _do_fill(page, step, out, timeout_ms) -> dict:
|
| 398 |
+
"""`web_fill` β type a value into a field."""
|
| 399 |
+
secret = _is_secret(step)
|
| 400 |
+
value = str(step.get("value") if step.get("value") is not None else "")
|
| 401 |
+
handle = _need(page, step, out, timeout_ms, "fill in")
|
| 402 |
+
if handle is None:
|
| 403 |
+
return out
|
| 404 |
+
if step.get("dryRun"):
|
| 405 |
+
# Not an approval gate β a TESTING affordance. It resolves the element and reports what
|
| 406 |
+
# WOULD be typed, so a journey can be rehearsed against a real page before it writes.
|
| 407 |
+
out["ok"], out["dryRun"] = True, True
|
| 408 |
+
out["value"] = _mask(value, secret)
|
| 409 |
+
out["wouldFill"] = _describe(handle)
|
| 410 |
+
return out
|
| 411 |
+
try:
|
| 412 |
+
handle.fill(value, timeout=timeout_ms)
|
| 413 |
+
except PWError as exc:
|
| 414 |
+
out["error"] = (f"The field {step.get('selector')!r} would not accept a value: "
|
| 415 |
+
f"{str(exc).splitlines()[0][:200]}")
|
| 416 |
+
return out
|
| 417 |
+
out["ok"], out["filled"] = True, _describe(handle)
|
| 418 |
+
out["value"] = _mask(value, secret)
|
| 419 |
+
out["secret"] = secret
|
| 420 |
+
return out
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
def _do_click(page, step, out, timeout_ms) -> dict:
|
| 424 |
+
"""`web_click` β click something, then let any navigation it triggers settle."""
|
| 425 |
+
handle = _need(page, step, out, timeout_ms, "click")
|
| 426 |
+
if handle is None:
|
| 427 |
+
return out
|
| 428 |
+
before = page.url
|
| 429 |
+
if step.get("dryRun"):
|
| 430 |
+
out["ok"], out["dryRun"] = True, True
|
| 431 |
+
out["wouldClick"] = _describe(handle)
|
| 432 |
+
return out
|
| 433 |
+
try:
|
| 434 |
+
handle.click(timeout=timeout_ms)
|
| 435 |
+
except PWError as exc:
|
| 436 |
+
out["error"] = (f"The element {step.get('selector')!r} could not be clicked: "
|
| 437 |
+
f"{str(exc).splitlines()[0][:200]}")
|
| 438 |
+
return out
|
| 439 |
+
# A click that navigates needs the new document before the next step reads it. `networkidle`
|
| 440 |
+
# is deliberately NOT used: it never fires on pages with polling, and the failure looks like
|
| 441 |
+
# a hung click rather than a chatty page.
|
| 442 |
+
try:
|
| 443 |
+
page.wait_for_load_state("domcontentloaded", timeout=timeout_ms)
|
| 444 |
+
except PWTimeout:
|
| 445 |
+
pass # a click that changes nothing is legitimate
|
| 446 |
+
out["ok"] = True
|
| 447 |
+
out["clicked"] = _describe(handle)
|
| 448 |
+
out["finalUrl"], out["navigated"] = page.url, (page.url != before)
|
| 449 |
+
out["title"] = _title(page)
|
| 450 |
+
out["value"] = out["title"] or page.url # see `_value_for` β one key for every kind
|
| 451 |
+
return out
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def _step_note(res: dict) -> str:
|
| 455 |
+
"""The one-line success summary a run log shows. Never a secret, never a whole page."""
|
| 456 |
+
for key in ("matched", "clicked", "filled", "proposed", "wouldClick", "wouldFill"):
|
| 457 |
+
if res.get(key) is not None:
|
| 458 |
+
return f"{key}={res[key]}"
|
| 459 |
+
return str(res.get("finalUrl") or res.get("title") or "")[:120]
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
def _describe(handle) -> str:
|
| 463 |
+
"""A short human description of an element, for the run log. Never its value."""
|
| 464 |
+
try:
|
| 465 |
+
tag = (handle.evaluate("e => e.tagName") or "?").lower()
|
| 466 |
+
name = handle.get_attribute("name") or handle.get_attribute("id") or ""
|
| 467 |
+
text = " ".join((handle.inner_text() or "").split())[:60]
|
| 468 |
+
return f"<{tag}{' ' + name if name else ''}>{(' ' + text) if text else ''}".strip()
|
| 469 |
+
except PWError:
|
| 470 |
+
return "<element>"
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
#: Attributes worth trying, in the order a human would: an id is stable, a name is stable, the
|
| 474 |
+
#: rest are increasingly cosmetic.
|
| 475 |
+
_REPAIR_ATTRS = ("id", "name", "data-testid", "aria-label", "placeholder", "value", "title")
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
def _do_repair(page, step, out, timeout_ms) -> dict:
|
| 479 |
+
"""`web_repair` β a selector stopped matching; PROPOSE one that does.
|
| 480 |
+
|
| 481 |
+
β IT PROPOSES AND DOES NOT SILENTLY SUBSTITUTE. A self-healing selector that quietly rebinds
|
| 482 |
+
to "whatever looks similar" is how an automation starts clicking the wrong button on a
|
| 483 |
+
redesigned page and reports success for weeks. The repaired selector comes back as a VALUE,
|
| 484 |
+
with the evidence it was chosen on, so a person (or a later step that explicitly passes
|
| 485 |
+
`apply`) decides. This is D-51/R1's self-healing idea at the only altitude that is safe
|
| 486 |
+
without Scrapling's full machinery.
|
| 487 |
+
"""
|
| 488 |
+
broken = str(step.get("selector") or "").strip()
|
| 489 |
+
hint = str(step.get("hint") or "").strip()
|
| 490 |
+
out["brokenSelector"] = broken
|
| 491 |
+
if not broken:
|
| 492 |
+
out["error"] = "This step says to repair a selector but names none."
|
| 493 |
+
return out
|
| 494 |
+
try:
|
| 495 |
+
if page.query_selector(broken) is not None:
|
| 496 |
+
out["ok"], out["repaired"] = True, False
|
| 497 |
+
out["proposed"] = out["value"] = broken
|
| 498 |
+
out["note"] = (f"{broken!r} still matches on {page.url}, so nothing needed repairing.")
|
| 499 |
+
return out
|
| 500 |
+
except PWError:
|
| 501 |
+
pass # an unparseable selector is exactly the case
|
| 502 |
+
if not hint:
|
| 503 |
+
out["error"] = (f"{broken!r} matches nothing on {page.url}, and no `hint` was given to "
|
| 504 |
+
f"search by. Give the visible text or the field name to look for.")
|
| 505 |
+
return out
|
| 506 |
+
|
| 507 |
+
# Search the live DOM for something the hint plausibly names, and build a STABLE selector for
|
| 508 |
+
# it. Runs in the page so one round trip does the whole scan.
|
| 509 |
+
try:
|
| 510 |
+
found = page.evaluate(
|
| 511 |
+
"""([hint, attrs]) => {
|
| 512 |
+
const want = hint.toLowerCase();
|
| 513 |
+
// β A LABEL IS NOT THE THING YOU ACT ON. The first version scored on text alone
|
| 514 |
+
// and, asked to repair a username field, proposed the <div> whose text contains
|
| 515 |
+
// "Username" β an exact match, perfectly useless: you cannot type into a div.
|
| 516 |
+
// So: follow a <label> to the control it names, and prefer elements you can
|
| 517 |
+
// actually act on over containers that merely CONTAIN the words.
|
| 518 |
+
const ACTABLE = new Set(['INPUT','SELECT','TEXTAREA','BUTTON','A','OPTION']);
|
| 519 |
+
const resolve = (el) => {
|
| 520 |
+
if (el.tagName === 'LABEL') {
|
| 521 |
+
const f = el.getAttribute('for');
|
| 522 |
+
const target = (f && document.getElementById(f)) || el.control ||
|
| 523 |
+
el.querySelector('input,select,textarea');
|
| 524 |
+
if (target) return target;
|
| 525 |
+
}
|
| 526 |
+
return el;
|
| 527 |
+
};
|
| 528 |
+
const score = (el) => {
|
| 529 |
+
const own = (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA')
|
| 530 |
+
? (el.getAttribute('placeholder') || el.value || '')
|
| 531 |
+
: (el.innerText || '');
|
| 532 |
+
const t = own.trim().toLowerCase();
|
| 533 |
+
let s = 0;
|
| 534 |
+
if (t && t === want) s = 100;
|
| 535 |
+
else {
|
| 536 |
+
for (const a of attrs) {
|
| 537 |
+
const v = (el.getAttribute(a) || '').toLowerCase();
|
| 538 |
+
if (v && v === want) { s = 95; break; }
|
| 539 |
+
}
|
| 540 |
+
}
|
| 541 |
+
if (!s && t && t.includes(want)) {
|
| 542 |
+
s = 60 - Math.min(30, t.length - want.length);
|
| 543 |
+
}
|
| 544 |
+
if (!s) {
|
| 545 |
+
for (const a of attrs) {
|
| 546 |
+
const v = (el.getAttribute(a) || '').toLowerCase();
|
| 547 |
+
if (v && v.includes(want)) { s = 50; break; }
|
| 548 |
+
}
|
| 549 |
+
}
|
| 550 |
+
if (!s) return 0;
|
| 551 |
+
// A container that merely wraps the match is the weakest possible evidence.
|
| 552 |
+
const kids = el.querySelectorAll('*').length;
|
| 553 |
+
if (!ACTABLE.has(el.tagName) && el.tagName !== 'LABEL') s -= Math.min(40, kids * 4);
|
| 554 |
+
return s;
|
| 555 |
+
};
|
| 556 |
+
const sel = (el) => {
|
| 557 |
+
if (el.id) return '#' + CSS.escape(el.id);
|
| 558 |
+
const nm = el.getAttribute('name');
|
| 559 |
+
if (nm) return el.tagName.toLowerCase() + '[name="' + nm + '"]';
|
| 560 |
+
const tid = el.getAttribute('data-testid');
|
| 561 |
+
if (tid) return '[data-testid="' + tid + '"]';
|
| 562 |
+
const ty = el.getAttribute('type');
|
| 563 |
+
if (el.tagName === 'INPUT' && ty) return 'input[type="' + ty + '"]';
|
| 564 |
+
const cls = (el.className || '').split(/\\s+/).filter(Boolean)[0];
|
| 565 |
+
return el.tagName.toLowerCase() + (cls ? '.' + CSS.escape(cls) : '');
|
| 566 |
+
};
|
| 567 |
+
let best = null;
|
| 568 |
+
for (const raw of document.querySelectorAll(
|
| 569 |
+
'a,button,input,select,textarea,[role=button],label,h1,h2,h3,td,th,span,div')) {
|
| 570 |
+
const s = score(raw);
|
| 571 |
+
if (s <= 0) continue;
|
| 572 |
+
const el = resolve(raw);
|
| 573 |
+
// Following a label to its input is a PROMOTION, not a tie: the whole point
|
| 574 |
+
// is to land on the actionable element.
|
| 575 |
+
const bonus = ACTABLE.has(el.tagName) ? 25 : 0;
|
| 576 |
+
const total = s + bonus;
|
| 577 |
+
if (!best || total > best.score) {
|
| 578 |
+
best = {score: total, selector: sel(el), tag: el.tagName.toLowerCase(),
|
| 579 |
+
via: (el === raw ? '' : raw.tagName.toLowerCase()),
|
| 580 |
+
text: ((el.innerText || el.value ||
|
| 581 |
+
el.getAttribute('name') || '')).trim().slice(0, 80)};
|
| 582 |
+
}
|
| 583 |
+
}
|
| 584 |
+
return best;
|
| 585 |
+
}""", [hint, list(_REPAIR_ATTRS)])
|
| 586 |
+
except PWError as exc:
|
| 587 |
+
out["error"] = f"The page could not be searched for a replacement: {str(exc)[:160]}"
|
| 588 |
+
return out
|
| 589 |
+
|
| 590 |
+
if not found:
|
| 591 |
+
out["error"] = (f"{broken!r} matches nothing on {page.url}, and nothing there matches the "
|
| 592 |
+
f"hint {hint!r} either. The page may have changed beyond repair, or the "
|
| 593 |
+
f"journey is on the wrong page.")
|
| 594 |
+
return out
|
| 595 |
+
# Confirm the proposal actually resolves β proposing a selector that does not match would be
|
| 596 |
+
# a repair that breaks on its own terms.
|
| 597 |
+
try:
|
| 598 |
+
confirmed = page.query_selector(found["selector"]) is not None
|
| 599 |
+
except PWError:
|
| 600 |
+
confirmed = False
|
| 601 |
+
if not confirmed:
|
| 602 |
+
out["error"] = (f"A replacement for {broken!r} was found ({found['selector']!r}) but it "
|
| 603 |
+
f"does not resolve on re-query, so it is not offered.")
|
| 604 |
+
return out
|
| 605 |
+
out["ok"], out["repaired"] = True, True
|
| 606 |
+
out["proposed"] = out["value"] = found["selector"] # see `_value_for`
|
| 607 |
+
out["confidence"] = found["score"]
|
| 608 |
+
out["evidence"] = (f"<{found['tag']}> {found['text']}"
|
| 609 |
+
+ (f" (found via its <{found['via']}>)" if found.get("via") else ""))
|
| 610 |
+
out["note"] = (f"{broken!r} no longer matches; {found['selector']!r} does, and it is the best "
|
| 611 |
+
f"match for {hint!r}. Nothing was changed β update the step to adopt it.")
|
| 612 |
+
return out
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
#: kind -> the function that performs it. `web_goto` needs no body: navigation is shared setup.
|
| 616 |
+
_ACTIONS = {"web_read": _do_read, "web_fill": _do_fill, "web_click": _do_click,
|
| 617 |
+
"web_repair": _do_repair, "web_goto": None}
|
| 618 |
+
|
| 619 |
+
#: β EVERY KIND SETS `value`, AND THAT IS A DELIBERATE CONTRACT RATHER THAN A CONVENIENCE.
|
| 620 |
+
#: The automation engine writes `result["value"]` into a record's cell through ONE arm. If each
|
| 621 |
+
#: kind returned its answer under its own key (`proposed`, `finalUrl`, `title`, β¦) that arm would
|
| 622 |
+
#: grow a per-kind switch β a second copy of this table, in another lane's file, that goes stale
|
| 623 |
+
#: the day a sixth kind lands. So the runner normalises: the natural answer per kind is ALSO
|
| 624 |
+
#: `value`, and the specific keys stay for readers who want them.
|
| 625 |
+
#: web_read the text or attribute that was read
|
| 626 |
+
#: web_goto the page title it landed on (else the URL)
|
| 627 |
+
#: web_click the title after the click (else the URL) β what the click led to
|
| 628 |
+
#: web_fill the value typed, MASKED when the field looks secret
|
| 629 |
+
#: web_repair the PROPOSED selector β the thing a person would paste back into the step
|
| 630 |
+
VALUE_BEARING_KINDS = tuple(sorted(_ACTIONS))
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
def _run_step(page, step: dict) -> dict:
|
| 634 |
+
"""Perform ONE step against the shared page.
|
| 635 |
+
|
| 636 |
+
β THE PAGE IS SHARED ON PURPOSE β it is what makes a JOURNEY possible: open a page, fill two
|
| 637 |
+
fields, click a button, read the result. Each step MAY carry its own `url`, which navigates
|
| 638 |
+
first; omitting it continues wherever the previous step left off. That single rule makes a
|
| 639 |
+
one-step call and a six-step journey the same code path.
|
| 640 |
+
"""
|
| 641 |
+
kind = str(step.get("kind") or "")
|
| 642 |
+
sid = str(step.get("id") or "")
|
| 643 |
+
url = str(step.get("url") or "").strip()
|
| 644 |
+
timeout_ms = int(step.get("timeoutMs") or 20000)
|
| 645 |
+
out = {"id": sid, "kind": kind, "ok": False}
|
| 646 |
+
if step.get("selector"):
|
| 647 |
+
out["selector"] = step["selector"]
|
| 648 |
+
started = time.monotonic()
|
| 649 |
+
|
| 650 |
+
if kind not in _ACTIONS:
|
| 651 |
+
out["error"] = (f"This runner does not implement {kind!r}. Implemented: "
|
| 652 |
+
f"{sorted(_ACTIONS)}.")
|
| 653 |
+
return out
|
| 654 |
+
if url:
|
| 655 |
+
out["url"] = url
|
| 656 |
+
if not url.lower().startswith(("http://", "https://")):
|
| 657 |
+
out["error"] = (f"Step {sid or '?'} has a URL that is not http(s) ({url!r}), so "
|
| 658 |
+
f"nothing was opened.")
|
| 659 |
+
return out
|
| 660 |
+
if not _goto(page, url, timeout_ms, out):
|
| 661 |
+
return out
|
| 662 |
+
elif page.url in ("about:blank", ""):
|
| 663 |
+
out["error"] = (f"Step {sid or '?'} ({kind}) continues from the previous page, but "
|
| 664 |
+
f"nothing has been opened yet. Give this step a `url`, or put a "
|
| 665 |
+
f"`web_goto` before it.")
|
| 666 |
+
return out
|
| 667 |
+
else:
|
| 668 |
+
out["url"] = page.url
|
| 669 |
+
|
| 670 |
+
if not _settle(page, step, out, timeout_ms):
|
| 671 |
+
return out
|
| 672 |
+
|
| 673 |
+
if kind == "web_goto":
|
| 674 |
+
# Navigation IS the whole action. Its value is the page it landed on, which the run log
|
| 675 |
+
# shows and a following step reads.
|
| 676 |
+
out["ok"] = True
|
| 677 |
+
out["finalUrl"] = page.url
|
| 678 |
+
out.setdefault("title", _title(page))
|
| 679 |
+
out["value"] = out.get("title") or page.url
|
| 680 |
+
else:
|
| 681 |
+
out = _ACTIONS[kind](page, step, out, timeout_ms)
|
| 682 |
+
|
| 683 |
+
out["elapsedMs"] = int((time.monotonic() - started) * 1000)
|
| 684 |
+
return out
|
| 685 |
+
|
| 686 |
+
|
| 687 |
+
results = []
|
| 688 |
+
_browse_started = time.monotonic()
|
| 689 |
+
try:
|
| 690 |
+
with sync_playwright() as pw:
|
| 691 |
+
launch_started = time.monotonic()
|
| 692 |
+
browser = pw.chromium.launch(args=["--no-sandbox", "--disable-dev-shm-usage"])
|
| 693 |
+
phase("browser_launch", launch_started)
|
| 694 |
+
say(f"BROWSER {browser.version}")
|
| 695 |
+
context = browser.new_context(
|
| 696 |
+
user_agent=("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like "
|
| 697 |
+
"Gecko) Chrome/125.0.0.0 Safari/537.36"),
|
| 698 |
+
viewport={"width": 1280, "height": 900})
|
| 699 |
+
page = context.new_page()
|
| 700 |
+
for idx, step in enumerate(STEPS, 1):
|
| 701 |
+
if _shutdown["asked"]:
|
| 702 |
+
results.append({"id": str(step.get("id") or ""), "kind": step.get("kind"),
|
| 703 |
+
"ok": False,
|
| 704 |
+
"error": "This job was stopped by the platform before the step "
|
| 705 |
+
"ran; nothing was read."})
|
| 706 |
+
continue
|
| 707 |
+
say(f"STEP {idx}/{len(STEPS)} {step.get('kind')} "
|
| 708 |
+
f"{str(step.get('url') or step.get('selector') or '')[:120]}")
|
| 709 |
+
res = _run_step(page, step)
|
| 710 |
+
say(f"STEP {idx}/{len(STEPS)} ok={res['ok']} "
|
| 711 |
+
f"{_step_note(res) if res['ok'] else res.get('error', '')[:180]}")
|
| 712 |
+
results.append(res)
|
| 713 |
+
# β A JOURNEY STOPS AT ITS FIRST FAILURE, and this is the difference between a list
|
| 714 |
+
# of independent reads and a SEQUENCE. Carrying on after a failed login would submit
|
| 715 |
+
# the next steps against whatever page we happened to be on β clicking real buttons
|
| 716 |
+
# on the wrong screen. The remaining steps are reported as not-attempted, so the run
|
| 717 |
+
# log distinguishes "did not run" from "ran and failed".
|
| 718 |
+
if not res["ok"] and len(STEPS) > 1:
|
| 719 |
+
for later in STEPS[idx:]:
|
| 720 |
+
results.append({"id": str(later.get("id") or ""), "kind": later.get("kind"),
|
| 721 |
+
"ok": False, "attempted": False,
|
| 722 |
+
"error": f"Not attempted: step {idx} "
|
| 723 |
+
f"({step.get('kind')}) failed first."})
|
| 724 |
+
say(f"JOURNEY halted at step {idx}/{len(STEPS)}; "
|
| 725 |
+
f"{len(STEPS) - idx} later step(s) not attempted")
|
| 726 |
+
break
|
| 727 |
+
browser.close()
|
| 728 |
+
except Exception as exc: # noqa: BLE001 β a boundary
|
| 729 |
+
phase("browse", _browse_started)
|
| 730 |
+
_fail(f"The browser itself failed inside the job ({type(exc).__name__}: "
|
| 731 |
+
f"{str(exc).splitlines()[0][:200]}).", kind="browser_crashed", steps=results)
|
| 732 |
+
phase("browse", _browse_started)
|
| 733 |
+
|
| 734 |
+
_emit({"ok": all(r.get("ok") for r in results), "runId": RUN_ID, "steps": results,
|
| 735 |
+
"phases": PHASES, "browserInstalled": INSTALL_BROWSER})
|
| 736 |
+
say(f"DONE ok={all(r.get('ok') for r in results)} steps={len(results)}")
|
platform/core/keychain.py
CHANGED
|
@@ -33,9 +33,27 @@ KEY = 'keychain'
|
|
| 33 |
#: consent screen and no app review β the three things that make every other provider on the
|
| 34 |
#: standing queue a multi-week errand. A type here is what lets the connectors directory show
|
| 35 |
#: them as CONNECTABLE rather than as another faded to-do.
|
| 36 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
#: which stored field feeds the masked preview, per type (first present wins)
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
MAX_ENTRIES = 40
|
| 40 |
MAX_FIELD_LEN = 500
|
| 41 |
|
|
@@ -151,8 +169,32 @@ def read_fields(rt, entry_id):
|
|
| 151 |
def odoo_creds(rt):
|
| 152 |
"""R3's resolver seam: the FIRST odoo-type entry's fields, else None (the caller falls
|
| 153 |
back to the environment). Deterministic order = insertion-id sort, so 'first' is stable."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
for row in list_entries(rt):
|
| 155 |
-
if row['type'] ==
|
| 156 |
return read_fields(rt, row['id'])
|
| 157 |
return None
|
| 158 |
|
|
@@ -235,6 +277,39 @@ def drop_user_secret(rt, username, provider):
|
|
| 235 |
return True
|
| 236 |
|
| 237 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
def test_entry(rt, entry_id, timeout=8):
|
| 239 |
"""A cheap, READ-ONLY liveness probe. odoo: xmlrpc `common.version()` (no auth, no data);
|
| 240 |
generic: shape-only. Always answers {ok, message} β an exception here is a result, not a
|
|
@@ -244,6 +319,18 @@ def test_entry(rt, entry_id, timeout=8):
|
|
| 244 |
if not isinstance(e, dict):
|
| 245 |
return {'ok': False, 'message': 'no such entry'}
|
| 246 |
fields = read_fields(rt, entry_id) or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
if e.get('type') != 'odoo':
|
| 248 |
return {'ok': bool(fields), 'message': f'{len(fields)} field(s) stored'}
|
| 249 |
url = str(fields.get('url') or '').rstrip('/')
|
|
|
|
| 33 |
#: consent screen and no app review β the three things that make every other provider on the
|
| 34 |
#: standing queue a multi-week errand. A type here is what lets the connectors directory show
|
| 35 |
#: them as CONNECTABLE rather than as another faded to-do.
|
| 36 |
+
#: ββ WAVE 31 (R2/R5) β `meta_ads` JOINS, AND IT IS THE FIRST END-TO-END KEYCHAINβHTTP CONNECTOR.
|
| 37 |
+
#: Every type above it is either read by nothing (`generic`, `stripe`, `shopify` store a key that
|
| 38 |
+
#: no data path consumes yet) or read by ONE resolver that talks XML-RPC (`odoo`). So `meta_ads`
|
| 39 |
+
#: is the first entry whose stored secret actually reaches an HTTP API in production, through
|
| 40 |
+
#: `meta_creds` below β which is why W31-T48's `how:` says *build the seam, do not assume it*.
|
| 41 |
+
#: β A TYPE MISSING FROM THIS SET 400s IN `add_entry`, which is the entire failure: the Connectors
|
| 42 |
+
#: directory would offer Meta Ads, an admin would paste a token, and the write would be refused
|
| 43 |
+
#: with "type must be one of odoo, generic, stripe, shopify" β a message about our data structure,
|
| 44 |
+
#: on a screen about their advertising account.
|
| 45 |
+
#: β R8 keeps the two doors separate on purpose: this is the TENANT's own token (GTM Lab pastes
|
| 46 |
+
#: theirs), while R5's "Connect Facebook" is the delegated-OAuth door Nango owns. Neither replaces
|
| 47 |
+
#: the other, and the schema work is blocked on neither.
|
| 48 |
+
ENTRY_TYPES = {'odoo', 'generic', 'stripe', 'shopify', 'meta_ads'}
|
| 49 |
#: which stored field feeds the masked preview, per type (first present wins)
|
| 50 |
+
#: β `access_token` JOINED IN WAVE 31, AND ITS ABSENCE WAS A REAL DEFECT FOUND BY BUILDING THE
|
| 51 |
+
#: SEAM RATHER THAN BY READING IT. Meta's credential is conventionally an `access_token`, and with
|
| 52 |
+
#: only `token` on this list `add_entry` stored the secret correctly and computed an EMPTY preview
|
| 53 |
+
#: β so the keychain table would show a Meta row with no hint at all, and an admin holding two
|
| 54 |
+
#: tokens could not tell which one is stored or whether the paste worked. Nothing raises; the
|
| 55 |
+
#: entry is fine; the only symptom is a blank cell on the screen the feature exists for.
|
| 56 |
+
_PREVIEW_FIELDS = ('api_key', 'access_token', 'value', 'password', 'token')
|
| 57 |
MAX_ENTRIES = 40
|
| 58 |
MAX_FIELD_LEN = 500
|
| 59 |
|
|
|
|
| 169 |
def odoo_creds(rt):
|
| 170 |
"""R3's resolver seam: the FIRST odoo-type entry's fields, else None (the caller falls
|
| 171 |
back to the environment). Deterministic order = insertion-id sort, so 'first' is stable."""
|
| 172 |
+
return _first_creds(rt, 'odoo')
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def meta_creds(rt):
|
| 176 |
+
"""W31 (R2): the FIRST `meta_ads` entry's fields, else None β the Meta connector's resolver.
|
| 177 |
+
|
| 178 |
+
β IT IS DELIBERATELY THE SAME SHAPE AS `odoo_creds` AND SHARES ITS IMPLEMENTATION, because
|
| 179 |
+
"which stored credential serves this connector?" is ONE question and this module was one
|
| 180 |
+
copy-paste away from having two answers to it ([[one-question-two-normalizers]]). The
|
| 181 |
+
difference between the two connectors belongs in the CONNECTOR, not in the lookup.
|
| 182 |
+
|
| 183 |
+
β AND IT FAILS CLOSED WITH NO ENVIRONMENT FALLBACK, which is where it deliberately DIFFERS
|
| 184 |
+
from `odoo_creds`'s caller. `harness/runtime.py::odoo_source` falls back to the process
|
| 185 |
+
environment for tenant #0 β a documented exception for the tenant whose `.env` this is. There
|
| 186 |
+
is no equivalent for Meta: `META_ADS_ACCESS_TOKEN` is the OWNER's measurement token (R8), and
|
| 187 |
+
handing it to a tenant whose admin has not stored one is precisely the leak the R3 rule
|
| 188 |
+
exists to prevent. A tenant with no entry gets None, and the connector says so.
|
| 189 |
+
"""
|
| 190 |
+
return _first_creds(rt, 'meta_ads')
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _first_creds(rt, etype):
|
| 194 |
+
"""The first entry of `etype`, decrypted β or None. Deterministic order = insertion-id sort,
|
| 195 |
+
so "first" is stable across reads rather than dict-order luck."""
|
| 196 |
for row in list_entries(rt):
|
| 197 |
+
if row['type'] == etype:
|
| 198 |
return read_fields(rt, row['id'])
|
| 199 |
return None
|
| 200 |
|
|
|
|
| 277 |
return True
|
| 278 |
|
| 279 |
|
| 280 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 281 |
+
# ββ W31 β THE LIVENESS-PROBE SLOT, AND WHY IT IS A SLOT AND NOT AN IMPORT.
|
| 282 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 283 |
+
#
|
| 284 |
+
# `test_entry` below wants to ASK each connector's API whether a stored credential works. The
|
| 285 |
+
# knowledge of HOW to ask lives in the connector modules β and those live in the API layer, which
|
| 286 |
+
# imports `core`, never the other way round (ARCHITECTURE.md's layering rule, restated in
|
| 287 |
+
# CLAUDE.md: *"core never imports up"*). A first draft of the Meta arm did the tempting thing β
|
| 288 |
+
# `sys.path.insert` into `aios-web/api` and `import connectors_meta` from inside `core` β and that
|
| 289 |
+
# is an architecture inversion with no gate on it, i.e. the kind that survives.
|
| 290 |
+
#
|
| 291 |
+
# β THE FIX IS THIS REPO'S OWN IDIOM, NOT A NEW ONE. `harness/datastore.py` has exactly this
|
| 292 |
+
# problem with the connector-pause flag and solves it with `set_paused_probe(fn)`: the lower layer
|
| 293 |
+
# exposes a SLOT and the upper layer fills it at import, with the comment *"an import back would
|
| 294 |
+
# be a cycle"*. Same shape, same reason. `verify_meta` asserts `core/` imports nothing from the
|
| 295 |
+
# API layer, so the inversion cannot come back quietly.
|
| 296 |
+
#
|
| 297 |
+
#: entry type -> `fn(fields: dict, timeout: int) -> {ok, message}`. Filled by the connector layer.
|
| 298 |
+
_PROBES = {}
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def register_prober(etype, fn):
|
| 302 |
+
"""Let the connector layer answer `test_entry` for ONE entry type.
|
| 303 |
+
|
| 304 |
+
β Additive and idempotent by design β a re-import must not double-register or raise, because
|
| 305 |
+
module import order is not something a caller controls.
|
| 306 |
+
"""
|
| 307 |
+
etype = str(etype or '').strip().lower()
|
| 308 |
+
if etype and callable(fn):
|
| 309 |
+
_PROBES[etype] = fn
|
| 310 |
+
return etype in _PROBES
|
| 311 |
+
|
| 312 |
+
|
| 313 |
def test_entry(rt, entry_id, timeout=8):
|
| 314 |
"""A cheap, READ-ONLY liveness probe. odoo: xmlrpc `common.version()` (no auth, no data);
|
| 315 |
generic: shape-only. Always answers {ok, message} β an exception here is a result, not a
|
|
|
|
| 319 |
if not isinstance(e, dict):
|
| 320 |
return {'ok': False, 'message': 'no such entry'}
|
| 321 |
fields = read_fields(rt, entry_id) or {}
|
| 322 |
+
# β W31 β A REGISTERED PROBER ANSWERS FOR ITS OWN TYPE, if the connector layer filled the
|
| 323 |
+
# slot. A shape check says "you pasted something", which is the least useful thing an
|
| 324 |
+
# admin can be told about a credential: a revoked or wrong-scoped token is the same SHAPE
|
| 325 |
+
# as a working one, so a connector that can ASK its API should.
|
| 326 |
+
prober = _PROBES.get(str(e.get('type') or ''))
|
| 327 |
+
if prober is not None:
|
| 328 |
+
try:
|
| 329 |
+
got = prober(dict(fields), timeout)
|
| 330 |
+
return got if isinstance(got, dict) else {'ok': False,
|
| 331 |
+
'message': 'the prober said nothing'}
|
| 332 |
+
except Exception as ex: # noqa: BLE001
|
| 333 |
+
return {'ok': False, 'message': f'{type(ex).__name__}: {str(ex)[:160]}'}
|
| 334 |
if e.get('type') != 'odoo':
|
| 335 |
return {'ok': bool(fields), 'message': f'{len(fields)} field(s) stored'}
|
| 336 |
url = str(fields.get('url') or '').rstrip('/')
|
platform/core/store.py
CHANGED
|
@@ -1,561 +1,682 @@
|
|
| 1 |
-
"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo β the platform's ONLY writable store
|
| 2 |
-
(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
|
| 3 |
-
|
| 4 |
-
Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
|
| 5 |
-
repo is created private on first write. Reads use a short in-process cache that writes refresh, so
|
| 6 |
-
within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
|
| 7 |
-
|
| 8 |
-
WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
|
| 9 |
-
functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) β every existing
|
| 10 |
-
`import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
|
| 11 |
-
`for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
|
| 12 |
-
(nurilab, gtmlab, β¦); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
|
| 13 |
-
instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
|
| 14 |
-
write burst can never starve another's flush window.
|
| 15 |
-
"""
|
| 16 |
-
import io
|
| 17 |
-
import os
|
| 18 |
-
import json
|
| 19 |
-
import threading
|
| 20 |
-
import time as _time
|
| 21 |
-
from pathlib import Path
|
| 22 |
-
|
| 23 |
-
try:
|
| 24 |
-
from dotenv import load_dotenv
|
| 25 |
-
load_dotenv(Path(__file__).resolve().parents[1] / '.env')
|
| 26 |
-
except Exception:
|
| 27 |
-
pass
|
| 28 |
-
|
| 29 |
-
from huggingface_hub import HfApi, hf_hub_download
|
| 30 |
-
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
|
| 31 |
-
|
| 32 |
-
REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
|
| 33 |
-
_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
|
| 34 |
-
_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
|
| 35 |
-
|
| 36 |
-
#: WAVE 29 (item 20 / ruling R11, contract C6) β THE CHANGE TOKEN'S PROCESS STAMP.
|
| 37 |
-
#:
|
| 38 |
-
#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a
|
| 39 |
-
#: browser holding `β¦:4` from the previous process could read `β¦:4` from the new one and conclude
|
| 40 |
-
#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a
|
| 41 |
-
#: change, which is the truthful answer: the client's copy predates this process entirely.
|
| 42 |
-
#:
|
| 43 |
-
#: β NOT a clock and never compared for order β a token is opaque and compared ONLY for equality.
|
| 44 |
-
REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}'
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def _token():
|
| 48 |
-
return os.environ.get('HF_TOKEN') or None
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
class Store:
|
| 52 |
-
"""One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush.
|
| 53 |
-
|
| 54 |
-
Every method body is the pre-wave-18 module function, verbatim except globalsβfields.
|
| 55 |
-
The behaviour contracts that matter are stated on each method and are UNCHANGED:
|
| 56 |
-
strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads.
|
| 57 |
-
"""
|
| 58 |
-
|
| 59 |
-
def __init__(self, repo):
|
| 60 |
-
self.repo = str(repo)
|
| 61 |
-
self._lock = threading.RLock()
|
| 62 |
-
self._cache = {}
|
| 63 |
-
# --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------
|
| 64 |
-
# One typed character in a Notes cell used to cost THREE synchronous network calls
|
| 65 |
-
# inside the component round-trip. update(..., flush='async') applies the mutation to
|
| 66 |
-
# the in-process cache synchronously (read-your-writes: every reader consults the
|
| 67 |
-
# cache first) and a background worker coalesces uploads. The restart-wipe protection
|
| 68 |
-
# is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
|
| 69 |
-
# strict-reads. Known, accepted: with a second concurrent writer the stale window
|
| 70 |
-
# widens from ~one round-trip to ~the flush gap.
|
| 71 |
-
self._owned = {} # name -> True once cache reflects our own last accepted write
|
| 72 |
-
self._dirty = {} # name -> cache is ahead of the hub
|
| 73 |
-
self._flushers = {} # name -> live worker thread
|
| 74 |
-
self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
|
| 75 |
-
self._last_flush = {} # name -> monotonic time of the last completed upload
|
| 76 |
-
# --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------
|
| 77 |
-
# name -> how many accepted writes this process has applied to that bucket, and when.
|
| 78 |
-
# Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for
|
| 79 |
-
# why that constraint is the whole feature rather than an optimisation.
|
| 80 |
-
self._revs = {}
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
def
|
| 233 |
-
"""
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
"
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
if
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
self._dirty
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
#
|
| 437 |
-
#
|
| 438 |
-
#
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
#
|
| 445 |
-
#
|
| 446 |
-
#
|
| 447 |
-
#
|
| 448 |
-
#
|
| 449 |
-
#
|
| 450 |
-
#
|
| 451 |
-
#
|
| 452 |
-
# =
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo β the platform's ONLY writable store
|
| 2 |
+
(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
|
| 3 |
+
|
| 4 |
+
Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
|
| 5 |
+
repo is created private on first write. Reads use a short in-process cache that writes refresh, so
|
| 6 |
+
within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
|
| 7 |
+
|
| 8 |
+
WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
|
| 9 |
+
functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) β every existing
|
| 10 |
+
`import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
|
| 11 |
+
`for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
|
| 12 |
+
(nurilab, gtmlab, β¦); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
|
| 13 |
+
instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
|
| 14 |
+
write burst can never starve another's flush window.
|
| 15 |
+
"""
|
| 16 |
+
import io
|
| 17 |
+
import os
|
| 18 |
+
import json
|
| 19 |
+
import threading
|
| 20 |
+
import time as _time
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
from dotenv import load_dotenv
|
| 25 |
+
load_dotenv(Path(__file__).resolve().parents[1] / '.env')
|
| 26 |
+
except Exception:
|
| 27 |
+
pass
|
| 28 |
+
|
| 29 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 30 |
+
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
|
| 31 |
+
|
| 32 |
+
REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
|
| 33 |
+
_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
|
| 34 |
+
_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
|
| 35 |
+
|
| 36 |
+
#: WAVE 29 (item 20 / ruling R11, contract C6) β THE CHANGE TOKEN'S PROCESS STAMP.
|
| 37 |
+
#:
|
| 38 |
+
#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a
|
| 39 |
+
#: browser holding `β¦:4` from the previous process could read `β¦:4` from the new one and conclude
|
| 40 |
+
#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a
|
| 41 |
+
#: change, which is the truthful answer: the client's copy predates this process entirely.
|
| 42 |
+
#:
|
| 43 |
+
#: β NOT a clock and never compared for order β a token is opaque and compared ONLY for equality.
|
| 44 |
+
REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}'
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _token():
|
| 48 |
+
return os.environ.get('HF_TOKEN') or None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class Store:
|
| 52 |
+
"""One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush.
|
| 53 |
+
|
| 54 |
+
Every method body is the pre-wave-18 module function, verbatim except globalsβfields.
|
| 55 |
+
The behaviour contracts that matter are stated on each method and are UNCHANGED:
|
| 56 |
+
strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads.
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
def __init__(self, repo):
|
| 60 |
+
self.repo = str(repo)
|
| 61 |
+
self._lock = threading.RLock()
|
| 62 |
+
self._cache = {}
|
| 63 |
+
# --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------
|
| 64 |
+
# One typed character in a Notes cell used to cost THREE synchronous network calls
|
| 65 |
+
# inside the component round-trip. update(..., flush='async') applies the mutation to
|
| 66 |
+
# the in-process cache synchronously (read-your-writes: every reader consults the
|
| 67 |
+
# cache first) and a background worker coalesces uploads. The restart-wipe protection
|
| 68 |
+
# is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
|
| 69 |
+
# strict-reads. Known, accepted: with a second concurrent writer the stale window
|
| 70 |
+
# widens from ~one round-trip to ~the flush gap.
|
| 71 |
+
self._owned = {} # name -> True once cache reflects our own last accepted write
|
| 72 |
+
self._dirty = {} # name -> cache is ahead of the hub
|
| 73 |
+
self._flushers = {} # name -> live worker thread
|
| 74 |
+
self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
|
| 75 |
+
self._last_flush = {} # name -> monotonic time of the last completed upload
|
| 76 |
+
# --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------
|
| 77 |
+
# name -> how many accepted writes this process has applied to that bucket, and when.
|
| 78 |
+
# Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for
|
| 79 |
+
# why that constraint is the whole feature rather than an optimisation.
|
| 80 |
+
self._revs = {}
|
| 81 |
+
# --- ββ W31-T13 (D-177(d)) β THE PER-KEY WRITE LOCK ------------------------------
|
| 82 |
+
# `self._lock` guards the CACHE. It used to be held across a sync `update`'s whole
|
| 83 |
+
# read-modify-write β a hub download, the caller's `fn`, and a blocking hub upload β so
|
| 84 |
+
# one write froze every reader in the process for two network round trips. `get()` takes
|
| 85 |
+
# the same lock, and `/nav`, `/workspace` and every permission check are `get()`.
|
| 86 |
+
# β IT IS NOT SAFE TO SIMPLY DROP THE LOCK. Two sync writers of one key would interleave
|
| 87 |
+
# read-modify-write and one update would vanish β corruption, not slowness. So the two
|
| 88 |
+
# jobs are separated: this lock serialises WRITERS OF ONE KEY (held across the network,
|
| 89 |
+
# never touching readers), and `self._lock` is taken only for the microseconds a cache
|
| 90 |
+
# read or write needs. Readers never wait on a network call again.
|
| 91 |
+
# β PER KEY, not one write lock: two tenants' buckets, or `user_tables` and `nav_meta`,
|
| 92 |
+
# have no reason to serialise against each other.
|
| 93 |
+
self._key_locks = {}
|
| 94 |
+
self._key_locks_guard = threading.Lock()
|
| 95 |
+
self._rev_at = {}
|
| 96 |
+
|
| 97 |
+
def available(self):
|
| 98 |
+
"""True when a token is configured β callers degrade gracefully without one."""
|
| 99 |
+
return bool(_token())
|
| 100 |
+
|
| 101 |
+
def _download(self, name):
|
| 102 |
+
"""Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent.
|
| 103 |
+
Any other failure (network, auth, rate-limit, corrupt JSON) RAISES β a transient read
|
| 104 |
+
error must never be mistaken for an empty store, or a caller could overwrite real data
|
| 105 |
+
with an empty dict (this is what wiped the user registry on restart)."""
|
| 106 |
+
try:
|
| 107 |
+
path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset',
|
| 108 |
+
token=_token(), force_download=True)
|
| 109 |
+
with open(path, encoding='utf-8') as f:
|
| 110 |
+
return json.load(f)
|
| 111 |
+
except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
|
| 112 |
+
return {}
|
| 113 |
+
except ValueError as e:
|
| 114 |
+
# Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in
|
| 115 |
+
# ValueError("Force download failedβ¦") with the 404 as __cause__ β so a brand-new
|
| 116 |
+
# tenant repo's very first strict read raised here and aborted its very first write.
|
| 117 |
+
# A genuinely-absent repo is the same fact as a genuinely-absent file: {}.
|
| 118 |
+
cause = getattr(e, '__cause__', None)
|
| 119 |
+
if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)):
|
| 120 |
+
return {}
|
| 121 |
+
raise
|
| 122 |
+
|
| 123 |
+
def get(self, name, fresh=False):
|
| 124 |
+
"""Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error,
|
| 125 |
+
hand back the last known-good cached value if we have one, else {}. Mutating callers go
|
| 126 |
+
through update()/_read_strict, which RAISE instead."""
|
| 127 |
+
with self._lock:
|
| 128 |
+
if not fresh and name in self._cache:
|
| 129 |
+
return json.loads(json.dumps(self._cache[name])) # hand back a copy
|
| 130 |
+
try:
|
| 131 |
+
data = self._download(name)
|
| 132 |
+
except Exception as e:
|
| 133 |
+
try: # lazy import β core stays layer-clean
|
| 134 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 135 |
+
_tel.error(f'store:get:{name}', e,
|
| 136 |
+
fallback='cached' if name in self._cache else 'empty')
|
| 137 |
+
except Exception:
|
| 138 |
+
pass
|
| 139 |
+
if name in self._cache:
|
| 140 |
+
return json.loads(json.dumps(self._cache[name]))
|
| 141 |
+
return {}
|
| 142 |
+
self._cache[name] = data
|
| 143 |
+
return json.loads(json.dumps(data))
|
| 144 |
+
|
| 145 |
+
def _key_lock(self, name):
|
| 146 |
+
"""The write lock for ONE key (W31-T13). Created on demand; never removed."""
|
| 147 |
+
n = str(name)
|
| 148 |
+
with self._key_locks_guard:
|
| 149 |
+
lk = self._key_locks.get(n)
|
| 150 |
+
if lk is None:
|
| 151 |
+
lk = self._key_locks[n] = threading.RLock()
|
| 152 |
+
return lk
|
| 153 |
+
|
| 154 |
+
def _read_strict(self, name):
|
| 155 |
+
"""Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
|
| 156 |
+
absent file), so a failed read ABORTS a read-modify-write instead of merging into {}.
|
| 157 |
+
|
| 158 |
+
β W31-T13 β THE DOWNLOAD IS OUTSIDE `self._lock` NOW, and the re-check after it is the
|
| 159 |
+
whole of what makes that safe. A hub read takes a round trip; holding the cache lock
|
| 160 |
+
across it is what froze every reader. But moving it out opens a window in which an ASYNC
|
| 161 |
+
writer can accept a row into the cache while our download is in flight β and assigning our
|
| 162 |
+
(older) hub copy afterwards would erase it. That is D-177's original symptom exactly
|
| 163 |
+
(`POST /rows` β 201, row gone), re-created by the lock split rather than by a call site.
|
| 164 |
+
So the dirty mark is re-read AFTER the download, under the short lock, and a key that
|
| 165 |
+
became dirty keeps its cache.
|
| 166 |
+
"""
|
| 167 |
+
data = self._download(name) # network, no lock held
|
| 168 |
+
with self._lock:
|
| 169 |
+
if self._dirty.get(name) and self._owned.get(name) and name in self._cache:
|
| 170 |
+
# Our copy is stale by construction: a pending upload is going to overwrite the
|
| 171 |
+
# hub anyway, so the fresh read cannot preserve a remote write β it can only
|
| 172 |
+
# destroy a local one we have already acknowledged. Same argument, and the same
|
| 173 |
+
# three conditions, as `update`'s pending-async branch below.
|
| 174 |
+
return json.loads(json.dumps(self._cache[name]))
|
| 175 |
+
self._cache[name] = data
|
| 176 |
+
return json.loads(json.dumps(data))
|
| 177 |
+
|
| 178 |
+
def exists(self, name):
|
| 179 |
+
"""True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
|
| 180 |
+
uncertainty β a seeder must never clobber a registry we merely failed to reach."""
|
| 181 |
+
try:
|
| 182 |
+
files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset')
|
| 183 |
+
return f'{name}.json' in files
|
| 184 |
+
except RepositoryNotFoundError:
|
| 185 |
+
return False
|
| 186 |
+
except Exception:
|
| 187 |
+
return True
|
| 188 |
+
|
| 189 |
+
# ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6)
|
| 190 |
+
def _bump(self, name):
|
| 191 |
+
"""Advance one bucket's write counter. THE CALLER HOLDS THE LOCK.
|
| 192 |
+
|
| 193 |
+
Called from the two places an accepted write becomes visible to readers of this process
|
| 194 |
+
β `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT
|
| 195 |
+
from `_flush_worker`: that uploads data a bump already accounted for, and counting it
|
| 196 |
+
again would report a change to every browser each time the coalescing worker woke.
|
| 197 |
+
"""
|
| 198 |
+
n = str(name)
|
| 199 |
+
self._revs[n] = self._revs.get(n, 0) + 1
|
| 200 |
+
self._rev_at[n] = _time.time()
|
| 201 |
+
|
| 202 |
+
def revision(self, name):
|
| 203 |
+
"""`{'rev', 'updated_at', 'token'}` for ONE bucket β a dict read, nothing else.
|
| 204 |
+
|
| 205 |
+
β THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why
|
| 206 |
+
this method exists at all. A browser asking "has anything changed" by re-fetching rows
|
| 207 |
+
costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way
|
| 208 |
+
in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s
|
| 209 |
+
per bucket β so two tabs polling saturate the server and every unrelated write queues
|
| 210 |
+
behind them. This performs no download, no `get()`, and no copy of the value.
|
| 211 |
+
β Anything added here that reads a bucket destroys the whole point (`verify_live_workspace`
|
| 212 |
+
counts `get`/`_read_strict`/`_download` on the serving path and goes red on one).
|
| 213 |
+
|
| 214 |
+
`rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend
|
| 215 |
+
ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops
|
| 216 |
+
being inert the moment D-4 flips, with the wire shape unchanged.
|
| 217 |
+
|
| 218 |
+
β WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write
|
| 219 |
+
by a SECOND container against the same HF dataset is invisible here. That is not a limit
|
| 220 |
+
this introduces β `get()` caches a bucket for the life of the process and never re-reads
|
| 221 |
+
it, so the other container's write is invisible to every reader on this backend already.
|
| 222 |
+
The token is exactly as fresh as the store it reports on. Postgres has no such gap.
|
| 223 |
+
β Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted:
|
| 224 |
+
an attachment is fetched on explicit user action, never polled.
|
| 225 |
+
"""
|
| 226 |
+
n = str(name)
|
| 227 |
+
with self._lock:
|
| 228 |
+
rev = int(self._revs.get(n, 0))
|
| 229 |
+
at = self._rev_at.get(n)
|
| 230 |
+
return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'}
|
| 231 |
+
|
| 232 |
+
def _upload(self, name, data):
|
| 233 |
+
"""The raw hub write. create_repo is confirmed ONCE per instance β it was a full extra
|
| 234 |
+
API round-trip on every put(), pure latency after the first success."""
|
| 235 |
+
api = HfApi(token=_token())
|
| 236 |
+
if not self._repo_ok:
|
| 237 |
+
api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
|
| 238 |
+
self._repo_ok = True
|
| 239 |
+
buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
|
| 240 |
+
api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo,
|
| 241 |
+
repo_type='dataset', commit_message=f'update {name}')
|
| 242 |
+
|
| 243 |
+
def upload_bytes(self, path_in_repo, data, message=None):
|
| 244 |
+
"""Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
|
| 245 |
+
Bytes are never cached in-process: large, cold, fetched on explicit user action."""
|
| 246 |
+
if not self.available():
|
| 247 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 248 |
+
with self._lock:
|
| 249 |
+
api = HfApi(token=_token())
|
| 250 |
+
if not self._repo_ok:
|
| 251 |
+
api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
|
| 252 |
+
self._repo_ok = True
|
| 253 |
+
api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
|
| 254 |
+
repo_id=self.repo, repo_type='dataset',
|
| 255 |
+
commit_message=message or f'add {path_in_repo}')
|
| 256 |
+
|
| 257 |
+
def download_bytes(self, path_in_repo):
|
| 258 |
+
"""Raw bytes back, or None when absent/unreachable β a missing attachment degrades to
|
| 259 |
+
"that file is gone", never to a broken page."""
|
| 260 |
+
if not self.available():
|
| 261 |
+
return None
|
| 262 |
+
try:
|
| 263 |
+
local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset',
|
| 264 |
+
token=_token())
|
| 265 |
+
return Path(local).read_bytes()
|
| 266 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 267 |
+
return None
|
| 268 |
+
except Exception:
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
def delete_path(self, path_in_repo):
|
| 272 |
+
"""Remove a stored file. Absent is SUCCESS β deleting what is already gone is the goal."""
|
| 273 |
+
if not self.available():
|
| 274 |
+
return False
|
| 275 |
+
try:
|
| 276 |
+
HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo,
|
| 277 |
+
repo_type='dataset',
|
| 278 |
+
commit_message=f'delete {path_in_repo}')
|
| 279 |
+
return True
|
| 280 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 281 |
+
return True
|
| 282 |
+
except Exception:
|
| 283 |
+
return False
|
| 284 |
+
|
| 285 |
+
def put(self, name, data):
|
| 286 |
+
"""Persist {name}.json (creates the private repo on first write). Updates the cache."""
|
| 287 |
+
if not self.available():
|
| 288 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 289 |
+
with self._lock:
|
| 290 |
+
self._upload(name, data)
|
| 291 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 292 |
+
self._owned[name] = True
|
| 293 |
+
self._last_flush[name] = _time.monotonic()
|
| 294 |
+
self._bump(name) # C6 β AFTER the upload: a raise here is not a write
|
| 295 |
+
return self._cache[name]
|
| 296 |
+
|
| 297 |
+
def _flush_worker(self, name):
|
| 298 |
+
"""Coalescing uploader for one key. Runs while dirty; retries with backoff on failure β
|
| 299 |
+
the cache stays the truth and the dirty mark survives, so nothing is lost silently.
|
| 300 |
+
|
| 301 |
+
ββ W31-T13 β THIS WORKER IS THE SECOND WRITER, AND THE PER-KEY LOCK MUST COVER IT.
|
| 302 |
+
β THE RACE THE LOCK SPLIT OPENED, spelled out because it is silent and survives a restart
|
| 303 |
+
as real data loss. `update`'s big lock used to span `_read_strict` + `fn` + the upload, so
|
| 304 |
+
this worker could never snapshot mid-update. With the cache lock narrowed it can:
|
| 305 |
+
async write -> `_dirty=True`, this worker scheduled, sleeps `_FLUSH_DELAY`
|
| 306 |
+
sync update t+0.5 -> takes the dirty branch, runs `fn`, enters `_upload` (no cache
|
| 307 |
+
lock held β that is the whole point of the ticket)
|
| 308 |
+
worker t+2.0 -> `self._lock` is FREE and `_dirty` is still True (the sync commit
|
| 309 |
+
has not happened yet), so it snapshots the PRE-UPDATE cache,
|
| 310 |
+
clears the mark, and uploads it AFTER the sync upload lands
|
| 311 |
+
result -> cache has the new value, the hub has the old one, `_dirty` is
|
| 312 |
+
False, and nothing will ever re-upload. Divergence with no
|
| 313 |
+
symptom until the next process boots and reads the hub.
|
| 314 |
+
β `user_tables` has ~40 sync writers against 2 async ones, so that window is the COMMON
|
| 315 |
+
case, not an exotic one β it is D-177's own shape, re-created one layer down.
|
| 316 |
+
β LOCK ORDER IS KEY-THEN-CACHE, EVERYWHERE. `_update_locked` takes them in that order, so
|
| 317 |
+
this must too; taking `self._lock` first on this path is the deadlock. Both sleeps stay
|
| 318 |
+
OUTSIDE the key lock β a worker that dozed while holding it would block every writer of
|
| 319 |
+
that key for `_FLUSH_MIN_GAP`, which is the freeze this ticket exists to remove.
|
| 320 |
+
"""
|
| 321 |
+
backoff = 2.0
|
| 322 |
+
while True:
|
| 323 |
+
_time.sleep(_FLUSH_DELAY)
|
| 324 |
+
wait, err, snapshot = None, None, None
|
| 325 |
+
with self._key_lock(name):
|
| 326 |
+
with self._lock:
|
| 327 |
+
gap = _time.monotonic() - self._last_flush.get(name, 0.0)
|
| 328 |
+
if gap < _FLUSH_MIN_GAP:
|
| 329 |
+
wait = _FLUSH_MIN_GAP - gap
|
| 330 |
+
elif not self._dirty.get(name):
|
| 331 |
+
self._flushers.pop(name, None)
|
| 332 |
+
return
|
| 333 |
+
else:
|
| 334 |
+
self._dirty[name] = False
|
| 335 |
+
snapshot = json.loads(json.dumps(self._cache.get(name, {})))
|
| 336 |
+
if wait is None:
|
| 337 |
+
try:
|
| 338 |
+
self._upload(name, snapshot)
|
| 339 |
+
with self._lock:
|
| 340 |
+
self._last_flush[name] = _time.monotonic()
|
| 341 |
+
backoff = 2.0
|
| 342 |
+
except Exception as e: # noqa: BLE001
|
| 343 |
+
with self._lock:
|
| 344 |
+
self._dirty[name] = True # keep it queued; cache is still the truth
|
| 345 |
+
err = e
|
| 346 |
+
if wait is not None:
|
| 347 |
+
_time.sleep(wait)
|
| 348 |
+
continue
|
| 349 |
+
if err is not None:
|
| 350 |
+
try:
|
| 351 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 352 |
+
_tel.error(f'store:flush:{name}', err)
|
| 353 |
+
except Exception:
|
| 354 |
+
pass
|
| 355 |
+
_time.sleep(backoff)
|
| 356 |
+
backoff = min(backoff * 2, 60.0)
|
| 357 |
+
|
| 358 |
+
def _schedule_flush(self, name):
|
| 359 |
+
"""Mark dirty + ensure exactly one live worker for the key. Caller holds the lock."""
|
| 360 |
+
self._dirty[name] = True
|
| 361 |
+
t = self._flushers.get(name)
|
| 362 |
+
if t is not None and t.is_alive():
|
| 363 |
+
return
|
| 364 |
+
t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True,
|
| 365 |
+
name=f'store-flush-{self.repo}:{name}')
|
| 366 |
+
self._flushers[name] = t
|
| 367 |
+
t.start()
|
| 368 |
+
|
| 369 |
+
def flush(self, name=None, timeout=30.0):
|
| 370 |
+
"""Block until pending async writes for `name` (or ALL keys) reach the hub β QA gates
|
| 371 |
+
and shutdown hooks use this; the app itself never needs to."""
|
| 372 |
+
deadline = _time.monotonic() + timeout
|
| 373 |
+
names = [name] if name else None
|
| 374 |
+
while _time.monotonic() < deadline:
|
| 375 |
+
with self._lock:
|
| 376 |
+
pending = {k for k, v in self._dirty.items() if v} | {
|
| 377 |
+
k for k, t in self._flushers.items() if t.is_alive()}
|
| 378 |
+
if names is not None:
|
| 379 |
+
pending &= set(names)
|
| 380 |
+
if not pending:
|
| 381 |
+
return True
|
| 382 |
+
_time.sleep(0.2)
|
| 383 |
+
return False
|
| 384 |
+
|
| 385 |
+
def _flush_now_at_exit(self):
|
| 386 |
+
"""Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
|
| 387 |
+
gap β a Space restart must not eat the last seconds of the async window.
|
| 388 |
+
|
| 389 |
+
β W31-T13 β DELIBERATELY NOT KEY-LOCKED, stated so the omission is not read as one. This
|
| 390 |
+
runs at interpreter exit with the process going away: a writer mid-`update` will never
|
| 391 |
+
finish, so waiting for its key lock could only turn a partial save into no save at all.
|
| 392 |
+
Best-effort is the correct contract here, and it is the one case where taking the lock
|
| 393 |
+
would be worse than skipping it.
|
| 394 |
+
"""
|
| 395 |
+
with self._lock:
|
| 396 |
+
dirty = [k for k, v in self._dirty.items() if v]
|
| 397 |
+
snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty}
|
| 398 |
+
for k in dirty:
|
| 399 |
+
self._dirty[k] = False
|
| 400 |
+
for k, d in snaps.items():
|
| 401 |
+
try:
|
| 402 |
+
self._upload(k, d)
|
| 403 |
+
except Exception:
|
| 404 |
+
pass # exiting anyway β nothing left to reschedule
|
| 405 |
+
|
| 406 |
+
def update(self, name, fn, flush='sync'):
|
| 407 |
+
"""Read-modify-write. STRICT first read (raises on a transient error rather than
|
| 408 |
+
starting from an empty dict), apply fn(data)->data, persist.
|
| 409 |
+
|
| 410 |
+
flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the
|
| 411 |
+
table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and
|
| 412 |
+
returns immediately; a coalescing worker commits. The strict read still guards the
|
| 413 |
+
FIRST async update of a key per process.
|
| 414 |
+
|
| 415 |
+
ββ W31-T13 (D-177(d)) β TWO LOCKS, AND THE SPLIT IS THE WHOLE TICKET.
|
| 416 |
+
`self._key_lock(name)` is held for the entire read-modify-write, so two writers of one key
|
| 417 |
+
still cannot interleave (last-write-wins is the contract; a lost update is not).
|
| 418 |
+
`self._lock` β the one every `get()` takes β is now held only for the microseconds a cache
|
| 419 |
+
read or write needs, never across a download or an upload. Before this, one sync write
|
| 420 |
+
froze every reader in the process for two network round trips, on a store whose warm
|
| 421 |
+
`get()` already costs 703 ms of deep copy on tenant #0.
|
| 422 |
+
β Uploading outside `self._lock` is not a new posture: `_flush_worker` has done exactly
|
| 423 |
+
that since wave 7, and `_repo_ok` racing is benign because `create_repo` is `exist_ok=True`.
|
| 424 |
+
"""
|
| 425 |
+
with self._key_lock(name):
|
| 426 |
+
return self._update_locked(name, fn, flush)
|
| 427 |
+
|
| 428 |
+
def _update_locked(self, name, fn, flush):
|
| 429 |
+
"""`update`'s body, with this key's write lock already held. See its docstring.
|
| 430 |
+
|
| 431 |
+
β THE CACHE LOCK IS TAKEN THREE TIMES AND RELEASED BETWEEN THEM, which is only safe
|
| 432 |
+
because THIS key's write lock is held throughout: nothing else can be mid-write on it, so
|
| 433 |
+
the state read in the first section is still ours in the third. A caller reaching this
|
| 434 |
+
without the key lock would have the old interleaving bug back.
|
| 435 |
+
"""
|
| 436 |
+
# β A FLAG, NOT A `None` SENTINEL. `None` can be a real cached value, and a sentinel that
|
| 437 |
+
# can equal real data is not a sentinel β this repo has paid for that exact shape before
|
| 438 |
+
# (`routes_nav`'s locked-mode default). The flag says "the cache answered", full stop.
|
| 439 |
+
data, from_cache = None, False
|
| 440 |
+
with self._lock:
|
| 441 |
+
if flush == 'async' and self._owned.get(name) and name in self._cache:
|
| 442 |
+
data, from_cache = json.loads(json.dumps(self._cache[name])), True
|
| 443 |
+
elif self._dirty.get(name) and self._owned.get(name) and name in self._cache:
|
| 444 |
+
# β A SYNC READ-MODIFY-WRITE MUST NOT DISCARD A PENDING ASYNC WRITE.
|
| 445 |
+
#
|
| 446 |
+
# THE BUG THIS LINE EXISTS FOR (owner report 2026-08-09, MEASURED on staging):
|
| 447 |
+
# `POST /rows` answered `201 {"rid":"100"}` and the row never existed.
|
| 448 |
+
# `user_tables.add_row` commits `flush='async'` β the row lives ONLY in
|
| 449 |
+
# `self._cache` until the coalescing worker uploads it, which is `_FLUSH_DELAY`
|
| 450 |
+
# (2s) away and up to `_FLUSH_MIN_GAP` (20s) if that key committed recently.
|
| 451 |
+
# Inside that window ANY sync writer of the SAME key took the `else` branch
|
| 452 |
+
# below, and `_read_strict` does `self._cache[name] = self._download(name)` β
|
| 453 |
+
# it REPLACES the cache with the hub's copy, erasing the only place the new row
|
| 454 |
+
# existed, and then `put()` uploaded that. Nothing raised. The 201 was truthful
|
| 455 |
+
# about `add_row`'s return value and false about the outcome.
|
| 456 |
+
#
|
| 457 |
+
# β `user_tables` has ~40 sync writers against 2 async ones, so this was never
|
| 458 |
+
# one bad call site β it is a property of the KEY. Wave 27 only made it fire
|
| 459 |
+
# every time, by putting the relation refresh on the add path.
|
| 460 |
+
#
|
| 461 |
+
# β AND RE-DOWNLOADING WAS PROTECTING NOTHING. Once a key is dirty our cache
|
| 462 |
+
# has already diverged and the pending upload was going to overwrite the hub
|
| 463 |
+
# anyway β so the fresh read cannot preserve a remote write, it can only destroy
|
| 464 |
+
# a local one we already acknowledged. This is the class's OWN documented
|
| 465 |
+
# contract, restored: see `_owned` above β *"read-your-writes: every reader
|
| 466 |
+
# consults the cache first"* β and `_flush_worker`'s *"the cache stays the truth
|
| 467 |
+
# and the dirty mark survives, so nothing is lost silently."*
|
| 468 |
+
#
|
| 469 |
+
# β NOT a weakening of `_read_strict`'s purpose. That exists so a FAILED read
|
| 470 |
+
# aborts the RMW instead of merging into `{}`; the cache is populated
|
| 471 |
+
# known-good local state, and this branch is only reachable when it is.
|
| 472 |
+
data, from_cache = json.loads(json.dumps(self._cache[name])), True
|
| 473 |
+
# else: the strict hub read happens BELOW, outside this lock (W31-T13)
|
| 474 |
+
|
| 475 |
+
# β W31-T13 β THE HUB READ AND THE CALLER'S `fn` RUN WITH NO CACHE LOCK HELD. `fn` is
|
| 476 |
+
# arbitrary caller code β `automation_engine`'s relation pass has run whole-tenant work
|
| 477 |
+
# inside one β and it used to execute with every reader in the process blocked behind it.
|
| 478 |
+
if not from_cache:
|
| 479 |
+
data = self._read_strict(name) # network; re-checks the dirty mark on the way back
|
| 480 |
+
result = fn(data)
|
| 481 |
+
data = result if result is not None else data
|
| 482 |
+
|
| 483 |
+
if flush == 'async':
|
| 484 |
+
if not self.available():
|
| 485 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 486 |
+
with self._lock:
|
| 487 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 488 |
+
self._owned[name] = True
|
| 489 |
+
self._bump(name) # C6 β the async write is ACCEPTED here, not at flush time
|
| 490 |
+
self._schedule_flush(name)
|
| 491 |
+
return data
|
| 492 |
+
|
| 493 |
+
# β THE SYNC COMMIT, SPELLED OUT HERE RATHER THAN THROUGH `put()`, AND THAT IS A CHOICE.
|
| 494 |
+
# `put()` holds `self._lock` across its own `_upload` β which is exactly the freeze this
|
| 495 |
+
# ticket removes β but it is PUBLIC with seven callers outside this file (`users.py`,
|
| 496 |
+
# `modules/ar.py`, `modules/map.py`, `runtime.py`, `store_backend.py`,
|
| 497 |
+
# `ops/seed_pg_from_hf.py`), and changing its locking changes all of their concurrency
|
| 498 |
+
# semantics at once. Duplicating four assignments with this comment beside them is the
|
| 499 |
+
# smaller blast radius; if `put` is ever split, this collapses back into a call.
|
| 500 |
+
if not self.available():
|
| 501 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 502 |
+
self._upload(name, data) # network, no cache lock held
|
| 503 |
+
with self._lock:
|
| 504 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 505 |
+
self._owned[name] = True
|
| 506 |
+
self._last_flush[name] = _time.monotonic()
|
| 507 |
+
# β AFTER THE UPLOAD, NEVER BEFORE β `put()`'s own law. `_upload` raises on failure,
|
| 508 |
+
# and a bump on that path would have `revision()` report a change that never landed.
|
| 509 |
+
self._bump(name)
|
| 510 |
+
# The upload above CARRIED whatever was pending, so the key is no longer ahead of
|
| 511 |
+
# the hub. Without this the flush worker wakes on a still-dirty mark and re-uploads
|
| 512 |
+
# identical content β a second commit against the 256-commits/hr budget for nothing.
|
| 513 |
+
# β AFTER the upload, for the same reason: on a raise the mark MUST survive (the cache
|
| 514 |
+
# is still the truth and the row must still be committed by the worker). Clearing it
|
| 515 |
+
# first would turn a transient upload error into the very silent loss the branch above
|
| 516 |
+
# exists to stop β which is why it sits here and not before `_upload`.
|
| 517 |
+
self._dirty[name] = False
|
| 518 |
+
return data
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
# ------------------------------------------------------------------- instances + module API
|
| 522 |
+
_INSTANCES = {}
|
| 523 |
+
_INSTANCES_LOCK = threading.RLock()
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
def for_repo(repo_id):
|
| 527 |
+
"""The HF Store bound to `repo_id` β one instance per repo per process (each with its own
|
| 528 |
+
cache and flush state).
|
| 529 |
+
|
| 530 |
+
β WAVE 20: this is now the HF-SPECIFIC factory. Callers that want "the right store for this
|
| 531 |
+
tenant, whatever the backend is" call `handle()` below. `for_repo` keeps its exact old
|
| 532 |
+
behaviour because the seed/migration tooling has to be able to name the FILE store
|
| 533 |
+
explicitly while the app runs on Postgres β a migration that could only reach the active
|
| 534 |
+
backend could not copy between them.
|
| 535 |
+
"""
|
| 536 |
+
rid = str(repo_id or '').strip() or REPO
|
| 537 |
+
with _INSTANCES_LOCK:
|
| 538 |
+
inst = _INSTANCES.get(rid)
|
| 539 |
+
if inst is None:
|
| 540 |
+
inst = Store(rid)
|
| 541 |
+
_INSTANCES[rid] = inst
|
| 542 |
+
return inst
|
| 543 |
+
|
| 544 |
+
|
| 545 |
+
# =============================================================================================
|
| 546 |
+
# WAVE 20 (owner ruling R1, closes DEBT D-4) β THE BACKEND SEAM.
|
| 547 |
+
#
|
| 548 |
+
# `core/store_backend.py` has carried this warning since EXIT-2b, and it was correct at the time:
|
| 549 |
+
#
|
| 550 |
+
# "Flipping STORE_BACKEND=pg does NOT redirect the ~40 existing callers that say
|
| 551 |
+
# `import core.store as store` β they are bound to the HF module directly. Rewiring them is
|
| 552 |
+
# task C-4 β¦ THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise."
|
| 553 |
+
#
|
| 554 |
+
# **R1 IS C-4.** The owner ruled the cutover on 2026-08-05 (the D-4 trigger that fired: a SECOND
|
| 555 |
+
# server process β `royal-imports/cfo-os` came back as a pinned LIVE environment beside staging,
|
| 556 |
+
# and two containers on one last-write-wins file store is the race B-3 was always about).
|
| 557 |
+
#
|
| 558 |
+
# THE REWIRE IS HERE RATHER THAN IN 28 FILES, and that is a deliberate choice over the obvious
|
| 559 |
+
# alternative of `sed`-ing every `import core.store as store` to `import core.store_backend`:
|
| 560 |
+
# * every one of those callers means "the store for the tenant I am serving", which is exactly
|
| 561 |
+
# what this module has always meant. The BACKEND is not their concern and making it their
|
| 562 |
+
# concern is how one of them gets missed;
|
| 563 |
+
# * a missed caller under a search-and-replace does not fail β it silently keeps writing to the
|
| 564 |
+
# file store while everything else writes to Postgres. That is the split-brain this whole
|
| 565 |
+
# ruling exists to end, reintroduced by the fix for it;
|
| 566 |
+
# * `store_backend.py`'s stated reason for refusing to do it this way β "no dual-read window and
|
| 567 |
+
# no way to compare the two stores' contents first" β is satisfied: `ops/seed_pg_from_hf.py`
|
| 568 |
+
# copies, then `--verify` diffs the two stores key-by-key before anything flips.
|
| 569 |
+
#
|
| 570 |
+
# β THE IMPORT IS LAZY AND MUST STAY LAZY. `core/store_pg.py` imports psycopg only inside
|
| 571 |
+
# `_pool()`, so an `hf` deployment installs no driver; importing it at module scope here would
|
| 572 |
+
# undo that and make the default backend depend on the optional dependency.
|
| 573 |
+
# =============================================================================================
|
| 574 |
+
|
| 575 |
+
def backend():
|
| 576 |
+
"""`'hf'` | `'pg'` β validated, resolved per call so a test can flip the env var.
|
| 577 |
+
|
| 578 |
+
Mirrors `core.store_backend.name()` deliberately rather than importing it: that module
|
| 579 |
+
imports THIS one, so reaching back would be a cycle. `verify_store_pg` asserts the two agree
|
| 580 |
+
on every value, which is the guard against them drifting apart.
|
| 581 |
+
"""
|
| 582 |
+
raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
|
| 583 |
+
if raw not in ('hf', 'pg'):
|
| 584 |
+
raise RuntimeError(
|
| 585 |
+
f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store) or 'pg' "
|
| 586 |
+
f"(Postgres, needs DATABASE_URL). Refusing to guess: a typo that silently served the "
|
| 587 |
+
f"other store is how data ends up in two places.")
|
| 588 |
+
return raw
|
| 589 |
+
|
| 590 |
+
|
| 591 |
+
def handle(repo_id=None, slug=None):
|
| 592 |
+
"""THE tenant-bound store handle for the ACTIVE backend β the one seam every caller crosses.
|
| 593 |
+
|
| 594 |
+
`repo_id` addresses the HF backend (a dataset repo); `slug` addresses Postgres (a schema).
|
| 595 |
+
Both are passed by `harness.runtime.get_runtime`, which knows both facts, so flipping the
|
| 596 |
+
backend never needs a lookup table between them β and neither identifier has to be invented
|
| 597 |
+
for the backend that does not use it.
|
| 598 |
+
"""
|
| 599 |
+
if backend() == 'pg':
|
| 600 |
+
import core.store_pg as _pg # noqa: PLC0415 β lazy: see the banner above
|
| 601 |
+
return _pg.PgStore(slug or os.environ.get('AIOS_TENANT') or 'royal-imports')
|
| 602 |
+
return for_repo(repo_id)
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
#: The HF default instance. Kept as a module global (not a property) because the seed/migration
|
| 606 |
+
#: tooling and `_flush_all_at_exit` both need the FILE store by name even when pg is active.
|
| 607 |
+
_DEFAULT = for_repo(REPO)
|
| 608 |
+
|
| 609 |
+
|
| 610 |
+
def _d():
|
| 611 |
+
"""The default tenant's store on the active backend β what every module function below uses.
|
| 612 |
+
|
| 613 |
+
β Resolved PER CALL, never cached. A cached default would freeze the backend at import time,
|
| 614 |
+
and import order is exactly what nobody controls: `api/main.py` imports half the platform
|
| 615 |
+
before it has read a single environment variable it did not inherit.
|
| 616 |
+
"""
|
| 617 |
+
return handle() if backend() == 'pg' else _DEFAULT
|
| 618 |
+
|
| 619 |
+
|
| 620 |
+
def available():
|
| 621 |
+
return _d().available()
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
def get(name, fresh=False):
|
| 625 |
+
return _d().get(name, fresh=fresh)
|
| 626 |
+
|
| 627 |
+
|
| 628 |
+
def _read_strict(name):
|
| 629 |
+
return _d()._read_strict(name)
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
def exists(name):
|
| 633 |
+
return _d().exists(name)
|
| 634 |
+
|
| 635 |
+
|
| 636 |
+
def upload_bytes(path_in_repo, data, message=None):
|
| 637 |
+
return _d().upload_bytes(path_in_repo, data, message=message)
|
| 638 |
+
|
| 639 |
+
|
| 640 |
+
def download_bytes(path_in_repo):
|
| 641 |
+
return _d().download_bytes(path_in_repo)
|
| 642 |
+
|
| 643 |
+
|
| 644 |
+
def delete_path(path_in_repo):
|
| 645 |
+
return _d().delete_path(path_in_repo)
|
| 646 |
+
|
| 647 |
+
|
| 648 |
+
def put(name, data):
|
| 649 |
+
return _d().put(name, data)
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
def flush(name=None, timeout=30.0):
|
| 653 |
+
return _d().flush(name=name, timeout=timeout)
|
| 654 |
+
|
| 655 |
+
|
| 656 |
+
def update(name, fn, flush='sync'):
|
| 657 |
+
return _d().update(name, fn, flush=flush)
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def revision(name):
|
| 661 |
+
"""The bucket's change token β wave 29, C6. See `Store.revision` for why it copies nothing.
|
| 662 |
+
|
| 663 |
+
β THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS
|
| 664 |
+
HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely
|
| 665 |
+
so a capability one backend lacks cannot be reached through the shared door and die at the
|
| 666 |
+
cutover β so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change
|
| 667 |
+
and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half
|
| 668 |
+
had no owner: a method is a capability the HF store has, a module function is a promise both
|
| 669 |
+
backends keep.
|
| 670 |
+
"""
|
| 671 |
+
return _d().revision(name)
|
| 672 |
+
|
| 673 |
+
|
| 674 |
+
def _flush_all_at_exit():
|
| 675 |
+
with _INSTANCES_LOCK:
|
| 676 |
+
instances = list(_INSTANCES.values())
|
| 677 |
+
for inst in instances:
|
| 678 |
+
inst._flush_now_at_exit()
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
import atexit # noqa: E402 (registered after the class it needs)
|
| 682 |
+
atexit.register(_flush_all_at_exit)
|
platform/core/user_tables.py
CHANGED
|
@@ -116,16 +116,69 @@ def register_connected(*table_keys):
|
|
| 116 |
return frozenset(_CONNECTED)
|
| 117 |
|
| 118 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
def is_connected(table_key, st=None):
|
| 120 |
"""Is this database backed by a connected source rather than by typing?
|
| 121 |
|
| 122 |
THREE ANSWERS, in falling order of authority, and the third is fenced deliberately:
|
| 123 |
1. the app layer registered it (`register_connected`) β the live path;
|
| 124 |
2. the stored definition says so (`connected: True`) β survives a process with no registrar;
|
| 125 |
-
3. β
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
|
|
|
| 129 |
"""
|
| 130 |
key = str(table_key or '').strip()
|
| 131 |
if key in _CONNECTED:
|
|
@@ -135,7 +188,7 @@ def is_connected(table_key, st=None):
|
|
| 135 |
return False
|
| 136 |
if table.get('connected') is True:
|
| 137 |
return True
|
| 138 |
-
return (key.startswith(
|
| 139 |
and table.get('source') == AUTOMATION_SOURCE
|
| 140 |
and table.get('recordMode') == AUTOMATION_RECORD_MODE)
|
| 141 |
|
|
@@ -2448,16 +2501,79 @@ def may_open(table_key, viewer, is_admin=False, st=None):
|
|
| 2448 |
return False
|
| 2449 |
|
| 2450 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2451 |
def nav_entries(viewer=None, is_admin=False, st=None):
|
| 2452 |
"""Registry-SHAPED dicts so the nav can render user tables beside real modules with no
|
| 2453 |
special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
|
| 2454 |
|
| 2455 |
Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2456 |
"""
|
|
|
|
|
|
|
| 2457 |
out = []
|
| 2458 |
-
for key, t in sorted(
|
| 2459 |
key=lambda kv: (kv[1].get('label') or '').lower()):
|
| 2460 |
-
if viewer is not None and not may_open(key, viewer, is_admin, st=
|
| 2461 |
continue
|
| 2462 |
out.append({'key': key, 'label': t.get('label') or key,
|
| 2463 |
'source': t.get('source') or BLANK_SOURCE,
|
|
|
|
| 116 |
return frozenset(_CONNECTED)
|
| 117 |
|
| 118 |
|
| 119 |
+
#: ββ W31-T25 / CONTRACT C2 β the KEY-CONVENTION leg's prefixes are a REGISTRY, not a literal.
|
| 120 |
+
#:
|
| 121 |
+
#: `is_connected`'s third leg used to read `key.startswith('ut_odoo_')` inline. Meta Ads is the
|
| 122 |
+
#: second connector to spawn locked databases (`ut_meta_*`), and the alternative was a second
|
| 123 |
+
#: predicate in the app layer answering the same question β one question, two normalizers, which
|
| 124 |
+
#: is a recorded defect class here ([[one-question-two-normalizers]]). So the connector layer
|
| 125 |
+
#: DECLARES its prefix and this module keeps the single evaluator.
|
| 126 |
+
#:
|
| 127 |
+
#: β `'ut_odoo_'` IS SEEDED RATHER THAN REGISTERED, AND THAT IS THE POINT OF THE LEG. The whole
|
| 128 |
+
#: reason a naming convention sits below the registry is to answer in a process where nothing has
|
| 129 |
+
#: registered anything β a cold worker that never imported `routes_odoo_tables` still owes the
|
| 130 |
+
#: right answer for `ut_odoo_gl_lines`, and an empty seed would have it fall through to
|
| 131 |
+
#: `MAX_ROWS`, i.e. re-cap a connected source against R6. A newly registered prefix carries the
|
| 132 |
+
#: weaker guarantee that `register_connected` already carries: it holds once its registrar has run.
|
| 133 |
+
_CONNECTED_PREFIXES = {'ut_odoo_'}
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def register_connected_prefix(*prefixes):
|
| 137 |
+
"""Declare a table-key prefix whose automation-sourced, records-locked tables are CONNECTED.
|
| 138 |
+
|
| 139 |
+
The connector layer calls this beside `register_connected` (which names exact keys): the
|
| 140 |
+
prefix answers for tables this process has not enumerated, the exact keys answer for the ones
|
| 141 |
+
it has. Returns the whole registered set, so a caller can assert what it just joined.
|
| 142 |
+
|
| 143 |
+
β A PREFIX ALONE NEVER MAKES A TABLE CONNECTED. `is_connected`'s third leg still demands
|
| 144 |
+
`source == AUTOMATION_SOURCE` and `recordMode == AUTOMATION_RECORD_MODE`, because a person who
|
| 145 |
+
names a database "Meta ads spend" gets a `ut_meta_β¦`-shaped key too β and exempting it from
|
| 146 |
+
the ceiling on the strength of its NAME would be exactly the silent hole the fence exists for.
|
| 147 |
+
"""
|
| 148 |
+
for prefix in prefixes:
|
| 149 |
+
p = str(prefix or '').strip()
|
| 150 |
+
if p:
|
| 151 |
+
_CONNECTED_PREFIXES.add(p)
|
| 152 |
+
return connected_prefixes()
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def connected_prefixes():
|
| 156 |
+
"""The registered key prefixes β the ONE list a caller may read to name the same set.
|
| 157 |
+
|
| 158 |
+
C2's parity gate reads this rather than re-spelling `ut_odoo_` / `ut_meta_` on the far side:
|
| 159 |
+
a second literal is a second normalizer wearing an assertion's clothes.
|
| 160 |
+
|
| 161 |
+
β AND IT IS THE REGISTRAR'S OWN RETURN VALUE, which is not tidiness β `verify_reachability`
|
| 162 |
+
went RED the hour this shipped, correctly: a public function whose only callers are `verify_*`
|
| 163 |
+
files is a feature no user can reach ([[reachable-is-not-the-same-as-built]]), and "another
|
| 164 |
+
lane's gate will call it" is not a caller. Routing the write door's answer through the read
|
| 165 |
+
door makes one construction site of the set instead of two, and makes the accessor
|
| 166 |
+
load-bearing rather than decorative.
|
| 167 |
+
"""
|
| 168 |
+
return frozenset(_CONNECTED_PREFIXES)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
def is_connected(table_key, st=None):
|
| 172 |
"""Is this database backed by a connected source rather than by typing?
|
| 173 |
|
| 174 |
THREE ANSWERS, in falling order of authority, and the third is fenced deliberately:
|
| 175 |
1. the app layer registered it (`register_connected`) β the live path;
|
| 176 |
2. the stored definition says so (`connected: True`) β survives a process with no registrar;
|
| 177 |
+
3. β a REGISTERED KEY PREFIX (`register_connected_prefix`; seeded `ut_odoo_`, and Meta Ads
|
| 178 |
+
joins it as `ut_meta_`), AND ONLY on a table that is automation-sourced with records
|
| 179 |
+
locked. A person who names a database "Odoo foo" gets exactly that key, and exempting it
|
| 180 |
+
from the ceiling on the strength of its name would be a silent hole. Their table is
|
| 181 |
+
`Blank`-sourced and records-mutable, so it cannot reach this leg.
|
| 182 |
"""
|
| 183 |
key = str(table_key or '').strip()
|
| 184 |
if key in _CONNECTED:
|
|
|
|
| 188 |
return False
|
| 189 |
if table.get('connected') is True:
|
| 190 |
return True
|
| 191 |
+
return (any(key.startswith(p) for p in _CONNECTED_PREFIXES)
|
| 192 |
and table.get('source') == AUTOMATION_SOURCE
|
| 193 |
and table.get('recordMode') == AUTOMATION_RECORD_MODE)
|
| 194 |
|
|
|
|
| 2501 |
return False
|
| 2502 |
|
| 2503 |
|
| 2504 |
+
#: ββ W31-T10 / CONTRACT C1 β THE TWO BUCKETS `may_open` READS, LENT FOR THE LENGTH OF ONE PASS.
|
| 2505 |
+
#:
|
| 2506 |
+
#: `core/store.py:Store.get` deep-copies the WHOLE tenant document on every call, hit or miss,
|
| 2507 |
+
#: under `self._lock` β a documented 35.8 MB ceiling. So asking `may_open` about N databases cost
|
| 2508 |
+
#: `1 + N` full copies of `user_tables`, plus up to another N of `object_shares` (its last leg),
|
| 2509 |
+
#: and `GET /nav` measured a median of **9,548 ms live / 24,741 ms in-process** on tenant #0.
|
| 2510 |
+
#:
|
| 2511 |
+
#: β THE OBVIOUS FIX IS THE FORBIDDEN ONE, and `may_open`'s own docstring is the record of why:
|
| 2512 |
+
#: wave 20 found TWO ideas of who owns a table (`routes_automation` admitted
|
| 2513 |
+
#: `createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a database in a picker
|
| 2514 |
+
#: and was refused on opening it. Inlining creator-or-admin at a call site re-creates exactly
|
| 2515 |
+
#: that. **`may_open` stays THE one resolver, unmodified and still asked about every table** β
|
| 2516 |
+
#: it is simply no longer charged for documents the caller is already holding.
|
| 2517 |
+
#:
|
| 2518 |
+
#: β WHY THE SHARES BUCKET IS IN HERE TOO, when `routes_automation._LentTables` intercepts only
|
| 2519 |
+
#: `user_tables`: `may_open`'s admit-by-grant leg calls `shares.may_see` β `role_for` β `grants`,
|
| 2520 |
+
#: which is `st.get('object_shares')` PER TABLE. It is a different key, so a `user_tables`-only
|
| 2521 |
+
#: lend passes it straight through β the exact residual C1 says needs its own answer. It is read
|
| 2522 |
+
#: LAZILY (an admin, or a table's creator, is admitted before that leg runs and never pays for
|
| 2523 |
+
#: it) and memoised for the pass.
|
| 2524 |
+
#:
|
| 2525 |
+
#: β THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through
|
| 2526 |
+
#: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the
|
| 2527 |
+
#: runtime through `__getattr__` β so this can serve a stale value for exactly two keys, for
|
| 2528 |
+
#: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level
|
| 2529 |
+
#: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS
|
| 2530 |
+
#: `createdBy`).
|
| 2531 |
+
_LENDABLE = (STORE_KEY, 'object_shares')
|
| 2532 |
+
|
| 2533 |
+
|
| 2534 |
+
class _Lent:
|
| 2535 |
+
"""A read-only `st` that serves already-read documents and passes everything else through."""
|
| 2536 |
+
|
| 2537 |
+
def __init__(self, runtime, docs):
|
| 2538 |
+
self._runtime, self._docs = runtime, dict(docs or {})
|
| 2539 |
+
|
| 2540 |
+
def get(self, name, fresh=False):
|
| 2541 |
+
if fresh or name not in _LENDABLE:
|
| 2542 |
+
return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
|
| 2543 |
+
if name not in self._docs:
|
| 2544 |
+
self._docs[name] = self._runtime.get(name) # lazily, once, for this pass
|
| 2545 |
+
return self._docs[name]
|
| 2546 |
+
|
| 2547 |
+
def __getattr__(self, name):
|
| 2548 |
+
return getattr(self._runtime, name)
|
| 2549 |
+
|
| 2550 |
+
|
| 2551 |
+
def lend(st=None, **docs):
|
| 2552 |
+
"""A handle that serves `docs` for `_LENDABLE` keys and defers everything else to `st`.
|
| 2553 |
+
|
| 2554 |
+
`lend(rt, user_tables=doc)` β keyword names are STORE KEYS. Call it once per read pass and
|
| 2555 |
+
hand the result to every predicate in that pass; see the note above for what makes it safe.
|
| 2556 |
+
"""
|
| 2557 |
+
return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None})
|
| 2558 |
+
|
| 2559 |
+
|
| 2560 |
def nav_entries(viewer=None, is_admin=False, st=None):
|
| 2561 |
"""Registry-SHAPED dicts so the nav can render user tables beside real modules with no
|
| 2562 |
special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
|
| 2563 |
|
| 2564 |
Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse.
|
| 2565 |
+
|
| 2566 |
+
β W31-T10 (C1): ONE document read for the whole listing. The filter is unchanged β the same
|
| 2567 |
+
`may_open`, asked about the same tables, in the same order β it is handed the document this
|
| 2568 |
+
function has already read instead of taking a fresh 35.8 MB copy per table. A caller that
|
| 2569 |
+
already holds the document passes its own `lend(...)` as `st` and this pays nothing at all.
|
| 2570 |
"""
|
| 2571 |
+
tables = all_tables(st)
|
| 2572 |
+
lent = lend(st, **{STORE_KEY: tables})
|
| 2573 |
out = []
|
| 2574 |
+
for key, t in sorted(tables.items(),
|
| 2575 |
key=lambda kv: (kv[1].get('label') or '').lower()):
|
| 2576 |
+
if viewer is not None and not may_open(key, viewer, is_admin, st=lent):
|
| 2577 |
continue
|
| 2578 |
out.append({'key': key, 'label': t.get('label') or key,
|
| 2579 |
'source': t.get('source') or BLANK_SOURCE,
|
platform/harness/meta_store.py
ADDED
|
@@ -0,0 +1,461 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""meta_store.py β the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
|
| 2 |
+
|
| 3 |
+
python platform/harness/meta_store.py --sync pull into tenant #0's mirror
|
| 4 |
+
python platform/harness/meta_store.py --sync --tenant gtmlab
|
| 5 |
+
python platform/harness/meta_store.py --status what is in the mirror now
|
| 6 |
+
|
| 7 |
+
β WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
|
| 8 |
+
hardwired to XML-RPC β it does `import core.odoo as O` and speaks `search_read` β so no amount of
|
| 9 |
+
spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
|
| 10 |
+
per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
|
| 11 |
+
delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
|
| 12 |
+
store with two loaders, not two stores.**
|
| 13 |
+
|
| 14 |
+
β THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
|
| 15 |
+
the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
|
| 16 |
+
`XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
|
| 17 |
+
`Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
|
| 18 |
+
different first hop.
|
| 19 |
+
|
| 20 |
+
β EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
|
| 21 |
+
carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
|
| 22 |
+
Ad Account 45 Β· Campaign 35 Β· Ad Set 54 Β· Ad 36 Β· Creative 58 Β· Insights 57
|
| 23 |
+
β AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
|
| 24 |
+
came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
|
| 25 |
+
nothing going red β which is exactly the "do not drop any column" instruction, broken silently.
|
| 26 |
+
The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
|
| 27 |
+
|
| 28 |
+
β `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
|
| 29 |
+
answers `403 (#200) Requires business_management permission`, which this token does not carry. One
|
| 30 |
+
column of 285. R6's second sentence β a limit that cannot be removed gets stated with its cause.
|
| 31 |
+
|
| 32 |
+
β IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
|
| 33 |
+
larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
|
| 34 |
+
(W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
|
| 35 |
+
"""
|
| 36 |
+
import argparse
|
| 37 |
+
import json
|
| 38 |
+
import os
|
| 39 |
+
import sys
|
| 40 |
+
import time
|
| 41 |
+
import urllib.error
|
| 42 |
+
import urllib.parse
|
| 43 |
+
import urllib.request
|
| 44 |
+
from pathlib import Path
|
| 45 |
+
|
| 46 |
+
_HERE = Path(__file__).resolve().parent
|
| 47 |
+
if str(_HERE.parent) not in sys.path:
|
| 48 |
+
sys.path.insert(0, str(_HERE.parent))
|
| 49 |
+
|
| 50 |
+
from harness import datastore # noqa: E402
|
| 51 |
+
|
| 52 |
+
GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
|
| 53 |
+
GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
|
| 54 |
+
|
| 55 |
+
#: Numeric columns, by name. Everything else is VARCHAR β including ids (see the header) and
|
| 56 |
+
#: including anything Graph returns as a nested object, which is stored as compact JSON text.
|
| 57 |
+
_INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
|
| 58 |
+
"inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
|
| 59 |
+
"account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
|
| 60 |
+
"min_daily_budget", "min_campaign_group_spend_cap"}
|
| 61 |
+
_DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
|
| 62 |
+
"inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
|
| 63 |
+
"daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
|
| 64 |
+
"daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
|
| 65 |
+
"lifetime_spend_cap", "lifetime_imps"}
|
| 66 |
+
|
| 67 |
+
#: β THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
|
| 68 |
+
#: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
|
| 69 |
+
#: the grid links in `meta_relational`.
|
| 70 |
+
ACCOUNT_FIELDS = (
|
| 71 |
+
"account_id account_status age amount_spent balance business_city business_country_code "
|
| 72 |
+
"business_name business_state business_street business_street2 business_zip capabilities "
|
| 73 |
+
"created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
|
| 74 |
+
"has_migrated_permissions id io_number is_attribution_spec_system_default "
|
| 75 |
+
"is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
|
| 76 |
+
"is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
|
| 77 |
+
"name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
|
| 78 |
+
"timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
|
| 79 |
+
).split()
|
| 80 |
+
|
| 81 |
+
CAMPAIGN_FIELDS = (
|
| 82 |
+
"account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
|
| 83 |
+
"campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
|
| 84 |
+
"created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
|
| 85 |
+
"last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
|
| 86 |
+
"promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
|
| 87 |
+
"special_ad_category special_ad_category_country spend_cap start_time status stop_time "
|
| 88 |
+
"topline_id updated_time"
|
| 89 |
+
).split()
|
| 90 |
+
|
| 91 |
+
ADSET_FIELDS = (
|
| 92 |
+
"account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
|
| 93 |
+
"bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
|
| 94 |
+
"campaign_active_time campaign_attribution campaign_id configured_status created_time "
|
| 95 |
+
"creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
|
| 96 |
+
"effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
|
| 97 |
+
"issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
|
| 98 |
+
"lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
|
| 99 |
+
"optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
|
| 100 |
+
"rf_prediction_id source_adset source_adset_id start_time status targeting "
|
| 101 |
+
"targeting_optimization_types time_based_ad_rotation_id_blocks "
|
| 102 |
+
"time_based_ad_rotation_intervals updated_time use_new_app_click"
|
| 103 |
+
).split()
|
| 104 |
+
|
| 105 |
+
AD_FIELDS = (
|
| 106 |
+
"account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
|
| 107 |
+
"adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
|
| 108 |
+
"conversion_domain created_time creative demolink_hash display_sequence effective_status "
|
| 109 |
+
"engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
|
| 110 |
+
"preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
|
| 111 |
+
"tracking_and_conversion_with_defaults tracking_specs updated_time"
|
| 112 |
+
).split()
|
| 113 |
+
|
| 114 |
+
CREATIVE_FIELDS = (
|
| 115 |
+
"account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
|
| 116 |
+
"branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
|
| 117 |
+
"category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
|
| 118 |
+
"destination_set_id dynamic_ad_voice effective_authorization_category "
|
| 119 |
+
"effective_instagram_media_id effective_object_story_id enable_direct_install "
|
| 120 |
+
"enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
|
| 121 |
+
"instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
|
| 122 |
+
"link_deep_link_url link_destination_display_url link_og_id link_url "
|
| 123 |
+
"messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
|
| 124 |
+
"object_type object_url place_page_set_id platform_customizations playable_asset_id "
|
| 125 |
+
"portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
|
| 126 |
+
"template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
|
| 127 |
+
"use_page_actor_override video_id"
|
| 128 |
+
).split()
|
| 129 |
+
|
| 130 |
+
INSIGHT_FIELDS = (
|
| 131 |
+
"account_currency account_id account_name action_values actions ad_id ad_name adset_id "
|
| 132 |
+
"adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
|
| 133 |
+
"clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
|
| 134 |
+
"cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
|
| 135 |
+
"date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
|
| 136 |
+
"impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
|
| 137 |
+
"optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
|
| 138 |
+
"social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
|
| 139 |
+
"video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
|
| 140 |
+
"video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
|
| 141 |
+
"video_play_actions video_thruplay_watched_actions website_purchase_roas"
|
| 142 |
+
).split()
|
| 143 |
+
|
| 144 |
+
#: table -> (graph edge on the account | None, measured fields, parent column)
|
| 145 |
+
SPECS = {
|
| 146 |
+
"meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
|
| 147 |
+
"meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
|
| 148 |
+
"meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
|
| 149 |
+
"meta_ads": ("ads", AD_FIELDS, "adset_id"),
|
| 150 |
+
"meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
#: The daily Insights grain (R2). One row per (ad, day) β the id is synthesised because Insights
|
| 154 |
+
#: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
|
| 155 |
+
INSIGHTS_TABLE = "meta_insights_daily"
|
| 156 |
+
INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
|
| 157 |
+
|
| 158 |
+
#: β INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
|
| 159 |
+
#: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
|
| 160 |
+
#: β Graph's way of saying the synchronous query is too heavy (the async report-run API is the
|
| 161 |
+
#: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
|
| 162 |
+
#: page 25 answer 200. So the window is walked in slices with every column intact:
|
| 163 |
+
#: 57 fields Β· ad level Β· 7d Β· limit 25 -> 200, 25 rows
|
| 164 |
+
#: 57 fields Β· account level Β· 7d -> 200, 7 rows
|
| 165 |
+
#: β Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over β
|
| 166 |
+
#: it drops columns R2 requires, and it does so invisibly.
|
| 167 |
+
INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
|
| 168 |
+
INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
|
| 169 |
+
INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _slices(days, size):
|
| 173 |
+
"""[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
|
| 174 |
+
from datetime import date, timedelta
|
| 175 |
+
end = date.today()
|
| 176 |
+
start = end - timedelta(days=max(1, days) - 1)
|
| 177 |
+
out, cur = [], start
|
| 178 |
+
while cur <= end:
|
| 179 |
+
stop = min(cur + timedelta(days=max(1, size) - 1), end)
|
| 180 |
+
out.append((cur.isoformat(), stop.isoformat()))
|
| 181 |
+
cur = stop + timedelta(days=1)
|
| 182 |
+
return out
|
| 183 |
+
|
| 184 |
+
#: β A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
|
| 185 |
+
#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
|
| 186 |
+
#: not a row cap on a connected source (R6 forbids that) β it is a runaway guard, and reaching it
|
| 187 |
+
#: is reported as a problem, never absorbed.
|
| 188 |
+
MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
|
| 189 |
+
PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
class MetaError(RuntimeError):
|
| 193 |
+
"""A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def token():
|
| 197 |
+
"""The Meta token from the environment or gitignored `platform/.env`; "" when absent.
|
| 198 |
+
|
| 199 |
+
β Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
|
| 200 |
+
`platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
|
| 201 |
+
(`core` never imports up). Fifteen lines is the price of that boundary.
|
| 202 |
+
"""
|
| 203 |
+
tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
|
| 204 |
+
if tok:
|
| 205 |
+
return tok.strip()
|
| 206 |
+
env = _HERE.parent / ".env"
|
| 207 |
+
if env.exists():
|
| 208 |
+
for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 209 |
+
if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
|
| 210 |
+
_, _, v = line.partition("=")
|
| 211 |
+
return v.strip().strip('"').strip("'")
|
| 212 |
+
return ""
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _get(path, tok, **params):
|
| 216 |
+
params["access_token"] = tok
|
| 217 |
+
url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
|
| 218 |
+
req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
|
| 219 |
+
try:
|
| 220 |
+
with urllib.request.urlopen(req, timeout=120) as r:
|
| 221 |
+
return json.loads(r.read().decode("utf-8", "replace"))
|
| 222 |
+
except urllib.error.HTTPError as e:
|
| 223 |
+
try:
|
| 224 |
+
msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
|
| 225 |
+
).get("message") or ""
|
| 226 |
+
except Exception:
|
| 227 |
+
msg = ""
|
| 228 |
+
raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
|
| 229 |
+
except Exception as e:
|
| 230 |
+
raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
#: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
|
| 234 |
+
#: why it cannot be treated as "the server is broken, give up".
|
| 235 |
+
_TOO_MUCH = "reduce the amount of data"
|
| 236 |
+
|
| 237 |
+
#: β THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
|
| 238 |
+
#: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
|
| 239 |
+
#: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" β the
|
| 240 |
+
#: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
|
| 241 |
+
#: β It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
|
| 242 |
+
#: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
|
| 243 |
+
#: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
|
| 244 |
+
_RATE_LIMITED = "too many calls"
|
| 245 |
+
_RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
|
| 246 |
+
_RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _walk(path, tok, log, **params):
|
| 250 |
+
"""Every page of an edge, paged by CURSOR under our own parameters.
|
| 251 |
+
|
| 252 |
+
β IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
|
| 253 |
+
first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
|
| 254 |
+
you're asking for"**, the retry at limit=25 succeeds β and then `next` carries the ORIGINAL
|
| 255 |
+
limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
|
| 256 |
+
Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
|
| 257 |
+
|
| 258 |
+
β It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
|
| 259 |
+
silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
|
| 260 |
+
every column per row.
|
| 261 |
+
|
| 262 |
+
-> (rows, hit_page_cap)
|
| 263 |
+
"""
|
| 264 |
+
out, pages = [], 0
|
| 265 |
+
limit = int(params.pop("limit", None) or PAGE)
|
| 266 |
+
after, waited = None, 0
|
| 267 |
+
while True:
|
| 268 |
+
call = dict(params, limit=limit)
|
| 269 |
+
if after:
|
| 270 |
+
call["after"] = after
|
| 271 |
+
try:
|
| 272 |
+
body = _get(path, tok, **call)
|
| 273 |
+
except MetaError as e:
|
| 274 |
+
if _TOO_MUCH in str(e) and limit > 5:
|
| 275 |
+
limit = max(5, limit // 4)
|
| 276 |
+
log(f" page too heavy for /{path} - retrying at limit={limit} "
|
| 277 |
+
f"(a payload ceiling, not a row cap; every column is still asked for)")
|
| 278 |
+
continue
|
| 279 |
+
if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
|
| 280 |
+
waited += 1
|
| 281 |
+
log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
|
| 282 |
+
f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
|
| 283 |
+
time.sleep(_RATE_WAIT)
|
| 284 |
+
continue
|
| 285 |
+
if _RATE_LIMITED in str(e).lower():
|
| 286 |
+
log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
|
| 287 |
+
f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
|
| 288 |
+
f"limit is per ad account and persists for minutes. Fix: re-run "
|
| 289 |
+
f"`--sync --only <table>` later; the upsert is idempotent.")
|
| 290 |
+
return out, True
|
| 291 |
+
raise
|
| 292 |
+
out.extend(body.get("data") or [])
|
| 293 |
+
pages += 1
|
| 294 |
+
after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
|
| 295 |
+
has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
|
| 296 |
+
if not has_next:
|
| 297 |
+
return out, False
|
| 298 |
+
if pages >= MAX_PAGES:
|
| 299 |
+
log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
|
| 300 |
+
f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
|
| 301 |
+
f"window with META_INSIGHTS_DAYS.")
|
| 302 |
+
return out, True
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def _cell(value):
|
| 306 |
+
"""One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
|
| 307 |
+
being dropped: R2 says every field the API returns, and `targeting` is a field."""
|
| 308 |
+
if value is None or isinstance(value, (str, int, float, bool)):
|
| 309 |
+
return value
|
| 310 |
+
return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _coltype(name):
|
| 314 |
+
if name in _INT:
|
| 315 |
+
return "BIGINT"
|
| 316 |
+
if name in _DBL:
|
| 317 |
+
return "DOUBLE"
|
| 318 |
+
return "VARCHAR" # ids included β see the header
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def _ensure(con, table, fields):
|
| 322 |
+
cols = ", ".join(f"{f} {_coltype(f)}" for f in fields)
|
| 323 |
+
con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
|
| 324 |
+
if "id" not in fields else
|
| 325 |
+
f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
|
| 326 |
+
have = {r[1] for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
|
| 327 |
+
for f in fields:
|
| 328 |
+
if f not in have:
|
| 329 |
+
con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {_coltype(f)}")
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def _upsert(con, table, fields, rows):
|
| 333 |
+
"""Delete-then-insert by id β the same idempotence `datastore._upsert` gives the Odoo half, so
|
| 334 |
+
a re-sync updates in place and can never append a second copy of the same object."""
|
| 335 |
+
if not rows:
|
| 336 |
+
return 0
|
| 337 |
+
cols = list(fields)
|
| 338 |
+
ids = [str(r.get("id") or "") for r in rows]
|
| 339 |
+
q = ",".join("?" for _ in ids)
|
| 340 |
+
con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
|
| 341 |
+
con.executemany(
|
| 342 |
+
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
|
| 343 |
+
[[_cell(r.get(c)) for c in cols] for r in rows])
|
| 344 |
+
return len(rows)
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def sync(tenant_key="royal-imports", tok=None, log=print, insights=True):
|
| 348 |
+
"""Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
|
| 349 |
+
|
| 350 |
+
Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
|
| 351 |
+
exactly as `odoo_relational.refresh` is.
|
| 352 |
+
"""
|
| 353 |
+
tok = tok or token()
|
| 354 |
+
report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
|
| 355 |
+
if not tok:
|
| 356 |
+
report["problems"].append(
|
| 357 |
+
"META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
|
| 358 |
+
"was pulled. This is a missing CREDENTIAL, not a missing capability.")
|
| 359 |
+
return report
|
| 360 |
+
|
| 361 |
+
path = datastore.path_for(tenant_key)
|
| 362 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 363 |
+
datastore.use_path(path)
|
| 364 |
+
con = datastore.connect()
|
| 365 |
+
log(f" mirror: {Path(path).name}")
|
| 366 |
+
|
| 367 |
+
accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
|
| 368 |
+
report["accounts"] = [a.get("id") for a in accts]
|
| 369 |
+
if not accts:
|
| 370 |
+
report["problems"].append("the token reaches no ad accounts")
|
| 371 |
+
return report
|
| 372 |
+
|
| 373 |
+
# ββ the five entity levels ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 374 |
+
for table, (edge, fields, _parent) in SPECS.items():
|
| 375 |
+
rows, capped = [], False
|
| 376 |
+
for acct in accts:
|
| 377 |
+
aid = acct["id"]
|
| 378 |
+
if edge is None:
|
| 379 |
+
rows.append(_get(aid, tok, fields=",".join(fields)))
|
| 380 |
+
else:
|
| 381 |
+
got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
|
| 382 |
+
rows.extend(got)
|
| 383 |
+
capped = capped or hit
|
| 384 |
+
_ensure(con, table, fields)
|
| 385 |
+
n = _upsert(con, table, fields, rows)
|
| 386 |
+
total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 387 |
+
report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 388 |
+
log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
|
| 389 |
+
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 390 |
+
if capped:
|
| 391 |
+
report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
|
| 392 |
+
|
| 393 |
+
# ββ the daily Insights grain ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 394 |
+
if insights:
|
| 395 |
+
rows, capped = [], False
|
| 396 |
+
for acct in accts:
|
| 397 |
+
for since, until in _slices(INSIGHTS_DAYS, INSIGHTS_SLICE):
|
| 398 |
+
got, hit = _walk(f"{acct['id']}/insights", tok, log,
|
| 399 |
+
fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
|
| 400 |
+
time_increment="1", limit=INSIGHTS_PAGE,
|
| 401 |
+
time_range=json.dumps({"since": since, "until": until}))
|
| 402 |
+
rows.extend(got)
|
| 403 |
+
capped = capped or hit
|
| 404 |
+
# β Insights rows carry no id. The key must be STABLE across runs or every re-sync
|
| 405 |
+
# appends a second copy of the same day β so it is composed from the grain itself.
|
| 406 |
+
for r in rows:
|
| 407 |
+
r["id"] = ":".join(str(r.get(k) or "") for k in
|
| 408 |
+
(f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
|
| 409 |
+
fields = ["id"] + INSIGHT_FIELDS
|
| 410 |
+
_ensure(con, INSIGHTS_TABLE, fields)
|
| 411 |
+
n = _upsert(con, INSIGHTS_TABLE, fields, rows)
|
| 412 |
+
total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
|
| 413 |
+
report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 414 |
+
log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
|
| 415 |
+
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 416 |
+
|
| 417 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 418 |
+
["meta", "done", 0, f"{INSIGHTS_DAYS}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
|
| 419 |
+
sum(t["in_mirror"] for t in report["tables"].values()),
|
| 420 |
+
time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 421 |
+
return report
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def status(tenant_key="royal-imports"):
|
| 425 |
+
"""What is in the mirror right now, per table. Never fetches."""
|
| 426 |
+
path = datastore.path_for(tenant_key)
|
| 427 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 428 |
+
datastore.use_path(path)
|
| 429 |
+
out = {}
|
| 430 |
+
con = datastore.connect()
|
| 431 |
+
for table in list(SPECS) + [INSIGHTS_TABLE]:
|
| 432 |
+
try:
|
| 433 |
+
out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 434 |
+
except Exception:
|
| 435 |
+
out[table] = None # table absent = never synced
|
| 436 |
+
return out
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def main(argv=None):
|
| 440 |
+
ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
|
| 441 |
+
ap.add_argument("--sync", action="store_true")
|
| 442 |
+
ap.add_argument("--status", action="store_true")
|
| 443 |
+
ap.add_argument("--tenant", default="royal-imports")
|
| 444 |
+
ap.add_argument("--no-insights", action="store_true")
|
| 445 |
+
a = ap.parse_args(argv)
|
| 446 |
+
if a.status:
|
| 447 |
+
for k, v in status(a.tenant).items():
|
| 448 |
+
print(f" {k:<20} {'(never synced)' if v is None else v}")
|
| 449 |
+
return 0
|
| 450 |
+
if not a.sync:
|
| 451 |
+
ap.print_help()
|
| 452 |
+
return 2
|
| 453 |
+
rep = sync(a.tenant, insights=not a.no_insights)
|
| 454 |
+
for p in rep["problems"]:
|
| 455 |
+
print(" PROBLEM:", p)
|
| 456 |
+
print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
|
| 457 |
+
return 1 if rep["problems"] else 0
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
if __name__ == "__main__":
|
| 461 |
+
sys.exit(main())
|
platform/harness/runtime.py
CHANGED
|
@@ -71,6 +71,47 @@ CONNECTOR_FLAGS_KEY = "connectors"
|
|
| 71 |
ENV_ODOO_FLAG_KEY = "odoo-env"
|
| 72 |
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
@dataclass
|
| 75 |
class TenantRuntime:
|
| 76 |
"""Everything a request needs to serve ONE tenant, and nothing about any other.
|
|
@@ -236,14 +277,81 @@ class TenantRuntime:
|
|
| 236 |
tenant's file returns real, plausible, wrong rows β no exception, no telemetry, and the
|
| 237 |
number reconciles against the wrong book. A loud failure is strictly better, and this is
|
| 238 |
the assertion EXIT-5 deletes when the connection becomes per-tenant.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
"""
|
| 240 |
want, have = self.datastore_path.resolve(), _ds.DB_PATH.resolve()
|
| 241 |
-
if want
|
| 242 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
f"tenant {self.key!r} expects the analytical store at {want}, but this process "
|
| 244 |
-
f"has {have} open. Refusing the read β a cross-tenant answer would
|
| 245 |
-
f"against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or
|
| 246 |
-
f"harness.datastore.use_path before serving this tenant.)")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
|
| 248 |
|
| 249 |
def register(key, builder):
|
|
|
|
| 71 |
ENV_ODOO_FLAG_KEY = "odoo-env"
|
| 72 |
|
| 73 |
|
| 74 |
+
class DatastoreMismatch(RuntimeError):
|
| 75 |
+
"""The process-wide DuckDB file is not the one this tenant's rows live in (W31-T45 / D-169).
|
| 76 |
+
|
| 77 |
+
β IT SUBCLASSES `RuntimeError` DELIBERATELY, AND THE REASON IS A LIVE TRAP RATHER THAN
|
| 78 |
+
TIDINESS. `harness.datastore.ro_con()` already raises a bare `RuntimeError` for *"the tenant
|
| 79 |
+
data store is completing its FIRST sync"*, and the API doors translate that into
|
| 80 |
+
`503 store_not_ready` β i.e. "wait a few minutes and retry". A cross-tenant mismatch NEVER
|
| 81 |
+
fixes itself by waiting: it is a pinned deployment serving the wrong file, and rendering it as
|
| 82 |
+
a retry banner would convert the loudest refusal in the system into a spinner.
|
| 83 |
+
|
| 84 |
+
So the type does two jobs at once. Being a `RuntimeError` keeps every existing
|
| 85 |
+
`except RuntimeError` working unchanged β including `verify_api::_guard_ok`, which is what
|
| 86 |
+
made the guard's own gate green before it had any production caller. Being a SUBCLASS lets
|
| 87 |
+
each door catch this FIRST and answer with its own status and sentence. β Order matters at
|
| 88 |
+
every catch site: a bare `except RuntimeError` placed above this one swallows it silently,
|
| 89 |
+
which is the exact shape of [[defects-that-mask-each-other]].
|
| 90 |
+
"""
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def mirror_cursor(rt):
|
| 94 |
+
"""THE door from a REQUEST to the analytical mirror: assert the tenant, THEN hand out a cursor.
|
| 95 |
+
|
| 96 |
+
ββ W31-T45 / D-169. Before this wave there were six independent `datastore.ro_con()` calls
|
| 97 |
+
across `routes_odoo_tables.py` and `odoo_relational.py` and NONE of them asserted anything,
|
| 98 |
+
while `TenantRuntime.assert_datastore_matches` β written for exactly this failure β had no
|
| 99 |
+
caller outside `verify_api.py`. Six call sites is six chances to forget; one door is one.
|
| 100 |
+
|
| 101 |
+
β THE ORDER IS THE POINT, and reversing it is the whole defect this closes. `ro_con()` opens
|
| 102 |
+
(or reuses) a cursor on the process-global `DB_PATH` and answers happily whichever tenant
|
| 103 |
+
asked β so a guard placed AFTER it has already let a cursor onto another tenant's file, and
|
| 104 |
+
any caller holding that cursor from an earlier line is unguarded regardless of what we assert
|
| 105 |
+
later. Assert first, hand out second, and there is no window.
|
| 106 |
+
|
| 107 |
+
β It raises `DatastoreMismatch` (a wrong FILE β never fixes itself) or a plain `RuntimeError`
|
| 108 |
+
(the store is mid-first-sync β retry). Callers must catch the subclass first; see its
|
| 109 |
+
docstring for why that ordering is a correctness rule rather than a style one.
|
| 110 |
+
"""
|
| 111 |
+
rt.assert_datastore_matches()
|
| 112 |
+
return _ds.ro_con()
|
| 113 |
+
|
| 114 |
+
|
| 115 |
@dataclass
|
| 116 |
class TenantRuntime:
|
| 117 |
"""Everything a request needs to serve ONE tenant, and nothing about any other.
|
|
|
|
| 277 |
tenant's file returns real, plausible, wrong rows β no exception, no telemetry, and the
|
| 278 |
number reconciles against the wrong book. A loud failure is strictly better, and this is
|
| 279 |
the assertion EXIT-5 deletes when the connection becomes per-tenant.
|
| 280 |
+
|
| 281 |
+
ββ W31-T45 / D-169 β IT HAS PRODUCTION CALLERS NOW, AND UNTIL THIS WAVE IT HAD NONE.
|
| 282 |
+
Its only caller anywhere was `verify_api.py`, i.e. the guard existed, was correct, was
|
| 283 |
+
tested, and governed nothing β while six mirror reads in `routes_odoo_tables.py` and
|
| 284 |
+
`odoo_relational.py` asserted nothing at all. What held the boundary was not this code but
|
| 285 |
+
a fact about the CUSTOMER LIST ("only tenant #0 has mirror databases"), and R2's Meta Ads
|
| 286 |
+
mirror for GTM Lab ends that fact. Every door now goes through `mirror_cursor()` below.
|
| 287 |
+
β This raises the priority of the D-29 / D-40 / D-66 tenant-residency family: they are the
|
| 288 |
+
rest of the same boundary, and this guard is one process-global away from them.
|
| 289 |
+
|
| 290 |
+
ββ THE PREDICATE IS "DOES THE OPEN FILE BELONG TO SOMEBODY ELSE", NOT "IS IT MY CANONICAL
|
| 291 |
+
PATH", and the difference was found by MOUNTING it rather than by reading it. A bare
|
| 292 |
+
`want != have` reads as the stricter, safer rule β and it refuses a case that cannot leak
|
| 293 |
+
anything: a process pinned by `datastore.use_path()` at a store no tenant's naming
|
| 294 |
+
convention produces (a provisioning script, a single-tenant worker, every hermetic gate
|
| 295 |
+
fixture in this repo). Nobody else reads that file, so there is no other book to reconcile
|
| 296 |
+
against. Mounted with the bare rule, `verify_scopes::section_read_through` went from
|
| 297 |
+
320/320 to a 409 on its first leg β the fixture was not wrong, the predicate was.
|
| 298 |
+
|
| 299 |
+
So: same path β fine, and that is the hot path with NO store read. Different path β ask
|
| 300 |
+
`known_tenants()` who owns the open one. A DIFFERENT tenant is the D-169 failure and
|
| 301 |
+
raises. NOBODY is a bespoke pin and is allowed, deliberately and narrowly.
|
| 302 |
+
|
| 303 |
+
β AND THE UNKNOWN-OWNER BRANCH IS FAIL-CLOSED, which is the half that is easy to get
|
| 304 |
+
backwards. "No known tenant owns this file" and "the tenant registry could not be read"
|
| 305 |
+
produce the same empty answer from a forward map, and they mean opposite things: the first
|
| 306 |
+
is a bespoke pin, the second is every tenant looking unowned β including the one whose
|
| 307 |
+
rows are open. A store blip must not turn this guard off, so an unreadable registry raises.
|
| 308 |
"""
|
| 309 |
want, have = self.datastore_path.resolve(), _ds.DB_PATH.resolve()
|
| 310 |
+
if want == have:
|
| 311 |
+
return
|
| 312 |
+
owner, registry_ok = self._store_owner(have)
|
| 313 |
+
if owner == self.key:
|
| 314 |
+
return
|
| 315 |
+
if owner or not registry_ok:
|
| 316 |
+
whose = (f"which belongs to tenant {owner!r}" if owner else
|
| 317 |
+
"and the tenant registry could not be read, so ownership is UNKNOWN "
|
| 318 |
+
"(refusing rather than guessing)")
|
| 319 |
+
raise DatastoreMismatch(
|
| 320 |
f"tenant {self.key!r} expects the analytical store at {want}, but this process "
|
| 321 |
+
f"has {have} open, {whose}. Refusing the read β a cross-tenant answer would "
|
| 322 |
+
f"reconcile against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or "
|
| 323 |
+
f"call harness.datastore.use_path before serving this tenant.)")
|
| 324 |
+
|
| 325 |
+
@staticmethod
|
| 326 |
+
def _store_owner(path):
|
| 327 |
+
"""`(slug_or_empty, registry_was_readable)` for the DuckDB file at `path`.
|
| 328 |
+
|
| 329 |
+
β FORWARD-MAPPED, never parsed out of the filename β `datastore.path_for` substitutes `_`
|
| 330 |
+
for every non-alphanumeric character and truncates at 60, so `a.b` and `a_b` produce the
|
| 331 |
+
same file and the name is genuinely not invertible. `tenant_for_store_path` above says the
|
| 332 |
+
same thing and is not reused here for one reason: it swallows every failure into `""`, and
|
| 333 |
+
this caller must be able to tell "nobody owns it" from "I could not find out".
|
| 334 |
+
|
| 335 |
+
β AND `known_tenants()` IS NOT REUSED EITHER, FOR THE SAME REASON ONE LAYER DOWN β it
|
| 336 |
+
cannot raise: `_tenant_records()` returns `{}` on any failure and `known_tenants` wraps
|
| 337 |
+
that in another `except: pass`. Calling it here would have made `registry_ok` a constant
|
| 338 |
+
`True` and the fail-closed branch above unreachable, i.e. a guard whose most important
|
| 339 |
+
clause is dead code that reads as if it fires ([[gate-can-report-green-on-nothing]]). The
|
| 340 |
+
control-plane bucket is therefore read HERE, where its failure is still visible.
|
| 341 |
+
"""
|
| 342 |
+
keys, registry_ok = set(_BUILDERS), True
|
| 343 |
+
try:
|
| 344 |
+
recs = store.get(TENANTS_KEY) or {}
|
| 345 |
+
keys |= {str(k).strip().lower() for k, v in recs.items() if isinstance(v, dict)}
|
| 346 |
+
except Exception: # noqa: BLE001
|
| 347 |
+
registry_ok = False
|
| 348 |
+
for key in sorted(keys):
|
| 349 |
+
try:
|
| 350 |
+
if _ds.path_for(key).resolve() == path:
|
| 351 |
+
return key, registry_ok
|
| 352 |
+
except Exception: # noqa: BLE001
|
| 353 |
+
continue
|
| 354 |
+
return "", registry_ok
|
| 355 |
|
| 356 |
|
| 357 |
def register(key, builder):
|
web/src/alerts/AlertsPane.tsx
CHANGED
|
@@ -20,11 +20,12 @@ import {
|
|
| 20 |
runAlert,
|
| 21 |
} from "./alertsApi";
|
| 22 |
import {
|
| 23 |
-
EMPTY_INBOX,
|
| 24 |
applyRead,
|
|
|
|
| 25 |
inboxOrder,
|
| 26 |
isAutomationOutcome,
|
| 27 |
isAutomationReview,
|
|
|
|
| 28 |
routeForTopic,
|
| 29 |
stampText,
|
| 30 |
} from "./alertsModel";
|
|
@@ -47,6 +48,7 @@ export default function AlertsPane({
|
|
| 47 |
onOpenView,
|
| 48 |
onOpenAutomation,
|
| 49 |
onInbox,
|
|
|
|
| 50 |
onToast,
|
| 51 |
}: {
|
| 52 |
onClose: () => void;
|
|
@@ -65,11 +67,29 @@ export default function AlertsPane({
|
|
| 65 |
/** Hand the freshly-read inbox back so the nav badge and the pane agree. */
|
| 66 |
onInbox: (inbox: Inbox) => void;
|
| 67 |
onToast: (message: string) => void;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
}) {
|
| 69 |
-
const [inbox, setInbox] = useState<Inbox>(
|
| 70 |
const [alerts, setAlerts] = useState<Alert[]>([]);
|
| 71 |
const [error, setError] = useState("");
|
| 72 |
const [busy, setBusy] = useState(false);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
const publish = useCallback(
|
| 75 |
(next: Inbox) => {
|
|
@@ -81,8 +101,18 @@ export default function AlertsPane({
|
|
| 81 |
|
| 82 |
const load = useCallback(async () => {
|
| 83 |
const [inboxRes, alertRes] = await Promise.all([fetchInbox(), fetchAlerts()]);
|
| 84 |
-
if (inboxRes.ok)
|
| 85 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
if (alertRes.ok) setAlerts(alertRes.value);
|
| 87 |
}, [publish]);
|
| 88 |
|
|
@@ -120,7 +150,13 @@ export default function AlertsPane({
|
|
| 120 |
|
| 121 |
const markAll = useCallback(() => {
|
| 122 |
const before = inbox;
|
| 123 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
void markRead(null, true).then((r) => {
|
| 125 |
if (!r.ok) {
|
| 126 |
publish(before);
|
|
@@ -167,6 +203,9 @@ export default function AlertsPane({
|
|
| 167 |
);
|
| 168 |
|
| 169 |
const rows = inboxOrder(inbox.items);
|
|
|
|
|
|
|
|
|
|
| 170 |
|
| 171 |
return (
|
| 172 |
<div className="shell-newdb-scrim" onClick={onClose}>
|
|
@@ -183,7 +222,12 @@ export default function AlertsPane({
|
|
| 183 |
<button
|
| 184 |
type="button"
|
| 185 |
className="alerts-markall"
|
| 186 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
onClick={markAll}
|
| 188 |
>
|
| 189 |
Mark all read
|
|
@@ -193,10 +237,22 @@ export default function AlertsPane({
|
|
| 193 |
{error ? <p className="shell-newdb-err">{error}</p> : null}
|
| 194 |
|
| 195 |
<div className="alerts-scroll">
|
| 196 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
<p className="alerts-empty">
|
| 198 |
Nothing new. An alert watches ONE view and tells you when a record it had
|
| 199 |
-
never matched arrives in it β make one from a view
|
| 200 |
</p>
|
| 201 |
) : (
|
| 202 |
<ul className="alerts-list">
|
|
|
|
| 20 |
runAlert,
|
| 21 |
} from "./alertsApi";
|
| 22 |
import {
|
|
|
|
| 23 |
applyRead,
|
| 24 |
+
canMarkAll,
|
| 25 |
inboxOrder,
|
| 26 |
isAutomationOutcome,
|
| 27 |
isAutomationReview,
|
| 28 |
+
paneView,
|
| 29 |
routeForTopic,
|
| 30 |
stampText,
|
| 31 |
} from "./alertsModel";
|
|
|
|
| 48 |
onOpenView,
|
| 49 |
onOpenAutomation,
|
| 50 |
onInbox,
|
| 51 |
+
seed,
|
| 52 |
onToast,
|
| 53 |
}: {
|
| 54 |
onClose: () => void;
|
|
|
|
| 67 |
/** Hand the freshly-read inbox back so the nav badge and the pane agree. */
|
| 68 |
onInbox: (inbox: Inbox) => void;
|
| 69 |
onToast: (message: string) => void;
|
| 70 |
+
/**
|
| 71 |
+
* ββ WAVE 31 Β· T23 (owner item 1) β THE FRAME'S OWN INBOX, AS THE SEED.
|
| 72 |
+
*
|
| 73 |
+
* β REQUIRED, NOT OPTIONAL, for the reason `onOpenAutomation` above is: an optional seed the
|
| 74 |
+
* frame forgot to pass degrades to `EMPTY_INBOX`, which is precisely the shipped bug, and no
|
| 75 |
+
* gate would see it. `tsc` fails the moment it is unmounted.
|
| 76 |
+
*
|
| 77 |
+
* The frame has ALREADY fetched this β it is what paints the rail badge the user just clicked.
|
| 78 |
+
* Starting from it means the panel opens knowing the count, so "Mark all read" is live from
|
| 79 |
+
* the first frame and stays live if this pane's own read fails.
|
| 80 |
+
*/
|
| 81 |
+
seed: Inbox;
|
| 82 |
}) {
|
| 83 |
+
const [inbox, setInbox] = useState<Inbox>(seed);
|
| 84 |
const [alerts, setAlerts] = useState<Alert[]>([]);
|
| 85 |
const [error, setError] = useState("");
|
| 86 |
const [busy, setBusy] = useState(false);
|
| 87 |
+
/**
|
| 88 |
+
* β THE STATE THE PANE DID NOT HAVE. Without it, "no rows yet" and "no rows, ever" are the
|
| 89 |
+
* same value, so an in-flight read β **3,280 ms live on tenant #0** β printed a confident
|
| 90 |
+
* "Nothing new." over an inbox nobody had looked at yet.
|
| 91 |
+
*/
|
| 92 |
+
const [phase, setPhase] = useState<"pending" | "ready" | "error">("pending");
|
| 93 |
|
| 94 |
const publish = useCallback(
|
| 95 |
(next: Inbox) => {
|
|
|
|
| 101 |
|
| 102 |
const load = useCallback(async () => {
|
| 103 |
const [inboxRes, alertRes] = await Promise.all([fetchInbox(), fetchAlerts()]);
|
| 104 |
+
if (inboxRes.ok) {
|
| 105 |
+
publish(inboxRes.value);
|
| 106 |
+
setError("");
|
| 107 |
+
setPhase("ready");
|
| 108 |
+
} else {
|
| 109 |
+
// β THE SEEDED INBOX SURVIVES A FAILED READ. Blanking it here would take the count β and
|
| 110 |
+
// with it the Mark-all control β away from a reader who can see the badge, which is the
|
| 111 |
+
// owner's second symptom. The failure is REPORTED (the message line below) and nothing the
|
| 112 |
+
// frame already knew is discarded.
|
| 113 |
+
setError(inboxRes.message);
|
| 114 |
+
setPhase("error");
|
| 115 |
+
}
|
| 116 |
if (alertRes.ok) setAlerts(alertRes.value);
|
| 117 |
}, [publish]);
|
| 118 |
|
|
|
|
| 150 |
|
| 151 |
const markAll = useCallback(() => {
|
| 152 |
const before = inbox;
|
| 153 |
+
// β W31-T23 β `applyRead(β¦, null, true)` zeroes the count from the ITEMS, which is right when
|
| 154 |
+
// we have the page and wrong when we do not: a failed read leaves `items` empty while the
|
| 155 |
+
// account really does have unread notifications, and the optimistic copy would still say 0.
|
| 156 |
+
// It does β and that is CORRECT here, because the request below clears the whole account
|
| 157 |
+
// (`ids: null`), so 0 is what the server is about to make true. The revert on failure is what
|
| 158 |
+
// keeps it honest.
|
| 159 |
+
publish({ ...applyRead(inbox, null, true), unread: 0 });
|
| 160 |
void markRead(null, true).then((r) => {
|
| 161 |
if (!r.ok) {
|
| 162 |
publish(before);
|
|
|
|
| 203 |
);
|
| 204 |
|
| 205 |
const rows = inboxOrder(inbox.items);
|
| 206 |
+
// β W31-T23 β the decision lives in `alertsModel` so `verify_alerts` can RUN it (this file is
|
| 207 |
+
// TSX and the gate's harness is node); this line is the mount.
|
| 208 |
+
const view = paneView(phase, rows.length, seed.unread);
|
| 209 |
|
| 210 |
return (
|
| 211 |
<div className="shell-newdb-scrim" onClick={onClose}>
|
|
|
|
| 222 |
<button
|
| 223 |
type="button"
|
| 224 |
className="alerts-markall"
|
| 225 |
+
// ββ W31-T23 (R6) β THE CONTROL THAT COULD NOT BE PRESSED. `inbox.unread === 0` was
|
| 226 |
+
// this pane's OWN state, zero until its fetch landed and zero for ever if it failed,
|
| 227 |
+
// so the one verb that clears the badge was disabled in exactly the two situations
|
| 228 |
+
// the owner reported. The seeded count is the account's, so the button is live from
|
| 229 |
+
// the first frame.
|
| 230 |
+
disabled={!canMarkAll(inbox.unread, busy)}
|
| 231 |
onClick={markAll}
|
| 232 |
>
|
| 233 |
Mark all read
|
|
|
|
| 237 |
{error ? <p className="shell-newdb-err">{error}</p> : null}
|
| 238 |
|
| 239 |
<div className="alerts-scroll">
|
| 240 |
+
{view === "pending" ? (
|
| 241 |
+
/* R6: the mark, never the word. `verify_icons` asserts this product ships no load
|
| 242 |
+
TEXT β and the sentence that used to occupy this space was worse than a word, it
|
| 243 |
+
was an ANSWER ("Nothing new.") to a question nobody had asked yet. */
|
| 244 |
+
<p className="alerts-empty" aria-busy="true">
|
| 245 |
+
<span className="lp-spin" role="status" aria-label="Loading alerts" />
|
| 246 |
+
</p>
|
| 247 |
+
) : view === "error" ? (
|
| 248 |
+
<p className="alerts-empty">
|
| 249 |
+
These notifications could not be loaded. The count beside the bell is still this
|
| 250 |
+
account’s β “Mark all read” above clears it.
|
| 251 |
+
</p>
|
| 252 |
+
) : view === "empty" ? (
|
| 253 |
<p className="alerts-empty">
|
| 254 |
Nothing new. An alert watches ONE view and tells you when a record it had
|
| 255 |
+
never matched arrives in it β make one from a view’s Β·Β·Β· menu.
|
| 256 |
</p>
|
| 257 |
) : (
|
| 258 |
<ul className="alerts-list">
|
web/src/alerts/alertsModel.ts
CHANGED
|
@@ -78,6 +78,61 @@ export interface Inbox {
|
|
| 78 |
|
| 79 |
export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
|
| 80 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
const str = (v: unknown): string => (typeof v === "string" ? v : "");
|
| 82 |
const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
|
| 83 |
|
|
|
|
| 78 |
|
| 79 |
export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
|
| 80 |
|
| 81 |
+
/**
|
| 82 |
+
* ββ WAVE 31 Β· T23 (owner item 1) β WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
|
| 83 |
+
*
|
| 84 |
+
* Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
|
| 85 |
+
* remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
|
| 86 |
+
* `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
|
| 87 |
+
* and `GET /notifications` answering β measured at **3,280 ms live** β and for ever after a
|
| 88 |
+
* failed fetch:
|
| 89 |
+
* Β· `items` was `[]`, so the pane printed **"Nothing new."** β a claim, not a wait;
|
| 90 |
+
* Β· `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
|
| 91 |
+
* already loaded could never be cleared.
|
| 92 |
+
* A confident sentence about somebody's inbox, and the one control that would fix it, both
|
| 93 |
+
* switched off by the same uninitialised state.
|
| 94 |
+
*
|
| 95 |
+
* β IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
|
| 96 |
+
* compiles and RUNS this module under node; markup it cannot reach is markup no control can
|
| 97 |
+
* mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
|
| 98 |
+
*/
|
| 99 |
+
export type PaneView = "pending" | "rows" | "empty" | "error";
|
| 100 |
+
|
| 101 |
+
/**
|
| 102 |
+
* `pending` while the first read is in flight Β· `error` when it failed and we have nothing to
|
| 103 |
+
* show Β· `rows` when there is something Β· `empty` ONLY when a successful read returned nothing.
|
| 104 |
+
*
|
| 105 |
+
* β ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
|
| 106 |
+
* the pane already holds notifications should not blank them β the reader loses real information
|
| 107 |
+
* to a transient. The error still reaches them as the pane's message line.
|
| 108 |
+
*/
|
| 109 |
+
export function paneView(
|
| 110 |
+
phase: "pending" | "ready" | "error",
|
| 111 |
+
rowCount: number,
|
| 112 |
+
seededUnread = 0
|
| 113 |
+
): PaneView {
|
| 114 |
+
if (rowCount > 0) return "rows";
|
| 115 |
+
if (phase === "pending") return "pending";
|
| 116 |
+
if (phase === "error") return "error";
|
| 117 |
+
// β A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
|
| 118 |
+
// there is something; the page we were given does not contain it. Saying "Nothing new" there
|
| 119 |
+
// is the same false confidence in a different costume.
|
| 120 |
+
return seededUnread > 0 ? "error" : "empty";
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
/**
|
| 124 |
+
* May "Mark all read" be pressed?
|
| 125 |
+
*
|
| 126 |
+
* β NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state β zero
|
| 127 |
+
* until its fetch lands, zero for ever if the fetch fails β so the control was dead in exactly
|
| 128 |
+
* the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
|
| 129 |
+
* the frame already holds, and a failed read does not take the verb away: `POST /notifications/
|
| 130 |
+
* read` with `ids: null` clears the account's inbox whether or not we managed to list it.
|
| 131 |
+
*/
|
| 132 |
+
export function canMarkAll(unread: number, busy = false): boolean {
|
| 133 |
+
return !busy && Math.max(0, Math.floor(unread)) > 0;
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
const str = (v: unknown): string => (typeof v === "string" ? v : "");
|
| 137 |
const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
|
| 138 |
|
web/src/automation/AutomationBuilder.tsx
CHANGED
|
@@ -591,6 +591,28 @@ export default function AutomationBuilder({
|
|
| 591 |
: null;
|
| 592 |
if (kind === "find_records")
|
| 593 |
return firstTable ? { table: firstTable.key, cond: null, limit: 25 } : null;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 594 |
/*
|
| 595 |
ββ 2026-08-07 (owner report) β THE ENRICH ACTION HAD NO BRANCH HERE, SO IT WAS
|
| 596 |
PERMANENTLY UNCLICKABLE. `disabled={!c.ready || busy || !seedFor(c.kind)}` and this
|
|
|
|
| 591 |
: null;
|
| 592 |
if (kind === "find_records")
|
| 593 |
return firstTable ? { table: firstTable.key, cond: null, limit: 25 } : null;
|
| 594 |
+
/*
|
| 595 |
+
ββ WAVE 31 Β· T38 (contract C5, owner ruling R10) β THE WEB READ, and this branch is here
|
| 596 |
+
because a gate demanded it rather than because anybody remembered.
|
| 597 |
+
|
| 598 |
+
β FLIPPING `ready:True` ON THE SERVER IS NOT MOUNTING AN ACTION. `verify_automation_ui`'s
|
| 599 |
+
seed check DERIVES its subject from the server catalog, so the moment `web_read` became
|
| 600 |
+
ready it went red with `MISSING: ['web_read']` β the action would have rendered faded,
|
| 601 |
+
labelled "Needs a database" (a claim about databases, which is not the problem) and been
|
| 602 |
+
permanently unclickable. That is D-79's scar exactly, and the check written after it is what
|
| 603 |
+
caught the repeat. A capability is reachable when the SERVER offers it, the ENGINE dispatches
|
| 604 |
+
it and the CLIENT can seed it; two out of three ships an invisible feature.
|
| 605 |
+
|
| 606 |
+
β THE SEED IS DELIBERATELY BLANK, and the server was changed to accept that. `url`,
|
| 607 |
+
`selector` and `field` are what a person types in the panel, so requiring them at validation
|
| 608 |
+
time would make a freshly-added step refuse itself β the wave-23 illegal-default-seed defect.
|
| 609 |
+
`_clean_action_config` stores an unconfigured step and the RUNNER fails closed with a
|
| 610 |
+
sentence naming what is missing, which is the same posture the enrich arm takes with
|
| 611 |
+
`profileField`. The values below mirror that validator's defaults exactly.
|
| 612 |
+
*/
|
| 613 |
+
if (kind === "web_read")
|
| 614 |
+
return { url: "", selector: "", field: firstField || "",
|
| 615 |
+
attr: "text", all: false, timeoutMs: 20000 };
|
| 616 |
/*
|
| 617 |
ββ 2026-08-07 (owner report) β THE ENRICH ACTION HAD NO BRANCH HERE, SO IT WAS
|
| 618 |
PERMANENTLY UNCLICKABLE. `disabled={!c.ready || busy || !seedFor(c.kind)}` and this
|
web/src/automation/AutomationDetail.tsx
CHANGED
|
@@ -112,7 +112,14 @@ interface Props {
|
|
| 112 |
* forever.
|
| 113 |
*/
|
| 114 |
liveStep: string;
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
onDeleted: () => void | Promise<void>;
|
| 117 |
}
|
| 118 |
|
|
@@ -587,6 +594,11 @@ export default function AutomationDetail({
|
|
| 587 |
return { targetTable };
|
| 588 |
};
|
| 589 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 590 |
const persist = async (announce = true, sched?: { cron: string; enabled: boolean }) => {
|
| 591 |
const res = await patchAutomation(automation.id, {
|
| 592 |
name,
|
|
@@ -596,7 +608,7 @@ export default function AutomationDetail({
|
|
| 596 |
});
|
| 597 |
setTouched(false);
|
| 598 |
if (announce) setMessage(heldNote() || "Saved.");
|
| 599 |
-
return res.automation
|
| 600 |
};
|
| 601 |
|
| 602 |
/**
|
|
@@ -616,7 +628,7 @@ export default function AutomationDetail({
|
|
| 616 |
setMessage("");
|
| 617 |
setCron(next.cron);
|
| 618 |
try {
|
| 619 |
-
await patchAutomation(automation.id, {
|
| 620 |
name,
|
| 621 |
kind,
|
| 622 |
config: buildConfig(),
|
|
@@ -624,7 +636,7 @@ export default function AutomationDetail({
|
|
| 624 |
});
|
| 625 |
setTouched(false);
|
| 626 |
setMessage(heldNote());
|
| 627 |
-
await onSaved(
|
| 628 |
} catch (e) {
|
| 629 |
setProblem(e instanceof AutomationError ? e.message : "That schedule was not saved.");
|
| 630 |
} finally {
|
|
@@ -652,7 +664,7 @@ export default function AutomationDetail({
|
|
| 652 |
setMessage("");
|
| 653 |
if (key !== "schedule") setShowCron(false);
|
| 654 |
try {
|
| 655 |
-
await patchAutomation(automation.id, {
|
| 656 |
name,
|
| 657 |
kind,
|
| 658 |
config: buildConfig(),
|
|
@@ -661,7 +673,7 @@ export default function AutomationDetail({
|
|
| 661 |
});
|
| 662 |
setTouched(false);
|
| 663 |
setMessage(heldNote());
|
| 664 |
-
await onSaved(
|
| 665 |
} catch (e) {
|
| 666 |
setProblem(e instanceof AutomationError ? e.message : "That trigger was not saved.");
|
| 667 |
} finally {
|
|
@@ -684,18 +696,41 @@ export default function AutomationDetail({
|
|
| 684 |
* permission rule, and the two copies disagreeing leaves a card that will not
|
| 685 |
* move for a reason nobody can read.
|
| 686 |
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 687 |
const save = async () => {
|
| 688 |
setSaving(true);
|
| 689 |
setProblem("");
|
| 690 |
setMessage("");
|
|
|
|
| 691 |
try {
|
| 692 |
-
|
| 693 |
-
await onSaved(id);
|
| 694 |
} catch (e) {
|
| 695 |
setProblem(e instanceof AutomationError ? e.message : "That could not be saved.");
|
|
|
|
| 696 |
} finally {
|
|
|
|
| 697 |
setSaving(false);
|
| 698 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 699 |
};
|
| 700 |
|
| 701 |
/**
|
|
@@ -708,8 +743,11 @@ export default function AutomationDetail({
|
|
| 708 |
setMessage("");
|
| 709 |
try {
|
| 710 |
if (touched) await persist(false);
|
| 711 |
-
|
| 712 |
-
|
|
|
|
|
|
|
|
|
|
| 713 |
} catch (e) {
|
| 714 |
setProblem(e instanceof AutomationError ? e.message : "That step was not changed.");
|
| 715 |
}
|
|
@@ -830,8 +868,8 @@ export default function AutomationDetail({
|
|
| 830 |
setProblem("");
|
| 831 |
setMessage("");
|
| 832 |
try {
|
| 833 |
-
await patchAutomation(automation.id, body);
|
| 834 |
-
await onSaved(
|
| 835 |
} catch (e) {
|
| 836 |
// β THE SERVER'S SENTENCE, VERBATIM. `clean_flow` refuses an empty group, an unready
|
| 837 |
// action kind, a valueless compare β each with a reason worth reading. A client that
|
|
|
|
| 112 |
* forever.
|
| 113 |
*/
|
| 114 |
liveStep: string;
|
| 115 |
+
/**
|
| 116 |
+
* β W31-T31 β `fresh` is the definition the WRITE JUST RETURNED, when the caller has one.
|
| 117 |
+
* Given it, the surface corrects that one row; without it, it falls back to a full reload. Every
|
| 118 |
+
* mutation door here already had the response in hand and was discarding it, which is what made
|
| 119 |
+
* a save cost two round trips. Optional on purpose: the doors that genuinely do not have a fresh
|
| 120 |
+
* definition (a delete, a run) still reload, and must.
|
| 121 |
+
*/
|
| 122 |
+
onSaved: (id?: string, fresh?: Automation) => void | Promise<void>;
|
| 123 |
onDeleted: () => void | Promise<void>;
|
| 124 |
}
|
| 125 |
|
|
|
|
| 594 |
return { targetTable };
|
| 595 |
};
|
| 596 |
|
| 597 |
+
/**
|
| 598 |
+
* β W31-T31 β RETURNS THE FRESH AUTOMATION, not just its id. The PATCH response already carries
|
| 599 |
+
* the whole updated definition; handing it to `onSaved` is what lets the surface correct one row
|
| 600 |
+
* instead of re-reading the entire list to learn what it was just told.
|
| 601 |
+
*/
|
| 602 |
const persist = async (announce = true, sched?: { cron: string; enabled: boolean }) => {
|
| 603 |
const res = await patchAutomation(automation.id, {
|
| 604 |
name,
|
|
|
|
| 608 |
});
|
| 609 |
setTouched(false);
|
| 610 |
if (announce) setMessage(heldNote() || "Saved.");
|
| 611 |
+
return res.automation;
|
| 612 |
};
|
| 613 |
|
| 614 |
/**
|
|
|
|
| 628 |
setMessage("");
|
| 629 |
setCron(next.cron);
|
| 630 |
try {
|
| 631 |
+
const { automation: fresh } = await patchAutomation(automation.id, {
|
| 632 |
name,
|
| 633 |
kind,
|
| 634 |
config: buildConfig(),
|
|
|
|
| 636 |
});
|
| 637 |
setTouched(false);
|
| 638 |
setMessage(heldNote());
|
| 639 |
+
await onSaved(fresh.id, fresh); // W31-T31: the response, not a second read
|
| 640 |
} catch (e) {
|
| 641 |
setProblem(e instanceof AutomationError ? e.message : "That schedule was not saved.");
|
| 642 |
} finally {
|
|
|
|
| 664 |
setMessage("");
|
| 665 |
if (key !== "schedule") setShowCron(false);
|
| 666 |
try {
|
| 667 |
+
const { automation: fresh } = await patchAutomation(automation.id, {
|
| 668 |
name,
|
| 669 |
kind,
|
| 670 |
config: buildConfig(),
|
|
|
|
| 673 |
});
|
| 674 |
setTouched(false);
|
| 675 |
setMessage(heldNote());
|
| 676 |
+
await onSaved(fresh.id, fresh); // W31-T31: the response, not a second read
|
| 677 |
} catch (e) {
|
| 678 |
setProblem(e instanceof AutomationError ? e.message : "That trigger was not saved.");
|
| 679 |
} finally {
|
|
|
|
| 696 |
* permission rule, and the two copies disagreeing leaves a card that will not
|
| 697 |
* move for a reason nobody can read.
|
| 698 |
*/
|
| 699 |
+
/**
|
| 700 |
+
* ββ WAVE 31 Β· T31 β "SAVINGβ¦" IS THE LIFETIME OF THE WRITE, AND NOTHING ELSE.
|
| 701 |
+
*
|
| 702 |
+
* β THE DEFECT, in the owner's words: *"It just say Saving... and takes a long time for me to
|
| 703 |
+
* change options and configs etc."* This function used to hold `saving` across `persist()` AND
|
| 704 |
+
* `onSaved()` β and `onSaved` reloads the whole list. So the label a person reads as "your edit is
|
| 705 |
+
* being written" was in fact reporting a write plus an unrelated full `GET /automations`, and the
|
| 706 |
+
* server-side half of that read is the one W31-T30 measured at seconds. Two round trips wearing
|
| 707 |
+
* one word.
|
| 708 |
+
*
|
| 709 |
+
* β AND THE SPLIT IS A CORRECTNESS FIX, NOT ONLY A SPEED ONE. The old `try` wrapped both calls in
|
| 710 |
+
* one `catch` that printed *"That could not be saved."* β so a save that SUCCEEDED and whose list
|
| 711 |
+
* refresh then failed told the person their work was lost while it sat safely in the store. The
|
| 712 |
+
* inverse of the bug being fixed, and the worse of the two. The write's failure is reported; the
|
| 713 |
+
* refresh's failure is not allowed to speak for it.
|
| 714 |
+
*/
|
| 715 |
const save = async () => {
|
| 716 |
setSaving(true);
|
| 717 |
setProblem("");
|
| 718 |
setMessage("");
|
| 719 |
+
let fresh: Automation | undefined;
|
| 720 |
try {
|
| 721 |
+
fresh = await persist();
|
|
|
|
| 722 |
} catch (e) {
|
| 723 |
setProblem(e instanceof AutomationError ? e.message : "That could not be saved.");
|
| 724 |
+
return;
|
| 725 |
} finally {
|
| 726 |
+
// β HERE, not after `onSaved`. This is the whole ticket.
|
| 727 |
setSaving(false);
|
| 728 |
}
|
| 729 |
+
try {
|
| 730 |
+
await onSaved(fresh.id, fresh);
|
| 731 |
+
} catch {
|
| 732 |
+
/* The save is already committed β see the note above. A failed refresh is not a failed save. */
|
| 733 |
+
}
|
| 734 |
};
|
| 735 |
|
| 736 |
/**
|
|
|
|
| 743 |
setMessage("");
|
| 744 |
try {
|
| 745 |
if (touched) await persist(false);
|
| 746 |
+
// β W31-T31 β the toggle RESPONSE is the new definition. Passing it on means a node switch
|
| 747 |
+
// costs the toggle itself, where it used to cost a toggle plus a whole-list read (plus a
|
| 748 |
+
// PATCH when the panel was dirty β three round trips for one click).
|
| 749 |
+
const { automation: fresh } = await toggleNode(automation.id, nodeId);
|
| 750 |
+
await onSaved(fresh.id, fresh);
|
| 751 |
} catch (e) {
|
| 752 |
setProblem(e instanceof AutomationError ? e.message : "That step was not changed.");
|
| 753 |
}
|
|
|
|
| 868 |
setProblem("");
|
| 869 |
setMessage("");
|
| 870 |
try {
|
| 871 |
+
const { automation: fresh } = await patchAutomation(automation.id, body);
|
| 872 |
+
await onSaved(fresh.id, fresh); // W31-T31: the response, not a second read
|
| 873 |
} catch (e) {
|
| 874 |
// β THE SERVER'S SENTENCE, VERBATIM. `clean_flow` refuses an empty group, an unready
|
| 875 |
// action kind, a valueless compare β each with a reason worth reading. A client that
|
web/src/automation/AutomationSurface.tsx
CHANGED
|
@@ -34,6 +34,7 @@ import {
|
|
| 34 |
listAutomations,
|
| 35 |
liveStepOf,
|
| 36 |
patchAutomation,
|
|
|
|
| 37 |
} from "./automationApi";
|
| 38 |
|
| 39 |
/**
|
|
@@ -284,6 +285,27 @@ export default function AutomationSurface() {
|
|
| 284 |
}
|
| 285 |
}, []);
|
| 286 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
useEffect(() => {
|
| 288 |
const ac = new AbortController();
|
| 289 |
void load(ac.signal);
|
|
@@ -653,7 +675,18 @@ export default function AutomationSurface() {
|
|
| 653 |
* the user asked for, which makes the write a no-op in the good case and nothing
|
| 654 |
* at all in the bad one.
|
| 655 |
*/
|
| 656 |
-
onSaved={async (id) => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 657 |
const next = await load();
|
| 658 |
if (id && next && id === wantedId.current) select(id);
|
| 659 |
}}
|
|
|
|
| 34 |
listAutomations,
|
| 35 |
liveStepOf,
|
| 36 |
patchAutomation,
|
| 37 |
+
rememberAutomation,
|
| 38 |
} from "./automationApi";
|
| 39 |
|
| 40 |
/**
|
|
|
|
| 285 |
}
|
| 286 |
}, []);
|
| 287 |
|
| 288 |
+
/**
|
| 289 |
+
* ββ WAVE 31 Β· T31 β one freshly-written automation replaces its own row, with no round trip.
|
| 290 |
+
*
|
| 291 |
+
* β IT TAKES THE GENERATION, and that is not decoration. `load`'s guard exists so a stale READ
|
| 292 |
+
* cannot repaint; a WRITE response is by definition newer than any read already in flight, so it
|
| 293 |
+
* must win the same race rather than sit outside it. Without the bump, a `GET /automations`
|
| 294 |
+
* issued before the save could resolve after the merge and paint the pre-save row back β the
|
| 295 |
+
* exact ghost `gen` was introduced for, arriving through the new door.
|
| 296 |
+
*
|
| 297 |
+
* β Returns FALSE when there is nothing to merge into (a cold surface has no memo), and the
|
| 298 |
+
* caller then does a real load. Painting a one-row list would be worse than a round trip.
|
| 299 |
+
*/
|
| 300 |
+
const mergeSaved = useCallback((fresh: Automation) => {
|
| 301 |
+
const next = rememberAutomation(fresh);
|
| 302 |
+
if (!next) return false;
|
| 303 |
+
gen.current += 1;
|
| 304 |
+
setData(next);
|
| 305 |
+
setError("");
|
| 306 |
+
return true;
|
| 307 |
+
}, []);
|
| 308 |
+
|
| 309 |
useEffect(() => {
|
| 310 |
const ac = new AbortController();
|
| 311 |
void load(ac.signal);
|
|
|
|
| 675 |
* the user asked for, which makes the write a no-op in the good case and nothing
|
| 676 |
* at all in the bad one.
|
| 677 |
*/
|
| 678 |
+
onSaved={async (id, fresh) => {
|
| 679 |
+
// ββ WAVE 31 Β· T31 β CORRECT THE ONE ROW WE WERE JUST TOLD ABOUT.
|
| 680 |
+
// `patchAutomation` and `toggleNode` return the updated definition; re-reading the
|
| 681 |
+
// whole list to learn it is the second of the two round trips "Savingβ¦" used to
|
| 682 |
+
// span. `mergeSaved` keeps the memo and the component in step (see
|
| 683 |
+
// `automationApi.rememberAutomation`) and takes the generation, because a write
|
| 684 |
+
// response is newer than any read still in flight β the same rule `load` follows,
|
| 685 |
+
// pointed the other way.
|
| 686 |
+
if (fresh && mergeSaved(fresh)) {
|
| 687 |
+
if (id && id === wantedId.current) select(id);
|
| 688 |
+
return;
|
| 689 |
+
}
|
| 690 |
const next = await load();
|
| 691 |
if (id && next && id === wantedId.current) select(id);
|
| 692 |
}}
|
web/src/automation/automationApi.ts
CHANGED
|
@@ -844,6 +844,36 @@ function send<T>(path: string, init?: RequestInit): Promise<T> {
|
|
| 844 |
* banner beat an empty rail that blames the reader.
|
| 845 |
* β Module scope, so it dies with the tab. It is a paint accelerator, never storage β nothing
|
| 846 |
* here may outlive a session, because the account decides the tenant [[aios-multi-tenancy]].
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 847 |
*/
|
| 848 |
const AUTOMATIONS_FRESH_MS = 5 * 60 * 1000;
|
| 849 |
|
|
@@ -870,6 +900,42 @@ export function listAutomations(abort?: AbortSignal): Promise<AutomationList> {
|
|
| 870 |
});
|
| 871 |
}
|
| 872 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 873 |
export function listTables(abort?: AbortSignal): Promise<{ tables: UserTable[] }> {
|
| 874 |
return send<{ tables: UserTable[] }>("/automations/tables", { signal: abort });
|
| 875 |
}
|
|
|
|
| 844 |
* banner beat an empty rail that blames the reader.
|
| 845 |
* β Module scope, so it dies with the tab. It is a paint accelerator, never storage β nothing
|
| 846 |
* here may outlive a session, because the account decides the tenant [[aios-multi-tenancy]].
|
| 847 |
+
*
|
| 848 |
+
* ββ WAVE 31 Β· T33 (D-176) β AND IT STAYS MODULE SCOPE. THE `sessionStorage` SEED WAS PROPOSED,
|
| 849 |
+
* MEASURED AGAINST THIS BOUNDARY, AND REFUSED. Recorded here rather than left as an absence,
|
| 850 |
+
* because the next reader will have the same good idea:
|
| 851 |
+
*
|
| 852 |
+
* D-176 is real β `data === null` on every F5, so a reload always lands on the skeleton. The
|
| 853 |
+
* proposed fix was a `sessionStorage` seed keyed `(tenant, uname, epoch)`. It cannot be made safe
|
| 854 |
+
* from this module, for two measured reasons:
|
| 855 |
+
*
|
| 856 |
+
* 1. **The client cannot name its tenant at first paint.** The tenant arrives on a RESPONSE
|
| 857 |
+
* header (`apiContract.ts::TENANT_HEADER`) and the session cookie is HttpOnly by design
|
| 858 |
+
* (`shell/session.ts` β *"there is no client-side 'am I logged in?' bit to cache"*). The seed
|
| 859 |
+
* is read synchronously by `useState(() => cachedAutomations())`, i.e. strictly before any
|
| 860 |
+
* response exists. A key that cannot be computed at read time is not a key.
|
| 861 |
+
* 2. **Nothing would clear it.** `forgetAutomations` β the invalidation door written for exactly
|
| 862 |
+
* this β has **ZERO product callers**; its only caller anywhere is the gate. Its twin
|
| 863 |
+
* `clearCustomersCache` IS called by Shell on sign-out/sign-in, which is what makes the grid's
|
| 864 |
+
* rows cache safe. So this memo is safe TODAY only because module scope dies with the tab, and
|
| 865 |
+
* `sessionStorage` survives both an F5 and a same-tab account switch.
|
| 866 |
+
*
|
| 867 |
+
* β The tempting discriminator β "only trust the seed when the navigation type is `reload`" β
|
| 868 |
+
* fails on the one case that matters: `checkTenant` calls `window.location.reload()` ON A TENANT
|
| 869 |
+
* SWAP, so the swap arrives wearing exactly the signature that would authorise the seed, and the
|
| 870 |
+
* previous tenant's automation names paint. The mechanism built to prevent the leak would deliver
|
| 871 |
+
* it.
|
| 872 |
+
*
|
| 873 |
+
* β ACCEPTED STATE, per this ticket's own second branch, with a testable exit: the seed becomes
|
| 874 |
+
* buildable the moment ONE of these exists β a tenant stamp readable synchronously at boot, or a
|
| 875 |
+
* product caller for `forgetAutomations` on sign-out. `verify_automation_ui.py` pins both halves
|
| 876 |
+
* so a future storage-backed memo cannot ship without its clear.
|
| 877 |
*/
|
| 878 |
const AUTOMATIONS_FRESH_MS = 5 * 60 * 1000;
|
| 879 |
|
|
|
|
| 900 |
});
|
| 901 |
}
|
| 902 |
|
| 903 |
+
/**
|
| 904 |
+
* ββ WAVE 31 Β· T31 β fold ONE freshly-written automation into the memo, and hand back the list.
|
| 905 |
+
*
|
| 906 |
+
* β WHY THIS EXISTS: `patchAutomation` and `toggleNode` both RETURN the updated definition, and
|
| 907 |
+
* the surface was throwing that away and issuing a full `GET /automations` to learn what it had
|
| 908 |
+
* just been told. Owner item 7, verbatim: *"It just say Saving... and takes a long time for me to
|
| 909 |
+
* change options and configs."* β the label spanned BOTH round trips, so the wait a person sees is
|
| 910 |
+
* the write plus a whole list read, on a route whose cold call is the expensive one.
|
| 911 |
+
*
|
| 912 |
+
* β THE MEMO AND THE COMPONENT MOVE TOGETHER, THROUGH THIS ONE FUNCTION, and that is the point of
|
| 913 |
+
* putting it here rather than folding in the component. `listMemo` seeds `AutomationSurface`'s
|
| 914 |
+
* FIRST paint (`useState(() => cachedAutomations())`), so a merge that updated only component state
|
| 915 |
+
* would leave the next mount painting the pre-save name from the memo β a rename that visibly
|
| 916 |
+
* un-does itself on navigation, which is a worse bug than the latency being fixed.
|
| 917 |
+
*
|
| 918 |
+
* β `at` IS DELIBERATELY NOT REFRESHED. One row being rewritten does not make the whole list newly
|
| 919 |
+
* fetched; extending the freshness window here would let a memo that has never been re-read live
|
| 920 |
+
* indefinitely as long as somebody keeps saving. The row is corrected, the clock is not touched.
|
| 921 |
+
*
|
| 922 |
+
* β Returns `null` when there is no memo to fold into β a COLD surface has nothing to correct, and
|
| 923 |
+
* its caller falls back to a real load rather than inventing a one-row list.
|
| 924 |
+
*/
|
| 925 |
+
export function rememberAutomation(fresh: Automation): AutomationList | null {
|
| 926 |
+
if (!listMemo || !fresh?.id) return null;
|
| 927 |
+
const prev = listMemo.data;
|
| 928 |
+
const seen = prev.automations.some((a) => a.id === fresh.id);
|
| 929 |
+
const data: AutomationList = {
|
| 930 |
+
...prev,
|
| 931 |
+
automations: seen
|
| 932 |
+
? prev.automations.map((a) => (a.id === fresh.id ? fresh : a))
|
| 933 |
+
: [...prev.automations, fresh],
|
| 934 |
+
};
|
| 935 |
+
listMemo = { at: listMemo.at, data };
|
| 936 |
+
return data;
|
| 937 |
+
}
|
| 938 |
+
|
| 939 |
export function listTables(abort?: AbortSignal): Promise<{ tables: UserTable[] }> {
|
| 940 |
return send<{ tables: UserTable[] }>("/automations/tables", { signal: abort });
|
| 941 |
}
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -84,9 +84,26 @@ export interface ConnectorPlatform {
|
|
| 84 |
desc: string;
|
| 85 |
}
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
type Load =
|
| 88 |
| { phase: "loading" }
|
| 89 |
-
| { phase: "ready"; rows: ConnectorRow[] }
|
| 90 |
| { phase: "error"; message: string };
|
| 91 |
|
| 92 |
const str = (v: unknown): string => (typeof v === "string" ? v : "");
|
|
@@ -110,6 +127,32 @@ export function parsePlatforms(raw: unknown): ConnectorPlatform[] {
|
|
| 110 |
return out;
|
| 111 |
}
|
| 112 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
export function parseDirectory(body: unknown): ConnectorRow[] {
|
| 114 |
const raw = (body as { connectors?: unknown } | null)?.connectors;
|
| 115 |
if (!Array.isArray(raw)) return [];
|
|
@@ -215,7 +258,7 @@ export default function ConnectorsPage({
|
|
| 215 |
});
|
| 216 |
return;
|
| 217 |
}
|
| 218 |
-
setLoad({ phase: "ready", rows: parseDirectory(body) });
|
| 219 |
} catch {
|
| 220 |
if (!dead) setLoad({ phase: "error", message: "Cannot reach the server." });
|
| 221 |
}
|
|
@@ -250,6 +293,18 @@ export default function ConnectorsPage({
|
|
| 250 |
// applied to a second server-composed payload).
|
| 251 |
<p className="conn-note">{load.message}</p>
|
| 252 |
) : null}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
{load.phase === "ready" && rows.length === 0 ? (
|
| 254 |
<p className="conn-note">
|
| 255 |
{query ? "No connector matches that." : "No connectors are available yet."}
|
|
@@ -265,6 +320,38 @@ export default function ConnectorsPage({
|
|
| 265 |
);
|
| 266 |
}
|
| 267 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
/**
|
| 269 |
* ONE CARD, EXPORTED β which is what lets `_conn_shot.tsx` paint the real thing.
|
| 270 |
*
|
|
|
|
| 84 |
desc: string;
|
| 85 |
}
|
| 86 |
|
| 87 |
+
/**
|
| 88 |
+
* ββ W31-T44 β ONE CONTRIBUTION THE SERVER COULD NOT COMPOSE (owner item 6b).
|
| 89 |
+
*
|
| 90 |
+
* The directory is composed from six registries, and until this wave a raise in ANY of them made
|
| 91 |
+
* the whole endpoint a 500 β so the page rendered one sentence and no cards, which is exactly the
|
| 92 |
+
* "Connectors disappears" the owner reported. Now each contribution fails alone and names itself
|
| 93 |
+
* here. β THE POINT IS THAT A SHORT LIST SAYS IT IS SHORT: a map missing a region must name the
|
| 94 |
+
* region, or the reader concludes the region does not exist.
|
| 95 |
+
*/
|
| 96 |
+
export interface ConnectorGap {
|
| 97 |
+
/** Which contribution β `odoo` | `oauth` | `token` | `scraper` | `builtin` | `planned` |
|
| 98 |
+
* `keychain`. Read as a STRING, never a union (the wave-9 law). */
|
| 99 |
+
source: string;
|
| 100 |
+
cause: string;
|
| 101 |
+
recommendation: string;
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
type Load =
|
| 105 |
| { phase: "loading" }
|
| 106 |
+
| { phase: "ready"; rows: ConnectorRow[]; gaps: ConnectorGap[] }
|
| 107 |
| { phase: "error"; message: string };
|
| 108 |
|
| 109 |
const str = (v: unknown): string => (typeof v === "string" ? v : "");
|
|
|
|
| 127 |
return out;
|
| 128 |
}
|
| 129 |
|
| 130 |
+
/**
|
| 131 |
+
* The gaps, read the way `parseDirectory` reads a row: anything without a `cause` is dropped,
|
| 132 |
+
* and a payload with no `incomplete` at all yields `[]`.
|
| 133 |
+
*
|
| 134 |
+
* β AN OLDER SERVER THEREFORE REPORTS NO GAPS, which is the honest degradation and NOT a silent
|
| 135 |
+
* one: a server that does not send the key is a server whose directory is all-or-nothing, so
|
| 136 |
+
* "no gaps" is true of every response it can produce.
|
| 137 |
+
*/
|
| 138 |
+
export function parseGaps(body: unknown): ConnectorGap[] {
|
| 139 |
+
const raw = (body as { incomplete?: unknown } | null)?.incomplete;
|
| 140 |
+
if (!Array.isArray(raw)) return [];
|
| 141 |
+
const out: ConnectorGap[] = [];
|
| 142 |
+
for (const item of raw) {
|
| 143 |
+
if (!item || typeof item !== "object") continue;
|
| 144 |
+
const g = item as Record<string, unknown>;
|
| 145 |
+
const cause = str(g.cause).trim();
|
| 146 |
+
if (!cause) continue;
|
| 147 |
+
out.push({
|
| 148 |
+
source: str(g.source).trim(),
|
| 149 |
+
cause,
|
| 150 |
+
recommendation: str(g.recommendation).trim(),
|
| 151 |
+
});
|
| 152 |
+
}
|
| 153 |
+
return out;
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
export function parseDirectory(body: unknown): ConnectorRow[] {
|
| 157 |
const raw = (body as { connectors?: unknown } | null)?.connectors;
|
| 158 |
if (!Array.isArray(raw)) return [];
|
|
|
|
| 258 |
});
|
| 259 |
return;
|
| 260 |
}
|
| 261 |
+
setLoad({ phase: "ready", rows: parseDirectory(body), gaps: parseGaps(body) });
|
| 262 |
} catch {
|
| 263 |
if (!dead) setLoad({ phase: "error", message: "Cannot reach the server." });
|
| 264 |
}
|
|
|
|
| 293 |
// applied to a second server-composed payload).
|
| 294 |
<p className="conn-note">{load.message}</p>
|
| 295 |
) : null}
|
| 296 |
+
{/*
|
| 297 |
+
ββ W31-T44 β THE LIST SAYS WHEN IT IS SHORT (owner item 6b).
|
| 298 |
+
|
| 299 |
+
β IT IS RENDERED ABOVE THE GRID AND NOT GATED ON `rows.length`, which is the whole
|
| 300 |
+
distinction the ticket is about. A directory that lost one registry still shows a
|
| 301 |
+
confident wall of cards, and "confidently short" is worse than empty: the reader has no
|
| 302 |
+
way to know a connector they HAVE is simply not on the page. The banner is what turns
|
| 303 |
+
that into "incomplete", which is the only honest state.
|
| 304 |
+
β Not gated on `query` either β a search that hides the gap while filtering is the same
|
| 305 |
+
omission one interaction later.
|
| 306 |
+
*/}
|
| 307 |
+
<DirectoryGaps gaps={load.phase === "ready" ? load.gaps : []} />
|
| 308 |
{load.phase === "ready" && rows.length === 0 ? (
|
| 309 |
<p className="conn-note">
|
| 310 |
{query ? "No connector matches that." : "No connectors are available yet."}
|
|
|
|
| 320 |
);
|
| 321 |
}
|
| 322 |
|
| 323 |
+
/**
|
| 324 |
+
* ββ W31-T44 (owner item 6b) β THE BANNER THAT SAYS THE LIST IS SHORT, EXPORTED FOR THE SAME
|
| 325 |
+
* REASON `ConnectorCard` IS.
|
| 326 |
+
*
|
| 327 |
+
* The page fetches on mount and `renderToStaticMarkup` does not run effects, so a static render
|
| 328 |
+
* of `ConnectorsPage` is a spinner forever β which is why W30 lifted the card out. This ticket's
|
| 329 |
+
* `done-when` is likewise visual ("the page can render incomplete"), so the thing that has to be
|
| 330 |
+
* looked at has to be reachable without a browser session. `_conn_shot.tsx` paints THIS.
|
| 331 |
+
*
|
| 332 |
+
* β IT IS RENDERED ABOVE THE GRID AND GATED ON NEITHER `rows.length` NOR `query`, which is the
|
| 333 |
+
* whole distinction the ticket is about. A directory that lost one registry still shows a
|
| 334 |
+
* confident wall of cards, and "confidently short" is worse than empty: the reader has no way to
|
| 335 |
+
* know a connector they HAVE is simply not on the page. A search that hid the banner while
|
| 336 |
+
* filtering would be the same omission one interaction later.
|
| 337 |
+
*/
|
| 338 |
+
export function DirectoryGaps({ gaps }: { gaps: ConnectorGap[] }) {
|
| 339 |
+
if (gaps.length === 0) return null;
|
| 340 |
+
return (
|
| 341 |
+
<div className="conn-note is-warn" role="status">
|
| 342 |
+
<strong>This list is incomplete.</strong>
|
| 343 |
+
<ul>
|
| 344 |
+
{gaps.map((g) => (
|
| 345 |
+
<li key={g.source + g.cause}>
|
| 346 |
+
{g.cause}
|
| 347 |
+
{g.recommendation ? ` β ${g.recommendation}` : ""}
|
| 348 |
+
</li>
|
| 349 |
+
))}
|
| 350 |
+
</ul>
|
| 351 |
+
</div>
|
| 352 |
+
);
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
/**
|
| 356 |
* ONE CARD, EXPORTED β which is what lets `_conn_shot.tsx` paint the real thing.
|
| 357 |
*
|
web/src/connectors/brandMarks.tsx
CHANGED
|
@@ -57,7 +57,9 @@ export type BrandName =
|
|
| 57 |
| "stripe"
|
| 58 |
| "shopify"
|
| 59 |
| "webhooks"
|
| 60 |
-
| "scraper"
|
|
|
|
|
|
|
| 61 |
|
| 62 |
/**
|
| 63 |
* Trigger and action KIND -> brand.
|
|
@@ -98,6 +100,12 @@ export const BRAND_OF_CONNECTOR: Record<string, BrandName> = {
|
|
| 98 |
webhooks: "webhooks",
|
| 99 |
instagram: "instagram",
|
| 100 |
tiktok: "tiktok",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
};
|
| 102 |
|
| 103 |
/** The house frame every mark is drawn in. 16Γ16 user units, sized by the caller. */
|
|
@@ -209,6 +217,34 @@ export function BrandMark({
|
|
| 209 |
<path d="m2 4.4 6 4.6 6-4.6" />
|
| 210 |
</svg>
|
| 211 |
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
if (name === "stripe")
|
| 213 |
return (
|
| 214 |
<svg {...f}>
|
|
|
|
| 57 |
| "stripe"
|
| 58 |
| "shopify"
|
| 59 |
| "webhooks"
|
| 60 |
+
| "scraper"
|
| 61 |
+
| "slack"
|
| 62 |
+
| "meta";
|
| 63 |
|
| 64 |
/**
|
| 65 |
* Trigger and action KIND -> brand.
|
|
|
|
| 100 |
webhooks: "webhooks",
|
| 101 |
instagram: "instagram",
|
| 102 |
tiktok: "tiktok",
|
| 103 |
+
// β WAVE 31 Β· R7 β the Nango breadth play's first two marks. `meta_ads` is the KEYCHAIN
|
| 104 |
+
// connector (a token an admin pastes today); `meta` would be the delegated-OAuth door the same
|
| 105 |
+
// company owns, so BOTH keys point at one mark rather than the map growing a near-duplicate.
|
| 106 |
+
slack: "slack",
|
| 107 |
+
meta: "meta",
|
| 108 |
+
meta_ads: "meta",
|
| 109 |
};
|
| 110 |
|
| 111 |
/** The house frame every mark is drawn in. 16Γ16 user units, sized by the caller. */
|
|
|
|
| 217 |
<path d="m2 4.4 6 4.6 6-4.6" />
|
| 218 |
</svg>
|
| 219 |
);
|
| 220 |
+
// β WAVE 31 Β· R7. Slack's mark is FOUR bars in four brand colours, drawn as rounded rects
|
| 221 |
+
// rather than the true interlocking glyph: at 15px the real lozenges collapse into mud, and a
|
| 222 |
+
// mark nobody can identify is worse than a two-letter tile. Colours are Slack's own.
|
| 223 |
+
if (name === "slack")
|
| 224 |
+
return (
|
| 225 |
+
<svg {...f}>
|
| 226 |
+
<rect x="2" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#36C5F0" />
|
| 227 |
+
<rect x="8.6" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#2EB67D" />
|
| 228 |
+
<rect x="6.6" y="2" width="2.8" height="5.4" rx="1.4" fill="#ECB22E" />
|
| 229 |
+
<rect x="6.6" y="8.6" width="2.8" height="5.4" rx="1.4" fill="#E01E5A" />
|
| 230 |
+
</svg>
|
| 231 |
+
);
|
| 232 |
+
// Meta's infinity loop, as two overlapping strokes in the brand blue. β `fill="none"` with a
|
| 233 |
+
// round cap: filled, the two lobes merge into a blob at this size.
|
| 234 |
+
if (name === "meta")
|
| 235 |
+
return (
|
| 236 |
+
<svg {...f} fill="none">
|
| 237 |
+
<path
|
| 238 |
+
d="M2.2 10.4c0-3 1.5-5.2 3.3-5.2 1.2 0 2 .8 3 2.4l1 1.6c.9 1.4 1.5 2.1 2.2 2.1.8 0
|
| 239 |
+
1.3-.8 1.3-2.1 0-1.6-.7-3.4-2-3.4-.9 0-1.7.7-2.6 2"
|
| 240 |
+
stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
|
| 241 |
+
/>
|
| 242 |
+
<path
|
| 243 |
+
d="M2.2 10.4c0 1.3.6 2 1.5 2 .8 0 1.4-.5 2.5-2.2l3-4.6"
|
| 244 |
+
stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
|
| 245 |
+
/>
|
| 246 |
+
</svg>
|
| 247 |
+
);
|
| 248 |
if (name === "stripe")
|
| 249 |
return (
|
| 250 |
<svg {...f}>
|
web/src/customer-grid/CustomerGrid.tsx
CHANGED
|
@@ -5927,7 +5927,16 @@ function CustomerGrid({
|
|
| 5927 |
is null unless the SERVER declared a limit on this response (R6's second sentence β
|
| 5928 |
short on screen, the full cause and recommendation one hover away, the same
|
| 5929 |
short-form/`title` split `countLabelCompact` uses for the same strip). */}
|
| 5930 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5931 |
<div className="cg-more">
|
| 5932 |
{foldNote && <span className="cg-more-note">{foldNote}</span>}
|
| 5933 |
{limitNote && (
|
|
|
|
| 5927 |
is null unless the SERVER declared a limit on this response (R6's second sentence β
|
| 5928 |
short on screen, the full cause and recommendation one hover away, the same
|
| 5929 |
short-form/`title` split `countLabelCompact` uses for the same strip). */}
|
| 5930 |
+
{/* ββ WAVE 31 Β· T22 (D-173) β THE LIMIT SENTENCE IS NO LONGER GATED ON `!capped`.
|
| 5931 |
+
β THE DEFECT IN ONE LINE: `capped` means "this grid has more rows than the display
|
| 5932 |
+
cap", i.e. it becomes TRUE at exactly the size that makes a server-declared limit
|
| 5933 |
+
worth reading β and the note was hidden precisely then. A limit reported only on small
|
| 5934 |
+
tables is a limit not reported at all, which is the violation R6's second sentence
|
| 5935 |
+
names ("if it can't be done, tell me why and recommend a fix"), not the limit itself.
|
| 5936 |
+
β The display cap and the server's limits are INDEPENDENT FACTS about one grid: one is
|
| 5937 |
+
how much we are painting, the other is what the server could not do. They belong in
|
| 5938 |
+
the same strip and neither may suppress the other. */}
|
| 5939 |
+
{(foldNote || limitNote || capabilityNote) && (
|
| 5940 |
<div className="cg-more">
|
| 5941 |
{foldNote && <span className="cg-more-note">{foldNote}</span>}
|
| 5942 |
{limitNote && (
|
web/src/customer-grid/apiBridge.ts
CHANGED
|
@@ -121,8 +121,41 @@ const CUSTOMERS_FRESH_MS = 5 * 60_000;
|
|
| 121 |
// customer pull each get the same 5-minute remount window without ever serving each other.
|
| 122 |
const rowsCache = new Map<string, { at: number; payload: CustomersPayload }>();
|
| 123 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
export function clearCustomersCache(): void {
|
| 125 |
rowsCache.clear();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
// β AND THE READ-THROUGH ROSTER, which is a per-SESSION fact, not a per-topic one. It names
|
| 127 |
// which tables this tenant serves through the mirror; a second account signing into the same
|
| 128 |
// browser is a different tenant's answer. Same boundary the rows cache is cleared on, and for
|
|
@@ -501,9 +534,16 @@ export async function fetchTableWindow(
|
|
| 501 |
*/
|
| 502 |
export async function fetchWorkspace(
|
| 503 |
scope: SurfaceScope = "customer",
|
| 504 |
-
opts: { announceFailure?: boolean } = {},
|
| 505 |
): Promise<GridWorkspace | null> {
|
| 506 |
const loud = opts.announceFailure === true;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 507 |
let res: Response;
|
| 508 |
try {
|
| 509 |
res = await fetch(`${API_V1}/workspace?scope=${encodeURIComponent(scope)}`,
|
|
@@ -530,7 +570,14 @@ export async function fetchWorkspace(
|
|
| 530 |
const w = ws?.workspace;
|
| 531 |
// A 200 carrying no workspace is still an ABSENCE, not a refusal β an older
|
| 532 |
// host, or a scope this one has nothing stored for. Unchanged, and quiet.
|
| 533 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 534 |
}
|
| 535 |
|
| 536 |
// --- the CHANGE POLLER (wave 29, item 20 / R11 / contract C6) ---------------
|
|
|
|
| 121 |
// customer pull each get the same 5-minute remount window without ever serving each other.
|
| 122 |
const rowsCache = new Map<string, { at: number; payload: CustomersPayload }>();
|
| 123 |
|
| 124 |
+
/**
|
| 125 |
+
* ββ WAVE 31 Β· T21 (owner item 7: *"It still takes a very long time to load from one Database
|
| 126 |
+
* into another"*) β THE WORKSPACE ENVELOPE, MEMOISED PER SCOPE.
|
| 127 |
+
*
|
| 128 |
+
* β WHAT WAS MISSING, stated exactly: `rowsCache` above has memoised ROWS since EXIT wave 1, and
|
| 129 |
+
* `fetchWorkspace` had NO cache of any kind. So switching back to a database whose rows were
|
| 130 |
+
* still warm STILL paid a full `/workspace` round trip β and that call is not cheap: it is
|
| 131 |
+
* `ut_assembly`, which on tenant #0 reads a **28.6 MB** document (703 ms of deep copy, warm) plus
|
| 132 |
+
* the whole `grid_events` workspace build. Measured live on `283b815`: **2,430 ms** for
|
| 133 |
+
* `?scope=customer`. Every databaseβdatabase hop paid it, both directions, for an envelope that
|
| 134 |
+
* had not changed.
|
| 135 |
+
*
|
| 136 |
+
* β THE FRESHNESS WINDOW IS THE ROWS CACHE'S, ON PURPOSE. Two windows over one surface would
|
| 137 |
+
* drift into a grid whose columns are newer than its rows, or the reverse β a shape this repo has
|
| 138 |
+
* already paid for. One constant, both memos.
|
| 139 |
+
*
|
| 140 |
+
* β AND IT IS EVICTED BY THE WRITE PATH, NOT ONLY BY TIME. Every accepted grid event fires
|
| 141 |
+
* `WORKSPACE_STALE_EVENT` β `useCustomerData.reread()`, which calls this with no `allowCached`,
|
| 142 |
+
* so it re-fetches AND overwrites this entry. Without that overwrite a person could hide a
|
| 143 |
+
* field, switch away, switch back, and be served the pre-write envelope β the write silently
|
| 144 |
+
* undone on screen, which is worse than the wait it replaced.
|
| 145 |
+
*/
|
| 146 |
+
const wsCache = new Map<string, { at: number; ws: GridWorkspace }>();
|
| 147 |
+
|
| 148 |
+
export function clearWorkspaceCache(): void {
|
| 149 |
+
wsCache.clear();
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
export function clearCustomersCache(): void {
|
| 153 |
rowsCache.clear();
|
| 154 |
+
// β THE ENVELOPE IS SESSION-SCOPED FOR THE SAME REASON THE ROWS ARE, and it carries MORE that
|
| 155 |
+
// is per-account than they do: `viewer`, `userOptions`, the user's own views and folders. One
|
| 156 |
+
// browser, two accounts, and a cached workspace served across the boundary would show the
|
| 157 |
+
// previous person's saved views under the new person's name.
|
| 158 |
+
clearWorkspaceCache();
|
| 159 |
// β AND THE READ-THROUGH ROSTER, which is a per-SESSION fact, not a per-topic one. It names
|
| 160 |
// which tables this tenant serves through the mirror; a second account signing into the same
|
| 161 |
// browser is a different tenant's answer. Same boundary the rows cache is cleared on, and for
|
|
|
|
| 534 |
*/
|
| 535 |
export async function fetchWorkspace(
|
| 536 |
scope: SurfaceScope = "customer",
|
| 537 |
+
opts: { announceFailure?: boolean; allowCached?: boolean } = {},
|
| 538 |
): Promise<GridWorkspace | null> {
|
| 539 |
const loud = opts.announceFailure === true;
|
| 540 |
+
// β W31-T21 β OPT IN, NEVER BY DEFAULT. Only the first-paint read of a surface may be served
|
| 541 |
+
// from the memo; the post-write `reread()` calls this with no options and therefore always goes
|
| 542 |
+
// to the server, which is what keeps the memo from becoming a second source of truth.
|
| 543 |
+
if (opts.allowCached === true) {
|
| 544 |
+
const hit = wsCache.get(scope);
|
| 545 |
+
if (hit && Date.now() - hit.at < CUSTOMERS_FRESH_MS) return hit.ws;
|
| 546 |
+
}
|
| 547 |
let res: Response;
|
| 548 |
try {
|
| 549 |
res = await fetch(`${API_V1}/workspace?scope=${encodeURIComponent(scope)}`,
|
|
|
|
| 570 |
const w = ws?.workspace;
|
| 571 |
// A 200 carrying no workspace is still an ABSENCE, not a refusal β an older
|
| 572 |
// host, or a scope this one has nothing stored for. Unchanged, and quiet.
|
| 573 |
+
const good = w && typeof w.storageKey === "string" && Array.isArray(w.views) ? w : null;
|
| 574 |
+
// β W31-T21 β EVERY SUCCESSFUL READ REFRESHES THE MEMO, including the post-write `reread()`
|
| 575 |
+
// that is not allowed to CONSUME it. That asymmetry is the whole safety argument: the write
|
| 576 |
+
// path can never leave a stale envelope behind for the next switch to serve.
|
| 577 |
+
// β A FAILURE LEAVES THE PREVIOUS ENTRY ALONE rather than poisoning it with `null` β the same
|
| 578 |
+
// "absent stays absent, never blank a live panel" posture `reread()` already takes.
|
| 579 |
+
if (good) wsCache.set(scope, { at: Date.now(), ws: good });
|
| 580 |
+
return good;
|
| 581 |
}
|
| 582 |
|
| 583 |
// --- the CHANGE POLLER (wave 29, item 20 / R11 / contract C6) ---------------
|
web/src/customer-grid/display.ts
CHANGED
|
@@ -382,6 +382,26 @@ export function automationStateLabel(v: CellValue): string {
|
|
| 382 |
*/
|
| 383 |
export const MAX_JSON_BYTES = 32 * 1024;
|
| 384 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 385 |
/** Does this text parse as JSON? The ONE test, shared by the preview, the cell and the viewer's
|
| 386 |
* save β three copies of a try/catch is how they end up disagreeing about `""` or `NaN`. */
|
| 387 |
export function jsonParse(v: CellValue): { ok: boolean; value?: unknown } {
|
|
@@ -432,6 +452,20 @@ export function jsonPreview(v: CellValue): string {
|
|
| 432 |
// about JSON and says nothing about the record.
|
| 433 |
// β The ARRAY LENGTH, never the document's own `n`: the count in a cell has to be checkable
|
| 434 |
// against what opening it shows ([[no-unverifiable-aggregates]]).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
const shown = postsWindowOf(value);
|
| 436 |
if (shown)
|
| 437 |
return shown.posts.length === 0
|
|
|
|
| 382 |
*/
|
| 383 |
export const MAX_JSON_BYTES = 32 * 1024;
|
| 384 |
|
| 385 |
+
/**
|
| 386 |
+
* β WAVE 31 Β· T22 (D-173) β is this parsed value the SERVER'S STAND-IN for a document it did not
|
| 387 |
+
* send, and if so how big was the real one? Returns a short size string, or null.
|
| 388 |
+
*
|
| 389 |
+
* β THE SHAPE IS `routes_tables._thin_json`'s OWN, and it is matched on `_truncated === true`
|
| 390 |
+
* plus a numeric `bytes` β never on the key alone, because a person's own document could contain
|
| 391 |
+
* a `_truncated` key and must not be reported as absent. `_url` is deliberately NOT required: the
|
| 392 |
+
* viewer keys its fetch on it, but a preview that refused to warn when it was missing would go
|
| 393 |
+
* quiet in exactly the degraded case that most needs a warning.
|
| 394 |
+
*/
|
| 395 |
+
export function truncatedDoc(value: unknown): string | null {
|
| 396 |
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
| 397 |
+
const v = value as Record<string, unknown>;
|
| 398 |
+
if (v._truncated !== true) return null;
|
| 399 |
+
const bytes = typeof v.bytes === "number" && isFinite(v.bytes) ? Math.max(0, v.bytes) : 0;
|
| 400 |
+
if (!bytes) return "not shown";
|
| 401 |
+
return bytes >= 1024 ? `${Math.round(bytes / 1024).toLocaleString()} KB not shown`
|
| 402 |
+
: `${bytes.toLocaleString()} bytes not shown`;
|
| 403 |
+
}
|
| 404 |
+
|
| 405 |
/** Does this text parse as JSON? The ONE test, shared by the preview, the cell and the viewer's
|
| 406 |
* save β three copies of a try/catch is how they end up disagreeing about `""` or `NaN`. */
|
| 407 |
export function jsonParse(v: CellValue): { ok: boolean; value?: unknown } {
|
|
|
|
| 452 |
// about JSON and says nothing about the record.
|
| 453 |
// β The ARRAY LENGTH, never the document's own `n`: the count in a cell has to be checkable
|
| 454 |
// against what opening it shows ([[no-unverifiable-aggregates]]).
|
| 455 |
+
// ββ WAVE 31 Β· T22 (D-173) β A DOCUMENT THE LIST DID NOT SEND SAYS SO, WITH ITS SIZE.
|
| 456 |
+
//
|
| 457 |
+
// β THE SWALLOW, and it is one branch below this one. `routes_tables._thin_json` replaces an
|
| 458 |
+
// oversized `json` cell with a stand-in β `{_truncated, bytes, _url}` β so the big document
|
| 459 |
+
// does not ride a list response (measured: `source_payload` was 95.6β98.5% of every IG grid's
|
| 460 |
+
// bytes). That stand-in is itself valid JSON with three keys, so it fell through to the generic
|
| 461 |
+
// object branch and the cell read **`{β¦} 3 keys`** β a confident, checkable-looking claim about
|
| 462 |
+
// a document that is not there, and indistinguishable from a real three-key document. R6's
|
| 463 |
+
// second sentence applies to our own wire: a value we declined to send is reported, never
|
| 464 |
+
// disguised.
|
| 465 |
+
// β THE SIZE IS THE SERVER'S OWN `bytes`, not a guess, and it is what makes the cell honest β
|
| 466 |
+
// "this column holds 41 KB you have not been shown" is a different fact from "3 keys".
|
| 467 |
+
const thinned = truncatedDoc(value);
|
| 468 |
+
if (thinned) return `{β¦} ${thinned}`;
|
| 469 |
const shown = postsWindowOf(value);
|
| 470 |
if (shown)
|
| 471 |
return shown.posts.length === 0
|
web/src/customer-grid/types.ts
CHANGED
|
@@ -3163,6 +3163,21 @@ export interface GridWorkspace {
|
|
| 3163 |
storageKey: string;
|
| 3164 |
views: SavedView[];
|
| 3165 |
activeViewId?: string;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3166 |
/**
|
| 3167 |
* 2026-08-04 β the FIELD CONTRACT, so it rides the cheap re-read a durable write triggers
|
| 3168 |
* (WORKSPACE_STALE) instead of the heavy rows call, which is deliberately never refetched.
|
|
|
|
| 3163 |
storageKey: string;
|
| 3164 |
views: SavedView[];
|
| 3165 |
activeViewId?: string;
|
| 3166 |
+
/**
|
| 3167 |
+
* ββ WAVE 31 Β· T20 β LIMITS THE ENVELOPE ITSELF COULD NOT MEET, which is a different question
|
| 3168 |
+
* from the rows payload's `limits` and needs its own answer.
|
| 3169 |
+
*
|
| 3170 |
+
* Today its one member is the read-through pid scope: on a mirror-backed grid whose population
|
| 3171 |
+
* exceeds one window (963,783 GL lines against a 5,000-row window) the workspace is built from
|
| 3172 |
+
* the DEFINITION plus a windowed count and the pid set is EMPTY β so cohort membership and a
|
| 3173 |
+
* shared view's `memberPids` are unresolved. Both consequences fail closed, and an unannounced
|
| 3174 |
+
* empty scope is exactly the silent limit R6's second sentence forbids.
|
| 3175 |
+
*
|
| 3176 |
+
* β ALWAYS PRESENT on a `ut_*` workspace, `[]` when nothing binds β a key a consumer has to
|
| 3177 |
+
* test for is a key a consumer forgets to test for. Merged into the payload's own limits by
|
| 3178 |
+
* `useCustomerData.withWorkspace`, which is its only reader.
|
| 3179 |
+
*/
|
| 3180 |
+
limits?: GridLimit[];
|
| 3181 |
/**
|
| 3182 |
* 2026-08-04 β the FIELD CONTRACT, so it rides the cheap re-read a durable write triggers
|
| 3183 |
* (WORKSPACE_STALE) instead of the heavy rows call, which is deliberately never refetched.
|
web/src/customer-grid/useCustomerData.ts
CHANGED
|
@@ -92,6 +92,21 @@ function withWorkspace(payload: CustomersPayload, ws: GridWorkspace): CustomersP
|
|
| 92 |
out.fields = ws.fields;
|
| 93 |
if (Array.isArray(ws.measures)) out.measures = ws.measures;
|
| 94 |
if (ws.measureSets && typeof ws.measureSets === "object") out.measureSets = ws.measureSets;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
if (ws.viewer && typeof ws.viewer.name === "string") out.viewer = ws.viewer;
|
| 96 |
if (Array.isArray(ws.userOptions)) out.userOptions = ws.userOptions;
|
| 97 |
const derived = ws.derived;
|
|
@@ -399,7 +414,13 @@ export function useCustomerData(
|
|
| 399 |
// this line would serialise the workspace behind it on the first grid a person opens.
|
| 400 |
// Only the ROWS request has to wait, because only it needs the answer.
|
| 401 |
const wsPromise = includeWorkspace
|
| 402 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 403 |
: Promise.resolve(null);
|
| 404 |
/**
|
| 405 |
* ββ W30-T42 (R7 / contract C2) β WHOLE BOOK, OR ONE WINDOW?
|
|
|
|
| 92 |
out.fields = ws.fields;
|
| 93 |
if (Array.isArray(ws.measures)) out.measures = ws.measures;
|
| 94 |
if (ws.measureSets && typeof ws.measureSets === "object") out.measureSets = ws.measureSets;
|
| 95 |
+
// ββ WAVE 31 Β· T22 β THE WORKSPACE'S OWN LIMITS REACH THE STRIP THAT RENDERS LIMITS.
|
| 96 |
+
//
|
| 97 |
+
// β W31-T20 put a second `limits` channel on the wire and this merge is its ONLY reader. The
|
| 98 |
+
// rows payload reports what the ROWS call could not do; the workspace envelope reports what the
|
| 99 |
+
// ENVELOPE could not do β and on a read-through grid too large for one window that is the
|
| 100 |
+
// whole message ("cohorts and shared-view membership are unresolved here"). Without this line
|
| 101 |
+
// the server would be emitting a sentence nothing displays, which is this repo's named
|
| 102 |
+
// [[flag-shipped-without-its-writer]] shape with the halves swapped.
|
| 103 |
+
// β UNION, DEDUPED BY SUBJECT, rows first: two channels can legitimately report the same
|
| 104 |
+
// subject, and printing "Limits apply to pids, pids" is how a reader learns to stop reading it.
|
| 105 |
+
if (Array.isArray(ws.limits) && ws.limits.length > 0) {
|
| 106 |
+
const seen = new Set((payload.limits ?? []).map((l) => l.subject));
|
| 107 |
+
const extra = ws.limits.filter((l) => l && l.subject && !seen.has(l.subject));
|
| 108 |
+
if (extra.length) out.limits = [...(payload.limits ?? []), ...extra];
|
| 109 |
+
}
|
| 110 |
if (ws.viewer && typeof ws.viewer.name === "string") out.viewer = ws.viewer;
|
| 111 |
if (Array.isArray(ws.userOptions)) out.userOptions = ws.userOptions;
|
| 112 |
const derived = ws.derived;
|
|
|
|
| 414 |
// this line would serialise the workspace behind it on the first grid a person opens.
|
| 415 |
// Only the ROWS request has to wait, because only it needs the answer.
|
| 416 |
const wsPromise = includeWorkspace
|
| 417 |
+
// β W31-T21 (owner item 7) β THE FIRST PAINT OF A SURFACE MAY BE SERVED FROM THE MEMO.
|
| 418 |
+
// This is the ONLY caller that opts in; `reread()` below deliberately does not, so a
|
| 419 |
+
// write always goes to the server and always refreshes what this line may serve. Without
|
| 420 |
+
// it, switching back to a database whose ROWS were still warm still paid a full
|
| 421 |
+
// `/workspace` (2,430 ms live for `?scope=customer` on `283b815`) for an envelope that
|
| 422 |
+
// had not changed.
|
| 423 |
+
? fetchWorkspace(scope, { announceFailure: firstPaint, allowCached: true })
|
| 424 |
: Promise.resolve(null);
|
| 425 |
/**
|
| 426 |
* ββ W30-T42 (R7 / contract C2) β WHOLE BOOK, OR ONE WINDOW?
|
web/src/index.css
CHANGED
|
@@ -3222,6 +3222,42 @@ body {
|
|
| 3222 |
database, not that it became something louder. */
|
| 3223 |
.shell-nav-auto .shell-nav-icon { opacity: 0.7; }
|
| 3224 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3225 |
/* A folder head is a LABEL, not a link: quieter, no hover, no cursor. */
|
| 3226 |
.shell-nav-group {
|
| 3227 |
font-size: var(--lp-fs-2xs);
|
|
@@ -11298,6 +11334,26 @@ a.cg-map-ctl-b { text-decoration: none; }
|
|
| 11298 |
}
|
| 11299 |
.conn-card-go:hover { text-decoration: underline; }
|
| 11300 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11301 |
/* === WAVE23 SHELL/HOME β E REGION END ==================================================== */
|
| 11302 |
|
| 11303 |
/* === WAVE23 GRID+FORMS β D REGION START ====================================================
|
|
|
|
| 3222 |
database, not that it became something louder. */
|
| 3223 |
.shell-nav-auto .shell-nav-icon { opacity: 0.7; }
|
| 3224 |
|
| 3225 |
+
/* ββ W31-T11 (owner item 6b) β THE SLOT ITSELF SAYS WHICH KIND OF ABSENCE THIS IS.
|
| 3226 |
+
`{automation ? β¦ : null}` drew literally nothing for three different situations β in flight,
|
| 3227 |
+
not in this workspace's catalogue, and could-not-be-read β and `/nav` measured a 9.5β14.0 s
|
| 3228 |
+
band, so "in flight" was the common one and read as the module disappearing.
|
| 3229 |
+
β NEITHER STATE IS A LINK, so both drop the pointer and the hover tint `.shell-nav-item`
|
| 3230 |
+
gives every row: an affordance on a row that cannot be opened is the fake affordance R8 bans.
|
| 3231 |
+
β NO NEW COLOUR TOKEN. Both reuse `--lp-muted`, so the rail keeps one ink for
|
| 3232 |
+
"present but not actionable" and this cannot drift from the folder head above it. */
|
| 3233 |
+
.shell-nav-item.is-pending,
|
| 3234 |
+
.shell-nav-item.is-unavailable {
|
| 3235 |
+
cursor: default;
|
| 3236 |
+
color: var(--lp-muted);
|
| 3237 |
+
}
|
| 3238 |
+
.shell-nav-item.is-pending:hover,
|
| 3239 |
+
.shell-nav-item.is-unavailable:hover { background: transparent; }
|
| 3240 |
+
.shell-nav-item.is-unavailable .shell-nav-icon { opacity: 0.45; }
|
| 3241 |
+
.shell-nav-label--muted { color: var(--lp-muted); }
|
| 3242 |
+
/* The mark inside a row is the rail's own spinner at icon size, so a pending row is the same
|
| 3243 |
+
height as the row it will become β a shorter placeholder makes the rail jump when it lands. */
|
| 3244 |
+
.shell-nav-item.is-pending .lp-spin {
|
| 3245 |
+
flex: 0 0 auto;
|
| 3246 |
+
width: 16px;
|
| 3247 |
+
height: 16px;
|
| 3248 |
+
}
|
| 3249 |
+
/* A quiet dot at the right edge: this row has something to say, and the `title` says it. It is
|
| 3250 |
+
NOT a status dot on an automation (W26/R14 bans those) β it marks the RAIL's own read state. */
|
| 3251 |
+
.shell-nav-note-dot {
|
| 3252 |
+
margin-left: auto;
|
| 3253 |
+
flex: 0 0 auto;
|
| 3254 |
+
width: 5px;
|
| 3255 |
+
height: 5px;
|
| 3256 |
+
border-radius: 50%;
|
| 3257 |
+
background: currentColor;
|
| 3258 |
+
opacity: 0.5;
|
| 3259 |
+
}
|
| 3260 |
+
|
| 3261 |
/* A folder head is a LABEL, not a link: quieter, no hover, no cursor. */
|
| 3262 |
.shell-nav-group {
|
| 3263 |
font-size: var(--lp-fs-2xs);
|
|
|
|
| 11334 |
}
|
| 11335 |
.conn-card-go:hover { text-decoration: underline; }
|
| 11336 |
|
| 11337 |
+
/* ββ W31-T44 (owner item 6b) β THE DIRECTORY SAYS WHEN IT IS SHORT.
|
| 11338 |
+
Amber, not red, and the distinction is the one `.cg-count-pending` already makes two thousand
|
| 11339 |
+
lines up: nothing is corrupt and the reader did nothing wrong β one registry did not answer, so
|
| 11340 |
+
some cards are missing. Red would claim an error they made. It reuses the yellow token family
|
| 11341 |
+
`.auto-banner.is-warn` uses rather than the two raw hexes the `.cg` chips predate, because a
|
| 11342 |
+
seventh spelling of "amber" in this file is how a palette stops being one. */
|
| 11343 |
+
.conn-note.is-warn {
|
| 11344 |
+
border: 1px solid var(--lp-yellow);
|
| 11345 |
+
background: var(--lp-yellow-tint);
|
| 11346 |
+
color: var(--lp-yellow-deep);
|
| 11347 |
+
border-radius: var(--lp-r-md);
|
| 11348 |
+
padding: 8px 12px;
|
| 11349 |
+
margin: 0 0 4px;
|
| 11350 |
+
}
|
| 11351 |
+
.conn-note.is-warn ul {
|
| 11352 |
+
margin: 4px 0 0;
|
| 11353 |
+
padding-left: 18px;
|
| 11354 |
+
}
|
| 11355 |
+
.conn-note.is-warn li + li { margin-top: 3px; }
|
| 11356 |
+
|
| 11357 |
/* === WAVE23 SHELL/HOME β E REGION END ==================================================== */
|
| 11358 |
|
| 11359 |
/* === WAVE23 GRID+FORMS β D REGION START ====================================================
|
web/src/shell/Shell.tsx
CHANGED
|
@@ -66,7 +66,7 @@ import type { SettingsSection } from "../settings/SettingsModal";
|
|
| 66 |
import { Brand } from "./Brand";
|
| 67 |
import { ErrorBoundary } from "./ErrorBoundary";
|
| 68 |
import LoginPage from "./LoginPage";
|
| 69 |
-
import { CONNECTORS_ROUTE, EMPTY_NAV_PREFS, ENVELOPE_KEYS, HOME_ROUTE, MAX_NAV_FOLDERS, appLink, databaseEntries, dbChipClass, defaultRoute, deleteTable, fetchNav, fetchNavPrefs, fetchTableFootprint, foldNav, postOpened, resolveRoute, saveNavMeta, saveNavPrefs, shapeNav, splitChrome } from "./nav";
|
| 70 |
import type { NavEntry, NavMetaPatch, NavPage, NavPrefs, Recent } from "./nav";
|
| 71 |
const HomePage = lazy(() => import("../home/HomePage"));
|
| 72 |
const TemplatePicker = lazy(() => import("../home/TemplatePicker"));
|
|
@@ -227,8 +227,12 @@ type Nav =
|
|
| 227 |
/** WAVE 23 C10 β the Home landing's recently-opened list, per user, off the nav payload. */
|
| 228 |
recents: Recent[];
|
| 229 |
empty?: string;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
}
|
| 231 |
-
| { phase: "error" };
|
| 232 |
|
| 233 |
const NO_UTILITY: NavPage[] = [];
|
| 234 |
/** Stable identity, like `NO_ENTRIES` β a fresh `[]` per render would re-run Home's memo. */
|
|
@@ -868,12 +872,18 @@ function ShellFrame() {
|
|
| 868 |
const { main, utility } = splitChrome(r.pages);
|
| 869 |
setNav({ phase: "ready", entries: shapeNav(main, APP_BASE), utility,
|
| 870 |
recents: r.recents,
|
| 871 |
-
...(r.empty ? { empty: r.empty } : {})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 872 |
}
|
| 873 |
// 401 is not "the nav is broken", it is "the session died under us" β
|
| 874 |
// conflating them would leave a signed-out user staring at a frame.
|
| 875 |
else if (r.status === 401) setSession({ phase: "anon" });
|
| 876 |
-
else setNav({ phase: "error"
|
|
|
|
| 877 |
});
|
| 878 |
return () => {
|
| 879 |
dead = true;
|
|
@@ -1460,6 +1470,33 @@ function ShellFrame() {
|
|
| 1460 |
// one accessor β which also returns the BRANDED `DatabaseEntry[]`, so handing a database
|
| 1461 |
// surface the raw `entries` is a compile error rather than one extra tile nobody looks at.
|
| 1462 |
const automation = entries.find((e) => e.key === "automation" && e.kind === "native");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1463 |
const dbEntries = databaseEntries(entries);
|
| 1464 |
// WAVE 23 C10 β the flyout's search. A plain label substring, case-folded: this list is at
|
| 1465 |
// most a few dozen rows, so anything cleverer (fuzzy, ranked) would be a scoring function
|
|
@@ -1622,6 +1659,35 @@ function ShellFrame() {
|
|
| 1622 |
)}
|
| 1623 |
<span className="shell-nav-label">{automation.label}</span>
|
| 1624 |
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1625 |
) : null}
|
| 1626 |
|
| 1627 |
{/* β WAVE 23 item 9 (R7) β THE DATABASE BUTTON, and the end of the always-on band.
|
|
@@ -2025,7 +2091,24 @@ function ShellFrame() {
|
|
| 2025 |
{/* Honest, and it names the fix. Inventing a nav here would show
|
| 2026 |
surfaces the server never granted. */}
|
| 2027 |
{nav.phase === "error" ? (
|
| 2028 |
-
<div className="shell-nav-note">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2029 |
) : null}
|
| 2030 |
{/* β Wave 18: this line is for a MISCONFIGURED ACCOUNT β one whose grants give it
|
| 2031 |
nothing β and it must not fire for a freshly provisioned TENANT, which has no
|
|
@@ -2420,6 +2503,13 @@ function ShellFrame() {
|
|
| 2420 |
onClose={() => setAlertsOpen(false)}
|
| 2421 |
onInbox={setInbox}
|
| 2422 |
onToast={setToast}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2423 |
onOpenView={(topic, viewId) => {
|
| 2424 |
const route = routeForTopic(topic);
|
| 2425 |
if (!route) return;
|
|
|
|
| 66 |
import { Brand } from "./Brand";
|
| 67 |
import { ErrorBoundary } from "./ErrorBoundary";
|
| 68 |
import LoginPage from "./LoginPage";
|
| 69 |
+
import { CONNECTORS_ROUTE, EMPTY_NAV_PREFS, ENVELOPE_KEYS, HOME_ROUTE, MAX_NAV_FOLDERS, NAV_TIMEOUT_STATUS, appLink, databaseEntries, dbChipClass, defaultRoute, deleteTable, fetchNav, fetchNavPrefs, fetchTableFootprint, foldNav, postOpened, resolveRoute, saveNavMeta, saveNavPrefs, shapeNav, splitChrome } from "./nav";
|
| 70 |
import type { NavEntry, NavMetaPatch, NavPage, NavPrefs, Recent } from "./nav";
|
| 71 |
const HomePage = lazy(() => import("../home/HomePage"));
|
| 72 |
const TemplatePicker = lazy(() => import("../home/TemplatePicker"));
|
|
|
|
| 227 |
/** WAVE 23 C10 β the Home landing's recently-opened list, per user, off the nav payload. */
|
| 228 |
recents: Recent[];
|
| 229 |
empty?: string;
|
| 230 |
+
/** β W31-T11 β registry keys this workspace's catalogue leaves out. Deliberate. */
|
| 231 |
+
omitted?: string[];
|
| 232 |
+
/** β W31-T11 β parts of the payload the SERVER could not read. Not deliberate. */
|
| 233 |
+
degraded?: string[];
|
| 234 |
}
|
| 235 |
+
| { phase: "error"; timedOut?: boolean };
|
| 236 |
|
| 237 |
const NO_UTILITY: NavPage[] = [];
|
| 238 |
/** Stable identity, like `NO_ENTRIES` β a fresh `[]` per render would re-run Home's memo. */
|
|
|
|
| 872 |
const { main, utility } = splitChrome(r.pages);
|
| 873 |
setNav({ phase: "ready", entries: shapeNav(main, APP_BASE), utility,
|
| 874 |
recents: r.recents,
|
| 875 |
+
...(r.empty ? { empty: r.empty } : {}),
|
| 876 |
+
// β W31-T11 β carried onto the state, not dropped here. A flag parsed off the
|
| 877 |
+
// wire and discarded before render is a wiring that goes nowhere, which is
|
| 878 |
+
// exactly how wave 20 shipped four of them.
|
| 879 |
+
...(r.omitted ? { omitted: r.omitted } : {}),
|
| 880 |
+
...(r.degraded ? { degraded: r.degraded } : {}) });
|
| 881 |
}
|
| 882 |
// 401 is not "the nav is broken", it is "the session died under us" β
|
| 883 |
// conflating them would leave a signed-out user staring at a frame.
|
| 884 |
else if (r.status === 401) setSession({ phase: "anon" });
|
| 885 |
+
else setNav({ phase: "error",
|
| 886 |
+
...(r.status === NAV_TIMEOUT_STATUS ? { timedOut: true } : {}) });
|
| 887 |
});
|
| 888 |
return () => {
|
| 889 |
dead = true;
|
|
|
|
| 1470 |
// one accessor β which also returns the BRANDED `DatabaseEntry[]`, so handing a database
|
| 1471 |
// surface the raw `entries` is a compile error rather than one extra tile nobody looks at.
|
| 1472 |
const automation = entries.find((e) => e.key === "automation" && e.kind === "native");
|
| 1473 |
+
/**
|
| 1474 |
+
* ββ W31-T11 (owner item 6b) β WHAT TO DRAW WHERE AUTOMATION GOES WHEN IT IS NOT THERE.
|
| 1475 |
+
*
|
| 1476 |
+
* β THE DEFECT, in one line: `{automation ? β¦ : null}` renders **literally nothing** for
|
| 1477 |
+
* three completely different situations, and the reader cannot tell them apart:
|
| 1478 |
+
* 1. `/nav` is in flight β `entries` is `NO_ENTRIES` until `phase === "ready"`, so the rail
|
| 1479 |
+
* shows its four static rows and a complete-looking gap. `/nav` measured a **9.5β14.0 s
|
| 1480 |
+
* band** on tenant #0, and `reference/ERROR 5.png` is exactly this screen. The owner read
|
| 1481 |
+
* it as the module disappearing, and reported it across several waves.
|
| 1482 |
+
* 2. the payload came back and this workspace's catalogue does not include the module β
|
| 1483 |
+
* deliberate, and the rail should stay quiet.
|
| 1484 |
+
* 3. the payload came back INCOMPLETE (`degraded`) or never came back at all β not
|
| 1485 |
+
* deliberate, and silence is a lie about what exists.
|
| 1486 |
+
* β ONLY CASE 2 DRAWS NOTHING. The other two get the slot, so an absence is never mistaken for
|
| 1487 |
+
* a workspace that simply does not have the feature. (The rail already had a region-level
|
| 1488 |
+
* spinner and error line four rows below β far enough away that nothing connected them to this
|
| 1489 |
+
* gap, which is why "there is a loading state" was true and did not help.)
|
| 1490 |
+
*/
|
| 1491 |
+
const automationSlot: "row" | "pending" | "unavailable" | "silent" = automation
|
| 1492 |
+
? "row"
|
| 1493 |
+
: nav.phase === "loading" || nav.phase === "idle"
|
| 1494 |
+
? "pending"
|
| 1495 |
+
: nav.phase === "error"
|
| 1496 |
+
? "unavailable"
|
| 1497 |
+
: nav.phase === "ready" && (nav.degraded?.length ?? 0) > 0
|
| 1498 |
+
? "unavailable"
|
| 1499 |
+
: "silent";
|
| 1500 |
const dbEntries = databaseEntries(entries);
|
| 1501 |
// WAVE 23 C10 β the flyout's search. A plain label substring, case-folded: this list is at
|
| 1502 |
// most a few dozen rows, so anything cleverer (fuzzy, ranked) would be a scoring function
|
|
|
|
| 1659 |
)}
|
| 1660 |
<span className="shell-nav-label">{automation.label}</span>
|
| 1661 |
</a>
|
| 1662 |
+
) : automationSlot === "pending" ? (
|
| 1663 |
+
/* β W31-T11 β THE PENDING STATE, IN AUTOMATION'S OWN SLOT. The shared spinner mark
|
| 1664 |
+
(R6: the mark, never the word "Loadingβ¦", which `verify_icons` enforces), on a row
|
| 1665 |
+
that occupies the space the real one will. `aria-busy` is what makes it a pending
|
| 1666 |
+
ROW to a screen reader rather than a decorative glyph. */
|
| 1667 |
+
<div
|
| 1668 |
+
className="shell-nav-item shell-nav-auto is-pending"
|
| 1669 |
+
aria-busy="true"
|
| 1670 |
+
aria-label="Automation is loading"
|
| 1671 |
+
>
|
| 1672 |
+
<span className="lp-spin" aria-hidden="true" />
|
| 1673 |
+
<span className="shell-nav-label shell-nav-label--muted">Automation</span>
|
| 1674 |
+
</div>
|
| 1675 |
+
) : automationSlot === "unavailable" ? (
|
| 1676 |
+
/* β W31-T11 β NOT LOADING, AND NOT ABSENT-ON-PURPOSE: the server said this payload
|
| 1677 |
+
is incomplete (`degraded`), or it never answered. The row states that rather than
|
| 1678 |
+
leaving a gap the reader has to interpret β which is the whole of owner item 6b. */
|
| 1679 |
+
<div
|
| 1680 |
+
className="shell-nav-item shell-nav-auto is-unavailable"
|
| 1681 |
+
title={
|
| 1682 |
+
nav.phase === "error" && nav.timedOut
|
| 1683 |
+
? "The navigation request timed out. Reload to retry."
|
| 1684 |
+
: "This list could not be loaded in full. Reload to retry."
|
| 1685 |
+
}
|
| 1686 |
+
>
|
| 1687 |
+
<AutoIcon />
|
| 1688 |
+
<span className="shell-nav-label shell-nav-label--muted">Automation</span>
|
| 1689 |
+
<span className="shell-nav-note-dot" aria-hidden="true" />
|
| 1690 |
+
</div>
|
| 1691 |
) : null}
|
| 1692 |
|
| 1693 |
{/* β WAVE 23 item 9 (R7) β THE DATABASE BUTTON, and the end of the always-on band.
|
|
|
|
| 2091 |
{/* Honest, and it names the fix. Inventing a nav here would show
|
| 2092 |
surfaces the server never granted. */}
|
| 2093 |
{nav.phase === "error" ? (
|
| 2094 |
+
<div className="shell-nav-note">
|
| 2095 |
+
{nav.timedOut
|
| 2096 |
+
? /* β W31-T11 β A DEADLINE IS A DIFFERENT FACT FROM A DEAD SERVER, and it earns
|
| 2097 |
+
its own sentence: before this ticket `fetchNav` had no timeout at all, so
|
| 2098 |
+
this state was UNREACHABLE and the rail waited for ever. */
|
| 2099 |
+
"Navigation took too long to load. Reload to retry."
|
| 2100 |
+
: "Navigation unavailable. Reload to retry."}
|
| 2101 |
+
</div>
|
| 2102 |
+
) : null}
|
| 2103 |
+
{/* β W31-T11 β A 200 THAT COULD NOT READ THE DATABASES SAYS SO. This is the case that
|
| 2104 |
+
had no surface at all: the server swallowed a store failure, answered 200 with every
|
| 2105 |
+
`ut_*` row missing, and the rail drew a confident, complete-looking list of nothing.
|
| 2106 |
+
It is NOT `phase: "error"` β the registry rows are real and usable β so it renders
|
| 2107 |
+
beside them rather than replacing them. */}
|
| 2108 |
+
{nav.phase === "ready" && (nav.degraded?.length ?? 0) > 0 ? (
|
| 2109 |
+
<div className="shell-nav-note">
|
| 2110 |
+
Some databases could not be loaded. Reload to retry.
|
| 2111 |
+
</div>
|
| 2112 |
) : null}
|
| 2113 |
{/* β Wave 18: this line is for a MISCONFIGURED ACCOUNT β one whose grants give it
|
| 2114 |
nothing β and it must not fire for a freshly provisioned TENANT, which has no
|
|
|
|
| 2503 |
onClose={() => setAlertsOpen(false)}
|
| 2504 |
onInbox={setInbox}
|
| 2505 |
onToast={setToast}
|
| 2506 |
+
/* ββ W31-T23 (owner item 1) β THE FRAME ALREADY KNOWS THE COUNT. It is what painted
|
| 2507 |
+
the badge the user just clicked, so handing it over means the panel opens knowing
|
| 2508 |
+
the account's number instead of starting at zero and asserting "Nothing new." while
|
| 2509 |
+
its own read is in flight (measured 3,280 ms live). It is also what keeps "Mark all
|
| 2510 |
+
read" pressable when that read FAILS β the badge and the verb that clears it must
|
| 2511 |
+
never be able to disagree. */
|
| 2512 |
+
seed={inbox}
|
| 2513 |
onOpenView={(topic, viewId) => {
|
| 2514 |
const route = routeForTopic(topic);
|
| 2515 |
if (!route) return;
|
web/src/shell/nav.ts
CHANGED
|
@@ -229,9 +229,49 @@ export interface Recent {
|
|
| 229 |
* misconfigured-account case it always did. Distinguishing them is what stops the rail saying
|
| 230 |
* "no surfaces are available" beside a hero saying "welcome, create your first database". */
|
| 231 |
export type NavResult =
|
| 232 |
-
| {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
| { ok: false; status: number };
|
| 234 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
/**
|
| 236 |
* `{recents:[β¦]}` β the list, fail-closed. An entry with no key or an unreadable stamp is
|
| 237 |
* DROPPED rather than rendered at the epoch, which would file it under "Older" for ever and
|
|
@@ -279,20 +319,66 @@ export function postOpened(key: string): void {
|
|
| 279 |
* in, say so). `status: 0` is our encoding for a transport failure, as in
|
| 280 |
* session.ts.
|
| 281 |
*/
|
| 282 |
-
export
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
try {
|
| 284 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
if (!res.ok) return { ok: false, status: res.status };
|
| 286 |
const body = await res.json();
|
| 287 |
const empty = typeof body?.empty === "string" ? body.empty : undefined;
|
|
|
|
|
|
|
| 288 |
return {
|
| 289 |
ok: true,
|
| 290 |
pages: parsePages(body),
|
| 291 |
recents: parseRecents(body),
|
| 292 |
...(empty ? { empty } : {}),
|
|
|
|
|
|
|
| 293 |
};
|
| 294 |
} catch {
|
| 295 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 296 |
}
|
| 297 |
}
|
| 298 |
|
|
|
|
| 229 |
* misconfigured-account case it always did. Distinguishing them is what stops the rail saying
|
| 230 |
* "no surfaces are available" beside a hero saying "welcome, create your first database". */
|
| 231 |
export type NavResult =
|
| 232 |
+
| {
|
| 233 |
+
ok: true;
|
| 234 |
+
pages: NavPage[];
|
| 235 |
+
recents: Recent[];
|
| 236 |
+
empty?: string;
|
| 237 |
+
/**
|
| 238 |
+
* β W31-T11 (owner item 6b) β REGISTRY KEYS THIS WORKSPACE'S CATALOGUE DOES NOT INCLUDE.
|
| 239 |
+
*
|
| 240 |
+
* A DELIBERATE absence. Every provisioned tenant carries a restricted `modules` list
|
| 241 |
+
* (`gtmlab`/`loopable`/`nurilab` are all `['analyst','automation']`), so this is normal and
|
| 242 |
+
* the rail draws nothing for these keys β as it always has. The value of naming them is
|
| 243 |
+
* that it makes the OTHER absence distinguishable.
|
| 244 |
+
*/
|
| 245 |
+
omitted?: string[];
|
| 246 |
+
/**
|
| 247 |
+
* β W31-T11 β PARTS OF THIS PAYLOAD THE SERVER COULD NOT READ.
|
| 248 |
+
*
|
| 249 |
+
* `["databases"]` means the `ut_*` merge raised and every database is missing from a
|
| 250 |
+
* **200 OK**. Before this the two absences were identical on the wire and the shell
|
| 251 |
+
* rendered `null` for both β which is pixel-identical to still-loading, with no timeout on
|
| 252 |
+
* `fetchNav` at all. That is the owner's *"Connectors and Automation still disappear"*
|
| 253 |
+
* report: not a permission bug, an unmarked failure.
|
| 254 |
+
*/
|
| 255 |
+
degraded?: string[];
|
| 256 |
+
}
|
| 257 |
| { ok: false; status: number };
|
| 258 |
|
| 259 |
+
/**
|
| 260 |
+
* How long the shell waits for `/nav` before calling it unavailable.
|
| 261 |
+
*
|
| 262 |
+
* β THE POINT IS NOT SPEED, IT IS THAT `phase:"loading"` CAN END. `fetchNav` had no
|
| 263 |
+
* `AbortSignal`, no deadline and no retry, so a request that never resolved left a rail that
|
| 264 |
+
* looked *almost* complete β the four static rows plus a spinner β for ever, and the owner read
|
| 265 |
+
* that screen as modules disappearing. **20 seconds is chosen to sit ABOVE the measured worst
|
| 266 |
+
* case, not near it**: `/nav` measured a 9.5β14.0 s band on tenant #0 before W31-T10, so a
|
| 267 |
+
* shorter deadline would have converted a slow success into a manufactured failure. It is a
|
| 268 |
+
* backstop for "never", not a performance budget.
|
| 269 |
+
*/
|
| 270 |
+
export const NAV_TIMEOUT_MS = 20_000;
|
| 271 |
+
|
| 272 |
+
/** Our encoding for "the deadline passed", distinct from 0 (transport) and any HTTP status. */
|
| 273 |
+
export const NAV_TIMEOUT_STATUS = -1;
|
| 274 |
+
|
| 275 |
/**
|
| 276 |
* `{recents:[β¦]}` β the list, fail-closed. An entry with no key or an unreadable stamp is
|
| 277 |
* DROPPED rather than rendered at the epoch, which would file it under "Older" for ever and
|
|
|
|
| 319 |
* in, say so). `status: 0` is our encoding for a transport failure, as in
|
| 320 |
* session.ts.
|
| 321 |
*/
|
| 322 |
+
export function parseKeyList(raw: unknown): string[] {
|
| 323 |
+
if (!Array.isArray(raw)) return [];
|
| 324 |
+
const out: string[] = [];
|
| 325 |
+
for (const k of raw) if (typeof k === "string" && k.trim()) out.push(k.trim());
|
| 326 |
+
return out;
|
| 327 |
+
}
|
| 328 |
+
|
| 329 |
+
export async function fetchNav(timeoutMs: number = NAV_TIMEOUT_MS): Promise<NavResult> {
|
| 330 |
+
// β `setTimeout` + `AbortController`, not `AbortSignal.timeout` β the latter is unavailable in
|
| 331 |
+
// the plain-node context `verify_login.py` compiles and RUNS this module in (see the header
|
| 332 |
+
// note on `appBase` being a parameter for exactly that reason). A helper that throws on import
|
| 333 |
+
// in the gate's environment is a helper that silently stops being tested.
|
| 334 |
+
const ctl = typeof AbortController === "function" ? new AbortController() : null;
|
| 335 |
+
let timedOut = false;
|
| 336 |
+
let timer: ReturnType<typeof setTimeout> | null = null;
|
| 337 |
+
// β THE DEADLINE IS A RACE, NOT ONLY AN ABORT β and this is the shape a probe found the hour
|
| 338 |
+
// it was written. Aborting the request tells the TRANSPORT to stop; it does not, on its own,
|
| 339 |
+
// make this function's promise settle. A `fetch` that ignores its signal therefore leaves the
|
| 340 |
+
// caller awaiting for ever, which is precisely the state the deadline exists to end: the shell
|
| 341 |
+
// would sit in `phase:"loading"` exactly as it did before, with an abort controller in the
|
| 342 |
+
// source to prove it had been handled ([[flag-shipped-without-its-writer]]). Racing settles
|
| 343 |
+
// this promise whatever the transport does, and the abort still fires so the request is not
|
| 344 |
+
// left running behind it.
|
| 345 |
+
const deadline =
|
| 346 |
+
timeoutMs > 0
|
| 347 |
+
? new Promise<null>((resolve) => {
|
| 348 |
+
timer = setTimeout(() => {
|
| 349 |
+
timedOut = true;
|
| 350 |
+
ctl?.abort();
|
| 351 |
+
resolve(null);
|
| 352 |
+
}, timeoutMs);
|
| 353 |
+
})
|
| 354 |
+
: null;
|
| 355 |
try {
|
| 356 |
+
const call = fetch(`${API_V1}/nav`, {
|
| 357 |
+
credentials: CREDENTIALS,
|
| 358 |
+
...(ctl ? { signal: ctl.signal } : {}),
|
| 359 |
+
});
|
| 360 |
+
const res = deadline ? await Promise.race([call, deadline]) : await call;
|
| 361 |
+
if (res === null) return { ok: false, status: NAV_TIMEOUT_STATUS };
|
| 362 |
if (!res.ok) return { ok: false, status: res.status };
|
| 363 |
const body = await res.json();
|
| 364 |
const empty = typeof body?.empty === "string" ? body.empty : undefined;
|
| 365 |
+
const omitted = parseKeyList((body as { omitted?: unknown } | null)?.omitted);
|
| 366 |
+
const degraded = parseKeyList((body as { degraded?: unknown } | null)?.degraded);
|
| 367 |
return {
|
| 368 |
ok: true,
|
| 369 |
pages: parsePages(body),
|
| 370 |
recents: parseRecents(body),
|
| 371 |
...(empty ? { empty } : {}),
|
| 372 |
+
...(omitted.length ? { omitted } : {}),
|
| 373 |
+
...(degraded.length ? { degraded } : {}),
|
| 374 |
};
|
| 375 |
} catch {
|
| 376 |
+
// β THE DEADLINE IS ITS OWN STATUS. A timeout reported as 0 (transport) would be told apart
|
| 377 |
+
// from a dead network by nothing, and the two want different words on screen: one is "we gave
|
| 378 |
+
// up waiting", the other is "we could not reach the server at all".
|
| 379 |
+
return { ok: false, status: timedOut ? NAV_TIMEOUT_STATUS : 0 };
|
| 380 |
+
} finally {
|
| 381 |
+
if (timer !== null) clearTimeout(timer);
|
| 382 |
}
|
| 383 |
}
|
| 384 |
|