diff --git a/Dockerfile b/Dockerfile index 502a1264b4baef91972c60da8ef35e5ff4368b0b..c4758ee1a369c0297b869664edd1ea9be0dd8e13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -126,8 +126,10 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ # S79-T37 (R25): the launcher sizes the web tier to the container. `request_class.py --env` reads # the cgroup CPU and memory limits (never os.cpu_count(), which reports the host) and exports the # uvicorn worker count, heavy concurrency and DuckDB caps. An env value set on the Space wins. -# Several workers start only with AIOS_MIRROR_VERSIONS=1, which opens the mirror read-only in each -# of them, and the shell holds W at 1 without it. 2 vCPU keeps one worker and today's caps. +# Several workers require an explicit AIOS_MIRROR_VERSIONS=1 input and the web role. +# Without that opt-in, --env exports one worker and the shell keeps W at 1. +# Measure unloaded staging freshness and peak RSS before raising its worker count. +# On 2 vCPU, two workers also require AIOS_UVICORN_WORKERS=2. # Several workers load the app through a factory: uvicorn imports a plain app string in its # supervisor too, and that process would run the boot threads without serving. CMD ["sh", "-c", "eval \"$(python api/request_class.py --env)\"; W=1; if [ \"${AIOS_MIRROR_VERSIONS:-0}\" = 1 ]; then W=\"${AIOS_UVICORN_WORKERS:-1}\"; fi; export AIOS_UVICORN_WORKERS=\"$W\"; APP=api.main:app; if [ \"$W\" -gt 1 ]; then APP=\"--factory api.request_class:web_app\"; fi; exec uvicorn $APP --host 0.0.0.0 --port ${PORT} --workers \"$W\" --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"] diff --git a/RELEASES.json b/RELEASES.json index 7ea789d78889c84bbf39bf67479a2e73b3f84b76..21fbe387178b0fd0c6983765d31c364da96de858 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,4 +1,4 @@ { - "current": "9464975c39b", + "current": "0a741b9b3", "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index 71b4e82051a5de0415e2a4d20e973d1069a01f4a..5bbe02664185cd63d64c5df83c2b1d6f5c3e0e8a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9464975c39b +0a741b9b3 diff --git a/api/amazon_relational.py b/api/amazon_relational.py index e16b31bec231a90f9443f3706f991a51381d8646..bf6b8176640c7dbf0eca0c34fa058f7407b8300d 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -284,7 +284,8 @@ _MONEY_INFIX = ("_life_time_value_",) #: multiplied by 100 in the read path (`grid_source`), so 0.15 reads "15%" and never "0.15%". #: ⛔ A rate whose scale was not measured stays out of both sets (see `oos_rate_week_*`). _PERCENT_SUFFIX = ("_percentage", "_percentage_b2_b") -_PERCENT = {"sales_by_asin_refund_rate", "sales_by_asin_refund_rate_b2_b", +_PERCENT = {"browse_node_metrics_occurrence_percentage_all_products", + "sales_by_asin_refund_rate", "sales_by_asin_refund_rate_b2_b", "cart_add_data_asin_cart_add_share", "click_data_asin_click_share", "impression_data_asin_impression_share", "purchase_data_asin_purchase_share", "cart_add_data_total_cart_add_rate", "click_data_total_click_rate", @@ -646,6 +647,8 @@ def _fields_for(cur, key, *, read_through=False): "source": "overlay", "default": name == pinned or name in shown or name in product or name == amazon_schema.STORE, "pinned": name == pinned, + **({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE} + if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}), **({"format": fmt} if fmt else {}), }, flow="amazon_relational")) for lookup in derived.get(name, ()): @@ -765,11 +768,53 @@ def restore_pending(rt=None): return True -def missing_source(key, rt=None, *, pending=None): +_UNRESOLVED = object() + + +def source_status(cur, key): + """D-2643: `{empty, code, reason}` from `amazon_store.table_empty_status` for a present table. + + An empty table always answers. A table with rows answers only beside a recorded Amazon + refusal, with `empty` False. None otherwise, and for keys outside the declared databases. + """ + table = key.removeprefix(PREFIX) + if key not in SOURCE or table not in amazon_store.DATABASE_TABLES: + return None + present = cur.execute(f"SELECT 1 FROM {_quoted(SOURCE[key])} LIMIT 1").fetchone() is not None + # `rows=1` marks a table that holds rows. The producer then answers a refusal only. + status = amazon_store.table_empty_status(cur, table, rows=int(present)) + if not status.get("code") or (present and status["code"] != "refused_by_amazon"): + return None + return {"empty": not present, "code": status["code"], "reason": status.get("reason") or ""} + + +def missing_source(key, rt=None, *, pending=None, cur=None): """The ONE sentence every door gives for an Amazon table this server's mirror does not hold yet. `pending` is a `restore_pending()` answer the caller already holds, so a listing reads the keychain once. + `cur` is the admitted mirror cursor. With it, a saved refusal or a measured empty report names + its own reason instead of the loading sentence. """ + table = key.removeprefix(PREFIX) + # Schema-only probe first: a mirror without sync bookkeeping keeps the loading sentences. + if (cur is not None and key != PREFIX + "inventory" + and table in amazon_store.DATABASE_TABLES and mirror_columns(cur, "_sync_state")): + status = amazon_store.table_empty_status(cur, table, rows=None) + if (status.get("code") and status.get("reason") + and status["reason"] != amazon_store.loading_reason(table)): + return {"missing": status["reason"], + "sourceStatus": {"empty": True, "code": status["code"], "reason": status["reason"]}} + if key == PREFIX + "inventory": + if rt is None: + from harness import runtime + tenant = runtime.tenant_for_store_path() + rt = runtime.get_runtime(tenant) if tenant else None + definitions = rt.get_projection("user_tables", drop=("rows",)) if rt is not None else {} + status = ((definitions or {}).get(key) or {}).get("sourceStatus") or {} + if status.get("empty") is True and isinstance(status.get("reason"), str) and status["reason"].strip(): + return {"missing": status["reason"]} + return {"missing": "Inventory request metadata is unavailable, " + "so retry Amazon sync to record the FBA ledger request."} label = key.removeprefix(PREFIX).replace("_", " ") if restore_pending(rt) if pending is None else pending: return {"missing": f"Amazon {label} is still loading after a server restart, " @@ -790,12 +835,15 @@ def _integer(value): return None if value in (None, "") else int(value) -def grid_source(cur, key, *, columns=None, allow_lookups=True): - """One binding from the current mirror schema, or an explicit unloaded-mirror reason.""" +def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVED): + """One binding from the current mirror schema, or an explicit unloaded-mirror reason. + + `status` is a `source_status` answer the caller already holds. + """ columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS] if not columns: - return missing_source(key) + return missing_source(key, cur=cur) names = {name for name, _dtype in columns} table = key.removeprefix(PREFIX) prepared = {row[0] for row in cur.execute( @@ -886,6 +934,10 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True): "needs_excluded": False, "cols": cols} if table not in amazon_store.DERIVED_TABLES: spec["page_key"] = f"{source}.rowid" + # D-2643: an empty table carries its code and reason to every door that reads this binding. + status = source_status(cur, key) if status is _UNRESOLVED else status + if status is not None: + spec["sourceStatus"] = status return spec @@ -894,12 +946,17 @@ def page_source(cur, key, definition=None, *, allow_lookups=True): columns = mirror_columns(cur, SOURCE[key], strict=True) mandatory = key in READ_THROUGH or bool((definition or {}).get("readThrough")) if not columns: - return missing_source(key) if mandatory else None + return missing_source(key, cur=cur) if mandatory else None + status = _UNRESOLVED if not mandatory: count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0] - if not _read_through(cur, key, count, columns=columns): - return None - return grid_source(cur, key, columns=columns, allow_lookups=allow_lookups) + # D-2643: an empty table, or one beside a recorded refusal, reads through, so its + # binding names its reason in the Data grid. + if count and not _read_through(cur, key, count, columns=columns): + status = source_status(cur, key) + if status is None: + return None + return grid_source(cur, key, columns=columns, allow_lookups=allow_lookups, status=status) def grid_sources(cur=None): @@ -913,7 +970,9 @@ def grid_sources(cur=None): keys = set(READ_THROUGH) for key in present - keys: count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0] - if _read_through(cur, key, count): + # D-2643: an empty or refused table registers too, so the grid opens it on the + # read-through door that carries its reason. + if _read_through(cur, key, count) or source_status(cur, key) is not None: keys.add(key) return {key: grid_source(cur, key) for key in SOURCE if key in keys} finally: @@ -1144,7 +1203,9 @@ def plan_fingerprint(cur, built): canonical = sorted(rows, key=lambda row: json.dumps( row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str)) # S78-T02: an empty population differs from an absent table, so a stale copy is rewritten. - empty = {"empty": True} if built.get(key) == [] else {} + # D-2643: a changed empty reason is a changed publication too. + empty = ({"empty": True, "sourceStatus": source_status(cur, key) + if mirror_columns(cur, SOURCE[key]) else None} if built.get(key) == [] else {}) tables.append({"key": key, "fields": _fields_for(cur, key), "rows": canonical, **empty}) payload = json.dumps(tables, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str) @@ -1377,6 +1438,9 @@ def apply_plan(rt, built, username="amazon", today=None): # S79-T13: each Ads table's description states its history window and any recorded limit. ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror) for key, *_rest in plans if key in _ADS_SPEC} + # D-2643: an emptied stored table keeps the code and reason its mirror answers with. + empty_status = {key: source_status(cur_mirror, key) for key, _label, _fields, rows in plans + if isinstance(rows, list) and not rows} for key, _label, _fields, rows in plans: if isinstance(rows, dict): continue @@ -1513,17 +1577,18 @@ def apply_plan(rt, built, username="amazon", today=None): saved = stored.get(row["_id"]) if numeric and isinstance(saved, dict): saved.update({name: row[name] for name in numeric if name in row}) - if key == PREFIX + "inventory": - status = rows.get("sourceStatus") if isinstance(rows, dict) else None - if status is not None: - doc[key]["sourceStatus"] = dict(status) - else: - doc[key].pop("sourceStatus", None) + status = rows.get("sourceStatus") if isinstance(rows, dict) else empty_status.get(key) + if status is not None: + doc[key]["sourceStatus"] = dict(status) + else: + doc[key].pop("sourceStatus", None) # DPP 1.8 attribution is explicit in the stored definition, not inferred from a # mutable display label or from the process having imported this module. doc[key]["connector"] = source_attribution(key) if ads_descriptions.get(key): doc[key]["description"] = ads_descriptions[key] + if key in (PREFIX + "review_topics", PREFIX + "review_trends"): + doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE if key == PREFIX + "listing_items": doc[key]["amazonListingIdentity"] = listing_version return doc diff --git a/api/application_registry.py b/api/application_registry.py index 878aa790f1238b776a98b99bae9bf14af4241218..bc5c7f81a18f77ee9b642ce384013d5e649c21a9 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -1418,9 +1418,10 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre def _nav_placement(app, index): """Put default Project management entries first without rewriting stored placements.""" placement = app.get("navPlacement") + # Every install stores the template default {"section": "app", "order": 0}, so order 0 is the + # default here, not a choice; any other stored placement wins. if (app.get("templateKind") == "amazon_profit" and (not placement or placement == {"section": "app", "order": 0})): - # A zero placement already asks for first place. Break its default tie in the rail. return {"section": "app", "order": -1} return placement or {"section": "app", "order": index} diff --git a/api/assistant_mirror.py b/api/assistant_mirror.py index 0b5eb67f4373ea7f6be9061af4c09e7b267fd678..1c42fbead0f78371ea066de1ef9388a622dc133e 100644 --- a/api/assistant_mirror.py +++ b/api/assistant_mirror.py @@ -166,6 +166,14 @@ class ReadinessSnapshot: return self._loading[key] +def amazon_empty_reason(status): + """The one-line reason in an Amazon `sourceStatus`, or "" when it names no code and reason.""" + if not isinstance(status, dict) or not status.get("empty") or not status.get("code"): + return "" + reason = status.get("reason") + return " ".join(reason.split()) if isinstance(reason, str) else "" + + def unavailable_reason(table_key, definition=None, *, snapshot=None): """The ONE sentence `list_databases` and `query_database` both give when the mirror holds no rows. @@ -177,6 +185,11 @@ def unavailable_reason(table_key, definition=None, *, snapshot=None): if key.startswith("ut_amzn_"): import amazon_relational as amazon if key in amazon.SOURCE: + # D-2643: the stored definition's empty reason, from the one Amazon producer. + status = (definition or {}).get("sourceStatus") if isinstance(definition, dict) else None + reason = amazon_empty_reason(status) + if reason: + return reason pending = snapshot.amazon_pending() if snapshot is not None else None return amazon.missing_source(key, pending=pending)["missing"] spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key) @@ -222,11 +235,38 @@ def binding(cur, table_key): which is not the same as the grid being empty. """ try: - return _rot()._source_for(cur, str(table_key or "")) + spec = _rot()._source_for(cur, str(table_key or "")) except Exception: # noqa: BLE001 # A spec that cannot be resolved is indistinguishable, to every caller, from a grid with # no binding: neither can be read. Fail closed to the reportable answer. return None + return amazon_lag(cur, str(table_key or ""), spec) + + +def amazon_lag(cur, table_key, spec): + """D-2642: a dated Amazon binding with rows also carries its newest row date, cadence and lag line. + + The keys come from `amazon_store.table_empty_status`, the one per-table producer. A trailing table + also gets a `source_behind` limit. A status that cannot be read carries the unknown lag line. + """ + if not spec or spec.get("missing") or not table_key.startswith("ut_amzn_"): + return spec + from harness import amazon_store + table = table_key.removeprefix("ut_amzn_") + if table not in amazon_store.LAG_DATE_COLUMNS: + return spec + try: + if cur.execute(f'SELECT 1 FROM "{amazon_store.table_name(table)}" LIMIT 1').fetchone() is None: + return spec + status = amazon_store.table_empty_status(cur, table, rows=1, lag=True) + dated = {name: status[name] for name in amazon_store.LAG_KEYS if name in status} + except Exception: # noqa: BLE001 + dated = amazon_store.unknown_lag(table) + spec = {**spec, "sourceStatus": {"empty": False, **(spec.get("sourceStatus") or {}), **dated}} + if dated.get("lag"): + spec["limits"] = [*(spec.get("limits") or []), + {"effect": "source_behind", "code": dated.get("lagCode"), "cause": dated["lag"]}] + return spec def is_bound(cur, table_key, *, snapshot=None): diff --git a/api/automation_engine.py b/api/automation_engine.py index f18bb3403cb3092ef0c6e318d750f2d18722b8cf..07d46ff706d3dcd061bb1f6c7ae6f60ec7cee07b 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -1111,7 +1111,7 @@ def retire_automation_stage_fields(rt, tables=None): changed["tables"].add(str(table_key)) return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") return {"tables": len(changed["tables"]), "fields": changed["fields"], "cells": changed["cells"]} @@ -1155,7 +1155,9 @@ def bind_unbound_fields(rt): "or delete it") return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + for _aid in sorted({a for a in plan.values() if a}): + _ut().index_flow_tables(_aid, sorted({tk for (tk, _fk), a in plan.items() if a == _aid}), rt) + _ut_update(rt, _up, flush="sync") bound = sum(1 for v in plan.values() if v) return bound, len(plan) - bound @@ -1341,7 +1343,10 @@ def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", t["recordMode"] = record_mode return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + if flow_tag: + # D-179: index first, so a failed stamp leaves a superset and never a missed table. + _ut().index_flow_tables(flow_tag, (key,), rt) + _ut_update(rt, _up, flush="sync") return key @@ -1353,7 +1358,7 @@ def ut_write_rows(rt, key, rows): if t is not None: t["rows"] = rows return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): @@ -1368,7 +1373,7 @@ def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): if tt is not None: tt["rows"] = rws return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") IG_FIELD_DESCRIPTIONS = { @@ -3746,6 +3751,11 @@ def create(rt, raw, username="", notes=None): # against a PREVIOUS definition, and a create has none — R10's columns must exist before a run, # so this call stays unconditional. _store_update(rt, _up, flush="async") + # D-179: a new flow has tagged nothing, so its flow-index entry starts complete and empty. + try: + _ut().index_flow_tables(defn["id"], (), rt, create=True) + except Exception as e: # noqa: BLE001 + print(f"[aios-auto] D-179 flow index entry not started: {type(e).__name__}") _presets_after_write(rt, defn, username) # R10: the columns exist before a run if defn.get("trigger"): _seed_event_state(rt, defn) @@ -4104,6 +4114,23 @@ def remove(rt, auto_id): # ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double # had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the # PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts. + # S85-T173 (D-179): a flow with an index entry skips the scan and patches only its tables, + # through a write that declares them (`user_tables.update_tables`). A flow with no entry, one + # that tagged before the index existed, keeps the scan and the whole write below. + _indexed = _ut().flow_index_tables(aid, rt) + if _indexed is not None: + if not _indexed: + return + _note = f"its automation was deleted {_stamp(_authored_now())}" + + def _disable_table(_tk, table): + for f in (table.get("fields") or []): + a = f.get("automation") if isinstance(f, dict) else None + if isinstance(a, dict) and str(a.get("flowId") or "") == aid: + a["disabled"] = True + a["statusNote"] = _note + _ut().update_tables(_indexed, _disable_table, rt, flush="async") + return try: import core.user_tables as _ut_defs # noqa: PLC0415 _scan = dict(_ut_defs.all_defs(rt) or {}) @@ -4130,7 +4157,7 @@ def remove(rt, auto_id): # person waits. Async for the reason given above — the disable is visible to the next read # immediately; only the upload is deferred, and it coalesces with any other pending write to # the same key instead of racing it. - rt.update(UT_STORE_KEY, _disable, flush="async") + _ut_update(rt, _disable, flush="async") # --------------------------------------------------------------------------------------------- @@ -5926,7 +5953,7 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): if needs: break if needs: - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") return changed @@ -6413,7 +6440,7 @@ def retract_foreign_presets(rt, log=print): for k in stats["kept"])) return stats stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") log(f"[aios-auto] retract: {stats['columns']} foreign column(s), {stats['cells']} cell(s) and " f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)") if stats["kept"]: @@ -6796,7 +6823,7 @@ def backfill_post_snapshot_grain(rt, log=print): table.setdefault("rows", {}).setdefault(rid, {}).update(values) return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind") return len(changes) @@ -6921,6 +6948,110 @@ def _as_derived_field(field): "type": "rollup", "rollup": _derived_profile_bag(key)} +def _ig_profile_plan(tbl, want): + """One profile table's migration as `(fields, rows, counts)`, or None when it has nothing to do. + + It edits copies and reads no store, so `migrate_ig_tables` runs it on the LOCKED table inside + the write: a field deleted after an earlier read is not in that table, so it is not re-added. + """ + counts = dict.fromkeys(("tables", "retyped", "converted", "stamped", "merged", "pinned", + "flagged", "droppedName", "droppedRecentPosts", + "droppedPostHashtags", "droppedTracked", "droppedTrackedCells"), 0) + fields = [dict(f) for f in tbl.get("fields") or []] + rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()} + # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched, + # so the cell conversions below can never run twice on the same value. + stale = [f for f in fields + if f.get("key") in want and f.get("type") != want[f["key"]] + and f.get("type") in ("text", "", None)] + stale_keys = {f["key"] for f in stale} + has_platform = any(f.get("key") == "platform" for f in fields) + needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()] + # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING. + # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the + # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in + # production needs them stamped here, and that is the whole reason this branch exists. + # + # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule, + # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which + # is exactly why these are safe to run on every write where a value conversion would not be. + pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \ + if PRESET_PINNED_KEY else None + flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \ + if PRESET_FLAG_KEY else None + # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write + # doors, and a migration is not exempt from it. If somebody has already flagged another + # column on this table, stamping the handle too would leave the table in a state its own + # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would + # silently bind to whichever came first in the field list. Leave it alone and pin only. + other_profile = next((f for f in fields if f is not flag_f + and isinstance(f.get("profile"), dict)), None) + want_pin = pin_f is not None and pin_f.get("pinned") is not True + want_flag = (flag_f is not None and other_profile is None + and not isinstance(flag_f.get("profile"), dict)) + vestigial = _vestigial_name_field(fields, rows) + retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} + if _retired_tracked_field(fields) is not None: + retired.add(TRACKED_KEY) + orphan_tracked = _orphan_tracked_rows(fields, rows) + if (not stale_keys and has_platform and not needs_stamp + and not want_pin and not want_flag and vestigial is None and not retired + and not orphan_tracked): + return None + counts["tables"] += 1 + if want_pin: + pin_f["pinned"] = True + counts["pinned"] += 1 + if want_flag: + flag_f["profile"] = {"source": PROFILE_SOURCE_IG} + counts["flagged"] += 1 + if vestigial is not None: + # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer + # exists is invisible everywhere except the next export, where it reappears as a column + # nobody declared. `_vestigial_name_field` has already proven every one of them blank. + fields = [f for f in fields if f is not vestigial] + for r in rows.values(): + r.pop("name", None) + counts["droppedName"] += 1 + if retired: + fields = [f for f in fields if f.get("key") not in retired] + for r in rows.values(): + for key in retired: + r.pop(key, None) + counts["droppedRecentPosts"] += int("posts" in retired) + counts["droppedPostHashtags"] += int("post_hashtags" in retired) + counts["droppedTracked"] += int(TRACKED_KEY in retired) + for rid in orphan_tracked: + # D-74: a tick whose column was already gone. Counted apart from the column drop + # because they answer different questions — "how many tables still declared it" and + # "how many rows were still carrying it after the declaration went". + rows[rid].pop(TRACKED_KEY, None) + counts["droppedTrackedCells"] += 1 + + for key in stale_keys: + conv = _MIGRATE_CONVERT.get(key) + if not conv: + continue # int/checkbox already store the right text shape + for r in rows.values(): + if str(r.get(key) or "").strip(): + new = conv(r[key]) + # "" from a converter means UNREADABLE, and the cell keeps its original text. + # A migration that empties a cell is indistinguishable from one that moved it. + if new: + r[key], counts["converted"] = new, counts["converted"] + 1 + for f in stale: + f["type"] = want[f["key"]] + counts["retyped"] += 1 + for rid in needs_stamp: + rows[rid]["platform"] = PLATFORM_INSTAGRAM + counts["stamped"] += 1 + if not has_platform: + fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields + rows, merged = _merge_candidates(rows) + counts["merged"] += merged + return fields, rows, counts + + def migrate_ig_tables(rt, log=print, only=""): """Bring every Instagram Profile and canonical child table onto one current contract. @@ -6986,115 +7117,36 @@ def migrate_ig_tables(rt, log=print, only=""): if only and not targets: return stats for table_key in targets: + # D-118: this lenient read only decides whether there is work. `_up` plans again on the + # locked table, so a field deleted after this read stays deleted and keeps its rows. tbl = ut_get(rt, table_key) or {} - fields = [dict(f) for f in tbl.get("fields") or []] - rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()} - # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched, - # so the cell conversions below can never run twice on the same value. - stale = [f for f in fields - if f.get("key") in want and f.get("type") != want[f["key"]] - and f.get("type") in ("text", "", None)] - stale_keys = {f["key"] for f in stale} - has_platform = any(f.get("key") == "platform" for f in fields) - needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()] - # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING. - # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the - # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in - # production needs them stamped here, and that is the whole reason this branch exists. - # - # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule, - # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which - # is exactly why these are safe to run on every write where a value conversion would not be. - pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \ - if PRESET_PINNED_KEY else None - flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \ - if PRESET_FLAG_KEY else None - # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write - # doors, and a migration is not exempt from it. If somebody has already flagged another - # column on this table, stamping the handle too would leave the table in a state its own - # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would - # silently bind to whichever came first in the field list. Leave it alone and pin only. - other_profile = next((f for f in fields if f is not flag_f - and isinstance(f.get("profile"), dict)), None) - want_pin = pin_f is not None and pin_f.get("pinned") is not True - want_flag = (flag_f is not None and other_profile is None - and not isinstance(flag_f.get("profile"), dict)) - vestigial = _vestigial_name_field(fields, rows) - retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} - if _retired_tracked_field(fields) is not None: - retired.add(TRACKED_KEY) - orphan_tracked = _orphan_tracked_rows(fields, rows) - if (not stale_keys and has_platform and not needs_stamp - and not want_pin and not want_flag and vestigial is None and not retired - and not orphan_tracked): + if _ig_profile_plan(tbl, want) is None: continue - stats["tables"] += 1 - if want_pin: - pin_f["pinned"] = True - stats["pinned"] += 1 - if want_flag: - flag_f["profile"] = {"source": PROFILE_SOURCE_IG} - stats["flagged"] += 1 - if vestigial is not None: - # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer - # exists is invisible everywhere except the next export, where it reappears as a column - # nobody declared. `_vestigial_name_field` has already proven every one of them blank. - fields = [f for f in fields if f is not vestigial] - for r in rows.values(): - r.pop("name", None) - stats["droppedName"] += 1 - if retired: - fields = [f for f in fields if f.get("key") not in retired] - for r in rows.values(): - for key in retired: - r.pop(key, None) - stats["droppedRecentPosts"] += int("posts" in retired) - stats["droppedPostHashtags"] += int("post_hashtags" in retired) - stats["droppedTracked"] += int(TRACKED_KEY in retired) - for rid in orphan_tracked: - # D-74: a tick whose column was already gone. Counted apart from the column drop - # because they answer different questions — "how many tables still declared it" and - # "how many rows were still carrying it after the declaration went". - rows[rid].pop(TRACKED_KEY, None) - stats["droppedTrackedCells"] += 1 - - for key in stale_keys: - conv = _MIGRATE_CONVERT.get(key) - if not conv: - continue # int/checkbox already store the right text shape - for r in rows.values(): - if str(r.get(key) or "").strip(): - new = conv(r[key]) - # "" from a converter means UNREADABLE, and the cell keeps its original text. - # A migration that empties a cell is indistinguishable from one that moved it. - if new: - r[key], stats["converted"] = new, stats["converted"] + 1 - for f in stale: - f["type"] = want[f["key"]] - stats["retyped"] += 1 - for rid in needs_stamp: - rows[rid]["platform"] = PLATFORM_INSTAGRAM - stats["stamped"] += 1 - if not has_platform: - fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields - rows, merged = _merge_candidates(rows) - stats["merged"] += merged - - field_changes = {} - - def _up(cur, _f=fields, _r=rows): + field_changes, planned = {}, {} + + def _up(cur): field_changes.clear() + planned.clear() cur = cur or {} - t = dict(cur.get(table_key) or {}) + if table_key not in cur: + return cur # deleted since the read: never re-create it + t = dict(cur[table_key] or {}) + plan = _ig_profile_plan(t, want) + if plan is None: + return cur old_keys = _ut()._field_list_keys(t) - t["fields"], t["rows"] = _f, _r + t["fields"], t["rows"], counts = plan new_keys = _ut()._field_list_keys(t) field_changes.update(removed=[key for key in old_keys if key not in new_keys], added=[key for key in new_keys if key not in old_keys]) + planned.update(counts) cur[table_key] = t return cur rt.update(UT_STORE_KEY, _up) + for name, count in planned.items(): + stats[name] += count + _ut()._log_store_write(rt, table_key, 'migrate_ig_tables') _ut()._log_field_list_change(rt, table_key, 'automation', 'migrate_ig_tables', field_changes) # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION. # @@ -7750,7 +7802,7 @@ def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print): cur[snap_key]["rows"] = merged return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") return written @@ -8401,7 +8453,7 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste _refresh_relations_inplace(cur, log=log) return cur - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables + _ut_update(rt, _up, flush="sync") # ONE coalesced update for all four tables queued = queue_pending_metric_snapshots( rt, str(defn.get("id") or ""), pending_metrics, workflow_id=str(defn.get("workflowId") or "")) @@ -9153,8 +9205,7 @@ def assemble_statements(defn, log=print): if left > 0: # R6's second sentence: a limit that bites is REPORTED, with its cause and what to do. notes.append(f"{left} more customers matched but one batch holds " - f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest, or " - f"narrow the tier filter") + f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest.") return batch, notes @@ -12805,7 +12856,7 @@ def _enrich_flush(rt, defn, username, acc, log): _refresh_relations_inplace(cur, log=log) return cur - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables + _ut_update(rt, _up, flush="sync") # ONE coalesced update for all three tables # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which # is the failure a chart cannot show you. capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"] @@ -12942,7 +12993,7 @@ def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=prin return cur if written: - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for every table + _ut_update(rt, _up, flush="sync") # ONE coalesced update for every table return inserted, capped, missing @@ -13518,7 +13569,7 @@ def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments _refresh_relations_inplace(cur, log=log) return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") try: import ig_master status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots) @@ -14084,9 +14135,9 @@ def compute_metric_cells(rt, table_key, today=None, tables=None, persist=False): if owned: _up(blob) if persist: - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") return len(changes) - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") return len(changes) @@ -14108,6 +14159,17 @@ def compute_metric_cells(rt, table_key, today=None, tables=None, persist=False): # posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The # moment one writes a number nothing else can re-derive, it has become a third copy. +def _ut_update(rt, fn, **kw): + """Write the whole `user_tables` document and log the D-118 write line for it.""" + out = rt.update(UT_STORE_KEY, fn, **kw) + try: + import sys + _ut()._log_store_write(rt, '*', sys._getframe(1).f_code.co_name) + except Exception: # noqa: BLE001 + pass # the log line never fails the write it describes + return out + + def _ut(): """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic. @@ -15549,7 +15611,7 @@ def compute_relation_cells(rt, table_key, tables=None): tt.setdefault("rows", {}).setdefault(r, {}).update(vals) return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") return len(changes) @@ -15702,7 +15764,7 @@ def refresh_relations(rt, log=print, tables=None, persist=True, *, target=None): return cur # A cold process must see recovered cells before their old refusal disappears. - saved = rt.update(UT_STORE_KEY, _up, flush='sync') + saved = _ut_update(rt, _up, flush='sync') if saved is False: raise RuntimeError('The relation refresh was not committed.') _persist_relation_limits(rt, results) @@ -15771,7 +15833,7 @@ def purge_subject(rt, handle): t["rows"] = nxt return cur - rt.update(UT_STORE_KEY, _up, flush="sync") + _ut_update(rt, _up, flush="sync") import ig_master for bucket, n in (ig_master.purge_handle(subject) or {}).items(): counts[f"master:{bucket}"] = n @@ -16097,7 +16159,11 @@ def clean_trigger(raw, previous=None): return None, str(exc) if key in TRIGGER_TABLE_KEYS: table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): + native_conditions = key == "event_field" and table in ("customer", "product") + if table and not table.startswith(UT_PREFIX) and not native_conditions: + if key == "event_field": + return None, ("Event conditions watch customer, product or blank databases (ut_*). " + f"{table!r} is not one") return None, ("event triggers watch blank databases (ut_*) this wave. " f"{table!r} is not one") out["table"] = table diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index 9518f992896635a90b478eb76acc9cd2d38e2e48..79746f0b544588f8eb0dc77f2279ee3233b451f4 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -638,35 +638,9 @@ def database_keys(): def read_database_enablement(rt): - """Return this tenant's fail-closed Amazon database enablement state. - - An absent document enables all declared tables. Unknown keys are ignored with a logged note. - Every stored value must be a real boolean. A malformed or unreadable document disables every row in the detail - response and marks the capability unavailable; it never guesses through corrupt tenant state. - """ - keys = database_keys() - try: - current = rt.get(ENABLEMENT_KEY) if rt is not None else None - except Exception as exc: # noqa: BLE001 - raise_if_resource_error(exc) - return {"available": False, "databases": {key: False for key in keys}} - # Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are - # the migration default rather than malformed state. - if current is None or current == {}: - return {"available": True, "databases": {key: True for key in keys}} - if not isinstance(current, dict): - return {"available": False, "databases": {key: False for key in keys}} - stored = current.get("databases") - if not isinstance(stored, dict): - return {"available": False, "databases": {key: False for key in keys}} - if any(type(value) is not bool for value in stored.values()): - return {"available": False, "databases": {key: False for key in keys}} - if set(stored) - set(keys): - logging.getLogger(__name__).warning("Ignoring unknown Amazon database enablement keys") - return {"available": True, - # Absent keys stay on so a later report addition does not silently disappear for - # every tenant that saved an older version of this document. - "databases": {key: stored.get(key, True) for key in keys}} + """Use the sync engine's fail-closed evaluator for this tenant's database choices.""" + from harness import amazon_store as az + return az.read_database_enablement(rt) def set_database_enabled(rt, database_key, enabled): diff --git a/api/deps.py b/api/deps.py index c23bf3a7369f3731898aaa96c8a30e21a67b9ee1..10f152500872e791025ddae7db75c1bb6310e292 100644 --- a/api/deps.py +++ b/api/deps.py @@ -776,10 +776,11 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No # (D-2630: describe answers the field list of a million-row table without counting it); # ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631); # ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1). + source_version = None try: if mode == "query_schema": - return [], fields, {"read": "connector mirror"}, definition.get("label") or database - if isinstance(mode, tuple) and mode and mode[0] == "aggregate": + rows, source_version = [], {"read": "connector mirror"} + elif isinstance(mode, tuple) and mode and mode[0] == "aggregate": if len(mode) > 2: values, total, groups, group_total, stamp = _assistant_mirror_groups( session, database, cur, spec, fields, mode[1], mode[2], @@ -792,9 +793,8 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys) if isinstance(extra, dict): extra["aggregates"], extra["total"] = values, total - return ([], fields, {"read": "connector mirror", "rows": total}, - definition.get("label") or database) - if isinstance(mode, tuple) and mode and mode[0] == "page": + rows, source_version = [], {"read": "connector mirror", "rows": total} + elif isinstance(mode, tuple) and mode and mode[0] == "page": rows, stamp, total = assistant_mirror.read_page( session, database, cur, spec, fields, offset=mode[1], limit=mode[2], wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys, @@ -810,12 +810,19 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No _assistant_refuse(exc.status, exc.code, exc.message) raise # unreachable; see above + if isinstance(extra, dict): + if isinstance(spec.get("sourceStatus"), dict): + extra["sourceStatus"] = copy.deepcopy(spec["sourceStatus"]) + if isinstance(spec.get("limits"), list): + extra["limits"] = copy.deepcopy(spec["limits"]) + # ⭐ PROVENANCE A CITATION CAN CARRY. `routes_query._system_prompt` runs # `json.dumps(source_version, default=str)`, so this stays plain JSON: when the mirror last # reconciled this table against its connector, and how many rows the answer was computed over. # The count is here because the model is told the record count separately, and a reader # checking one against the other should find the same integer. - source_version = {"read": "connector mirror", "refreshed": stamp, "rows": total} + if source_version is None: + source_version = {"read": "connector mirror", "refreshed": stamp, "rows": total} return rows, fields, source_version, definition.get("label") or database @@ -1294,6 +1301,10 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None, out["groups"] = copy.deepcopy(extra["groups"]) out["total_groups"] = extra["group_total"] out["source_version"]["refreshed"] = copy.deepcopy(extra["group_stamp"]) + if isinstance(extra.get("sourceStatus"), dict): + out["sourceStatus"] = copy.deepcopy(extra["sourceStatus"]) + if isinstance(extra.get("limits"), list): + out["limits"] = copy.deepcopy(extra["limits"]) return out @@ -1830,10 +1841,11 @@ def _view_as_user(request, actor, tenant, rt): if not target: raise _view_as_refused() rec = copy.deepcopy(_session_registry(target).get(target)) - if (not rec or not rec.get("active", True) or rec.get("role", "user") != "user" - or str(rec.get("tenant") or "").strip().lower() != tenant): + if not rec or not rec.get("active", True) or rec.get("role", "user") != "user": raise _view_as_refused() member = users._public(target, rec) + if member.get("tenant") != tenant: + raise _view_as_refused() audit_key = (tenant, actor["username"], target) if getattr(request.state, "_view_as_audited", None) != audit_key: from core import activity_log diff --git a/api/intelligence_store.py b/api/intelligence_store.py index bdfdf08aaf42a72b1a2550315f175b79e3bb1b98..acdad3245411349b7d05cd33882ecd3cd8df2fa4 100644 --- a/api/intelligence_store.py +++ b/api/intelligence_store.py @@ -1628,6 +1628,7 @@ def decide_proposal(session, proposal_id, decision): raise IntelligenceError("person_required", "A person must review this change.", 403) if decision not in ("approve", "reject"): raise IntelligenceError("bad_decision", "Choose Approve or Decline.") + saved = {} def decide(raw): saved.clear() diff --git a/api/loopy_bulk.py b/api/loopy_bulk.py index b8aa4d364318912af68ac0c517177d0586053f34..5b5d8a435fdc4160a962c54465c78bd96ef1cc83 100644 --- a/api/loopy_bulk.py +++ b/api/loopy_bulk.py @@ -5,7 +5,7 @@ from urllib.parse import quote from loopy_chat_link_tools import _name_field, _phone_field from loopy_chat_links import validate_message, wa_link -from loopy_diagnostics import ScopedOdoo, amount, bind_functions, identities, plain +from loopy_diagnostics import ScopedOdoo, amount, ar_scope, bind_functions, identities, plain from loopy_file_tools import _secure_link, read_all CUSTOMERS = "customer_data" @@ -72,25 +72,18 @@ def _group(rows, today): def validate(session, records, overdue, grouped, *, today, partner_ids): """Tie each grouped amount and overdue age to independent, permission-bounded Odoo reads.""" import wakeups - from core import odoo as O, perm_scope, perms + from core import odoo as O if session.tenant != session.runtime.key: raise ValueError("The chase list tenant must match its runtime.") - def narrowed(database): - grant = perm_scope.assistant_entry(session.user, database) or {} - return (bool(grant.get("filter")) - or perm_scope.row_scope_applies(session.user, database, st=session.runtime) - or "All" not in perms.allowed_bu_labels(session.user) - or any(value is not None for value in perm_scope.derive_pool_scope(session.user, database))) - - invoice_wall, customer_wall = narrowed(INVOICES), narrowed(CUSTOMERS) - ids = identities(records, "odoo_id") shown_ids = identities(overdue, "odoo_id") source = session.runtime.odoo_source() if source is None: raise ValueError("Connect Odoo before building a chase list.") - restrictions = {"account.move": ([("id", "in", ids)] if invoice_wall else []) - + ([("partner_id", "in", partner_ids)] if customer_wall else [])} + # Unrestricted validation must see invoices absent from the permitted mirror snapshot. + restrictions = ar_scope(session, customers=True, native=True, records={ + INVOICES: records, CUSTOMERS: [{"pid": pid} for pid in partner_ids]}) + walls = {field: set(values) for field, _operator, values in restrictions["account.move"]} domain, _accepts = wakeups._ar_rule(today, 0) with source._bound(): client = ScopedOdoo(O.get_odoo(), restrictions) @@ -101,7 +94,8 @@ def validate(session, records, overdue, grouped, *, today, partner_ids): partner = row.get("partner_id") pid = partner[0] if isinstance(partner, (list, tuple)) and partner else partner if (type(pid) is not int or pid <= 0 - or customer_wall and pid not in partner_ids or invoice_wall and row["id"] not in ids): + or "partner_id" in walls and pid not in walls["partner_id"] + or "id" in walls and row["id"] not in walls["id"]): raise ValueError("Odoo returned an unreadable invoice.") native_rows.append({"odoo_id": row["id"], "partner_id": pid, "residual": row.get("amount_residual_signed"), "due_date": row.get("invoice_date_due")}) diff --git a/api/main.py b/api/main.py index 63590dbf10d4fe0adb4f75cbcb00cde00ab6b1f3..13bfa1a31c95332d6418b62bc488e6a7ec9af869 100644 --- a/api/main.py +++ b/api/main.py @@ -388,27 +388,22 @@ def public_freeze_notice(): @app.get("/api/health") -async def health(challenge: str = ""): - """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not - reveal anything about the deployment beyond "the process is up". No version, no tenant list, - no config: a health endpoint is the one URL every scanner finds first. - - ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned - release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated - caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind - a session — and the authoritative copy is the `VERSION` file in the Space repo, which - `deploy_web.space_version()` reads without needing the app to be up at all. - - W-265 adds a possession challenge for the deployer. It reveals only whether this process was - born after one high-entropy, single-deploy nonce was stamped; it never returns the nonce, - version, Space id, or configuration. A normal liveness request keeps the exact - ``{"ok": true}`` response shape. - """ +def health(challenge: str = ""): + """Check a strict store read without disclosing tenants, configuration or exception text.""" + # A cached availability flag cannot detect a connection lost after a successful first read. + try: + ok = bool(store.available()) + if ok: + store._read_strict("sync_settings") + except Exception: + ok = False + body = {"ok": ok} if challenge: expected = os.environ.get("AIOS_DEPLOY_NONCE") or "" - return {"ok": True, - "deployReady": bool(expected and hmac.compare_digest(challenge, expected))} - return {"ok": True} + body["deployReady"] = bool(ok and expected and hmac.compare_digest(challenge, expected)) + if not ok: + return JSONResponse(body, status_code=503, headers={"Cache-Control": "no-store"}) + return body def _mirror_answers(rt=None): @@ -1195,9 +1190,10 @@ def _pull_meta(why, tenant_key=None): rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key, log=lambda *_a: None, insights_days=7) outcomes[key] = mirror_store._sync_result("meta", rep)[1] - _status(rt, "meta_ads", "done" if outcomes[key] else "failed") + detail = _sync_now_failure(" ".join(rep.get("problems") or [])) if not outcomes[key] else "" + _status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail) for problem in rep.get("problems") or []: - print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}") + print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}") except Exception as exc: print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}") _failed = globals().get("_publish_connector_status") @@ -1526,6 +1522,20 @@ def _sync_now_cause(cause): return "unclassified", "The connector reported a failure. Inspect its sync report and retry." raw = str(cause) text = raw.lower() + # Authored transport causes precede generic timeout, credential and connection matching. + meta_transport = ( + ("meta_dns", "Meta Ads could not resolve the provider's host name. Check the server's DNS connection."), + ("meta_certificate", "Meta Ads could not verify the server certificate. Check the server's trusted certificates and clock."), + ("meta_tls", "Meta Ads could not establish a secure connection. Check the server's TLS connection."), + ("timeout", "The Meta Ads request timed out. Retry the sync after checking the server's network connection."), + ("meta_refused", "The connection to Meta Ads was refused. Check the server's firewall and proxy settings."), + ("meta_reset", "The connection to Meta Ads was reset. Retry the sync after checking the server's network connection."), + ("meta_route", "The server cannot reach the Meta Ads network. Check its network route and firewall settings."), + ("meta_network", "The network request to Meta Ads failed. Check the server's network and proxy settings, then retry the sync."), + ) + for classifier, sentence in meta_transport: + if sentence.split(". ", 1)[0].lower() + "." in text: + return classifier, sentence causes = ( ("not_offered", ("not_offered", "not offered", "does not offer this report"), "Amazon does not offer this report to this seller. Its last published rows are kept."), @@ -3215,15 +3225,17 @@ def _entity_source_counter(ds): """ import inspect as _inspect_counts try: - if "after_entity" not in _inspect_counts.signature(ds.sync_all).parameters: - return None rt = ds._resident_odoo_runtime() + supports_hook = "after_entity" in _inspect_counts.signature(ds.sync_all).parameters except Exception: # noqa: BLE001 return None tenant = getattr(rt, "key", None) if not tenant: return None + # A new pass cannot inherit a failed pass's counts when its adapter lacks the hook. counted = _PASS_SOURCE_COUNTS[tenant] = {} + if not supports_hook: + return None per_connection = {} def after_entity(key): @@ -3380,6 +3392,11 @@ def _store_resync_loop(): return import time as _t from core import connector_sync + from harness import amazon_store as _phase_amazon + import odoo_relational as _phase_odoo + import sync_alerts as _phase_alerts + if _phase_alerts.environment() in {"LIVE", "STAGING"}: + _phase_amazon.register_schedule_offset(_phase_odoo.slot_offset_seconds) _restore_sync_request_desks() # Restore/rebuild precedes cadence/source hydration as well as the Odoo backfill. import amazon_relational as _ar_boot @@ -4155,6 +4172,8 @@ def _seed_mirror_if_absent(): def restore(tenant_key, rt): try: + if _ds.odoo_mirror_restored(tenant_key, rt): + return result = _ds.restore_odoo_mirror(tenant_key, rt) except Exception as exc: result = refused(tenant_key, exc) @@ -4336,7 +4355,7 @@ def _boot_mirrors(): # With several web workers only the primary restores the mirror. The others read it. _seed_mirror_if_absent() if _role_boot.role() == "web": - # Readiness also waits for the worker's boot hand-offs and the mirror's own completeness. + # Readiness checks this process's Odoo mirror and reads while worker catch-up continues. _role_boot.mark_ready("mirror", True) # Every connector but Odoo reaches the web process through the worker's hand-off. print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker") @@ -4348,7 +4367,7 @@ def _boot_mirrors(): _restore_amazon_mirror("boot") _restore_declared_mirrors("boot") finally: - # Every boot hand-off is on disk now; the web process's readiness waits for this line. + # Publish restore progress without holding the web's installed Odoo reads. _role_boot.note_boot_restore() return _restore_amazon_mirror("boot") diff --git a/api/odoo_relational.py b/api/odoo_relational.py index cba4877f67388ef890300d56aebc16ed6bb66ed0..bd5a194b45543f2fed76a5141fa2fba98ca634f5 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -7819,12 +7819,15 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio key = table_key_for(base_key, connection_id) source_count = None source_mode = "unavailable" + source_basis = "unavailable" source_failure = None if source_counts and bucket in source_counts: source_count, source_mode = source_counts[bucket] + source_basis = "mirror_pull" elif source_client is not None: try: source_count, source_mode = _source_domain_count(source_client, bucket) + source_basis = "reconciliation" except Exception as exc: # noqa: BLE001 source_failure = { "code": "source_reconciliation_failed", "type": type(exc).__name__, @@ -7887,6 +7890,7 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio receipts[key] = { **identity, "state": "partial" if partial else "succeeded", "sourceCount": source_count, "sourceCountMode": source_mode, + "sourceCountBasis": source_basis, "sourceReconciliation": source_reconciliation, "mirrorCount": count, "materializedCount": materialized_count, "servingCount": count, "readThrough": read_through, "duplicateCount": 0, @@ -7908,7 +7912,8 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio errors[key] = failure receipts[key] = { **identity, "state": "failed", "sourceCount": source_count, - "sourceCountMode": source_mode, "sourceReconciliation": "unconfirmed", + "sourceCountMode": source_mode, "sourceCountBasis": source_basis, + "sourceReconciliation": "unconfirmed", "mirrorCount": None, "materializedCount": None, "servingCount": None, "readThrough": is_read_through_key(key), "duplicateCount": None, "freshness": {"state": "failed", "capturedAt": captured_at}, diff --git a/api/process_role.py b/api/process_role.py index 54edc51c565e4bdeabb6d990a821bdb3ea27fd8d..dd08e33d2e06b2d0978c1800e374039c06d8c606 100644 --- a/api/process_role.py +++ b/api/process_role.py @@ -439,8 +439,7 @@ _READY_DEFAULT = {"mirror": "The mirror is not installed yet.", "tenant_keys": "The tenant data keys have not been checked yet."} _READINESS = {} _BOOT_RESTORE = {} -# The first full pass latches the boot conditions: a later worker restart or a fresh hand-off -# must not pull a warm web process out of rotation. +# Record the first completed local readiness check for diagnostics. _READY_LATCH = {"at": None} @@ -622,15 +621,11 @@ def readiness(mirror_live=None, role_name=None, root=None): continue row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")} checks[name], detail[name] = bool(row["ok"]), row["detail"] - boot = role_name == "web" and _READY_LATCH["at"] is None if "mirror" in checks and checks["mirror"]: if mirror_live is False: checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading." - elif boot and _waits_for_worker_boot() and not worker_boot_restored(root): - checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore." - elif boot and pending_handoffs(root): - checks["mirror"] = False - detail["mirror"] = "Mirror hand-offs are not installed yet: " + ", ".join(pending_handoffs(root)) + "." + # The web serves its installed Odoo snapshot while worker restores and catch-up continue. + # Connector reads enforce their own mirror readiness before serving that connector's rows. failing = [name for name in checks if not checks[name]] if not failing and _READY_LATCH["at"] is None: _READY_LATCH["at"] = time.time() diff --git a/api/request_class.py b/api/request_class.py index a6530004109345763c728a9f322f0f31cb28ae08..51f77b42b56b043f3c68fff5b288154658420148 100644 --- a/api/request_class.py +++ b/api/request_class.py @@ -536,7 +536,7 @@ def _replay(msgs, receive): # ----------------------------------------------------------------------------- box sizing (R25) -#: The 2 vCPU tier: today's values, exactly. Staging (cpu-basic, 2 vCPU, 16 GB) runs these. +#: The nominal 2 vCPU profile, independent of the container's measured memory limit. TODAY = {"workers": 1, "heavyConcurrency": 2, "webDuckdbMemory": "6GB", "webDuckdbThreads": 2, "workerDuckdbMemory": "4GB", "workerDuckdbThreads": 1} #: Launch env name for each sizing value. An explicit env value always wins over the derived one. @@ -655,15 +655,13 @@ def launch_role(environ=None): def launch_env(environ=None, box=None): - """{env name: value} the launcher exports. Explicit env wins, except it never buys workers - without the read-only mirror: AIOS_MIRROR_VERSIONS=0 or a role other than web holds one worker. - """ + """Return launch exports, allowing several workers only for web with AIOS_MIRROR_VERSIONS=1.""" environ = os.environ if environ is None else environ box = box or detect_box() explicit = {key for key, name in SIZING_ENV.items() if str(environ.get(name) or "").strip()} forced = web_workers(environ) if "workers" in explicit else None role = launch_role(environ) - if str(environ.get(VERSIONS_ENV) or "").strip() == "0" or role != "web": + if str(environ.get(VERSIONS_ENV) or "").strip() != "1" or role != "web": # Only the web role versions its mirror writes. Every other role restores in place. forced = 1 values = sizing(box["cpus"], box["memoryBytes"], workers=forced) diff --git a/api/routes_amazon_events.py b/api/routes_amazon_events.py new file mode 100644 index 0000000000000000000000000000000000000000..227317623ff44060aa8e9d15b7ad6ab2bd3b0203 --- /dev/null +++ b/api/routes_amazon_events.py @@ -0,0 +1,180 @@ +"""Receive signed Amazon listing events and administer tenant subscriptions.""" +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import time + +from fastapi import APIRouter, Body, Depends, HTTPException, Request + +from deps import Session, err +from routes_admin import admin_gate + + +router = APIRouter() +NOTIFICATION_TYPE = "BRANDED_ITEM_CONTENT_CHANGE" +AWS_REGION = "us-east-2" +AWS_ACCOUNT_SHA256 = "ef5273b4f80cbdf7da2c921e9679b69e0772f2d585e9c47a13425022750a96b5" +SUBSCRIPTIONS_FIELD = "amazonNotifications" + + +def verify_event(headers, body: bytes, *, now=None): + """Verify the producer's timestamp and exact body before JSON decoding.""" + token = os.environ.get("AIOS_AMAZON_EVENT_TOKEN") + if not token: + return False, 503, "amazon_event_token_not_configured" + lowered = {str(key).lower(): str(value) for key, value in headers.items()} + stamp = lowered.get("x-aios-timestamp", "") + signature = lowered.get("x-aios-signature", "") + try: + valid_stamp = abs((time.time() if now is None else float(now)) - int(stamp)) <= 300 + except ValueError: + valid_stamp = False + expected = hmac.new(token.encode(), stamp.encode() + b"\n" + body, hashlib.sha256).hexdigest() + if not valid_stamp or not hmac.compare_digest(expected.encode(), signature.encode()): + return False, 403, "amazon_event_signature_invalid" + return True, 200, "" + + +def _partner_bus(fields, result): + """Read the returned destination's partner bus name for the owner's bind step.""" + from harness import amazon_client as client + + if result.get("eventBusName"): + return result["eventBusName"] + token = client.notifications_token(fields) + destinations = client._request( + client._NOTIFICATION_DESTINATIONS_PATH, token, + region=fields.get("region") or client.DEFAULT_REGION, + sandbox=client.is_sandbox(fields)).get("payload") or [] + for destination in destinations: + if destination.get("destinationId") == result["destinationId"]: + return ((destination.get("resource") or {}).get("eventBridge") or {}).get("name") + return None + + +@router.post("/connectors/amazon_sp/notifications") +def setup_notifications(body: dict = Body(...), session: Session = Depends(admin_gate)): + """Ensure subscriptions and save their IDs on the selected tenant connection.""" + from core import keychain + from harness import amazon_client as client + + account = str(body.get("awsAccountId") or "") + if (len(account) != 12 or not account.isascii() or not account.isdigit() + or hashlib.sha256(account.encode()).hexdigest() != AWS_ACCOUNT_SHA256 + or body.get("awsRegion", AWS_REGION) != AWS_REGION): + raise err(422, "amazon_notification_destination_invalid", + "Use the designated AIOS AWS account in us-east-2.") + roles = body.get("grantedRoles", []) + if not isinstance(roles, list) or any(not isinstance(role, str) for role in roles): + raise err(422, "amazon_notification_roles_invalid", "grantedRoles must be a list of role names.") + rt = session.runtime + connection_id = keychain.resolve_connection_id(rt, "amazon_sp", body.get("connectionId")) + row = keychain._row_for(rt, connection_id, "amazon_sp") + if row is None: + raise err(404, "amazon_connection_not_found", "This tenant has no matching Amazon connection.") + try: + fields = keychain.amazon_creds(rt, connection_id=connection_id) + if not fields: + raise err(404, "amazon_connection_not_found", "This Amazon connection has no credential.") + result = client.ensure_notifications(fields, account, AWS_REGION, granted_roles=roles) + except client.AmazonError: + raise err(503, "amazon_notifications_unavailable", "Amazon notification setup could not complete.") from None + except keychain.KeychainLocked: + raise err(503, "keychain_locked", "The tenant keychain is locked.") from None + try: + event_bus_name = _partner_bus(fields, result) + except client.AmazonError: + # Subscription IDs remain durable if the subsequent destination read fails. + event_bus_name = None + + def save(bucket): + entry = (bucket.get("entries") or {}).get(row["id"]) + if (not isinstance(entry, dict) or entry.get("type") != "amazon_sp" + or keychain.connection_id_of(entry, row["id"]) != connection_id): + raise err(409, "amazon_connection_changed", "The Amazon connection changed during setup.") + previous = entry.get(SUBSCRIPTIONS_FIELD) or {} + subscriptions = dict(previous.get("subscriptions") or {}) + # Preserve prior IDs when a destination-only pass skips all subscription types. + for kind, state in result["subscriptions"].items(): + if state.get("subscriptionId"): + subscriptions[kind] = dict(state) + entry[SUBSCRIPTIONS_FIELD] = { + "connectionId": connection_id, "destinationId": result["destinationId"], + "eventBusName": event_bus_name, "subscriptions": subscriptions} + return bucket + + try: + rt.update(keychain.KEY, save) + except HTTPException: + raise + except Exception: + raise err(503, "store_unavailable", "Amazon subscription IDs could not be saved.") from None + if not event_bus_name: + raise err(503, "amazon_partner_bus_name_missing", + "Amazon did not return the partner event bus name. Subscription IDs were saved.") + return {**result, "connectionId": connection_id, "eventBusName": event_bus_name} + + +def tenant_for_subscription(subscription_id): + """Resolve one live tenant connection by subscription metadata without decrypting credentials.""" + from core import keychain + from harness.runtime import get_runtime, known_tenants + + matches = [] + for tenant in known_tenants(): + rt = get_runtime(tenant) + bucket = rt.get(keychain.KEY, fresh=True) or {} + for entry_id, entry in (bucket.get("entries") or {}).items(): + if not isinstance(entry, dict) or entry.get("type") != "amazon_sp": + continue + connection_id = keychain.connection_id_of(entry, entry_id) + if keychain._row_for(rt, connection_id, "amazon_sp") is None: + continue + states = (entry.get(SUBSCRIPTIONS_FIELD) or {}).get("subscriptions") or {} + if any(state.get("subscriptionId") == subscription_id for state in states.values()): + matches.append(rt) + if len(matches) > 1: + raise err(409, "amazon_subscription_ambiguous", "The subscription belongs to multiple connections.") + return matches[0] if matches else None + + +@router.post("/connectors/amazon_sp/events") +async def listing_event(request: Request): + """Record listing attributes only after signature, type and subscription admission.""" + from harness import amazon_listing_changes + + raw = await request.body() + valid, status, reason = verify_event(request.headers, raw) + if not valid: + raise err(status, reason, "Amazon event authentication failed.") + try: + envelope = json.loads(raw) + detail = envelope.get("detail") if isinstance(envelope, dict) else None + if not isinstance(detail, dict): + raise ValueError + except (ValueError, UnicodeDecodeError): + raise err(422, "amazon_event_envelope_invalid", "A full EventBridge envelope is required.") from None + if detail.get("NotificationType") != NOTIFICATION_TYPE: + raise err(422, "amazon_notification_type_unsupported", "Only BRANDED_ITEM_CONTENT_CHANGE is supported.") + metadata, payload = detail.get("NotificationMetadata"), detail.get("Payload") + if (not isinstance(metadata, dict) or not isinstance(payload, dict) + or not isinstance(metadata.get("SubscriptionId"), str) or not metadata["SubscriptionId"]): + raise err(422, "amazon_event_payload_invalid", "Notification metadata and listing payload are required.") + try: + rt = tenant_for_subscription(metadata["SubscriptionId"]) + if rt is None: + raise err(404, "amazon_subscription_unknown", "The Amazon notification subscription is not registered.") + count = amazon_listing_changes.record(rt, { + "asin": payload.get("Asin"), "marketplaceId": payload.get("MarketplaceId"), + "attributesChanged": payload.get("AttributesChanged"), "source": "notification", + "notificationId": metadata.get("NotificationId"), "publishedAt": metadata.get("PublishTime")}) + except HTTPException: + raise + except ValueError: + raise err(422, "amazon_listing_change_invalid", "The listing change fields are incomplete or invalid.") from None + except Exception: + raise err(503, "store_unavailable", "The Amazon listing change could not be recorded.") from None + return {"ok": True, "recorded": count} diff --git a/api/routes_amazon_projects.py b/api/routes_amazon_projects.py index 83d1fa7a6ece23be66c8a99585c3c9881abbd4fa..fd6e800e6f9a24bbaf4d001a9770a75bcab45ad3 100644 --- a/api/routes_amazon_projects.py +++ b/api/routes_amazon_projects.py @@ -15,10 +15,12 @@ from typing import Any from fastapi import APIRouter, Body, Depends, HTTPException, Query from core import amazon_projects as projects +from core import amazon_project_tasks as project_tasks from core import amazon_project_results as project_results -from core import users +from core import perm_scope, users from deps import Session, err, require_session from modules import amazon_profit +from modules import amazon_project_filters as project_filters from modules import sku_action_results import routes_amazon_profit import routes_sku_actions @@ -32,7 +34,7 @@ NOT_FOUND = "That project does not exist." #: Contract field name to store field. FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "owner", "startOn": "start_on", "targetOn": "target_on", "progress": "progress", "notes": "notes", - "completedOn": "completed_on", "resultConfigs": "result_configs"} + "completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"} #: Project JSON keys a client may echo back. They are ignored, never written. READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy", "rollup", "impact", "via")) @@ -49,6 +51,8 @@ def _call(fn, *args, **kwargs): raise err(503, "projects_store_unavailable", "Projects need the workspace database.") from None except ValueError as exc: raise err(400, "invalid_project", str(exc)) from None + except perm_scope.Denied as exc: + raise err(403, "forbidden", str(exc)) from None except Exception as exc: # noqa: BLE001 print(f"[amazon-projects] {getattr(fn, '__name__', fn)} failed: {type(exc).__name__}: {exc}", file=sys.stderr) @@ -118,7 +122,7 @@ def _project(record, today): "targetCount": record.get("target_count", 0), "createdAt": record["created_at"], "updatedAt": record["updated_at"], "createdBy": record["created_by"], "rollup": record.get("rollup"), "impact": impact or None, - "resultConfigs": record.get("result_configs", [])} + "resultConfigs": record.get("result_configs", []), "contributors": record.get("contributors", [])} def _item(grain, key, listing, pictures): @@ -158,6 +162,20 @@ def _owner_candidates(session): def _owner_fields(body, session, values, previous=None): """Resolve new assignments against the current tenant roster on every write.""" + if "contributors" in body: + if not isinstance(body["contributors"], list): + raise err(400, "invalid_project", "Send contributors as a list of workspace usernames.") + candidates = {item["username"]: item for item in _owner_candidates(session)} + members = [] + for username in body["contributors"]: + if not isinstance(username, str) or not username.strip(): + raise err(400, "invalid_project", "Each contributor must be a workspace username.") + username = username.strip().lower() + identity = candidates.get(username) + if identity is None: + raise err(400, "invalid_project", f"Workspace user '{username}' cannot contribute to this project.") + members.append({"tenant": session.tenant, "username": identity["username"], "name": identity["name"]}) + values = {**values, "contributors": members} if "ownerUsername" not in body: if values.get("owner") is not None: old = (previous or {}).get("owner") @@ -175,7 +193,7 @@ def _owner_fields(body, session, values, previous=None): old = projects.owner_identity((previous or {}).get("owner"), session.tenant) if old and old["username"] == username: return {**values, "owner": previous["owner"]} - raise err(400, "invalid_project", "Select a current workspace user.") + raise err(400, "invalid_project", f"Workspace user '{username}' cannot own this project.") return {**values, "owner": _call(projects.owner_value, session.tenant, identity)} @@ -496,14 +514,14 @@ def row_projects(grain: str | None = None, key: str | None = None, today = projects._today() if grain is None and key is None: records = _call(projects.list_projects, runtime) - return {"grain": None, "key": None, "canEdit": can_edit, + return {"grain": None, "key": None, "canEdit": can_edit, "today": today.isoformat(), "projects": [_project(r, today) for r in records], "summary": projects.summary(records)} pair = _pair(grain, key) above, cause = _mirror(runtime, lambda cur: projects.lineage(cur, *pair), []) pairs = [pair] + [p for p in above if p != pair] listed = _distinct(_call(projects.projects_for, runtime, pairs), pairs) out = [{**_project(record, today), "via": {"grain": via[0], "key": via[1]}} for via, record in listed] - return _with_cause({"grain": pair[0], "key": pair[1], "canEdit": can_edit, "projects": out, + return _with_cause({"grain": pair[0], "key": pair[1], "canEdit": can_edit, "today": today.isoformat(), "projects": out, "summary": projects.summary([record for _via, record in listed])}, cause) @@ -545,12 +563,53 @@ def one_project(project_id: str, session: Session = Depends(require_session)): runtime = _reader(session) record = _call(projects.get, runtime, project_id) targets, cause = _targets(runtime, project_id) - activity = _call(projects.activity, runtime, project_id) + activity = _activity_page(session, project_id)["activity"] return _with_cause({"canEdit": bool(routes_amazon_profit._may_edit(session)), "project": _project(record, projects._today()), "targets": targets, "activity": activity}, cause) +def _changes_for(runtime, pairs, since): + """This tenant's listing changes for the project targets, empty when the reader cannot answer.""" + from harness import amazon_listing_changes + try: + return amazon_listing_changes.for_runtime(runtime).changes_for(pairs, since) + except Exception as exc: # noqa: BLE001 + # An absent or unreadable mirror omits Amazon rows and keeps app activity readable. + print(f"[amazon-projects] listing changes unavailable: {type(exc).__name__}: {exc}", file=sys.stderr) + return [] + + +def _activity_page(session, project_id, cursor=None): + """Use one session-bound cursor and the same activity normalization for both project doors.""" + from routes_tables import _activity_cursor + runtime = session.runtime + pairs = _call(projects.targets, runtime, project_id) + query = {"amazonProject": project_id, "targets": pairs} + position = _activity_cursor(session, query, token=cursor) if cursor is not None else None + actors = {username: str(record.get("name") or username) + for username, record in users._tenant_accounts(session.tenant)} + tasks = _call(project_tasks.list_tasks, runtime, project_id) + page = _call(projects.activity_page, runtime, project_id, cursor=position, + actor_names=actors, task_names={task["id"]: task["name"] for task in tasks}, + read_changes=lambda since: _changes_for(runtime, pairs, since)) + for entry in page["activity"]: + field = entry.get("field") or "" + if field == "Done by filter" or field.endswith("no longer meets the filter"): + entry.update(action=field, field="tasks") + page["nextCursor"] = _activity_cursor(session, query, position=page.pop("next_cursor")) + return page + + +@router.get("/{project_id}/activity") +def project_activity(project_id: str, cursor: str | None = Query(None), + session: Session = Depends(require_session)): + """Page every project activity entry newest first through the Command Center read permission.""" + runtime = _reader(session) + _call(projects.get, runtime, project_id) + return _activity_page(session, project_id, cursor) + + @router.get("/{project_id}/results") def project_result_figures(project_id: str, session: Session = Depends(require_session)): """Serve current target results only after effective grants and independent total validation.""" @@ -606,6 +665,7 @@ def project_result_series(project_id: str, start: str | None = Query(None, alias available = _result_sources(session, targets + selected_targets) con = routes_sku_actions._results_mirror(runtime) reason = None + sku_count_reason = None try: skus = projects.resolve_target_skus(con, targets) if selected: @@ -617,6 +677,7 @@ def project_result_series(project_id: str, start: str | None = Query(None, alias reason = "Today is incomplete. No completed days are available in this range." except projects.MirrorUnreadable as exc: skus, reason = [], str(exc) + sku_count_reason = reason figures = {} def read(key): @@ -627,8 +688,10 @@ def project_result_series(project_id: str, start: str | None = Query(None, alias figures[metric] = _result_figure(con, {"metric": metric}, bounds, skus, available) return _series_figure(key, figures.get(metric, {}), bounds) - return project_results.series(record, bounds=bounds, read=read, + answer = project_results.series(record, bounds=bounds, read=read, independent=lambda key: _series_economics_total(con, key, bounds, skus, available)) + return {**answer, "skuCount": len(skus) if sku_count_reason is None else None, + "skuCountReason": sku_count_reason} except HTTPException: raise except (project_results.ValidationFailed, sku_action_results.ValidationFailed): @@ -681,23 +744,263 @@ def _target_answer(runtime, project_id, result): return _with_cause({"project": _project(result["project"], projects._today()), "targets": targets}, cause) +def _matching_targets(session, body): + """Resolve the whole permitted filter population before selecting a response page.""" + if not isinstance(body, dict) or "filter" not in body: + raise err(400, "invalid_target", "Send a grain and filter tree.") + grain = body.get("grain") + key = _call(project_filters.table_key, grain) + session.require(key) + try: + keys = project_filters.matching_keys(_cells_store(session.runtime), grain, body["filter"], + user=session.user) + except perm_scope.Denied: + raise err(403, "forbidden", f"Your account cannot read {key}.") from None + except perm_scope.UnknownTable: + raise err(503, "project_targets_unavailable", f"Target table {key} is unavailable.") from None + except perm_scope.Unresolvable as exc: + raise err(503, "project_targets_unavailable", + f"Cannot read {key}: {exc.cause} {exc.recommendation}") from None + except ValueError as exc: + raise err(400, "invalid_target", str(exc)) from None + except Exception as exc: # noqa: BLE001 + print(f"[amazon-projects] filter read failed: {type(exc).__name__}", file=sys.stderr) + raise err(503, "project_targets_unavailable", f"Target table {key} could not be read. Try again shortly.") from None + return [(grain, key) for key in keys] + + +def _require_target_grains(session, pairs): + for grain in dict.fromkeys(grain for grain, _key in pairs): + session.require(project_filters.table_key(grain)) + + +def _remove_target_pairs(body): + return _pairs(body["targets"]) if "targets" in body else [_pair(body.get("grain"), body.get("key"))] + + +def _add_target_pairs(session, body): + """Keep valid explicit targets and return each malformed target's refusal.""" + if isinstance(body, dict) and "filter" in body: + if "targets" in body: + raise err(400, "invalid_target", "Send either targets or a grain and filter tree.") + return _matching_targets(session, body), [] + values = body.get("targets") if isinstance(body, dict) else None + if not isinstance(values, list) or not values: + raise err(400, "invalid_target", "Send targets as a list of grain and key.") + pairs, refused = [], [] + for value in values: + if not isinstance(value, dict): + refused.append({"key": None, "reason": "Send each target as a grain and key."}) + continue + try: + pairs.append(projects.target(value.get("grain"), value.get("key"))) + except ValueError as exc: + refused.append({"key": value.get("key"), "reason": str(exc)}) + _require_target_grains(session, pairs) + return pairs, refused + + +@router.post("/{project_id}/targets/match") +def match_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)): + """Count every permitted filter match and return one page of its keys.""" + runtime = _reader(session) + _call(projects.get, runtime, project_id) + limit = body.get("limit", project_filters.MATCH_PAGE) if isinstance(body, dict) else project_filters.MATCH_PAGE + offset = body.get("offset", 0) if isinstance(body, dict) else 0 + if type(limit) is not int or limit < 1 or type(offset) is not int or offset < 0: + raise err(400, "invalid_target", "Use a positive whole limit and a nonnegative whole offset.") + pairs = _matching_targets(session, body) + return {"grain": body["grain"], "total": len(pairs), "keys": [key for _grain, key in pairs[offset:offset + limit]]} + + @router.post("/{project_id}/targets") def add_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)): """Assign the project to more SKUs, ASINs or Parent ASINs.""" runtime = _writer(session) - pairs = _pairs((body or {}).get("targets") if isinstance(body, dict) else None) - result = _call(projects.add_targets, runtime, project_id, pairs, by=_by(session)) + _call(projects.get, runtime, project_id) + pairs, refused = _add_target_pairs(session, body) + result = (_call(projects.add_targets, runtime, project_id, pairs, by=_by(session)) if pairs + else {"project": _call(projects.get, runtime, project_id)}) _repaint(runtime, pairs) - return _target_answer(runtime, project_id, result) + return {**_target_answer(runtime, project_id, result), "refused": refused} @router.post("/{project_id}/targets/remove") def remove_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)): - """Unassign one SKU, ASIN or Parent ASIN from the project.""" + """Unassign a target list or the legacy grain and key pair from the project.""" runtime = _writer(session) if not isinstance(body, dict): raise err(400, "invalid_target", "Name the grain and key to remove.") - pair = _pair(body.get("grain"), body.get("key")) - result = _call(projects.remove_targets, runtime, project_id, [pair], by=_by(session)) - _repaint(runtime, [pair]) + pairs = _remove_target_pairs(body) + _require_target_grains(session, pairs) + result = _call(projects.remove_targets, runtime, project_id, pairs, by=_by(session)) + _repaint(runtime, pairs) return _target_answer(runtime, project_id, result) + + +# --------------------------------------------------------------------------- tasks and dependencies + +TASK_FIELDS = {"name": "name", "assignee": "assignee", "startOn": "start_on", "dueOn": "due_on", + "done": "done", "contingencyFor": "contingency_for", "parentId": "parent_id", + "position": "position", "targets": "targets", "targetFilter": "target_filter", + "completionFilter": "completion_filter"} +TASK_EDITS = frozenset(TASK_FIELDS) - {"parentId", "position"} +DEPENDENCY_FIELDS = {"fromTask": "from_task", "toTask": "to_task", "type": "type", "lagDays": "lag_days"} + + +def _task(record): + result = {"id": record["id"], "projectId": record["project_id"], "parentId": record["parent_id"], + "position": record["position"], "name": record["name"], "assignee": record["assignee"], + "startOn": record["start_on"], "dueOn": record["due_on"], "done": record["done"], + "doneAt": record["done_at"], "contingencyFor": record["contingency_for"], + "active": record["active"], "createdBy": record["created_by"]} + if any(field in record for field in project_tasks.TARGET_FIELDS): + result.update(targets=record.get("targets"), targetFilter=record.get("target_filter"), + completionFilter=record.get("completion_filter"), filterStatus=record.get("filter_status"), + filterClosedAt=record.get("filter_closed_at")) + return result + + +def _task_filters(session, record): + """Use current source permissions before saving task targets or completion conditions.""" + pairs = record.get("targets") or [] + grains = list(dict.fromkeys(pair["grain"] for pair in pairs)) + saved = record.get("target_filter") + if saved is not None: + grains.append(saved["grain"]) + for grain in dict.fromkeys(grains): + session.require(project_filters.table_key(grain)) + for tree in ([saved["filter"]] if saved and grain == saved["grain"] else []) + ( + [record["completion_filter"]] if record.get("completion_filter") is not None else []): + _call(project_filters.check_task_filter, _cells_store(session.runtime), grain, tree, user=session.user) + + +def _task_matcher(session): + def match(_runtime, grain, tree, *, user): + key = project_filters.table_key(grain) + session.require(key) + st = _cells_store(session.runtime) + project_filters.validate_task_tree(tree, perm_scope.scoped_fields(session.user, key, st=st)) + return project_filters.matching_keys(st, grain, tree, user=user) + return match + + +def _dependency(record): + return {"id": record["id"], "projectId": record["project_id"], "fromTask": record["from_task"], + "toTask": record["to_task"], "type": record["type"], "lagDays": record["lag_days"]} + + +def _planning_fields(body, fields, kind, required=()): + if not isinstance(body, dict) or not body: + raise err(400, f"invalid_{kind}", f"Send the {kind} fields as an object.") + unknown = sorted(set(body) - set(fields)) + if unknown: + raise err(400, f"invalid_{kind}", f"Unknown {kind} fields: {', '.join(unknown)}.") + if set(required) - set(body): + raise err(400, f"invalid_{kind}", f"Send these {kind} fields: {', '.join(required)}.") + return {fields[key]: value for key, value in body.items()} + + +def _task_answer(runtime, project_id, record, shifted=()): + # Task writes commit leaf progress before the target cells read it. + project = _call(projects.get, runtime, project_id) + _repaint(runtime, _target_pairs(runtime, project_id)) + return {"task": _task(record), "shifted": [_task(row) for row in shifted], + "project": _project(project, projects._today())} + + +@router.get("/{project_id}/tasks") +def list_project_tasks(project_id: str, session: Session = Depends(require_session)): + """Read every task as a flat row and every dependency in this tenant's project.""" + runtime = _reader(session) + return {"tasks": [_task(row) for row in _call(project_tasks.list_tasks, runtime, project_id, + matcher=_task_matcher(session), today=projects._today(), + by=amazon_profit.WRITER)], + "dependencies": [_dependency(row) for row in + _call(project_tasks.list_dependencies, runtime, project_id)]} + + +@router.post("/{project_id}/tasks") +def create_project_task(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)): + """Create a task under an optional parent with one assignee and its own dates.""" + runtime = _writer(session) + values = _planning_fields(body, TASK_FIELDS, "task", required=("name",)) + validated = _call(project_tasks._fields, values) + _task_filters(session, validated) + record = _call(project_tasks.create, runtime, project_id, by=_by(session), **values) + return _task_answer(runtime, project_id, record) + + +@router.patch("/{project_id}/tasks/{task_id}") +def edit_project_task(project_id: str, task_id: str, body: Any = Body(default=None), + session: Session = Depends(require_session)): + """Reschedule dates first and retain every shifted dependent through the other field edits.""" + runtime = _writer(session) + values = _planning_fields(body, {key: TASK_FIELDS[key] for key in TASK_EDITS}, "task") + # Reschedule accepts only a tenant task id, so bind it to the URL's project first. + previous = _call(project_tasks.get, runtime, project_id, task_id) + validated = _call(project_tasks._fields, values, previous) + _task_filters(session, validated) + if "contingency_for" in values: + rows = {row["id"]: row for row in _call(project_tasks.list_tasks, runtime, project_id)} + _call(project_tasks._check_contingency, rows, task_id, validated["contingency_for"]) + shifted = [] + if {"start_on", "due_on"} & values.keys(): + result = _call(project_tasks.reschedule, runtime, task_id, + values.pop("start_on", previous["start_on"]), + values.pop("due_on", previous["due_on"]), _by(session)) + record, shifted = result["task"], result["shifted"] + else: + record = previous + if values: + record = _call(project_tasks.update, runtime, project_id, task_id, values, by=_by(session)) + return _task_answer(runtime, project_id, record, shifted) + + +@router.post("/{project_id}/tasks/{task_id}") +def activate_project_task(project_id: str, task_id: str, session: Session = Depends(require_session)): + """Activate a dormant backup through an explicit editor action.""" + runtime = _writer(session) + record = _call(project_tasks.activate, runtime, project_id, task_id, by=_by(session)) + return _task_answer(runtime, project_id, record) + + +@router.post("/{project_id}/tasks/{task_id}/move") +def move_project_task(project_id: str, task_id: str, body: Any = Body(default=None), + session: Session = Depends(require_session)): + """Reparent or reorder a task through the core's ancestor and position checks.""" + runtime = _writer(session) + values = _planning_fields(body, {key: TASK_FIELDS[key] for key in ("parentId", "position")}, "task") + record = _call(project_tasks.move, runtime, project_id, task_id, by=_by(session), **values) + return _task_answer(runtime, project_id, record) + + +@router.delete("/{project_id}/tasks/{task_id}") +def delete_project_task(project_id: str, task_id: str, session: Session = Depends(require_session)): + """Delete the task and its subtree, then repaint its project's target cells.""" + runtime = _writer(session) + result = _call(project_tasks.delete, runtime, project_id, task_id, by=_by(session)) + _repaint(runtime, _target_pairs(runtime, project_id)) + return {"ok": True, "id": result["deleted"], "taskIds": result["task_ids"], "progress": result["progress"]} + + +@router.post("/{project_id}/dependencies") +def add_project_dependency(project_id: str, body: Any = Body(default=None), + session: Session = Depends(require_session)): + """Link two project tasks with a dependency type and signed day lag.""" + runtime = _writer(session) + values = _planning_fields(body, DEPENDENCY_FIELDS, "dependency", required=("fromTask", "toTask")) + for field in ("from_task", "to_task"): + _call(project_tasks.get, runtime, project_id, values[field]) + record = _call(project_tasks.add_dependency, runtime, project_id, by=_by(session), **values) + return {"dependency": _dependency(record), "shifted": [_task(row) for row in record["shifted"]]} + + +@router.delete("/{project_id}/dependencies") +def delete_project_dependency(project_id: str, body: Any = Body(default=None), + session: Session = Depends(require_session)): + """Unlink a dependency by id within this tenant's project.""" + runtime = _writer(session) + values = _planning_fields(body, {"id": "dependency_id"}, "dependency", required=("id",)) + record = _call(project_tasks.delete_dependency, runtime, project_id, by=_by(session), **values) + return {"ok": True, "id": record["id"]} diff --git a/api/routes_auth.py b/api/routes_auth.py index 8a8796220f01a64255531eb2b7b83896bf0fc891..801e882e5edf4eef2e6d85154b6eae850fabb876 100644 --- a/api/routes_auth.py +++ b/api/routes_auth.py @@ -75,6 +75,8 @@ def _verify(uname, pw): HOTFIX 2026-09-28: a store `PoolTimeout` used to read as "no such account", so a correct password got 401 `invalid_credentials` and was charged to the lockout budget.""" try: + if uname and pw and not users.store.available(): + raise users.RegistryUnavailable("The account store is unavailable") return users.verify(uname, pw) if (uname and pw) else None except users.RegistryUnavailable as exc: raise _auth_store_down("verify", exc) diff --git a/api/routes_brand.py b/api/routes_brand.py index 9c85792eaa3cf9703c223f358ea6e985e7af3213..88c9b7dfa3c0598ea9ca1e303949b99638ceed8a 100644 --- a/api/routes_brand.py +++ b/api/routes_brand.py @@ -35,6 +35,7 @@ from __future__ import annotations import os import re +from xml.etree import ElementTree from pathlib import Path from fastapi import APIRouter, Depends @@ -49,8 +50,7 @@ router = APIRouter(prefix="/api/v1") #: path are refused at the door (400) rather than resolved and then defended against. _KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,31}$") -#: A year, immutable. The artwork is versioned by its own key: a NEW mark for a vendor is a new file -#: with the same name, so a rebrand ships with a deploy and a hard reload, not with a cache bust. +#: The client versions each artwork URL before it can enter this immutable cache. #: ⚠ Deliberately the same string as `routes_assets._IMMUTABLE` — two asset doors telling a browser #: two different things about the same class of file would be a defect nobody could see. _IMMUTABLE = "public, max-age=31536000, immutable" @@ -86,6 +86,21 @@ def _roots(): return out +def _has_artwork_magic(path): + """Accept PNG signatures or an SVG document root, never a Git LFS pointer.""" + try: + with path.open("rb") as stream: + if path.suffix == ".png": + return stream.read(8) == b"\x89PNG\r\n\x1a\n" + if path.suffix == ".svg": + return ElementTree.parse(stream).getroot().tag in ( + "svg", "{http://www.w3.org/2000/svg}svg", + ) + except (OSError, ElementTree.ParseError, ValueError, LookupError): + return False + return False + + def brand_path(key): """The file for a brand key, or None. Pure and importable, so the gate answers this question without a request — the same shape `routes_connectors.directory` uses for the same reason.""" @@ -97,8 +112,11 @@ def brand_path(key): p = root / f"{k}{ext}" # ⚠ `is_file()` and not `exists()`: a DIRECTORY named `instagram.png` would otherwise # resolve and then fail inside FileResponse as a 500. - if p.is_file(): - return p + try: + if p.is_file() and p.stat().st_size > 129 and _has_artwork_magic(p): + return p + except OSError: + continue return None @@ -111,7 +129,7 @@ def available(): if not root.is_dir(): continue for p in root.iterdir(): - if p.is_file() and p.suffix in _MEDIA and _KEY_RE.match(p.stem): + if p.suffix in _MEDIA and _KEY_RE.match(p.stem) and brand_path(p.stem) is not None: keys.add(p.stem) return sorted(keys) diff --git a/api/routes_external.py b/api/routes_external.py index 79f0dc7a7ce6f0f219e93f3746aea28224471cf6..394ea4d350fdfc2e7d231ee8c2a1f3ae5cf6ddd7 100644 --- a/api/routes_external.py +++ b/api/routes_external.py @@ -14,6 +14,8 @@ import re import time import uuid from contextlib import contextmanager +from contextvars import ContextVar +from http.client import responses as _HTTP_STATUS_REASONS from urllib.parse import parse_qs, quote, unquote, urlencode, urlsplit, urlunsplit from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response @@ -49,6 +51,19 @@ _QUERY_MAX_FILTER_SIBLINGS = 50 _QUERY_KEYS = frozenset({"fields", "filters", "sorts", "limit", "cursor", "aggregates", "groupBy"}) +def _query_grouping_help(text): + """Describe native Table grouping in the external query's existing syntax help.""" + help_text = ("A stored Table groups one field without a month unit, using the Table grid's aggregate reader. " + "Its group order is count descending, then key ascending. " + "A query_groups_unavailable refusal names its cause and a query_database call without groupBy that reads the rows.") + return text.replace( + "Only a connected database groups. Any other database answers query_groups_unavailable.", + help_text).replace( + "Only a connected database, such as an Odoo sales lines mirror, groups. " + "Any other database answers 409 query_groups_unavailable: read its records by page, " + "or ask for aggregates without groupBy.", help_text) + + def _register_query_paging_contract(): """Keep the legacy and grouped query schemas aligned with the API's bounded page reader.""" from tool_groups.data import GROUP @@ -68,6 +83,7 @@ _VIEW_DRAFT_CONFIG_KEYS = frozenset({ "filters", "filterConj", "sorts", "groupBy", "visible", "order", "display", "memberPids", }) +_VIEW_OPERATION_PLAN = ContextVar("external_view_operation_plan", default=None) _CONFIRMATION_BUCKET = "external_confirmations" _CONFIRMATION_TTL_SECONDS = 5 * 60 _OPERATION_RETENTION_SECONDS = 24 * 60 * 60 @@ -780,16 +796,21 @@ def _encode_query_cursor(session: Session, query_digest: str, return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") +def _invalid_query_cursor(cause: str): + return err(400, "query_cursor_invalid", cause + " Repeat the original query without cursor " + "for a fresh first page. Reuse each returned next_cursor unchanged with the same account and connection.") + + def _read_query_cursor(session: Session, raw) -> dict: """Authenticate a cursor before reading the source publication.""" if not isinstance(raw, str) or not raw or len(raw) > _QUERY_MAX_CURSOR: - raise err(400, "query_cursor_invalid", "The query cursor is invalid.") + raise _invalid_query_cursor("The query cursor must be a nonempty next_cursor string within the cursor size limit.") try: padded = raw + "=" * (-len(raw) % 4) payload = json.loads(base64.b64decode( padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8")) except (ValueError, TypeError, UnicodeError, json.JSONDecodeError) as exc: - raise err(400, "query_cursor_invalid", "The query cursor is invalid.") from exc + raise _invalid_query_cursor("The query cursor is not a complete, unchanged next_cursor value.") from exc if (not isinstance(payload, dict) or set(payload) not in ({"v", "q", "s", "o", "p"}, {"v", "q", "s", "o", "p", "i"}) or type(payload.get("v")) is not int or payload["v"] != 1 @@ -798,15 +819,15 @@ def _read_query_cursor(session: Session, raw) -> dict: or not isinstance(payload.get("q"), str) or not isinstance(payload.get("s"), str) or not isinstance(payload.get("p"), str)): - raise err(400, "query_cursor_invalid", "The query cursor is invalid.") + raise _invalid_query_cursor("The query cursor has an unsupported format.") publication = payload.get("i") if "i" in payload and (not isinstance(publication, dict) or set(publication) != {"source", "snapshot_id"} or any(not isinstance(value, str) or not value for value in publication.values())): - raise err(400, "query_cursor_invalid", "The query cursor is invalid.") + raise _invalid_query_cursor("The query cursor has an unsupported format.") expected = _cursor_proof(session, payload["q"], payload["s"], payload["o"], publication) if not payload["p"].isascii() or not hmac.compare_digest(payload["p"], expected): - raise err(400, "query_cursor_invalid", "The query cursor is invalid.") + raise _invalid_query_cursor("The query cursor cannot be verified for this account and connection.") return payload @@ -1847,7 +1868,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str: "", ]) - lines.extend(query_syntax_section()) + lines.extend(_query_grouping_help(line) for line in query_syntax_section()) if public: lines.extend([ "## Your read scope", @@ -2709,6 +2730,40 @@ def list_databases(session: Session = Depends(require_external_analyst)): "write_scope": "Use the listed tools to make changes with your own permissions."} +_SCHEMA_READ_REFUSAL_FIXES = { + "refused": ( + "The schema read was refused by the database's source. Ask an Administrator to check the source before retrying."), + "not_read_through": ( + "This database has no connected read path. Ask an Administrator to repair its source connection, then describe it again."), + "cross_tenant_store": ( + "This connection cannot read that workspace. Reconnect to the intended workspace before describing the database."), + "store_not_ready": ( + "This database is temporarily unavailable. Try describing it again shortly."), + "schema_unavailable": ( + "This database's field definitions are temporarily unavailable. Try describing it again shortly."), + "schema_not_loaded": ( + "This connected database has no saved fields yet. Describe it again after its source sync completes."), + "not_connected": ( + "This database has no connected source. Call table.list_databases and choose a returned database key."), + "scope_not_applicable": ( + "This database's access restriction could not be applied. Ask an Administrator to repair it without widening access."), + "filter_not_applicable": ( + "This database's access filter could not be applied. Ask an Administrator to repair it without removing the restriction."), + "filter_unsupported": ( + "This database's access filter cannot be evaluated. Ask an Administrator to repair it without removing the restriction."), + "forbidden_table": ( + "Current permissions do not allow this database. Call table.list_databases and choose a readable database."), + "odoo_definition_not_published": ( + "This database's field definitions have not been published. Describe it again after its source sync completes."), + "odoo_mirror_binding_incomplete": ( + "This database's source binding is incomplete. Ask an Administrator to repair the source, then describe it again."), + "odoo_mirror_not_loaded": ( + "This database's source is not loaded. Describe it again after its source sync completes."), + "canonical_source_unavailable": ( + "This database's current source is unavailable. Try describing it again after its source sync completes."), +} + + @router.get("/databases/{database}/schema") def database_schema(database: str, session: Session = Depends(require_external_analyst)): """Return the field contract after resolving the ordinary strict Assistant read wall.""" @@ -2718,9 +2773,20 @@ def database_schema(database: str, session: Session = Depends(require_external_a session = copy.copy(session) session.runtime = user_tables.lend(session.runtime) _assert_database_scope(session, database) - resolved = _resolve_external_scope( - session, databases=[database], include_schema=True, strict_schema=True, - ) + try: + resolved = _resolve_external_scope( + session, databases=[database], include_schema=True, strict_schema=True, + ) + except HTTPException as exc: + info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} + repair = _SCHEMA_READ_REFUSAL_FIXES.get(info.get("code")) + if not repair: + raise + message = _mcp_plain(info.get("message"), repair) + if repair not in message: + message = (message.rstrip() + " " + repair).strip() + raise HTTPException(status_code=exc.status_code, headers=exc.headers, + detail={**exc.detail, "error": {**info, "message": message}}) from exc schema = copy.deepcopy(resolved[0]["schema"]) schema["fields"] = [{**field, **filter_field_contract(field)} for field in schema.get("fields", [])] return external_freshness.add(schema, session) @@ -2788,6 +2854,49 @@ def query_as(rt, tenant, username, database, payload, scope=None) -> dict: return {"ok": False, "error": {"code": refused["code"], "message": message}} +def _query_field_labels(payload, fields): + """Resolve exact, unique readable labels while preserving canonical keys and query values.""" + keys = {field["key"] for field in fields if isinstance(field.get("key"), str)} + labels = {} + for field in fields: + label = field.get("label") + if isinstance(label, str) and label and field.get("key") in keys: + labels.setdefault(label, set()).add(field["key"]) + missing = [] + + def resolve(name): + if isinstance(name, str) and name not in keys: + matches = labels.get(name, set()) + if len(matches) == 1: + return next(iter(matches)) + missing.append(name) + return name + + def filters(node): + if isinstance(node, list): + for child in node: + filters(child) + elif isinstance(node, dict): + if "colId" in node: + node["colId"] = resolve(node["colId"]) + for name in ("nodes", "children"): + if name in node: + filters(node[name]) + + resolved = copy.deepcopy(payload) + if isinstance(resolved.get("fields"), list): + resolved["fields"] = [resolve(name) for name in resolved["fields"]] + filters(resolved.get("filters")) + for name in ("sorts", "aggregates", "groupBy"): + if isinstance(resolved.get(name), list): + for index, item in enumerate(resolved[name]): + if name == "groupBy" and isinstance(item, str): + resolved[name][index] = resolve(item) + elif isinstance(item, dict) and "field" in item: + item["field"] = resolve(item["field"]) + return resolved, missing, keys + + def _query_database(session: Session, database: str, payload: dict): try: if isinstance(payload, dict): @@ -2804,6 +2913,16 @@ def _query_database(session: Session, database: str, payload: dict): return result except HTTPException as exc: info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} + if info.get("code") == "assistant_unknown_field" and isinstance(payload, dict): + schema = database_schema(database, session) + resolved, missing, visible_keys = _query_field_labels(payload, schema.get("fields", [])) + if resolved != payload: + return _query_database(session, database, resolved) + message = (deps.assistant_unknown_field_message(missing, visible_keys) + if missing else info.get("message", "")) + info["message"] = message.replace( + "Use a field key returned by describe_database.", + "Call data.describe_database for this database and use a returned field key.") cause = {"assistant_store_not_ready": deps._assistant_source_cause_of(info.get("message")) or "syncing", "store_not_ready": "syncing", "amazon_mirror_not_loaded": deps._assistant_source_cause_of(info.get("message")) or "loading"}.get(info.get("code")) @@ -2813,7 +2932,11 @@ def _query_database(session: Session, database: str, payload: dict): # S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules. info["message"] = bad_filter_message(info.get("message")) if info.get("code") in QUERY_SYNTAX_CODES and isinstance(info.get("message"), str): - info["message"] = with_query_syntax(info["message"]) + info["message"] = _query_grouping_help(with_query_syntax(info["message"])) + if info.get("code") == "query_groups_unavailable": + info["retry_query"] = {"database": database, **{ + key: copy.deepcopy(value) for key, value in payload.items() + if key not in {"groupBy", "aggregates", "cursor"}}} if info.get("code") == "query_cursor_stale": fix = _MCP_FIXES["query_cursor_stale"] if fix not in info["message"]: @@ -2922,6 +3045,90 @@ def _typed_selected_stop_identity(session, database, filters): return identity +def _query_source_metadata(*sources): + """Copy supplied canonical status and merge supplied limits without inferring a source cause.""" + out = {} + for source in sources: + if isinstance(source.get("sourceStatus"), dict): + out["sourceStatus"] = copy.deepcopy(source["sourceStatus"]) + if isinstance(source.get("limits"), list): + limits = out.setdefault("limits", []) + for limitation in source["limits"]: + if limitation not in limits: + limits.append(copy.deepcopy(limitation)) + return out + + +def _query_table_groups(session, database, scoped, aggregates, groups, offset, limit): + """Adapt the Table grid's scoped aggregate without reading a row window.""" + import registry_source + import routes_tables + from core import relations + + def unavailable(cause): + raise err(409, "query_groups_unavailable", cause + " Call table with action query_database " + "using retry_query, without groupBy or aggregates, then follow page.next_cursor.") + + if scoped.get("source_kind") != "materialised_user_table": + unavailable("This source has no Table aggregate reader.") + if len(groups) != 1 or groups[0].get("unit"): + unavailable("The Table aggregate reader groups one field at a time without a month unit.") + if relations.current_query_link_pin() is not None: + unavailable("The Table aggregate reader cannot apply this link pin.") + grant = perm_scope.assistant_entry(session.user, database) + if session.admin and (grant or {}).get("filter"): + unavailable("The Table grid bypasses administrator row filters, so it cannot answer this scoped grouping.") + + aggs, claimed = [], {} + for item in aggregates: + field = item.get("field") + if field is None: + continue + fn = "average" if item["fn"] == "avg" else item["fn"] + if field in claimed and claimed[field] != fn: + unavailable("The Table aggregate reader returns one summary per field. Send separate grouped calls for each summary.") + if field not in claimed: + aggs.append({"colId": field, "agg": fn}) + claimed[field] = fn + + tree = scoped.get("filters") or {} + body = {"filters": copy.deepcopy(tree.get("nodes") or []), "filterConj": tree.get("conj", "and"), + "groupBy": groups[0]["field"], "groupOffset": offset, "groupLimit": limit, "aggs": aggs} + before = registry_source.generation(session, database) + if before is None: + unavailable("This Table store cannot track changes for stable group paging.") + out = routes_tables.table_aggregates(database, body=body, session=session) + registry_source.require_generation(session, database, before) + source_metadata = _query_source_metadata(scoped, out) + limits = source_metadata.get("limits", []) + for limitation in limits: + # The standing editable-row budget does not mean this aggregate was refused. + if (limitation.get("effect") in {"aggregate_ignored", "group_ignored", "filter_ignored", "filter_unresolved"} + or (limitation.get("subject"), limitation.get("effect")) in { + ("pids", "unresolved"), ("bytes", "refused")}): + unavailable(str(limitation.get("cause") or "The Table aggregate reader cannot compute this grouping or summary.") + + " " + str(limitation.get("recommendation") or "Use describe_database to choose supported fields.")) + if out.get("groups") is None: + unavailable("The Table aggregate reader returned no grouped result.") + + def values(container, count): + if any(field not in (container or {}) for field in claimed): + unavailable("The Table aggregate reader returned an incomplete requested summary.") + return [{**item, "value": count if item.get("field") is None + else container[item["field"]]} for item in aggregates] + + return {**{key: copy.deepcopy(value) for key, value in scoped.items() if key != "records"}, + **source_metadata, + "source_version": {"table_generation": _json_digest(before)}, + "limits": limits, + "total_rows": out["total_rows"], "total_groups": out["total_groups"], + "server_aggregates": values(out["aggregates"], out["total_rows"]), + "groups": [{"keys": {groups[0]["field"]: group["key"]}, + "aggregates": values(group["aggregates"], group["count"])} + for group in out["groups"]], + "group_order": "count_desc_key_asc"} + + def _query_database_page(session: Session, database: str, payload: dict): if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS: raise err(400, "query_arguments_invalid", @@ -2955,6 +3162,7 @@ def _query_database_page(session: Session, database: str, payload: dict): scope_filters = None if identity is not None else payload.get("filters") scoped = _external_read_scope( session, database, fields=None, filters=scope_filters, mode="query_schema") + schema_metadata = _query_source_metadata(scoped) if identity is not None and ( scoped.get("source_kind") != "runtime_cache" or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters): @@ -3002,16 +3210,19 @@ def _query_database_page(session: Session, database: str, payload: dict): if groups: if not aggregates or sorts: raise err(400, "query_groups_invalid", - "Grouped queries require aggregates and return groups in key order.") - if not mirror: - raise err(409, "query_groups_unavailable", - "Server grouping requires a connector mirror. Use a mirror database.") + "Grouped queries require aggregates and no sorts.") digest = _json_digest({"database": database, **payload, "groupBy": groups, "limit": limit, "cursor": None}) options = {"groupBy": groups, "offset": 0, "limit": limit} - summary = _external_read_scope( - session, database, filters=payload.get("filters"), - mode=("aggregate", aggregates, options)) + + def group_page(offset): + if not mirror: + return _query_table_groups(session, database, scoped, aggregates, groups, offset, limit) + return _external_read_scope( + session, database, filters=payload.get("filters"), + mode=("aggregate", aggregates, {**options, "offset": offset})) + + summary = group_page(0) snap = _json_digest({"source_version": summary.get("source_version"), "fields": visible_fields, "total_groups": summary["total_groups"]}) @@ -3021,9 +3232,7 @@ def _query_database_page(session: Session, database: str, payload: dict): raise err(409, "query_cursor_stale", "The permitted result set changed. Start again without a cursor.") if offset: - summary = _external_read_scope( - session, database, filters=payload.get("filters"), - mode=("aggregate", aggregates, {**options, "offset": offset})) + summary = group_page(offset) if _json_digest({"source_version": summary.get("source_version"), "fields": visible_fields, "total_groups": summary["total_groups"]}) != snap: @@ -3040,17 +3249,19 @@ def _query_database_page(session: Session, database: str, payload: dict): sorts=[], page={"limit": limit, "returned": returned, "offset": offset, "has_more": next_cursor is not None, "next_cursor": next_cursor}, - caps={"max_page_size": _QUERY_MAX_LIMIT, "max_group_fields": 4, + caps={"max_page_size": _QUERY_MAX_LIMIT, "max_group_fields": 4 if mirror else 1, "max_sorts": _QUERY_MAX_SORTS, "max_aggregates": _QUERY_MAX_AGGREGATES, "max_filter_depth": _QUERY_MAX_FILTER_DEPTH, "max_filter_nodes": _QUERY_MAX_FILTER_NODES, "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS}) + result.update(_query_source_metadata(schema_metadata, summary)) external_freshness.add(result, session) _assert_finite_query_value(result) external_activity.record_ok(session, database) return result server_aggregates = None + summary = {} paged_total = None if mirror: # ⭐⭐ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates @@ -3124,6 +3335,7 @@ def _query_database_page(session: Session, database: str, payload: dict): "max_filter_nodes": _QUERY_MAX_FILTER_NODES, "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS}, } + result.update(_query_source_metadata(schema_metadata, summary, first, page)) external_freshness.add(result, session) _assert_finite_query_value(result) external_activity.record_ok(session, database) @@ -3204,13 +3416,14 @@ def _query_database_page(session: Session, database: str, payload: dict): "max_filter_nodes": _QUERY_MAX_FILTER_NODES, "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS}, } + result.update(_query_source_metadata(schema_metadata, scoped)) external_freshness.add(result, session) _assert_finite_query_value(result) external_activity.record_ok(session, database) return result -def _view_context(session, database, event): +def _view_context(session, database, event, *, schema_only=False): import routes_grid scope = routes_grid._scope_or_400( @@ -3233,18 +3446,21 @@ def _view_context(session, database, event): from routes_products import MODULE, product_assembly session.require(MODULE) - g = product_assembly(session, consume_corrections=False) + g = product_assembly(session, consume_corrections=False, + **({"with_rows": False} if schema_only else {})) else: from routes_customers import grid_assembly session.require(routes_grid.MODULE) - g = grid_assembly(session, scope=scope, consume_corrections=False) + g = grid_assembly(session, scope=scope, consume_corrections=False, + **({"with_rows": False} if schema_only else {})) except HTTPException as exc: info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} raise err(exc.status_code, info.get("code", "view_refused"), f"Database {database}: {info.get('message') or 'The app refused access.'}") from exc ctx = routes_grid._ctx( session, g["fields"], g["pids"], scope_key=scope, + **({"permission_fields": (g.get("_permission_scope") or {}).get("fields")} if schema_only else {}), measure_keys=frozenset(m["key"] for m in g["measures"]), resolved_ids=frozenset(g["measure_sets"]), cohort_ids=frozenset(c["id"] for c in g["lists"]), @@ -3331,7 +3547,53 @@ def _validate_personal_view_draft(session: Session, database: str, raw: dict) -> _external_read_scope(session, database, fields=[], filters=None) -def _saved_view_filters(config): +def _resolve_saved_view_cell_ref(session, reference): + """Authorize a saved operand through the current workbook and scoped cell read doors.""" + from core import sheets + from core.sheet_structure import col_to_index + + session.require("sheets") + ctx = _tool_context(session) + try: + book = sheets.get_workbook(session.runtime, reference["workbook_id"], + before_normalize=ctx.require_reader) + ctx.require_reader(book) + except (HTTPException, sheets.SheetRefusal) as exc: + code = (exc.detail.get("error", {}).get("code") + if isinstance(exc, HTTPException) else exc.code) + if code == "workbook_missing": + raise err(404, "workbook_missing", "The referenced workbook is unavailable or unreadable. " + "Choose a workbook you can read, or use a literal value.") from exc + raise + tab = sheets.find_tab(book, reference["tab"]) + address = reference["ref"].replace("$", "").upper() + match = re.fullmatch(r"([A-Z]+)([1-9][0-9]*)", address) + column, row = col_to_index(match[1]), int(match[2]) + + def refuse(cause): + raise err(403, "view_filters_unsupported", "Cell reference refused: " + cause + + ". Choose a readable workbook and cell, or use a literal value.") + + if column > tab["cols"] or row > tab["rows"]: + refuse("the cell is outside the tab dimensions") + if (tab.get("visibility", "visible") != "visible" or row in tab.get("hiddenRows", []) + or column in tab.get("hiddenCols", [])): + refuse("the cell is hidden") + # The external projection masks out-of-key source values before recomputing formulas. + cell = tab.get("cells", {}).get(address, {}) + sources = ([cell] if cell.get("kind") != "formula" else + [item for sheet in book.get("sheets", []) for item in sheet.get("cells", {}).values()]) + if any(item.get("kind") == "bound" and not external_access.database_allowed( + _key_scope(session)[1], (item.get("binding") or {}).get("database")) for item in sources): + refuse("the cell source is outside this connection's database scope") + projected = sheets.find_tab(ctx.project_book(book), tab["id"]) + cell = projected.get("cells", {}).get(address, {}) + if cell.get("state") == "refused" or cell.get("error") or cell.get("refusal"): + refuse("the cell is unreadable under the current source permissions or needs Refresh") + return True + + +def _saved_view_filters(config, *, resolve_cell_ref=None): """Convert declared query groups to native View groups and refuse any lossy normalization.""" value = config.get("filters", []) _assert_query_filter_bounds([value] if isinstance(value, dict) else value) @@ -3351,22 +3613,29 @@ def _saved_view_filters(config): or not isinstance(raw[key], list) or not raw[key]): refuse() return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]} - if any(key in raw for key in ("valueRef", "value2Ref")): - refuse("Cell references cannot be saved through this tool because workbook and cell access " - "cannot be verified here. Use literal values or a readable rhs field instead.") op = raw.get("op") if not isinstance(op, str) or op not in aios_grid.FILTER_OPS: name = json.dumps(op[:120]) if isinstance(op, str) else "(missing or invalid)" refuse(f"Unsupported filter operator {name}. Use a supported operator: " + ", ".join(sorted(aios_grid.FILTER_OPS)) + ".") if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked", - "dateWindow", "dateMode", "rhs"} + "dateWindow", "dateMode", "rhs", "valueRef", "value2Ref"} or not isinstance(raw.get("colId"), str)): refuse() for key in ("value", "value2"): value = raw.get(key) if isinstance(value, (dict, list)) or value is not None and len(str(value)) > 500: refuse() + for key in ("valueRef", "value2Ref"): + if key not in raw: + continue + reference = aios_grid.clean_filter_cell_ref(raw[key]) + if reference is None: + refuse(f"{key} needs {{workbook_id, tab, ref: 'A1'}} and optional boolean locked. " + "Use one cell address, or a literal value.") + if resolve_cell_ref is None or resolve_cell_ref(reference) is not True: + refuse("Cell references require a caller-authorised workbook and cell resolver. " + "Use a readable workbook and cell, or a literal value.") return copy.deepcopy(raw) if isinstance(value, dict): @@ -3471,6 +3740,108 @@ def _view_share_preview(session: Session, args: dict): } +def _view_filter_members(nodes, *, new=False, parent=None, seed=None): + """Address filter nodes by their C1 identity without cleaning their values.""" + members = {} + for index, node in enumerate(nodes): + node_id = node.get("id") + if not node_id: + if not new: + raise err(409, "view_refused", "This View needs stable filter ids before its filters can be edited. " + "Open it in Table after the View update is available.") + node_id = uuid.uuid5(uuid.NAMESPACE_URL, f"external-view-filter:{seed}:{parent}:{index}").hex + if not isinstance(node_id, str) or not node_id or "filter:" + node_id in members: + raise err(400, "view_filters_unsupported", "Each filter needs its own text id.") + value = {**copy.deepcopy(node), "id": node_id} + children = value.pop("children", None) + conjunction = value.pop("conj", "and") if children is not None else None + members["filter:" + node_id] = {"node": value, "parent": parent} + if children is not None: + value["children"] = [] + members["filterConj:" + node_id] = conjunction + nested = _view_filter_members(children, new=new, parent=node_id, seed=seed) + if members.keys() & nested.keys(): + raise err(400, "view_filters_unsupported", "Each filter needs its own text id.") + members.update(nested) + return members + + +def _view_members(view, config_keys, fields, *, new_filters=False, seed=None, stored=False): + """Translate the published draft config to C1 members, leaving validation to Table. + + `stored` reads positions as the document holds them, with no seed for a legacy rule.""" + config = view.get("config") or {} + members = {key: copy.deepcopy(view[key]) for key in ("name", "note") if key in view} + if "visible" in config_keys: + visible = set(config.get("visible", fields)) + members.update({"hidden:" + key: key not in visible for key in fields}) + if "sorts" in config_keys: + for index, rule in enumerate(config.get("sorts") or []): + pos = rule.get("pos") or (None if stored else f"{index + 1:08x}") + members["sort:" + rule["colId"]] = {"dir": rule["dir"], **({"pos": pos} if pos else {})} + if "order" in config_keys: + positions = config.get("positions") or {} + members.update({"position:" + key: positions.get(key) or f"{index + 1:08x}" + for index, key in enumerate(config.get("order") or []) + if positions.get(key) or not stored}) + if "filters" in config_keys: + members.update(_view_filter_members(config.get("filters") or [], new=new_filters, seed=seed)) + if "memberPids" in config_keys: + members.update({"memberPid:" + str(pid): True for pid in config.get("memberPids") or []}) + for key, member in (("groupBy", "group"), ("display", "display"), ("filterConj", "filterConj")): + if key in config_keys: + members[member] = copy.deepcopy(config.get(key, "and" if key == "filterConj" else None)) + return members + + +def _view_operation_plan(database, owner, before, raw, fields, *, execution_id=None): + """Pin only requested member changes so confirmation never re-diffs a stale View snapshot.""" + from core import write_execution + execution_id = execution_id or write_execution.new_id() + keys = set(raw["config"]) + old_config = before.get("config") or {} + for key in ("filters", "order"): + if raw["config"].get(key, []) == old_config.get(key, []): + keys.discard(key) + old_sorts = old_config.get("sorts") or [] + if raw["config"].get("sorts", []) == [ + {key: value for key, value in rule.items() if key != "pos"} for rule in old_sorts]: + keys.discard("sorts") + prior = _view_members(before, keys, fields) + # The preview shows the stored value. A seeded position the document never held is not a before value. + shown = _view_members(before, keys, fields, stored=True) + desired = _view_members(raw, keys, fields, new_filters=True, seed=execution_id) + if "sorts" in keys: + same_order = [rule["colId"] for rule in raw["config"].get("sorts") or []] == [ + rule["colId"] for rule in old_sorts] + for index, rule in enumerate(raw["config"].get("sorts") or []): + if same_order: + desired["sort:" + rule["colId"]]["pos"] = prior["sort:" + rule["colId"]]["pos"] + if "locked" in rule and bool(rule["locked"]) != next((bool(old.get("locked")) for old in old_sorts + if old["colId"] == rule["colId"]), False): + raise err(409, "view_refused", "Change sort locks in Table before editing this View through the assistant.") + # Position members have no remove spelling. Omitted columns keep their stored positions. + for member in prior.keys() - desired.keys(): + if member.startswith("position:") or member.startswith("filterConj:"): + desired[member] = copy.deepcopy(prior[member]) + # An omitted draft property leaves that member alone. + if "note" not in raw: + desired.pop("note", None) + prior.pop("note", None) + ops = [] + for member in sorted(prior.keys() | desired.keys()): + if (member in prior, prior.get(member)) == (member in desired, desired.get(member)): + continue + ops.append({"view_id": raw["id"], "member": member, + "op": "set" if member in desired else "remove", + "value": copy.deepcopy(desired.get(member)), "client_op_id": uuid.uuid5( + uuid.NAMESPACE_URL, f"external-view-op:{execution_id}:{raw['id']}:{member}").hex}) + return {"database": database, "view_id": raw["id"], "owner": owner, + "createdBy": before.get("createdBy"), "ops": ops, + "before": {op["member"]: copy.deepcopy(shown.get(op["member"])) for op in ops}, + "config_keys": sorted(keys)} + + def _view_event(session, kind, args): from core import grid_events @@ -3498,23 +3869,52 @@ def _view_event(session, kind, args): if kind == "view_upsert": # Resolve the ordinary View/database wall before the additional T31 draft validation so # the established refusal vocabulary and connector record-scope guard stay intact. - raw = {**raw, "config": _saved_view_filters(raw["config"])} + raw = {**raw, "config": _saved_view_filters( + raw["config"], resolve_cell_ref=lambda ref: _resolve_saved_view_cell_ref(session, ref))} event["view"] = raw _validate_personal_view_draft(session, database, raw) existing = ctx.table.find_view(view_id, prefer_owner=session.uname) if (existing and str(existing[0]).strip().lower() != str(session.uname).strip().lower()): raise err(404, "view_unavailable", - f"View {view_id}: another user's personal draft is read-only.") + f"View {view_id}: another user's personal draft is read-only. Ask the view's owner for edit access.") from core import shares share_record = shares.grants("view", view_id, st=session.runtime) if share_record.get("owner") or share_record.get("entries"): raise err(409, "view_is_shared", "This View has entered the sharing workflow and cannot be changed by the " "personal-draft tool. Use a separately confirmed View change.") + plan = _VIEW_OPERATION_PLAN.get() + if plan is not None and (kind != "view_upsert" or not existing + or plan["database"] != database or plan["view_id"] != view_id + or plan["owner"] != existing[0] or plan["createdBy"] != existing[1].get("createdBy")): + raise err(409, "view_refused", "This View is no longer available for the previewed edit. Open it and preview again.") + if kind == "view_upsert" and existing: + readable = {field["key"] for field in ctx.fields} + config_keys = set(plan["config_keys"] if plan is not None else raw["config"]) + prior_config = {key: value for key, value in existing[1].get("config", {}).items() if key in config_keys} + if set(_view_referenced_fields(prior_config)) - readable: + raise err(403, "field_unavailable", "This View includes a field you can no longer read. " + "Open it in Table with current access before previewing this edit.") + if "memberPids" in config_keys: + allowed_pids = {str(pid) for pid in ctx.allowed_pids} + if any(str(pid) not in allowed_pids for pid in prior_config.get("memberPids") or []): + raise err(403, "out_of_scope", "This View includes records outside your current access. " + "Open it with current record access before previewing this edit.") + if not callable(getattr(grid_events, "clean_view_member", None)): + raise err(503, "view_refused", "View edits are unavailable until Table's View update is installed. Try again after the update.") + plan = plan or _view_operation_plan(database, existing[0], existing[1], raw, + [field["key"] for field in ctx.fields], execution_id=event["id"]) + if not plan["ops"]: + raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.") try: before = ctx.table.find_view(view_id, prefer_owner=session.uname) - rerender = grid_events.handle_one(event, ctx) + rerender = False + for item in ([{"id": op["client_op_id"], "type": "view_op", **op} for op in plan["ops"]] + if plan is not None else [event]): + rerender = grid_events.handle_one(item, ctx) or rerender + if ctx.out.refusals or ctx.out.toast: + break except grid_events.StoreUnavailable as exc: raise err(503, "store_unavailable", "The View is unavailable right now.") from exc if ctx.out.refusals: @@ -3525,7 +3925,7 @@ def _view_event(session, kind, args): raise err(403, "view_refused", f"View {view_id}: {ctx.out.toast}") after = ctx.table.find_view(view_id, prefer_owner=session.uname) changed = before != after - if not changed: + if not changed and plan is None: raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.") # ⛔⛔ `handle_one`'s BOOL IS A RERENDER FLAG, NOT A SUCCESS FLAG, and reading it as success is # the mistake this comment exists to stop the next reader making. Its own contract block @@ -3543,8 +3943,30 @@ def _view_event(session, kind, args): (not after or str(after[0]).strip().lower() != str(session.uname).strip().lower())): raise err(409, "view_not_personal", f"View {view_id}: the personal draft was not saved.") - return {"database": database, "view_id": view_id, "rerender": bool(rerender), - "draft": "personal", "shared": False} + result = {"database": database, "view_id": view_id, "rerender": bool(rerender), + "draft": "personal", "shared": False} + if plan is not None: + observed = _view_members(after[1], plan["config_keys"], [field["key"] for field in ctx.fields]) + # A different member's diff cannot acknowledge an ignored operation. + for op in plan["ops"]: + member = op["member"] + expected = (True, op["value"]) if op["op"] == "set" else (False, None) + if (member in observed, observed.get(member)) != expected: + # A handler that wrote nothing keeps the published silent no-op refusal. + if not changed: + raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.") + raise err(409, "view_refused", f"View {view_id}: a requested change was not saved. " + "Open it and preview again.") + result["_view_operation_plan"] = plan + result["_view_operation_diff"] = [ + {"path": "/views/" + view_id + "/" + op["member"], + "before": copy.deepcopy(plan["before"].get(op["member"])), + "after": copy.deepcopy(observed.get(op["member"]))} for op in plan["ops"]] + result["_view_operation_diff"] = [item for item in result["_view_operation_diff"] + if item["before"] != item["after"]] + if not result["_view_operation_diff"]: + raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.") + return result def _trusted_activity_via(session): @@ -4031,6 +4453,54 @@ def _field_link_tables(*fields): return tables +_CONNECTED_SOURCE_FIELD_REASON = ( + "This database's source fields come from its connector and cannot be added, changed or deleted here." +) +_CONNECTED_SOURCE_FIELD_FIX = ( + "Add the field to a writable Table database instead. Call table with action list_databases to find one. " + "Then call table with action database.field_create and arguments containing action field_create, " + "its database key and the same field." +) + + +def _connected_formula_create(session, args): + """Identify formula creation that belongs to the connected database's workspace.""" + from core import user_tables + + raw = args.get("field") + return (args.get("action") == "field_create" and isinstance(raw, dict) + and raw.get("type") == "formula" + and not user_tables.is_user_table(str(args.get("database") or "").strip(), st=session.runtime)) + + +def _create_connected_formula(session, args): + """Use the app's field event to admit and persist a connected-source formula definition.""" + import routes_tables + from core import grid_events, write_execution + + database = str(args.get("database") or "").strip() + _assert_database_scope(session, database) + _assert_write_scope(session, "database_object") + routes_tables._managed_or_refuse(session, database, mutation=True) + event = {"id": write_execution.new_id(), "type": "field_upsert", "field": args["field"]} + ctx = _view_context(session, database, event, schema_only=True) + key = str(args["field"].get("key") or "")[:80] + if key in {field["key"] for field in ctx.fields} or key in ctx.hidden_keys: + raise err(400, "field_refused", "That field key is already in use or unavailable. Choose a new custom_ key.") + try: + grid_events.handle_one(event, ctx) + except grid_events.StoreUnavailable as exc: + raise err(503, "store_unavailable", "The formula field could not be saved. Try again.") from exc + if ctx.out.refusals: + refusal = ctx.out.refusals[0] + raise err(400, "field_refused", refusal.get("message") or "The app refused this formula field.") + written = ctx.out.writes.get(event["id"], {}) + if written.get("refused") or not isinstance(written.get("value"), dict): + raise err(409, "field_refused", "The formula field was not saved. Check its key and definition.") + return {"database": database, "action": "field_create", "field_key": key, + "id": key, "_connected_formula": True} + + def _plan_database_write(session, args): import routes_tables from core import user_tables @@ -4044,6 +4514,10 @@ def _plan_database_write(session, args): _assert_write_scope(session, "database_object") if action == "rename": return _plan_database_rename(session, args) + if _connected_formula_create(session, args): + if not isinstance(session.runtime, _PreviewRuntime): + session = Session(session.tenant, session.user, session.claims, _PreviewRuntime(session.runtime)) + return _create_connected_formula(session, args) routes_tables._managed_or_refuse(session, database, mutation=True) if action == "record_create": routes_tables._adds_or_refuse(database) @@ -4051,8 +4525,14 @@ def _plan_database_write(session, args): tables_before = _database_write_tables(session.runtime, (database,)) table = copy.deepcopy(tables_before.get(database) or {}) if not table or not user_tables.is_user_table(database, st=session.runtime): - raise err(403, "connected_source_read_only", - "Records from connected sources are read only.") + reason = (_CONNECTED_SOURCE_FIELD_REASON if action.startswith("field_") + else "Records from connected sources are read only.") + raw_field = args.get("field") if isinstance(args.get("field"), dict) else {} + if action == "field_create" and str(raw_field.get("type") or "").strip().lower() == "link": + # R22 keeps this door closed. The administrator's registry Link door is the REST one. + reason += (" An administrator creates a Link field on a connected registry table " + "through POST /api/v1/tables/{key}/fields.") + raise err(403, "connected_source_read_only", reason) revision = _bucket_revision(_database_write_source(session.runtime), user_tables.STORE_KEY, tables_before) if type(revision.get("rev")) is not int: @@ -4136,11 +4616,13 @@ def _plan_database_write(session, args): if not isinstance(raw, dict): raise err(400, "tool_arguments_invalid", "field must be an object.") have = list(table.get("fields") or []) + # The preview definition's own ceiling, the one the REST door reads. None means no cap. + field_cap = user_tables.field_limit(database, session.runtime, definition=table) field = user_tables._clean_field( raw, recurrence_validator=recurrence_validator, valid_keys={str(item.get("key")) for item in have if item.get("key")}) if (not field or any(item.get("key") == field.get("key") for item in have) - or len(have) >= user_tables.MAX_FIELDS + or (field_cap is not None and len(have) >= field_cap) or not user_tables.dynamic_link_filter_valid(field, have) or not user_tables.flow_bound(field.get("automation"), session.runtime)): raise err(400, "field_refused", "The field does not pass the database's schema rules.") @@ -4228,6 +4710,8 @@ def _execute_database_object(session, args, *, plan=None): return database_schema(database, session) if action == "query": return _query_database(session, database, args.get("query") or {}) + if _connected_formula_create(session, args): + return _create_connected_formula(session, args) plan = plan or _plan_database_write(session, args) if action == "rename": import routes_nav @@ -4881,7 +5365,7 @@ def _is_website_note(name, arguments): def _preview_external_write(session, name, arguments, *, issue_token=True, execution=None, purpose=None, about=None, - intent=None): + intent=None, view_operation_plan=None): from core import write_execution if isinstance(arguments, dict) and "about" in arguments: if about is not None: @@ -4904,20 +5388,32 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu tenant=session.tenant, user=copy.deepcopy(session.user), claims=copy.deepcopy(session.claims), runtime=preview_runtime) execution = execution or {'seed': uuid.uuid4().hex, 'timestamp': time.time()} - with write_execution.replay(execution): - if _is_website_note(name, normalised): - import website_tools - result = website_tools.plan_internal_note(_tool_context(preview_session, preview=True), normalised["arguments"]) - normalised = {"action": "add_internal_note", "arguments": copy.deepcopy(result["arguments"])} - else: - result = _dispatch_external_tool(_tool_context(preview_session, preview=True), name, normalised, about=about) + view_binding = _VIEW_OPERATION_PLAN.set(view_operation_plan) + try: + with write_execution.replay(execution): + if _is_website_note(name, normalised): + import website_tools + result = website_tools.plan_internal_note(_tool_context(preview_session, preview=True), normalised["arguments"]) + normalised = {"action": "add_internal_note", "arguments": copy.deepcopy(result["arguments"])} + else: + result = _dispatch_external_tool(_tool_context(preview_session, preview=True), name, normalised, about=about) + finally: + _VIEW_OPERATION_PLAN.reset(view_binding) if isinstance(result, dict) and result.get("refused"): raise err(409, "write_partial_refused", "Part of this change could not be accepted. Nothing was previewed.") remote_note = _is_website_note(name, normalised) - plan = result if name == "database_object" or remote_note or _is_members_write(name, normalised) else None - if plan is not None: + view_plan = result.get("_view_operation_plan") if isinstance(result, dict) else None + plan = result if ((name == "database_object" and not result.get("_connected_formula")) + or remote_note or _is_members_write(name, normalised)) else None + if view_plan is not None: + diff = copy.deepcopy(result["_view_operation_diff"]) + warnings = [] + affected = {"count": len(view_plan["ops"]), "ids": [view_plan["view_id"]]} + current_revision = {key: copy.deepcopy(view_plan[key]) + for key in ("database", "view_id", "owner", "createdBy")} + elif plan is not None: diff = copy.deepcopy(plan.get("diff") or []) warnings = list(plan.get("warnings") or []) affected = copy.deepcopy(plan.get("affected") or {}) @@ -4934,7 +5430,9 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu affected = _affected_for(name, normalised, diff, result) current_revision = { key: revision for key, revision in preview_runtime.base_revisions().items() - if key in documents + # Formula promotion also updates its personal workspace when there is nothing to remove. + if key in documents or (isinstance(result, dict) and result.get("_connected_formula") + and preview_runtime._local_revisions.get(key)) } if not diff: raise err(409, "write_not_changed", "The requested write would not change anything.") @@ -4955,7 +5453,7 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu _canonical_json(diff, capped=False) diff_digest = _json_digest(diff) - shown_diff, diff_summary = (diff, None) if plan is not None else _summarise_diff(diff, projected) + shown_diff, diff_summary = (diff, None) if plan is not None or view_plan is not None else _summarise_diff(diff, projected) if diff_summary is not None: warnings.append(f"The preview shows {diff_summary['sample']} of {diff_summary['changes']} changes. " "confirm_change applies all of them.") @@ -4978,6 +5476,8 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu "execution": execution, "about": about, "about_digest": _json_digest(about), "buildKind": build_kind, } + if view_plan is not None: + payload["view_operation_plan"] = copy.deepcopy(view_plan) if purpose is not None: payload.update(purpose=purpose, purpose_digest=_json_digest(purpose)) if remote_note: @@ -5096,7 +5596,7 @@ def _store_operation(session, payload, preview, staged, intent=None): app = ((doc or {}).get("applications") or {}).get(app_id) or {} access["databases"].extend(_databases_in_arguments(app)) # Foreign handlers own their object permissions. Pin their consulted documents without calling them on replay. - if payload["tool"] not in external_tools.TOOLS: + if payload["tool"] not in external_tools.TOOLS and "view_operation_plan" not in payload: access["documents"] = {key: _json_digest(value) for key, value in staged._initial.items() if key not in {external_access.BUCKET, external_oauth.BUCKET}} access["after_documents"] = {key: _json_digest(staged._docs[key]) for key in access["documents"]} @@ -5181,7 +5681,7 @@ def _operation_access(session, item, *, confirming=False): "Ask the owner to restore access before checking again.") for database in set(access["databases"]): _assert_database_scope(session, database) - if _is_website_note(name, args): + if _is_website_note(name, args) or (name == "database_object" and _connected_formula_create(session, args)): _external_read_scope(session, database, fields=[], mode="schema") else: _external_read_scope(session, database, fields=[]) @@ -5192,23 +5692,29 @@ def _operation_access(session, item, *, confirming=False): import routes_tables action = args["action"] record_pid = str(args.get("record_id") or "").strip() - g = routes_tables.ut_write_ctx(session, args["database"], - pid=int(record_pid) if action in {"record_update", "record_delete"} and record_pid.isdigit() else None) + formula = _connected_formula_create(session, args) + g = ({"fields": _view_context(session, args["database"], {}, schema_only=True).fields} + if formula else routes_tables.ut_write_ctx(session, args["database"], + pid=int(record_pid) if action in {"record_update", "record_delete"} and record_pid.isdigit() else None)) if action in {"record_create", "record_update"} and item["status"] == "committed": if any(int(rid) not in g["pids"] for rid in item["affected"]["ids"]): raise err(403, "operation_unavailable", "This connection cannot read every record covered by the operation. " "Query the database with current access, or ask the owner to restore access before checking again.") fields = set(args.get("values") or {}) + if formula and item["status"] == "committed": + fields.add(str(args["field"]["key"])[:80]) if args.get("field_key") and action != "field_delete": fields.add(args["field_key"]) if not fields <= {field["key"] for field in g["fields"]}: raise err(403, "operation_unavailable", "This connection cannot read every field covered by the operation. " "Call data with action describe_database, then ask the owner to restore required access before checking again.") - if name in {"view_upsert", "view_delete", "view_object"}: - view_id = args.get("view_id") or (args.get("view") or {}).get("id") - ctx_view = _view_context(session, args["database"], {}) + if name in {"view_upsert", "view_delete", "view_object"} or "view_operation_plan" in payload: + view_plan = payload.get("view_operation_plan") + view_id = view_plan["view_id"] if view_plan else args.get("view_id") or (args.get("view") or {}).get("id") + database = view_plan["database"] if view_plan else args["database"] + ctx_view = _view_context(session, database, {}) found = ctx_view.table.find_view(view_id, prefer_owner=session.uname) deleting = name == "view_delete" or args.get("action") == "delete" if (found and str(found[0]).strip().lower() != session.uname.lower()) or (not found and not deleting): @@ -5248,7 +5754,7 @@ def _operation_access(session, item, *, confirming=False): "This connection cannot read the operation's item with its current access. " "Read the item with this connection, or ask the owner to restore access before checking again.") return session - if access["documents"] and _json_digest(user) != access["user"]: + if (access["documents"] or "view_operation_plan" in payload) and _json_digest(user) != access["user"]: raise err(403, "operation_unavailable", "The account changed since the preview, so receipt access cannot be verified. " "Read the item with this connection before previewing another change.") @@ -5475,6 +5981,30 @@ def _expired_fix(payload): + str(_CONFIRMATION_TTL_SECONDS // 60) + " minutes.") +def _expired_confirmation(session, payload, *, intent=None): + """Return expiry context and a newly scoped preview without committing the change.""" + window = _CONFIRMATION_TTL_SECONDS + age = max(0, int(time.time()) - int(payload["expiry"]) + window) + details = {"preview_age_seconds": age, "confirmation_window_seconds": window} + message = (f"This preview is {age} seconds old. The confirmation window is {window} seconds. " + + _expired_fix(payload)) + try: + preview = _preview_external_write( + session, payload["tool"], copy.deepcopy(payload["arguments"]), + purpose=payload.get("purpose"), about=payload.get("about"), intent=intent) + except HTTPException as exc: + info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} + details["fresh_preview_error"] = info.get("code", "tool_unavailable") + message += " A fresh preview could not be prepared with current access and data." + except sheets.SheetRefusal as exc: + details["fresh_preview_error"] = exc.code + message += " A fresh preview could not be prepared with current access and data." + else: + details["fresh_preview"] = {key: value for key, value in preview.items() if not key.startswith("_")} + message += " Review error.fresh_preview before confirming its new confirmation_token." + return err(409, "confirmation_expired", message, details) + + def _confirm_fix(args): """Name what a confirm_change call got wrong and the shape that succeeds.""" allowed = ("confirmation_token", "confirmation_phrase", "intent") @@ -5507,7 +6037,7 @@ def _confirm_change(session, args): if payload.get("environment") != _write_environment(): raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX) if int(payload.get("expiry") or 0) <= int(time.time()): - raise err(409, "confirmation_expired", _expired_fix(payload)) + raise _expired_confirmation(fresh_session, payload, intent=stored.get("intent")) if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE: raise err(400, "confirmation_phrase_required", "Enter the exact stronger confirmation phrase shown in the preview.") @@ -5522,17 +6052,23 @@ def _confirm_change(session, args): _confirmation_claim(fresh_session, operation_id) except _OperationAlreadyClaimed: return _operation_status(fresh_session, {"operation_id": operation_id}) + except HTTPException as exc: + if isinstance(exc.detail, dict) and exc.detail.get("error", {}).get("code") == "confirmation_expired": + raise _expired_confirmation(fresh_session, payload, intent=stored.get("intent")) from None + raise remote_note = _is_website_note(tool, arguments) capability = None try: recomputed = _preview_external_write( fresh_session, tool, arguments, issue_token=False, execution=execution, - purpose=payload.get("purpose"), about=payload.get("about")) + purpose=payload.get("purpose"), about=payload.get("about"), + view_operation_plan=payload.get("view_operation_plan")) current = recomputed["_confirmation"] if (payload.get("arguments_digest") != current.get("arguments_digest") or payload.get("environment") != current.get("environment") or payload.get("revision_digest") != current.get("revision_digest") or payload.get("diff_digest") != current.get("diff_digest") + or payload.get("view_operation_plan") != current.get("view_operation_plan") or (remote_note and any(payload.get(key) != current.get(key) for key in ("remote_plan_digest", "target_digest", "source_fingerprint"))) or any((key in payload, payload.get(key)) != (key in current, current.get(key)) @@ -5546,12 +6082,20 @@ def _confirm_change(session, args): ctx, recomputed["_plan"], confirmation_token=args["confirmation_token"]) result = website_tools.execute_confirmed_note(ctx, recomputed["_plan"], capability=capability) return _finish_website_note(fresh_session, payload, recomputed["_plan"], result) - if tool == "database_object": + if tool == "database_object" and recomputed.get("_plan") is not None: result = _execute_database_object( fresh_session, arguments, plan=recomputed.get("_plan")) elif _is_members_write(tool, arguments): # The account writer compares the pinned expect itself, so no runtime guard is needed. result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"]) + elif payload.get("view_operation_plan") is not None: + # Only signed C1 ops use Table's member merge. Other writers retain document CAS. + view_binding = _VIEW_OPERATION_PLAN.set(payload["view_operation_plan"]) + try: + with write_execution.replay(execution): + result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"]) + finally: + _VIEW_OPERATION_PLAN.reset(view_binding) else: from datetime import datetime, timezone commit_started = datetime.now(timezone.utc) @@ -5578,7 +6122,7 @@ def _confirm_change(session, args): _confirmation_status(fresh_session, operation_id, "failed") raise before_revision = copy.deepcopy(recomputed.get("current_revision") or {}) - if tool == "database_object": + if tool == "database_object" and recomputed.get("_plan") is not None: from core import user_tables import routes_nav bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY @@ -5589,6 +6133,8 @@ def _confirm_change(session, args): after_revision = {"rev": produced_rev} elif _is_members_write(tool, arguments): after_revision = copy.deepcopy(result["revision"]) + elif payload.get("view_operation_plan") is not None: + after_revision = {"view_id": result["view_id"], "members": copy.deepcopy(result["_view_operation_diff"])} else: after_revision = copy.deepcopy(guarded_session.runtime._expected) after_revision_digest = _json_digest(after_revision) @@ -5624,7 +6170,7 @@ def _confirm_change(session, args): _confirmation_status(fresh_session, operation_id, "committed", response) if isinstance(stored.get("arguments_ref"), dict): _drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")]) - if (tool != "database_object" and not _is_members_write(tool, arguments) + if ((tool != "database_object" or recomputed.get("_plan") is None) and not _is_members_write(tool, arguments) and receipt.get("persisted") is True): try: targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments, @@ -5665,13 +6211,13 @@ _MCP_CODE_GROUPS = ( tab_missing folder_missing unknown_table unknown_field view_missing view_not_found sheet_structure_tab_missing unknown_sheet_template cell_missing """), - ("This action is not available right now.", """ + ("The requested service or data source is unavailable. Try again shortly.", """ tool_unavailable tool_capability_unsupported capability_unavailable external_access_unavailable store_unavailable store_not_ready store_catching_up oauth_unavailable not_connected schema_not_loaded query_groups_unavailable confirmation_revision_unavailable assistant_source_too_large amazon_mirror_not_loaded query_link_unavailable """), - ("The change could not be accepted. Review it before trying again.", """ + ("This operation conflicts with the item's editing rules. Read its settings before trying again.", """ tool_refused write_partial_refused field_refused view_refused refused connected_source_read_only computed_field records_read_only loopable_managed preset_field_locked zone_membership_read_only not_a_user_table @@ -5740,6 +6286,39 @@ MCP_RESULT_CODES = { code: reason for reason, codes in _MCP_CODE_GROUPS for code in codes.split() } MCP_RESULT_CODES.update({ + "forbidden": "This connection does not have permission for this operation. Ask an Administrator to check its access.", + "tool_not_found": "The requested tool is not published on this connection. Read loopable_guide for the available tools.", + "tool_unavailable": "The tool could not complete this request. Try the same call again shortly.", + "tool_refused": "The request does not meet this tool's requirements. Read loopable_guide for its accepted actions and arguments.", + "tool_capability_unsupported": "This tool does not support the requested capability. Read its guide for supported actions.", + "capability_unavailable": "The requested capability is not available to this connection. Read loopable_guide for supported actions.", + "external_access_unavailable": "This connection's access could not be checked. Try again shortly.", + "store_unavailable": "The workspace's saved data could not be read. Try the same call again shortly.", + "store_not_ready": "The workspace's saved data is not ready for this request. Try again after loading completes.", + "store_catching_up": "The workspace is still receiving its saved data. Try again after loading completes.", + "oauth_unavailable": "This connection's authorization service is unavailable. Try connecting again shortly.", + "not_connected": "This database has no connected source. Call table with action list_databases and choose a connected database.", + "schema_not_loaded": "This database's saved fields are not loaded. Try again after its source sync completes.", + "query_groups_unavailable": "This query's grouping or summaries could not be calculated. Read its permitted records by page instead.", + "confirmation_revision_unavailable": "The database version could not be checked. Nothing was saved. Read it and preview the change again.", + "assistant_source_too_large": "This source exceeds the read size this action supports. Use a paged Table query instead.", + "amazon_mirror_not_loaded": "This Amazon database's saved data is not loaded. Try again after its source sync completes.", + "query_link_unavailable": "This query's linked database could not be read. Describe both databases and check their access before retrying.", + "connected_source_read_only": _CONNECTED_SOURCE_FIELD_REASON + " " + _CONNECTED_SOURCE_FIELD_FIX, + "write_partial_refused": "Some requested edits were refused by the fields' editing rules. Read the refused items before previewing a corrected change.", + "field_refused": "This field's editing rules prevent the requested change. Read its definition before previewing a corrected change.", + "view_refused": "This View's settings or access rules prevent the requested change. Read the View before previewing a corrected change.", + "refused": "This operation could not satisfy the item's access or validation rules. Read the item's settings before retrying.", + "computed_field": "This field is calculated from other values and cannot be edited directly. Change its permitted inputs instead.", + "records_read_only": "This database's records cannot be edited through this connection. Use a writable Table database for new records.", + "loopable_managed": "Loopable manages this system database. Use its dedicated App tools to change it.", + "preset_field_locked": "This field is defined by the database's preset and cannot be changed here. Add a separate field to a writable Table database.", + "zone_membership_read_only": "Zone membership is managed by its assignment rules. Use the Zone tools to change those rules.", + "not_a_user_table": "This operation requires a writable Table database. Call table with action list_databases to choose one.", + "metric_unavailable": ("This metric has no validated value within your current access and window. " + "Ask an Administrator to check source availability and metric access, or choose another window."), + "metric_window_invalid": ("The dates do not form a valid named window. Use real ISO dates in order, " + "and a complete calendar month for calendar_month."), "workbook_name_ambiguous": "More than one readable workbook has this exact name. Use one of the candidate ids.", "view_filters_unsupported": "These filters cannot be saved without changing their meaning. Use colId, op and value leaves, with conj and children groups.", "confirmation_session_required": "Read this confirmation through its authenticated connection.", @@ -5792,6 +6371,8 @@ MCP_RESULT_CODES.update({ "request_rate_limited": "This AI has reached its hourly request cap. Try again later.", }) MCP_RESULT_CODES.update(READ_REFUSAL_FIXES) +for _schema_code, _schema_fix in _SCHEMA_READ_REFUSAL_FIXES.items(): + MCP_RESULT_CODES.setdefault(_schema_code, _schema_fix) _MCP_INTERNAL_PROSE = re.compile( r"\b(?:DuckDB|Postgres(?:ql)?|SQLite|Polars|Pandas|SQLAlchemy|PyArrow|HyperFormula)\b|" r"\b(?:api|web|platform)_[a-z_]+\b|\b(?:verify_\w+|\w+Error)\b|" @@ -5881,14 +6462,14 @@ class _MCPGuideText(str): _MCP_RESULT_OPAQUE = frozenset({ - "records", "rows", "cells", "values", "value", "before", "after", "arguments", "diff", + "records", "rows", "groups", "cells", "values", "value", "before", "after", "arguments", "diff", "confirmation_token", "confirmation_phrase", "diff_digest", "current_revision", - "revision_before", "revision_after", "filters", "sorts", "config", "binding", "bindings", + "revision_before", "revision_after", "filters", "sorts", "config", "binding", "bindings", "sourceStatus", }) _MCP_RESULT_IDENTIFIERS = frozenset({ "resourceUri", "deepLink", "pdfLink", "downloadUrl", "src", "websiteUrl", "fileId", "link", "href", "id", "ids", "operation_id", "key", "keys", "pid", "pids", "database", "databases", "sourceDatabase", - "compatibleDatabases", "field", "fields", "field_key", "colId", "metric_key", "groupBy", + "compatibleDatabases", "field", "fields", "field_key", "colId", "metric_key", "groupBy", "group_order", "workbook_id", "worksheet", "tab", "view_id", "app_id", "catalog_id", "chart_id", "record_id", "contextId", "path", "paths", "address", "addresses", "range", "cursor", "next_cursor", "token", "source_version", "code", "tool", "action", "person", "agent", "owner", "author", "from", "to", @@ -5899,7 +6480,7 @@ _MCP_RESULT_OBJECTS = frozenset({ "data", "error", "databases", "fields", "tab", "tabs", "workbook", "workbooks", "worksheet", "worksheets", "file", "files", "request", "requests", "template", "templates", "application", "applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools", - "annotations", "serverInfo", "environment", + "annotations", "serverInfo", "environment", "fresh_preview", }) # A link descriptor names its target database key. It is an identifier, never prose (D-9307). _MCP_RESULT_DESCRIPTORS = frozenset({"link"}) @@ -5924,7 +6505,7 @@ def _mcp_result_copy(value, labels=None, *, field=None, metadata=False): return copy.deepcopy(value) result = {} for key, child in value.items(): - if key == "retry_query" and value.get("code") == "query_cursor_stale": + if key == "retry_query" and value.get("code") in {"query_cursor_stale", "query_groups_unavailable"}: # These are the caller's original arguments, not explanatory metadata. result[key] = copy.deepcopy(child) elif key == "about": @@ -7034,11 +7615,12 @@ def _dispatch_mcp(req, session: Session, base_url: str = ""): #: Known refusal codes whose MCP answer carries a repair. _MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX, + "connected_source_read_only": _CONNECTED_SOURCE_FIELD_FIX, **READ_REFUSAL_FIXES} def _mcp_next_call(code, message): - """Append the supported repair once for a known read or missing-item refusal.""" + """Append the supported repair once for a known refusal.""" text = str(message or "").rstrip() fix = _MCP_NEXT_CALL.get(code, "") if code == "assistant_unknown_database": @@ -7182,6 +7764,7 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): if purpose is not None and not (writing and (confirm or direct_wiki)): raise err(400, "tool_arguments_invalid", "Send purpose only with a previewed build.") if name == "query_database" and isinstance(arguments, dict): + _assert_query_filter_bounds(arguments.get("filters")) external_tools.validate_query_filter_operators(arguments.get("filters")) _assert_tool_database_scope(session, name, scoped_arguments) if name == "confirm_change": @@ -7244,8 +7827,8 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): import routes_sheet if exc.code == "tool_arguments_invalid" and name == "confirm_change": exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments)) - if exc.code == "tool_arguments_invalid" and _is_query_call(name, arguments): - exc = sheets.SheetRefusal(exc.code, with_query_syntax(exc.message)) + if exc.code in {"tool_arguments_invalid", "query_aggregates_invalid"} and _is_query_call(name, arguments): + exc = sheets.SheetRefusal(exc.code, _query_grouping_help(with_query_syntax(exc.message))) if exc.code in _MCP_NEXT_CALL: exc = sheets.SheetRefusal(exc.code, _mcp_next_call(exc.code, exc.message)) translated = routes_sheet._refuse(exc) @@ -7255,9 +7838,11 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): except HTTPException as exc: if exc.status_code == 401: _auth_refusal() - info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} + info = exc.detail.get("error", exc.detail) if isinstance(exc.detail, dict) else {} + info = info if isinstance(info, dict) else {} retry = ({"retry_query": info["retry_query"]} - if info.get("code") == "query_cursor_stale" and "retry_query" in info else {}) + if info.get("code") in {"query_cursor_stale", "query_groups_unavailable"} + and "retry_query" in info else {}) details = info.get("details") if info.get("code") in {"build_about_required", "build_about_invalid", "call_intent_required", "call_intent_invalid"} and isinstance(details, dict): @@ -7269,12 +7854,22 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): retry["suggestions"] = details.get("suggestions", []) if info.get("code") == "workbook_name_ambiguous" and isinstance(details, dict): retry["candidates"] = details.get("candidates", []) - message = info.get("message", "The app refused this operation.") + if info.get("code") == "confirmation_expired" and isinstance(details, dict): + retry.update({key: details[key] for key in ( + "preview_age_seconds", "confirmation_window_seconds", "fresh_preview", "fresh_preview_error") + if key in details}) + public_code = _mcp_public_code(info.get("code"), exc.status_code) + detail = exc.detail if isinstance(exc.detail, str) else None + if detail == _HTTP_STATUS_REASONS.get(exc.status_code): + detail = None + message = info.get("message") or info.get("reason") or detail + if not isinstance(message, str) or not message.strip(): + message = MCP_RESULT_CODES[public_code] if info.get("code") in {"build_about_required", "build_about_invalid"} and isinstance(message, str): message += _mcp_about_fix(called) if info.get("code") in _MCP_NEXT_CALL: message = _mcp_next_call(info["code"], message) - out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry) + out = _tool_refusal(name, info.get("code") or public_code, message, exc.status_code, **retry) except (TypeError, ValueError, KeyError, AttributeError): out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.") except Exception as exc: diff --git a/api/routes_feedback.py b/api/routes_feedback.py index 11da5150425b6a46f1d53557f51c1896d0d1c0f0..1772b02902c154ba5a82a036527cbee37b479d08 100644 --- a/api/routes_feedback.py +++ b/api/routes_feedback.py @@ -375,6 +375,11 @@ font-size: var(--lp-fs-xs); } """ +def _feedback_date(stamp): + """Format an operator date without a leading zero on the day.""" + return f"{stamp:%b} {stamp.day}, {stamp.year}" + + @router.get("/feedback/log") def feedback_log(tenant: str = "", session: Session = Depends(require_session)): """Render the existing operator log with one row per distinct AI limitation.""" @@ -386,7 +391,7 @@ def feedback_log(tenant: str = "", session: Session = Depends(require_session)): rows = [] for row in data["rows"]: stamp = datetime.fromtimestamp(float(row["ts"]), timezone.utc) - timestamp = stamp.strftime("%d %b %Y, %H:%M:%S UTC") + date = _feedback_date(stamp) limitation = row.get("category") == "limitation" question = row.get("question") if limitation else row.get("text") detail = (f"{row.get('tool', '')}.{row.get('action', '')}" if limitation @@ -404,10 +409,12 @@ def feedback_log(tenant: str = "", session: Session = Depends(require_session)): filer_details.append( f'{escape(str(user))}: {count:,} {label}, last ' f'') + f'{_feedback_date(last)} ' + f'{last:%H:%M:%S} UTC') rows.append( f'' - f'' + f'' + f'{stamp:%H:%M:%S} UTC' f'{escape(str(row["tenant"]))}' f'{escape(str(question or ""))}' f'{escape(detail)}{"".join(filer_details)}' @@ -417,7 +424,8 @@ def feedback_log(tenant: str = "", session: Session = Depends(require_session)): '

Team feedback

' f'

{data["total"]:,} {item_label}

') if rows: - body += ('
' + body += ('
Latest
' + '' '' '' '' + "".join(rows) + '
DateTimeWorkspaceQuestionCodeCount
') diff --git a/api/routes_grid.py b/api/routes_grid.py index d97bf19adc74475a2172bc5bd954d4b8d1bafcdf..d942214b31b8cc0f76e609b9d8507f1cfb6f1041 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -28,6 +28,7 @@ MODULE = "customer_data" #: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a #: legitimate client — refuse it rather than doing 500 store writes on one request. _MAX_EVENTS = 24 +_MAX_VIEW_OPS = 200 def row_patch_receipt(scope): @@ -105,16 +106,17 @@ def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw): # Getting any one of them from the other topic is the "validated against the wrong field # contract" near-miss the wave-15 routes_products header warned about. scope_key = str(kw.get("scope_key") or "") + admission = kw.get('admission') or session.runtime if scope_key == "product": import modules.product_data as pd from routes_products import MODULE as _PMOD, pd_fields from registry_source import MetadataStore - module, canonical = _PMOD, pd_fields(consolidated=True, st=MetadataStore(session.runtime)) + module, canonical = _PMOD, pd_fields(consolidated=True, st=MetadataStore(admission)) kw.setdefault("table", pd.table_ops(session.runtime)) hidden = perm_scope.hidden_keys(session.user, module, canonical if permission_fields is None else permission_fields, - st=session.runtime) + st=admission) elif scope_key.startswith("ut_"): # Wave 18 C3-UT: a user table has no module in the permission wall (its wall is # `user_tables.may_open`, already applied by the assembly this route ran first), so @@ -129,7 +131,7 @@ def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw): import core.table_store as table_store kw.setdefault("table", table_store.make( "customer_table_workspace", st=session.runtime)) - hidden = perm_scope.hidden_keys(session.user, module, canonical, st=session.runtime) + hidden = perm_scope.hidden_keys(session.user, module, canonical, st=admission) return grid_events.EventCtx( uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin, # C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the @@ -406,44 +408,20 @@ def _workspace_values(values, session, module, fields): def _published_rev(ws): - """S82-T156: the store revision the workspace was read at, which a view write stamps.""" + """The store change revision published with this workspace snapshot.""" rev = ws.get("rev") if isinstance(ws, dict) else None return rev if isinstance(rev, int) and not isinstance(rev, bool) else None -def _settle_refused_view_write(ctx, event): - """Store a no-op receipt for a refused stale view write, so its resend re-reads and stops. - - The refusal rolls back the event's own receipt, and the outbox resends the same id. Without - this receipt every resend is refused again and the browser's save queue never moves (D-634). - """ - from core import edit_receipts, grid_events - - if not getattr(ctx, "durable_receipts", False) or not isinstance(event, dict): - return - identity = event.get("opId", event.get("id")) - if not identity: - return - try: - identity = edit_receipts.op_id(identity) - dropped = {"rerender": True, "write": {"opId": identity}, "refusals": [], - "doc": None, "toast": None} - edit_receipts.run_once( - grid_events._store_of(ctx), ctx.uname, - edit_receipts.operation(ctx.scope_key, event.get("type"), event.get("pid")), - identity, lambda: dropped) - except Exception as exc: # noqa: BLE001 - print(f"[grid/events] refused view write {identity!r}: resend receipt not saved " - f"({type(exc).__name__})") - - def _workspace_filter_removals(workspace, fields, measures, hidden=()): - """Clean saved display filters on a detached wire copy, retaining removal reasons.""" + """Clean dead View members on a detached wire copy, retaining filter removal reasons.""" from copy import deepcopy + from core.table_store import prune_view_field_members import aios_grid workspace = dict(workspace) - valid_keys = {field['key'] for field in fields} | {measure['key'] for measure in measures} + field_keys = {field['key'] for field in fields} + valid_keys = field_keys | {measure['key'] for measure in measures} cohort_ids = {str(entry['id']) for entry in workspace.get('lists') or () if entry.get('id')} view_ids = {str(view['id']) for view in workspace.get('views') or () if view.get('id')} hidden = set(hidden or ()) @@ -457,6 +435,7 @@ def _workspace_filter_removals(workspace, fields, measures, hidden=()): cfg['filters'] = aios_grid.clean_filter_tree( cfg.get('filters'), valid_keys, cohort_ids=cohort_ids, visible_view_ids=view_ids, report=reports) + prune_view_field_members(view, live_keys=field_keys, prune_filters=False) removals = [] for report in [*(view.get('filterRemovals') or ()), *reports]: if (isinstance(report, dict) and report.get('colId') not in hidden @@ -501,6 +480,8 @@ def workspace(scope: str = "customer", # session (its wall is the table's own, inside the assembly). if not (scope == "product" or scope.startswith("ut_")): session.require(MODULE) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") # ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The # storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the # table's own wall (creator or admin, `user_tables.may_open`). @@ -599,14 +580,22 @@ def workspace(scope: str = "customer", # R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own # first-cohort contract change. Same reason as the customer branch below. workspace["fields"] = g["fields"] - # The measure channel is EMPTY on this topic (customer-grain descope) — stated - # explicitly so the client's pickers grey rather than guess. + # Count workspace-wide references using the same producer as Customers and user tables. + try: + from routes_tables import _field_usage + admission = g.get('_admission', session.runtime) + usage = _field_usage(session, PRODUCT_MODULE, g['fields'], None, + admission, 'product_table_workspace', st=admission) + except Exception: + usage = {} + if usage: + workspace['fields'] = [dict(field, usage=dict(usage[field['key']])) + if usage.get(field.get('key')) else field + for field in g['fields']] + # The model owns this session's permitted Product catalogue. workspace["measures"] = g["measures"] workspace["measureSets"] = g["measure_sets"] - # `g["derived"]` — the same correction as the user-table branch above, for the same - # reason: `product_assembly` builds this topic's cohort cells and this route discarded - # them. The measure half stays empty on this topic by descope, which is a different - # statement and one the offer already makes. + # Product cohort and measure cells share the existing read-only derived channel. workspace["derived"] = _workspace_values(g["derived"], session, PRODUCT_MODULE, g["fields"]) workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} try: @@ -2114,7 +2103,7 @@ def grid_events_route(body: dict = Body(default=None), Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the same sessions the read door (`/workspace`) admits, or a tenant without the customer module can SEE its own user tables and not write to them.""" - from core import grid_events, shared_overlay, view_audit + from core import grid_events, shared_overlay from routes_customers import grid_assembly from routes_tables import _app_write_via, _managed_or_refuse @@ -2124,7 +2113,11 @@ def grid_events_route(body: dict = Body(default=None), events = [body] # a single event object, the legacy shape if not isinstance(events, list): raise err(400, "bad_events", "expected {events: [...]}") - if len(events) > _MAX_EVENTS: + view_ops = [event for event in events + if isinstance(event, dict) and event.get("type") == "view_op"] + if len(view_ops) > _MAX_VIEW_OPS: + raise err(400, "too_many_view_ops", f"at most {_MAX_VIEW_OPS} View operations per request") + if len(events) - len(view_ops) > _MAX_EVENTS: raise err(400, "too_many_events", f"at most {_MAX_EVENTS} events per request (the client's resend window)") @@ -2147,11 +2140,20 @@ def grid_events_route(body: dict = Body(default=None), # and lands in the product bucket. The measure/cohort context is honestly EMPTY there: # `clean_measure_field` refuses measure creates on this surface by construction (the # customer-grain descope), which is the fail-closed shape, not an accident. + view_only = bool(events) and all( + isinstance(event, dict) and event.get('type') in ( + 'view_op', 'view_select', 'folder_create', 'folder_rename', 'folder_delete', + 'folder_duplicate', 'item_move', 'folder_reorder') + and event.get('pid') is None and not event.get('pids') for event in events) + table_key = (scope + '_table_workspace' if scope.startswith('ut_') else + 'product_table_workspace' if scope == 'product' else 'customer_table_workspace') + actor = grid_events.request_admission(session, table_key, + schema_only=view_only and not scope.startswith('ut_')) if scope == "product": from routes_products import MODULE as PRODUCT_MODULE, product_assembly session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4) - g = product_assembly(session, consume_corrections=False) + g = product_assembly(actor, consume_corrections=False, with_rows=not view_only) elif scope.startswith("ut_"): # Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the # measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here). @@ -2163,9 +2165,10 @@ def grid_events_route(body: dict = Body(default=None), # permission wall and the admission context — and `ut_write_ctx` returns the SAME six keys # this route reads, with a pid set derived from exactly the row ids `scoped_pool` would # have kept. What it does not do is materialise the rows nobody here looks at. - from routes_tables import ut_write_ctx + from routes_tables import ut_assembly, ut_write_ctx - g = ut_write_ctx(session, scope) + g = (ut_assembly(actor, scope, with_rows=False, fields_only=True) if view_only else + ut_write_ctx(actor, scope)) # ⭐⭐ W31-T20 / D-174 — A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the # half that is easy to skip because the code already "fails closed" without it. When a # read-through grid's population exceeds one window the pid set is EMPTY, and every @@ -2188,7 +2191,18 @@ def grid_events_route(body: dict = Body(default=None), f"{lim.get('cause') or 'the row set is unresolved'}. " f"{lim.get('recommendation') or ''}".strip()) else: - g = grid_assembly(session, scope=scope, consume_corrections=False) + g = grid_assembly(actor, scope=scope, consume_corrections=False, with_rows=not view_only) + record_admitter = None + if view_only: + from routes_tables import _builtin_note_context, ut_write_ctx + checked_pids = {} + def record_admitter(pid): + if pid not in checked_pids: + permitted = (ut_write_ctx(actor, scope, pid=pid) if scope.startswith('ut_') else + _builtin_note_context( + actor, 'product_data' if scope == 'product' else MODULE, pid)) + checked_pids[pid] = pid in permitted['pids'] + return checked_pids[pid] # ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`, # `clean_measure_field` had an empty admission list and every measure-column create over # HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every @@ -2198,6 +2212,8 @@ def grid_events_route(body: dict = Body(default=None), # The product and customer assemblies read `ws` with these pids and no consume: lend it once. lend = None if scope.startswith("ut_") else g.get("ws") ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, via=via, + admission=g.get('_admission', actor.runtime), + record_admitter=record_admitter, durable_receipts=True, workspace_lend=lend if isinstance(lend, dict) else None, permission_fields=(g.get('_permission_scope') or {}).get('fields'), @@ -2211,9 +2227,25 @@ def grid_events_route(body: dict = Body(default=None), # Per-event results so the client can tell which of a batch landed — the component's own # bridge has no response channel at all, so this is strictly more than the embed gets. - results = [] + results = [None] * len(events) + grouped = {} + for index, one in enumerate(events): + if isinstance(one, dict) and one.get("type") == "view_op": + grouped.setdefault(str(one.get("view_id") or "").strip(), []).append((index, one)) + dispatched = set() try: - for one in events: + for index, one in enumerate(events): + if isinstance(one, dict) and one.get("type") == "view_op": + view_id = str(one.get("view_id") or "").strip() + if view_id in dispatched: + continue + dispatched.add(view_id) + batch = grouped[view_id] + eid = one.get("client_op_id", one.get("opId", one.get("id"))) + rows = grid_events.handle_view_ops([event for _, event in batch], ctx) + for (slot, _), row in zip(batch, rows): + results[slot] = row + continue eid = str(one.get("opId", one.get("id")) or "") if isinstance(one, dict) else "" # ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler # appends to a shared list, so the refusals belonging to THIS event are exactly the @@ -2221,11 +2253,7 @@ def grid_events_route(body: dict = Body(default=None), # answer "the batch was refused" and leave the caller to guess which member, which is # the same class of unfalsifiable answer the channel exists to end. _before = len(ctx.out.refusals) - try: - rerender = grid_events.handle_one(one, ctx) - except view_audit.ViewRevisionStale: - _settle_refused_view_write(ctx, one) - raise + rerender = grid_events.handle_one(one, ctx) row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid, "rerender": bool(rerender)} mine = ctx.out.refusals[_before:] @@ -2233,7 +2261,7 @@ def grid_events_route(body: dict = Body(default=None), # `refused` is the SHAPE a caller branches on; the first reason is the one that # stopped this write (a handler returns at its first refusal). row["refused"] = mine[0] - results.append(row) + results[index] = row except (shared_overlay.WriteUnavailable, grid_events.StoreUnavailable) as exc: refusal = err(503, "store_unavailable", "This edit could not be confirmed as saved. Retry with the same op id. " diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 8453980db7a7c1c33f46cbecc452313da2855af0..a7c814d85107be5c917bb1bec545fb1e1a83c2dd 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -2224,5 +2224,7 @@ def live_limits_put(body: dict = Body(default=None), from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402 +from routes_amazon_events import router as amazon_events_router # noqa: E402 router.include_router(amazon_deletion_router) +router.include_router(amazon_events_router) diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index c0312620ac51eee064dfd723aa7b6da93971e543..bdb297280047a5e10387428f0209c4007184c90f 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -1633,10 +1633,11 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups) else: spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups) - if key == amazon.PREFIX + "inventory" and spec is not None and not spec.get("missing"): + if (key == amazon.PREFIX + "inventory" and spec is not None + and not spec.get("missing") and 'sourceStatus' not in spec): status = amazon.amazon_store.inventory_table_status(cur) spec = {**spec, 'sourceStatus': { - name: status[name] for name in ('empty', 'code', 'reason') if name in status}} + name: status[name] for name in ('empty', 'code', 'reason', 'recommendation') if name in status}} except Exception as exc: # noqa: BLE001 from core.request_timing import raise_if_resource_error raise_if_resource_error(exc) @@ -2749,11 +2750,11 @@ def _user_tables_token(rt): raise err(503, "schema_unavailable", "table definition revisions are unavailable") from exc -def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD): +def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD, hold=None): """`read()`, reused while this tenant's `user_tables` revision is unchanged. - ⛔ `clone` runs on the way in AND on every hit, so a caller that edits what it was handed - cannot edit the next request's answer. `keep(value)` decides what may be held: an empty + ⛔ `hold` (default `clone`) copies the value on the way in and `clone` runs on every hit, so a + caller that edits what it was handed cannot edit the next request's answer. `keep(value)` decides what may be held: an empty answer never is (it is the shape a failed read takes, and caching it would make one outage outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure. `token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round @@ -2771,7 +2772,7 @@ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD): return clone(hit[1]) value = read() if token is not None and tenant and keep(value): - held = (token, clone(value)) + held = (token, (hold or clone)(value)) with _REV_MEMO_LOCK: if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX: _REV_MEMO.pop(next(iter(_REV_MEMO)), None) @@ -2779,11 +2780,111 @@ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD): return value +class _DefsView(dict): + """One request's view of the held definitions: a table is deep-copied the first time it is read. + + Keys, `in` and `len` read the held mapping directly. Every door that hands out a definition + (`[]`, `get`, `values`, `items`, `pop`, `dict(view)`, `copy`, `deepcopy`) hands out this + request's own copy, so a caller edit never reaches the memo or another request. + """ + + # `user_tables._with_activity` copies a mapping with `dict()` unless it is marked lazy. + lazy_tables = True + __slots__ = ("_mine",) + + def __init__(self, held): + super().__init__(held) + self._mine = set() + + def __getitem__(self, key): + value = dict.__getitem__(self, key) + if key not in self._mine: + value = deepcopy(value) + dict.__setitem__(self, key, value) + self._mine.add(key) + return value + + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + self._mine.add(key) + + def __delitem__(self, key): + dict.__delitem__(self, key) + self._mine.discard(key) + + # A Python `__iter__` sends `dict(view)`, `{**view}` and `|` through `keys()` and `__getitem__`. + def __iter__(self): + return iter(dict.keys(self)) + + def get(self, key, default=None): + return self[key] if dict.__contains__(self, key) else default + + def setdefault(self, key, default=None): + if not dict.__contains__(self, key): + self[key] = default + return self[key] + + def pop(self, key, *default): + if not dict.__contains__(self, key): + if default: + return default[0] + raise KeyError(key) + value = self[key] + del self[key] + return value + + def popitem(self): + if not self: + raise KeyError("popitem(): dictionary is empty") + key = next(reversed(dict.keys(self))) + return key, self.pop(key) + + def update(self, *args, **kwargs): + for key, value in dict(*args, **kwargs).items(): + self[key] = value + + def values(self): + return [self[key] for key in dict.keys(self)] + + def items(self): + return [(key, self[key]) for key in dict.keys(self)] + + def copy(self): + return dict(self.items()) + + __copy__ = copy + + def __deepcopy__(self, memo): + return {key: deepcopy(dict.__getitem__(self, key), memo) for key in dict.keys(self)} + + def __reduce__(self): + return dict, (self.copy(),) + + def __or__(self, other): + if not isinstance(other, dict): + return NotImplemented + merged = self.copy() + merged.update(other) + return merged + + def __ror__(self, other): + if not isinstance(other, dict): + return NotImplemented + merged = dict(other) + merged.update(self.items()) + return merged + + def __ior__(self, other): + self.update(other) + return self + + def _page_defs(session): """Return one schema-only `(snapshot, revision)` for a page and its scoped targets. Shared metadata and source plans belong to the fresh lend, never to the revision memo. Projection and revision failures refuse instead of falling back to a whole document. + A memo hit lends a `_DefsView`, so a page copies only the definitions it reads. """ import copy import core.store as store @@ -2799,7 +2900,7 @@ def _page_defs(session): try: defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt, schema_only=True), - keep=keep, clone=copy.deepcopy, token=token) + keep=keep, clone=_DefsView, hold=copy.deepcopy, token=token) except Exception as exc: # noqa: BLE001 from core.request_timing import raise_if_resource_error raise_if_resource_error(exc) @@ -4068,6 +4169,24 @@ def resolved_fields(table_key, defn): return list(declared.values()) +def _source_status_limits(table_key, spec): + """Carry a declared source cause through the existing row and aggregate metadata. + + An empty source names why it is empty. A source that keeps rows beside a refusal (`empty` + false, with `code` and `reason`) names the refusal, so the count hover is not silent about it. + """ + status = spec.get('sourceStatus') or {} + if not status.get('reason'): + return [] + if status.get('empty'): + return [{'subject': table_key, 'effect': 'source_empty', 'code': status.get('code'), + 'cause': status['reason'], 'recommendation': status.get('recommendation') or ''}] + if not status.get('code'): + return [] + return [{'subject': table_key, 'effect': 'source_refused', 'code': status['code'], + 'cause': status['reason'], 'recommendation': status.get('recommendation') or ''}] + + def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None, search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False, st=None): @@ -4282,7 +4401,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= # that number would tell a scoped reader exactly how many rows they are not allowed to see. base_where = (f'({spec["where"]}) AND {wall_sql}' if (spec["where"] and wall_sql) else (wall_sql or spec["where"])) - limits = [] + limits = _source_status_limits(table_key, spec) if tree is _TREE_UNSET: tree = _json_arg(filters, "filters") sort_spec = _json_arg(sorts, "sorts") or [] @@ -5189,10 +5308,6 @@ def odoo_table_rows(table_key: str, defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"] cols, limits, frm = read["cols"], read["limits"], read["frm"] source_status = spec.get('sourceStatus') or {} - if source_status.get('empty') and source_status.get('reason'): - limits = [*limits, {'subject': table_key, 'effect': 'source_empty', - 'code': source_status.get('code'), 'cause': source_status['reason'], - 'recommendation': ''}] base_where, wall_params = read["base_where"], read["wall_params"] where, params, sort_spec = read["where"], read["params"], read["sort_spec"] _gsql, _gparams = _group_pin(cols, groupBy, groupKey) diff --git a/api/routes_products.py b/api/routes_products.py index b0acc543cdf480123638fd328f9357aa0fa1ad87..f73aee52efb18d3dcb696ff7737ff82ef47daa73 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -433,6 +433,11 @@ def _project_enabled_raw_rows(rows, enabled, *, drop_source=False): return out +#: R27 (D-3503): the pool serves the oldest active record and flags newer ones. Nobody picks one. +DUPLICATE_CODE_RECOMMENDATION = ('The oldest active Odoo record is the canonical record for each ' + 'shared SKU. Newer records are flagged for review.') + + def _duplicate_code_limits(rows_src, fields): """Report collapsed products only through permitted rows and the visible code field.""" import modules.product_data as pd @@ -446,19 +451,21 @@ def _duplicate_code_limits(rows_src, fields): codes = ', '.join(report['codes']) return [{'subject': 'code', 'effect': 'duplicate_codes', 'cause': f"Shared SKU codes {codes}: {report['gap']} fewer rows than active Odoo products for these codes.", - 'recommendation': 'Ask the owner to choose the canonical Odoo record for each shared SKU.'}] + 'recommendation': DUPLICATE_CODE_RECOMMENDATION}] def product_assembly(session: Session, scope: str = "product", storage_key: str = "", consume_corrections: bool = False, *, with_rows: bool = True, needed=None): from routes_customers import grid_assembly from core import perm_scope + session = grid_events.request_admission(session, 'product_table_workspace', schema_only=not with_rows) team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE) - return grid_assembly(session, scope, storage_key, + result = grid_assembly(session, scope, storage_key, build=lambda borrowed: _compute_product_assembly( borrowed, scope, storage_key, False, with_rows=with_rows, needed=needed), pool_stamp=lambda: _measure_stamp(session.runtime, team_id) if with_rows else 0, with_rows=with_rows, needed=needed) + return dict(result, _admission=session.runtime) def _compute_product_assembly(session: Session, scope: str = "product", storage_key: str = "", @@ -772,12 +779,15 @@ def patch_product(pid: int, body: dict = Body(default=None), raise err(403, "out_of_scope", "that product is not in your catalogue") import core.perm_scope as perm_scope + table = pd.table_ops(session.runtime) + table.overlay_readback = {} + admission = g.get('_admission') ctx = grid_events.EventCtx( uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], admin=session.admin, fallback_ws=None, seen_ids={}, hidden_keys=perm_scope.hidden_keys( - session.user, MODULE, g["fields"], st=session.runtime), - scope_key="product", table=pd.table_ops(session.runtime), st=session.runtime) + session.user, MODULE, g["fields"], st=admission or session.runtime), + scope_key="product", table=table, st=session.runtime, admission=admission) if notes is not None: return _patch_cell_notes(MODULE, pid, notes, ctx, source_row_exists=lambda key, rid, admitted: key == MODULE and int(rid) in g['pids']) @@ -797,8 +807,8 @@ def patch_product(pid: int, body: dict = Body(default=None), # only the per-user stratum would find nothing and report every accepted shared write as # `refused` — a correct write reported as a failure, which is the read-path-cannot-witness- # the-write-path shape ([[read-path-cannot-witness-write-path]]). - stored = (grid_events.table_workspace(ctx, allowed_pids=None) - .get("overlays") or {}).get(str(pid)) or {} + stored = dict(((g.get('ws') or {}).get('overlays') or {}).get(str(pid)) or {}) + stored.update(table.overlay_readback.get(str(pid)) or {}) stored = {**stored, **((pd.shared_cells([pid], st=session.runtime).get(str(pid))) or {})} accepted = {k: stored.get(k) for k in updates if k in stored} refused = sorted(k for k in updates diff --git a/api/routes_query.py b/api/routes_query.py index b89887e9bcb1cf339bac5274f898f327d6e5d60d..def1943c46686fbcd249d3a97fdae4cfbe997046 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -1089,6 +1089,15 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s chosen = None offered = {tool["function"]["name"] for tool in tools} steps = 0 + message_deliveries = [] + + def completed_reply(said): + # A delivery receipt owns the copy only when this turn did nothing else. + if (message_deliveries and all(message_deliveries) + and len(turn["intentTexts"]) == 1 and not turn.get("pendingDelete")): + return "Sent" + return said + while True: # Joined messages remain transient and are admitted before every model step. checkpoint = _turn_checkpoint(turn, messages) @@ -1122,6 +1131,8 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s return None, None, provider, "stopped" if spec is not None and not reason and snapshot is not None: turn["toolCalls"].append({"name": "build_view", "args": _safe(spec), "effect": "read", "ok": False, "ref": None}) + if spec is None and not reason: + said = completed_reply(said) return spec, said, provider, reason calls = spec["tool_calls"] if not isinstance(calls, list) or not calls: @@ -1130,7 +1141,7 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s continue if closed == "stopped": return None, None, provider, "stopped" - return None, said or "The assistant did not request a tool.", provider, None + return None, completed_reply(said) or "The assistant did not request a tool.", provider, None if any(not isinstance(call, dict) or not isinstance(call.get("name"), str) for call in calls): closed = _close_model_turn(turn) if closed == "joined": @@ -1206,6 +1217,10 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s for recorded in turn["toolCalls"]: if recorded.get("name") == "remember" and recorded.get("ref") == result["id"]: recorded.pop("linkNote", None) + message_deliveries.append( + name == "agent_message" and isinstance(result, dict) and result.get("ok") is True + and bool(result.get("request_id")) + and not any(result.get(key) for key in ("refused", "error", "pending", "pendingDelete"))) if isinstance(result, dict) and result.get("stopped"): return None, None, provider, "stopped" content = json.dumps(result, default=str, ensure_ascii=False) diff --git a/api/routes_shares.py b/api/routes_shares.py index 66d8c7bbd81f9940b4ed808d69da3d1aaea3ca05..a76f43418359cab15f7be4106d5b3833fc2be1d7 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1,6 +1,6 @@ """routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). - GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} + GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, maxGrantableRole, people} PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) PUT /api/v1/share/{kind}/{oid}/owner <- {owner: ""} (R6d — `field` ONLY) GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} @@ -60,9 +60,8 @@ SECOND ONE IS GONE RATHER THAN SOFTENED.** It read *"'Widen a users-scoped view NOT closed — an `edit` grantee may add `*` — it is CAPPED at Can view"*, and that was an accurate account of R4 that the owner overruled the moment it was put in front of them. *"Grant themselves ownership and lock you out"* stays closed by (3) and `set_grants`' sticky owner. A `view` grantee -still cannot share at all, and still gets `403`. The client greys the editor for -non-administrators and offers "Can edit" to anyone it greys in; that is a courtesy, and these -checks are the wall. +still cannot share at all, and still gets `403`. The client caps both role pickers with +`maxGrantableRole`; these checks remain the wall. ⚠ **WHAT `GET` NOW DISCLOSES.** The grant record carries `by`, and this route returns the record verbatim — so anybody who may read a view's grant list also learns who added each person, a @@ -1247,7 +1246,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)): # registry has no owner to return from `may_administer`, even though the same caller may safely # claim their own object through the PUT path below. Reflect that exact claim predicate here: # a Member who owns an unshared View receives the people picker; a collaborator still does not. - if not rec["owner"] and not may_admin and _owns_object(session, kind, oid): + claiming = not rec["owner"] and _owns_object(session, kind, oid) + if claiming: may_admin = True # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — @@ -1279,6 +1279,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)): **rec, "role": role, "mayAdminister": may_admin, + "maxGrantableRole": "edit" if claiming else shares.max_grantable_role( + kind, oid, session.uname, is_admin=_admin_share, st=session.runtime), "ownerLabel": owner_label, # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry # them. `assignable_people` serves bare display names because `user`-kind CELLS store diff --git a/api/routes_sheet.py b/api/routes_sheet.py index ead24165665cb8299b44179e207780ce28c3f643..42a3e77e57f7b7f62059b1379994898d6390e6e0 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -2190,6 +2190,16 @@ def _refresh_context(book, binding, memo=None, *, tab=None, session=None): payload["inputs"] = memo["inputs"] # Relative filters and measure windows can change their answer at midnight without a write. encoded = json.dumps(binding, sort_keys=True) + scope = binding.get("scope") or {} + if session is not None and scope.get("view_id"): + if memo is None: + memo = {} + try: + payload["view"] = _view_filter(session, binding.get("database"), scope["view_id"], + scope.get("view_name", ""), memo=memo.setdefault("views", {})) + except HTTPException as exc: + # Resolution owns the per-cell refusal when a saved view disappears. + payload["view"] = {"refusal": exc.detail} if (_binding_is_volatile(binding, graph, closure) or any(word in encoded for word in ('"window"', '"within"', '"ending"', '"view_id"'))): payload["today"] = _sheet_today(session) @@ -7573,7 +7583,7 @@ def _asked_key(asked, home, fr): @router.post("/sheets/{workbook_id}/resolve-bindings") def resolve_bindings(workbook_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): + session: Session = Depends(require_session), response: Response = None): """Values or named refusals for the NAMED bound cells only (contract C1, rulings R2 and R48). ⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL. @@ -7629,7 +7639,10 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None), pairs in it rather than by the number of cells. """ with _sheet_request_scope(session): - return _resolve_bindings_scoped(workbook_id, body, session) + answer = _resolve_bindings_scoped(workbook_id, body, session) + if response is not None: + response.headers["X-Sheet-Resolved-Cells"] = str(len(answer)) + return answer def _resolve_bindings_scoped(workbook_id: str, body, session: Session): @@ -7637,7 +7650,8 @@ def _resolve_bindings_scoped(workbook_id: str, body, session: Session): import formula_runtime as fr try: - book = sheets.get_workbook(_store(session), workbook_id) + book = sheets.get_workbook(_store(session), workbook_id, + before_normalize=lambda doc: _require_reader(session, doc)) except sheets.SheetRefusal as exc: raise _refuse(exc) @@ -7655,29 +7669,65 @@ def _resolve_bindings_scoped(workbook_id: str, body, session: Session): view_memo = {} by_binding = {} answer = {} - for asked in (payload.get("cells") or ()): - label = str(asked or "") - if not label: - continue - cell = bound.get(_asked_key(label, home, fr)) - if cell is None: - continue + requested = {str(asked): _asked_key(asked, home, fr) + for asked in payload.get("cells") or () if str(asked or "")} + selected = set(requested.values()) & bound.keys() + graph = _read_graph(session, book, fr, None) + previous = getattr(session, "_sheet_refresh_input", None) + cache = None + + def viewer_input(key): + nonlocal cache + if key in selected: + return None + from core import sheet_binding + if cache is None: + cache = sheet_binding.read_refresh_cache(_store(session), workbook_id) + cell = bound[key] binding = cell.get("binding") or {} - own_tab = _asked_key(label, home, fr).rpartition("!")[0] - slot = json.dumps([own_tab, binding], sort_keys=True, default=str) - if slot not in by_binding: - try: - by_binding[slot] = _resolve_cell_binding(session, binding, book, own_tab, - view_memo=view_memo)["value"] - except HTTPException as exc: - detail = getattr(exc, "detail", None) or {} - info = detail.get("error") if isinstance(detail, dict) else None - by_binding[slot] = { - "value": None, - "refusal": { - "code": (info or {}).get("code", "sheet_unresolved"), - "message": (info or {}).get("message", "This cell could not be refreshed."), - }, - } - answer[label] = by_binding[slot] + database = _require_database(session, binding.get("database")) + slot = _binding_scope_key(session, database, binding) + _require_aggregate_field(session, binding) + own_tab, _, addr = key.rpartition("!") + cell_key = str(sheets.find_tab(book, own_tab)["id"]) + "!" + addr + hit = (cache.get("refreshCells") or {}).get(cell_key, {}).get(slot) + # A viewer's filter request never resolves an unnamed input or borrows an owner's value. + if not hit or hit["cell"].get("state") == "refused": + raise err(409, "sheet_refresh_required", "Refresh this input cell before refreshing its dependents.") + return _public_refresh_snapshot(hit["cell"]) + + viewer = _role_on_workbook(session, book) == "view" + if viewer: + session._sheet_refresh_input = viewer_input + try: + for label, key in requested.items(): + cell = bound.get(key) + if cell is None: + continue + binding = cell.get("binding") or {} + own_tab = key.rpartition("!")[0] + slot = json.dumps([own_tab, binding], sort_keys=True, default=str) + if slot not in by_binding: + try: + resolved = graph["bound_values"].get(key) + if isinstance(resolved, HTTPException): + raise resolved + if resolved is None: + resolved = _resolve_cell_binding(session, binding, book, own_tab, + view_memo=view_memo) + graph["bound_values"][key] = resolved + by_binding[slot] = resolved["value"] + except HTTPException as exc: + detail = getattr(exc, "detail", None) or {} + info = detail.get("error") if isinstance(detail, dict) else None + by_binding[slot] = { + "value": None, + "refusal": { + "code": (info or {}).get("code", "sheet_unresolved"), + "message": (info or {}).get("message", "This cell could not be refreshed."), + }, + } + answer[label] = by_binding[slot] + finally: + session._sheet_refresh_input = previous return answer diff --git a/api/routes_sku_actions.py b/api/routes_sku_actions.py index 378a1da6d14ba4195e63e33e3b24024fd7c6e02a..989a641bfd0dd2e7ae122e4e04718e64df7ffc04 100644 --- a/api/routes_sku_actions.py +++ b/api/routes_sku_actions.py @@ -173,12 +173,12 @@ def _results_sources(session): def _results_mirror(runtime): """Open a private cursor only after asserting the caller's tenant mirror binding.""" from pathlib import Path - from harness import datastore, runtime as runtime_module + from harness import runtime as runtime_module try: runtime.assert_datastore_matches() if not Path(runtime.datastore_path).is_file(): raise err(503, 'sku_action_results_source_unavailable', 'This workspace has no Amazon mirror yet.') - return datastore.ro_cursor() + return runtime_module.mirror_cursor(runtime, require_ready=False) except runtime_module.DatastoreMismatch: raise err(503, 'sku_action_results_store_mismatch', 'The workspace mirror binding does not match.') from None except HTTPException: diff --git a/api/routes_tables.py b/api/routes_tables.py index 0d85992b6d44ef2a4391a5c1041a84267d713620..3e6392232dcf77aec71b972287ea2a7b8fafab33 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -54,6 +54,7 @@ from fastapi import APIRouter from fastapi import HTTPException, Request, Response from deps import Session, err, require_session +from harness.runtime import DatastoreMismatch from routes_grid import row_patch_receipt router = APIRouter(prefix="/api/v1") @@ -773,6 +774,31 @@ def query_activity(session: Session, *, object=None, by=None, since=None, until= _with_workspace=_with_workspace) +class _ActivityAppReads: + """One page's App registry reads, shared by the index's two registry loads (S85-T169).""" + + _READS = frozenset(('get', 'get_many', 'list_keys', 'get_projection', 'get_entry', 'exists')) + + def __init__(self, runtime): + self._runtime, self._memo = runtime, {} + + def __getattr__(self, name): + attr = getattr(self._runtime, name) + if name not in self._READS or not callable(attr): + return attr + + def read(*args, **kwargs): + if kwargs.get('fresh'): + return attr(*args, **kwargs) + identity = repr((name, args, sorted(kwargs.items()))) + if identity not in self._memo: + self._memo[identity] = attr(*args, **kwargs) + from copy import deepcopy + return deepcopy(self._memo[identity]) + + return read + + def _query_activity_page(session: Session, *, object=None, by=None, since=None, until=None, cursor=None, limit=100, scan_limit=10000, _with_workspace=False): """Resolve one activity page within its caller's request scope.""" @@ -884,8 +910,11 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, if 'app' in obj: from routes_assets import application_index if applications is None: + # The index loads the registry twice. This page lists and reads it once. + reader = copy(session) + reader.runtime = _ActivityAppReads(session.runtime) applications = {app['id'] for app in - application_index(session=session)['applications']} + application_index(session=reader)['applications']} admitted = obj['app'] in applications elif 'workbook' in obj: import routes_sheet @@ -1140,6 +1169,8 @@ def scoped_pool(session: Session, table_key: str, st=None): # short one. `scoped_pids` below takes the same refusal and answers a different # question with it, because an envelope needs no row. raise err(409, "window_required", str(e)) + except DatastoreMismatch as e: + raise err(409, "cross_tenant_store", str(e)) from e except RuntimeError as e: raise err(503, "store_not_ready", str(e)) rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base, @@ -1181,7 +1212,7 @@ def _new_locked_shared_field(session, table_key, pid, key, body): or any(proposed.get(name) for name in ( "formula", "rollup", "if", "automation", "metric", "measure", "link"))): raise err(400, "bad_field", "Only a user-created value column can be added here.") - fields = _ut().clean_fields([proposed]) + fields = _ut().clean_fields([proposed], table_key=table_key, st=session.runtime) if not fields or fields[0]["key"] != key or not _locked_user_field(fields[0]): raise err(400, "bad_field", "The column name or type is invalid.") field = fields[0] @@ -1536,6 +1567,8 @@ def scoped_pids(session: Session, table_key: str, limits=None, st=None): if limits is not None: limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) return frozenset(), fields_base, defn + except DatastoreMismatch as e: + raise err(409, "cross_tenant_store", str(e)) from e except RuntimeError as e: raise err(503, "store_not_ready", str(e)) # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection @@ -1750,7 +1783,7 @@ _USAGE_KEYS = ("views", "filters", "rollups", "automations", "permRules") _USAGE_DISPLAY_REFS = ("dateField", "stackField", "titleField", "colorField", "sizeField", "coordField") -#: `{(tenant, table_key): (token, {field_key: usage})}` — see `_field_usage` for why this is a +#: `{(tenant, table_key, field_keys): (token, {field_key: usage})}` — see `_field_usage` for why this is a #: memo rather than three fresh documents per assembly. Bounded: cleared past the cap rather than #: grown, the same rule `core.measure_resolve`'s memos keep. _USAGE_MEMO = {} @@ -1948,7 +1981,8 @@ def _field_usage(session, table_key, fields, defn, ws_st, ws_key, st=None): if not keys: return {} lent = st if st is not None else session.runtime - memo_key = (str(getattr(session, "tenant", "") or ""), str(table_key)) + # Registry callers lend already-walled fields, so a narrower reader cannot seed missing counts. + memo_key = (str(getattr(session, "tenant", "") or ""), str(table_key), tuple(sorted(keys))) token = _usage_token(lent, ws_key) if token is not None: hit = _USAGE_MEMO.get(memo_key) @@ -2043,9 +2077,61 @@ def _field_usage(session, table_key, fields, defn, ws_st, ws_key, st=None): return tally +def _table_page_store(runtime): + """Keep SQL pages on the bound store while preserving explicit snapshot readers.""" + from core import grid_events + from routes_customers import _AssemblyStore + while isinstance(runtime, (grid_events.AdmissionSnapshot, _AssemblyStore)): + runtime = runtime._runtime + return runtime + + +def _scoped_table_rows(session, table_key, st, *, cursor=None, limit=None): + """Read one scoped page or follow every continuation for a complete assembly.""" + from core import perm_scope, store, store_pg + try: + defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) + fields = _source_contract_fields(table_key, [dict(f) for f in (defn.get('fields') or [])], st=st) + runtime = _table_page_store(session.runtime) + sql_pages = isinstance(perm_scope._window_store(runtime), store_pg.PgStore) + reader = runtime if sql_pages and ('rows' not in defn or store.is_projected(defn)) else st + rows = [] + while True: + page = perm_scope.scoped_table_page(session.user, table_key, st=reader, + cursor=cursor, limit=1024 if limit is None else limit) + rows.extend(page['rows']) + cursor = page['nextCursor'] + # An empty permitted page can still have later readable records. + if limit is not None or cursor is None: + break + except ValueError as exc: + raise err(400, 'bad_table_page', str(exc)) from exc + except perm_scope.Denied as exc: + raise err(403, 'forbidden', str(exc)) from exc + except perm_scope.UnknownTable as exc: + raise err(404, 'unknown_table', str(exc)) from exc + except perm_scope.Unresolvable as exc: + raise err(409, 'scope_unavailable', str(exc)) from exc + except DatastoreMismatch as exc: + raise err(409, 'cross_tenant_store', str(exc)) from exc + except RuntimeError as exc: + raise err(503, 'store_not_ready', str(exc)) from exc + return frozenset(row['pid'] for row in rows), rows, fields, defn, cursor + + def ut_assembly(session: Session, table_key: str, storage_key: str = "", consume_corrections: bool = False, with_rows: bool = True, st=None, - *, fields_only: bool = False, needed=None): + *, fields_only: bool = False, needed=None, cursor=None, limit=None): + from core import grid_events + actor = grid_events.request_admission(session, table_key + '_table_workspace') + result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st, + fields_only=fields_only, needed=needed, cursor=cursor, limit=limit) + return dict(result, _admission=actor.runtime) + + +def _ut_assembly(session: Session, table_key: str, storage_key: str = "", + consume_corrections: bool = False, with_rows: bool = True, st=None, + *, fields_only: bool = False, needed=None, cursor=None, limit=None): if not with_rows and st is None: if not fields_only: import routes_odoo_tables as connected @@ -2062,7 +2148,7 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", borrowed, table_key, storage_key, False, False, fields_only=fields_only, needed=needed), pool_stamp=lambda: 0, with_rows=not fields_only, needed=needed) return _compute_ut_assembly(session, table_key, storage_key, False, with_rows, st, - fields_only=fields_only, needed=needed) + fields_only=fields_only, needed=needed, cursor=cursor, limit=limit) def _metadata_pids(session, table_key, ctx, st): @@ -2112,7 +2198,7 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" consume_corrections: bool = False, with_rows: bool = True, st=None, *, fields_only: bool = False, needed=None, include_usage: bool = True, include_measures: bool = True, include_views: bool = True, - metadata_only: bool = True): + metadata_only: bool = True, cursor=None, limit=None): """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so `/workspace` and the events route consume any of the three interchangeably. @@ -2145,7 +2231,10 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" # object then serves `object_shares` to the grant legs downstream, so the assembly stops # reading that bucket once per question asked of it. if st is None: - st = _table_lend(session.runtime, table_key, defs_only=not with_rows) + from core import perm_scope, store_pg + sql_pages = isinstance(perm_scope._window_store(_table_page_store(session.runtime)), store_pg.PgStore) + st = _table_lend(session.runtime, table_key, defs_only=not with_rows or sql_pages) + next_cursor = None metadata_membership = False if fields_only: defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) @@ -2156,9 +2245,8 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" fields_base = list(defn.get("fields") or []) pids, rows_src = frozenset(), [] elif with_rows: - # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only - # a READ route passes one — see `scoped_pool`'s own note and contract C5. - pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) + pids, rows_src, fields_base, defn, next_cursor = _scoped_table_rows( + session, table_key, st, cursor=cursor, limit=limit) else: defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) metadata_membership = (metadata_only and table_key != "ut_activity" and @@ -2368,6 +2456,7 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" workspace['canAddField'] = _ut().may_edit_database( table_key, session.uname, is_admin=session.admin, st=st) return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, + "nextCursor": next_cursor, "fields": fields, "views": views, "lists": lists, "hidden": hidden, "derived": derived, # ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a @@ -2685,6 +2774,13 @@ def list_tables(brief: bool = Query(False), "created": t.get("created") or "", "rowCount": (counts.get(key) if ut.materialises(key, defn=t) and type(counts.get(key)) is int and counts[key] >= 0 else None)}) + # The stored definition's empty-source cause, from the definition already in hand. + # No sourceStatus adds no key, so the row keeps its shape. + if isinstance(t.get("sourceStatus"), dict): + from routes_odoo_tables import _source_status_limits + limits = _source_status_limits(key, t) + if limits: + row["limits"] = limits out.append(row) if brief: # ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and @@ -2865,9 +2961,11 @@ def create_table(body: dict = Body(default=None), recurrence_validator = lambda raw: govern_date_recurrence( session, "same", base_fields, raw) try: + # A top-level `subject` is a claim `ut.create` refuses. Never drop it silently. key = ut.create(label, session.uname, fields=raw_fields, source=source, st=session.runtime, - recurrence_validator=recurrence_validator) + recurrence_validator=recurrence_validator, + subject=body.get("subject")) except ValueError as exc: raise err(400, getattr(exc, 'code', 'bad_recurrence'), str(exc)) except Exception: @@ -2911,7 +3009,7 @@ def table_footprint(table_key: str, session: Session = Depends(require_session)) like the delete itself: only someone who could delete may case the joint. ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row - COUNT needs the whole document, and only on a materialised table.""" + COUNT needs stored rows on a materialised table, or the mirror on a read-through table.""" defn = _defn_or_refuse(session, table_key, defs_only=True) if not (session.admin or defn.get("createdBy") == session.uname): raise err(403, "forbidden", "only the database's creator or an admin can delete it") @@ -2949,18 +3047,29 @@ def table_footprint(table_key: str, session: Session = Depends(require_session)) auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) except Exception: pass - # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where - # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, - # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the - # projection is not what makes it wrong. - # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 - # in a dialog headed "what dies with this database"). Booked rather than fixed here: this - # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` - # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. if _ut().materialises(table_key, st=s, defn=defn): rows = len((_ut().get_one(table_key, st=s) or {}).get("rows") or {}) else: - rows = 0 + from routes_odoo_tables import _mirror_cur, population + cur = None + try: + cur, mismatch = _mirror_cur(session, table_key) + if mismatch: + raise err(409, 'cross_tenant_store', mismatch) + rows = population(table_key, cur=cur, rt=s) + if rows is None: + raise err(503, 'footprint_count_unavailable', + 'The row count is not held here. This database has no readable mirror binding.') + except HTTPException: + raise + except RuntimeError as exc: + raise err(503, 'store_not_ready', str(exc)) from exc + except Exception as exc: + raise err(503, 'footprint_count_unavailable', + 'The mirror row count could not be read. Retry when the mirror is available.') from exc + finally: + if cur is not None: + cur.close() return {"rows": rows, "fields": fields, "views": len(views), "sharedUsers": len(g.get("entries") or []), "automations": sorted(auto, key=lambda a: a["name"].lower())} @@ -3136,15 +3245,98 @@ def _amazon_profit_period_cells(table_key, fields, merged, st): if not diagnostic_pids: return merged label = _amazon_recorded_period_label(st) + from modules.amazon_profit import BLANK_REASONS + keys = {field.get("key") for field in fields} out = dict(merged) for pid in diagnostic_pids: - out[pid] = dict(merged[pid], period=label) + cells = dict(merged[pid], period=label) + # A legacy refusal blanked figures and lived only in Period. An empty reason cell beside a + # blank figure carries it, so the blank still says why. + for note, figures in BLANK_REASONS.items(): + if (note in keys and not str(cells.get(note) or "").strip() + and any(key in keys and cells.get(key) in (None, "") for key in figures)): + cells[note] = merged[pid]["period"] + out[pid] = cells return out +def _stored_link_pin_ids(session, target, related): + """Target ids of an offered one-hop stored link, read from the cell the source rows door serves. + + None when the pin is not that shape, so the caller keeps the original refusal. The offer, both + grants, the source row wall and the source field wall all apply, as they do for the source grid. + """ + import core.relations as relations + import routes_odoo_tables as connected + from routes_grid import _may_open, related_path_entries + + if isinstance(related, str): + try: + related = json.loads(related) + except ValueError: + return None + if not isinstance(related, dict): + return None + source, via = str(related.get("table") or ""), related.get("via") + path = connected._via_path(via) + if path is not None and len(path) != 1: + return None + try: + record = int(related.get("record")) + except (TypeError, ValueError): + return None + ut = _ut() + if (not source.startswith(ut.KEY_PREFIX) or not _may_open(session, source) + or not _may_open(session, target)): + return None + entries = related_path_entries(session, source, 1) + entry = relations.pick_path(entries, path) if path else relations.pick(entries, via) + if entry is None or str(entry.get("target")) != str(target) or not entry.get("served"): + return None + bag, link = entry.get("via") or {}, entry.get("link") or {} + field = str(bag.get("field") or "") + # The same manual-link predicate `routes_odoo_tables._relation_leg_ids` applies. + manual = bool(field and link and not link.get("inverse") + and (link.get("selectionMode") == "manual" or not link.get("on"))) + if bag.get("direction") != "forward" or not manual: + return None + if not ut.materialises(source, st=session.runtime): + return None + g = ut_assembly(session, source, consume_corrections=False) + if (record not in g["pids"] or field in (g.get("hidden") or ()) + or field not in {f.get("key") for f in g["fields"]}): + return None + base = next((r for r in g["rows_src"] if str(r.get("pid")) == str(record)), None) + if base is None: + return None + # The merge order of `table_rows`: definition row, then this user's overlay, then shared cells. + cells = {k: v for k, v in base.items() if k != "pid"} + overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(record)) + if isinstance(overlay, dict): + cells.update(overlay) + shared = (g.get("shared_cells") or {}).get(str(record)) + if isinstance(shared, dict): + cells.update(shared) + return {int(pid) for pid in connected._relation_cell_ids(cells.get(field))} + + +def _offered_link_rows(session, table_key, rows, related, refusal): + """Narrow by a stored link edge `/related` offers when the shared resolver refused it (D-13545). + + Any other refusal, or a pin of another shape, re-raises `refusal` unchanged. + """ + detail = refusal.detail if isinstance(refusal.detail, dict) else {} + if refusal.status_code != 400 or (detail.get("error") or {}).get("code") != "unknown_relation": + raise refusal + ids = _stored_link_pin_ids(session, table_key, related) + if ids is None: + raise refusal + return [row for row in rows if int(row["pid"]) in ids] + + @router.get("/tables/{table_key}/rows") def table_rows(table_key: str, session: Session = Depends(require_session), - related: str = None, cursor: str = None, limit: int = 100): + related: str = None, cursor: str = None, limit: int = None): """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, identity}` — so the client's generic topic fetch consumes it with zero new parsing. @@ -3158,7 +3350,8 @@ def table_rows(table_key: str, session: Session = Depends(require_session), if table_key == 'ut_activity': if related: raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.') - page = query_activity(session, cursor=cursor, limit=limit, _with_workspace=True) + page = query_activity(session, cursor=cursor, limit=100 if limit is None else limit, + _with_workspace=True) g = page.pop('_grid') json_keys = {field['key'] for field in g['fields'] if field.get('type') == 'json'} rows = [{name: json.dumps(value, ensure_ascii=False, separators=(',', ':')) @@ -3183,42 +3376,21 @@ def table_rows(table_key: str, session: Session = Depends(require_session), # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the # value it replaced (contract C5). - if _ut()._entry_reader(session.runtime) is None: - _lent = _table_lend(session.runtime, table_key) - g = ut_assembly(session, table_key, st=_lent, consume_corrections=False, needed=source_needed) - else: - from registry_source import with_runtime - from routes_customers import _AssemblyStore - # One request owns these borrowed documents. A cold memo build and its page share reads; - # a warm page reads records/shared cells afresh and lends Zone documents to both readers. - original_rt = (session.runtime._runtime if isinstance(session.runtime, _AssemblyStore) - else session.runtime) - if not isinstance(session.runtime, _AssemblyStore): - session = with_runtime(session, _AssemblyStore(original_rt)) - g = ut_assembly(session, table_key, with_rows=False, needed=source_needed) - assembly_rt = session.runtime - entry_key = (_ut().STORE_KEY, ("entry", str(table_key))) - if entry_key in assembly_rt.docs: - # A cold build already read this entry. The page only borrows it within this request. - entry = assembly_rt.docs[entry_key] - else: - # A warm memo holds no records. Read the original runtime, avoiding a whole-entry copy. - entry = _table_entry(original_rt, table_key) - _lent = _ut().lend(assembly_rt, user_tables={str(table_key): entry} if entry else {}) - # The page borrows the same envelope as workspace; only records and shared cells are fresh. - pids, rows_src, _fields_base, _defn = scoped_pool(session, table_key, st=_lent) - g['pids'] = pids - from core import perm_scope - g['rows_src'] = [perm_scope.strip_row(row, g['hidden']) for row in rows_src - if row['pid'] in g['pids']] - g['shared_cells'] = { - pid: {key: value for key, value in cells.items() if key not in g['hidden']} - for pid, cells in _ut_shared_cells(session, table_key, g['pids']).items()} - if table_key in _AMAZON_PROFIT_TABLES: - # Schema metadata omits personal cells; restore them after row admission through the bound reader. - personal = _ops(session, table_key, st=original_rt).workspace( - session.uname, consume_corrections=False) - g['ws'] = dict(g.get('ws') or {}, overlays=personal.get('overlays') or {}) + from core import perm_scope, store_pg + try: + sql_pages = isinstance(perm_scope._window_store(_table_page_store(session.runtime)), store_pg.PgStore) + _lent = _table_lend(session.runtime, table_key, defs_only=sql_pages) + except DatastoreMismatch as exc: + raise err(409, 'cross_tenant_store', str(exc)) from exc + except RuntimeError as exc: + raise err(503, 'store_not_ready', str(exc)) from exc + # Bare GET callers consume a complete book; a cursor alone requests one default page. + page_limit = 100 if cursor is not None and limit is None else limit + # Canonical evaluators lend one bounded batch and must retain every row in that batch. + if hasattr(session.runtime, '_canonical_needed'): + page_limit = None + g = ut_assembly(session, table_key, st=_lent, consume_corrections=False, + needed=source_needed, cursor=cursor, limit=page_limit) # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge # this line does not perform since the route was written (owner item 3, 2026-08-09: # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). @@ -3287,7 +3459,10 @@ def table_rows(table_key: str, session: Session = Depends(require_session), # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. _stamp_ai_states(table_key, g["fields"], rows, st=_lent) from routes_odoo_tables import narrow_related_rows - rows = narrow_related_rows(session, table_key, rows, related) + try: + rows = narrow_related_rows(session, table_key, rows, related) + except HTTPException as exc: + rows = _offered_link_rows(session, table_key, rows, related, exc) rows, g['fields'], metric_refusals = scope_linked_metrics( session, table_key, rows, g['fields'], g['measures'], related) if related: @@ -3305,7 +3480,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session), _walled = [] if rows or not perm_scope.row_scope_applies(session.user, table_key) else [{ "subject": "rows", "effect": "filtered", "cause": "an access rule an administrator set for your account matches no record in " - "this database", + "this page", "recommendation": "ask an administrator to review your access rule"}] # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the # only thing that ever set this key; the write door never stopped working and every client @@ -3322,10 +3497,12 @@ def table_rows(table_key: str, session: Session = Depends(require_session), for zone in _ut().zones(table_key, st=_lent)] from core import connector_spec rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime) - rows = _project_cell_notes(table_key, rows, projection_fields, session.runtime, + rows = _project_cell_notes(table_key, rows, projection_fields, + session.runtime if sql_pages else _lent, hidden=g.get('hidden') or ()) rows = _exclude_sourceless(rows, projection_fields) return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"], + "nextCursor": g['nextCursor'], "refusals": metric_refusals, "zones": _zones, "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, @@ -3571,16 +3748,25 @@ def import_rows(table_key: str, body: dict = Body(default=None), # could not see a column the user had just created — it reads the DEFINITION. Same defect shape # as the rename (item 6a): two places to look, and the surfaces disagreed silently. -def _field_or_refuse(session, table_key, fkey="", *, raw=None): - """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. +def _registry_field_schema(session, table_key): + """Read the caller's complete registry schema without loading connector records.""" + from routes_grid import workspace + scope = {'product_data': 'product', 'customer_data': 'customer'}[table_key] + return workspace(scope=scope, session=session, fields_only=True)['workspace']['fields'] - ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and - `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they - write through the `user_tables` write doors), so no projected snapshot survives into a write. - ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 - draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" - defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + +def _field_or_refuse(session, table_key, fkey="", *, raw=None, create=False): + """Admit schema edits through table and field walls, with administrator registry Link creates as the sole exception.""" ut = _ut() + if create and not fkey and table_key in ut.LINKABLE_MODULE_KEYS: + if not isinstance(raw, dict) or str(raw.get('type') or '').strip().lower() != 'link': + raise err(400, 'not_a_user_table', + 'A connected database accepts Link fields here. Its source owns the other column types.') + if not session.admin: + raise err(403, 'forbidden', 'Only an administrator can add a Link field to a connected database.') + session.require(table_key) + return {'fields': _registry_field_schema(session, table_key)} + defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) if not ut.is_user_table(table_key, st=session.runtime): raise err(400, "not_a_user_table", "only a user-created database has an editable schema. A connected source " @@ -3635,15 +3821,16 @@ def _admit_if_sources(session, table_key, body, definition, fkey=''): raise err(403 if exc.code == 'if_source_forbidden' else 400, exc.code, str(exc)) -def _admit_link_selection(session, body): +def _admit_link_selection(session, body, *, hidden_code='link_selection_forbidden'): """S76-T33: a Link condition may name only target fields this session can read.""" raw = body if isinstance(body, dict) else {} bag = raw.get("link") if not isinstance(bag, dict) or not isinstance(bag.get("selectionFilter"), dict): return - target = str(bag.get("table") or "") - if not target.startswith(_ut().KEY_PREFIX): - return # a registry target is checked in core, which fails closed + target = _ut()._field_key(bag.get("table")) + registry_target = target in _ut().LINKABLE_MODULE_KEYS + if not target.startswith(_ut().KEY_PREFIX) and not registry_target: + return import core.perm_scope as perm_scope if not perm_scope.may_read(session.user, target, st=session.runtime): raise err(403, "link_selection_forbidden", @@ -3656,21 +3843,52 @@ def _admit_link_selection(session, body): stack.extend(node.get("children") or []) rhs = node.get("rhs") if isinstance(node.get("rhs"), dict) else {} named.update(ref for ref in (node.get("colId"), rhs.get("colId")) if isinstance(ref, str) and ref) - fields = (_table_entry(session.runtime, target) or {}).get("fields") or [] - if named & set(_ut_hidden(session, target, fields, st=session.runtime) or ()): - raise err(403, "link_selection_forbidden", + if registry_target: + session.require(target) + fields = _registry_field_schema(session, target) + hidden = named - {field['key'] for field in fields} + else: + fields = (_table_entry(session.runtime, target) or {}).get("fields") or [] + hidden = _ut_hidden(session, target, fields, st=session.runtime) or () + if named & set(hidden): + raise err(403, hidden_code, "The link condition names a field you cannot read in the linked database.") +def _add_registry_link(session, table_key, body, definition): + """Store an admitted registry Link in the canonical shared overlay without changing source columns.""" + from core import shared_overlay + from modules import product_data + from routes_customers import _shared_key + + ut = _ut() + bucket = product_data.TABLE_KEY if table_key == 'product_data' else _shared_key() + have = list(definition.get('fields') or []) + list(shared_overlay.fields(bucket, st=session.runtime).values()) + try: + field = ut._clean_field(body) + if (not field or any(prior.get('key') == field['key'] for prior in have) + or not ut.dynamic_link_filter_valid(field, have)): + raise ValueError('The Link field needs a unique name and a valid linked database.') + field.update(custom=True, shared=True, createdBy=session.uname) + field = shared_overlay.put_field(bucket, field['key'], field, st=session.runtime) + except ValueError as exc: + raise err(400, getattr(exc, 'code', 'bad_field'), str(exc)) from exc + return _with_dropped(ut, {'ok': True, 'value': field, 'refused': False, 'reason': '', 'field': field}, body) + + @router.post("/tables/{table_key}/fields", status_code=201) @_c8_write def add_field(table_key: str, body: dict = Body(default=None), session: Session = Depends(require_session)): _managed_or_refuse(session, table_key, mutation=True) - definition = _field_or_refuse(session, table_key) + definition = _field_or_refuse(session, table_key, raw=body, create=True) _admit_if_sources(session, table_key, body, definition) - _admit_link_selection(session, body) ut = _ut() + registry_link = table_key in ut.LINKABLE_MODULE_KEYS + _admit_link_selection(session, body, hidden_code=( + 'link_condition_hidden' if registry_link else 'link_selection_forbidden')) + if registry_link: + return _add_registry_link(session, table_key, body, definition) from routes_grid import govern_date_recurrence # Only recurrence admission needs the merged workspace. Plain creates need no row audience. visible_fields = (ut_write_ctx(session, table_key)['fields'] @@ -3788,8 +4006,12 @@ def _refusal_sentence(ut, session, body, table_key="", fkey=""): if kind and kind not in ut.UT_FIELD_TYPES: return (f"{kind!r} is not a column type here (types: " f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") + limit = ut.field_limit(table_key, session.runtime) + if limit is None: + return (f"the column was refused. Check the name and type " + f"(types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") return (f"the column was refused. Check the name and type, or the table may be at its " - f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") + f"{limit}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") @router.patch("/tables/{table_key}/fields/{fkey}") @@ -4423,37 +4645,13 @@ def view_audit_trail(table_key: str, since: float = Query(default=0), @router.delete("/tables/{table_key}/views/{view_id}") def delete_table_view(table_key: str, view_id: str, request: Request, - rev: int = Query(default=None), body: dict = Body(default=None), session: Session = Depends(require_session)): - """Delete one saved View through the DIRECT door, guarded by the revision the caller read. - - ⭐⭐ W50-T25 — THIS ROUTE IS WHERE THE 409 BECOMES OBSERVABLE. C3's refusal is - `409 view_revision_stale {rev_current}`; the events channel cannot answer it yet because - `api/main.py` (lane F's file) is where a core exception becomes an HTTP status. Landing the - same refusal on a route this ticket owns means the shape is proven in this wave rather than - asserted, and the mapping the combine copies into `main.py` is a copy of something running. - - `rev` may arrive as a query parameter or in the body. ABSENT MEANS NO CHECK, and the delete - is accepted and logged with `rev: null` — which is what keeps the whole change safe before - W50-T28 wires the client: nothing on the wire sends a `rev` today, so nothing can be refused. - - ⚡ W50-T26 OWNS TRASH. When its `.trash[]` door lands, this route keeps its wall, - its guard and its audit row and gains the move; nothing here has to be unpicked for it. - """ - from core import view_audit - + """Delete an owned saved View and accept ignored legacy request metadata.""" _managed_or_refuse(session, table_key, mutation=True) vid = str(view_id or "").strip() if not vid: raise err(400, "view_required", "choose a view to delete") - payload = body if isinstance(body, dict) else {} - sent = rev if rev is not None else payload.get("rev") - try: - sent = None if sent is None else int(sent) - except (TypeError, ValueError): - raise err(400, "bad_revision", "the view revision must be a whole number") - ops = _ops(session, table_key) # ⛔ `prefer_owner` — W48C-T02's trap, and an audit row is exactly where it would hurt. # A bare first-hit scan returns whichever stratum the document yielded first, so an admin @@ -4465,17 +4663,7 @@ def delete_table_view(table_key: str, view_id: str, request: Request, owner = found[0] if owner != session.uname and not session.admin: raise err(403, "view_not_yours", "only the owner can delete this saved view") - try: - ops.delete_view(owner, vid, actor=session.uname, - origin=_audit_origin(request), rev=sent) - except view_audit.ViewRevisionStale as stale: - raise HTTPException(status_code=409, detail={"error": { - "code": "view_revision_stale", - "message": "someone else changed this view list while you had it open. " - "Reload and try again.", - "rev_current": stale.rev_current}}) - except view_audit.RevisionUnverifiable as unknown: - raise err(503, "view_revision_unverifiable", str(unknown)) + ops.delete_view(owner, vid, actor=session.uname, origin=_audit_origin(request)) from core import shares shares.drop_objects([('view', vid)], st=ops.st) return {"deleted": vid, "owner": owner} @@ -4557,32 +4745,20 @@ def list_view_trash(table_key: str, session: Session = Depends(require_session)) @router.post("/tables/{table_key}/views/{view_id}/restore") def restore_table_view(table_key: str, view_id: str, request: Request, - rev: int = Query(default=None), - body: dict = Body(default=None), + body: dict = Body(default=None), session: Session = Depends(require_session)): """Put one trashed view back, with every assignment its delete cascaded away. - Guarded by the same revision the caller read, refused with the same `409 view_revision_stale` - body as the delete beside it, and audited as C3's `restore` action. - ⛔ AN ID COLLISION IS A 409 THAT NAMES THE LIVE VIEW, never a silent re-key: a restored view under a fresh id would leave every grant, assignment and Sheet cell that names the old id pointing at something else, and it would look like it worked. """ from core import table_store as ts - from core import view_audit _managed_or_refuse(session, table_key, mutation=True) vid = str(view_id or "").strip() if not vid: raise err(400, "view_required", "choose a view to restore") - payload = body if isinstance(body, dict) else {} - sent = rev if rev is not None else payload.get("rev") - try: - sent = None if sent is None else int(sent) - except (TypeError, ValueError): - raise err(400, "bad_revision", "the view revision must be a whole number") - ops = _ops(session, table_key) # ⛔ AN UNREADABLE DOCUMENT IS A 503, NOT A 404. `find_trashed` deliberately does not swallow # its read (unlike the older `find_view` beside it), because answering `None` here would make @@ -4605,7 +4781,7 @@ def restore_table_view(table_key: str, view_id: str, request: Request, raise err(403, "view_not_yours", "only the owner can restore this saved view") try: landed = ops.restore_view(owner, vid, actor=session.uname, - origin=_audit_origin(request), rev=sent) + origin=_audit_origin(request)) except ts.ViewIdInUse as clash: raise HTTPException(status_code=409, detail={"error": { "code": "view_id_in_use", @@ -4614,14 +4790,6 @@ def restore_table_view(table_key: str, view_id: str, request: Request, "name": clash.name}}) except ts.ViewNotInTrash as missing: raise err(404, "view_not_in_trash", str(missing)) - except view_audit.ViewRevisionStale as stale: - raise HTTPException(status_code=409, detail={"error": { - "code": "view_revision_stale", - "message": "someone else changed this view list while you had it open. " - "Reload and try again.", - "rev_current": stale.rev_current}}) - except view_audit.RevisionUnverifiable as unknown: - raise err(503, "view_revision_unverifiable", str(unknown)) return {"restored": vid, "owner": owner, "name": landed.get("name")} @@ -4687,30 +4855,15 @@ def preview_view_purge(table_key: str, view_id: str, @router.delete("/tables/{table_key}/views/{view_id}/purge") def purge_table_view(table_key: str, view_id: str, - rev: int = Query(default=None), session: Session = Depends(require_session)): - """Destroy one trashed view now, after every cell scoped to it has taken its filter.""" - from core import view_audit - + """Destroy one trashed view after its dependent cells have taken its filter.""" ops, vid, owner, _entry = _trash_owner_or_refuse( session, table_key, view_id, mutation=True) - try: - sent = None if rev is None else int(rev) - except (TypeError, ValueError): - raise err(400, "bad_revision", "the view revision must be a whole number") from core import table_store as ts try: - name, adopted = ops.purge_view(owner, vid, rev=sent, actor=session.uname) + name, adopted = ops.purge_view(owner, vid, actor=session.uname) except ts.ViewNotInTrash as missing: raise err(404, "view_not_in_trash", str(missing)) - except view_audit.ViewRevisionStale as stale: - raise HTTPException(status_code=409, detail={"error": { - "code": "view_revision_stale", - "message": "someone else changed this view list while you had it open. " - "Reload and try again.", - "rev_current": stale.rev_current}}) - except view_audit.RevisionUnverifiable as unknown: - raise err(503, "view_revision_unverifiable", str(unknown)) except Exception as exc: # noqa: BLE001 # ⛔ THE ADOPTION FAILED, SO NOTHING WAS DESTROYED. `purge_view` copies the filter before # it touches the bucket, so this refusal leaves the view exactly where it was. diff --git a/api/routes_usage.py b/api/routes_usage.py index 011d849472b9241a77a7a7d3d0815c5852c0e056..afd19d8fb956ce5b77868b165aa8ea1fad626e26 100644 --- a/api/routes_usage.py +++ b/api/routes_usage.py @@ -12,15 +12,52 @@ reporting feature into a wall the owner explicitly did not ask for this wave. `?scope=tenant` folds every account and is refused for a non-admin — how much AI another employee used is not a fact this product hands out sideways. """ -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query import usage_ledger import turn_audit +import usage_activity from deps import Session, err, require_session router = APIRouter(prefix="/api/v1") +@router.get("/usage/activity/mcp") +def mcp_activity(from_: str = Query(default=None, alias="from"), + to: str = None, user: str = None, tool: str = None, + cursor: str = None, limit: int = 100, + session: Session = Depends(require_session)): + """Return complete MCP tool totals and one page of value-free event metadata.""" + if not session.admin: + raise err(403, "admin_required", "Usage activity is available to administrators only.") + try: + return usage_activity.read_mcp(session, from_=from_, to=to, user=user, + tool=tool, cursor=cursor, limit=limit) + except PermissionError: + raise err(403, "forbidden", "The usage runtime must belong to this workspace.") from None + except usage_activity.McpCursorInvalid: + raise err(400, "usage_mcp_cursor_invalid", "Reload MCP activity to continue.") from None + except usage_activity.McpCursorStale: + raise err(409, "usage_mcp_cursor_stale", "Retained MCP activity changed. Reload to continue.") from None + except ValueError as exc: + raise err(400, "bad_request", str(exc)) from None + + +@router.get("/usage/activity") +def activity(from_: str = Query(default=None, alias="from"), + to: str = None, user: str = None, kind: str = None, + session: Session = Depends(require_session)): + """Return retained activity counts for this tenant's administrators.""" + if not session.admin: + raise err(403, "admin_required", "Usage activity is available to administrators only.") + try: + return usage_activity.read(session, from_=from_, to=to, user=user, kind=kind) + except PermissionError: + raise err(403, "forbidden", "The usage runtime must belong to this workspace.") from None + except ValueError as exc: + raise err(400, "bad_request", str(exc)) from None + + @router.get("/usage/turns") def turns(scope: str = "me", week: str = None, offset: int = 0, limit: int = 50, session: Session = Depends(require_session)): diff --git a/api/sync_worker.py b/api/sync_worker.py index 184b55e7cb9d7af42ef9575c3ceddd593152fe38..d61b51443246cc0cc7933a91d99d46058ac4dc7f 100644 --- a/api/sync_worker.py +++ b/api/sync_worker.py @@ -7,6 +7,7 @@ It keeps its own DuckDB files, so it never opens the web process's royal.duckdb. Its heavy jobs wait until it holds the store's sync writer lease, so two Spaces never both sync. It loads the app as `api.main`, the name the web uses, so the process holds one main module (D-781). """ +import faulthandler import importlib import os import sys @@ -115,6 +116,8 @@ def boot_app_migrations(app_main): def main(argv=None): + # Enable before app imports so native failures include every Python thread's stack. + faulthandler.enable(all_threads=True) import sync_runtime args = sys.argv[1:] if argv is None else argv mode = next((sync_runtime.JOB_FLAGS[flag] for flag in args if flag in sync_runtime.JOB_FLAGS), None) diff --git a/api/test_odoo_rank_refresh.py b/api/test_odoo_rank_refresh.py index 95b44a7af73aa230b4e7ca850c7ff60af43cc06a..7490a1d1bf1c0724528d691f398422f1decc3a81 100644 --- a/api/test_odoo_rank_refresh.py +++ b/api/test_odoo_rank_refresh.py @@ -232,7 +232,144 @@ def _with_collaborators(source, nodes, ns): FREEZE_OFF = SimpleNamespace(enabled=lambda: False) +def observe_s85_t113_growing_source(*, capture=True, mirror_rows=7706, hook=True, + prior_counts=None): + """Reconcile a synthetic pull after Amazon takes 45 minutes and Odoo grows by 69 rows.""" + from contextlib import nullcontext + from datetime import datetime, timedelta, timezone + import _fake_store + import automation_engine + import connector_onboarding + import core.odoo as odoo + import core.user_tables as ut + import odoo_relational as rel + import rollup_sql + import routes_connectors as connectors + import routes_odoo_tables + from harness import runtime + + key = rel.STOCK_QUANT_KEY + source = {'count': mirror_rows, 'seconds': 0, 'calls': []} + + class GrowingSource: + def search_count(self, model, domain): + assert model == 'stock.quant' + source['calls'].append((source['seconds'], copy.deepcopy(domain), source['count'])) + # A count of all locations must fail the internal-stock reconciliation. + return source['count'] if domain == [('location_id.usage', '=', 'internal')] else 300000 + + def __getattr__(self, method): + raise AssertionError('unexpected source method: ' + method) + + rt = _fake_store.FakeStore() + rt.key = rt.tenant = 's85_t113_fixture' + rt.odoo_source = lambda: SimpleNamespace(_bound=nullcontext) + rt.assert_datastore_matches = lambda: None + old = (datetime.now(timezone.utc) - timedelta(days=2)).isoformat() + rt.put(rel.CONFIG_KEY, {'grids': {key: True}, 'lastGoodSyncAt': old}) + + def sync_all(*, after_entity=None): + if after_entity: + after_entity('stock_quant') + + def plain_sync_all(): + return None + + fixture_ds = SimpleNamespace(sync_all=sync_all if hook else plain_sync_all, + _resident_odoo_runtime=lambda: rt, ENTITIES={'stock_quant': {'model': 'stock.quant'}}) + tree = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8')) + wanted = {'_entity_source_counter', '_capture_resync_source_counts', '_rebuild_odoo_relational'} + nodes = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in wanted] + namespace = {'routes_connectors': connectors, '_resync_tenant': lambda: (rt.key, 'fixture'), + '_rebuild_meta_relational': lambda *a, **k: {}, + '_rebuild_amazon_relational': lambda *a, **k: {}, + '_PASS_SOURCE_COUNTS': {rt.key: dict(prior_counts or {})}} + exec(compile(ast.Module(body=nodes, type_ignores=[]), 'main.py:S85-T113', 'exec'), namespace) + + def plan(*_a, **_k): + return {'stock_quant': [{'_id': str(i)} for i in range(1, mirror_rows + 1)]} + + def amazon(why): + assert why == 'resync' + source['seconds'] += 45 * 60 + source['count'] += 69 + + with ExitStack() as stack, redirect_stdout(io.StringIO()) as output: + stack.enter_context(patch.object(socket.socket, 'connect', side_effect=AssertionError('no network'))) + stack.enter_context(_fake_store.installed(rt)) + stack.enter_context(patch.object(runtime, 'get_runtime', return_value=rt)) + stack.enter_context(patch.object(odoo, 'get_odoo', return_value=GrowingSource())) + stack.enter_context(patch.object(rel, 'default_connection_id', return_value=None)) + stack.enter_context(patch.object(rel, 'available', return_value=True)) + stack.enter_context(patch.object(rel, 'enabled_buckets', return_value={'stock_quant'})) + stack.enter_context(patch.object(rel, 'plan', side_effect=plan)) + stack.enter_context(patch.object(rel, 'apply_plan', side_effect=lambda *_a, **_k: {key: {'rows': 0}})) + stack.enter_context(patch.object(rel, '_canonical_mirror_ready', return_value=False)) + stack.enter_context(patch.object(rel, 'sync_discovered_models', return_value={})) + stack.enter_context(patch.object(rel, '_refresh_declared_schema', return_value={})) + stack.enter_context(patch.object(rel, 'authoritative_schema_snapshot', return_value=({}, []))) + stack.enter_context(patch.object(ut, 'register_connected')) + stack.enter_context(patch.object(ut, 'record_odoo_schema_snapshot')) + stack.enter_context(patch.object(connector_onboarding, 'mirror_cursor', return_value=object())) + stack.enter_context(patch.object(routes_odoo_tables, 'population', return_value=mirror_rows)) + stack.enter_context(patch.object(automation_engine, 'refresh_relations')) + stack.enter_context(patch.object(rollup_sql, 'compute')) + real_refresh, reports = rel.refresh, [] + + def refresh(*args, **kwargs): + reports.append((kwargs, real_refresh(*args, **kwargs))) + return reports[-1][1] + + stack.enter_context(patch.object(rel, 'refresh', side_effect=refresh)) + after_entity = namespace['_entity_source_counter'](fixture_ds) if capture else None + fixture_ds.sync_all(after_entity=after_entity) if hook else fixture_ds.sync_all() + counts = namespace['_capture_resync_source_counts'](rt.key) if capture else None + amazon('resync') + ok = namespace['_rebuild_odoo_relational']('resync', None, {}, tenant=rt.key, + cadence_seconds=86400, source_counts=counts) + kwargs, report = reports[-1] if reports else ({}, {}) + failures = rel.blocking_failures( + connectors._odoo_result_database_failures(rt, None, rel.health_view(report))[0], report) + freshness = rel.sync_freshness(rt) + return SimpleNamespace(counts=counts, ok=ok, freshness=freshness, calls=source['calls'], rt=rt, + refresh_kwargs=kwargs, failures=failures, receipt=(report.get('databases') or {}).get(key) or {}, + seconds=source['seconds'], source_count=source['count'], log=output.getvalue(), + remaining_counts=namespace['_PASS_SOURCE_COUNTS']) + + class RankRefreshTests(unittest.TestCase): + def test_s85_t113_freshness_uses_the_pull_count_after_a_growing_source(self): + import odoo_relational as rel + fixed = observe_s85_t113_growing_source() + control = observe_s85_t113_growing_source(capture=False) + self.assertEqual(fixed.seconds, 2700) + self.assertEqual(fixed.source_count, 7775) + self.assertEqual(fixed.calls, [(0, [('location_id.usage', '=', 'internal')], 7706)]) + self.assertEqual(fixed.counts, {'stock_quant': (7706, 'search_count')}) + self.assertEqual(fixed.remaining_counts, {}) + self.assertEqual(fixed.refresh_kwargs['source_counts'], fixed.counts) + self.assertEqual((fixed.receipt['sourceReconciliation'], fixed.receipt['sourceCount'], + fixed.receipt['mirrorCount'], fixed.receipt['sourceCountBasis']), + ('matched', 7706, 7706, 'mirror_pull')) + self.assertTrue(fixed.ok) + self.assertFalse(fixed.freshness['stale']) + self.assertEqual(fixed.rt.get(rel._ut().ODOO_SYNC_STATUS_KEY)['phase'], 'ok') + self.assertEqual(control.calls, [(2700, [('location_id.usage', '=', 'internal')], 7775)]) + self.assertEqual((control.receipt['sourceReconciliation'], control.receipt['sourceCountBasis']), + ('mismatch', 'reconciliation')) + self.assertFalse(control.ok) + self.assertEqual(control.rt.get(rel._ut().ODOO_SYNC_STATUS_KEY)['phase'], 'error') + self.assertTrue(control.freshness['stale']) + self.assertEqual(control.failures[rel.STOCK_QUANT_KEY]['code'], 'reconciliation_incomplete') + + def test_s85_t113_freshness_never_reuses_an_unfinished_pass_count_without_a_hook(self): + observed = observe_s85_t113_growing_source(hook=False, + prior_counts={'stock_quant': (1, 'search_count')}) + self.assertTrue(observed.ok) + self.assertEqual(observed.counts, {'stock_quant': (7706, 'search_count')}) + self.assertEqual(observed.calls, [(0, [('location_id.usage', '=', 'internal')], 7706)]) + self.assertEqual(observed.remaining_counts, {}) + def setUp(self): self.stack = ExitStack() self.addCleanup(self.stack.close) @@ -921,7 +1058,8 @@ class RankRefreshTests(unittest.TestCase): return datetime.fromisoformat(text.replace('Z', '+00:00')).timestamp() def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None, - reports=None, checkpoints=None, take=None, slot_flags=None, status=None): + reports=None, checkpoints=None, take=None, slot_flags=None, status=None, + stop_at=None): """Drive the real loop and its real slot resolver on a clock that advances at each wait.""" import odoo_relational as rel source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8')) @@ -939,8 +1077,9 @@ class RankRefreshTests(unittest.TestCase): state = {} def wait(seconds): + self.assertLessEqual(seconds, 60.0, 'the Amazon check tick must remain at most 60 seconds') sleeps.append(seconds) - if len(sleeps) >= max_sleeps: + if len(sleeps) >= max_sleeps or stop_at is not None and now[0] + seconds >= stop_at: raise StopIteration('bounded synthetic resident loop') now[0] += seconds return False @@ -1010,22 +1149,21 @@ class RankRefreshTests(unittest.TestCase): last = self.utc('2026-10-23T06:05:00Z') events, sleeps, odoo = self.run_slot_loop( {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}, - boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=3) + boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=4000, + stop_at=self.utc('2026-10-25T06:00:01Z')) slot, next_slot = self.utc('2026-10-24T06:00:00Z'), self.utc('2026-10-25T06:00:00Z') boot_work = [e for e in events if e[1] < slot] # Restore only at boot: the saved Amazon mirror, no mirror pass, no count, no store half. self.assertEqual(boot_work, [('amazon', self.utc('2026-10-24T02:30:00Z'), 'boot', False)]) self.assertIn('2026-10-24T06:00:00Z (13:00 Asia/Jakarta)', self.loop_output) - # The first Odoo pass fires once, at the slot plus at most one second. - self.assertEqual(sleeps[0], slot + 1 - self.utc('2026-10-24T02:30:00Z')) - self.assertEqual([e[1] for e in odoo], [slot + 1, slot + 1, next_slot + 1, next_slot + 1]) - self.assertTrue(0 < odoo[0][1] - slot <= 1) - # No second Odoo pass before the next day's slot: the wait after the pass targets it. - self.assertEqual(sleeps[1], 86400) + # Every real 60-second check runs before and between the two daily Odoo passes. + self.assertEqual(set(sleeps), {60.0}) + self.assertGreater(len(sleeps), 1440) + self.assertEqual([e[1] for e in odoo], [slot, slot, next_slot, next_slot]) # The slot is stamped BEFORE the mirror pass reads Odoo, and the status names the next one. import odoo_relational as rel stamp = self.slot_store.docs[rel.CONFIG_KEY][rel.RESYNC_MEMORY_FIELD]['slotPassAt'] - self.assertEqual(rel._memory_epoch(stamp), next_slot + 1) + self.assertEqual(rel._memory_epoch(stamp), next_slot) self.assertEqual(self.slot_state['nextPassAt'], '2026-10-26T06:00:00Z') def test_h73_a_missed_slot_runs_once_at_boot_then_at_the_next_slot(self): @@ -1034,11 +1172,11 @@ class RankRefreshTests(unittest.TestCase): boot = self.utc('2026-10-24T09:00:00Z') events, sleeps, odoo = self.run_slot_loop( {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}, - boot=boot, max_sleeps=2) + boot=boot, max_sleeps=4000, stop_at=self.utc('2026-10-25T06:00:01Z')) next_slot = self.utc('2026-10-25T06:00:00Z') mirror = [e[1] for e in odoo if e[0] == 'mirror'] - self.assertEqual(mirror, [boot, next_slot + 1]) - self.assertEqual(sleeps, [next_slot + 1 - boot, 86400]) + self.assertEqual(mirror, [boot, next_slot]) + self.assertEqual(set(sleeps), {60.0}) self.assertNotIn(('amazon', boot, 'boot', False), events) def test_h73_a_restart_inside_a_slot_pass_does_not_run_it_twice(self): @@ -1046,7 +1184,6 @@ class RankRefreshTests(unittest.TestCase): import odoo_relational as rel stamp = self.utc('2026-10-24T06:00:01Z') yesterday = self.utc('2026-10-23T06:30:00Z') - to_next_slot = self.utc('2026-10-25T06:00:01Z') - self.utc('2026-10-24T06:40:00Z') base = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': stamp, 'fingerprints': {'res_partner': (3, 7)}} # Ended: `slotDoneAt` after the claim, or (older memory) `passAt` after it. No Odoo call. @@ -1054,7 +1191,7 @@ class RankRefreshTests(unittest.TestCase): {'passAt': self.utc('2026-10-24T06:30:00Z')}): events, sleeps, odoo = self.run_slot_loop( dict(base, **ended), boot=self.utc('2026-10-24T06:40:00Z'), max_sleeps=1) - self.assertEqual((odoo, sleeps), ([], [to_next_slot])) + self.assertEqual((odoo, sleeps), ([], [60.0])) # Killed: the end marker predates the claim. One pass at boot, stamped start and end. boot = self.utc('2026-10-24T06:40:00Z') events, sleeps, odoo = self.run_slot_loop( @@ -1080,7 +1217,7 @@ class RankRefreshTests(unittest.TestCase): events, sleeps, odoo = self.run_slot_loop( dict(base, slotPassAt=boot, slotDoneAt=yesterday, slotResumedFor=slot_at, slotResumes=3), boot=fourth, max_sleeps=1) - self.assertEqual((odoo, sleeps), ([], [self.utc('2026-10-25T06:00:01Z') - fourth])) + self.assertEqual((odoo, sleeps), ([], [60.0])) self.assertNotIn('never finished', self.loop_output) self.assertIn('slot_pass_interrupted: the 13:00 Asia/Jakarta slot pass was interrupted by ' '4 restarts and waits for the next slot at 2026-10-25T06:00:00Z', @@ -1110,7 +1247,7 @@ class RankRefreshTests(unittest.TestCase): self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot]) self.assertIn('the 13:00 Asia/Jakarta slot pass started at 2026-10-27T06:00:01Z never ' 'finished; running it again now (resume 1 of 3)', self.loop_output) - self.assertEqual(sleeps, [self.utc('2026-10-28T06:00:01Z') - boot]) + self.assertEqual(sleeps, [60.0]) def test_h1_slot_resumes_are_bounded_and_reset_by_a_new_slot(self): """H1: a killed slot pass resumes while its count is under 3, then waits; a new slot starts over""" @@ -1145,7 +1282,7 @@ class RankRefreshTests(unittest.TestCase): last = self.utc('2026-10-23T06:05:00Z') events, _sleeps, _odoo = self.run_slot_loop( {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}, - boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=2) + boot=self.utc('2026-10-24T05:59:30Z'), max_sleeps=2) slot = self.utc('2026-10-24T06:00:00Z') kinds = [e[0] for e in events if e[1] > slot] amazon = [i for i, e in enumerate(e for e in events if e[1] > slot) @@ -1164,7 +1301,7 @@ class RankRefreshTests(unittest.TestCase): {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}, boot=boot, max_sleeps=1, tenant='acme') self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot]) - self.assertEqual(sleeps, [86400]) + self.assertEqual(sleeps, [60.0]) self.assertNotIn('nextPassAt', self.slot_state) def test_h73_manual_tenant0_has_no_slot(self): @@ -1197,7 +1334,8 @@ class RankRefreshTests(unittest.TestCase): take = lambda **kwargs: held.append(tuple(kwargs.get('exclude_requested_by') or ())) or [] _events, _sleeps, odoo = self.run_slot_loop( {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}, - boot=self.utc('2026-10-24T04:30:00Z'), max_sleeps=2, take=take) + boot=self.utc('2026-10-24T04:30:00Z'), max_sleeps=100, + stop_at=self.utc('2026-10-24T06:00:01Z'), take=take) self.assertTrue(odoo, 'the slot pass ran') # Boot 90 min before the slot: the drain outside the slot holds first load, and so does the claim. self.assertTrue(len(held) >= 2 and all(h == ('first load',) for h in held), held) @@ -1223,21 +1361,23 @@ class RankRefreshTests(unittest.TestCase): self.assertTrue(held and all('first load' in exclusions for exclusions in held)) self.assertEqual(job, {'state': 'running', 'rowsDone': 10}) - def test_first_load_runs_far_from_the_slot(self): - """A first-load continuation keeps running when the next slot is more than two hours away""" + def test_first_load_stays_with_its_actor_far_from_the_slot(self): + """The independent actor retains first-load work while the resident loop polls its desk.""" last = self.utc('2026-10-23T06:05:00Z') held = [] - take = lambda **kwargs: held.append(tuple(kwargs.get('exclude_requested_by') or ())) or [] + take = lambda **kwargs: held.append(kwargs) or [] self.run_slot_loop({'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}, boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=1, take=take) - self.assertEqual(held[0], ()) + self.assertTrue(held) + self.assertTrue(all('amazon_sp' in call['exclude_providers'] for call in held)) + self.assertTrue(all(call['exclude_requested_by'] == ('first load',) for call in held)) - def test_the_slot_pass_refreshes_every_amazon_table(self): - """Live 10-29: orders pulled at 16:20Z by a late pass were deferred by the next day's 06:00Z slot""" + def test_the_slot_pass_preserves_each_amazon_tables_cadence(self): + """A slot pass retains Amazon's per-table cadence and the resident slot flag.""" last = self.utc('2026-10-23T06:05:00Z') flags = [] self.run_slot_loop({'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}, - boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=2, slot_flags=flags) + boot=self.utc('2026-10-24T05:59:30Z'), max_sleeps=2, slot_flags=flags) resync = [flag for args, flag in flags if args[:1] == ('resync',)] self.assertEqual(resync, [True]) self.assertTrue(all(not flag for args, flag in flags if args[:1] != ('resync',)), flags) @@ -1245,8 +1385,13 @@ class RankRefreshTests(unittest.TestCase): from datetime import datetime, timezone now = datetime(2026, 10, 29, 10, 16, tzinfo=timezone.utc) ticks = {'orders': {'state': 'done', 'updated_at': '2026-10-28T18:20:03+00:00'}} - self.assertIn('orders', az.sync_plan(ticks, ['orders'], now=now)['deferred']) - self.assertEqual(az.sync_plan(ticks, ['orders'], now=now, slot_pass=True)['tables'], ['orders']) + due = az.sync_plan(ticks, ['orders'], now=now) + self.assertEqual(due['tables'], ['orders']) + self.assertEqual(az.sync_plan(ticks, ['orders'], now=now, slot_pass=True), due) + recent = {'orders': {'state': 'done', 'updated_at': now.isoformat()}} + deferred = az.sync_plan(recent, ['orders'], now=now) + self.assertIn('orders', deferred['deferred']) + self.assertEqual(az.sync_plan(recent, ['orders'], now=now, slot_pass=True), deferred) def test_slot_near_and_late(self): """The quiet window remains two hours; the slot clock alone cannot make a pass late.""" @@ -1296,7 +1441,6 @@ class RankRefreshTests(unittest.TestCase): import odoo_relational as rel last = self.utc('2026-10-22T06:05:00Z') boot = self.utc('2026-10-24T07:00:00Z') - next_slot = self.utc('2026-10-25T06:00:00Z') memory = {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}} held = {'account_move_line': {'phase': 'backfill', 'pulled': 0}} marker = 'account_move_line.cols.amount_residual' @@ -1307,7 +1451,7 @@ class RankRefreshTests(unittest.TestCase): memory, boot=boot, max_sleeps=2, reports=[held, held], checkpoints=[before, after, after, after]) self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot, boot + 60]) - self.assertEqual(sleeps, [60.0, next_slot + 1 - (boot + 60)]) + self.assertEqual(sleeps, [60.0, 60.0]) self.assertEqual([e for e in events if e[0] in ('meta', 'amazon', 'rebuild')], []) stamps = rel.read_resync_memory(self.slot_store) self.assertEqual((stamps['slotPassAt'], stamps['slotDoneAt']), (boot, boot)) @@ -1318,7 +1462,7 @@ class RankRefreshTests(unittest.TestCase): events, sleeps, odoo = self.run_slot_loop( memory, boot=boot, max_sleeps=1, reports=[held], checkpoints=[before, before]) self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot]) - self.assertEqual(sleeps, [next_slot + 1 - boot]) + self.assertEqual(sleeps, [60.0]) self.assertNotIn('odoo backfill continues', self.loop_output) # ── Hotfix 10-31: a good pass after the slot satisfies it, so a killed claim does not alarm ── @@ -1373,7 +1517,7 @@ class RankRefreshTests(unittest.TestCase): memory, boot=boot, max_sleeps=1, config=self.stored_config(memory)) # The loop reached the slot block: it named the next pass and waits for that slot. self.assertEqual(self.slot_state['nextPassAt'], '2026-11-01T06:00:00Z') - self.assertEqual(sleeps, [next_slot + 1 - boot]) + self.assertEqual(sleeps, [60.0]) self.assertEqual(odoo, []) self.assertNotIn('never finished', self.loop_output) self.assertNotIn('slot_pass_interrupted', self.loop_output) diff --git a/api/tool_groups/__init__.py b/api/tool_groups/__init__.py index 181379ed743cbc47ba6086c4af612e7b57970b28..29002182eca3595d133f3e834bb0266460419aa7 100644 --- a/api/tool_groups/__init__.py +++ b/api/tool_groups/__init__.py @@ -197,7 +197,10 @@ def resolve(name, arguments): }, "required": ["action"], "additionalProperties": False, }) args = arguments.get("arguments", {}) - external_tools.validate_arguments(args, entry["schema"]) + if entry.get("legacy_name") in {"query_database", "list_databases"}: + args = external_tools.table_read_arguments(entry["legacy_name"], args, entry["schema"]) + else: + external_tools.validate_arguments(args, entry["schema"]) except sheets.SheetRefusal as exc: if exc.code != "tool_arguments_invalid" or entry is None: raise diff --git a/api/tool_groups/data.py b/api/tool_groups/data.py index b39066e8082d60bace63da3aaa2714255b159c3b..ec0262cec1a373d1f854a55c31af07aa875176d2 100644 --- a/api/tool_groups/data.py +++ b/api/tool_groups/data.py @@ -45,11 +45,18 @@ READ_ARGUMENT_HELP = ( #: The one skill.md section every query refusal names. QUERY_SYNTAX_SECTION = "Query syntax" QUERY_FILTER_EXAMPLE = {"conj": "and", "nodes": [{"colId": "", "op": "eq", "value": ""}]} +QUERY_GROUPING_HELP = ( + "Connector mirrors group up to four fields and support month units. Their groups arrive in key order. " + "A stored Table groups one field without a month unit, using the Table grid's aggregate reader. " + "Its group order is count descending, then key ascending. " + "A query_groups_unavailable refusal names its cause. Call table with action query_database " + "using error.retry_query, without groupBy or aggregates, then follow page.next_cursor." +) QUERY_SYNTAX_FIX = ( "Filters use {conj, nodes: [{colId, op, value}]}, for example " + json.dumps(QUERY_FILTER_EXAMPLE, separators=(", ", ": ")) + ". " "groupBy is a list of 1 to 4 field keys, or {field, unit: month}, with at least one aggregate and no sorts. " - "Only a connected database groups. Any other database answers query_groups_unavailable. " + + QUERY_GROUPING_HELP + " " "See " + QUERY_SYNTAX_SECTION + " in skill.md." ) #: Query refusal codes that carry QUERY_SYNTAX_FIX. @@ -115,9 +122,7 @@ def query_syntax_section(): "- aggregates: [{fn, field}], fn count, count_distinct, sum, avg, min or max, at most 16. " "count may omit field. sum and avg need a numeric field (query_aggregate_type otherwise).", "- groupBy: 1 to 4 field keys, or {field, unit: month} for a date field. A grouped query needs at least " - "one aggregate and no sorts (query_groups_invalid otherwise). Only a connected database, such as an " - "Odoo sales lines mirror, groups. Any other database answers 409 query_groups_unavailable: read its " - "records by page, or ask for aggregates without groupBy.", + "one aggregate and no sorts (query_groups_invalid otherwise). " + QUERY_GROUPING_HELP, "- Paging: default 100. " + QUERY_PAGING_HELP + " Only fields, filters, sorts, limit, cursor, aggregates and groupBy are " "accepted (query_arguments_invalid otherwise).", @@ -158,8 +163,8 @@ def query_guide(rows): "Only use operators declared for that field. Combine date bounds in an AND group's nodes.", "", "## Grouping and paging", "", "Use groupBy with 1 to 4 readable field keys. For calendar months, use an object with " - "field set to a readable date field's key and unit set to month. Grouping needs a connected database, " - "at least one aggregate and no sorts. Groups arrive in key order.", + "field set to a readable date field's key and unit set to month. " + "Grouping needs at least one aggregate and no sorts. " + QUERY_GROUPING_HELP, "The top-level aggregates cover the whole permitted, filtered result, never one page. " "Each groups entry has keys and aggregates for that entire group. " "page.returned counts groups on the current page. records is empty for grouped queries. " diff --git a/api/tool_groups/metrics.py b/api/tool_groups/metrics.py index 5329a30fd8f4fa37cb82573c2a929ec406e532de..806137ad621152d05448c9679149e1250fc115f3 100644 --- a/api/tool_groups/metrics.py +++ b/api/tool_groups/metrics.py @@ -64,8 +64,12 @@ class _LimitRefused(_Unavailable): LAST_VALIDATED_KEY = "metric_last_validated" #: The newest entries kept. An evicted key is refused until it validates again (rule 8). LAST_VALIDATED_ENTRIES = 2000 -_UNAVAILABLE = "This metric has no validated value within your current access and window." -_LIMITED = "This window has no validated value yet, and the live call limit is reached." +_UNAVAILABLE = ("This metric has no validated value within your current access and window. " + "Ask an Administrator to check source availability and metric access, or choose another window.") +_LIMITED = ("This window has no validated value yet, and the live call limit is reached. " + "Retry after the live call limit resets.") +_WINDOW_INVALID = ("The dates do not form a valid named window. Use real ISO dates in order, " + "and a complete calendar month for calendar_month.") _WARNINGS_LIMITED = "Warnings were not checked because the live call limit is reached." @@ -186,7 +190,7 @@ def _window(window): or end.day != calendar.monthrange(start.year, start.month)[1]): raise ValueError except ValueError: - raise deps.err(400, "metric_window_invalid", "Use valid dates for the named window.") from None + raise deps.err(400, "metric_window_invalid", _WINDOW_INVALID) from None return copy.deepcopy(window) diff --git a/api/usage_activity.py b/api/usage_activity.py new file mode 100644 index 0000000000000000000000000000000000000000..55c907801bd011dc4b9f94441895bcbeb4a3e095 --- /dev/null +++ b/api/usage_activity.py @@ -0,0 +1,491 @@ +"""Join tenant activity counts without changing authored state or hiding source failures.""" + +from calendar import monthrange +from collections import Counter +from datetime import date, datetime, time, timedelta, timezone +import base64 +import binascii +import hashlib +import hmac +import json +import math + +from core import activity_log, audit, history_rows, user_tables, view_audit +import turn_audit + +KINDS = ("mcp", "api", "sign_in", "record_edit", "field_edit", "cell_edit", "view_edit", "ai_turn") +SOURCE_KINDS = { + "security_events": ("mcp", "api", "sign_in"), + "activity_events": ("record_edit", "field_edit"), + "value_history": ("cell_edit",), + "view_audit": ("view_edit",), + "turn_audit": ("ai_turn",), +} +PAGE = 200 +UTC = timezone.utc + + +def _now(): + return datetime.now(UTC) + + +def _timestamp(value): + if isinstance(value, (float, int)) and not isinstance(value, bool) and math.isfinite(value): + return datetime.fromtimestamp(value, UTC) + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if not isinstance(value, datetime) or value.tzinfo is None: + raise ValueError("Activity timestamps must include a timezone.") + return value.astimezone(UTC) + + +def _months_before(day, months): + year, month0 = divmod(day.year * 12 + day.month - 1 - months, 12) + month = month0 + 1 + return day.replace(year=year, month=month, day=min(day.day, monthrange(year, month)[1])) + + +def date_range(from_=None, to=None, *, now=None): + """Return an inclusive UTC date range within the last thirteen calendar months.""" + now = _timestamp(now) if now is not None else _now() + try: + end = date.fromisoformat(to) if to else now.date() + start = date.fromisoformat(from_) if from_ else end - timedelta(days=29) + except (TypeError, ValueError): + raise ValueError("Use dates in YYYY-MM-DD format.") from None + if start > end: + raise ValueError("From must be on or before To.") + if start < _months_before(now.date(), 13) or end > now.date(): + raise ValueError("Choose dates within the last 13 months.") + return start, end, now + + +def _unlimited_prefix(reader): + """Expand readers without a cursor until a short result proves exhaustion.""" + limit = PAGE + while True: + rows = reader(limit) + if len(rows) < limit: + return rows + limit *= 2 + + +def _tenant_row(row, tenant): + # Schema-bound sources can omit a stamp, but an explicit foreign stamp is never admitted. + return row.get("tenant", tenant) == tenant + + +def _security_record_mode(runtime): + """Use the canonical read-only layout decision for bound PostgreSQL security events.""" + from core import account_rows + backend = account_rows.records_backend(runtime) + if not backend: + return False + pg, slug = backend + with account_rows.read_snapshot(slug) as con: + return account_rows._record_mode(con, pg, slug, account_rows.EVENTS_KEY) + + +def _security(session, start, stop, now, source): + rows = _unlimited_prefix(lambda limit: audit.read( + session.tenant, limit=limit, now=now.timestamp(), + kinds={"external.query", "external.write", "auth.login"}, viewer=session.user)) + source["retention"] = {"days": 400, "legacyRowsPerPartition": audit.MAX_EVENTS} + if not _security_record_mode(session.runtime): + stored = (session.runtime.get(audit.BUCKET) or {}).get("events", []) + for external in (True, False): + partition = [row for row in stored if _tenant_row(row, session.tenant) + and (row.get("kind") in audit.EXTERNAL_KINDS) == external] + if len(partition) >= audit.MAX_EVENTS: + source["limitations"].append({"code": "legacy_ring", "cause": + "Legacy security documents retain at most 2000 events per partition."}) + held = min(_timestamp(row["ts"]) for row in partition).isoformat() + source["heldFrom"] = max(source["heldFrom"], held) + for row in rows: + if not _tenant_row(row, session.tenant): + continue + kind = "sign_in" if row["kind"] == "auth.login" else ( + "mcp" if row.get("detail", {}).get("via") == "mcp" else "api") + yield _timestamp(row["ts"]), row.get("principal") or "", kind + + +def _activity(session, start, stop, now, source): + cursor = None + source["retention"] = {"months": activity_log.RETENTION_MONTHS} + with activity_log.read_scope(session.runtime): + while True: + page = activity_log.read(session.runtime, can_read_object=lambda obj: True, + since=start, until=min(stop, now), cursor=cursor, limit=PAGE) + for row in page["records"]: + if _tenant_row(row, session.tenant) and row["kind"] in {"record", "field"}: + yield _timestamp(row["occurred_at"]), row.get("by") or "", row["kind"] + "_edit" + next_cursor = page["next_cursor"] + if next_cursor is None: + break + if next_cursor == cursor: + raise RuntimeError("Activity cursor did not advance.") + cursor = next_cursor + + +def _history_tables(rt, now): + """Include retained history for removed tables without reading their former record values.""" + tables = set(user_tables.all_defs(rt, schema_only=True)) + backend = history_rows.backend(rt) + since = now - timedelta(days=history_rows.RETENTION_DAYS) + if isinstance(backend, history_rows.PgHistoryStore): + try: + with backend._connection() as con: + cursor = con.execute(f'SELECT DISTINCT "table" FROM {backend.table} WHERE at >= %s', (since,)) + while rows := cursor.fetchmany(PAGE): + tables.update(row[0] for row in rows) + except Exception as exc: + if getattr(exc, "sqlstate", None) != "42P01": + raise + elif callable(getattr(backend, "table_keys", None)): + tables.update(backend.table_keys(since=since)) + return sorted(tables) + + +def _cells(session, start, stop, now, source): + source["retention"] = {"days": history_rows.RETENTION_DAYS} + for table in _history_tables(session.runtime, now): + if table == activity_log.DATABASE_KEY: + continue + history = user_tables.value_history(table, st=session.runtime, + retention_days=history_rows.RETENTION_DAYS, extended_entitled=True) + for row in history["events"]: + if _tenant_row(row, session.tenant) and row.get("kind") in {"edit", "restore"}: + for _ in row.get("changes", {}): + yield _timestamp(row["at"]), row.get("actor") or "", "cell_edit" + + +def _views(session, start, stop, now, source): + source["retention"] = {"days": None, "documentRowsPerTable": view_audit.RING_CAP} + from core.view_templates import _WS_KEYS + keys = set(_WS_KEYS.values()) | {key for key in session.runtime.list_keys("ut_") + if key.endswith("_table_workspace")} + for table in sorted(keys): + home = view_audit.home(session.runtime, table) + data = None + if home == view_audit.DOCUMENT: + # Read outside rows(), whose legacy document fallback swallows store failures. + data = session.runtime.get(table) or {} + ring = data.get(view_audit.AUDIT_KEY) or [] + if len(ring) >= view_audit.RING_CAP: + held = min(_timestamp(row["ts"]) for row in ring).isoformat() + source["heldFrom"] = max(source["heldFrom"], held) + source["limitations"].append({"code": "view_ring", "cause": + "A view document retains only its newest 2000 audit rows."}) + rows = _unlimited_prefix(lambda limit: view_audit.rows(session.runtime, + table_key=table, since=math.nextafter(start.timestamp(), -math.inf), limit=limit, data=data)) + for row in rows: + if _tenant_row(row, session.tenant): + yield _timestamp(row["ts"]), row.get("actor") or "", "view_edit" + + +def _turns(session, start, stop, now, source): + source["retention"] = {"days": None, "linesPerWeek": turn_audit.MAX_LINES, + "bytesPerWeek": turn_audit.MAX_BYTES} + monday = start.date() - timedelta(days=start.weekday()) + while monday <= min(stop, now).date(): + year, week, _ = monday.isocalendar() + week = f"{year}-W{week:02d}" + offset = 0 + capped, oldest = None, None + while True: + page = turn_audit.read(session.runtime, session.uname, tenant_wide=True, + week=week, offset=offset, limit=PAGE) + if offset == 0 and page["overflow"]: + capped = {"code": "turn_week_cap", "week": week, + "lost": page["overflow"], "limit": turn_audit.MAX_LINES, + "cause": "Weekly turn documents evict oldest rows above the line or byte limit."} + source["limitations"].append(capped) + for row in page["lines"]: + if _tenant_row(row, session.tenant): + stamp = _timestamp(row["ts"]) + oldest = min(oldest, stamp) if oldest is not None else stamp + yield stamp, row.get("user") or "", "ai_turn" + if not page["hasMore"]: + break + if not page["lines"]: + raise RuntimeError("Turn cursor did not advance.") + offset += len(page["lines"]) + if capped is not None: + held = oldest or datetime.combine(monday + timedelta(days=7), time.min, UTC) + capped["heldFrom"] = held.isoformat() + source["heldFrom"] = max(source["heldFrom"], held.isoformat()) + monday += timedelta(days=7) + + +READERS = {"security_events": _security, "activity_events": _activity, + "value_history": _cells, "view_audit": _views, "turn_audit": _turns} + + +def _source_error(exc): + # Exception text can hold database addresses, credentials or customer values. + from core.tenant_keys import TenantKeyLocked + if isinstance(exc, TenantKeyLocked): + return {"code": "tenant_data_key_unavailable", "type": "TenantKeyLocked", + "message": "The tenant data key could not be read. Retry after restoring its configured backend."} + return {"code": "source_unreadable", "type": type(exc).__name__, + "message": "This activity source could not be read. Retry."} + + +class McpCursorInvalid(ValueError): + """The cursor does not belong to this caller and query, or has an invalid shape.""" + + +class McpCursorStale(ValueError): + """The retained selection changed since the first page and requires a fresh read.""" + + +def _mcp_cursor(session, query, *, position=None, token=None): + """Sign pagination state against the current principal, tenant, permissions and query.""" + import aios_session + # Cookie refresh changes the idle deadline on each GET, but preserves the authenticated session. + claims = {key: value for key, value in (getattr(session, "claims", None) or {}).items() if key != "i"} + binding = json.dumps([session.tenant, session.uname, claims, + session.user, query], sort_keys=True, default=str) + + def proof(value): + body = json.dumps(value, sort_keys=True, separators=(",", ":")) + return aios_session._sign(("usage-mcp:v1:" + binding + body).encode()) + + if token is not None: + try: + if not isinstance(token, str) or not 1 <= len(token) <= 4096: + raise ValueError() + raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True) + value, signature = json.loads(raw) + if not isinstance(signature, str) or not hmac.compare_digest(signature, proof(value)): + raise ValueError() + if (not isinstance(value, dict) + or set(value) != {"asOf", "snapshot", "offset", "total"} + or not isinstance(value["snapshot"], str) or len(value["snapshot"]) != 64 + or any(char not in "0123456789abcdef" for char in value["snapshot"]) + or type(value["offset"]) is not int or type(value["total"]) is not int + or not 0 < value["offset"] < value["total"] + or not isinstance(value["asOf"], str)): + raise ValueError() + _timestamp(value["asOf"]) + return value + except (ValueError, TypeError, UnicodeError, binascii.Error, OverflowError, RecursionError): + raise McpCursorInvalid() from None + raw = json.dumps([position, proof(position)], separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def _mcp_events(session, now, source): + """Exhaust canonical audit reads without changing the source's retention policy.""" + source["retention"] = {"days": audit.RETENTION_SECONDS // 86400, + "legacyRowsPerPartition": audit.MAX_EVENTS} + rows = _unlimited_prefix(lambda limit: audit.read(session.tenant, limit=limit, + now=now.timestamp(), kinds=audit.EXTERNAL_KINDS, viewer=session.user)) + if not _security_record_mode(session.runtime): + stored = (session.runtime.get(audit.BUCKET) or {}).get("events", []) + partition = [row for row in stored if _tenant_row(row, session.tenant) + and row.get("kind") in audit.EXTERNAL_KINDS] + if len(partition) >= audit.MAX_EVENTS: + source["limitations"].append({"code": "legacy_ring", "cause": + "Legacy security documents retain at most 2000 events per partition."}) + held = min(_timestamp(row["ts"]) for row in partition).isoformat() + source["heldFrom"] = max(source["heldFrom"], held) + return rows + + +def _mcp_text(value): + """Expose only bounded metadata text with the canonical credential redaction.""" + if not isinstance(value, str) or not value.strip(): + return "unknown" + clean = audit.redact_intent(value) + return "unknown" if clean == "[invalid]" else clean + + +def _mcp_metadata(row): + """Project the explicit metadata allowlist and leave absent historical fields unknown.""" + detail = row.get("detail") or {} + return {"tool": _mcp_text(detail.get("tool")), "action": _mcp_text(detail.get("action")), + "outcome": _mcp_text(detail.get("outcome")), "code": _mcp_text(detail.get("code")), + "time": _timestamp(row["ts"]).isoformat(), "user": _mcp_text(row.get("principal")), + "database": _mcp_text(detail.get("database")), "keyId": _mcp_text(detail.get("key_id")), + "keyLabel": _mcp_text(detail.get("key_label"))} + + +def read_mcp(session, *, from_=None, to=None, user=None, tool=None, cursor=None, limit=100, now=None): + """Return full selected MCP totals and a page pinned to a signed, ephemeral source fingerprint.""" + if not session.admin: + raise PermissionError("Usage activity requires a tenant administrator.") + if not session.tenant or view_audit.tenant_of(session.runtime) != session.tenant: + raise PermissionError("The usage runtime must belong to the session tenant.") + if type(limit) is not int or not 1 <= limit <= PAGE: + raise ValueError("Limit must be 1 to 200.") + user = str(user).strip() if user else None + if user and (len(user) > 256 or "\x00" in user): + raise ValueError("Choose a valid user.") + tool = str(tool).strip() if tool else None + if tool and (len(tool) > 500 or "\x00" in tool): + raise ValueError("Choose a valid tool.") + query = {"from": from_ or None, "to": to or None, "user": user, "tool": tool, "limit": limit} + position = _mcp_cursor(session, query, token=cursor) if cursor is not None else None + now = _timestamp(now) if now is not None else _now() + as_of = _timestamp(position["asOf"]) if position else now + if as_of > now: + raise McpCursorInvalid() + first, last, _ = date_range(from_, to, now=as_of) + start = datetime.combine(first, time.min, UTC) + stop = datetime.combine(last + timedelta(days=1), time.min, UTC) + floor = now - timedelta(seconds=audit.RETENTION_SECONDS) + source = {"source": "security_events", "kinds": ["mcp"], "heldFrom": floor.isoformat(), + "retention": {"days": audit.RETENTION_SECONDS // 86400, + "legacyRowsPerPartition": audit.MAX_EVENTS}, + "status": "ok", "error": None, "limitations": [], "total": None} + if floor > start: + source["limitations"].append({"code": "retention", "cause": + "This source no longer holds the beginning of the requested range."}) + report = {"tenant": session.tenant, "from": first.isoformat(), "to": last.isoformat(), + "timezone": "UTC", "filters": {"user": user, "tool": tool}, "asOf": as_of.isoformat(), + "limit": limit, "complete": False, "total": None, "byTool": [], "events": [], + "nextCursor": None, "source": source} + try: + rows = _mcp_events(session, now, source) + selected = [] + for row in rows: + if (not _tenant_row(row, session.tenant) or row.get("kind") not in audit.EXTERNAL_KINDS + or row.get("detail", {}).get("via") != "mcp"): + continue + at = _timestamp(row["ts"]) + if not (start <= at < stop and at <= as_of) or (user is not None and row.get("principal") != user): + continue + metadata = _mcp_metadata(row) + if tool is None or metadata["tool"] == tool: + selected.append(metadata) + except Exception as exc: + source.update(status="error", error=_source_error(exc)) + return report + + import aios_session + counts, occurrences = Counter(), Counter() + events = [] + for metadata in selected: + body = json.dumps([session.tenant, metadata], sort_keys=True, separators=(",", ":")) + identity = aios_session._sign(("usage-mcp-event:v1:" + body).encode()) + occurrences[identity] += 1 + events.append({"id": f"{identity}.{occurrences[identity]}", **metadata}) + counts[(metadata["tool"], metadata["outcome"])] += 1 + # Identical events retain distinct occurrence ids, so pagination never deduplicates source rows. + events.sort(key=lambda row: (row["time"], row["id"]), reverse=True) + snapshot = hashlib.sha256(json.dumps([row["id"] for row in events], separators=(",", ":")).encode()).hexdigest() + if position and (position["snapshot"] != snapshot or position["total"] != len(events)): + raise McpCursorStale() + offset = position["offset"] if position else 0 + following = offset + limit + if following < len(events): + report["nextCursor"] = _mcp_cursor(session, query, position={ + "asOf": as_of.isoformat(), "snapshot": snapshot, "offset": following, "total": len(events)}) + source.update(total=len(events), status="partial" if source["limitations"] else "ok") + report.update(complete=source["status"] == "ok", total=len(events), events=events[offset:following], + byTool=[{"tool": name, "total": sum(n for (t, _), n in counts.items() if t == name), + "byOutcome": {outcome: counts[(name, outcome)] for t, outcome in sorted(counts) if t == name}} + for name in sorted({t for t, _ in counts})]) + return report + + +def read(session, *, from_=None, to=None, user=None, kind=None, now=None): + """Return a count cube and separate retention and error metadata for every selected source.""" + if not session.admin: + raise PermissionError("Usage activity requires a tenant administrator.") + if not session.tenant or view_audit.tenant_of(session.runtime) != session.tenant: + raise PermissionError("The usage runtime must belong to the session tenant.") + first, last, now = date_range(from_, to, now=now) + start = datetime.combine(first, time.min, UTC) + stop = datetime.combine(last + timedelta(days=1), time.min, UTC) + wanted = str(kind or "").strip().lower() + if wanted and wanted not in KINDS: + raise ValueError("Kind must be one of: " + ", ".join(KINDS) + ".") + user = str(user).strip() if user else None + if user and (len(user) > 256 or "\x00" in user): + raise ValueError("Choose a valid user.") + selected = (wanted,) if wanted else KINDS + floors = {"security_events": now - timedelta(seconds=audit.RETENTION_SECONDS), + "activity_events": activity_log.retention_cutoff(now), + "value_history": now - timedelta(days=history_rows.RETENTION_DAYS), + "view_audit": start, "turn_audit": start} + sources, counts = [], Counter() + totals = {key: 0 for key in selected} + for name, reader in READERS.items(): + kinds = [key for key in SOURCE_KINDS[name] if key in selected] + if not kinds: + continue + source = {"source": name, "kinds": kinds, "heldFrom": floors[name].isoformat(), + "status": "ok", "error": None, "limitations": [], "total": None} + if floors[name] > start: + source["limitations"].append({"code": "retention", "cause": + "This source no longer holds the beginning of the requested range."}) + local = Counter() + try: + for at, actor, event_kind in reader(session, start, stop, now, source): + if (start <= at < stop and at <= now and event_kind in kinds + and (user is None or actor == user)): + local[(at.date().isoformat(), actor, event_kind)] += 1 + except Exception as exc: + source.update(status="error", error=_source_error(exc)) + for key in kinds: + totals[key] = None + else: + source["total"] = sum(local.values()) + source["byKind"] = {key: sum(n for (_, _, k), n in local.items() if k == key) for key in kinds} + source["status"] = "partial" if source["limitations"] else "ok" + counts.update(local) + totals.update(source["byKind"]) + sources.append(source) + complete = all(source["status"] == "ok" for source in sources) + users = sorted({actor for _, actor, _ in counts}) + by_user = [{"user": actor, "byKind": { + key: None if totals[key] is None else sum(n for (_, a, k), n in counts.items() if a == actor and k == key) + for key in selected}} for actor in users] + by_day = [] + day = first + while day <= last: + values = {key: None if totals[key] is None else sum( + n for (d, _, k), n in counts.items() if d == day.isoformat() and k == key) for key in selected} + for source in sources: + if day < _timestamp(source["heldFrom"]).date(): + for key in source["kinds"]: + values[key] = None + by_day.append({"day": day.isoformat(), "byKind": values}) + day += timedelta(days=1) + return {"tenant": session.tenant, "from": first.isoformat(), "to": last.isoformat(), "timezone": "UTC", + "filters": {"user": user, "kind": wanted or None}, "complete": complete, + "total": sum(totals.values()) if all(n is not None for n in totals.values()) else None, + "availableTotal": sum(counts.values()), "byKind": totals, "byUser": by_user, + "byDay": by_day, "counts": [{"day": day, "user": actor, "kind": key, "count": n} + for (day, actor, key), n in sorted(counts.items())], "sources": sources} + + +def validate(report, direct_counts): + """Reconcile every kind and day-user-kind cell against independently read source counts.""" + actual = Counter({(row["day"], row["user"], row["kind"]): row["count"] for row in report["counts"]}) + differences = {kind: {"served": report["byKind"][kind], + "source": sum(n for (_, _, key), n in direct_counts.items() if key == kind)} + for kind in report["byKind"]} + readable = all(source["status"] != "error" for source in report["sources"]) + users_match = {row["user"] for row in report["byUser"]} == {actor for _, actor, _ in direct_counts} + for row in report["byUser"]: + for kind, value in row["byKind"].items(): + users_match = users_match and value == sum( + n for (_, actor, key), n in direct_counts.items() if actor == row["user"] and key == kind) + days_match = True + for row in report["byDay"]: + for kind, value in row["byKind"].items(): + missing = any(kind in source["kinds"] and (source["status"] == "error" or + row["day"] < _timestamp(source["heldFrom"]).date().isoformat()) for source in report["sources"]) + expected = None if missing else sum(n for (day, _, key), n in direct_counts.items() + if day == row["day"] and key == kind) + days_match = days_match and value == expected + aggregates_match = (report["availableTotal"] == sum(direct_counts.values()) + and report["total"] == sum(direct_counts.values())) + return {"ok": readable and actual == direct_counts and all( + pair["served"] == pair["source"] for pair in differences.values()) + and users_match and days_match and aggregates_match, + "byKind": differences, "cubeMatches": actual == direct_counts, "sourcesReadable": readable, + "usersMatch": users_match, "daysMatch": days_match, "aggregatesMatch": aggregates_match} diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 45bf2104c62a304572c489bbd28ec9956b7d07eb..f661d1613c6134055552b44bdd7691de17ecfb42 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -2893,12 +2893,20 @@ def _clean_display(raw, valid_keys): # next server round trip. if mode == 'grid': coord_field = raw.get('coordField') - return {'mode': 'grid', 'coordField': coord_field} if coord_field in valid_keys else None + if raw.get('groupSummary') is not True: + return {'mode': 'grid', 'coordField': coord_field} if coord_field in valid_keys else None + # The C1 display member carries the grid's group summary toggle. Store only True. + out = {'mode': 'grid', 'groupSummary': True} + if coord_field in valid_keys: + out['coordField'] = coord_field + return out # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown # mode is still rejected rather than accidentally aliased into a real one. Every already # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. mode = _LEGACY_MODES.get(mode, mode) out = {'mode': mode} + if raw.get('groupSummary') is True: + out['groupSummary'] = True # `coordField` is the older per-View fallback. Keep it in the regular field-ref loop so a # deleted source fails safely; the current source is served separately as database setup. for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'): @@ -3332,6 +3340,18 @@ def view_filter_cycle(view_id, nodes, filters_of): return walk(view_id, [view_id], {view_id}) +def clean_filter_cell_ref(raw): + """Validate a saved external cell address without choosing its reader or changing its shape.""" + if (not isinstance(raw, dict) or set(raw) - {'workbook_id', 'tab', 'ref', 'locked'} + or any(not isinstance(raw.get(key), str) or not raw[key].strip() + or raw[key] != raw[key].strip() or len(raw[key]) > 120 + for key in ('workbook_id', 'tab', 'ref')) + or not re.fullmatch(r'\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}', raw['ref']) + or 'locked' in raw and not isinstance(raw['locked'], bool)): + return None + return copy.deepcopy(raw) + + def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, visible_view_ids=None, report=None): """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). @@ -3494,6 +3514,7 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, leaf['rankScope'] = rank_scope rhs = _clean_rhs(node.get('rhs'), valid_keys) if isinstance(node.get('rhs'), dict) and rhs is None: + # An invalid comparison must not become an inactive empty literal. if report is not None: report.append({'id': leaf.get('id', ''), 'colId': column, 'rhsColId': node['rhs'].get('colId'), @@ -3501,6 +3522,12 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, continue if rhs is not None: leaf['rhs'] = rhs + # The save host authorizes these addresses before this structural cleaner runs. + refs = {key: clean_filter_cell_ref(node[key]) + for key in ('valueRef', 'value2Ref') if key in node} + if any(ref is None for ref in refs.values()): + continue + leaf.update(refs) # C3: a record lock is explicit on the leaf, not a view-wide presentation flag. # Once stored, grid_events preserves the containing filter config unless it is copied # to a new View; a later caller cannot turn this `True` into an editable-looking false. diff --git a/platform/core/amazon_project_tasks.py b/platform/core/amazon_project_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..696103bef3ae37075d13b9350775872efc46fecd --- /dev/null +++ b/platform/core/amazon_project_tasks.py @@ -0,0 +1,725 @@ +"""Tenant-bound task trees and leaf progress for Amazon projects.""" + +from collections import defaultdict, deque +from contextlib import contextmanager +from datetime import date, timedelta +from decimal import Decimal, InvalidOperation, localcontext +import json +import uuid + +from core import amazon_projects as projects + +TASKS = 'amazon_project_tasks' +DEPENDENCIES = 'amazon_project_dependencies' +FILTERS = 'amazon_project_task_filters' +TABLES = (TASKS, DEPENDENCIES, FILTERS) +TASK_COLUMNS = ('id', 'project_id', 'parent_id', 'position', 'name', 'assignee', 'start_on', + 'due_on', 'done_at', 'contingency_for', 'active', 'created_by') +DEPENDENCY_COLUMNS = ('id', 'project_id', 'from_task', 'to_task', 'type', 'lag_days') +DEPENDENCY_ENDPOINTS = {'FS': ('due_on', 'start_on'), 'SS': ('start_on', 'start_on'), + 'FF': ('due_on', 'due_on'), 'SF': ('start_on', 'due_on')} +TARGET_FIELDS = ('targets', 'target_filter', 'completion_filter') +FILTER_STATE = ('filter_status', 'filter_closed_at', 'closed_targets', 'filter_missing') +EDITABLE = frozenset(('name', 'assignee', 'start_on', 'due_on', 'done', 'contingency_for', *TARGET_FIELDS)) +_UNCHANGED = object() + + +def _tree(value, *, empty=True): + nodes = value if isinstance(value, list) else value.get('nodes') if isinstance(value, dict) else None + if not isinstance(nodes, list) or (not empty and not nodes): + raise ValueError('Send a saved filter tree with conditions.') + try: + return json.loads(json.dumps(value, allow_nan=False)) + except (TypeError, ValueError): + raise ValueError('Send a saved filter tree with JSON values.') from None + + +def _target_fields(current): + values = current.get('targets') + if values is not None: + if not isinstance(values, list): + raise ValueError('Send targets as a list of grain and key pairs.') + pairs, seen = [], set() + for value in values: + if not isinstance(value, dict) or set(value) != {'grain', 'key'}: + raise ValueError('Send each task target as a grain and key.') + pair = projects.target(value['grain'], value['key']) + if pair[0] not in ('sku', 'asin'): + raise ValueError('Task targets use sku or asin.') + if pair not in seen: + seen.add(pair) + pairs.append(pair) + current['targets'] = projects._pair_list(pairs) + saved = current.get('target_filter') + if saved is not None: + if not isinstance(saved, dict) or set(saved) != {'grain', 'filter'} or saved['grain'] not in ('sku', 'asin'): + raise ValueError('Send targetFilter as a sku or asin grain and filter tree.') + current['target_filter'] = dict(grain=saved['grain'], filter=_tree(saved['filter'])) + if current.get('targets') and saved is not None: + raise ValueError('Send either a target list or one saved target filter.') + if current.get('completion_filter') is not None: + current['completion_filter'] = _tree(current['completion_filter'], empty=False) + if not current.get('targets') and saved is None: + raise ValueError('A completion filter needs task targets.') + return current + + +def _position(value): + try: + number = Decimal(str(value)) + except (InvalidOperation, ValueError): + raise ValueError('position must be a finite number.') from None + if isinstance(value, bool) or not number.is_finite(): + raise ValueError('position must be a finite number.') + return number + + +def _fields(values, previous=None): + current = dict(previous or {}) + current.update(values) + current['name'] = projects._text(current.get('name'), 'name') + current['assignee'] = projects._text(current.get('assignee'), 'assignee', optional=True) + for field in ('start_on', 'due_on'): + current[field] = projects._date(current.get(field), field) + if current['start_on'] and current['due_on'] and current['due_on'] < current['start_on']: + raise ValueError('due_on must be on or after start_on.') + current['contingency_for'] = projects._text(current.get('contingency_for'), 'contingency_for', optional=True) + if 'done' in values: + if type(values['done']) is not bool: + raise ValueError('done must be true or false.') + current['done_at'] = (current.get('done_at') or projects._now()) if values['done'] else None + else: + current.setdefault('done_at', None) + current.pop('done', None) + return _target_fields(current) + + +def _record(values): + row = dict(zip(TASK_COLUMNS, values)) + row['position'] = str(_position(row['position'])) + row['active'] = bool(row['active']) + row['done'] = row['done_at'] is not None + for field in ('start_on', 'due_on'): + row[field] = projects._date(row[field], field) + return row + + +def _check_parent(rows, task_id, parent_id): + """Walk ancestors without a Python call stack or a fixed depth ceiling.""" + visited = {task_id} + while parent_id is not None: + if parent_id in visited: + raise ValueError('A task cannot move under itself or its descendant.') + if parent_id not in rows: + raise ValueError('The parent task must belong to this project.') + visited.add(parent_id) + parent_id = rows[parent_id]['parent_id'] + + +def _check_contingency(rows, task_id, contingency_for): + if contingency_for is not None and (contingency_for == task_id or contingency_for not in rows): + raise ValueError('The backup task must name another task in this project.') + + +def _dependency_fields(kind, lag_days): + if not isinstance(kind, str) or kind not in DEPENDENCY_ENDPOINTS: + raise ValueError('Dependency type must be FS, SS, FF or SF.') + if type(lag_days) is not int or not -2147483648 <= lag_days <= 2147483647: + raise ValueError('lag_days must be an integer number of days.') + return kind, lag_days + + +def _check_dependency(rows, dependencies, from_task, to_task): + if from_task not in rows or to_task not in rows: + raise ValueError('Both dependency tasks must belong to this project.') + if from_task == to_task: + raise ValueError('A task cannot depend on itself.') + outgoing = defaultdict(list) + for dependency in dependencies: + outgoing[dependency['from_task']].append(dependency['to_task']) + pending, seen = [to_task], set() + while pending: + task_id = pending.pop() + if task_id == from_task: + source, target = rows[from_task], rows[to_task] + raise ValueError(f'Dependency cycle between "{source["name"]}" ({from_task}) ' + f'and "{target["name"]}" ({to_task}).') + if task_id not in seen: + seen.add(task_id) + pending.extend(outgoing[task_id]) + + +def _constraint_gap(rows, dependency): + source, target = rows[dependency['from_task']], rows[dependency['to_task']] + source_field, target_field = DEPENDENCY_ENDPOINTS[dependency['type']] + if source[source_field] is None or target[target_field] is None: + raise ValueError(f'Dependency "{source["name"]}" to "{target["name"]}" ' + f'requires predecessor {source_field} and successor {target_field}.') + return ((date.fromisoformat(source[source_field]) - date.fromisoformat(target[target_field])).days + + dependency['lag_days']) + + +def _propagate(rows, dependencies, task_id): + """Shift each affected task once after all its predecessors, preserving its duration.""" + incoming, outgoing, indegree = defaultdict(list), defaultdict(list), dict.fromkeys(rows, 0) + for dependency in dependencies: + incoming[dependency['to_task']].append(dependency) + outgoing[dependency['from_task']].append(dependency['to_task']) + indegree[dependency['to_task']] += 1 + pending, affected = [task_id], set() + while pending: + current = pending.pop() + if current not in affected: + affected.add(current) + pending.extend(outgoing[current]) + ready, shifted, visited = deque(key for key, count in indegree.items() if count == 0), [], 0 + while ready: + current = ready.popleft() + visited += 1 + if current in affected: + days = max([0] + [_constraint_gap(rows, dependency) for dependency in incoming[current]]) + if days: + changed = dict(rows[current]) + try: + for field in ('start_on', 'due_on'): + if changed[field] is not None: + changed[field] = (date.fromisoformat(changed[field]) + timedelta(days=days)).isoformat() + except OverflowError: + raise ValueError(f'Dependency shift exceeds the calendar for task "{changed["name"]}".') from None + rows[current] = changed + shifted.append(changed) + for successor in outgoing[current]: + indegree[successor] -= 1 + if indegree[successor] == 0: + ready.append(successor) + if visited != len(rows): + raise ValueError('The stored task dependencies contain a cycle.') + return shifted + + +def _subtree(rows, task_id): + children = defaultdict(list) + for row in rows.values(): + children[row['parent_id']].append(row['id']) + pending, found = [task_id], [] + seen = set() + while pending: + current = pending.pop() + if current not in seen: + seen.add(current) + found.append(current) + pending.extend(children[current]) + return found + + +def _leaf_progress(rows, fallback): + """Count completed active leaves, excluding every dormant backup subtree.""" + children = defaultdict(list) + for row in rows: + children[row['parent_id']].append(row) + pending, leaves = list(children[None]), [] + while pending: + row = pending.pop() + if not row['active']: + continue + active_children = [child for child in children[row['id']] if child['active']] + if active_children: + pending.extend(active_children) + else: + leaves.append(row) + done, total = sum(row['done'] for row in leaves), len(leaves) + return {'done': done, 'total': total, + 'progress': round(100 * done / total) if total else (0 if rows else fallback)} + + +def _backend(rt): + return PgTaskStore(projects._backend(rt)) + + +def list_tasks(rt, project_id, *, matcher=None, today=None, by=None): + """Read every task as flat rows with parent ids and exact fractional positions.""" + backend = _backend(rt) + project_id = projects._text(project_id, 'project_id') + if matcher is not None: + _evaluate_project(rt, backend, project_id, matcher, today=today, by=by) + return backend.list_tasks(project_id) + + +def get(rt, project_id, task_id): + """Read one task within its tenant and project, or raise KeyError.""" + task_id = projects._text(task_id, 'task_id') + for row in list_tasks(rt, project_id): + if row['id'] == task_id: + return row + raise KeyError('Task not found in this project.') + + +def progress(rt, project_id): + """Return completed leaf counts and their percentage, retaining manual progress without tasks.""" + project_id = projects._text(project_id, 'project_id') + return _backend(rt).progress(project_id) + + +def _mutate(rt, project_id, task_id, action, values, by): + project_id = projects._text(project_id, 'project_id') + task_id = projects._text(task_id, 'task_id') + by = projects._text(by, 'by') + before, after, result = _backend(rt).mutate(project_id, task_id, action, values, by) + projects._log(rt, project_id, by, 'tasks', before, after) + return result + + +def create(rt, project_id, *, name, by, parent_id=None, position=None, assignee=None, + start_on=None, due_on=None, done=False, contingency_for=None, created_by=None, + targets=None, target_filter=None, completion_filter=None): + """Create one task with at most one assignee and an optional dormant backup link.""" + values = _fields(dict(name=name, assignee=assignee, start_on=start_on, due_on=due_on, + done=done, contingency_for=contingency_for)) + if targets is not None or target_filter is not None or completion_filter is not None: + values.update(targets=targets, target_filter=target_filter, completion_filter=completion_filter) + _target_fields(values) + values.update(parent_id=projects._text(parent_id, 'parent_id', optional=True), + position=None if position is None else str(_position(position)), + created_by=projects._text(created_by or by, 'created_by')) + return _mutate(rt, project_id, uuid.uuid4().hex, 'create', values, by) + + +def update(rt, project_id, task_id, changes, *, by): + """Change task fields without allowing a second tree or activation write door.""" + if not isinstance(changes, dict) or not changes or set(changes) - EDITABLE: + raise ValueError(f'Update only: {", ".join(sorted(EDITABLE))}.') + return _mutate(rt, project_id, task_id, 'update', dict(changes), by) + + +def move(rt, project_id, task_id, *, by, parent_id=_UNCHANGED, position=None): + """Reparent or reorder one task after checking the full ancestor chain.""" + values = dict(position=None if position is None else str(_position(position))) + if parent_id is not _UNCHANGED: + values['parent_id'] = projects._text(parent_id, 'parent_id', optional=True) + return _mutate(rt, project_id, task_id, 'move', values, by) + + +def activate(rt, project_id, task_id, *, by): + """Include a backup task and its descendants after an explicit human activation.""" + return _mutate(rt, project_id, task_id, 'activate', {}, by) + + +def delete(rt, project_id, task_id, *, by): + """Delete one task and its full subtree in the project's write transaction.""" + return _mutate(rt, project_id, task_id, 'delete', {}, by) + + +def list_dependencies(rt, project_id): + """Read all dependency edges within one tenant and project without creating storage.""" + return _backend(rt).list_dependencies(projects._text(project_id, 'project_id')) + + +def add_dependency(rt, project_id, from_task, to_task, *, type='FS', lag_days=0, by): + """Link two tasks and shift the successor and its dependents to satisfy the new constraint.""" + project_id = projects._text(project_id, 'project_id') + by = projects._text(by, 'by') + kind, lag_days = _dependency_fields(type, lag_days) + record = dict(id=uuid.uuid4().hex, project_id=project_id, + from_task=projects._text(from_task, 'from_task'), + to_task=projects._text(to_task, 'to_task'), type=kind, lag_days=lag_days) + result = _backend(rt).add_dependency(record) + projects._log(rt, project_id, by, 'dependencies', None, result) + return result + + +def delete_dependency(rt, project_id, dependency_id, *, by): + """Remove one project dependency without moving either task earlier.""" + project_id = projects._text(project_id, 'project_id') + by = projects._text(by, 'by') + record = _backend(rt).delete_dependency(project_id, projects._text(dependency_id, 'dependency_id')) + projects._log(rt, project_id, by, 'dependencies', record, None) + return record + + +def reschedule(rt, task_id, start_on, due_on, by): + """Reschedule one tenant task and return every dependent shifted to satisfy its constraints.""" + task_id, by = projects._text(task_id, 'task_id'), projects._text(by, 'by') + values = {field: projects._date(value, field) for field, value in + (('start_on', start_on), ('due_on', due_on))} + before, result = _backend(rt).reschedule(task_id, values) + projects._log(rt, result['task']['project_id'], by, 'tasks', before, result) + return result + + +def validate(rt, project_id, candidate=None): + """Reconcile progress against a separate authored-task read without the display calculator.""" + backend = _backend(rt) + project_id = projects._text(project_id, 'project_id') + with backend.projects._connection() as con: + project = backend._project(con, project_id) + rows = (con.execute(f'SELECT id, parent_id, active, done_at FROM {backend.t(TASKS)} ' + 'WHERE tenant = %s AND project_id = %s', + (backend.tenant_slug, project_id)).fetchall() + if backend._exists(con) else []) + children, completed = defaultdict(list), {} + for task_id, parent_id, active, done_at in rows: + if active: + children[parent_id].append(task_id) + completed[task_id] = done_at is not None + pending, leaves = deque(children[None]), [] + while pending: + task_id = pending.popleft() + if children[task_id]: + pending.extend(children[task_id]) + else: + leaves.append(task_id) + done, total = sum(completed[task_id] for task_id in leaves), len(leaves) + expected = round(100 * done / total) if total else (0 if rows else project['progress']) + actual = project['progress'] if candidate is None else candidate + return {'check': 'project progress ties to active leaf tasks', 'ok': actual == expected, + 'expected': expected, 'actual': actual, 'done': done, 'total': total} + + +def _evaluate_project(rt, backend, project_id, matcher, *, today, by): + day = projects._date(today or projects._today(), 'today') + by = projects._text(by, 'by') + rows = backend.list_tasks(project_id) + if not any(row.get('completion_filter') is not None for row in rows): + return [] + events = [] + with backend._write(project_id) as (con, project): + rows = backend._rows(con, project_id) + ancestors = {item['id']: item for item in rows} + for row in rows: + if row.get('completion_filter') is None: + continue + # A dormant backup and its subtree need human activation before evaluation. + current = row + while current['active'] and current['parent_id'] is not None: + current = ancestors[current['parent_id']] + if not current['active']: + continue + principal = (backend.tenant_slug, row.get('filter_by', row['created_by'])) + if row.get('filter_closed_at'): + pairs = row['closed_targets'] + elif row.get('target_filter') is not None: + saved = row['target_filter'] + pairs = projects._pair_list((saved['grain'], key) for key in + matcher(rt, saved['grain'], saved['filter'], user=principal)) + else: + pairs = row.get('targets') or [] + met = set() + for grain in dict.fromkeys(pair['grain'] for pair in pairs): + keys = set(matcher(rt, grain, row['completion_filter'], user=principal)) + met.update((grain, pair['key']) for pair in pairs if pair['grain'] == grain and pair['key'] in keys) + missing = [pair for pair in pairs if (pair['grain'], pair['key']) not in met] + after = dict(row, filter_status={'met': len(met), 'total': len(pairs), + 'message': f'met {len(met)} of {len(pairs)}', 'evaluated_on': day}) + if row.get('filter_closed_at'): + previous = {(pair['grain'], pair['key']) for pair in row.get('filter_missing', [])} + for pair in missing: + if (pair['grain'], pair['key']) not in previous: + events.append((row['id'], f'{projects.GRAIN_LABELS[pair["grain"]]} {pair["key"]} ' + 'no longer meets the filter', row, dict(after, target=pair))) + elif not row['done'] and pairs and not missing: + after.update(done=True, done_at=projects._now(), filter_closed_at=projects._now(), + closed_targets=pairs) + events.append((row['id'], 'Done by filter', row, after)) + after['filter_missing'] = missing + backend._save(con, after) + backend._sync_progress(con, project_id, project['progress']) + for _task_id, action, before, after in events: + projects._log(rt, project_id, by, action, before, after) + return [dict(task_id=task_id, action=action) for task_id, action, _before, _after in events] + + +def evaluate_filter_tasks(rt, matcher, *, today, by): + """Evaluate every saved task criterion on refresh through the injected tenant matcher.""" + backend = _backend(rt) + events = [] + for project_id in backend.filter_projects(): + events.extend(_evaluate_project(rt, backend, project_id, matcher, today=today, by=by)) + return events + + +def validate_filter_tasks(rt, project_id, matcher): + """Reconcile stored completion counts against a separate target and matcher read.""" + backend = _backend(rt) + checks = [] + for row in backend.list_tasks(projects._text(project_id, 'project_id')): + if row.get('filter_status') is None: + continue + user = (backend.tenant_slug, row.get('filter_by', row['created_by'])) + saved = row.get('target_filter') + if row.get('filter_closed_at'): + targets = row['closed_targets'] + elif saved is not None: + targets = [{'grain': saved['grain'], 'key': key} + for key in matcher(rt, saved['grain'], saved['filter'], user=user)] + else: + targets = row.get('targets') or [] + matches = {grain: set(matcher(rt, grain, row['completion_filter'], user=user)) + for grain in {pair['grain'] for pair in targets}} + met = sum(pair['key'] in matches[pair['grain']] for pair in targets) + actual = row['filter_status'] + checks.append({'task': row['id'], 'expected': {'met': met, 'total': len(targets)}, + 'actual': {key: actual[key] for key in ('met', 'total')}, + 'ok': actual['met'] == met and actual['total'] == len(targets)}) + return {'check': 'task filter counts tie to target rows', 'ok': all(row['ok'] for row in checks), + 'tasks': checks} + + +class PgTaskStore: + """Use the existing tenant backend, write guard and project row lock for task authorship.""" + + def __init__(self, backend): + self.projects = backend + self.tenant_slug = backend.tenant_slug + self.t = backend.t + + def _exists(self, con, table=TASKS): + return con.execute('SELECT to_regclass(%s)', (self.t(table),)).fetchone()[0] is not None + + def _project(self, con, project_id, lock=False): + if not self.projects._exists(con): + raise KeyError('Project not found.') + return self.projects._one(con, project_id, lock=lock) + + def ensure_schema(self, con): + con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(TASKS)} ( + tenant TEXT NOT NULL, id TEXT NOT NULL, project_id TEXT NOT NULL, + parent_id TEXT, position NUMERIC NOT NULL, name TEXT NOT NULL, + assignee TEXT, start_on DATE, due_on DATE, done_at TEXT, + contingency_for TEXT, active BOOLEAN NOT NULL DEFAULT TRUE, created_by TEXT NOT NULL, + PRIMARY KEY (tenant, id), UNIQUE (tenant, project_id, id), + FOREIGN KEY (tenant, project_id) REFERENCES {self.t(projects.PROJECTS)} (tenant, id) + ON DELETE CASCADE, + FOREIGN KEY (tenant, project_id, parent_id) REFERENCES {self.t(TASKS)} (tenant, project_id, id) + DEFERRABLE INITIALLY DEFERRED, + CHECK (parent_id IS NULL OR parent_id <> id), + CHECK (contingency_for IS NULL OR contingency_for <> id), + CHECK (start_on IS NULL OR due_on IS NULL OR start_on <= due_on))''') + con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_tasks_siblings" ' + f'ON {self.t(TASKS)} (tenant, project_id, parent_id, position, id)') + con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(DEPENDENCIES)} ( + tenant TEXT NOT NULL, id TEXT NOT NULL, project_id TEXT NOT NULL, + from_task TEXT NOT NULL, to_task TEXT NOT NULL, type TEXT NOT NULL, + lag_days INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (tenant, id), + UNIQUE (tenant, project_id, from_task, to_task, type), + FOREIGN KEY (tenant, project_id) REFERENCES {self.t(projects.PROJECTS)} (tenant, id) + ON DELETE CASCADE, + FOREIGN KEY (tenant, project_id, from_task) REFERENCES {self.t(TASKS)} (tenant, project_id, id) + ON DELETE CASCADE, + FOREIGN KEY (tenant, project_id, to_task) REFERENCES {self.t(TASKS)} (tenant, project_id, id) + ON DELETE CASCADE, + CHECK (from_task <> to_task), CHECK (type IN ('FS', 'SS', 'FF', 'SF')))''') + con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_dependencies_target" ' + f'ON {self.t(DEPENDENCIES)} (tenant, project_id, to_task)') + con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(FILTERS)} ( + tenant TEXT NOT NULL, id TEXT NOT NULL, project_id TEXT NOT NULL, config TEXT NOT NULL, + PRIMARY KEY (tenant, id), + FOREIGN KEY (tenant, project_id, id) REFERENCES {self.t(TASKS)} (tenant, project_id, id) + ON DELETE CASCADE)''') + + @contextmanager + def _write(self, project_id): + self.projects._write_guard() + with self.projects._connection() as con: + project = self._project(con, project_id, lock=True) + self.ensure_schema(con) + yield con, project + + def _rows(self, con, project_id): + rows = con.execute(f'SELECT {", ".join(TASK_COLUMNS)} FROM {self.t(TASKS)} ' + 'WHERE tenant = %s AND project_id = %s ' + 'ORDER BY parent_id NULLS FIRST, position, id', + (self.tenant_slug, project_id)).fetchall() + records = [_record(row) for row in rows] + configs = {} + if self._exists(con, FILTERS): + configs = {task_id: json.loads(value) for task_id, value in con.execute( + f'SELECT id, config FROM {self.t(FILTERS)} WHERE tenant = %s AND project_id = %s', + (self.tenant_slug, project_id)).fetchall()} + return [dict(row, **configs.get(row['id'], {})) for row in records] + + def filter_projects(self): + with self.projects._connection() as con: + if not self._exists(con, FILTERS): + return [] + return [row[0] for row in con.execute( + f'SELECT DISTINCT project_id FROM {self.t(FILTERS)} WHERE tenant = %s ORDER BY project_id', + (self.tenant_slug,)).fetchall()] + + def list_tasks(self, project_id): + with self.projects._connection() as con: + self._project(con, project_id) + return self._rows(con, project_id) if self._exists(con) else [] + + def _dependencies(self, con, project_id): + if not self._exists(con, DEPENDENCIES): + return [] + rows = con.execute(f'SELECT {", ".join(DEPENDENCY_COLUMNS)} FROM {self.t(DEPENDENCIES)} ' + 'WHERE tenant = %s AND project_id = %s ORDER BY id', + (self.tenant_slug, project_id)).fetchall() + return [dict(zip(DEPENDENCY_COLUMNS, row)) for row in rows] + + def list_dependencies(self, project_id): + with self.projects._connection() as con: + self._project(con, project_id) + return self._dependencies(con, project_id) + + def add_dependency(self, record): + with self._write(record['project_id']) as (con, project): + rows = {row['id']: row for row in self._rows(con, record['project_id'])} + dependencies = self._dependencies(con, record['project_id']) + _check_dependency(rows, dependencies, record['from_task'], record['to_task']) + if any(all(edge[field] == record[field] for field in ('from_task', 'to_task', 'type')) + for edge in dependencies): + raise ValueError('This task dependency already exists.') + shifted = _propagate(rows, [*dependencies, record], record['to_task']) + columns = ('tenant', *DEPENDENCY_COLUMNS) + con.execute(f'INSERT INTO {self.t(DEPENDENCIES)} ({", ".join(columns)}) VALUES ' + f'({", ".join("%s" for _ in columns)})', + (self.tenant_slug, *(record[field] for field in DEPENDENCY_COLUMNS))) + for row in shifted: + self._save(con, row) + self._sync_progress(con, record['project_id'], project['progress']) + return dict(record, shifted=shifted) + + def delete_dependency(self, project_id, dependency_id): + with self._write(project_id) as (con, _project): + record = next((row for row in self._dependencies(con, project_id) + if row['id'] == dependency_id), None) + if record is None: + raise KeyError('Dependency not found in this project.') + con.execute(f'DELETE FROM {self.t(DEPENDENCIES)} ' + 'WHERE tenant = %s AND project_id = %s AND id = %s', + (self.tenant_slug, project_id, dependency_id)) + return record + + def reschedule(self, task_id, values): + with self.projects._connection() as con: + found = (con.execute(f'SELECT project_id FROM {self.t(TASKS)} WHERE tenant = %s AND id = %s', + (self.tenant_slug, task_id)).fetchone() if self._exists(con) else None) + if found is None: + raise KeyError('Task not found in this tenant.') + with self._write(found[0]) as (con, project): + rows = {row['id']: row for row in self._rows(con, found[0])} + if task_id not in rows: + raise KeyError('Task not found in this project.') + old = dict(rows) + rows[task_id] = _fields(values, rows[task_id]) + rows[task_id]['done'] = rows[task_id]['done_at'] is not None + shifted = _propagate(rows, self._dependencies(con, found[0]), task_id) + shifted = [row for row in shifted if row['id'] != task_id] + for row in [rows[task_id], *shifted]: + self._save(con, row) + self._sync_progress(con, found[0], project['progress']) + before = {'task': old[task_id], 'shifted': [old[row['id']] for row in shifted]} + return before, {'task': rows[task_id], 'shifted': shifted} + + def progress(self, project_id): + with self.projects._connection() as con: + project = self._project(con, project_id) + rows = self._rows(con, project_id) if self._exists(con) else [] + return _leaf_progress(rows, project['progress']) + + def _next_position(self, rows, parent_id, task_id): + positions = [_position(row['position']) for row in rows.values() + if row['parent_id'] == parent_id and row['id'] != task_id] + last = max(positions, default=Decimal(0)) + with localcontext() as context: + context.prec = max(28, len(last.as_tuple().digits) + abs(last.as_tuple().exponent) + 2) + return str(last + 1) + + def _save(self, con, record, *, insert=False): + if insert: + columns = ('tenant', *TASK_COLUMNS) + con.execute(f'INSERT INTO {self.t(TASKS)} ({", ".join(columns)}) VALUES ' + f'({", ".join("%s" for _ in columns)})', + (self.tenant_slug, *(record[field] for field in TASK_COLUMNS))) + else: + fields = tuple(field for field in TASK_COLUMNS if field not in ('id', 'project_id', 'created_by')) + con.execute(f'UPDATE {self.t(TASKS)} SET ' + + ', '.join(f'{field} = %s' for field in fields) + + ' WHERE tenant = %s AND project_id = %s AND id = %s', + (*(record[field] for field in fields), self.tenant_slug, record['project_id'], record['id'])) + if any(field in record for field in TARGET_FIELDS): + config = {field: record[field] for field in (*TARGET_FIELDS, *FILTER_STATE, 'filter_by') if field in record} + con.execute(f'INSERT INTO {self.t(FILTERS)} (tenant, id, project_id, config) VALUES (%s, %s, %s, %s) ' + 'ON CONFLICT (tenant, id) DO UPDATE SET config = excluded.config', + (self.tenant_slug, record['id'], record['project_id'], json.dumps(config, allow_nan=False))) + + def _delete(self, con, project_id, ids): + # Delete descendants first so database cascade triggers add no depth ceiling. + for part in projects._chunks(list(reversed(ids))): + marks = ', '.join('%s' for _ in part) + args = (self.tenant_slug, project_id, *part) + self._delete_dependencies(con, project_id, part) + con.execute(f'DELETE FROM {self.t(FILTERS)} ' + f'WHERE tenant = %s AND project_id = %s AND id IN ({marks})', args) + con.execute(f'UPDATE {self.t(TASKS)} SET contingency_for = NULL ' + f'WHERE tenant = %s AND project_id = %s AND contingency_for IN ({marks})', args) + con.execute(f'DELETE FROM {self.t(TASKS)} ' + f'WHERE tenant = %s AND project_id = %s AND id IN ({marks})', args) + + def _delete_dependencies(self, con, project_id, ids): + marks = ', '.join('%s' for _ in ids) + con.execute(f'DELETE FROM {self.t(DEPENDENCIES)} ' + f'WHERE tenant = %s AND project_id = %s ' + f'AND (from_task IN ({marks}) OR to_task IN ({marks}))', + (self.tenant_slug, project_id, *ids, *ids)) + + def _sync_progress(self, con, project_id, fallback): + result = _leaf_progress(self._rows(con, project_id), fallback) + con.execute(f'UPDATE {self.t(projects.PROJECTS)} SET progress = %s, updated_at = %s ' + 'WHERE tenant = %s AND id = %s', + (result['progress'], projects._now(), self.tenant_slug, project_id)) + return result + + def mutate(self, project_id, task_id, action, values, by): + with self._write(project_id) as (con, project): + rows = {row['id']: row for row in self._rows(con, project_id)} + before = rows.get(task_id) + if action != 'create' and before is None: + raise KeyError('Task not found in this project.') + if action == 'delete': + ids = _subtree(rows, task_id) + self._delete(con, project_id, ids) + counts = self._sync_progress(con, project_id, project['progress']) + return before, None, {'deleted': task_id, 'task_ids': ids, 'progress': counts['progress']} + if action == 'create': + after = dict(values, id=task_id, project_id=project_id, + active=values['contingency_for'] is None) + elif action == 'update': + after = _fields(values, before) + if after['contingency_for'] != before['contingency_for'] and after['contingency_for'] is not None: + after['active'] = False + elif action == 'move': + after = dict(before, **values) + elif action == 'activate': + after = dict(before, active=True) + else: + raise ValueError('Unknown task change.') + if set(TARGET_FIELDS) & values.keys(): + if before and before.get('filter_closed_at') and values.get('done') is not False: + raise ValueError('Reopen and reset the task before changing its targets or filter.') + after['filter_by'] = by + for field in FILTER_STATE: + after.pop(field, None) + if values.get('done') is False: + for field in FILTER_STATE: + after.pop(field, None) + if after.get('completion_filter') is not None: + after['filter_by'] = by + _check_parent(rows, task_id, after['parent_id']) + _check_contingency(rows, task_id, after['contingency_for']) + if after['position'] is None: + after['position'] = self._next_position(rows, after['parent_id'], task_id) + shifted = [] + if action == 'update' and {'start_on', 'due_on'} & values.keys(): + rows[task_id] = after + shifted = _propagate(rows, self._dependencies(con, project_id), task_id) + after = rows[task_id] + self._save(con, after, insert=action == 'create') + for row in shifted: + if row['id'] != task_id: + self._save(con, row) + self._sync_progress(con, project_id, project['progress']) + saved = next(row for row in self._rows(con, project_id) if row['id'] == task_id) + return before, saved, saved diff --git a/platform/core/amazon_projects.py b/platform/core/amazon_projects.py index 6004b0298213b0752ee8b2d9df37dd7e33b4d512..3fddd9b8abcff538be82cb16772ae3f129a7eaac 100644 --- a/platform/core/amazon_projects.py +++ b/platform/core/amazon_projects.py @@ -1,7 +1,7 @@ """Per-tenant PostgreSQL Amazon projects, each assigned to many SKUs, ASINs and Parent ASINs. No request reads the `user_tables` document for a project (C6). Core imports no harness module -(verify_layering) and no App module (rule 13). Both tables are created on the first write. +(verify_layering) and no App module (rule 13). Tables are created on the first write. A door called with `by` appends one `core.activity_log` event per create, field or target change. """ @@ -9,13 +9,16 @@ from contextlib import contextmanager from datetime import date, datetime, timezone from decimal import Decimal, InvalidOperation, ROUND_HALF_UP import json +import hashlib +import heapq import re import sys import uuid PROJECTS = 'amazon_projects' TARGETS = 'amazon_project_targets' -TABLES = (PROJECTS, TARGETS) +MEMBERS = 'amazon_project_members' +TABLES = (PROJECTS, TARGETS, MEMBERS) GRAINS = ('sku', 'asin', 'parent_asin') #: R18: the fixed status set of SDLC 82. @@ -32,7 +35,7 @@ PROJECT_COLUMNS = ('id', 'tenant', 'name', 'status', 'priority', 'owner', 'start 'created_by') TARGET_COLUMNS = ('project_id', 'grain', 'key', 'added_at') EDITABLE = frozenset(('name', 'status', 'priority', 'owner', 'start_on', 'target_on', 'progress', 'notes', - 'completed_on', 'result_configs')) + 'completed_on', 'result_configs', 'contributors')) RESULT_CONFIGS_KEY = '_resultConfigs' DATES = ('start_on', 'target_on', 'completed_on') NOTES_BYTES = 65536 @@ -45,6 +48,11 @@ ACTIVITY_FIELDS = {'name': 'name', 'priority': 'priority', 'owner': 'owner', 'st 'target_on': 'targetOn', 'progress': 'progress', 'notes': 'notes', 'completed_on': 'completedOn', 'result_configs': 'resultConfigs', 'status': 'status'} GRAIN_LABELS = {'sku': 'SKU', 'asin': 'ASIN', 'parent_asin': 'Parent ASIN'} +LISTING_ACTIVITY_FIELDS = {'title': 'Title', 'item_name': 'Title', 'image': 'Main image', + 'main_image': 'Main image', 'main_image_url': 'Main image', + 'main_product_image_locator': 'Main image', + 'description': 'Description', 'product_description': 'Description', + 'bullet_point': 'Bullet points'} #: R39: an old `ut_amazon_projects` status becomes (status, progress). MOVE_STATUS = {'Open': ('In progress', 0), 'Done': ('Completed', 100)} @@ -81,6 +89,19 @@ def owner_identity(value, tenant=None): return None +def _member_values(values, tenant): + """Normalize unique contributor identities bound to the project's tenant.""" + if not isinstance(values, (list, tuple)): + raise ValueError('Contributors must be a list of workspace users.') + members = {} + for identity in values: + if not isinstance(identity, dict) or identity.get('tenant') != tenant: + raise ValueError('Each contributor must belong to this workspace.') + canonical = owner_identity(owner_value(tenant, identity), tenant) + members[canonical['username']] = {key: canonical[key] for key in ('username', 'name')} + return sorted(members.values(), key=lambda member: member['username']) + + def _in_sql(values): return ', '.join(f"'{value}'" for value in values) @@ -160,10 +181,11 @@ def target(grain, key): def _targets(values): """Unique (grain, key) pairs in first-seen order, from dicts or pairs.""" - out = [] + out, seen = [], set() for value in values or (): pair = _target(value.get('grain'), value.get('key')) if isinstance(value, dict) else _target(*value) - if pair not in out: + if pair not in seen: + seen.add(pair) out.append(pair) return out @@ -179,6 +201,8 @@ def _fields(values, *, previous=None): result['progress'] = _progress(result.get('progress')) for field in DATES: result[field] = _date(result.get(field), field) + if result['start_on'] is not None and result['target_on'] is not None and result['target_on'] < result['start_on']: + raise ValueError('Target date cannot be before the start date.') from core import amazon_project_results result['result_configs'] = amazon_project_results.configurations(result.get('result_configs', ())) if (values.get('completed_on') is not None and result['status'] != COMPLETED): @@ -216,6 +240,9 @@ def _record(values, columns=PROJECT_COLUMNS): value = values[offset] if len(values) > offset else None result[field] = json.loads(value) if isinstance(value, str) else value result['result_configs'] = (result.get('impact') or {}).get(RESULT_CONFIGS_KEY, []) + value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None + members = json.loads(value) if isinstance(value, str) else value + result['contributors'] = sorted(members or [], key=lambda member: member['username']) return result @@ -262,12 +289,13 @@ def summary(projects): def create(rt, *, name, status='Backlog', priority='No priority', owner=None, start_on=None, target_on=None, progress=0, notes=None, targets=(), created_by=None, - completed_on=None, source_id=None, by=None, result_configs=()): + completed_on=None, source_id=None, by=None, result_configs=(), contributors=()): """Insert one project and its targets in one transaction. `source_id` is unique per tenant.""" values = _fields(dict(name=name, status=status, priority=priority, owner=owner, start_on=start_on, target_on=target_on, progress=progress, notes=notes, completed_on=completed_on, result_configs=result_configs)) backend = _backend(rt) + values['contributors'] = _member_values(contributors, backend.tenant_slug) stamp = _now() values.update(id=uuid.uuid4().hex, tenant=backend.tenant_slug, created_at=stamp, updated_at=stamp, created_by=_text(created_by or by, 'created_by', optional=True), @@ -275,6 +303,7 @@ def create(rt, *, name, status='Backlog', priority='No priority', owner=None, st record = backend.insert(values, _targets(targets), stamp) if by is not None: _log(rt, record['id'], by, 'project', None, record['name']) + _log_members(rt, record['id'], by, [], record['contributors']) return record @@ -298,11 +327,12 @@ def update(rt, project_id, changes, by=None): continue if before[field] != after[field]: _log(rt, project_id, by, name, before[field], after[field]) + _log_members(rt, project_id, by, before['contributors'], after['contributors']) return after def delete(rt, project_id, by=None): - """Delete one project and every target of it in one transaction.""" + """Delete one project, its targets and its contributors in one transaction.""" project_id = _text(project_id, 'project_id') backend = _backend(rt) name = backend.get(project_id)['name'] if by is not None else None @@ -558,6 +588,12 @@ def _log(rt, project_id, by, field, before, after): """Append one project event. A failed append is logged, never undoes the committed write.""" from core import activity_log try: + if field == 'dependencies': + from core import amazon_project_tasks + names = {task['id']: task['name'] for task in amazon_project_tasks.list_tasks(rt, project_id)} + before, after = [dict(value, from_name=names.get(value['from_task'], value['from_task']), + to_name=names.get(value['to_task'], value['to_task'])) + if isinstance(value, dict) else value for value in (before, after)] activity_log.append(rt, kind='app', object={'app': ACTIVITY_KEY, 'id': project_id}, by=_text(by, 'by'), changes={field: {'before': before, 'after': after}}) except Exception as exc: # noqa: BLE001 @@ -565,6 +601,16 @@ def _log(rt, project_id, by, field, before, after): file=sys.stderr) +def _log_members(rt, project_id, by, before, after): + """Append one event for each contributor username added or removed.""" + old = {member['username']: member for member in before} + new = {member['username']: member for member in after} + for username in sorted(old.keys() - new.keys()): + _log(rt, project_id, by, 'contributors', old[username], None) + for username in sorted(new.keys() - old.keys()): + _log(rt, project_id, by, 'contributors', None, new[username]) + + def _shown(value): """A target list as one line of text, else the value.""" if isinstance(value, list): @@ -572,30 +618,254 @@ def _shown(value): return ', '.join(v['metric'] for v in value) return ', '.join(f'{GRAIN_LABELS.get(t.get("grain"), t.get("grain"))} {t.get("key")}' for t in value if isinstance(t, dict)) + if isinstance(value, dict) and 'username' in value: + return value['name'] identity = owner_identity(value) return identity['name'] if identity else value -def activity(rt, project_id, limit=ACTIVITY_LIMIT): - """`[{at, by, action, field, from, to}]` for one project, newest first.""" - from core import activity_log - wanted = {'app': ACTIVITY_KEY, 'id': _text(project_id, 'project_id')} - page = activity_log.read(rt, can_read_object=lambda obj: obj == wanted, object=wanted, limit=limit) +def _activity_value(value, task_names): + """Show task and dependency names without exposing their stored record payloads.""" + if isinstance(value, dict): + task = value.get('task', value) + if isinstance(task, dict) and 'name' in task and 'username' not in task: + return task['name'] + if 'from_task' in value and 'to_task' in value: + return ' -> '.join(value.get(key.replace('_task', '_name')) or task_names.get(value[key], value[key]) + for key in ('from_task', 'to_task')) + return _shown(value) + + +TASK_ACTIVITY_FIELDS = ('name', 'assignee', 'start_on', 'due_on', 'done') +CLIPPED = ' [truncated]' + + +def _skip_space(text, position): + while position < len(text) and text[position].isspace(): + position += 1 + return position + + +def _clipped_members(preview): + """Read the members of a clipped JSON object that end before the cut, entering an open task.""" + decoder, out, position = json.JSONDecoder(), {}, 1 + while True: + try: + key, position = decoder.raw_decode(preview, _skip_space(preview, position)) + except ValueError: + return out + position = _skip_space(preview, position) + if not isinstance(key, str) or preview[position:position + 1] != ':': + return out + position = _skip_space(preview, position + 1) + try: + value, position = decoder.raw_decode(preview, position) + except ValueError: + # A reschedule logs {shifted, task}, so its task is often the member the cut opens. + return {'task': _clipped_members(preview[position:])} if ( + key == 'task' and preview[position:position + 1] == '{') else out + position = _skip_space(preview, position) + # A scalar that runs into the cut may itself be cut, so keep it only before a comma or brace. + if preview[position:position + 1] not in (',', '}'): + return out + out[key] = value + if preview[position] == '}': + return out + position += 1 + + +def _task_fields(value, truncated, actor_names): + """One task side as its person-readable fields, and whether the log clipped it.""" + clipped = bool(truncated) and isinstance(value, str) and value.endswith(CLIPPED) + if clipped: + preview = value[:-len(CLIPPED)] + value = _clipped_members(preview) if preview.startswith('{') else {} + if not isinstance(value, dict): + return {}, True + if 'name' not in value and isinstance(value.get('task'), dict): + value = value['task'] + fields = {key: value[key] for key in TASK_ACTIVITY_FIELDS + if key in value and (value[key] is None or isinstance(value[key], (str, bool)))} + if isinstance(fields.get('assignee'), str): + fields['assignee'] = actor_names.get(fields['assignee'], fields['assignee']) + return fields, clipped + + +def _task_change(before, after, truncated, actor_names): + """A changed task as its name and each readable field that differs, never its stored record.""" + (old, old_cut), (new, new_cut) = (_task_fields(side, truncated, actor_names) for side in (before, after)) + partial = old_cut or new_cut + changed = {key for key in TASK_ACTIVITY_FIELDS if (key in old and key in new and old[key] != new[key]) + or (not partial and (key in old) != (key in new))} + sides = [] + for fields, cut in ((old, old_cut), (new, new_cut)): + side = {key: value for key, value in fields.items() if key == 'name' or key in changed} + # The marker tells the client a clipped side compares only the fields both sides kept. + sides.append(json.dumps(side, ensure_ascii=False) + CLIPPED if cut else side) + return sides + + +def _activity_target_pair(change, truncated): + """Recover an absent side only from a bounded JSON target-list pair.""" + before, after = change.get('before'), change.get('after') + marker = ' [truncated]' + if truncated and all(isinstance(value, str) and value.endswith(marker) for value in (before, after)): + old, new = before[:-len(marker)], after[:-len(marker)] + if old == 'null' and new.startswith('['): + before = None + elif new == 'null' and old.startswith('['): + after = None + # Retain the preview marker because clipped arrays cannot establish a complete target list. + return before, after + + +def _activity_target_preview(value, truncated): + """Show complete target identities from a bounded JSON array without guessing its missing tail.""" + marker = ' [truncated]' + if not truncated or not isinstance(value, str) or not value.endswith(marker): + return value + preview = value[:-len(marker)] + if preview and not preview.startswith('['): + return value + decoder, targets, position = json.JSONDecoder(), [], 1 + while position < len(preview): + while position < len(preview) and preview[position].isspace(): + position += 1 + try: + target, position = decoder.raw_decode(preview, position) + except ValueError: + break + if (not isinstance(target, dict) or target.get('grain') not in GRAIN_LABELS + or not isinstance(target.get('key'), str) or not target['key']): + break + targets.append(target) + while position < len(preview) and preview[position].isspace(): + position += 1 + if position == len(preview) or preview[position] != ',': + break + position += 1 + shown = _shown(targets) + return f'{shown}. Target list incomplete' if shown else 'Target list incomplete' + + +def _activity_entries(event, actor_names, task_names): + """Project one log event into named activity entries.""" out = [] - for event in page['records']: - for field, change in (event.get('changes') or {}).items(): - before, after = change.get('before'), change.get('after') - if field == 'project': - action, field = ('created' if before is None else 'deleted'), None - elif field == 'targets': - action = 'added targets' if before is None else 'removed targets' - else: - action = 'changed' - out.append({'at': event['occurred_at'], 'by': event['by'], 'action': action, 'field': field, - 'from': _shown(before), 'to': _shown(after)}) + for index, (field, change) in enumerate((event.get('changes') or {}).items()): + before, after = change.get('before'), change.get('after') + if field == 'project': + action, field = ('created' if before is None else 'deleted'), None + elif field == 'targets': + before, after = _activity_target_pair(change, event.get('changes_truncated')) + action = ('added' if before is None else 'removed' if after is None else 'changed') + ' targets' + before = _activity_target_preview(before, event.get('changes_truncated')) + after = _activity_target_preview(after, event.get('changes_truncated')) + elif field == 'contributors': + action = 'added contributor' if before is None else 'removed contributor' + elif field in ('tasks', 'dependencies'): + action = ('added' if before is None else 'removed' if after is None else 'changed') + ' ' + ( + 'task' if field == 'tasks' else 'dependency') + else: + action = 'changed' + if action == 'changed task': + shown = _task_change(before, after, event.get('changes_truncated'), actor_names) + else: + shown = [_activity_value(value, task_names) for value in (before, after)] + out.append({'id': f'loopable:{event["id"]}:{index}', 'at': event['occurred_at'], + 'by': event['by'], 'byUsername': event['by'], + 'byName': actor_names.get(event['by'], event['by']), + 'action': action, 'field': field, 'from': shown[0], 'to': shown[1]}) return out +def activity_page(rt, project_id, *, cursor=None, limit=ACTIVITY_LIMIT, read_changes=None, + actor_names=None, task_names=None): + """Merge bounded log pages with an injected C4 reader under one snapshot cutoff.""" + from core import activity_log + if type(limit) is not int or not 1 <= limit <= ACTIVITY_LIMIT: + raise ValueError(f'limit must be between 1 and {ACTIVITY_LIMIT}.') + state = cursor or {'until': _now(), 'app': None, 'amazon': None} + if (not isinstance(state, dict) or set(state) != {'until', 'app', 'amazon'} + or not isinstance(state['until'], str)): + raise ValueError('Reload activity to continue.') + until = datetime.fromisoformat(state['until']).astimezone(timezone.utc).isoformat() + actor_names, task_names = actor_names or {}, task_names or {} + wanted = {'app': ACTIVITY_KEY, 'id': _text(project_id, 'project_id')} + + def app_entries(): + position = state['app'] + # Re-read the boundary event because one event can contain several fields. + before = [position[0], position[1] + 1] if position else None + while True: + page = activity_log.read(rt, can_read_object=lambda obj: obj == wanted, object=wanted, + cursor=before, until=until, limit=limit) + for event in page['records']: + for index, entry in enumerate(_activity_entries(event, actor_names, task_names)): + at = entry['at'] + next_position = [at, event['id'], index] + if position and at == position[0] and event['id'] == position[1] and index <= position[2]: + continue + yield ((at, 1, event['id'], -index), entry, 'app', next_position) + before = page['next_cursor'] + if before is None: + return + + def amazon_entries(): + position = state['amazon'] + skipped = 0 + + def candidates(): + nonlocal skipped + for change in read_changes(None) if read_changes is not None else (): + at = datetime.fromisoformat(change['detected_at']).astimezone(timezone.utc).isoformat() + if at > until: + continue + identity = hashlib.sha256(json.dumps(change, sort_keys=True, default=str).encode()).hexdigest() + key = (at, identity) + if position: + if key > tuple(position[:2]): + continue + if key == tuple(position[:2]) and skipped < position[2]: + skipped += 1 + continue + yield key, change + + # C4 has no page argument, so retain only this response's bounded candidate window. + selected = heapq.nlargest(limit + 1, candidates(), key=lambda item: item[0]) + previous, count = None, 0 + for (at, identity), change in selected: + key = (at, identity) + count = count + 1 if key == previous else ( + position[2] + 1 if position and key == tuple(position[:2]) else 1) + previous = key + attribute = change['attribute'] + field = LISTING_ACTIVITY_FIELDS.get(attribute, attribute.replace('_', ' ').capitalize()) + entry = {'id': f'amazon:{identity}:{count}', 'at': at, 'by': 'Amazon', + 'byUsername': 'amazon', 'byName': 'Amazon', 'source': 'amazon', + 'action': 'listing changed', 'field': field, + 'from': change['before_value'], 'to': change['after_value'], + 'attribute': change['attribute'], 'before': change['before_value'], + 'after': change['after_value'], 'detectedAt': change['detected_at'], + 'detectionSource': change['source'], 'sku': change['sku'], 'asin': change['asin'], + 'marketplaceId': change['marketplace_id']} + yield ((at, 0, identity, -count), entry, 'amazon', [at, identity, count]) + + merged = heapq.merge(app_entries(), amazon_entries(), key=lambda item: item[0], reverse=True) + rows = [] + next_state = dict(state) + for _key, entry, source, position in merged: + if len(rows) == limit: + return {'activity': rows, 'next_cursor': next_state} + rows.append(entry) + next_state[source] = position + return {'activity': rows, 'next_cursor': None} + + +def activity(rt, project_id, limit=ACTIVITY_LIMIT, **kwargs): + """Return the first merged activity page for legacy project detail consumers.""" + return activity_page(rt, project_id, limit=limit, **kwargs)['activity'] + + # --------------------------------------------------------------------------- the move (R39) def _loose_date(value): @@ -815,7 +1085,7 @@ class PgProjectStore: return con.execute('SELECT to_regclass(%s)', (self.t(PROJECTS),)).fetchone()[0] is not None def ensure_schema(self, con): - """Create both project tables and their indexes, only on a write.""" + """Create project tables and their indexes, only on a write.""" con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(PROJECTS)} ( id TEXT NOT NULL, tenant TEXT NOT NULL, name TEXT NOT NULL, status TEXT NOT NULL {STATUS_CHECK}, @@ -834,6 +1104,11 @@ class PgProjectStore: key TEXT NOT NULL, added_at TEXT NOT NULL, PRIMARY KEY (tenant, project_id, grain, key), FOREIGN KEY (tenant, project_id) REFERENCES {self.t(PROJECTS)} (tenant, id) ON DELETE CASCADE)''') + con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(MEMBERS)} ( + tenant TEXT NOT NULL, project_id TEXT NOT NULL, + username TEXT NOT NULL, name TEXT NOT NULL, added_at TEXT NOT NULL, + PRIMARY KEY (tenant, project_id, username), + FOREIGN KEY (tenant, project_id) REFERENCES {self.t(PROJECTS)} (tenant, id) ON DELETE CASCADE)''') # One row's lookup is one indexed read on this index. con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" ' f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)') @@ -847,19 +1122,26 @@ class PgProjectStore: self.ensure_schema(con) yield con - def _select_sql(self, where, prefix=''): + def _select_sql(self, where, prefix='', members='NULL'): cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS) count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c ' 'WHERE c.tenant = p.tenant AND c.project_id = p.id)') - return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()} ' + return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members} ' f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}') def _rollup_projection(self): # Old schemas read blank figures until a guarded write adds the columns. return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'" + def _members_projection(self, con): + if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None: + return 'NULL' + return ("(SELECT jsonb_agg(jsonb_build_object('username', m.username, 'name', m.name)) " + f'FROM {self.t(MEMBERS)} AS m WHERE m.tenant = p.tenant AND m.project_id = p.id)') + def _one(self, con, project_id, lock=False): - row = con.execute(self._select_sql(' AND p.id = %s') + (' FOR UPDATE OF p' if lock else ''), + row = con.execute(self._select_sql(' AND p.id = %s', members=self._members_projection(con)) + + (' FOR UPDATE OF p' if lock else ''), (self.tenant_slug, project_id)).fetchone() return _record(row) @@ -870,12 +1152,25 @@ class PgProjectStore: def _insert_targets(self, con, project_id, pairs, stamp): cols = ('tenant',) + TARGET_COLUMNS - for part in _chunks(pairs, 100): + for part in _chunks(pairs): con.execute(f'INSERT INTO {self.t(TARGETS)} ({", ".join(cols)}) VALUES ' + ', '.join(f'({_marks(len(cols))})' for _ in part) + ' ON CONFLICT (tenant, project_id, grain, key) DO NOTHING', tuple(v for grain, key in part for v in (self.tenant_slug, project_id, grain, key, stamp))) + def _replace_members(self, con, project_id, members, previous, stamp): + wanted = {member['username'] for member in members} + removed = sorted(member['username'] for member in previous if member['username'] not in wanted) + for part in _chunks(removed): + con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s ' + f'AND username IN ({_marks(len(part))})', (self.tenant_slug, project_id, *part)) + for part in _chunks(members, 100): + con.execute(f'INSERT INTO {self.t(MEMBERS)} (tenant, project_id, username, name, added_at) VALUES ' + + ', '.join(f'({_marks(5)})' for _ in part) + + ' ON CONFLICT (tenant, project_id, username) DO UPDATE SET name = EXCLUDED.name', + tuple(value for member in part for value in + (self.tenant_slug, project_id, member['username'], member['name'], stamp))) + def _touch(self, con, project_id, stamp): current = self._one(con, project_id, lock=True) impact = _with_result_configs(None, current['result_configs']) @@ -891,6 +1186,7 @@ class PgProjectStore: con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) ' f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values) self._insert_targets(con, record['id'], pairs, stamp) + self._replace_members(con, record['id'], record['contributors'], [], stamp) return self._one(con, record['id']) def get(self, project_id): @@ -903,7 +1199,9 @@ class PgProjectStore: with self._write() as con: current = self._one(con, project_id, lock=True) candidate = _fields(changes, previous=current) - fields = sorted(EDITABLE - {'result_configs'}) + if 'contributors' in changes: + candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug) + fields = sorted(EDITABLE - {'result_configs', 'contributors'}) changed_anchor = any(candidate[k] != current[k] for k in ('status', 'completed_on', 'result_configs')) impact = _with_result_configs(None if changed_anchor else current['impact'], candidate['result_configs']) con.execute(f'UPDATE {self.t(PROJECTS)} SET ' @@ -911,10 +1209,14 @@ class PgProjectStore: + ' WHERE tenant = %s AND id = %s', tuple(candidate[field] for field in fields) + (stamp, json.dumps(impact), self.tenant_slug, project_id)) + if 'contributors' in changes: + self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp) return self._one(con, project_id) def delete(self, project_id): with self._write() as con: + con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s', + (self.tenant_slug, project_id)) con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s', (self.tenant_slug, project_id)) row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id', @@ -927,7 +1229,9 @@ class PgProjectStore: with self._connection() as con: if not self._exists(con): return [] - rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id'), (self.tenant_slug,)).fetchall() + rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id', + members=self._members_projection(con)), + (self.tenant_slug,)).fetchall() return [_record(row) for row in rows] def targets(self, project_id): @@ -942,13 +1246,14 @@ class PgProjectStore: with self._connection() as con: if not self._exists(con): return out + members = self._members_projection(con) for part in _chunks(pairs): match = ' OR '.join(['(t.grain = %s AND t.key = %s)'] * len(part)) cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS) count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c ' 'WHERE c.tenant = p.tenant AND c.project_id = p.id)') rows = con.execute( - f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()} ' + f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members} ' f'FROM {self.t(TARGETS)} AS t ' f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id ' f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id', diff --git a/platform/core/audit.py b/platform/core/audit.py index 4b91bcb46158c131d553200caed25e29dad5a83a..37cb35dde8eaacefee19c9dd86eebd9782a7be83 100644 --- a/platform/core/audit.py +++ b/platform/core/audit.py @@ -13,7 +13,7 @@ from core import account_rows, freeze_notice, platform_admin AUDIT_KINDS = frozenset({ "auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled", "auth.mfa_failed", "auth.logout", "admin.grant", "credential.added", - "credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled", + "credential.rotated", "credential.revoked", "amazon.access", "amazon.sync", "amazon.throttled", "amazon.authorization_lost", "amazon.deletion_scheduled", "amazon.personal_read", "data.purge", "data.restore", "config.changed", "customer.personal_read", "external.query", diff --git a/platform/core/connector_spec.py b/platform/core/connector_spec.py index 079e1a303cb07648b5565fda42c30b726c526ac9..5efb4bf13e19700c7f5b5c2e4a9c6adf63002fa5 100644 --- a/platform/core/connector_spec.py +++ b/platform/core/connector_spec.py @@ -812,6 +812,18 @@ AMAZON_TABLES = ( identity=("metric_id",), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), + ConnectorTableSpec( + key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics", + source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin", + identity=("asin", "marketplace_id", "sentiment", "topic"), + links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), + ), + ConnectorTableSpec( + key=_AMZN + "review_trends", native="customerFeedback/2024-06-01/items/{asin}/reviews/trends", + source="amzn_review_trends", label="Amazon Listings: Review trends", pinned="asin", + identity=("asin", "marketplace_id", "sentiment", "topic", "month"), + links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), + ), ConnectorTableSpec( key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance", source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku", @@ -3238,6 +3250,53 @@ AMAZON_LABELS = { "parent_asin_source": "Parent ASIN source", "relationships": "Relationships", }, + _AMZN + "review_topics": { + "_data_origin": "Data origin", + "_report_document_id": "Report document ID", + "_report_row_number": "Report row", + "_report_type": "Report type", + "_snapshot_date": "Synced on", + "asin": "ASIN", + "marketplace_id": "Marketplace ID", + "sentiment": "Sentiment", + "topic": "Topic", + "item_name": "Product name", + "country_code": "Country", + "date_range_start_date": "Review period start", + "date_range_end_date": "Review period end", + "asin_metrics_number_of_mentions": "ASIN mentions", + "asin_metrics_occurrence_percentage": "ASIN occurrence", + "asin_metrics_star_rating_impact": "ASIN star rating impact", + "parent_asin_metrics_number_of_mentions": "Parent ASIN mentions", + "parent_asin_metrics_occurrence_percentage": "Parent ASIN occurrence", + "parent_asin_metrics_star_rating_impact": "Parent ASIN star rating impact", + "browse_node_metrics_occurrence_percentage_all_products": "Browse node occurrence", + "browse_node_metrics_star_rating_impact_all_products": "Browse node star rating impact", + "child_asin_metrics_most_mentions_asin": "Child ASIN with most mentions", + "child_asin_metrics_most_mentions_number_of_mentions": "Most child ASIN mentions", + "child_asin_metrics_least_mentions_asin": "Child ASIN with least mentions", + "child_asin_metrics_least_mentions_number_of_mentions": "Least child ASIN mentions", + "review_snippet_1": "Review snippet 1", + "review_snippet_2": "Review snippet 2", + "review_snippet_3": "Review snippet 3", + "review_snippets": "Review snippets", + "subtopics": "Subtopics", + }, + _AMZN + "review_trends": { + "_data_origin": "Data origin", + "_report_document_id": "Report document ID", + "_report_row_number": "Report row", + "_report_type": "Report type", + "_snapshot_date": "Synced on", + "asin": "ASIN", + "marketplace_id": "Marketplace ID", + "sentiment": "Sentiment", + "topic": "Topic", + "month": "Month", + "date_range_start_date": "Review period start", + "date_range_end_date": "Review period end", + "asin_metrics_occurrence_percentage": "ASIN occurrence", + }, _AMZN + "replenishment": { "_data_origin": "Data origin", "_report_document_id": "Report document ID", @@ -3404,6 +3463,12 @@ AMAZON_DEFAULT_COLUMNS = { _AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date", "active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue", "revenue_penetration", "subscriber_retention_for30_days", "program_type"), + _AMZN + "review_topics": ("sentiment", "topic", "asin_metrics_number_of_mentions", + "asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact", + "parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products", + "review_snippet_1", "review_snippet_2", "review_snippet_3"), + _AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage", + "date_range_start_date", "date_range_end_date"), _AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks", "cart_adds", "purchases", "backend_keywords", "source_basis"), _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank", @@ -3527,7 +3592,7 @@ AMAZON_GROUPS = { **{_AMZN + t: "Listings" for t in ( "listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings", "listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended", - "b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes")}, + "b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends")}, **{_AMZN + t: "Advertising" for t in ( "brand_analytics", "search_query_performance", "search_catalog_performance", "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance", diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py index 9bdb24135679a5e6083b09906f5b1396383f1e26..947192850271c25735a301380855545465237c70 100644 --- a/platform/core/external_tools.py +++ b/platform/core/external_tools.py @@ -140,6 +140,21 @@ _QUERY_FILTER = {"oneOf": [ _object({"link": _STRING, "where": _query_filter_node(allow_link=False)}, ("link", "where"), additionalProperties=False), ]} +_CLIENT_QUERY_LEAF = {"anyOf": [ + _QUERY_FILTER_NODE, + _object({"field": _STRING, "op": _FILTER_OP, "value": {}}, + ("field", "op"), additionalProperties=False), + _object({"field": _STRING, "operator": {"enum": ["="]}, "value": {}}, + ("field", "operator"), additionalProperties=False), +]} +_CLIENT_QUERY_NODES = {"type": "array", "minItems": 1, "maxItems": 50, + "items": _CLIENT_QUERY_LEAF} +_CLIENT_QUERY_FILTER = {"anyOf": [ + _QUERY_FILTER, _CLIENT_QUERY_LEAF, _CLIENT_QUERY_NODES, + *[_object({conj: _CLIENT_QUERY_NODES}, (conj,), additionalProperties=False) + for conj in ("and", "or")], + _object(additionalProperties={"type": ["string", "number", "boolean", "null"]}), +]} FILTER_NODE_REF = "#/$defs/filter_node" FILTER_NODE_EXAMPLE = {"conj": "and", "nodes": [ {"colId": "status", "op": "eq", "value": "open"}, @@ -606,7 +621,9 @@ TOOLS = { ("workbook_id", "key", "name", "description"), additionalProperties=False), _save_template, write=True), "list_databases": _tool("List the databases you can access or see in Loopable.", - _object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("ai_apps",)), + _object({"limit": {"type": "integer", "minimum": 1, + "description": "Optional client hint. The complete permitted database list is returned."}}, + additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("ai_apps",)), "describe_database": _tool("Read the fields you have permission to use in a database.", _object({"database": _STRING}, ("database",), additionalProperties=False), lambda ctx, args: ctx.describe_database(args["database"]), grants=("ai_apps",)), @@ -615,7 +632,7 @@ TOOLS = { _object({ "database": _STRING, "fields": {"type": "array", "items": _STRING, "maxItems": 200}, - "filters": _QUERY_FILTER, + "filters": _CLIENT_QUERY_FILTER, "sorts": {"type": "array", "maxItems": 8, "items": _object({ "field": _STRING, "direction": {"enum": ["asc", "desc"]}, }, ("field", "direction"), additionalProperties=False)}, @@ -633,6 +650,8 @@ TOOLS = { _object({"field": _STRING, "unit": {"enum": ["month"]}}, ("field",), additionalProperties=False), ]}}, + "group_by": {"type": "array", "minItems": 1, "maxItems": 4, + "items": _STRING, "description": "Client alias for groupBy. Send only one spelling."}, }, ("database",), additionalProperties=False), lambda ctx, args: ctx.query_database(args["database"], { key: value for key, value in args.items() if key != "database" @@ -786,7 +805,7 @@ def validate_query_filter_operators(filters): stack.extend(node[key] for key in ("nodes", "children", "where") if key in node) -def validate_arguments(arguments, schema): +def validate_arguments(arguments, schema, *, named=False): """Validate grouped arguments against the selected action's JSON schema.""" if isinstance(arguments, dict) and "filters" in schema.get("properties", {}): validate_query_filter_operators(arguments.get("filters")) @@ -850,13 +869,97 @@ def validate_arguments(arguments, schema): return False return True + def problems(value, rule, path="arguments"): + if matches(value, rule): + return [] + if not isinstance(rule, dict): + return [path] + for branch in ("anyOf", "oneOf"): + if branch in rule: + candidates = [problems(value, child, path) for child in rule[branch]] + return min(candidates, key=lambda found: ( + -max((item.count(".") + item.count("[") for item in found), default=0), len(found))) + if isinstance(value, dict) and rule.get("type") == "object": + properties = rule.get("properties", {}) + found = [path + "." + key for key in rule.get("required", ()) if key not in value] + for key, child in value.items(): + found.extend(problems(child, properties.get(key, rule.get("additionalProperties", True)), + path + "." + str(key))) + return found or [path] + if isinstance(value, list) and rule.get("type") == "array": + found = [item for index, child in enumerate(value) + for item in problems(child, rule.get("items", True), f"{path}[{index}]")] + return found or [path] + return [path] + if not isinstance(arguments, dict) or not matches(arguments, schema): - raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.") + detail = (" Invalid arguments: " + ", ".join(dict.fromkeys(problems(arguments, schema))) + "." + if named else "") + raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema." + detail) encoded = json.dumps(arguments, ensure_ascii=False, allow_nan=False) if any(prefix in encoded for prefix in ("lpb1.", "lpo1.", "lpr1.", "lpc1.")): raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.") +def table_read_arguments(name, arguments, schema=None): + """Admit supported client spellings before the existing scoped Table read.""" + schema = schema if schema is not None else TOOLS[name]["inputSchema"] + try: + try: + validate_arguments(arguments, schema, named=True) + except sheets.SheetRefusal as exc: + if name != "query_database" or exc.code != "tool_arguments_invalid": + raise + aggregate = schema["properties"]["aggregates"] + item = aggregate["items"] + # Preserve the semantic code only when aggregate member values cause the schema failure. + transport = {**schema, "properties": {**schema["properties"], "aggregates": { + **aggregate, "items": {**item, "properties": {key: {} for key in item["properties"]}}}}} + validate_arguments(arguments, transport, named=True) + accepted = ", ".join(item["properties"]["fn"]["enum"]) + raise sheets.SheetRefusal("query_aggregates_invalid", exc.message + " Use aggregates like " + '[{"fn": "count"}, {"fn": "sum", "field": ""}]. ' + "fn must be one of: " + accepted + ". When supplied, field must be a readable field key string.") from exc + args = copy.deepcopy(arguments) + if name != "query_database": + return args + if "group_by" in args: + if "groupBy" in args: + raise sheets.SheetRefusal("tool_arguments_invalid", "Send group_by or groupBy, not both.") + args["groupBy"] = args.pop("group_by") + filters = args.get("filters") + + def leaf(node): + validate_arguments({"filters": node}, _object({"filters": _CLIENT_QUERY_LEAF}), named=True) + if "field" not in node: + return node + return {"colId": node["field"], "op": "eq" if "operator" in node else node["op"], + **({"value": node["value"]} if "value" in node else {})} + + if isinstance(filters, list): + args["filters"] = [leaf(node) for node in filters] + elif isinstance(filters, dict): + if set(filters) in ({"and"}, {"or"}): + conj = next(iter(filters)) + args["filters"] = {"conj": conj, "nodes": [leaf(node) for node in filters[conj]]} + elif set(filters) & {"colId", "field", "op", "operator"}: + args["filters"] = {"conj": "and", "nodes": [leaf(filters)]} + elif not set(filters) & {"conj", "nodes", "link", "where", "children", "and", "or"}: + if not filters or len(filters) > 50: + raise sheets.SheetRefusal("tool_arguments_invalid", "filters must contain 1 to 50 field equality values.") + args["filters"] = {"conj": "and", "nodes": [ + {"colId": field, "op": "eq", "value": value} for field, value in filters.items()]} + validate_arguments({"filters": args["filters"]}, _object({"filters": _QUERY_FILTER}), named=True) + return args + except sheets.SheetRefusal as exc: + if exc.code != "tool_arguments_invalid": + raise + example = {"database": "", "filters": {"conj": "and", "nodes": [ + {"colId": "", "op": "eq", "value": ""}]}} if name == "query_database" else {} + shape = {"action": name, "arguments": example} + raise sheets.SheetRefusal(exc.code, exc.message + " Call table with " + json.dumps(shape) + ".") from exc + + def dispatch(ctx, name, arguments): """Dispatch one declared tool. The API translates core and grant refusals to MCP content.""" tool = TOOLS.get(name) if isinstance(name, str) else None @@ -872,6 +975,8 @@ def dispatch(ctx, name, arguments): else: ctx.session.require(grant) schema = tool["inputSchema"] + if name in {"query_database", "list_databases"}: + arguments = table_read_arguments(name, arguments, schema) if not isinstance(arguments, dict): raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.") missing = set(schema["required"]) - arguments.keys() diff --git a/platform/core/filter_eval.py b/platform/core/filter_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..5c71a12539f1f56878bfcb7914a59137cfe09072 --- /dev/null +++ b/platform/core/filter_eval.py @@ -0,0 +1,827 @@ +"""core/filter_eval.py — the filter tree evaluated against ROWS, in Python (wave 15, C-EVAL). + +THE THIRD ENGINE, and the one with a different job. There are now three readers of the same +filter tree and it is worth being precise about why: + + * `customer-grid/useVisibleRows.ts` — the CLIENT, over in-memory rows. The reference. + * `harness/filter_sql.py` — compiles the tree to SQL for the DuckDB semantic tables. + * **this module** — evaluates the tree against wire-shaped row dicts in the + API/host process, where there is no SQL and the rows are + already in hand. + +It exists because a PERMANENT (permission) filter has to be applied server-side, to rows the +server is about to send, on a surface whose columns are not a DuckDB table. Neither existing +engine can do that: the client one is TypeScript, and the SQL one needs a table. + +⛔ TWO ENTRY POINTS, OPPOSITE DEFAULTS, AND THE DIFFERENCE IS THE WHOLE POINT. + + matches(...) DISPLAY semantics. A faithful mirror of the client, including the places + where the client IGNORES a leaf. Refereed against the client by + `aios-web/verify_filter_parity.py` — if this disagrees with the browser about + one row, a saved view means two different things in two places. + + permits(...) PERMISSION semantics. Same evaluator, `strict=True`: anything the display + engine would IGNORE, this DENIES. + +Why they cannot be the same function (C-EVAL amendment 1, and the reason this module has a +docstring this long). The client engine is TRI-STATE — `True` / `False` / `None`, where `None` +means "this leaf is INACTIVE, parent ignores it entirely". A leaf naming a column that does not +exist returns `None` (`useVisibleRows.ts:318`), and a group whose children are all inactive is +itself inactive (:273). For a DISPLAY filter that is exactly right: a half-built condition must +not zero the grid, and a stale column reference must not either. + +For a PERMISSION filter it is a catastrophe. `agent is Tara` is the only thing standing between +a sales rep and the whole book; rename or delete the `agent` field and that leaf becomes +`None` — ignored — and the rep silently receives every customer. The wall would be one field +rename away from open, and nothing would go red. So `permits()` inverts every ignore into a +denial: an unanswerable permission filter is an OUTAGE, not a licence. + +The asymmetry is a PARAMETER (`strict`) rather than a second implementation, deliberately: a +mirror must copy the original's asymmetries, and the only way to be sure it still does next +month is for both behaviours to come out of the same walk. + +⚠ WHAT THIS MODULE DOES NOT RE-IMPLEMENT. Activeness (`is_rule_active`), the JS-`Number` +coercion (`to_num`), the op vocabularies, cohort-id parsing and the date-window/anchor resolvers +all already exist in `filter_sql.py` and `windows.py`, and are already held against the client by +`verify_filter_engine.py` (525 checks) and `verify_windows.py` (123). Importing them is not +laziness — a second copy of "what counts as an active rule" is precisely the drift those gates +exist to catch, and it would be a copy the gates do not cover. + +IMPORT EDGE. This module imports `harness.filter_sql` at module level. That is the interim +layering edge `("filter_eval.py", "harness.filter_sql")`, retired when `filter_sql` moves into +core (lane G amend). +""" +import json +import math +from collections import namedtuple + +from harness.filter_sql import ( # noqa: F401 (COHORT_OPS re-exported for callers) + COHORT_FIELD, COHORT_OPS, COHORT_OP_ALIASES, FILTER_OPS, MULTI_VALUE_OPS, RANK_OPS, + is_rule_active, parse_cohort_ids, parse_filter_values, to_num, + _resolve_anchor, _window_bounds, +) + +#: The CLIENT's numeric family, which is WIDER than `filter_sql.NUMERIC_TYPES`. `rating` (an +#: integer 1..max) and `formula` (computed in the browser and injected at the field's key) compare +#: as numbers here because this engine sees the same assembled row the client does. They are +#: absent from the SQL port for a good reason — they are client-only strata that can never reach a +#: DuckDB table — so the two lists differing is correct, not drift. Mirrors `isNumericType`. +NUMERIC_TYPES = frozenset({'currency', 'int', 'pct', 'rating', 'formula', 'autonumber'}) + +#: `created_time` is a DATETIME rendered read-only, and it filters as a DATE (truncated to the +#: day below). Mirrors `isDateType`. +DATE_TYPES = frozenset({'date', 'created_time', 'last_modified_time'}) + +#: The date ops that actually compare against an ANCHOR. Everything else falls to the +#: active-and-always-true default WITHOUT resolving one — ordering matters, see `_match_date`. +DATE_CMP_OPS = frozenset({'eq', 'neq', 'lt', 'lte', 'gt', 'gte'}) + + +class EvalCtx: + """Everything the engine needs besides the row and the columns. + + All four members share one property — each is an answer a single ROW cannot compute — and + therefore share one failure mode: absent, the condition matches NOTHING rather than + everything. That is the widening sin the cohort/measure/rank leaves are each written to + avoid, and it is why none of these defaults to a permissive value. + """ + + __slots__ = ('cohort_sets', 'measure_sets', 'rank_sets', 'today') + + def __init__(self, cohort_sets=None, measure_sets=None, rank_sets=None, today=None): + #: {cohort_id: set(pid)} — the cohorts THIS reader may see. + self.cohort_sets = cohort_sets or {} + #: {rule_id: set(pid)} — resolved measure conditions (CG-8). + self.measure_sets = measure_sets or {} + #: {id(leaf) or leaf-key: set(pid)} — resolved rank slices (C-OPS). + self.rank_sets = rank_sets or {} + #: The TENANT's today, ISO. Never the server clock at call time — a date filter that + #: means something different per process is not a filter. + self.today = today + + +_EMPTY_CTX = EvalCtx() + + +def _is_group(node): + return isinstance(node, dict) and isinstance(node.get('children'), list) + + +def _is_cohort(node): + return isinstance(node, dict) and node.get('colId') == COHORT_FIELD + + +def _is_measure(node): + """Mirrors `isMeasureRule`: a leaf carrying a `window` IS a measure condition.""" + return isinstance(node, dict) and node.get('window') is not None + + +def _rhs_col(rhs): + """Mirrors `rhsColId`: a STATISTIC names no column.""" + if not isinstance(rhs, dict) or rhs.get('kind') == 'stat': + return None + return rhs.get('colId') + + +def _as_text(v): + """The client's `String(raw ?? "").toLowerCase()`. `None` becomes '' (never 'None').""" + return ('' if v is None else str(v)).lower() + + +def _as_day(v): + """Day-granularity truncation. A `created_time` cell is a DATETIME and every comparand in + this vocabulary is a DATE — lexically the longer string sorts AFTER its own day, so without + this `on or before ` would silently exclude today's rows. Identity for a pure date.""" + return ('' if v is None else str(v))[:10] + + +def _cmp_num(op, a, b): + """`default: True` is the documented ACTIVE-and-always-true case — an operator that exists + but does not apply to this type (`contains` on a number) narrows nothing and asks nothing. + An operator in NO vocabulary never reaches here; `_eval` refuses it first.""" + if op == 'gt': + return a > b + if op == 'gte': + return a >= b + if op == 'lt': + return a < b + if op == 'lte': + return a <= b + if op == 'eq': + return a == b + if op == 'neq': + return a != b + return True + + +def _cmp_str(op, a, v): + if op == 'eq': + return a == v + if op == 'neq': + return a != v + if op == 'lt': + return a < v + if op == 'lte': + return a <= v + if op == 'gt': + return a > v + if op == 'gte': + return a >= v + return True + + +def _cmp_text(op, a, v): + if op == 'contains': + return v in a + if op == 'doesNotContain': + return v not in a + if op == 'eq': + return a == v + if op == 'neq': + return a != v + return True + + +def _as_choice_text(raw): + """The client's ``String(raw ?? "")`` over a CHOICE-SET cell, lowercased. + + ⚠ NOT ``_as_text``. A multi-select value can still arrive from the wire as a LIST (see + ``_is_blank``, which already carries that case), and Python's ``str(['a', 'b'])`` renders + ``"['a', 'b']"`` where JS's ``String(["a","b"])`` renders ``"a,b"`` — the comma-joined form + the storage layer uses and the one the membership test below has to see. + """ + if isinstance(raw, (list, tuple)): + return ','.join('' if x is None else str(x) for x in raw).lower() + return _as_text(raw) + + +def _cell_members(raw): + """The MEMBERS of one cell, mirroring `useVisibleRows.cellMembers`. + + A multi-select cell is a comma-joined set; a single-valued cell is a one-member set, so + ``hasAnyOf`` over a select column means exactly what ``isAnyOf`` does there. A BLANK cell has + NO members, which is the blank decision the three engines pin as a lock-step case. + """ + return [m for m in (part.strip() for part in _as_choice_text(raw).split(',')) if m] + + +def _match_multi_value(op, raw, values): + """⭐ W51-T07 — the five multi-value operators, COMPOSED from `_cmp_text`. + + ⛔ THIS FUNCTION IS THE POINT OF THE WHOLE AMENDMENT. This module imports ``FILTER_OPS`` from + ``filter_sql``, and its only vocabulary guard is ``op not in FILTER_OPS -> False``. The moment + the five ops joined that set, the guard began PASSING them straight into ``_cmp_text``, whose + tail returns True for an operator it does not know — so every row would have matched, in the + module that is BOTH the server-side display mirror (``matches``) and the PERMISSION wall + (``permits``). A bare ``return False`` would have been no better: the server would then + disagree with the browser about real rows, which is the drift this module exists to prevent. + + ``isAnyOf`` is the OR of ``_cmp_text('eq', ...)`` over the values, so it is the union of the + equivalent single-value leaves by construction rather than by testing. + """ + want = [v.strip().lower() for v in values] + # ⛔ NO VALUES = UNANSWERABLE = MATCHES NOTHING, uniformly. A SECOND layer: `_eval` already + # calls such a leaf inactive via `is_rule_active`. Written anyway because the vacuous cases + # go the WIDENING way — `all(... for v in [])` is True, so `hasAllOf []` would match every + # row, on a permission wall. `filter_sql` answers `1=0` for the identical input. + if not want: + return False + if op in ('isAnyOf', 'isNoneOf'): + a = _as_choice_text(raw) + hit = any(_cmp_text('eq', a, v) for v in want) + return hit if op == 'isAnyOf' else not hit + members = _cell_members(raw) + if op == 'hasAllOf': + return all(v in members for v in want) + hit = any(v in members for v in want) + return hit if op == 'hasAnyOf' else not hit + + +def _customer_tag_rules(nodes): + """Find every Customer type operand, including operands inside OR groups.""" + for node in nodes: + if _is_group(node): + yield from _customer_tag_rules(node['children']) + elif isinstance(node, dict) and node.get('colId') == 'customer_type': + yield node + + +def _match_customer_tags(rule, raw): + """Test exact JSON string-list membership, or return None when the rule is uncomputable.""" + if not isinstance(raw, str) or rule.get('rhs') is not None or _is_measure(rule): + return None + try: + tags = json.loads(raw) + except (ValueError, RecursionError): + return None + if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags): + return None + op = rule.get('op') + if op in ('isEmpty', 'isNotEmpty'): + return (not tags) if op == 'isEmpty' else bool(tags) + if op not in ('eq', 'neq', 'contains', 'doesNotContain') and op not in MULTI_VALUE_OPS: + return None + values = rule.get('values') if op in MULTI_VALUE_OPS else [rule.get('value')] + if (not isinstance(values, list) or not values + or any(not isinstance(value, str) or not value for value in values)): + return None + hits = [value in tags for value in values] + hit = all(hits) if op == 'hasAllOf' else any(hits) + return not hit if op in ('neq', 'doesNotContain', 'isNoneOf', 'hasNoneOf') else hit + + +def _is_blank(raw): + """The condition-builder blank contract: absent, empty choice set, or ``'(none)'``. + + Connector grids intentionally render a missing attribute as the visible ``'(none)'`` sentinel, + while multi-select wire values can arrive as a list before their comma-joined storage form. + Both are absence for ``isEmpty``; numeric 0 and false remain real values. + """ + if raw is None: + return True + if isinstance(raw, (list, tuple, set, frozenset)): + return not raw or all(_is_blank(value) for value in raw) + return isinstance(raw, str) and raw.strip() in ('', '(none)') + + +def _match_numeric(rule, raw, other, has_rhs): + a = to_num(raw) + op = rule.get('op') + if has_rhs: + return _cmp_num(op, a, to_num(other)) + if op == 'between': + v, v2 = rule.get('value'), rule.get('value2') + if v == '' or v2 is None or v2 == '': + return True # both bounds required + lo, hi = to_num(v), to_num(v2) + if lo > hi: + lo, hi = hi, lo + return lo <= a <= hi + if rule.get('value') == '': + return True # inactive until a value is typed + return _cmp_num(op, a, to_num(rule.get('value'))) + + +def _match_date(rule, raw, other, has_rhs, today): + a = _as_day(raw) + op = rule.get('op') + + # "is within [the past month]" — a RANGE test, and the only date op reading `dateWindow`. + if op == 'within': + if not today: + return False + bounds = _window_bounds(rule.get('dateWindow'), today) + if not bounds: + return False # unresolvable range -> matches nothing + if a == '': + # A blank date is not "within" any range. Without this, '' >= a null lower bound is + # vacuously true and every never-ordered customer lands inside every window. + return False + lo, hi = bounds + return (lo is None or a >= lo) and (hi is None or a <= hi) + + if has_rhs: + return _cmp_str(op, a, _as_day(other)) + if op == 'between': + v, v2 = rule.get('value'), rule.get('value2') + if v == '' or v2 is None or v2 == '': + return True + lo, hi = (v, v2) if v < v2 else (v2, v) + return lo <= a <= hi + + # An op that does not apply to a date (`contains`) is ACTIVE-and-always-true, decided BEFORE + # any anchor is resolved — there is no date in that question to resolve. Ordering this after + # the anchor would turn `contains` on a date from always-true into matches-nothing, which is + # a different answer under `or`. + if op not in DATE_CMP_OPS: + return True + if not rule.get('dateMode') and rule.get('value') == '': + return True # inactive until a value is typed + mode = rule.get('dateMode') or 'exact' + # `today` is required by a RELATIVE anchor and by nothing else, which is what keeps every + # pre-anchor saved view working in a payload that carries no today. + if mode != 'exact' and not today: + return False + v = _resolve_anchor(mode, rule.get('value'), today or '1970-01-01') + if v is None: + # Unresolvable (junk date, missing N) -> matches nothing. The honest answer to a + # question with no date in it. + return False + return _cmp_str(op, a, v) + + +def _match_filter(rule, row, columns, today): + """One LEAF against one row, dispatched on the field's declared type. + + `isEmpty`/`isNotEmpty` are handled FIRST and type-independently: they must never fall + through to the "blank value -> inactive" guards below, which would turn them into + always-true no-ops. + """ + field = columns.get(rule.get('colId')) or {} + ftype = field.get('type') + raw = row.get(rule.get('colId')) + op = rule.get('op') + + if op == 'isEmpty': + return _is_blank(raw) + if op == 'isNotEmpty': + return not _is_blank(raw) + + # A RANK op can never be answered from one row. `_eval` routes rank leaves to their resolved + # set and one should never arrive here; this is the SECOND layer, and it exists because of + # what the first layer's absence would do — `_cmp_num`'s `default: True` would make `topN` + # match EVERY row, a filter that widens under a count that agrees with it. + if op in RANK_OPS: + return False + + # ⭐ W51-T07 — the MULTI-VALUE family, ON THE OP and before the type dispatch. Every branch + # below ends in a blank-value short-circuit and these leaves carry a blank `value`; one + # fall-through and a permission wall reading `Agent is any of [...]` admits every row. + if op in MULTI_VALUE_OPS: + if ftype in DATE_TYPES: + return False + values = parse_filter_values(rule.get('values')) + if ftype in NUMERIC_TYPES: + if op not in ('isAnyOf', 'isNoneOf') or not values: + return False + number = to_num(raw) + hit = any(_cmp_num('eq', number, to_num(value)) for value in values) + return hit if op == 'isAnyOf' else not hit + return _match_multi_value(op, raw, values) + + rhs = rule.get('rhs') + has_rhs = rhs is not None + other = row.get(_rhs_col(rhs) or '') if has_rhs else None + + if ftype in NUMERIC_TYPES: + return _match_numeric(rule, raw, other, has_rhs) + if ftype in DATE_TYPES: + return _match_date(rule, raw, other, has_rhs, today) + + a = _as_text(raw) + if has_rhs: + return _cmp_text(op, a, _as_text(other)) + if rule.get('value') == '': + return True + return _cmp_text(op, a, _as_text(rule.get('value'))) + + +def _eval(node, row, columns, ctx, strict): + """One node against one row. `True` / `False` / `None`, where `None` = INACTIVE. + + `strict` is the permission mode: every `None` this function would return becomes `False` + instead. Read the two behaviours side by side rather than trusting either alone — the + difference between them IS the security property (see the module docstring). + """ + ignore = False if strict else None + + if _is_group(node): + saw_active = False + all_true = True + any_true = False + for child in node['children']: + v = _eval(child, row, columns, ctx, strict) + if v is None: + continue # inactive -> ignored entirely + saw_active = True + all_true = all_true and v + any_true = any_true or v + if not saw_active: + # A group with no active children is itself inactive. In strict mode an empty or + # wholly-unanswerable permission group DENIES: a wall that evaluates to "nothing to + # say" must not mean "let them through". + return ignore + return any_true if node.get('conj') == 'or' else all_true + + if not isinstance(node, dict): + return ignore + + # A COHORT leaf, BEFORE the column lookup: `__cohort__` is not a column, so the lookup below + # would call it INACTIVE — and inactive means every row on screen under a count nobody doubts. + if _is_cohort(node): + if not is_rule_active(node, columns): + return ignore + ids = parse_cohort_ids(node.get('value')) + sets = [ctx.cohort_sets.get(cid) for cid in ids] + # ANY named cohort we were given no membership for makes the WHOLE condition + # unanswerable, and unanswerable matches nothing — for `is none of` as well. Answering + # "everything" to `is none of [a cohort we cannot see]` is the widening sin wearing a + # plausible face, and PARTIAL evaluation is worse: `is none of [A, B]` quietly becoming + # `is none of [A]` would show every row in B under a count nobody would doubt. + if any(s is None for s in sets): + return False + pid = row.get('pid') + op = node.get('op') + op = op if op in COHORT_OPS else COHORT_OP_ALIASES.get(op, 'anyOf') + if op == 'allOf': + return all(pid in s for s in sets) + in_any = any(pid in s for s in sets) + return (not in_any) if op == 'noneOf' else in_any + + # A MEASURE leaf, also before the column lookup and for the same reason: a measure's colId + # ("revenue") is not a column of this table. An ACTIVE rule whose set has not been resolved + # is PENDING and matches NOTHING — never everything. + if _is_measure(node): + if not is_rule_active(node, columns): + return ignore + s = ctx.measure_sets.get(node.get('id')) if node.get('id') else None + if s is None: + return False + return row.get('pid') in s + + if strict and node.get('colId') == 'customer_type': + return ('customer_type' in columns + and _match_customer_tags(node, row.get('customer_type')) is True) + + # ── a column leaf ──────────────────────────────────────────────────────────────────────── + if node.get('colId') not in columns: + # THE LEAF THIS WHOLE MODULE'S `strict` FLAG EXISTS FOR. Display: ignore it. Permission: + # deny — see the module docstring's field-rename walkthrough. + return ignore + if node.get('rhs') is not None and _rhs_col(node['rhs']) not in columns: + # An rhs naming a column this table does not have is ignored by the SAME rule as an + # unknown LEFT column. Treating one side as "ignore" and the other as "match nothing" + # would make `a > deleted` behave differently from `deleted > a`. + return ignore + if not is_rule_active(node, columns): + return ignore + + # A RANK leaf is a MEMBERSHIP test, like a cohort or a measure: the set was computed in a + # prior pass over the other rows. No set = REFUSE, never "ignore" and never "true" — both of + # those SHOW rows. + if node.get('op') in RANK_OPS: + # ``_rank_key`` gained an explicit scope in Wave 43. Accept the old four-part key + # for callers which resolved their own set before that addition; a missing answer is + # still a refusal, never a permissive fallback. + s = ctx.rank_sets.get(_rank_key(node)) + if s is None: + s = ctx.rank_sets.get(_legacy_rank_key(node)) + return (row.get('pid') in s) if s is not None else False + + # An operator in NO vocabulary at all. The comparators end in `default: True`, right for an + # operator that exists but does not apply to this type; NOT right for one that exists + # nowhere (`inTertile` out of a stale saved view), which would match every row. + if node.get('op') not in FILTER_OPS: + return False + + return _match_filter(node, row, columns, ctx.today) + + +def _rank_key(node): + """The client keys `rankSets` by leaf IDENTITY (a Map keyed on the node object). Python + dicts cannot key on a dict, so the caller that resolves rank sets and this evaluator agree + on a derived key. Kept in ONE place so the two cannot drift apart.""" + return (_legacy_rank_key(node) + (_rank_scope_token(node),)) + + +def _legacy_rank_key(node): + """The Wave-42 four-part resolver key, retained for supplied membership sets.""" + return (node.get('id') or '', node.get('colId') or '', node.get('op') or '', + str(node.get('value') if node.get('value') is not None else '')) + + +def _rank_scope_token(node): + """A stable, serialisable identity for one rank leaf's scope configuration. + + ``rankScope`` is deliberately optional: old views keep their global meaning. A grouped + rank may name its group field explicitly (``{mode:'group', field:'agent'}``) or defer to + the view's ``groupBy`` through the concise ``'group'`` shape. The resolver below refuses + the latter when no group field was supplied, rather than silently widening it to global. + """ + raw = node.get('rankScope') if isinstance(node, dict) else None + if isinstance(raw, dict): + mode = raw.get('mode', raw.get('scope')) + field = raw.get('field', raw.get('groupBy')) + if mode == 'group': + return 'group:' + (str(field).strip()[:80] if field is not None else '') + return 'global' + return 'group:' if raw == 'group' else 'global' + + +def _rank_scope(node, group_by): + """``('global' | 'group', field | None)`` for a cleaned or directly-posted rank leaf.""" + raw = node.get('rankScope') if isinstance(node, dict) else None + if raw == 'group': + field = group_by + elif isinstance(raw, dict) and raw.get('mode', raw.get('scope')) == 'group': + field = raw.get('field', raw.get('groupBy', group_by)) + else: + return 'global', None + field = str(field or '').strip()[:80] + return ('group', field or None) + + +def _rank_leaves(nodes, columns): + """ACTIVE rank leaves in tree order; measure leaves are not rank questions.""" + out = [] + for node in nodes or (): + if _is_group(node): + out.extend(_rank_leaves(node.get('children'), columns)) + elif (isinstance(node, dict) and node.get('op') in RANK_OPS + and not _is_measure(node) and is_rule_active(node, columns)): + out.append(node) + return out + + +def _strip_rank_leaves(nodes): + """The residue that defines the ranking domain (the client performs the same pass).""" + out = [] + for node in nodes or (): + if _is_group(node): + children = _strip_rank_leaves(node.get('children')) + if children: + out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', + 'children': children}) + elif not (isinstance(node, dict) and node.get('op') in RANK_OPS + and not _is_measure(node)): + out.append(node) + return out + + +def _rank_number(raw): + """A rank value, where blank/unparseable is ABSENT rather than numeric zero.""" + if raw is None or raw == '': + return None + if isinstance(raw, bool): + return float(raw) # JavaScript ``Number(true)`` parity. + try: + value = float(raw) + except (TypeError, ValueError): + return None + return value if math.isfinite(value) else None + + +def _rank_bound(value, lo, hi): + """The exact whole-number bounds the rank builder writes; junk remains unanswerable.""" + try: + number = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(number) or int(number) != number: + return None + number = int(number) + return number if lo <= number <= hi else None + + +def _pid_order(pid): + """Stable ascending pid order even for mixed legacy string/integer wire ids.""" + try: + return 0, int(pid) + except (TypeError, ValueError): + return 1, str(pid) + + +def _group_value(row, key): + """A hashable, deterministic grouping value; list/dict cells remain one visible group.""" + value = row.get(key) + if isinstance(value, (list, dict, set)): + return repr(value) + try: + hash(value) + except TypeError: + return repr(value) + return value + + +RankResolution = namedtuple('RankResolution', 'sets unanswerable') + + +def resolve_rank_sets(tree, rows, fields, ctx=None, group_by=None): + """Resolve top/bottom/percentile/average slices over the server's filtered row domain. + + Rank filters cannot be compiled as a row-level SQL ``WHERE`` clause, but they *can* and must + be answered before an in-memory API/host payload is filtered. The residue (all non-rank + conditions) defines the domain, ties are selected by ascending ``pid`` in both directions, + and ``rankScope: 'group'`` independently ranks every ``group_by`` bucket. A grouped request + without an actual group key is unanswerable and gets an empty membership set; it never falls + back to a global rank. + """ + columns = _columns_map(fields) + nodes, conj = tree_parts(tree) + leaves = _rank_leaves(nodes, columns) + if not leaves: + return RankResolution({}, 0) + + base_ctx = ctx or _EMPTY_CTX + residue = _strip_rank_leaves(nodes) + residue_tree = {'nodes': residue, 'conj': conj} + domain = [row for row in (rows or ()) + if matches(residue_tree, row, columns, base_ctx)] + sets, unanswerable = {}, 0 + + for leaf in leaves: + key = _rank_key(leaf) + field = columns.get(leaf.get('colId')) or {} + if field.get('type') not in NUMERIC_TYPES: + sets[key] = set() + unanswerable += 1 + continue + scope, scoped_by = _rank_scope(leaf, group_by) + if scope == 'group' and scoped_by not in columns: + sets[key] = set() + unanswerable += 1 + continue + groups = {} + for row in domain: + group_key = _group_value(row, scoped_by) if scope == 'group' else None + groups.setdefault(group_key, []).append(row) + + selected, answered = set(), False + for group_rows in groups.values(): + ranked = [] + for row in group_rows: + value = _rank_number(row.get(leaf.get('colId'))) + if value is not None: + ranked.append((value, row.get('pid'))) + if not ranked: + continue + answered = True + asc = sorted(ranked, key=lambda item: (item[0], _pid_order(item[1]))) + desc = sorted(ranked, key=lambda item: (-item[0], _pid_order(item[1]))) + count = len(asc) + op = leaf.get('op') + keep = None + if op == 'topN': + bound = _rank_bound(leaf.get('value'), 1, 10000) + keep = desc[:bound] if bound is not None else None + elif op == 'bottomN': + bound = _rank_bound(leaf.get('value'), 1, 10000) + keep = asc[:bound] if bound is not None else None + elif op in ('inTopPct', 'inBottomPct'): + pct = _rank_bound(leaf.get('value'), 1, 100) + take = min(count, max(1, math.ceil(count * pct / 100))) if pct else None + order = desc if op == 'inTopPct' else asc + keep = order[:take] if take is not None else None + elif op in ('aboveAvg', 'belowAvg'): + mean = sum(value for value, _ in asc) / count + keep = [(value, pid) for value, pid in asc + if (value > mean if op == 'aboveAvg' else value < mean)] + elif op in ('inQuartile', 'inDecile'): + buckets = 4 if op == 'inQuartile' else 10 + wanted = _rank_bound(leaf.get('value'), 1, buckets) + if wanted is not None and count >= buckets: + keep = [item for index, item in enumerate(asc) + if min(buckets, (index * buckets) // count + 1) == wanted] + if keep is None: + selected = set() + answered = False + break + selected.update(pid for _, pid in keep) + sets[key] = selected + if domain and not answered: + unanswerable += 1 + return RankResolution(sets, unanswerable) + + +def _columns_map(fields): + """`[{key,type,…}]` -> `{key: field}`. Accepts a mapping unchanged, so callers holding + either shape need no adapter.""" + if isinstance(fields, dict): + return fields + out = {} + for f in fields or (): + if isinstance(f, dict) and f.get('key'): + out[f['key']] = f + return out + + +def evaluate(tree, row, fields, ctx=None): + """The TRI-STATE the client actually has: `True` / `False` / `None`, where `None` means the + tree is INACTIVE — it asks nothing, so it narrows nothing. + + Exposed because collapsing `None` into `True` (which `matches` does, correctly, for its own + purpose) destroys a distinction some callers must keep. The `memberPids` union is the one + that bit: the client's rule is `members.has(pid) || matchFilterTree(…)` and that union sits + INSIDE `if (filters.length)`, so an INACTIVE tree must show every row and ignore the member + list — whereas an ACTIVE tree that merely fails must still admit hand-picked members. A + caller holding only `matches` cannot tell those two apart and will narrow an inactive view + down to its hand-picked rows. (`filter_sql.compile_filter_tree` keeps the same distinction + by returning None rather than a predicate; this is the row-engine's equivalent.) + + Activeness is ROW-INDEPENDENT — every `None` here comes from `is_rule_active` or an empty + group, none of which reads the row — so a caller may compute it once per tree if it wishes. + """ + columns = _columns_map(fields) + nodes, conj = tree_parts(tree) + return _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, False) + + +def matches(tree, row, fields, ctx=None): + """DISPLAY semantics — does `row` survive this filter tree? Mirrors the client exactly. + + A wholly INACTIVE tree returns True: no narrowing, every row shows. That is what the browser + does with a half-built condition, and this function's whole job is to agree with the browser. + Use `evaluate()` instead when you need to tell "inactive" from "matched" (see its docstring). + """ + v = evaluate(tree, row, fields, ctx) + return True if v is None else bool(v) + + +def visible_pids(tree, rows, fields, ctx=None, member_pids=None, group_by=None): + """The client's whole row pipeline for one tree, member union included — the shape a caller + should reach for instead of assembling `matches` in a loop. + + Mirrors `useVisibleRows`: an INACTIVE tree yields every row (and IGNORES `member_pids`, + because the union lives inside `if (filters.length)`); an ACTIVE tree yields the rows it + matches UNION the hand-picked members. + """ + ctx = ctx or _EMPTY_CTX + # This is the whole-payload entry point, so it is the one place a server can answer a rank + # filter without trusting a browser to have done a prior pass. Preserve supplied cohort / + # measure answers, but replace rank memberships with this payload's authoritative answer. + # Direct ``evaluate`` intentionally stays a one-row primitive and therefore stays + # fail-closed when a caller has not supplied a membership set. + resolved = resolve_rank_sets(tree, rows, fields, ctx, group_by=group_by) + if resolved.sets: + rank_sets = dict(ctx.rank_sets) + rank_sets.update(resolved.sets) + ctx = EvalCtx(ctx.cohort_sets, ctx.measure_sets, rank_sets, ctx.today) + members = set(member_pids or ()) + out, active = [], False + verdicts = [] + for r in rows: + v = evaluate(tree, r, fields, ctx) + if v is not None: + active = True + verdicts.append((r, v)) + for r, v in verdicts: + if not active: + out.append(r.get('pid')) + elif v or (members and r.get('pid') in members): + out.append(r.get('pid')) + return out + + +def permits(tree, row, fields, ctx=None): + """PERMISSION semantics — may this reader SEE `row` under this permanent filter? + + Same evaluator, every ignore inverted into a denial. An empty/absent tree means "no + restriction declared" and admits (the caller decides whether a module with no entry is + reachable at all — that is `perm_scope`'s question, not this one); anything PRESENT but + unanswerable DENIES. + """ + nodes, conj = tree_parts(tree) + if not nodes: + return True # nothing declared -> nothing to enforce here + columns = _columns_map(fields) + tag_rules = list(_customer_tag_rules(nodes)) + # An uncomputable tag operand denies the whole wall, including an OR branch. + if tag_rules and ('customer_type' not in columns or any( + _match_customer_tags(rule, row.get('customer_type')) is None for rule in tag_rules)): + return False + v = _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, True) + return bool(v) + + +def tree_parts(tree): + """Read C-PERM amendment 2's `FilterTree` — `{conj?, nodes}` — into `(nodes, conj)`. + + Accepts a BARE LIST too, because that is what `ViewConfig.filters` is and what + `clean_filter_tree` returns; an absent conj is `'and'`, mirroring `ViewConfig.filterConj`. + One reader, so the absent-conj default lives here instead of at every call site. + """ + if isinstance(tree, dict): + nodes = tree.get('nodes') + conj = 'or' if tree.get('conj') == 'or' else 'and' + return (nodes if isinstance(nodes, list) else []), conj + return (tree if isinstance(tree, list) else []), 'and' diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 3cb2a87a01616f912ad81c7a3bdbfb4054187f6d..535faf32905cda50a8c811a8db395856bf9f88c9 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -39,7 +39,9 @@ down, and this note is the marker for that work. """ import datetime as dt import json as _json_ev # wave 23 (C7): the json field's parse wall +import re from dataclasses import dataclass, field +from contextlib import contextmanager, nullcontext from functools import wraps # ⚠ ALIASED ON PURPOSE. `field` is imported above for the dataclasses that need it, and this file # uses `field` as a LOCAL VARIABLE in a dozen handlers (`field = {'key': key, …}`). A @@ -136,6 +138,87 @@ class EventResult: write_id: str = '' +class AdmissionSnapshot: + """Lend one request's workspace and grants to admission, never to a mutation callback.""" + + def __init__(self, runtime, uname, table_key, *, schema_only=False): + self._runtime, self.uname, self.table_key = runtime, uname, table_key + self.schema_only = bool(schema_only) + self._documents = {} + self._grid_admission = self + + def __getattr__(self, name): + return getattr(self._runtime, name) + + def _holds(self, name): + return name in (self.table_key, 'object_shares') + + def invalidate(self, *names): + """Discard admission documents after canonical writes or fresh reads.""" + for name in names or tuple(self._documents): + self._documents.pop(name, None) + + def _document(self, name): + from copy import deepcopy + if name not in self._documents: + if self.schema_only and name == self.table_key: + value = self._runtime.get_projection(name, drop=('overlays',)) + else: + read = getattr(self._runtime, '_read_strict', self._runtime.get) + value = read(name) + self._documents[name] = value or {} + return deepcopy(self._documents[name]) + + def get(self, name, fresh=False): + if fresh: + self.invalidate(name) + if self._holds(name) and not fresh: + return self._document(name) + return self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name) + + def _read_strict(self, name): + if self._holds(name): + return self._document(name) + return getattr(self._runtime, '_read_strict', self._runtime.get)(name) + + def get_entry(self, name, entry): + if self._holds(name): + return self._document(name).get(str(entry)) + return self._runtime.get_entry(name, entry) + + def get_projection(self, name, drop=()): + if self._holds(name): + from core.store import _project + return _project(self._document(name), frozenset(drop)) + return self._runtime.get_projection(name, drop=drop) + + def update(self, name, *args, **kwargs): + try: + return self._runtime.update(name, *args, **kwargs) + finally: + self.invalidate(name) + + def put(self, name, *args, **kwargs): + try: + return self._runtime.put(name, *args, **kwargs) + finally: + self.invalidate(name) + + +def request_admission(session, table_key, *, schema_only=False): + """Bind a detached host session to a fresh tenant, user and table admission snapshot.""" + from copy import copy + actor = copy(session) + runtime = session.runtime + if not (isinstance(runtime, AdmissionSnapshot) and runtime.uname == session.uname + and runtime.table_key == table_key and runtime.schema_only == bool(schema_only)): + while isinstance(runtime, AdmissionSnapshot): + runtime = runtime._runtime + runtime = AdmissionSnapshot(runtime, session.uname, table_key, schema_only=schema_only) + actor.runtime = runtime + return actor + + @dataclass class EventCtx: """Everything the handler is allowed to know, and nothing about how it is being served. @@ -176,6 +259,7 @@ class EventCtx: # Core can normalise a recurrence's finite shape, but it cannot decide whether a named # same/linked field is visible to this caller without reaching upward into the API layer. recurrence_validator: Optional[Callable[[object], Optional[dict]]] = None + record_admitter: Optional[Callable[[int], bool]] = None # Value-history restore reuses the ordinary cell wall. This stamp changes only the audit # event kind and admits ``None`` as "remove the prior cell" after every normal field/row # permission check has passed; browser-authored edits never set it. @@ -207,6 +291,7 @@ class EventCtx: durable_receipts: bool = False #: The route's assembly workspace, lent to the first event only (`handle_one`). workspace_lend: Optional[dict] = None + admission: Optional[AdmissionSnapshot] = None def __post_init__(self): # A caller that forgets `seen_ids` gets per-call dedup rather than a crash — the SET is @@ -238,6 +323,30 @@ def _store_of(ctx): return inner if inner is not None else store +def _admission_ctx(ctx): + """Copy only admission readers while keeping the mutation context on its canonical store.""" + snapshot = getattr(ctx, 'admission', None) + if snapshot is None: + return ctx + from copy import copy + from dataclasses import replace + st = _store_of(ctx) + from core.edit_receipts import CheckedStore + if isinstance(st, CheckedStore): + st = st.st + while hasattr(st, '_runtime'): + st = st._runtime + source = snapshot._runtime + while hasattr(source, '_runtime'): + source = source._runtime + if (st is not source or snapshot.uname != ctx.uname + or snapshot.table_key != _tops(ctx).table_key): + raise ValueError('admission snapshot does not match this tenant, user and table') + table = copy(_tops(ctx)) + table._st = snapshot + return replace(ctx, st=snapshot, table=table, admission=None) + + def _blobs(ctx): """The BYTE store for this ctx — `_store_of`'s sibling, split out because it is NOT yet equivalent and pretending otherwise would hide a live residency hole. @@ -279,48 +388,6 @@ def _ag_overview_id(): EVENT_ORIGIN = {'client': 'grid-events', 'ip': None} -def _event_rev(event): - """THE STORE REVISION THIS BROWSER DECLARED IT WAS LOOKING AT, or None if it declared none. - - ⭐⭐ W51-T33 — THE ONE LINK THAT WAS MISSING, AND IT IS THE WHOLE TICKET. - `CustomerGrid.tsx::stampViewWrite` has put `rev` on every view write since W50-T28, - `table_store._audited_update` has enforced it on both backends since W50-T25, and - `main.py::_view_revision_stale` has answered `409 view_revision_stale {rev_current}` for any - door that raises — while THIS channel, the one the browser actually writes through, read the - stamp nowhere and passed no `rev=` at all. Every layer was green and the guard was inert - ([[a-rev-guard-inert-on-the-events-channel]]): two people editing one view still produced two - 200s and the first edit still vanished. Reading the value here is the only thing that was - missing, which is why this helper is four lines and not a subsystem. - - ⛔ THE EXCEPTION IS NOT CAUGHT ANYWHERE BELOW, DELIBERATELY. `ViewRevisionStale` is a - `RuntimeError` precisely so the `except Exception` fallbacks dotted through these handlers - cannot swallow it (`view_audit.py`'s own class docstring says so), and `main.py:232`'s - handler names THIS channel as the door it exists for: *"the grid events channel above all, - which raises straight out of `table_store` with nothing between it and the client"*. Catching - it here to convert it into a toast would make a shipped, documented handler unreachable and - answer 200 for a write that was refused. - - ⛔ ABSENT IS None, AND None IS ACCEPTED — the pre-C3 behaviour, unchanged. A write that - declares no revision is simply not asked the question, which is what keeps every unstamped - door (and every caller that predates the stamp) writing exactly as it does today. - - ⚠ ANYTHING UNREADABLE AS AN INTEGER IS TREATED AS ABSENT RATHER THAN AS ZERO, and that is - not tidiness. `store_pg.revision()` reports a MISSING key as rev 0, so coercing junk to 0 - would turn a malformed stamp into a confident claim about an empty bucket — the one value - most likely to disagree with the store and refuse a write nobody was racing. - ⚠ `bool` is an `int` in Python, so `True` would arrive as revision 1. Refused by name. - """ - if not isinstance(event, dict): - return None - raw = event.get('rev') - if raw is None or isinstance(raw, bool): - return None - try: - return int(raw) - except (TypeError, ValueError): - return None - - def _tops(ctx): """The table-ops object this ctx operates on — the customer table unless the caller chose another topic. ONE resolution point so 'which bucket does this write land in' has exactly @@ -467,7 +534,7 @@ def _field_share_role(ctx, key, definition): _storage_key, topic = _field_share_keys(ctx) return _shares.role_for( 'field', _shares.field_oid(topic, key), ctx.uname, - is_admin=ctx.admin, st=ctx.st or getattr(ctx.table, 'st', None)) + is_admin=ctx.admin, st=_store_of(_admission_ctx(ctx))) except Exception: return None @@ -734,7 +801,8 @@ def _drop_dead_application_refs(ctx, views): def workspace_corrections(st, table_key, uname, assembly): """Only the workspace render wire spends pending acknowledgements, after the memo read.""" - if not assembly.get('_pending_corrections'): + # Only the memoised path sets the flag. An absent flag is unknown, so read the workspace. + if assembly.get('_pending_corrections') is False: return assembly from core.table_store import TableStore fields = TableStore(table_key, st=st).workspace(uname, consume_corrections=True)['fields'] @@ -753,7 +821,7 @@ def workspace_corrections(st, table_key, uname, assembly): def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False, - *, include_views=True): + *, include_views=True, admission=True): """Durable Airtable-style view/schema/overlay state, with an honest session fallback. Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED @@ -778,6 +846,8 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids` stays an EXPLICIT argument because the folder leg deliberately omits it. """ + original_ctx = ctx + ctx = _admission_ctx(ctx) if admission else ctx uname = ctx.uname if _store_of(ctx).available(): # Wave 48 C3: this is the only runtime transition door. It either promotes every @@ -788,6 +858,7 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do include_document=bool(capture_document)) if capture_document: ws, ctx.workspace_document = read + original_ctx.workspace_document = ctx.workspace_document else: ws = read if not include_views: @@ -939,6 +1010,295 @@ def _clean_secondary_view(raw): } +_SERVER_VIEW_MEMBERS = frozenset({ + 'cohortLock', 'createdBy', 'updatedAt', 'filterRemovals', 'kind', +}) +_VIEW_POSITION = re.compile(r'[0-9A-Za-z]{1,64}\Z') + + +@dataclass +class _ViewMemberCtx: + """Lend create-time filter accounting without changing the request context.""" + source: EventCtx + filter_removals: list = field(default_factory=list) + filter_budget: Optional[list] = None + + def __getattr__(self, name): + return getattr(self.source, name) + + +def _view_member_refusal(member, code, message): + return None, {'member': member, 'code': code, 'message': message} + + +def _view_field_key(key, valid_keys): + return '__zone__' if key == 'zone' and '__zone__' in valid_keys else key + + +def _clean_view_sort(value): + if not isinstance(value, dict) or value.get('dir') not in ('asc', 'desc'): + return None + result = {'dir': value['dir']} + # Legacy sort arrays have no position. Explicit positions follow the C1 alphabet. + if 'pos' in value: + if not isinstance(value['pos'], str) or not _VIEW_POSITION.fullmatch(value['pos']): + return None + result['pos'] = value['pos'] + if value.get('locked') is True: + result['locked'] = True + return result + + +def _clean_view_summary(value): + import aios_grid as _ag + if not isinstance(value, dict): + return None + agg = value.get('agg') + # Share is a stored picker choice. It is not a query aggregate. + if agg not in (*_ag.SUMMARY_AGGS, 'none', 'share'): + return None + result = {'agg': agg} + if agg == 'percentile': + try: + result['p'] = max(0, min(100, round(float(value.get('p'))))) + except (TypeError, ValueError, OverflowError): + return None + return result + + +def _clean_view_filter(value, ctx, valid_keys): + import aios_grid as _ag + node = value['node'] + # C1 creates an empty group before separate operations attach its children. + if node.get('children') == []: + cleaned = {'conj': 'or' if node.get('conj') == 'or' else 'and', 'children': []} + if isinstance(node.get('id'), str): + cleaned['id'] = node['id'][:64] + return {'node': cleaned, 'parent': value.get('parent')} + migrated = _ag.migrate_zone_projection_config({'filters': [node]})['filters'] + filter_removals = getattr(ctx, 'filter_removals', None) + if filter_removals is None: + filter_removals = [] + # Measures belong to filters only. Every other member uses table fields. + nodes = _ag.clean_filter_tree( + migrated, valid_keys | set(ctx.measure_keys), + budget=getattr(ctx, 'filter_budget', None), cohort_ids=set(ctx.cohort_ids), + visible_view_ids={str(view['id']) for view in ctx.visible_views + if isinstance(view, dict) and view.get('id')}, + report=filter_removals) + if not nodes: + return None + cleaned = nodes[0] + if isinstance(node.get('children'), list) and isinstance(node.get('id'), str): + cleaned['id'] = node['id'][:64] + return {'node': cleaned, 'parent': value.get('parent')} + + +def clean_view_member(member, value, ctx, prior): + """Return one cleaned C1 member or a refusal before either write door applies it.""" + def refuse(code, message): + return _view_member_refusal(member, code, message) + + if not isinstance(member, str): + return refuse('unknown_member', 'Choose a supported View setting.') + family, separator, key = member.partition(':') + if family in _SERVER_VIEW_MEMBERS: + return refuse('server_member', 'The server sets this View setting.') + field_families = ('hidden', 'sort', 'width', 'position', 'summary') + scalar_families = ('group', 'display', 'name', 'note', 'locked', 'colorBy', + 'rowHeightMode', 'frozenCount', 'important', 'secondaryView', + 'application', 'fullscreen', 'filterConj') + if (family not in (*field_families, *scalar_families, 'filter', 'memberPid') + or (family in (*field_families, 'filter', 'memberPid') and not separator) + or (family in scalar_families and separator and family != 'filterConj')): + return refuse('unknown_member', 'Choose a supported View setting.') + valid_keys = {item['key'] for item in ctx.fields} + if family in field_families: + key = _view_field_key(key, valid_keys) + if key not in valid_keys: + return refuse('unknown_field', 'Choose a field in this table.') + if family == 'hidden' or family in ('locked', 'important', 'fullscreen'): + if not isinstance(value, bool): + return refuse('invalid_boolean', 'Choose true or false for this View setting.') + return value, None + if family == 'sort': + cleaned = _clean_view_sort(value) + return ((cleaned, None) if cleaned is not None else + refuse('invalid_sort', 'Choose a sort direction and a valid position.')) + if family == 'width': + try: + return max(60, min(600, round(float(value)))), None + except (TypeError, ValueError, OverflowError): + return refuse('invalid_width', 'Choose a numeric column width.') + if family == 'position': + if isinstance(value, str) and _VIEW_POSITION.fullmatch(value): + return value, None + return refuse('invalid_position', 'Use 1 to 64 letters or digits for the column position.') + if family == 'summary': + cleaned = _clean_view_summary(value) + return ((cleaned, None) if cleaned is not None else + refuse('invalid_summary', 'Choose a supported column summary.')) + if family == 'memberPid': + try: + pid = int(key) + except (TypeError, ValueError): + pid = None + admit_record = getattr(ctx, 'record_admitter', None) + if pid not in (ctx.allowed_pids or ()) and not ( + isinstance(pid, int) and 0 < pid < 2**63 and callable(admit_record) + and admit_record(pid)): + return refuse('unknown_record', 'Choose a record available in this table.') + if not isinstance(value, bool): + return refuse('invalid_boolean', 'Choose true or false for record membership.') + return value, None + if family == 'filter': + if (not key or len(key) > 64 or not isinstance(value, dict) + or not isinstance(value.get('node'), dict) + or (value.get('parent') is not None and + (not isinstance(value['parent'], str) or not value['parent'] + or len(value['parent']) > 64))): + return refuse('invalid_filter', 'Choose a valid filter condition and parent group.') + try: + cleaned = _clean_view_filter(value, ctx, valid_keys) + except (TypeError, ValueError, OverflowError): + cleaned = None + return ((cleaned, None) if cleaned is not None else + refuse('invalid_filter', 'Choose a filter available in this table.')) + if family == 'filterConj': + if (separator and (not key or len(key) > 64)) or value not in ('and', 'or'): + return refuse('invalid_conjunction', 'Choose "and" or "or" for the filter group.') + return value, None + if family in ('group', 'colorBy'): + value = _view_field_key(value, valid_keys) + if value is None or (isinstance(value, str) and value in valid_keys): + return value, None + return refuse('unknown_field', 'Choose a field in this table.') + if family == 'name': + name = str(value or '').strip()[:120] + return ((name, None) if name else + refuse('invalid_name', 'Give this View a name.')) + if family == 'note': + return str(value or '')[:2000], None + if family == 'rowHeightMode': + if value in ('compact', 'short', 'medium', 'tall'): + return value, None + return refuse('invalid_row_height', 'Choose a supported row height.') + if family == 'frozenCount': + try: + return max(1, min(8, int(value))), None + except (TypeError, ValueError, OverflowError): + return refuse('invalid_frozen_count', 'Choose a numeric frozen column count.') + if family == 'secondaryView': + try: + cleaned = _clean_secondary_view(value) + except (TypeError, ValueError, OverflowError): + cleaned = None + return ((cleaned, None) if value is None or cleaned is not None else + refuse('invalid_secondary_view', 'Choose a related database and linking field.')) + if family == 'application': + if value is None: + return None, None + from core import view_templates + try: + cleaned = view_templates.application_ref({'config': {'application': value}}) + except (TypeError, ValueError, OverflowError): + cleaned = None + if cleaned is None: + return refuse('invalid_application_ref', + 'That Application reference is not a valid typed identity.') + live = _live_application_ids(ctx) + return (None if live is not None and cleaned['id'] not in live else cleaned), None + if family == 'display': + import aios_grid as _ag + if isinstance(value, dict) and value.get('mode') == 'timeseries': + return refuse('view_mode_retired', + 'Time-series Views are retired. Choose an available View display mode.') + try: + cleaned = _ag._clean_display(value, valid_keys) + except (TypeError, ValueError, OverflowError): + cleaned = None + return ((cleaned, None) if value is None or cleaned is not None + or value in ({'mode': 'grid'}, {'mode': 'grid', 'groupSummary': False}) else + refuse('invalid_display', 'Choose an available View display mode.')) + + +def _clean_view_config(cfg, ctx, prior, display): + """Project legacy create arrays through the C1 member cleaner.""" + import aios_grid as _ag + cleaner_ctx = _ViewMemberCtx(ctx, filter_budget=[_ag.MAX_FILTER_NODES]) + + def clean(member, value, default=None): + cleaned, refusal = clean_view_member(member, value, cleaner_ctx, prior) + return default if refusal else cleaned + + valid_keys = {item['key'] for item in ctx.fields} + config = { + 'filters': [], 'filterConj': clean('filterConj', cfg.get('filterConj', 'and'), 'and'), + 'important': clean('important', cfg.get('important', False), False), + 'sorts': [], 'groupBy': clean('group', cfg.get('groupBy')), + 'colorBy': clean('colorBy', cfg.get('colorBy')), + 'rowHeightMode': clean('rowHeightMode', cfg.get('rowHeightMode', 'short'), 'short'), + 'visible': [], 'widths': {}, 'summary': {}, 'memberPids': [], + 'secondaryView': clean('secondaryView', cfg.get('secondaryView')), + } + raw_filters = cfg.get('filters') if isinstance(cfg.get('filters'), list) else [] + for index, node in enumerate(raw_filters[:_ag.MAX_FILTER_SIBLINGS]): + node_id = str(node.get('id') or f'legacy-{index}') if isinstance(node, dict) else f'legacy-{index}' + cleaned = clean('filter:' + node_id, {'node': node, 'parent': None}) + if cleaned is not None: + config['filters'].append(cleaned['node']) + raw_sorts = cfg.get('sorts') if isinstance(cfg.get('sorts'), list) else [] + for rule in raw_sorts[:20]: + if not isinstance(rule, dict) or not isinstance(rule.get('colId'), str): + continue + key = _view_field_key(rule['colId'], valid_keys) + cleaned = clean('sort:' + key, rule) + if cleaned is not None: + config['sorts'].append({'colId': key, **cleaned}) + # Order remains an array. S85-T02 owns position seeding and order derivation. + order = cfg.get('order') if isinstance(cfg.get('order'), list) else [] + config['order'] = list(dict.fromkeys( + _view_field_key(key, valid_keys) for key in order + if isinstance(key, str) and _view_field_key(key, valid_keys) in valid_keys)) + config['order'] += [item['key'] for item in ctx.fields if item['key'] not in config['order']] + visible = cfg.get('visible') if isinstance(cfg.get('visible'), list) else [] + for key in dict.fromkeys(_view_field_key(key, valid_keys) for key in visible if isinstance(key, str)): + hidden = clean('hidden:' + key, False) + if hidden is False: + config['visible'].append(key) + for source, family in (('widths', 'width'), ('summary', 'summary')): + values = cfg.get(source) if isinstance(cfg.get(source), dict) else {} + entries = list(values.items())[:400] if family == 'summary' else values.items() + for key, value in entries: + if not isinstance(key, str): + continue + key = _view_field_key(key, valid_keys) + cleaned = clean(family + ':' + key, value) + if cleaned is not None: + config[source][key] = cleaned + pids = cfg.get('memberPids') if isinstance(cfg.get('memberPids'), list) else [] + for pid in pids: + if type(pid) is int and clean('memberPid:' + str(pid), True) is True: + config['memberPids'].append(pid) + application, refusal = clean_view_member('application', cfg.get('application'), cleaner_ctx, prior) + if refusal: + return None, cleaner_ctx.filter_removals, refusal + if cfg.get('groupSummary') is True: + # A legacy root toggle lands in the display member, where viewOps.ts reads it. + display = {**(display or {'mode': 'grid'}), 'groupSummary': True} + if application is not None: + config['application'] = application + elif display is not None: + config['display'] = display + if cfg.get('frozenCount') is not None: + frozen = clean('frozenCount', cfg['frozenCount']) + if frozen is not None: + config['frozenCount'] = frozen + if 'fullscreen' in cfg: + config['fullscreen'] = clean('fullscreen', cfg['fullscreen'], False) + return config, cleaner_ctx.filter_removals, None + + def _shared_folder_record(fid, folder): """C9's wire record for a folder granted to somebody else: `{id, name, order?, icon?}`. @@ -980,6 +1340,17 @@ def _shared_view_role(tops, view_id, owner, uname, st=None): return role if role in ('view', 'edit') else None +_VIEW_READ_ONLY_MESSAGE = "Ask the view's owner for edit access." + + +def _may_edit_view(tops, view_id, owner, view, uname, admin): + """Admit the stored creator, an administrator or an edit grantee on this tenant's View.""" + creator = str(view.get('createdBy') or owner).strip().lower() + return (admin or creator == str(uname).strip().lower() + or _shared_view_role(tops, view_id, owner, uname, + st=getattr(tops, 'st', None)) == 'edit') + + def _granted_views(ctx, uname): """{view_id: STAMPED view} for every view GRANTED to `uname` by name (wave 21, item 9, C1). @@ -998,10 +1369,26 @@ def _granted_views(ctx, uname): import core.shares as shares tops = _tops(ctx) st = getattr(tops, 'st', None) + snapshot = getattr(st, '_grid_admission', None) + lookup = {} + if (isinstance(snapshot, AdmissionSnapshot) and isinstance(tops, _tstore.TableStore) + and tops._settings_rows is None): + tops.migrate_legacy_shared_views() + lookup['document'] = snapshot.get(tops.table_key) ids = (shares.shared_with(uname, kind='view', st=st) or {}).get('view') or [] + views = dict(lookup, settings=None) if lookup else {} + if ids and not lookup and isinstance(tops, _tstore.TableStore): + # S85-T169: resolve the settings backend and the workspace document once for all ids. + views['settings'] = rows = tops._settings_rows + if rows is None: + tops.migrate_legacy_shared_views() + try: + views['document'] = st.get(tops.table_key) or {} + except Exception: + views = {'settings': None} # each find_view keeps its own fail-closed read out = {} for vid in ids: - found = tops.find_view(vid) if hasattr(tops, 'find_view') else None + found = tops.find_view(vid, **views) if hasattr(tops, 'find_view') else None if not found: continue owner, view = found @@ -1038,7 +1425,7 @@ def _granted_views(ctx, uname): except Exception: folder_ids = [] for fid in folder_ids: - found = tops.find_folder(fid) if hasattr(tops, 'find_folder') else None + found = tops.find_folder(fid, **lookup) if hasattr(tops, 'find_folder') else None if not found: continue # a folder on another topic contributes nothing here owner, folder, inside = found @@ -1209,7 +1596,7 @@ def _refuse(ctx, code, message, key='', *, event='field_upsert'): try: out = getattr(ctx, 'out', None) if out is not None and hasattr(out, 'refusals'): - if len(out.refusals) < 24: # one per event in the window; never unbounded + if sum(item.get('event') != 'view_op' for item in out.refusals) < 24: out.refusals.append({ 'event': str(event or 'field_upsert')[:40], 'key': str(key or '')[:80], @@ -1403,14 +1790,309 @@ def _activity_action(fn): return wrapped +def _view_op_reply(ctx, result, *, rerender=False): + """Expose one C1 result without receipt history or a shared toast.""" + row = {key: result[key] for key in ('opId', 'member', 'applied') if key in result} + row['rerender'] = bool(rerender) + if result.get('refused'): + row['refused'] = {key: result['refused'][key] for key in ('reason', 'member', 'message')} + ctx.out.refusals.append({'event': 'view_op', **row['refused']}) + if isinstance(row.get('opId'), str): + ctx.out.writes[row['opId']] = row + return row + + +def _admit_view_op(view, candidate, member, *, owner, actor, admin): + """Check the resolved operation against the current locked View inside its update.""" + import aios_grid as _ag + family = member.partition(':')[0] + if family == 'locked' and not (admin or str(view.get('createdBy') or owner).lower() + == str(actor).lower()): + return {'reason': 'lock_owner_only', + 'message': 'Only the View creator or an administrator can change its lock.'} + cfg, next_cfg = view.get('config') or {}, candidate.get('config') or {} + if (family in ('display', 'application') and view.get('locked') + and view.get('kind') != 'system' and view.get('id') != 'all-customers' + and _tstore._view_member_value(view, member) != _tstore._view_member_value(candidate, member)): + return {'reason': 'view_mode_locked', 'message': 'Unlock this View before changing its display.'} + filters, sorts = next_cfg.get('filters') or [], next_cfg.get('sorts') or [] + if (_ag.preserve_locked_view_records(cfg, filters, sorts) != (filters, sorts) + or (family == 'filterConj' and filters + and cfg.get('filterConj', 'and') != next_cfg.get('filterConj', 'and') + and _ag.preserve_locked_view_records(cfg, [], sorts)[0])): + return {'reason': 'locked_view_record', + 'message': 'Copy this View to change its locked filter or sort records.'} + return None + + +@contextmanager +def _cohort_name_transaction(st): + """Keep a cohort View write and its synchronization on one bounded PostgreSQL connection.""" + from core import edit_receipts, store_pg + backend = edit_receipts._backend(st) + if isinstance(backend, edit_receipts.PgReceipts): + with backend._connection() as con, con.transaction(): + with store_pg.PgStore(backend.slug).transaction(con): + yield con, backend.slug + else: + yield None, None + + +def _sync_cohort_view_name(ctx, tops, owner, view_id): + """Copy the canonical View name while its source remains locked through the cohort write.""" + from core import store_pg + sets = _cohorts(ctx) + cohort_name = (sets.all_for(owner).get(view_id) or {}).get('name') + + def synchronize(data): + view = ((data.get(owner) or {}).get('views') or {}).get(view_id) or {} + name = view.get('name') + if name and cohort_name != name: + sets.rename(owner, view_id, name) + return True + return False + + with _cohort_name_transaction(tops.st) as (con, slug): + if con is not None: + rows = tops._settings_rows + if rows is not None: + with rows._reading(): + synchronize(rows._settings(con, (owner,))) + else: + row = con.execute(store_pg._q( + 'SELECT value FROM {tbl} WHERE key = %s FOR UPDATE', + 'store_kv', slug), (tops.table_key,)).fetchone() + synchronize(row[0] if row else {}) + else: + # Detached stores provide source exclusion through their update contract. + class Unchanged(Exception): + pass + + def change(data, _audit): + if not synchronize(data): + raise Unchanged + return data + + try: + tops._audited_update(change, flush='sync', owners=(owner,)) + except Unchanged: + pass + + +def handle_view_ops(events, ctx): + """Authorize one View batch and apply its cleaned members through the owner-row writer.""" + from copy import deepcopy + from core import edit_receipts, view_settings_rows + import aios_grid as _ag + from harness import measure_filter as _mfc + + events = [dict(event, client_op_id=event.get( + 'client_op_id', event.get('opId', event.get('id')))) for event in events] + if not events: + return [] + ctx.workspace_lend = None + view_id = str(events[0].get('view_id') or '').strip() + + def refusal(event, reason, message): + return {'opId': event.get('client_op_id'), 'member': event.get('member'), + 'applied': False, 'refused': { + 'reason': reason, 'member': event.get('member'), 'message': message}} + + def refuse_all(reason, message): + return [_view_op_reply(ctx, refusal(event, reason, message)) for event in events] + + if not _store_of(ctx).available(): + raise StoreUnavailable('The View store is unavailable.') + tops = _tops(ctx) + admission_tops = _tops(_admission_ctx(ctx)) + owner = ctx.uname + workspace = admission_tops._read_owners(owner).get(owner) or {} + prior = (workspace.get('views') or {}).get(view_id) + projected = next((view for view in ctx.visible_views + if isinstance(view, dict) and view.get('id') == view_id), {}) + _pinned_per_user = ((prior or projected).get('kind') == 'system' + or view_id in ('all-customers', _ag_overview_id())) + assigned = admission_tops.assigned_views(ctx.uname, ctx.allowed_pids) + is_assigned = view_id in assigned + if is_assigned: + prior = assigned[view_id] + workspace = admission_tops._read_owners(_tstore.ASSIGNED_KEY).get(_tstore.ASSIGNED_KEY) or {} + elif view_id.startswith('assigned:') or projected.get('assigned'): + return refuse_all('view_gone', 'That assigned View is no longer available to you.') + elif not _pinned_per_user: + found = admission_tops.find_view(view_id) + if found and found[0] != ctx.uname: + owner, prior = found + workspace = admission_tops._read_owners(owner).get(owner) or {} + prior = (workspace.get('views') or {}).get(view_id) + seed_view = not prior and _pinned_per_user and bool(projected) + if seed_view: + # A pinned projection gets its own source, never another user's same-id source. + prior = {key: deepcopy(value) for key, value in projected.items() + if key in ('id', 'name', 'kind', 'locked', 'note', 'config')} + prior['createdBy'] = ctx.uname + prior['kind'] = 'system' + if not isinstance(prior, dict): + return refuse_all('view_gone', 'That View is no longer available.') + if not is_assigned and not _pinned_per_user and not _may_edit_view( + admission_tops, view_id, owner, prior, ctx.uname, ctx.admin): + return refuse_all('view_read_only', _VIEW_READ_ONLY_MESSAGE) + + def admit(view, candidate, member): + if not is_assigned and not _pinned_per_user and not _may_edit_view( + tops, view_id, owner, view, ctx.uname, ctx.admin): + return {'reason': 'view_read_only', 'message': _VIEW_READ_ONLY_MESSAGE} + return _admit_view_op(view, candidate, member, owner=owner, + actor=ctx.uname, admin=ctx.admin) + + receipts = view_settings_rows.ViewOpReceipts(workspace, getattr(tops.st, 'key', tops.table_key)) + action = edit_receipts.operation(tops.table_key, 'view_op', view_id) + preview = deepcopy(prior) + prepared, slots, answers, known = [], [], [None] * len(events), {} + for index, event in enumerate(events): + member, operation = _tstore._view_member_identity(event.get('member')), event.get('op') + identity = event.get('client_op_id') + + def refuse(reason, message): + answers[index] = refusal(event, reason, message) + + try: + edit_receipts.op_id(identity) + except ValueError: + refuse('invalid_op_id', 'Give this View operation a client operation id.') + continue + if str(event.get('view_id') or '').strip() != view_id: + refuse('view_gone', 'That View is no longer available.') + continue + replay = receipts.result(ctx.uname, action, identity) + if replay: + answers[index] = replay + continue + if identity in known: + prepared.append(next(item for item in prepared if item['client_op_id'] == identity)) + slots.append(index) + continue + family, _, key = member.partition(':') if isinstance(member, str) else ('', '', '') + if is_assigned and family in ('filter', 'filterConj', 'memberPid'): + refuse('assigned_filter_fixed', 'The assigned View fixes its filters and record membership.') + continue + if operation not in ('set', 'remove', 'undo'): + refuse('invalid_op', 'Choose set, remove or undo for this View setting.') + continue + present, value = operation != 'remove', event.get('value') + if operation == 'undo': + original = known.get(value) if isinstance(value, str) else None + original = original or receipts.result(ctx.uname, action, value) + if (not original or not original.get('applied') + or _tstore._view_member_identity(original.get('member')) != member): + refuse('changed_since', 'That change is not available to undo.') + continue + present, value = original['beforePresent'], original['before'] + if not present: + if family not in ('sort', 'width', 'filter', 'summary', 'memberPid') and operation != 'undo': + refuse('invalid_op', 'Set a value for this View setting.') + continue + sample = {'sort': {'dir': 'asc', 'pos': 'V'}, 'width': 100, + 'filter': {'node': {'children': []}, 'parent': None}, + 'summary': {'agg': 'none'}, 'memberPid': False}.get(family) + # Undo can remove an optional scalar that was absent before the user's edit. + if operation == 'undo' and family not in ('sort', 'width', 'filter', 'summary', 'memberPid'): + sample = _tstore._view_member_value(preview, member)[1] + cleaned, denied = clean_view_member(member, sample, ctx, preview) + else: + cleaned, denied = clean_view_member(member, value, ctx, preview) + if denied: + code = 'target_gone' if denied['code'] == 'unknown_field' else denied['code'] + refuse(code, denied['message']) + continue + if operation == 'set' and family == 'sort' and 'pos' not in cleaned: + refuse('invalid_position', 'Use 1 to 64 letters or digits for the position.') + continue + if family in ('hidden', 'sort', 'width', 'position', 'summary'): + member = family + ':' + _view_field_key(key, {field['key'] for field in ctx.fields}) + cfg = preview.get('config') or {} + candidate = deepcopy(preview) + if family == 'position' and 'positions' not in candidate.setdefault('config', {}): + candidate['config']['positions'] = _tstore._view_position_seed(cfg.get('order') or []) + before_present, before = _tstore._view_member_value(candidate, member) + try: + _tstore._set_view_member(candidate, member, cleaned, present=present) + except _tstore._ViewOpRefused as exc: + refuse(exc.reason, str(exc)) + continue + next_cfg = candidate.get('config') or {} + filters, sorts = next_cfg.get('filters') or [], next_cfg.get('sorts') or [] + if family == 'filter' and present: + cycle = _ag.view_filter_cycle(view_id, filters, lambda vid: ( + next(((view.get('config') or {}).get('filters') or [] for view in ctx.visible_views + if isinstance(view, dict) and view.get('id') == vid), []))) + if cycle: + refuse('view_cycle', 'Choose a View that does not refer back to this View.') + continue + if is_assigned and family in ('name', 'note', 'locked'): + refuse('unknown_member', 'Choose a presentation setting for this assigned View.') + continue + preview = candidate + after_present, after = _tstore._view_member_value(preview, member) + known[identity] = {'opId': identity, 'member': member, 'applied': True, + 'before': before, 'beforePresent': before_present, + 'after': after, 'afterPresent': after_present} + prepared.append({**event, 'member': member, + 'value': cleaned if operation == 'set' else event.get('value')}) + slots.append(index) + + cohort_name = view_id in ctx.cohort_ids and any(event.get('member') == 'name' for event in events) + with _cohort_name_transaction(tops.st) if cohort_name else nullcontext(): + if prepared: + try: + landed = tops.apply_view_ops( + owner, view_id, prepared, actor=ctx.uname, origin=EVENT_ORIGIN, + assigned_recipient=ctx.uname if is_assigned else None, + seed_view=prior if seed_view else None, + admit=admit) + except edit_receipts.ReceiptUnavailable as exc: + raise StoreUnavailable(str(exc)) from exc + finally: + if ctx.admission is not None: + ctx.admission.invalidate(tops.table_key) + for index, result in zip(slots, landed): + answers[index] = result + preview = (tops.assigned_views(ctx.uname, ctx.allowed_pids).get(view_id, {}) if is_assigned else + ((tops._read_owners(owner).get(owner) or {}).get('views') or {}).get(view_id, {})) + if view_id in ctx.cohort_ids and any( + result.get('applied') and result.get('member') == 'name' for result in answers): + _sync_cohort_view_name(ctx, tops, owner, view_id) + unresolved = {str(rule.get('id') or '') for rule in measure_leaves( + preview.get('config', {}).get('filters') or []) if _mfc.rule_complete(rule)} + rerender = bool(unresolved - set(ctx.resolved_ids) - {''}) + return [_view_op_reply(ctx, answer, rerender=rerender and answer.get('applied', False)) + for answer in answers] + + +class _UnwrittenViewUpsert(Exception): + """Abort the receipt transaction for an existing View's whole-save refusal.""" + + def _edit_receipt(fn): @wraps(fn) def wrapped(event, ctx): from copy import copy, deepcopy from dataclasses import replace from core import edit_receipts + def apply_event(event, write_ctx): + try: + return fn(event, write_ctx) + finally: + # Canonical writers bypass the admission handle, including row-backed settings. + if write_ctx.admission is not None: + write_ctx.admission.invalidate() + if not isinstance(event, dict): return False + # C1 stores receipts with its owner-row update, rather than the event wrapper. + if event.get('type') == 'view_op': + identity = event.get('client_op_id', event.get('opId', event.get('id'))) + return fn(dict(event, id=identity, client_op_id=identity), ctx) identity = event.get('opId', event.get('id')) active_id = edit_receipts.current_op_id() # Trusted staged previews own their confirmation ledger and must not gain transport metadata. @@ -1423,7 +2105,7 @@ def _edit_receipt(fn): if len(ctx.seen_ids) > 400: for stale in list(ctx.seen_ids)[:200]: ctx.seen_ids.pop(stale, None) - return fn(dict(event, id=identity), ctx) + return apply_event(dict(event, id=identity), ctx) if not identity: return False try: @@ -1442,12 +2124,16 @@ def _edit_receipt(fn): if table is not None: table._st = checked write_ctx = replace(ctx, st=checked, table=table, workspace_lend=lend) - rerender = fn(event, write_ctx) + rerender = apply_event(event, write_ctx) write = dict(ctx.out.writes.get(identity) or {}) write['opId'] = identity result = deepcopy({'rerender': bool(rerender), 'write': write, 'refusals': ctx.out.refusals[before:], 'doc': ctx.out.doc, 'toast': ctx.out.toast}) + if event.get('type') == 'view_upsert' and any( + row.get('code') in ('view_ops_required', 'view_read_only') + for row in result['refusals']): + raise _UnwrittenViewUpsert(result) edit_receipts.observe_event(result) return result @@ -1457,6 +2143,8 @@ def _edit_receipt(fn): _store_of(ctx), ctx.uname, edit_receipts.operation(ctx.scope_key, event.get('type'), event.get('pid')), identity, apply) + except _UnwrittenViewUpsert as exc: + receipt = exc.args[0] except edit_receipts.ReceiptUnavailable as exc: raise StoreUnavailable(str(exc)) from exc receipt = receipt.get('event') or receipt @@ -1528,6 +2216,8 @@ def handle_one(event, ctx): event_id = event['id'] ctx.out.write_id = event_id kind = event.get('type') + if kind == 'view_op': + return any(row['rerender'] for row in handle_view_ops([event], ctx)) if kind == 'overlay_patch' and str(scope_key) == 'ut_activity': return _refuse(ctx, 'activity_read_only', 'Activity records cannot be edited.', event=kind) @@ -1631,19 +2321,39 @@ def handle_one(event, ctx): if kind == 'view_upsert': raw = event.get('view') - if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict): + if not isinstance(raw, dict): return False view_id = str(raw.get('id') or '').strip()[:120] - name = str(raw.get('name') or '').strip()[:120] + prior = (ws.get('views') or {}).get(view_id) or {} + tops = _tops(ctx) + assigned = tops.assigned_views(uname, allowed_pids) if _store_of(ctx).available() else {} + is_assigned = raw.get('assigned') or prior.get('assigned') or view_id in assigned + # Stored system Views and pinned ids keep personal presentation in the caller's stratum. + _pinned_per_user = (prior.get('kind') == 'system' + or view_id in ('all-customers', _ag_overview_id())) + _save_as = uname + if _store_of(ctx).available() and not _pinned_per_user and view_id not in assigned: + found = tops.find_view(view_id, prefer_owner=uname) + if found: + _save_as, prior = found + if prior.get('kind') == 'system': + _save_as, prior = uname, (ws.get('views') or {}).get(view_id) or {} + elif not _may_edit_view(tops, view_id, _save_as, prior, uname, admin): + return _refuse(ctx, 'view_read_only', _VIEW_READ_ONLY_MESSAGE, + view_id, event=kind) + if ctx.durable_receipts and (prior or view_id in assigned): + return _refuse(ctx, 'view_ops_required', + 'This page is out of date. Reload to keep editing this view.', + view_id, event=kind) + if not isinstance(raw.get('config'), dict): + return False + name, name_refusal = clean_view_member('name', raw.get('name'), ctx, prior) requested_name = name - if not view_id or not name: + if not view_id or name_refusal: return False - display = raw['config'].get('display') - if isinstance(display, dict) and display.get('mode') == 'timeseries': - return _folder_refuse( - ctx, kind, 'view_mode_retired', - 'Time-series Views are retired. Choose an available View display mode.', - key=view_id) + display, display_refusal = clean_view_member('display', raw['config'].get('display'), ctx, prior) + if display_refusal and display_refusal['code'] == 'view_mode_retired': + return _folder_refuse(ctx, kind, display_refusal['code'], display_refusal['message'], key=view_id) if 'permissions' in raw: return _refuse( ctx, 'view_permissions_retired', @@ -1654,7 +2364,7 @@ def handle_one(event, ctx): # here for the same reason the page does it: a missing embed build must # never break page import. import aios_grid as _ag - cfg = _ag.migrate_zone_projection_config(raw['config']) + cfg = raw['config'] # A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered, # shown, grouped or coloured by, because it is not a column of this table. Hence the # widened key set here and the plain `valid_keys` everywhere below. @@ -1671,8 +2381,6 @@ def handle_one(event, ctx): # so this needs no new plumbing; it is the value nobody had handed to the validator. # ⚠ Edited by lane C, not this file's owner: `platform/core/grid_events.py` is in NO fence # and lane C was the last session open. See `mailbox/C.md` NOTE C-24. - _view_ids = {str(v.get('id')) for v in (visible_views or ()) - if isinstance(v, dict) and v.get('id')} # ⛔⛔ THE CYCLE REFUSAL'S HOST HALF (C4), and it is the ticket's `done-when` rather than a # guard bolted on afterwards. View X filtered on "is in View Y" where Y is filtered on X # cannot be resolved by anything: X needs Y needs X. The browser refuses it too, with a @@ -1691,81 +2399,11 @@ def handle_one(event, ctx): '. A view cannot filter on a view that filters on it, because neither can ' 'be worked out without the other.', view_id) return False - filter_removals = [] - filters = _ag.clean_filter_tree(cfg.get('filters'), - valid_keys | set(measure_keys), - cohort_ids=set(cohort_ids), - visible_view_ids=_view_ids, report=filter_removals) - sorts = [] - for rule in list(cfg.get('sorts') or [])[:20]: - if (isinstance(rule, dict) and rule.get('colId') in valid_keys - and rule.get('dir') in ('asc', 'desc')): - clean_sort = {'colId': rule['colId'], 'dir': rule['dir']} - if rule.get('locked') is True: - clean_sort['locked'] = True - sorts.append(clean_sort) - order = list(dict.fromkeys( - key for key in list(cfg.get('order') or []) if key in valid_keys)) - order += [field['key'] for field in fields if field['key'] not in order] - visible = list(dict.fromkeys( - key for key in list(cfg.get('visible') or []) if key in valid_keys)) - widths = {} - raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {} - for key, value in raw_widths.items(): - if key in valid_keys: - try: - widths[key] = max(60, min(600, round(float(value)))) - except (TypeError, ValueError): - pass - # ⭐⭐ W50-T03 / R2 — THE SUMMARY CHOICE IS A VIEW SETTING, SO IT HAS TO SURVIVE THIS DOOR. - # - # ⛔ THE BUG THIS CLOSES IS SILENT AND LOOKS LIKE A CLIENT BUG. This whitelist REBUILDS the - # stored config from named keys, so a key it does not name is not rejected loudly — it is - # simply absent from what gets written. The reader picks Median, the cell shows Median, and - # the choice disappears the next time the host copy wins a reconcile. Exactly the shape - # `secondaryView` above records having paid for once already. - # - # ⚠ `'none'` IS A CHOICE, NOT AN ABSENCE (types.ts::SummaryChoice): R2 defaults numeric - # kinds to Sum, so a reader turning a money column's summary OFF must be able to SAY so, - # and an absent key means "nothing chosen" and re-defaults to Sum. Dropping `'none'` here - # would make that switch un-turn-off-able. - import aios_grid as _ag_summary - summary = {} - raw_summary = cfg.get('summary') if isinstance(cfg.get('summary'), dict) else {} - for key, value in list(raw_summary.items())[:400]: - if key not in valid_keys or not isinstance(value, dict): - continue - agg = value.get('agg') - # The vocabulary is `aios_grid.SUMMARY_AGGS` plus `'none'` — NOT `FIELD_AGGS`, which is - # `field.agg`'s six-name write vocabulary and rejects `count_distinct`/`percentile`. - # - # ⭐⭐ W51-T02 / OWNER RULING R3 — AND PLUS `'share'`, ALLOWLISTED BY NAME RIGHT HERE - # RATHER THAN BY WIDENING `aios_grid.SUMMARY_AGGS`. - # - # R3 puts "% of total" on the bottom-bar picker, so `share` is a choice a reader can - # store and it has to survive this door — this whitelist REBUILDS the config from - # named keys, so an unlisted value is not rejected loudly, it simply never gets - # written and the choice vanishes on the next reconcile (the bug the note above - # closed for `count_distinct`). - # ⛔ IT IS NOT A WIRE AGG AND MUST NOT JOIN `aios_grid.SUMMARY_AGGS`. That tuple is - # C1's eight-name clause, `verify_grid_ux.py::summary_bar_shape` holds it byte-for-byte - # against the client's `iconShapes.SUMMARY_AGGS`, and it is what the `/aggregates` - # route compiles into SQL. No engine has a "share of the view" aggregate: a share is a - # DIVISION of two ordinary ones (a numeric column's sum, anything else's count) and - # the client does it. So the picker's extra name is admitted where choices are STORED - # and nowhere a query is COMPILED — which is also why `aios_grid.py`, another lane's - # file, is never opened for this. - if agg != 'none' and agg != 'share' and agg not in _ag_summary.SUMMARY_AGGS: - continue - choice = {'agg': agg} - if agg == 'percentile': - try: - choice['p'] = max(0, min(100, round(float(value.get('p'))))) - except (TypeError, ValueError): - # A percentile with no readable point is not a percentile. Drop the whole - # choice rather than store one that would answer at a point nobody picked. - continue - summary[key] = choice + config, filter_removals, config_refusal = _clean_view_config(cfg, ctx, prior, display) + if config_refusal: + _refuse(ctx, config_refusal['code'], config_refusal['message'], view_id, event=kind) + return False + filters, sorts = config['filters'], config['sorts'] # C-LOCK (wave 2026-08-02, item 12's mechanism): a view may be LOCKED to one cohort. # `cohort_ids` is the same permission list the cohort filter leaf uses — a lock naming # a cohort this session cannot see is DROPPED at write, exactly like a cohort rule. @@ -1782,84 +2420,10 @@ def handle_one(event, ctx): _is_locked_view = view_id in set(cohort_ids) if _is_locked_view: cohort_lock = view_id - config = { - 'filters': filters, - # conjunction joining the ROOT-level conditions; anything but an - # explicit 'or' means 'and' (legacy views had an implicit AND). - 'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and', - **({'cohortLock': cohort_lock} - if cohort_lock and cohort_lock in set(cohort_ids) else {}), - # ⭐⭐ WAVE 32 · R5 / CONTRACT C4 — "Mark important" (raised by SESSION C as ASK C-12). - # - # ⛔ WITHOUT THIS LINE THE MARK IS A LIE THAT SURVIVES UNTIL RELOAD. This dict is - # REBUILT FROM AN ALLOWLIST on every autosave — a column resize is enough — so a key - # the client sends and this literal does not name is silently dropped. The badge would - # paint, the write would answer 200, and the mark would be gone on the next read: the - # exact shape T24's done-when says "persists across a reload" to catch, and - # [[a-migration-that-runs-on-the-next-write]] on a write path. - # - # ⚠ UNCONDITIONAL, NOT SPREAD-CONDITIONAL LIKE `cohortLock` ABOVE, and the asymmetry is - # the whole point: R5's mark must be REMOVABLE. A key written only when present means - # unmarking sends `important: false`, the allowlist drops it, and the stored `true` - # survives — a mark you can set and never clear. - # ⚠ `is True`, not truthy: the client sends a real boolean, and a string `"false"` must - # not become a mark. - # ⚠ NO PERMISSION GATE, deliberately. R5 makes this a personal legibility mark, not a - # lock; the source/grant write wall below already decides who may write this view at - # all, and adding a second wall here would be the two-normalizers shape this wave is - # already fixing elsewhere. - 'important': cfg.get('important') is True, - 'sorts': sorts, - 'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None, - 'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None, - 'rowHeightMode': (cfg.get('rowHeightMode') - if cfg.get('rowHeightMode') in ('compact', 'short', 'medium', 'tall') - else 'short'), - 'order': order, - 'visible': visible, - 'widths': widths, - 'summary': summary, - 'memberPids': [pid for pid in list(cfg.get('memberPids') or []) - if isinstance(pid, int) and pid in allowed_pids], - # ⭐⭐ W41-T20 clause 5 / C6 -- see `_clean_secondary_view` above. - 'secondaryView': _clean_secondary_view(cfg.get('secondaryView')), - } - # W45 C3: a View either owns a direct display spec OR references one reusable - # Application. It never stores both. The identity/revision cleaner is shared with the - # template projection, and this is its production write path (not a verifier-only helper). - from core import view_templates as _view_templates - application = None - if cfg.get('application') is not None: - application = _view_templates.application_ref({'config': cfg}) - if application is None: - _refuse(ctx, 'invalid_application_ref', - 'that Application reference is not a valid typed identity', view_id) - return False - # S67-T39 / R36: a reference to a DELETED App is dropped, not stored, so the View - # saves as a plain view of its database. Unknown registry state keeps the ref. - _live = _live_application_ids(ctx) - if _live is not None and application['id'] not in _live: - application = None - else: - config['application'] = application - # Wave-6 item 10: a direct non-grid display mode rides the view config, validated - # structurally (mode whitelist, refs must be this table's fields). A bound Application's - # config remains in the registry and is resolved on read, so an offered clone is dropped. - if application is None: - display = _ag._clean_display(cfg.get('display'), valid_keys) - if display: - config['display'] = display - # Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling - # sideways. Clamped 1..8 — 0 would unfreeze the identity column the whole - # surface pivots on; absent = the legacy single frozen column. - if cfg.get('frozenCount') is not None: - try: - config['frozenCount'] = max(1, min(8, int(cfg['frozenCount']))) - except (TypeError, ValueError): - pass + if cohort_lock and cohort_lock in set(cohort_ids): + config['cohortLock'] = cohort_lock # The prior stored View supplies lock state and its immutable filter records. Sharing # itself is exclusively the modern grant registry checked at the foreign-view wall below. - prior = (ws.get('views') or {}).get(view_id) or {} # A visible foreign view is a grant projection. The modern foreign-view wall below # resolves the owner's personal source and checks the grant registry before deciding # whether this is a create; there is no legacy bucket to consult as a second authority. @@ -1902,12 +2466,6 @@ def handle_one(event, ctx): config['display'] = dict(prior['config']['display']) else: config.pop('display', None) - # A SYSTEM VIEW is per-user state and is never a share source. `kind` is - # host-validated below; read the existing value when present so a browser cannot change - # the residency/identity of a stored View. - _kind = (prior.get('kind') if not is_new - else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom') - else 'custom')) # C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever # is already stored. Enforced here rather than hidden in the client. This composes with # C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze @@ -1915,7 +2473,10 @@ def handle_one(event, ctx): # ⚠ Inert in practice TODAY — a user can only ever write their own per-user workspace, # so `uname == creator` always holds. Correct plumbing for shared views, not a live gate. may_lock = admin or uname == creator - locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked')) + locked, lock_refusal = clean_view_member( + 'locked', raw.get('locked', False) if may_lock else prior.get('locked', False), ctx, prior) + if lock_refusal: + locked = bool(prior.get('locked')) if not is_new else False # C3: the LOCK's actual job — a locked view's display mode is frozen, so a locked Kanban # stays Kanban. This half IS live per-user and is the part the owner asked for. Filters, # sorts and columns stay editable by design. @@ -1947,7 +2508,7 @@ def handle_one(event, ctx): else 'custom', 'locked': locked, 'createdBy': creator, - 'note': str(raw.get('note') or '')[:2000], + 'note': clean_view_member('note', raw.get('note'), ctx, prior)[0], 'config': config, 'filterRemovals': filter_removals} # System/list projections are not all persisted in this table store, but custom views @@ -1972,7 +2533,7 @@ def handle_one(event, ctx): # win the merge, so the receiver sees their private copy forever and calls sharing # broken). Role `edit` (or admin) writes THROUGH to the owner's record — config only; # `createdBy` stays pinned from the stored source and grants remain the access wall. - # Role `view` (or no grant) refuses with a toast instead of forking. + # A reader receives a result-row refusal before any View mutation. # # ⛔⛔ WAVE 33 · T13 (owner item 3) — A PINNED PER-USER ID IS NOT A FOREIGN VIEW, AND # TREATING IT AS ONE IS A CROSS-USER WRITE THAT REPORTS SUCCESS. @@ -1994,14 +2555,12 @@ def handle_one(event, ctx): # VIEW IS PER-USER STATE AND MUST NEVER BE SHARED", enforced there by pinning # `perms = {'edit': 'personal'}`. A per-user view can therefore never BE a grant, which # is the only thing this wall exists to protect — so exempting it narrows nothing. - _pinned_per_user = (_kind == 'system' or view_id in ('all-customers', _ag_overview_id())) # W48-T20 / C4 — an assignment is neither the owner's view nor a generic foreign # share. Its filter lives in TableStore's ``__assigned__`` projection and this write # door accepts only the recipient's presentation. In particular, a forged ``edit`` # grant or a client flip of ``assigned`` cannot route this through the generic # share-through-to-owner branch below. - assigned = _tops(ctx).assigned_views(uname, allowed_pids) if _store_of(ctx).available() else {} - if raw.get('assigned') or prior.get('assigned') or view_id in assigned: + if is_assigned: if view_id not in assigned: ctx.out.toast = 'That assigned view is no longer available to you.' return False @@ -2014,34 +2573,14 @@ def handle_one(event, ctx): # load-bearing: it replaces any optimistic browser filter with the locked server # projection before another action can use it. return True - _save_as = uname - if _store_of(ctx).available() and not _pinned_per_user: - _foreign = _tops(ctx).find_view(view_id) - if _foreign and _foreign[0] != uname: - _o_uname, _stored = _foreign - if not admin: - if _shared_view_role(_tops(ctx), view_id, _o_uname, uname, - st=getattr(_tops(ctx), 'st', None)) != 'edit': - ctx.out.toast = ('That view is shared with you read-only. Ask its ' - 'owner for edit access.') - return False - view['createdBy'] = _stored.get('createdBy', _o_uname) - _save_as = _o_uname if _store_of(ctx).available(): # ⛔ W50-T25 / D8 — `_save_as` IS THE BUCKET THE WRITE LANDED IN and is therefore # the audit `owner`; `uname` is who DID it. On the foreign-owner branch above they # differ, and a row that conflated them would name the wrong person in the one # record whose entire job is naming the right one. - # ⭐⭐ W51-T33 — `rev=` IS THE STAMP THE BROWSER ALREADY SENDS, finally read. - # See `_event_rev`. The refusal is NOT caught: `main.py:232` turns - # `ViewRevisionStale` into `409 view_revision_stale {rev_current}` for exactly this - # door, and `RevisionUnverifiable` into a 503 beside it, so both reach the client in - # the shape `routes_tables.py`'s direct view routes already answer. - # S82-T156: a stale stamp that undoes no later change lands (D-634). view = (_tops(ctx).save_view( _save_as, view, reserved_names=_reserved_view_names, - actor=uname, origin=EVENT_ORIGIN, rev=_event_rev(event), - rebase=True) or view) + actor=uname, origin=EVENT_ORIGIN) or view) else: _session_ready() _view_names = list(_reserved_view_names) @@ -2307,8 +2846,9 @@ def handle_one(event, ctx): if ftype in ('link', 'rollup') or (ftype == 'if' and scope_key.startswith('ut_')): return _refuse( ctx, 'wrong_door', - 'a link or rollup column belongs to the table schema, not to a personal ' - 'overlay - create it from the Fields panel instead', key) + 'Create this schema field from the Fields panel through ' + 'POST /api/v1/tables/{key}/fields. On a connected registry database, ' + 'only an administrator can create a Link field here.', key) readonly = ftype in _ag2.READONLY_CUSTOM_TYPES # The editable family must SAY it is overlay (the client's own contract); the # read-only pair (formula / created_time) is stored under the cohort column's @@ -2594,7 +3134,8 @@ def handle_one(event, ctx): fresh_fields = _shared_overlay.fields(shared_storage_key, st=ctx.table.st) or {} else: fresh_fields = (table_workspace( - ctx, allowed_pids=allowed_pids, consume_corrections=False).get('fields') or {}) + ctx, allowed_pids=allowed_pids, consume_corrections=False, + admission=False).get('fields') or {}) durable_field = fresh_fields.get(key) if source_backed: durable_field = _source_echo(field, durable_field) @@ -3073,67 +3614,21 @@ def handle_one(event, ctx): placed, clean, {sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES}) - # ⚠ NO allowed_pids ON PURPOSE — this is app.py's bare `_cl_table_workspace(uname)` - # call at the old :6268, preserved exactly. Omitting the pool drops memberPids from - # any foreign shared view (fail-closed), which is harmless here because the folder - # legs read only view IDS and the folder maps. Passing the pool would be a behavior - # change inside a pure move. - # S82-T156: the revision this tree is computed at, so a stale stamped move can land - # guarded there when no view it files was moved after the stamp (D-634). - _reader = getattr(_tops(ctx), 'revision', None) - _tree_rev = (_reader() if callable(_reader) and _event_rev(event) is not None - and _store_of(ctx).available() else None) - _res = _mutate(table_workspace(ctx, consume_corrections=False)) - if _res is None: - # Refused: nothing is written — and now the caller is TOLD, with the cause `_mutate` - # recorded at the line that decided it. The fallback exists so a future `return None` - # added without a `_no(...)` degrades to a generic sentence rather than to silence. + # Folder gestures mutate the latest owner row inside the write, without a whole-tree stamp. + try: + _receipt = _tops(ctx).save_folders(uname, mutate=_mutate, + actor=uname, origin=EVENT_ORIGIN) + except _tstore.FolderTreeError as _fte: + return _folder_refuse(ctx, kind, + str(getattr(_fte, 'reason', '') or 'folder_tree_illegal'), + str(_fte), key=str(getattr(_fte, 'folder_id', '') or '')) + if _receipt is None: return _folder_refuse( ctx, kind, why.get('code') or 'folder_change_refused', why.get('message') or ('That folder change could not be saved, so nothing was ' 'altered. Reload the page and try again.'), key=why.get('key') or '') - # ⭐⭐ W41-T12's HANDOVER, TAKEN. `save_folders` VALIDATES before it writes and RAISES - # `FolderTreeError` on a cycle or an over-deep chain, having written nothing. Letting that - # escape would 500 the whole event window; catching it and dropping the result would put - # it straight back into the silent-200 path this ticket exists to close. `str(exc)` is - # already the sentence a person reads and already C7-clean, and `folder_id` is an - # attribute, so nothing is parsed out of the message. - # - # ⭐⭐ REACHABLE NOW, AND THIS COMMENT SAID THE OPPOSITE UNTIL /wave-validate CHANGED IT. - # It read: 'UNREACHABLE FROM THIS DOOR TODAY ... `clean_folders` STRIPS `parent` entirely, - # so no nesting can reach the validator through here and neither raise can fire. This is - # the forward contract with T12: the day `clean_folders` learns `parent`, the refusal is - # already wired and worded.' - # - # That day is 2026-08-25. `aios_grid.clean_folders` carries `parent` through, the - # `folder_create` branch above forwards it, and `validate_folder_tree` now fires for real: - # an 11-deep chain raises `too_deep` naming the folder, a cycle raises `cycle`, and a - # dangling parent REPAIRS to root with a receipt this branch turns into a toast. Driven - # both ways, including the negative control that a payload with no `parent` anywhere comes - # back byte-identical with zero repairs. - # - # ⛔ LEFT AS A CORRECTION RATHER THAN DELETED. A comment asserting a guard is dead is - # exactly what stops the next reader checking, and this one would have outlived the fix. - try: - # ⭐⭐ W51-T33 — THE SECOND STAMPED VIEW WRITE, and it is stamped by the client's - # own decision: `CustomerGrid.tsx:10075` emits the views-rail `item_move` through - # `stampViewWrite`, with the comment *"`move` is one of C3's seven audited actions - # and it is a view WRITE like any other"*. `save_folders` has taken `rev=` since - # W50-T25, so the value had a door on both sides and no line between them. - # ⚠ THE OTHER FOUR KINDS THAT LAND HERE ARE UNAFFECTED. `folder_create`, - # `folder_rename`, `folder_delete` and `folder_reorder` carry no stamp, and neither - # does `ViewSidebar.tsx:2022`'s folder-nest `item_move`, so `_event_rev` answers None - # for every one of them and the comparison is not asked — byte-identical behaviour. - _receipt = _tops(ctx).save_folders(uname, _res[0], _res[1], - actor=uname, origin=EVENT_ORIGIN, - rev=_event_rev(event), - rebase_at=_tree_rev) - except _tstore.FolderTreeError as _fte: - return _folder_refuse(ctx, kind, - str(getattr(_fte, 'reason', '') or 'folder_tree_illegal'), - str(_fte), key=str(getattr(_fte, 'folder_id', '') or '')) # ⭐⭐ THE DURABILITY HALF — the owner's actual complaint. See `_commit_folder_write`: the # write above lands in `Store._cache` and the hub commit is deferred by up to ~22s, so # a second process (or the same one after a restart) cannot read it back. Forced here, @@ -3903,7 +4398,8 @@ def handle_one(event, ctx): _history_after.update(updates) _history_after.update(dict(row_events)) _history_st = ctx.st - if _history_table.startswith('ut_') and ctx.table is not None: + if ctx.table is not None and (_history_table.startswith('ut_') + or _history_table in ('customer', 'product')): # One receipt carries every committed field delta, even when legacy hooks fan out. changes = {key: {'before': _history_before.get(key), 'after': _history_after.get(key)} @@ -3914,7 +4410,7 @@ def handle_one(event, ctx): 'type': 'event_field', 'table': _history_table, 'rowId': str(pid), 'field': _k, 'after': _v, 'st': ctx.table.st, 'user': uname, 'receiptId': event_id, 'changes': changes}) - elif _history_table in ('customer', 'product') and getattr(_history_st, 'key', None): + if _history_table in ('customer', 'product') and getattr(_history_st, 'key', None): from core import received_changes changes = {key: {'before': _history_before.get(key), 'after': _history_after.get(key)} for key in set(updates) | {key for key, _value in row_events} diff --git a/platform/core/module_tables.py b/platform/core/module_tables.py index ac2ff946972c3880af591e7b01ee3f8bfdb12510..82cd30ee3d9564e52a964db7136ea77ccb77842e 100644 --- a/platform/core/module_tables.py +++ b/platform/core/module_tables.py @@ -42,12 +42,12 @@ def declared_sources(definition): def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None, label=None, - fields=None, owned_keys=None, sources=None, stale=None, repair=None, + fields=None, owned_keys=None, sources=None, field_limit=None, stale=None, repair=None, error=ModuleTableRefused): """Create or adopt module tables; add missing columns, never retype one. Returns the spec keys. `specs` holds `{key, label, fields}` dicts with stored field definitions, plus optional - `sources`. A stored label that differs is rewritten. `stale(spec, stored)` and + `sources` and `fieldLimit`. A stored label that differs is rewritten. `stale(spec, stored)` and `repair(spec, table)` let the caller add its own repairs to the one store write. """ ut = _ut() @@ -60,10 +60,15 @@ def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None, spec["ownedKeys"] = list(owned_keys) if sources is not None: spec[SOURCES_KEY] = list(sources) + if field_limit is not None: + spec['fieldLimit'] = field_limit specs = [spec] elif table_key is not None: raise TypeError("Pass either specs or table_key.") specs = [dict(spec) for spec in specs] + for spec in specs: + if 'fieldLimit' in spec and (type(spec['fieldLimit']) is not int or spec['fieldLimit'] <= 0): + raise ValueError('A module fieldLimit must be a positive integer.') keys = [spec["key"] for spec in specs] ut.register_module_tables(*keys) @@ -71,12 +76,16 @@ def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None, have = ut.get_one(spec["key"], st) if have is None or have.get("label") != spec["label"]: return True + if have.get(ut.MODULE_OWNED_KEY) is not True: + return True if not {f["key"] for f in spec["fields"]} <= {f.get("key") for f in have.get("fields") or []}: return True if SOURCES_KEY in spec and declared_sources(have) != _clean_sources(spec[SOURCES_KEY]): return True if "ownedKeys" in spec and have.get("ownedKeys") != spec["ownedKeys"]: return True + if 'fieldLimit' in spec and have.get('fieldLimit') != spec['fieldLimit']: + return True return bool(stale and stale(spec, have)) todo = [spec for spec in specs if _is_stale(spec)] @@ -93,19 +102,25 @@ def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None, if table is None: # A module table takes no cap slot (register_module_tables), so nothing refuses here. table = cur[key] = {"key": key, "label": want_label, "source": ut.AUTOMATION_SOURCE, + ut.MODULE_OWNED_KEY: True, "createdBy": username or owner, "created": created, "fields": [dict(f) for f in wanted], "rows": {}} if SOURCES_KEY in spec: table[SOURCES_KEY] = list(_clean_sources(spec[SOURCES_KEY])) if "ownedKeys" in spec: table["ownedKeys"] = list(spec["ownedKeys"]) + if 'fieldLimit' in spec: + table['fieldLimit'] = spec['fieldLimit'] continue + table[ut.MODULE_OWNED_KEY] = True if table.get("label") != want_label: table["label"] = want_label if SOURCES_KEY in spec: table[SOURCES_KEY] = list(_clean_sources(spec[SOURCES_KEY])) if "ownedKeys" in spec: table["ownedKeys"] = list(spec["ownedKeys"]) + if 'fieldLimit' in spec: + table['fieldLimit'] = spec['fieldLimit'] present = {f.get("key") for f in table.get("fields") or []} if repair is not None: repair(spec, table) diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index a136cfdaecacf52a69c523f13e29c4461ecdd66b..9479ce74350210c26d1f712bd617cadd62967fe8 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -908,7 +908,7 @@ def _enrich_stored_wall(leaves, rows, fields, module, st): def _wall_source_revisions(st, table_key, tree): """Every derived dependency contributes its storage revision, including mirror syncs.""" - from harness import filter_eval as fe + from core import filter_eval as fe leaves = _wall_leaf_keys(fe.tree_parts(tree)[0]) if not leaves: return {} @@ -1065,7 +1065,7 @@ def _enrich_for_wall(tree, rows, fields, module, st): set difference and reaches no store at all. """ try: - from harness import filter_eval as fe + from core import filter_eval as fe # ⛔ `tree_parts`, NEVER `tree['nodes']`. A `FilterTree` is a PAIR and a BARE LIST is also # a legal shape (C-PERM amendment 2); a second reader of it here would answer "no leaves" # for a wall the evaluator one line down reads perfectly well. @@ -1369,7 +1369,7 @@ def apply_row_scope(rows, user, module, fields, ctx=None, st=None, defn=None, re caller's held table definition to a storeless door without changing the ordinary wall. `permits`, never `matches` — an unanswerable permanent filter DENIES rather than being - ignored. See `harness/filter_eval`'s docstring for the field-rename walkthrough that makes + ignored. See `core/filter_eval`'s docstring for the field-rename walkthrough that makes the difference a leak rather than a preference. ⭐⭐ `st` IS OWNER I16's SECOND HALF AND IT IS OPTIONAL SO THE FIRST HALF CANNOT MOVE. @@ -1396,7 +1396,7 @@ def apply_row_scope(rows, user, module, fields, ctx=None, st=None, defn=None, re tree = (e or {}).get('filter') if not tree: return list(rows or ()) - from harness import filter_eval as fe + from core import filter_eval as fe if tree: # ⛔⛔ THE ENRICHMENT ANSWERS THE WALL AND MUST NEVER REACH A CALLER. It merges a # column's value onto a COPY of each row so `permits` can evaluate a leaf naming it; @@ -1446,7 +1446,7 @@ def assistant_apply_row_scope(rows, user, module, fields, ctx=None, st=None, def tree = e.get('filter') if not tree: return list(rows or ()) - from harness import filter_eval as fe + from core import filter_eval as fe if tree: # Same rule as `apply_row_scope`: judge on the copy, return the ORIGINAL. See the block # there for the measured leak this prevents. @@ -1477,7 +1477,7 @@ def validate_assistant_filter(tree, fields): """ if tree is None: return None - from harness import filter_eval as fe + from core import filter_eval as fe from core.filter_contract import MAX_FILTER_VALUES by_key = {str(field.get('key')): field for field in (fields or ()) @@ -1988,7 +1988,7 @@ def _derive_from_filter(user, module): return perms.scope_team_id(user), perms.scope_agent(user) return None, None - from harness import filter_eval as fe + from core import filter_eval as fe nodes, conj = fe.tree_parts(tree) if conj == 'or': return None, None @@ -2265,7 +2265,7 @@ def describe_scope(user, module, fields, st=None): tree = (entry(user, module) or {}).get('filter') parts, rows = [], [] if tree: - from harness import filter_eval as fe + from core import filter_eval as fe hidden = set(hidden_keys(user, module, list(by_key.values()), st=st)) if by_key else set() nodes, conj = fe.tree_parts(tree) nodes = [n for n in nodes if isinstance(n, dict)] @@ -3012,7 +3012,8 @@ def _window_number_sql(expr, *, postgres=False, json_sql=None): def _window_wall(user, key, fields, columns, ctx=None, *, postgres=False, id_sql=None, st=None): """Compile each permanent leaf separately so an inactive OR branch stays false.""" - from harness import filter_eval as fe, filter_sql as fs + from core import filter_eval as fe + from harness import filter_sql as fs tree = None if perms.is_admin(user) else (entry(user, key) or {}).get('filter') if not tree: return 'TRUE', [] @@ -3362,7 +3363,7 @@ class _UserTableWindow: 'type': 'int' if f.get('type') in ('rating', 'formula', 'autonumber') else 'date' if f.get('type') in ('created_time', 'last_modified_time') else f.get('type', 'text')} for f in self.fields if f.get('key')} - from harness import filter_eval + from core import filter_eval nodes, _conj = filter_eval.tree_parts((entry(user, self.key) or {}).get('filter')) def rhs_keys(nodes): diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index 21f6bdc9bf0669e49df1c5f5cf105bd34292c20b..33aca53662df47609a07543f60dcbdc7140a40ab 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -621,9 +621,10 @@ def _pool(): # default. Thirty seconds still reuses a normal request burst without creating a resident # database client. limits = _pg_limits() + # No server-side prepares: a failed pipeline or PgBouncer leaves psycopg naming an absent statement. connection_kwargs = dict(connect_timeout=limits['connect'], keepalives=1, keepalives_idle=30, keepalives_interval=10, keepalives_count=3, - tcp_user_timeout=limits['network_ms']) + tcp_user_timeout=limits['network_ms'], prepare_threshold=None) if sslrootcert is not None: connection_kwargs['sslrootcert'] = sslrootcert # The pool applies transaction settings and lends connections with checked socket polls. @@ -723,6 +724,9 @@ class SharedReadTimeout(TimeoutError): def _bump_write_generation(name, tenant_slug): _request_timing.clear_read_memo() key = (_effective_slug(tenant_slug), str(name)) + layouts = _layout_memo() + if layouts is not None: + layouts.pop(key, None) with _READ_FLIGHT_LOCK: _WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1 with _WHOLE_TEXT_LOCK: @@ -885,15 +889,38 @@ def _validated_content(name, drops, tenant_slug): return False, None +#: Accumulator attribute holding one request's settings-layout answers (S85-T169). +_LAYOUT_MEMO = '_store_pg_layout_probes' + + +def _layout_memo(): + """`{(slug, name): migrated}` for the current request, or None outside one or in a caller transaction.""" + acc = _request_timing.CURRENT.get() + if acc is None or acc.closed or _CALLER_TRANSACTION.get() is not None: + return None + with acc.lock: + return acc.__dict__.setdefault(_LAYOUT_MEMO, {}) + + def _view_rows(name, tenant_slug=None): - """Resolve an explicitly migrated workspace without creating schema or settings.""" + """Resolve an explicitly migrated workspace without creating schema or settings. + + A request reads each workspace's layout marker once. Its own write to that key reads it again. + """ from core import view_settings_rows as settings if not settings.supports(name): return None + layouts = _layout_memo() + key = (_effective_slug(tenant_slug), str(name)) + if layouts is not None and key in layouts: + return settings.PgRows(name, key[0]) if layouts[key] else None with _pool().connection() as con: row = con.execute(_q("SELECT value ->> '__layout__' FROM {tbl} WHERE key = %s", 'store_kv', tenant_slug), (str(name),)).fetchone() - return settings.PgRows(name, _effective_slug(tenant_slug)) if row and row[0] == settings.LAYOUT else None + migrated = bool(row and row[0] == settings.LAYOUT) + if layouts is not None: + layouts[key] = migrated + return settings.PgRows(name, _effective_slug(tenant_slug)) if migrated else None def _view_rows_many(names, tenant_slug=None): @@ -2358,7 +2385,8 @@ def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row): raise settings.MigrationRequired('row-backed settings require explicit owner-row mutations') if _ut_marker(name, tenant_slug, data): raise _RowsLayoutFound(name) - result = fn(data) + with _lend_holding(con, tenant_slug): + result = fn(data) data = result if result is not None else data payload = _payload(name, data, con, tenant_slug) # The write, the audit rows and COMMIT go in one round trip on this module's pool. @@ -2394,6 +2422,22 @@ def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row): return data +@contextmanager +def _lend_holding(con, tenant_slug): + """Serve the callback's nested store calls on the connection that holds the row lock. + + A second pooled borrow under the lock deadlocks a full pool and starves sign-in (D-3480). + """ + if _CALLER_TRANSACTION.get() is not None: + yield + return + token = _CALLER_TRANSACTION.set(_EnlistedPool(con, _effective_slug(tenant_slug))) + try: + yield + finally: + _CALLER_TRANSACTION.reset(token) + + class _RowsLayoutFound(Exception): """The locked `user_tables` value is the row-mode marker: roll back and write rows instead.""" diff --git a/platform/core/table_store.py b/platform/core/table_store.py index 6073dc3464a6e7415032fb23ab2ad80c342c8de8..35229edef5d07f8a9bbad7d5cb34335c64b8576c 100644 --- a/platform/core/table_store.py +++ b/platform/core/table_store.py @@ -16,8 +16,8 @@ A LIST (membership/formula semantics) is deliberately a different store from a V (presentation/query state) — see modules/customer_data.py's customer_lists key. """ import copy -import hashlib import json +import re import time import core.store as store @@ -88,6 +88,9 @@ RESERVED_MEMBERS = (SHARED_KEY, ASSIGNED_KEY, AUDIT_KEY) #: is NOT in `RESERVED_MEMBERS` (that tuple names TOP-LEVEL members that are not users). TRASH_KEY = 'trash' +#: `find_view(settings=...)` default: resolve the settings-rows backend on this call. +SETTINGS_UNRESOLVED = object() + #: R9's retention, in days. ⛔ A CONSTANT RATHER THAN A LITERAL: it is read by `delete_view`, #: by `trash()` and by the gate that proves a 31-day-old entry purges while a 29-day-old one #: does not, and three copies of `30` is how two of them drift. @@ -97,146 +100,180 @@ TRASH_RETENTION_DAYS = 30 #: matching `view_audit.row`'s `ts` and `store_pg.revision`'s `updated_at` on both backends). TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60 -# ══════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ S82-T156 (D-634) — A STALE VIEW WRITE IS REFUSED ONLY WHEN IT WOULD UNDO A LATER CHANGE. -# -# `/workspace` publishes the store revision, and the browser stamps it on a view write. The -# revision moves on EVERY write to the table (a view select, a cell, a field note), and the -# browser re-reads only after some writes. So "stamp differs" alone refuses one tab's own second -# write, and the outbox then resends that refused write forever. -# -# Each view write records its author, landed revision and the member digests from before it. -# The next before-image isolates each write's changes. Known requester writes are skipped. -# A foreign or unknown writer's changed member stays protected when this write would replace it. -# -# ⛔ The record lives in the owner's own stratum beside `views`, so it rides the same update and -# the same settings row. `workspace()` never projects it. -# ⚠ An unstamped write tags itself with a revision read just before its update. A write that -# lands between that read and the update makes the tag one low, so a conflict in that window can -# be missed. The stamped path tags exactly. -# ══════════════════════════════════════════════════════════════════════════════════════════════ +_VIEW_OP_MEMBERS = 'viewOpMembers' +_VIEW_OP_TOP = frozenset(('name', 'note', 'locked')) +_VIEW_OP_CONFIG = frozenset(('group', 'display', 'filterConj', 'colorBy', 'rowHeightMode', + 'frozenCount', 'important', 'secondaryView', 'application', 'fullscreen')) +_VIEW_OP_FAMILIES = frozenset(('hidden', 'sort', 'filter', 'width', 'position', 'summary', 'memberPid')) +_VIEW_OP_POSITION = re.compile(r'[0-9A-Za-z]{1,64}\Z') -#: The per-owner member holding each view's recent write record. -VIEW_WRITES_KEY = 'viewWrites' -# Folder ids are surface-scoped and keep history separate from view ids. -FOLDER_WRITES_KEY = 'folderWrites' -#: Writes kept per view. Older senders need a matching trimmed-author boundary. -VIEW_WRITES_KEEP = 8 -#: Views kept per stratum. The least recently written record goes first. -VIEW_WRITES_VIEWS = 400 -#: Members the server sets or derives, so two writers never disagree about them. -_DERIVED_VIEW_MEMBERS = frozenset({'updatedAt', 'filterRemovals', 'createdBy', 'hasShares', - 'sharedOut'}) +def _view_member_identity(member): + """Use one receipt and last-writer identity for equivalent record ids.""" + if isinstance(member, str) and member.startswith('memberPid:'): + try: + return 'memberPid:' + str(int(member.partition(':')[2])) + except ValueError: + pass + return member -def _member_digest(value): - text = json.dumps(value, sort_keys=True, separators=(',', ':'), default=str) - return hashlib.sha1(text.encode('utf-8')).hexdigest()[:16] +class _ViewOpRefused(ValueError): + def __init__(self, reason, message): + super().__init__(message) + self.reason = reason -def view_members(view): - """`{member: digest}` for a saved view, one member per config key, or None for no view.""" - if not isinstance(view, dict): - return None - out = {} - for key, value in view.items(): - if key in _DERIVED_VIEW_MEMBERS: - continue - if key == 'config' and isinstance(value, dict): - out.update((f'config.{name}', _member_digest(item)) for name, item in value.items()) - else: - out[str(key)] = _member_digest(value) - return out +def _view_filter_nodes(nodes, parent=None): + for node in nodes: + yield node, parent, nodes + if isinstance(node.get('children'), list): + yield from _view_filter_nodes(node['children'], node.get('id')) -def _note_write(ws, view_id, at, kind, before, book_key, *, author=None): - """Append one write to the selected bounded history in this owner's stratum.""" - if at is None or not view_id or not isinstance(ws, dict): + +def prune_view_field_members(view, *, live_keys=None, removed_keys=(), prune_filters=True): + """Drop dead field references from a mutable View without changing live member values.""" + cfg = view.get('config') if isinstance(view, dict) else None + if not isinstance(cfg, dict): return - book = ws.get(book_key) - if not isinstance(book, dict): - book = ws[book_key] = {} - entry = book.get(view_id) if isinstance(book.get(view_id), dict) else {} - log = [row for row in entry.get('log') or () if isinstance(row, dict)] - author = str(author or '').strip().lower() or None - log.append({'rev': int(at), 'kind': kind, 'before': before, 'author': author}) - since = int(entry.get('since') or 0) - trimmed = entry.get('trimmed') - trimmed = dict(trimmed) if isinstance(trimmed, dict) else {} - while len(log) > VIEW_WRITES_KEEP: - removed = log.pop(0) - removed_author = removed.get('author') - # Only a known, uninterrupted author run can excuse writes outside the ring. - if not removed_author or removed_author != trimmed.get('author'): - trimmed = {'author': removed_author, 'after': since} - since = max(since, int(removed.get('rev') or 0)) - book[view_id] = {'since': since, 'log': log} - if trimmed: - book[view_id]['trimmed'] = trimmed - if len(book) > VIEW_WRITES_VIEWS: - # jsonb keeps no key order, so recency is read off each record's newest revision. - def newest(vid): - rows = (book.get(vid) or {}).get('log') or () - return max((int(row.get('rev') or 0) for row in rows if isinstance(row, dict)), - default=0) - for vid in sorted(book, key=newest)[:len(book) - VIEW_WRITES_VIEWS]: - book.pop(vid, None) - - -def note_view_write(ws, view_id, at, kind, before, *, author=None): - """Append one view, gone or place write to this stratum's record.""" - _note_write(ws, view_id, at, kind, before, VIEW_WRITES_KEY, author=author) - - -def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY): - """`(entries after sent, judgeable)` for one view, oldest first.""" - book = (ws.get(book_key) or {}) if isinstance(ws, dict) else {} - entry = book.get(view_id) if isinstance(book, dict) else None - if not isinstance(entry, dict): - return [], True - later = [row for row in entry.get('log') or () - if isinstance(row, dict) and row.get('kind') in kinds - and int(row.get('rev') or 0) > int(sent)] - return later, int(sent) >= int(entry.get('since') or 0) - - -def stale_view_conflict(ws, view_id, mine, sent, *, requester=None): - """Whether saving `mine` over `view_id` undoes a change made after revision `sent`.""" - later, judgeable = _later_writes(ws, view_id, sent, ('view', 'gone')) - requester = str(requester or '').strip().lower() or None - if not judgeable: - entry = (ws.get(VIEW_WRITES_KEY) or {}).get(view_id) or {} - trimmed = entry.get('trimmed') - trimmed = trimmed if isinstance(trimmed, dict) else {} - if not (requester and trimmed.get('author') == requester - and int(sent) >= int(trimmed.get('after', entry.get('since') or 0))): - return True - current = ((ws.get('views') or {}).get(view_id)) - now, want = view_members(current) or {}, view_members(mine) or {} - for index, row in enumerate(later): - if requester and row.get('author') == requester: - continue - if not isinstance(current, dict): - return True - before = row.get('before') or {} - after = (later[index + 1].get('before') or {}) if index + 1 < len(later) else now - # The next before-image isolates this writer's changes from the requester's later edits. - if any(before.get(member) != after.get(member) and want.get(member) != now.get(member) - for member in set(before) | set(after)): - return True - return False - - -def stale_place_conflict(ws, view_id, folder, sent): - """Whether filing `view_id` in `folder` undoes a move made after revision `sent`.""" - later, judgeable = _later_writes(ws, view_id, sent, ('place',)) - if not later: - return False - if not judgeable: - return True - current = ((ws.get('itemFolders') or {}).get('views') or {}).get(view_id) - base = (later[0].get('before') or {}).get('folderId') - return current != base and folder != current + removed = set(removed_keys) + + def keep(key): + return key not in removed and (live_keys is None or key in live_keys) + + for name in ('visible', 'order'): + if isinstance(cfg.get(name), list): + cfg[name] = [key for key in cfg[name] if keep(key)] + for name in ('widths', 'positions', 'summary'): + if isinstance(cfg.get(name), dict): + cfg[name] = {key: value for key, value in cfg[name].items() if keep(key)} + if isinstance(cfg.get('sorts'), list): + cfg['sorts'] = [rule for rule in cfg['sorts'] + if isinstance(rule, dict) and keep(rule.get('colId'))] + for name in ('groupBy', 'colorBy'): + if cfg.get(name) is not None and not keep(cfg[name]): + cfg.pop(name, None) + + def filters(nodes): + result = [] + for node in nodes: + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + node = {**node, 'children': filters(node['children'])} + elif not keep(node.get('colId')) or (node.get('rhsColId') and not keep(node['rhsColId'])): + continue + result.append(node) + return result + + if prune_filters and isinstance(cfg.get('filters'), list): + cfg['filters'] = filters(cfg['filters']) + + +def _view_position_seed(keys): + # Match the browser's initial column and legacy sort positions. + alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + result = {} + for index, key in enumerate(dict.fromkeys(keys), 1): + number, digits = index, '' + while number: + number, remainder = divmod(number, len(alphabet)) + digits = alphabet[remainder] + digits + result[key] = digits.rjust(8, '0') + 'V' + return result + + +def _view_member_value(view, member): + family, _, key = member.partition(':') + cfg = view.get('config') or {} + if family in _VIEW_OP_TOP: + return member in view, copy.deepcopy(view.get(member)) + if family == 'hidden': + return True, key not in cfg.get('visible', []) + if family == 'memberPid': + return True, int(key) in cfg.get('memberPids', []) + if family == 'sort': + hit = next((rule for rule in cfg.get('sorts', []) if rule.get('colId') == key), None) + return hit is not None, ({k: copy.deepcopy(v) for k, v in hit.items() if k != 'colId'} + if hit is not None else None) + if family in ('filter', 'filterConj') and key: + hit = next(((node, parent) for node, parent, _ in + _view_filter_nodes(cfg.get('filters', [])) if node.get('id') == key), None) + if family == 'filter': + return hit is not None, {'node': copy.deepcopy(hit[0]), 'parent': hit[1]} if hit else None + return bool(hit and 'children' in hit[0]), hit[0].get('conj', 'and') if hit else None + maps = {'width': 'widths', 'summary': 'summary', 'position': 'positions'} + if family in maps: + values = cfg.get(maps[family]) or {} + return key in values, copy.deepcopy(values.get(key)) + name = 'groupBy' if member == 'group' else member + return name in cfg, copy.deepcopy(cfg.get(name)) + + +def _set_view_member(view, member, value, *, present=True): + family, _, key = member.partition(':') + cfg = view.setdefault('config', {}) + if family == 'hidden': + visible = list(cfg.get('visible') or []) + cfg['visible'] = ([item for item in visible if item != key] if value else + list(dict.fromkeys([*visible, key]))) + elif family == 'memberPid': + pids = list(cfg.get('memberPids') or []) + cfg['memberPids'] = ([pid for pid in pids if pid != int(key)] if not present or not value + else list(dict.fromkeys([*pids, int(key)]))) + elif family == 'sort': + rules = list(cfg.get('sorts') or []) + if any('pos' not in rule for rule in rules): + seeds = _view_position_seed(rule['colId'] for rule in rules) + rules = [{**rule, 'pos': rule.get('pos', seeds[rule['colId']])} for rule in rules] + rules = [rule for rule in rules if rule.get('colId') != key] + if present: + rules.append({'colId': key, **copy.deepcopy(value)}) + cfg['sorts'] = sorted(rules, key=lambda rule: (rule['pos'], rule['colId'])) + elif family == 'filter': + nodes = cfg.setdefault('filters', []) + found = list(_view_filter_nodes(nodes)) + target = next((node for node, _, _ in found if node.get('id') == key), None) + parent = value.get('parent') if present else None + destination = nodes + if parent is not None: + destination = next((node.get('children') for node, _, _ in found + if node.get('id') == parent and isinstance(node.get('children'), list)), None) + descendants = ({node.get('id') for node, _, _ in + _view_filter_nodes([target])} if target else set()) + if destination is None or parent in descendants: + raise _ViewOpRefused('target_gone', 'That filter group is no longer available.') + for node, _, siblings in found: + if node.get('id') == key: + siblings.remove(node) + if present: + node = copy.deepcopy(value['node']) + if target and isinstance(target.get('children'), list) and 'children' in node: + node['children'] = target['children'] + node['id'] = key + destination.append(node) + elif family == 'filterConj' and key: + target = next((node for node, _, _ in _view_filter_nodes(cfg.get('filters', [])) + if node.get('id') == key and isinstance(node.get('children'), list)), None) + if target is None: + raise _ViewOpRefused('target_gone', 'That filter group is no longer available.') + target['conj'] = value + elif family in ('width', 'summary', 'position'): + name = {'width': 'widths', 'summary': 'summary', 'position': 'positions'}[family] + values = cfg.setdefault(name, {}) + if present: + values[key] = copy.deepcopy(value) + else: + values.pop(key, None) + if family == 'position': + cfg['order'] = sorted(values, key=lambda item: (values[item], item)) + else: + target = view if family in _VIEW_OP_TOP else cfg + name = 'groupBy' if member == 'group' else member + if present: + target[name] = copy.deepcopy(value) + else: + target.pop(name, None) class ViewNotInTrash(LookupError): @@ -785,7 +822,7 @@ class TableStore: return self._st.get(self.table_key) or {} def revision(self): - """The revision `_audited_update(rev=)` compares for this workspace, or None if unreadable.""" + """The workspace change revision, or None if unreadable.""" rows = self._settings_rows if rows is not None: with rows._reading() as con: @@ -810,30 +847,8 @@ class TableStore: rev = self.revision() return rev, self._st.get(self.table_key) or {} - def _next_rev(self, rev): - """The revision a write lands as: exact after a passed stamp, else read just before.""" - if rev is not None: - return int(rev) + 1 - current = self.revision() - return None if current is None else current + 1 - - def _land_stale(self, owner, sent, stale, conflict, write, at=None): - """Land a stale stamped write that undoes no later change, else raise its refusal. - - `write(rev)` repeats the write guarded at `rev`. `at` pins that revision for a write - whose payload was computed from the workspace as it stood then. - """ - for _attempt in range(1 if at is not None else 3): - current, data = self._read_owners_at(owner) - if current is None or conflict((data or {}).get(owner) or {}, int(sent)): - break - try: - return write(current if at is None else at) - except view_audit.ViewRevisionStale as again: - stale = again - raise view_audit.ViewRevisionStale(rev_current=stale.rev_current, rev_sent=int(sent)) - - def log_view_activity(self, view_id, change, *, by, before=None, after=None, via=None): + def log_view_activity(self, view_id, change, *, by, before=None, after=None, via=None, + member=None, overwrote=None): """Append one View event through the exact workspace store after its write succeeds.""" from core.user_tables import _activity_writer append = _activity_writer(self._st, by) @@ -844,10 +859,17 @@ class TableStore: if workspace == self.table_key), self.table_key.removesuffix('_table_workspace')) if isinstance(via, dict): via = via.get('client') or None + changes = {} + if member is not None: + changes.update(member={'before': None, 'after': member}, + by={'before': None, 'after': by}, + overwrote={'before': None, 'after': overwrote}) + changes[change] = {'before': before, 'after': after} append(kind='view', object={'database': database, 'id': str(view_id)}, by=by, - via=via, changes={change: {'before': before, 'after': after}}) + via=via, changes=changes) - def find_view(self, view_id, prefer_owner=None, include_trashed=False): + def find_view(self, view_id, prefer_owner=None, include_trashed=False, *, document=None, + settings=SETTINGS_UNRESOLVED): """`(owner_username, view)` for a view living in ANY personal stratum, else None. ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted @@ -883,7 +905,8 @@ class TableStore: vid = str(view_id or '').strip() if not vid: return None - rows = self._settings_rows + # A caller looping over ids passes the backend it resolved once for this read (S85-T169). + rows = self._settings_rows if settings is SETTINGS_UNRESOLVED else settings if rows is not None: hit = rows.find_view(vid, prefer_owner) if hit is not None or not include_trashed: @@ -895,9 +918,10 @@ class TableStore: if isinstance(record, dict): return owner, dict(record, id=vid) return None - self.migrate_legacy_shared_views() + if document is None: + self.migrate_legacy_shared_views() try: - data = self._st.get(self.table_key) or {} + data = document if document is not None else self._st.get(self.table_key) or {} except Exception: return None mine = str(prefer_owner or '').strip() @@ -932,7 +956,7 @@ class TableStore: return str(who), dict(record, id=vid) return None - def find_folder(self, folder_id): + def find_folder(self, folder_id, *, document=None): """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any personal stratum, else None. `find_view`'s sibling, and here for the same reason. @@ -954,7 +978,7 @@ class TableStore: if rows is not None: return rows.find_folder(fid) try: - data = self._st.get(self.table_key) or {} + data = document if document is not None else self._st.get(self.table_key) or {} except Exception: return None for username, ws in data.items(): @@ -1015,7 +1039,7 @@ class TableStore: self.migrate_legacy_shared_views() rev = None try: - # S82-T156: `rev` is what the browser stamps on a view write (D-634). + # Publish the store change token independently of view write admission. rev, data = self._read_owners_at(username, SHARED_KEY, ASSIGNED_KEY) ws = data.get(username, {}) or {} # A collision acknowledgement is protocol state, not part of a field definition. @@ -1119,39 +1143,8 @@ class TableStore: return (out, data) if include_document else out # ---------------------------------------------------------------- write - def _audited_update(self, apply, *, rev=None, flush='async', owners=None): - """ONE workspace transaction carrying the C3 revision guard and its audit rows. - - ⭐⭐ W50-T25 (ruling R9 / contract C3) — THE DOOR EVERY VIEW MUTATION GOES THROUGH. - `apply(data, record)` receives the whole workspace document as the store's locked read - returned it, plus `record(...)` — the ONLY way to write an audit row. Calling `record` - inside `apply` is what makes the row and the workspace change the SAME commit: two - transactions would let the change land while the trail row failed, and the trail would - then have a hole exactly where the bug was (owner instruction 19). - - `rev` is the `store_kv.rev` the client last read, or None for no check. - - TWO ENFORCEMENT PATHS, and the difference is stated rather than implied: - - * IN-TRANSACTION (`view_audit.guards(st)`) — the handle's `update` takes `expect_rev`, - so the comparison happens against the value its own `SELECT … FOR UPDATE` returned - and NO writer can slip between the check and the write. `core/store_pg.py` alone - offers this. - * ADVISORY (everything else, which is every caller today) — read `revision()`, compare, - then write. A real TOCTOU window: another writer can land in between, so this refuses - every stale write it can SEE but cannot promise it saw all of them. - - ⛔ THE ADVISORY PATH IS NEVER SILENTLY SKIPPED. A handle that cannot answer `revision()` - at all makes the guard unanswerable, and an unanswerable guard fails CLOSED: it raises - `RevisionUnverifiable` rather than writing as though nothing had been asked. - - ⛔⛔ AND ADVISORY IS WHAT PRODUCTION GETS TODAY. `harness/runtime.TenantRuntime.update` - is `(self, name, fn, flush="sync")` with no keyword forward, and every API caller hands - this class `st=session.runtime`. One line in that file — outside W50-T25's fence — flips - every view write onto the in-transaction path and moves the trail's home from the - `__audit__` document ring to `control.audit`. Until then the ring IS the trail, and it is - atomic on Postgres too, because `apply` runs inside that backend's own transaction. - """ + def _audited_update(self, apply, *, flush='async', owners=None): + """Commit a workspace mutation and its audit rows in one store transaction.""" st = self._st row_backend = self._settings_rows # A LIST, because one folder move changes several views' placements and each is its own @@ -1160,6 +1153,7 @@ class TableStore: rows = [] def _apply(data): + rows.clear() data = data if isinstance(data, dict) else {} def record(*, action, owner, view_id, actor=None, origin=None, @@ -1170,7 +1164,7 @@ class TableStore: return None values = dict(table_key=self.table_key, owner=owner, view_id=view_id, action=action, actor=actor or owner, origin=origin, - before=before, after=after, rev=rev) + before=before, after=after) if row_backend is not None: entry = view_audit.row(tenant=row_backend.tenant_slug, **values) else: @@ -1184,27 +1178,12 @@ class TableStore: if row_backend is not None: if owners is None: raise ValueError('a row settings mutation needs explicit owner rows') - return row_backend.update(_apply, owners=owners, rev=rev, audit_rows=rows) - if view_audit.guards(st): - return st.update(self.table_key, _apply, flush=flush, - expect_rev=rev, audit_row=rows) - if rev is not None: - reader = getattr(st, 'revision', None) - if not callable(reader): - raise view_audit.RevisionUnverifiable( - 'this deployment cannot tell which revision of the view list you are ' - 'looking at, so the change was not saved. Reload and try again.') - try: - current = int((reader(self.table_key) or {}).get('rev') or 0) - except Exception as exc: # noqa: BLE001 - raise view_audit.RevisionUnverifiable( - 'the view list revision could not be read, so the change was not saved. ' - 'Reload and try again.') from exc - if current != int(rev): - raise view_audit.ViewRevisionStale(rev_current=current, rev_sent=int(rev)) + return row_backend.update(_apply, owners=owners, audit_rows=rows) + if view_audit.home(st) == view_audit.CONTROL_AUDIT: + return st.update(self.table_key, _apply, flush=flush, audit_row=rows) return st.update(self.table_key, _apply, flush=flush) - def _update(self, username, change, shared=None, *, rev=None, audit=None): + def _update(self, username, change, shared=None, *, audit=None): """Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the tenant-wide one, in ONE transaction. @@ -1251,7 +1230,7 @@ class TableStore: # the in-process cache (read-your-writes for every subsequent render); the hub write # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'. owners = (username, SHARED_KEY) if shared is not None else (username,) - return self._audited_update(_up, rev=rev, flush='async', owners=owners) + return self._audited_update(_up, flush='async', owners=owners) def rename_choice_values(self, username, change): """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME). @@ -1353,68 +1332,39 @@ class TableStore: self._update(username, _set) - def save_folders(self, username, folders, item_folders, *, - actor=None, origin=None, rev=None, rebase_at=None): - """Replace the folder stratum wholesale (wave-8 I11), once the tree is proven legal. - - Wholesale rather than per-folder because the caller has ALREADY sanitised the complete - picture through aios_grid.clean_folders / clean_item_folders, and those two are - interdependent: a placement is only legal while its folder exists, so committing them - separately would leave a window where a reader sees an item filed into a folder that is - not there yet. One write, one consistent state. - - ⭐⭐ W41-T12 (R1 + C9) — AND IT VALIDATES NOW. That first paragraph used to say the - caller had already VALIDATED, and until wave 41 the shape was flat enough for that to be - harmless. `parent` changes it: a depth cap and a cycle refusal that live only in the - caller are a convenience, the same two here are a wall, and nesting is the first folder - feature where the difference can hang a reader instead of merely looking wrong. - `validate_folder_tree` above carries the rules and the reasoning. - - ⛔ VALIDATION RUNS BEFORE `_update`, NEVER INSIDE ITS CALLBACK. `_update` pre-creates - five strata on the cached document and commits with flush='async', so a raise from - inside the callback would abandon a half-touched transaction whose commit is already in - flight. Raising first means a refused tree leaves the store exactly as it was. - - RETURNS the accepted picture and every repair made along the way: - `{'folders': …, 'itemFolders': …, 'repairs': [{'reason': …, …}]}`. It returned None - before; the return value is what lets a caller say WHY a folder came back at the root. - RAISES `FolderTreeError` on a cycle or an over-deep chain, having written nothing. - - `rebase_at` (S82-T156) is the revision the caller read the tree at. A stale `rev` then - lands, guarded at `rebase_at`, when it undoes no later view placement or folder parent change. + def save_folders(self, username, folders=None, item_folders=None, *, mutate=None, + actor=None, origin=None): + """Save a validated folder picture or apply `mutate(ws)` to the latest stored workspace. + + A mutation returns `(folders, item_folders)` or None to refuse without writing. + Validation precedes folder, placement and audit changes inside the update closure. + Events use this callback without a whole-tree revision guard, so unrelated gestures survive. + The receipt contains `folders`, `itemFolders` and `repairs`; illegal trees raise `FolderTreeError`. """ - tree, placed, repairs = validate_folder_tree(folders, item_folders) + tree, placed, repairs = (validate_folder_tree(folders, item_folders) if mutate is None + else ({}, {}, [])) + + class _FolderMutationRefused(Exception): + pass + # ⭐ W50-T25 — THIS IS C3's `move` DOOR, and the placement map is keyed by SURFACE # FIRST (`{'views': {viewId: folderId}}`). Reading item ids off the top level finds the # surface names instead and matches nothing, which is `find_folder`'s own recorded trap # one layer down; the audit hook would then silently record zero moves. was = {} - landing = {} - - def _parents(raw): - return {json.dumps([surface, str(row['id'])]): row.get('parent') - for surface, rows in (raw or {}).items() if isinstance(rows, list) - for row in rows if isinstance(row, dict) and row.get('id')} - - parents = _parents(tree) def _set(ws): + nonlocal tree, placed, repairs + if mutate is not None: + picture = mutate(ws) + if picture is None: + raise _FolderMutationRefused() + tree, placed, repairs = validate_folder_tree(*picture) was.clear() was.update({str(k): v for k, v in ((ws.get('itemFolders') or {}).get('views') or {}).items()}) - before_parents = _parents(ws.get('folders')) - for folder_id in sorted(set(before_parents) | set(parents)): - if before_parents.get(folder_id) != parents.get(folder_id): - _note_write(ws, folder_id, landing.get('at'), 'parent', - {'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY, - author=actor or username) ws['folders'] = tree ws['itemFolders'] = placed - now = placed.get('views') or {} - for view_id in sorted(set(was) | {str(k) for k in now}): - if was.get(view_id) != now.get(view_id): - note_view_write(ws, view_id, landing.get('at'), 'place', - {'folderId': was.get(view_id)}, author=actor or username) def _audit(ws, record): now = (ws.get('itemFolders') or {}).get('views') or {} @@ -1434,31 +1384,10 @@ class TableStore: before=dict(base, folderId=before_folder), after=dict(base, folderId=after_folder)) - def _write(at_rev): - landing['at'] = self._next_rev(at_rev) - self._update(username, _set, rev=at_rev, audit=_audit) - - def _conflict(ws, sent): - filed = (placed.get('views') or {}) - held = ((ws.get('itemFolders') or {}).get('views') or {}) - held_parents = _parents(ws.get('folders')) - for folder_id in set(parents) | set(held_parents): - current = held_parents.get(folder_id) - if parents.get(folder_id) == current: - continue - later, judgeable = _later_writes(ws, folder_id, sent, ('parent',), - book_key=FOLDER_WRITES_KEY) - if later and (not judgeable or current != (later[0].get('before') or {}).get('parent')): - return True - return any(stale_place_conflict(ws, vid, filed.get(vid), sent) - for vid in set(filed) | set(held) if filed.get(vid) != held.get(vid)) - try: - _write(rev) - except view_audit.ViewRevisionStale as stale: - if rev is None or rebase_at is None: - raise - self._land_stale(username, rev, stale, _conflict, _write, at=rebase_at) + self._update(username, _set, audit=_audit) + except _FolderMutationRefused: + return None # Fresh containers: the caller inspecting its receipt must not be able to reach into # the document the store is holding and edit it from underneath the cache. return {'folders': {s: list(rows) for s, rows in tree.items()}, @@ -1468,7 +1397,7 @@ class TableStore: def save_view_order(self, username, order): """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail. - Wholesale, like `save_folders` above and for the same reason: the client sends the full + Wholesale: the client sends the full list it is looking at, not a delta, because a partial order cannot say where an UNNAMED view went. @@ -1825,7 +1754,7 @@ class TableStore: return receipt def assign_view(self, owner, view, recipient, allowed_pids=(), *, - actor=None, origin=None, rev=None): + actor=None, origin=None): """Create one immutable assigned-view projection for ``recipient``. The filter snapshot is deliberately separate from the presentation @@ -1856,8 +1785,11 @@ class TableStore: } def _up(data, audit): record['updatedAt'] = write_execution.utcnow().isoformat() - bucket = data.setdefault(ASSIGNED_KEY, {}).setdefault('views', {}) + assigned = data.setdefault(ASSIGNED_KEY, {}) + bucket = assigned.setdefault('views', {}) bucket.setdefault(who, {})[assignment_id] = record + for member in (assigned.get(_VIEW_OP_MEMBERS) or {}).get(assignment_id, {}).values(): + member.update(by=actor or owner, client_op_id=None) # ⭐ W50-T25 — C3's `share` action. The audit `owner` is the SOURCE VIEW's owner, # not the recipient and not necessarily the actor: the trail answers "what happened # to whose view", and an assignment is something done TO the owner's record. @@ -1866,7 +1798,7 @@ class TableStore: before=None, after=dict(view_audit.summary(source) or {}, recipient=who)) return data - self._audited_update(_up, rev=rev, flush='async', owners=(ASSIGNED_KEY,)) + self._audited_update(_up, flush='async', owners=(ASSIGNED_KEY,)) self.log_view_activity(source_id, 'assigned', by=actor or owner, after=who, via=origin) return dict(record) @@ -1934,14 +1866,211 @@ class TableStore: if not rows: views.pop(who, None) if not views: - data.pop(ASSIGNED_KEY, None) + assigned = data[ASSIGNED_KEY] + assigned.pop('views', None) + if not assigned: + data.pop(ASSIGNED_KEY, None) return data self._audited_update(lambda data, _audit: _up(data), owners=(ASSIGNED_KEY,)) return dismissed['value'] + def _view_op_fields(self, workspace): + """Resolve current user-table fields without reading source records.""" + own = workspace.get('fields') or {} + if view_settings_rows.supports(self.table_key): + from core import user_tables + definition = user_tables.get_definition(self._shared_stratum_key(), st=self._st) or {} + return ({field['key'] for field in definition.get('fields', [])} + | {key for key in own if key.startswith(('custom_', 'measure_'))} + | self._tenant_wide_keys()) + # The event adapter checks built-in fields against its scoped field contract. + return None + + def apply_view_ops(self, owner, view_id, ops, *, actor, origin, + assigned_recipient=None, admit=None, seed_view=None): + """Apply one cleaned, authorized batch to the locked View and commit receipts with its member changes.""" + from core import edit_receipts + if owner in RESERVED_MEMBERS: + raise ValueError('reserved username') + view_id, actor = str(view_id), str(actor) + recipient = (str(assigned_recipient).strip().lower() + if assigned_recipient is not None else None) + if recipient is not None and (not recipient or recipient != actor.strip().lower() + or recipient != str(owner).strip().lower()): + raise ValueError('assigned presentation belongs to its recipient') + if seed_view is not None and (recipient is not None or owner != actor + or seed_view.get('id') != view_id + or seed_view.get('kind') != 'system'): + raise ValueError('only a personal system View can be seeded') + row_owner = ASSIGNED_KEY if recipient is not None else owner + requested = copy.deepcopy(list(ops)) + action = edit_receipts.operation(self.table_key, 'view_op', view_id) + results, activity = [], [] + + def apply(data, audit): + results.clear() + activity.clear() + ws = data.setdefault(row_owner, {}) + assignment = (((ws.get('views') or {}).get(recipient) or {}).get(view_id) + if recipient is not None else None) + receipts = view_settings_rows.ViewOpReceipts(ws, getattr(self._st, 'key', self.table_key)) + target_fields = {} + for operation in requested: + member = _view_member_identity(operation.get('member')) + identity = operation.get('client_op_id') + base = {'opId': identity, 'member': member, 'applied': False} + + def refuse(reason, message, **extra): + return {**base, 'refused': {'reason': reason, 'member': member, + 'message': message, **extra}} + + def land(): + view = (ws.get('views') or {}).get(view_id) + if (recipient is not None and isinstance(assignment, dict) + and isinstance(assignment.get('filter'), dict)): + view = {'id': view_id, 'kind': 'assigned', 'name': assignment.get('name'), + 'config': {**copy.deepcopy(assignment.get('presentation') or {}), + **copy.deepcopy(assignment['filter'])}} + if view is None and seed_view is not None: + view = copy.deepcopy(seed_view) + if not isinstance(view, dict) or view_id in (ws.get(TRASH_KEY) or {}): + return refuse('view_gone', 'That View is no longer available.') + if operation.get('view_id', view_id) != view_id: + return refuse('view_gone', 'That View is no longer available.') + if not isinstance(member, str): + return refuse('unknown_member', 'Choose a supported View setting.') + family, separator, key = member.partition(':') + if recipient is not None: + if family in ('filter', 'filterConj', 'memberPid'): + return refuse('assigned_filter_fixed', + 'The assigned View fixes its filters and record membership.') + if family in _VIEW_OP_TOP: + return refuse('unknown_member', 'Choose a presentation setting for this assigned View.') + if (family not in _VIEW_OP_TOP | _VIEW_OP_CONFIG | _VIEW_OP_FAMILIES + or (family in _VIEW_OP_FAMILIES and not key) + or (family == 'filterConj' and separator and not key) + or (separator and family in _VIEW_OP_TOP | _VIEW_OP_CONFIG + and family != 'filterConj')): + return refuse('unknown_member', 'Choose a supported View setting.') + kind, value = operation.get('op'), operation.get('value') + if kind not in ('set', 'remove', 'undo'): + return refuse('invalid_op', 'Choose set, remove or undo for this View setting.') + if 'keys' not in target_fields: + field_ws = (self._read_owners(owner).get(owner) or {} + if recipient is not None else ws) + target_fields['keys'] = self._view_op_fields(field_ws) + fields = target_fields['keys'] + target = key if family in ('hidden', 'sort', 'width', 'position', 'summary') else None + if target is not None and fields is not None and target not in fields: + return refuse('target_gone', 'That field is no longer available in this table.') + candidate = copy.deepcopy(view) + cfg = candidate.setdefault('config', {}) + if family == 'position' and 'positions' not in cfg: + cfg['positions'] = _view_position_seed(cfg.get('order') or []) + if family == 'sort': + rules = cfg.get('sorts') or [] + seeds = _view_position_seed(rule['colId'] for rule in rules) + cfg['sorts'] = [{**rule, 'pos': rule.get('pos', seeds[rule['colId']])} + for rule in rules] + before_present, before = _view_member_value(candidate, member) + writers = (ws.get(_VIEW_OP_MEMBERS) or {}).get(view_id) or {} + matching = ([writer for name, writer in writers.items() + if _view_member_identity(name) == member] if family == 'memberPid' else + [writers[member]] if member in writers else []) + previous = writers.get(member) or next(iter(matching), None) + present = kind != 'remove' + if kind == 'undo': + original = receipts.result(actor, action, value) + if (not original or not original.get('applied') + or _view_member_identity(original.get('member')) != member): + return refuse('changed_since', 'That change is not available to undo.', by=actor) + if ((before_present, before) != (original['afterPresent'], original['after']) + or any(writer.get('client_op_id') != value or writer.get('by') != actor + for writer in matching)): + by = previous.get('by') if previous else 'another editor' + return refuse('changed_since', f'{by} changed this View setting after your edit.', by=by) + present, value = original['beforePresent'], original['before'] + elif kind == 'remove' and family not in ('sort', 'width', 'filter', 'summary', 'memberPid'): + return refuse('invalid_op', 'Set a value for this View setting.') + if (family in ('group', 'colorBy') and present and value is not None + and fields is not None and value not in fields): + return refuse('target_gone', 'That field is no longer available in this table.') + if family == 'filter' and present: + node = value.get('node') if isinstance(value, dict) else None + if not isinstance(node, dict): + return refuse('invalid_filter', 'Choose a valid filter condition and parent group.') + columns = [item.get('colId') for item, _, _ in _view_filter_nodes([node]) + if item.get('colId')] + if fields is not None and any( + column not in fields and not column.startswith(('__', 'measure_')) + for column in columns): + return refuse('target_gone', 'That field is no longer available in this table.') + if kind == 'set' and family in ('sort', 'position'): + position = value.get('pos') if isinstance(value, dict) else value + if not isinstance(position, str) or not _VIEW_OP_POSITION.fullmatch(position): + return refuse('invalid_position', 'Use 1 to 64 letters or digits for the position.') + if family == 'sort' and (not isinstance(value, dict) or value.get('dir') not in ('asc', 'desc')): + return refuse('invalid_sort', 'Choose a sort direction and a valid position.') + try: + _set_view_member(candidate, member, value, present=present) + except _ViewOpRefused as exc: + return refuse(exc.reason, str(exc)) + if admit is not None: + denied = admit(view, candidate, member) + if denied: + return refuse(denied['reason'], denied['message']) + if family == 'name': + names = ([name for who, vid, name in data.view_index() if vid != view_id] + if isinstance(data, view_settings_rows.ScopedDocument) else + [other.get('name') for who, workspace in data.items() + if who not in RESERVED_MEMBERS and isinstance(workspace, dict) + for vid, other in (workspace.get('views') or {}).items() + if vid != view_id and isinstance(other, dict)]) + candidate['name'] = _unique_name(candidate.get('name'), names) + after_present, after = _view_member_value(candidate, member) + candidate['updatedAt'] = write_execution.utcnow().isoformat() + if recipient is not None: + assignment['presentation'] = { + name: value for name, value in candidate['config'].items() + if name not in ('filters', 'filterConj', 'memberPids')} + assignment['updatedAt'] = candidate['updatedAt'] + else: + ws.setdefault('views', {})[view_id] = candidate + result = {**base, 'applied': True, 'before': before, 'after': after, + 'beforePresent': before_present, 'afterPresent': after_present, + 'by': actor, 'overwrote': previous.get('by') if previous else None} + writers = ws.setdefault(_VIEW_OP_MEMBERS, {}).setdefault(view_id, {}) + if family == 'memberPid': + for name in tuple(writers): + if name != member and _view_member_identity(name) == member: + writers.pop(name) + writers[member] = {'by': actor, 'client_op_id': identity} + audit(action='rename' if family == 'name' and before != after else None, + owner=owner, view_id=view_id, actor=actor, origin=origin, + before=view_audit.summary(view), after=view_audit.summary(candidate)) + activity.append(copy.deepcopy(result)) + return result + + try: + edit_receipts.op_id(identity) + except ValueError: + results.append(refuse('invalid_op_id', 'Give this View operation a client operation id.')) + continue + results.append(edit_receipts.run_once(receipts, actor, action, identity, land)) + return data + + stored = self._audited_update(apply, flush='sync', owners=(row_owner,)) + if stored is None or stored is False: + raise edit_receipts.ReceiptUnavailable('The View operation was not saved. Retry this client operation id.') + for result in activity: + self.log_view_activity(view_id, 'changed', by=actor, member=result['member'], + before=result['before'], after=result['after'], + overwrote=result['overwrote'], via=origin) + return copy.deepcopy(results) + def save_view(self, username, view, reserved_names=(), *, - actor=None, origin=None, rev=None, action=None, rebase=False): + actor=None, origin=None, action=None): """Upsert a SavedView into its creator's personal workspace. Sharing is exclusively a grant-registry concern; this method never chooses a second home @@ -1963,9 +2092,6 @@ class TableStore: `view_upsert` with a fresh id — `grid_events` L2408 says so in as many words), and `restore` belongs to W50-T26's trash door. Both are honest only when the caller names them; nothing is guessed. - - `rebase=True` (S82-T156) lands a stale `rev` when the write undoes no change made after - it, guarded again at the current revision. A real conflict still raises `ViewRevisionStale`. """ view_id = str((view or {}).get('id') or '').strip() if not view_id: @@ -1982,7 +2108,6 @@ class TableStore: raise ValueError('Saved Views use modern grants; view-level permissions are retired') accepted = {} activity_before = {} - landing = {} def _up(data, audit): # View names are tenant-global across personal source records. A grant is only an @@ -2010,32 +2135,18 @@ class TableStore: prior = (ws.get('views') or {}).get(view_id) activity_before['value'] = view_audit.summary(prior) ws.setdefault('views', {})[view_id] = payload - note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior), - author=actor or username) audit(action=action or view_audit.derive_action(prior, payload), owner=username, view_id=view_id, actor=actor, origin=origin, before=view_audit.summary(prior), after=view_audit.summary(payload)) return data - def _write(at_rev): - landing['at'] = self._next_rev(at_rev) - self._audited_update(_up, rev=at_rev, flush='async', owners=(username,)) - - try: - _write(rev) - except view_audit.ViewRevisionStale as stale: - if rev is None or not rebase: - raise - self._land_stale(username, rev, stale, - lambda ws, sent: stale_view_conflict( - ws, view_id, requested, sent, requester=actor or username), - _write) + self._audited_update(_up, flush='async', owners=(username,)) self.log_view_activity(view_id, 'saved', by=actor or username, before=activity_before.get('value'), after=view_audit.summary(accepted), via=origin) return dict(accepted) - def delete_view(self, username, view_id, *, actor=None, origin=None, rev=None): + def delete_view(self, username, view_id, *, actor=None, origin=None): """Delete an owner source View and its dependent assignments, never another user's. ⭐⭐ W50-T25 — C3's `delete` action, and `actor` is a SEPARATE parameter from `username` @@ -2065,7 +2176,6 @@ class TableStore: # a caller that named no actor must not produce a trash entry blaming nobody. who = str(actor or owner or '') or None removed = {} - at = self._next_rev(rev) def _up(data, audit): views = data.get(owner, {}).get('views') or {} @@ -2073,8 +2183,6 @@ class TableStore: removed.clear() if isinstance(before, dict): removed['value'] = view_audit.summary(before) - note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before), - author=who) views.pop(vid, None) audit(action='delete' if isinstance(before, dict) else None, owner=owner, view_id=vid, actor=actor, origin=origin, @@ -2114,7 +2222,7 @@ class TableStore: purge_trash(bucket, now=stamp) return data - self._audited_update(_up, rev=rev, flush='async', owners=(owner, ASSIGNED_KEY)) + self._audited_update(_up, flush='async', owners=(owner, ASSIGNED_KEY)) if removed: self.log_view_activity(vid, 'deleted', by=who, before=removed['value'], via=origin) if row_backed: @@ -2207,7 +2315,7 @@ class TableStore: return False return True - def purge_view(self, username, view_id, *, rev=None, actor=None, origin=None): + def purge_view(self, username, view_id, *, actor=None, origin=None): """R11's "Delete forever": destroy one trashed view NOW. `(name, adopted_cell_count)`. ⛔ THE ADOPTION RUNS FIRST AND A FAILURE REFUSES THE PURGE. Unlike the retention sweep on @@ -2219,8 +2327,7 @@ class TableStore: inventing an eighth here would widen a vocabulary two gates pin. The `delete` row written when the view entered the trash is still the record of its removal. Booked as a finding. - ⚠ IT STILL GOES THROUGH `_audited_update`, for the C3 revision guard and the one locked - read-modify-write. `action=None` is how that door spells "no row" (see `delete_view`). + The locked update keeps trash removal atomic. `action=None` requests no audit row. """ vid = str(view_id or '').strip() owner = str(username or '').strip().lower() @@ -2262,7 +2369,7 @@ class TableStore: audit(action=None, owner=owner, view_id=vid) return current - self._audited_update(_up, rev=rev, flush='async', owners=(owner,)) + self._audited_update(_up, flush='async', owners=(owner,)) if removed: self.log_view_activity(vid, 'purged', by=actor or owner, before=removed['value'], via=origin) @@ -2373,7 +2480,7 @@ class TableStore: return str(who), dict(entry, id=vid) return None - def restore_view(self, username, view_id, *, actor=None, origin=None, rev=None): + def restore_view(self, username, view_id, *, actor=None, origin=None): """Move one trashed view back into `views`, with every assignment the delete cascaded. ⭐⭐ R9's *"one-click Restore"*, and it is an AUDITED action: `restore` is one of C3's @@ -2448,7 +2555,7 @@ class TableStore: before=None, after=view_audit.summary(payload)) return data - self._audited_update(_up, rev=rev, flush='async', owners=(owner, ASSIGNED_KEY)) + self._audited_update(_up, flush='async', owners=(owner, ASSIGNED_KEY)) self.log_view_activity(vid, 'restored', by=actor or owner, after=view_audit.summary(restored), via=origin) return dict(restored) @@ -2597,8 +2704,8 @@ class TableStore: Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula columns — the caller enforces the prefix). The stored overlay VALUES for the key are scrubbed with it: a deleted column's cells must not linger as orphan data that would - silently resurface if the key were ever reused. Views referencing the key self-heal on - their next autosave (an unknown colId is dropped) — the rule every stale key rides. + silently resurface if the key were ever reused. This user's Views lose the field's + members in this transaction. The workspace reader cleans other users' projections. ⭐⭐ W36-T25 — AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can @@ -2616,6 +2723,8 @@ class TableStore: for row in ws['overlays'].values(): if isinstance(row, dict): row.pop(key, None) + for view in ws['views'].values(): + prune_view_field_members(view, removed_keys=(key,)) self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None)) @@ -2756,7 +2865,12 @@ class TableStore: entry['options'] = merge_choice_options(entry.get('options'), values) fields[key] = entry - self._update(username, _patch) + committed = self._update(username, _patch) + readback = getattr(self, 'overlay_readback', None) + if isinstance(readback, dict): + from copy import deepcopy + readback[str(int(pid))] = deepcopy( + (((committed or {}).get(username) or {}).get('overlays') or {}).get(str(int(pid))) or {}) def make(table_key, st=None): diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 92d11fbd4c11a4bc98ee7458ea6e3ff14b78c20c..800dadd539ce2d9e47c4bb940950632cbf759241 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -50,6 +50,7 @@ import inspect import json as _json import math import re +import sys import uuid as _uuid from copy import deepcopy from contextlib import contextmanager @@ -315,6 +316,8 @@ def counts_toward_table_cap(table_key, definition=None): #: A module's own databases (Command Center's four) are not what people create: no cap slot either. _MODULE_OWNED = set() +# Only ensure_module_table stamps this stored declaration; public create builds its own metadata. +MODULE_OWNED_KEY = 'moduleOwned' def register_module_tables(*keys): @@ -438,6 +441,7 @@ def strip_materialised(st=None): return doc s.update(STORE_KEY, _strip, flush='sync') + _log_store_write(s, '*', 'strip_materialised') try: out['after'] = len(_json.dumps(s.get(STORE_KEY) or {}, default=str)) except Exception: # noqa: BLE001 @@ -740,6 +744,7 @@ def record_connector_health(table_key, state, *, st=None, last_success=None, return document landed = _st(st).update(STORE_KEY, _set, flush='sync') + _log_store_write(_st(st), key, 'record_connector_health') return dict((landed.get(key) or {}).get(CONNECTOR_HEALTH_FIELD) or receipt) @@ -829,8 +834,7 @@ def purge_credential_tables(entry_type, connection_id, rt=None, st=None): return out -#: Command Center's SKU table carries 64 module columns, and a person must still be able to add a field. -MAX_FIELDS = 100 +MAX_FIELDS = 60 #: the badge the nav paints for a table with no connector behind it BLANK_SOURCE = 'Blank' #: automation-created tables carry their maker instead (wave 18 C4-AUTO) @@ -930,11 +934,26 @@ def records_mutable(table_key, st=None): this human-door predicate. """ table = get_definition(table_key, st, fallback=get_one) or {} - if not table: - return False - if str(table_key) in _LOCKED_RECORD_KEYS: - return False - return table.get('recordMode') != AUTOMATION_RECORD_MODE + return _records_mutable_definition(table_key, table) + + +def _records_mutable_definition(table_key, definition): + """Evaluate the record lock from an already admitted stored definition.""" + return bool(definition) and str(table_key) not in _LOCKED_RECORD_KEYS \ + and definition.get('recordMode') != AUTOMATION_RECORD_MODE + + +def field_limit(table_key=None, st=None, *, definition=None): + """Return the module or user field ceiling, or None for a records-locked connector.""" + if definition is None: + definition = get_definition(table_key, st) if table_key else {} + module_owned = (str(table_key or (definition or {}).get('key')) in _MODULE_OWNED + or (definition or {}).get(MODULE_OWNED_KEY) is True) + if (definition and not module_owned + and not _records_mutable_definition(table_key, definition)): + return None + limit = (definition or {}).get('fieldLimit') + return limit if type(limit) is int and limit > 0 else MAX_FIELDS #: The field types a user table may declare. ⚠ Kept a SUBSET of #: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section). `created_time` stays @@ -1327,15 +1346,52 @@ def _log_field_list_change(target, table_key, by, via, change): f'removed={removed} added={added} rev={rev} updated_at={stamp}', flush=True) +def _store_write_state(target, name=STORE_KEY): + """D-118: `(dirty, owned, sha)` of one bucket in the store behind `target`, None where the backend keeps none. + + Reads the three per-name maps without the store lock: `get()` holds that lock across a download. + """ + st, key = target, str(name) + try: + for _ in range(8): + if isinstance(st, _Lent): + st = st._runtime + elif callable(getattr(type(st), 'store_key', None)) and callable(getattr(type(st), '_store', None)): + key, st = st.store_key(key), st._store() # a TenantRuntime namespaces the key + else: + break + if st is store: + st = store._d() + return tuple(m.get(key) if isinstance(m, dict) else None + for m in (vars(st).get('_dirty'), vars(st).get('_owned'), vars(st).get('_base'))) + except Exception: # noqa: BLE001 + return None, None, None + + +def _log_store_write(target, table_key, op): + """D-118: one stderr line per committed `user_tables` write: table, op, dirty, owned, commit sha. + + It carries no cell value and never fails the write it describes. + """ + if _is_preview_store(target): + return + dirty, owned, sha = _store_write_state(target) + print(f'[user-tables] D-118 write: table={table_key} op={op} dirty={dirty} owned={owned} ' + f'sha={sha}', file=sys.stderr, flush=True) + + def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, row_ids=(), field_keys=(), flush='sync', audit_restore_ids=(), **update_kwargs): """Commit a table mutation and log each changed selected record or field once.""" # None selects all identities. Empty tuples select none. Guards stay inside mutate/update. target = _st(st) + op = sys._getframe(1).f_code.co_name # the door, for the D-118 write line original_mutate = mutate row_ids = None if row_ids is None else tuple(str(r) for r in row_ids) # D-118: the field keys this write removed or added, from the final store attempt only. field_list_change = {} + # D-179: flows newly tagged on this table, from the final attempt; the flow index learns them. + new_flows = set() def public_result(result): if (isinstance(result, dict) and isinstance(result.get('ut_activity'), dict) @@ -1368,6 +1424,7 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, selected = {str(r): selected[str(r)] for r in row_ids if str(r) in selected} before_rows = deepcopy(selected) prior_field_keys = _field_list_keys(prior_table) + prior_flows = _flow_tags(prior_table) outbox = ((current or {}).get('ut_activity') or {}).get('_activity_outbox') # A record-mode outbox writes per entry; a replaced document escalates to table mode. queued = {} if getattr(outbox, 'lazy_records', False) else deepcopy(outbox or {}) @@ -1390,13 +1447,17 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, field_list_change.update( removed=[k for k in prior_field_keys if k not in after_field_keys], added=[k for k in after_field_keys if k not in prior_field_keys]) + new_flows.clear() + new_flows.update(_flow_tags(table if isinstance(table, dict) else {}) - prior_flows) return document append = _activity_writer(target, by) if append is None: result = target.update(STORE_KEY, mutate, flush=flush, **update_kwargs) if result is not False: + _log_store_write(target, table_key, op) _log_field_list_change(target, table_key, by, via, field_list_change) + _index_new_flows(target, table_key, new_flows) return public_result(result) row_ids = None if row_ids is None else tuple(str(r) for r in row_ids) field_keys = None if field_keys is None else frozenset(str(k) for k in field_keys) @@ -1468,7 +1529,9 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, result = target.update(STORE_KEY, apply, flush=flush, **update_kwargs) if result is not False: + _log_store_write(target, table_key, op) _log_field_list_change(target, table_key, by, via, field_list_change) + _index_new_flows(target, table_key, new_flows) from core import store_pg enlisted = store_pg._CALLER_TRANSACTION.get() is not None if batch is not None and not enlisted: @@ -1478,6 +1541,100 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, return public_result(result) +#: D-179: `{flowId: [table_key, ...]}`, the tables each automation tagged. An entry is COMPLETE +#: or ABSENT: only a new flow starts one (`create=True`), every tagging door extends it, and a +#: flow with no entry (one tagged before the index existed) keeps the whole-scan path. +FLOW_INDEX_KEY = 'user_tables_flow_index' + + +def _flow_tags(table): + """The `automation.flowId` values on one table's fields.""" + return {str((f.get('automation') or {}).get('flowId') or '') + for f in ((table or {}).get('fields') or []) + if isinstance(f, dict) and isinstance(f.get('automation'), dict)} - {''} + + +def _index_new_flows(target, table_key, flows): + """Extend each indexed flow in `flows` with `table_key`; a failure is logged, never raised.""" + for fid in sorted(flows or ()): + try: + index_flow_tables(fid, (table_key,), target) + except Exception as exc: # noqa: BLE001 + print(f'[user-tables] D-179 flow index not extended: flow={fid} table={table_key} ' + f'({type(exc).__name__})', file=sys.stderr, flush=True) + + +def flow_index_tables(flow_id, st=None): + """The table keys `flow_id` tagged, or None when the index has no entry for it.""" + try: + index = _st(st).get(FLOW_INDEX_KEY) or {} + except Exception: # noqa: BLE001 + return None + entry = index.get(str(flow_id)) if isinstance(index, dict) else None + return [str(k) for k in entry] if isinstance(entry, list) else None + + +def index_flow_tables(flow_id, table_keys=(), st=None, *, create=False): + """Name `table_keys` under `flow_id`. Only `create` starts an entry: a new flow tagged nothing yet. + + A failed extend DROPS the entry, so a flow is never served by an index that misses a table. + The start is async: a lost start leaves no entry, which is the safe answer. An extend is sync. + """ + fid = str(flow_id or '').strip() + keys = sorted({str(k) for k in (table_keys or ()) if str(k or '').strip()}) + if not fid: + return + if not create: + have = flow_index_tables(fid, st) + if have is None or set(keys) <= set(have): + return + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + entry = cur.get(fid) + if create: + cur[fid] = keys + elif isinstance(entry, list): + cur[fid] = sorted(set(entry) | set(keys)) + return cur + try: + _st(st).update(FLOW_INDEX_KEY, _up, flush='async' if create else 'sync') + except Exception: # noqa: BLE001 + if create: + raise + _st(st).update(FLOW_INDEX_KEY, lambda cur: {k: v for k, v in (cur or {}).items() + if k != fid}, flush='sync') + + +def update_tables(table_keys, mutate, st=None, *, flush='sync'): + """Run `mutate(key, table)` on each named table in ONE write that declares those tables first. + + On the row layout `lock_records` loads only the named definitions, so no whole document is read. + A named table that no longer exists is skipped. + """ + keys = sorted({str(k) for k in (table_keys or ()) if str(k or '').strip()}) + target = _st(st) + op = sys._getframe(1).f_code.co_name + + def _apply(cur): + cur = cur if isinstance(cur, dict) else {} + lock = getattr(cur, 'lock_records', None) + if callable(lock): + lock({k: [] for k in keys}) + for k in keys: + table = cur.get(k) + if isinstance(table, dict): + mutate(k, table) + return cur + if not keys: + return None + result = target.update(STORE_KEY, _apply, flush=flush) + if result is not False: + for k in keys: + _log_store_write(target, k, op) + return result + + #: Wave 22 (contract C3) — the ROW-EVENT seam. The API layer APPENDS listeners (the automation #: engine's trigger hook, registered by `routes_automation` — the one module that imports both #: sides); platform code only ever EMITS. This is what keeps the Notion loop-prevention law @@ -1899,13 +2056,17 @@ def allocate_autonumbers(table, values): return values -def clean_fields(raw, recurrence_validator=None): +def clean_fields(raw, recurrence_validator=None, *, table_key=None, st=None, definition=None): """Validate a caller-supplied field list into the stored shape — refuse junk, never store it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every base column of a user table is user-editable by construction), or None when nothing survives. Unknown types are DROPPED, not coerced — a column silently retyped is a lie.""" - out, seen = [], set() - for f in (raw or [])[:MAX_FIELDS]: + if definition is None: + definition = get_definition(table_key, st) if table_key else {} + limit = field_limit(table_key, st, definition=definition) + admitted = {f.get('key') for f in (definition or {}).get('fields') or [] if isinstance(f, dict)} + out, seen, added = [], set(), 0 + for f in raw or []: if not isinstance(f, dict): continue label = ' '.join(str(f.get('label') or f.get('key') or '').split())[:80] @@ -1914,6 +2075,9 @@ def clean_fields(raw, recurrence_validator=None): ftype = str(f.get('type') or 'text').strip().lower() if not key or key in seen or ftype not in UT_FIELD_TYPES or not label: continue + # A lower ceiling refuses new fields but never slices off an admitted definition. + if limit is not None and key not in admitted and len(admitted) + added >= limit: + continue seen.add(key) origin = str(f.get('origin') or 'user') authority = str(f.get('valueAuthority') or 'user-entered') @@ -2072,6 +2236,8 @@ def clean_fields(raw, recurrence_validator=None): continue entry['defaultValue'] = f['defaultValue'] out.append(entry) + if key not in admitted: + added += 1 return out or None @@ -2509,13 +2675,27 @@ def is_managed_system_table(table_key, st=None, definition=None): else (get_one(table_key, st) or {}))) +class SubjectClaimRefused(ValueError): + """A create door refused a source `subject` claim (D-250).""" + code = 'subject_claim_refused' + + +#: The sentence a refused `subject` claim returns to the client. +SUBJECT_CLAIM_REFUSAL = ('Only a connector creates a database for a source subject. ' + 'A database made in Table claims no subject. Remove "subject" and try again.') + + def create(label, username, fields=None, source=None, st=None, *, managed_by=None, capability=None, - recurrence_validator=None): + recurrence_validator=None, subject=None): """Create a blank table. Returns its key, or None if refused. Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) — - the caller is a UI form and a refusal is a message, not an exception. + the caller is a UI form and a refusal is a message, not an exception. A `subject` claim + raises `SubjectClaimRefused`: one subject, one database binds connector tables only. """ + # D-250: only `odoo_relational._ensure_table_inplace` stamps `subject`, under its conflict check. + if subject is not None or (isinstance(source, dict) and 'subject' in source): + raise SubjectClaimRefused(SUBJECT_CLAIM_REFUSAL) if managed_by is not None: from core import route_system if managed_by != route_system.MANAGED_BY or not route_system.is_internal_writer(capability): @@ -2571,6 +2751,7 @@ def create(label, username, fields=None, source=None, st=None, *, managed_by=Non return cur _st(st).update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it + _log_store_write(_st(st), key, 'create') from core import shares shares.repair_published_database_grants(_st(st), table_keys=(key,)) return key @@ -2602,6 +2783,7 @@ def delete(table_key, st=None): cur.pop(key, None) return cur s.update(STORE_KEY, _drop, flush='sync') + _log_store_write(s, table_key, 'delete') # A zone is tenant data attached to this table's row-id namespace. Leaving it behind would # let a newly-created table with the same key inherit somebody else's geographic boundary. @@ -2694,18 +2876,46 @@ def delete(table_key, st=None): pass -def set_fields(table_key, fields, st=None, *, by=None, via=None): +def _rebase_field_list(edited, base, locked): + """D-118: the locked field list with only the caller's edits of `base` applied, by key. + + A key the caller left as it read it keeps the locked definition. A key absent from `locked` + is written only when `base` never held it, so a field deleted after the read stays deleted. + """ + def keyed(fields): + return {str(f['key']): f for f in fields or [] if isinstance(f, dict) and f.get('key')} + + before, after = keyed(base), keyed(edited) + out = [] + for field in locked or []: + key = str(field.get('key')) + if key in before and key not in after: + continue # the caller removed it + out.append(after[key] if key in after and after[key] != before.get(key) else field) + held = {str(f.get('key')) for f in locked or []} + return out + [f for key, f in after.items() if key not in before and key not in held] + + +def set_fields(table_key, fields, st=None, *, by=None, via=None, base=None): """Replace a table's base-field contract (wave 18 — the automation engine grows tables it - creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless.""" + creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless. + + `base` is the field list the caller read before it edited `fields`. With it, the write + applies only the caller's changes to the locked list (`_rebase_field_list`). + """ _validate_link_selection_fields(fields, st) - clean = clean_fields(fields) + clean = clean_fields(fields, table_key=table_key, st=st) if not clean or not is_user_table(table_key, st): return False def _set(cur): t = cur.get(str(table_key)) if t is not None: - replacement = deepcopy(clean) + wanted = fields if base is None else _rebase_field_list(fields, base, t.get('fields')) + # Re-evaluate against the locked definition, including fields admitted by a concurrent writer. + replacement = clean_fields(wanted, table_key=table_key, definition=t) + if not replacement: + return cur previous = {f['key']: f for f in t.get('fields') or []} for field in replacement: if field.get('type') == 'autonumber': @@ -2927,6 +3137,7 @@ def set_label(table_key, label, st=None): return cur _st(st).update(STORE_KEY, _set, flush='sync') + _log_store_write(_st(st), table_key, 'set_label') return label @@ -3081,7 +3292,7 @@ class LinkSelectionError(ValueError): def clean_link_selection_filter(raw, fields=None): """Clean a complete filter-kit restriction without dropping any submitted condition.""" import aios_grid - from harness.filter_eval import is_rule_active + from core.filter_eval import is_rule_active if (not isinstance(raw, dict) or set(raw) - {'conj', 'nodes'} or raw.get('conj', 'and') not in ('and', 'or') @@ -3147,7 +3358,7 @@ def clean_link_selection_filter(raw, fields=None): def link_selection_permits(field, row, fields): """Apply a Link restriction to an already permission-scoped target row and its real schema.""" - from harness.filter_eval import permits + from core.filter_eval import permits bag = field.get('link') if isinstance(field, dict) else None if not isinstance(bag, dict) or 'selectionFilter' not in bag: return True @@ -3157,7 +3368,7 @@ def link_selection_permits(field, row, fields): def filter_link_selection(field, rows, fields): """Narrow permission-scoped picker candidates using the Link's canonical restriction.""" - from harness.filter_eval import permits + from core.filter_eval import permits bag = field.get('link') if isinstance(field, dict) else None if not isinstance(bag, dict) or 'selectionFilter' not in bag: return list(rows) @@ -3180,7 +3391,7 @@ def _validate_link_selection_fields(fields, st=None): def _validate_link_selection_values(fields, values, tables): """Refuse restricted Link writes against the target rows in the same store snapshot.""" - from harness.filter_eval import permits + from core.filter_eval import permits for field in fields or (): bag = field.get('link') if isinstance(field, dict) else None key = field.get('key') @@ -4004,6 +4215,7 @@ def stamp_ai_enrich(table_key, col_id, marks, st=None): return cur _st(st).update(STORE_KEY, _set, flush='async') + _log_store_write(_st(st), table_key, 'stamp_ai_enrich') return len(clean) @@ -5046,7 +5258,8 @@ def add_field(table_key, raw, st=None, recurrence_validator=None, *, by=None, vi return None defn = get_definition(table_key, st) or {} have = [f for f in (defn.get('fields') or [])] - if len(have) >= MAX_FIELDS: + limit = field_limit(table_key, st, definition=defn) + if limit is not None and len(have) >= limit: return None field = _clean_field(raw, recurrence_validator=recurrence_validator, valid_keys={str(f.get('key')) for f in have if f.get('key')}) @@ -5072,7 +5285,8 @@ def add_field(table_key, raw, st=None, recurrence_validator=None, *, by=None, vi if str(table_key) == 'ut_activity': t = cur['ut_activity'] = _activity_definition(t) if t is not None: - if (len(t.get('fields') or []) >= MAX_FIELDS + limit = field_limit(table_key, definition=t) + if ((limit is not None and len(t.get('fields') or []) >= limit) or any(f.get('key') == field['key'] for f in t.get('fields') or [])): return cur if field.get('type') == 'autonumber': @@ -5202,7 +5416,8 @@ def sync_reciprocal_link(table_key, field_key, st=None): # link with no reciprocal is exactly what every link was before this function existed, and # it still resolves its own cell. `reciprocal` is simply not stamped on the bag, so the # next `sync` on a table with room creates it — this is a skip, not a tombstone. - if len(target.get('fields') or []) >= MAX_FIELDS: + limit = field_limit(target_key, definition=target) + if limit is not None and len(target.get('fields') or []) >= limit: # ⭐ W41-T15 — the skip the paragraph above describes, now REPORTED. It was already # deliberate; it was never legible from outside the function. _refuse('target_full', @@ -5234,6 +5449,7 @@ def sync_reciprocal_link(table_key, field_key, st=None): return cur _st(st).update(STORE_KEY, _sync, flush='sync') + _log_store_write(_st(st), table_key, 'sync_reciprocal_link') return result @@ -5593,6 +5809,7 @@ def patch_cell_notes(table_key, row_id, notes, ctx, *, op_id, source_row_exists= result = edit_receipts.CheckedStore(ctx.st).update(STORE_KEY, merge, flush='sync') if not saved or result is None or result is False: raise edit_receipts.ReceiptUnavailable('The cell note was not saved. Retry this opId.') + _log_store_write(ctx.st, table_key, 'patch_cell_notes') return {'pid': row_id, 'opId': identity, 'value': {CELL_NOTES_KEY: deepcopy(delta)}} # Match the row PATCH receipt identity and envelope so its active transaction is reused. diff --git a/platform/core/view_audit.py b/platform/core/view_audit.py index 238f1bfde7ffc82876f1fb0528167ce75c522cc7..f7ab391ceca9c2a17641256f0feeae1664729ab8 100644 --- a/platform/core/view_audit.py +++ b/platform/core/view_audit.py @@ -1,60 +1,8 @@ -"""view_audit.py — THE ONE WRITER of the saved-View audit trail, and the rev guard beside it. - -Owner instruction 19, verbatim: *"Views Karen Ganley created (titled like 'Avi YTD sales') -disappeared; nobody deleted them. Research thoroughly; created views cannot disappear."* -Ruling R9 answers it with an audit trail (who, when, what, from where) over create, rename, -move, duplicate, share, delete and restore, plus rev-guarded view writes. Contract C3 fixes the -row shape. This module owns both halves; nothing else in the repo appends an audit row. - -⭐⭐ WHY THIS FILE EXISTS AT ALL: the disappearance was invisible because nothing recorded the -write. `verify_store_concurrency.py`'s header measures the loss that was found by forensics -(one HF commit in which `karen@teamroyalimports.com` lost 3 views and 49 overlay rows while -`leadership` gained 4), and forensics is what you do when there is no trail. The trail is the -instrument that makes the NEXT one a two-minute lookup instead of a wave. - -──────────────────────────────────────────────────────────────────────────────────────────────── -⛔⛔ THE ROW COMMITS IN THE SAME TRANSACTION AS THE WORKSPACE WRITE. THAT IS THE WHOLE POINT. - -Two transactions means the workspace write commits, the trail row fails, and the trail then has a -hole in exactly the place the bug was. So the row is built by `row()` and appended INSIDE the -read-modify-write closure `core/table_store.py` already hands to `st.update(...)` — the closure -runs inside `store_pg.update`'s `with con.transaction():` on Postgres, and inside the HF store's -single journalled apply off it. One write, one commit, one failure mode, on every backend. - -HOMES, and `home()` is the only place that chooses: - - * `DOCUMENT` (default, and the one that ships today) — a reserved top-level member - `__audit__` of the SAME workspace document, ring-capped to `RING_CAP` newest rows. - Atomic by construction, on every backend, because it rides the closure. - * `CONTROL_AUDIT` (Postgres, `control.audit`) — used only when the store handle's `update` - actually accepts `audit_row=`, i.e. when it is `core.store_pg` / `PgStore` directly. - -⛔⛔ AND TODAY NOTHING IN PRODUCTION REACHES THE SECOND HOME. `harness/runtime.TenantRuntime.update` -is `update(self, name, fn, flush="sync")` with no `**kw`, and EVERY API caller passes -`st=session.runtime` (`routes_tables._ops`, `routes_grid`, `routes_shares`, `automation_engine`). -So `store_pg.update(audit_row=…)` is written, gated and unreachable until `harness/runtime.py` -passes the two keywords through — a file outside W50-T25's fence, booked as a handover rather -than widened into it. Because the document home is atomic on Postgres too, that gap costs -locality, not correctness: the trail is complete either way. - -──────────────────────────────────────────────────────────────────────────────────────────────── -⛔ AN ORDINARY CONFIG AUTOSAVE IS NOT AN AUDITED ACTION, and the done-when's seven names are the -whole list. Every filter, sort, group, colour and column resize autosaves through -`table_store.save_view`; a row per keystroke makes the trail unreadable and expensive, and -`store_kv` carries the ring inside the workspace document that five or six reads per assembly -already deep-copy. `derive_action()` below therefore answers `None` for a same-name re-save, and -that `None` means NO ROW, not a row with a blank action. - -⛔ `duplicate` IS SUPPORTED HERE AND NEVER DERIVED. The events door cannot tell a duplicate from -a create: `grid_events` L2408's own comment says the client duplicates by emitting *"the ordinary -per-item duplicate events it already owns (view_upsert with a fresh id)"*, which is -indistinguishable from a create at the door. So a fresh id is honestly logged `create`, and the -marker that would let it say `duplicate` is part of the W50-T28 client handover. - -⛔ THE `owner` IS THE BUCKET THE WRITE LANDED IN — never the actor, never `find_view`'s first -hit. `find_view` without `prefer_owner` returns whichever stratum the dict yielded first, which -is how W48C-T02 starred a colleague's overlay record; an audit row that inherited that mistake -would blame the wrong person in the one record that exists to say who did it. +"""Build and read saved-View audit rows in the workspace mutation's transaction. + +Document stores keep a bounded ring. PostgreSQL handles use control.audit when supported. +Migrated workspaces write and read their scoped settings audit rows through PgRows. +Generic store revision exceptions remain here for Sheet, limiters and other store consumers. """ import time @@ -80,17 +28,7 @@ PG_ACTION_PREFIX = 'view.' class ViewRevisionStale(RuntimeError): - """A view mutation carried a `rev` older than the bucket's current one. - - Carries `rev_current` (what the store holds now) and `rev_sent` (what the client believed). - The API turns it into `409 view_revision_stale {rev_current}` so the client can refetch and - retry once rather than merely toasting; C3 names that status and that payload key. - - ⚠ A `RuntimeError` rather than a new base, deliberately: it must NOT be catchable by the - `except Exception` fallbacks that already dot the event handlers as "the store hiccupped", - and it must not be a `StoreUnavailable` subclass, or `routes_grid` would answer 503 and the - client would show "saved" for a write that was refused. - """ + """A generic store write failed its expected revision check.""" def __init__(self, rev_current=None, rev_sent=None, message=None): self.rev_current = None if rev_current is None else int(rev_current) @@ -142,9 +80,7 @@ def row(*, tenant, table_key, owner, view_id, action, actor, that function's own comment pins as epoch seconds on BOTH backends. A trail whose timestamps were in a different unit from the change token beside them could not be read against it. - `rev` is `None` when the caller sent none. That is the done-when's *"one without `rev` is - accepted and logged as `rev: null`"* and it is what makes this change safe to ship before - W50-T28 wires the client: nothing on the wire carries `rev` today, so nothing can be refused. + View doors leave the legacy audit revision null. Generic callers may supply it. """ act = str(action or '').strip() if act not in ACTIONS: @@ -228,14 +164,7 @@ def home(st, table_key=None): def guards(st): - """Whether this handle enforces `expect_rev` INSIDE its locking read. - - False means the caller must fall back to the ADVISORY check (`revision()` then compare then - write), which has a real TOCTOU window: another writer can land between the two. It is still - the check C3 asks for and it still refuses the stale write it can see; it simply cannot - promise it saw every one. Stated rather than implied, because the fallback is what every - caller gets until `harness/runtime.py` passes the keywords through. - """ + """Whether this generic store handle declares support for expected revision checks.""" upd = getattr(st, 'update', None) return upd is not None and _accepts(upd, 'expect_rev') diff --git a/platform/core/view_settings_rows.py b/platform/core/view_settings_rows.py index fa175f3eef7b6457c3c5be1cb5f0c9d3067db145..999fd89d4fadc14dd1ff9bd3b66cbe4724bfcbb0 100644 --- a/platform/core/view_settings_rows.py +++ b/platform/core/view_settings_rows.py @@ -146,6 +146,33 @@ class ScopedDocument(dict): self.view_index = index +class ViewOpReceipts: + """Lend run_once receipt storage inside the locked owner row and its replayed closure.""" + + MEMBER = 'viewOpReceipts' + + def __init__(self, workspace, tenant): + self.workspace = workspace + self.key = str(tenant) + + def get(self, name): + from core.edit_receipts import TABLE_NAME + return copy.deepcopy(self.workspace.get(self.MEMBER, {})) if name == TABLE_NAME else {} + + def update(self, name, fn, flush=None): + from core.edit_receipts import TABLE_NAME + if name != TABLE_NAME: + raise ValueError('view operations can only lend their own receipts') + value = fn(self.get(name)) + self.workspace[self.MEMBER] = copy.deepcopy(value) + return copy.deepcopy(value) + + def result(self, actor, action, identity): + key = json.dumps([str(actor), str(action), str(identity)], separators=(',', ':')) + receipt = self.workspace.get(self.MEMBER, {}).get(key) + return copy.deepcopy(receipt['result']) if receipt else None + + class PgRows: """One tenant and workspace, addressed through the existing bounded PostgreSQL pool.""" @@ -209,7 +236,7 @@ class PgRows: 'AND NOT (r.row_key = ANY(%s::text[]))', self.tenant_slug), (self.name, list(RESERVED))).fetchall() - def update(self, fn, *, owners, rev=None, audit_rows=None): + def update(self, fn, *, owners, audit_rows=None): """Lock the marker, mutate scoped rows, and append audit entries in one transaction.""" pg = _pg() pg.check_write(self.tenant_slug, f'update view settings {self.name!r}') @@ -220,9 +247,7 @@ class PgRows: with pg._pool().connection() as con: with con.transaction(): _limits(con) - _stamp, current = self._head(con, lock=True, create=True) - if rev is not None and int(rev) != int(current): - raise view_audit.ViewRevisionStale(current, rev) + self._head(con, lock=True, create=True) before = self._settings(con, owners, lock=True) data = ScopedDocument(copy.deepcopy(before), lambda: self._index(con)) result = fn(data) diff --git a/platform/core/view_templates.py b/platform/core/view_templates.py index 5b92ba9b67c7d437f6164bb63a7d65c57d984302..b317f7c5bc89f6fd9a1f31839530d99475edfd14 100644 --- a/platform/core/view_templates.py +++ b/platform/core/view_templates.py @@ -111,10 +111,21 @@ def _view(view_id, name, note, visible, *, mode='grid', stack_field=None, sorts= # contract can move and a ut_ table's columns are whatever its owner made. #: Command Center columns every grain view shows after its ids, score first. Each reason column -#: sits right after the figures it explains (S82-T64). +#: sits right after the figures it explains (S82-T64). Every declared metric is shown (S85-T125). _CC_HEALTH = ['score', 'priority', 'stock_health', 'profit_health', 'ads_health', 'listing_health', - 'units_sold', 'net_sales', 'net_profit', 'margin', 'cost_note', 'ctr', 'acos', 'ads_note', - 'tacos', 'tacos_note', 'days_of_cover', 'stock_note', 'return_rate', 'returns_note'] + 'units_sold', 'net_sales', 'fees', 'sponsored_ads', 'net_proceeds', 'sales_note', + 'cogs_total', 'net_profit', 'margin', 'cost_note', + 'ad_impressions', 'ad_clicks', 'ad_spend', 'ad_sales', 'ad_orders', 'ctr', 'cvr', 'cpc', 'roas', + 'acos', 'ads_note', 'tacos', 'tacos_note', + 'fba_on_hand', 'inbound_units', 'days_of_cover', 'stock_note', + 'return_units', 'return_rate', 'returns_note', + 'sessions', 'unit_session_pct', 'buy_box_pct', 'listing_note', + 'units_30d', 'net_sales_30d', 'ad_spend_30d', 'health_note'] +#: The SKU table adds the per unit cost and the Odoo channel figures to the same columns. +_CC_SKU = (_CC_HEALTH[:_CC_HEALTH.index('cogs_total')] + ['cogs_per_unit'] + + _CC_HEALTH[_CC_HEALTH.index('cogs_total'):_CC_HEALTH.index('cost_note') + 1] + + ['odoo_channel_sales', 'odoo_channel_profit', 'odoo_channel_margin', 'odoo_margin_source'] + + _CC_HEALTH[_CC_HEALTH.index('cost_note') + 1:]) # Projects use the native Projects popup; retained legacy table data has no creation template. TEMPLATES = [ @@ -255,13 +266,13 @@ TEMPLATES = [ _view( 'tpl_amazon_sku_profit_all_skus', 'All SKUs', 'Every SKU with the columns that matter first, lowest score first.', - ['sku', 'picture', 'asin', 'parent_asin', 'title', 'odoo_product'] + _CC_HEALTH + ['cogs_source', 'projects'], + ['sku', 'picture', 'asin', 'parent_asin', 'title', 'odoo_product'] + _CC_SKU + ['cogs_source', 'projects'], sorts=[{'colId': 'score', 'dir': 'asc'}], ), _view( 'tpl_amazon_sku_profit_priority_high', 'Priority High', 'SKUs that need attention first, lowest score first.', - ['sku', 'picture', 'asin', 'parent_asin', 'title', 'odoo_product'] + _CC_HEALTH + ['cogs_source', 'projects'], + ['sku', 'picture', 'asin', 'parent_asin', 'title', 'odoo_product'] + _CC_SKU + ['cogs_source', 'projects'], sorts=[{'colId': 'score', 'dir': 'asc'}], filters=[{'colId': 'priority', 'op': 'eq', 'value': 'High'}], ), diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index f66da1a5436131318798110e7f058f8b0732892d..b8df411b08d94b450fe4ca91cfd900ba9e744a7e 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -115,6 +115,8 @@ _entity_pacers = { "getInventorySummaries": {"last": None, "lock": threading.Lock()}, "searchListingsItems": {"last": None, "lock": threading.Lock()}, "searchCatalogItems": {"last": None, "lock": threading.Lock()}, + "getItemReviewTopics": {"last": None, "lock": threading.Lock()}, + "getItemReviewTrends": {"last": None, "lock": threading.Lock()}, "getMyFeesEstimates": {"last": None, "lock": threading.Lock()}, "listAllFulfillmentOrders": {"last": None, "lock": threading.Lock()}, "getFulfillmentOrder": {"last": None, "lock": threading.Lock()}, @@ -872,11 +874,39 @@ GROUP_A_TABLES = { "snapshot": True, "validates_against": "Amazon raw seller metric count and total offer metrics vs stored replenishment rows", }, + "review_topics": { + "label": "Amazon review topics", + "api": "getItemReviewTopics", + "role": "Brand Analytics / Selling Partner Insights", + "role_match": "any", + "min_interval_s": 7 * 86400, + "cadence_note": "Amazon refreshes English review topics weekly in US, UK, France, Italy, Germany, Spain and Japan. Every listing ASIN is requested.", + "snapshot": True, + "validates_against": "Amazon returned positive and negative topic counts per ASIN vs stored topics for the same pass", + }, + "review_trends": { + "label": "Amazon review trends", + "api": "getItemReviewTrends", + "role": "Brand Analytics / Selling Partner Insights", + "role_match": "any", + "min_interval_s": 7 * 86400, + "cadence_note": "Amazon refreshes six months of English review trends weekly in US, UK, France, Italy, Germany, Spain and Japan. Every listing ASIN is requested.", + "snapshot": True, + "validates_against": "Amazon raw trend point counts and occurrence percentages per ASIN, sentiment, topic and month vs stored rows for the same pass", + }, } REPORT_TYPES.update(GROUP_A_TABLES) # These are visible declarations, never enablement keys or request allow-list entries. DECLARED_DATASETS = { + "browse_node_return_topics": { + "label": "Amazon browse node return topics", + "operations": ["getBrowseNodeReturnTopics", "getBrowseNodeReturnTrends"], + "role": "Brand Analytics / Selling Partner Insights", + "role_match": "any", + "cadence_note": "Not requested by this connector.", + "reason": "Browse node return topics and trends describe category returns, not item review trends. Their readers are not built in this release.", + }, "manage_inventory_variants": { "label": "Amazon manage inventory alternatives", "report_types": ["GET_FBA_MYI_ALL_INVENTORY_DATA","GET_AFN_INVENTORY_DATA"], @@ -962,13 +992,6 @@ DECLARED_DATASETS = { "cadence_note": "Read once a day by the rival job, outside the sync pass.", "reason": "The daily rival job reads offer prices through getItemOffersBatch. The sync does not request them.", }, - "review_topics": { - "label": "Amazon review topics and trends", - "operations": ["getItemReviewTopics","getItemReviewTrends","getBrowseNodeReturnTopics","getBrowseNodeReturnTrends"], - "role": "Selling Partner Insights", - "cadence_note": "Not requested by this connector.", - "reason": "Per ASIN review and return topics are not built in this release; the limit is Amazon's per ASIN request rate.", - }, "promotions": { "label": "Amazon promotion calendar", "operations": ["searchPromotions","getPromotion"], @@ -1080,13 +1103,23 @@ ROLES_TAKEN = ( ROLES_REFUSED = ("Buyer Communication", "Buyer Solicitation") NOTIFICATIONS_SCOPE = "sellingpartnerapi::notifications" +# Destination support: https://developer-docs.amazon.com/sp-api/docs/notification-type-values NOTIFICATION_TYPES = { - "REPORT_PROCESSING_FINISHED": {"roles": ROLES_TAKEN, "payload_version": "2020-09-04"}, + # The report example uses 2020-09-04 despite the page's schema prose saying 1.0. + "REPORT_PROCESSING_FINISHED": { + "roles": ROLES_TAKEN, "payload_version": "2020-09-04", "destinations": ("SQS",)}, "FBA_INVENTORY_AVAILABILITY_CHANGES": { - "roles": ("Amazon Fulfillment",), "payload_version": "1.0", "table": "inventory"}, + "roles": ("Amazon Fulfillment",), "payload_version": "1.0", "table": "inventory", + "destinations": ("SQS",)}, "LISTINGS_ITEM_STATUS_CHANGE": { - "roles": ("Product Listing",), "payload_version": "1.0", "table": "listings"}, - "ANY_OFFER_CHANGED": {"roles": ("Pricing",), "payload_version": "1.0", "table": "pricing"}, + "roles": ("Product Listing",), "payload_version": "1.0", "table": "listings", + "destinations": ("EventBridge",)}, + "BRANDED_ITEM_CONTENT_CHANGE": { + "roles": ("Product Listing",), "payload_version": "1.0", "table": "listings", + "destinations": ("EventBridge",), "requires_brand_owner": True}, + "ANY_OFFER_CHANGED": { + "roles": ("Pricing",), "payload_version": "1.0", "table": "pricing", + "destinations": ("SQS",)}, } _NOTIFICATION_DESTINATIONS_PATH = "notifications/v1/destinations" _NOTIFICATION_SUBSCRIPTIONS_PATH = "notifications/v1/subscriptions" @@ -1169,6 +1202,12 @@ OPERATIONS = { "method": "GET", "path": "listings/2021-08-01/items/{sellerId}"}, "searchCatalogItems": {"name": "searchCatalogItems", "roles": ("Product Listing",), "method": "GET", "path": "catalog/2022-04-01/items"}, + "getItemReviewTopics": {"name": "getItemReviewTopics", + "roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any", + "method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/topics"}, + "getItemReviewTrends": {"name": "getItemReviewTrends", + "roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any", + "method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/trends"}, "listAllFulfillmentOrders": {"name": "listAllFulfillmentOrders", "roles": ("Amazon Fulfillment",), "method": "GET", "path": "fulfillmentOutbound/2020-07-01/fulfillmentOrders"}, "getFulfillmentOrder": {"name": "getFulfillmentOrder", "roles": ("Amazon Fulfillment",), @@ -1481,7 +1520,7 @@ def _default_interval(operation): "listInventory": AWD_INVENTORY_INTERVAL, "listTransactions": TRANSACTIONS_INTERVAL, "listAllFulfillmentOrders": 0.5, "getFulfillmentOrder": 0.5, "getSellingPartnerMetrics": 1.0, "listOfferMetrics": 1.0, - **ENTITY_INTERVALS, **INBOUND_INTERVALS}.get(operation, 1.0) + **ENTITY_INTERVALS, **INBOUND_INTERVALS, **FEEDBACK_INTERVALS}.get(operation, 1.0) def _observe_rate(operation, scope, headers): @@ -1801,6 +1840,7 @@ def transactions(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now= ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5, "searchListingsItems": 0.2, "searchCatalogItems": 0.5, "getMyFeesEstimates": 2.0} +FEEDBACK_INTERVALS = {"getItemReviewTopics": 1.0, "getItemReviewTrends": 1.0} INBOUND_INTERVALS = {"listInboundPlans": 0.5, "getInboundPlan": 0.5, "listShipmentItems": 0.5, "getShipment": 0.5, "getShipmentItemsByShipmentId": 0.5, "getShipmentItems": 0.5} #: Amazon's 400 text for an Amazon Warehousing and Distribution plan in the FBA inbound list. @@ -2159,6 +2199,48 @@ def inbound_shipment_items(fields, *, timeout=_TIMEOUT, sleep=time.sleep, now=ti return InboundRecords(records, skipped) +def item_review_feedback(fields, asins, *, operation="getItemReviewTopics", timeout=_TIMEOUT, + sleep=time.sleep, now=time.monotonic): + """Read every child ASIN, retaining known absences separately from invalid responses.""" + if operation not in FEEDBACK_INTERVALS: + raise AmazonOperationRefused("unapproved Customer Feedback reader") + if not set(OPERATIONS[operation]["roles"]).intersection(ROLES_TAKEN): + raise AmazonOperationRefused("Customer Feedback needs Brand Analytics or Selling Partner Insights") + identifiers = sorted(set(asins)) + if any(not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in identifiers): + raise AmazonError("Customer Feedback contains an invalid child ASIN. Correct the listings catalogue and retry.") + if not identifiers: + return [] + marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID + params = {"marketplaceId": marketplace} + if operation == "getItemReviewTopics": + params["sortBy"] = "MENTIONS" + root = "topics" if operation == "getItemReviewTopics" else "reviewTrends" + token, issued = access_token(fields, timeout=timeout), now() + records = [] + for asin in identifiers: + _pace_entity(operation, FEEDBACK_INTERVALS[operation], sleep=sleep, now=now) + if now() - issued >= 3000: + token, issued = access_token(fields, timeout=timeout), now() + body = _request(OPERATIONS[operation]["path"].replace("{asin}", asin), token, + region=str(fields.get("region") or DEFAULT_REGION), sandbox=is_sandbox(fields), + timeout=timeout, throttle_fix="continue every child ASIN at the Customer Feedback rate limit", + **params) + if body is None: + records.append({"asin": asin, "marketplaceId": marketplace, "_no_feedback_reason": + f"Amazon {operation} returned 204 No Content for this ASIN and marketplace."}) + continue + if (not isinstance(body, dict) or body.get("asin") != asin or body.get("marketplaceId") != marketplace + or not isinstance(body.get(root), dict)): + raise AmazonError(f"Amazon {operation} returned an invalid response for the requested child ASIN and marketplace. Retry after Amazon restores its response.") + for key in ("positiveTopics", "negativeTopics"): + topics = body[root].get(key) + if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)): + raise AmazonError(f"Amazon {operation} returned invalid {key}. No partial snapshot was stored.") + records.append(body) + return records + + def group_a_records(fields, operation, *, start=None, end=None, asins=(), timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic): """Fixed GET readers, every page and every own-ASIN batch; no record ceiling.""" @@ -2389,7 +2471,7 @@ def _notifications_post(operation, token, body, *, notification_type=None, def ensure_notifications(fields, aws_account_id, aws_region, *, granted_roles, timeout=_TIMEOUT): - """Ensure the EventBridge destination and four approved subscriptions, returning safe causes.""" + """Ensure only EventBridge subscriptions, returning their IDs and safe skip causes.""" account, aws_region = str(aws_account_id), str(aws_region) if (not re.fullmatch(r"[0-9]{12}", account) or not re.fullmatch(r"[a-z]{2}(?:-[a-z]+)+-[0-9]+", aws_region)): @@ -2412,6 +2494,11 @@ def ensure_notifications(fields, aws_account_id, aws_region, *, granted_roles, t # Destination setup remains grantless even when the seller has granted no event roles. seller_token, states = None, {} for event_type, spec in NOTIFICATION_TYPES.items(): + if "EventBridge" not in spec["destinations"]: + states[event_type] = {"state": "skipped", "reason": + "Amazon delivers this notification only to SQS. " + "Set up an Amazon Simple Queue Service (SQS) queue and subscribe this type to it."} + continue if not set(spec["roles"]).intersection(granted_roles): states[event_type] = {"state": "skipped", "reason": "Amazon requires one of these granted roles for " + event_type + ": " @@ -2438,7 +2525,7 @@ def ensure_notifications(fields, aws_account_id, aws_region, *, granted_roles, t timeout=timeout).get("payload") or {} if not subscription.get("subscriptionId"): raise AmazonError("Amazon Notifications returned no subscription ID.") - states[event_type] = {"state": "subscribed"} + states[event_type] = {"state": "subscribed", "subscriptionId": subscription["subscriptionId"]} except AmazonError as exc: reason = str(exc) for value in (grantless, seller_token, *(str(fields.get(key) or "") @@ -2485,6 +2572,9 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT, try: with urllib.request.urlopen(req, timeout=timeout) as r: _observe_rate(operation, scope, getattr(r, "headers", None)) + if (operation in FEEDBACK_INTERVALS + and getattr(r, "status", None) == 204): + return None return json.loads(r.read().decode("utf-8") or "{}") except urllib.error.HTTPError as e: _observe_rate(operation, scope, e.headers) @@ -3390,7 +3480,7 @@ def report_document(fields, report_type, start=None, end=None, options=None, def read(path): try: return _request(path, token, region=region, timeout=timeout, sandbox=sbx) - except AmazonThrottled as exc: + except AmazonError as exc: exc.report_id = rid raise @@ -3409,8 +3499,10 @@ def report_document(fields, report_type, start=None, end=None, options=None, meta = read(f"reports/2021-06-30/documents/{error_id}") if meta.get("url"): reason = _download_document(meta["url"], str(meta.get("compressionAlgorithm") or "")) - raise AmazonReportError(f"Amazon ended Search Query Performance as FATAL (reportId {rid})", - reason, state) + error = AmazonReportError(f"Amazon ended Search Query Performance as FATAL (reportId {rid})", + reason, state) + error.report_id = rid + raise error if state == "CANCELLED" and not (got.get("errorMessage") or got.get("errorDocumentId") or got.get("reportDocumentId")): # S66-T42 (lead, observed live 2026-09-25): Amazon documents CANCELLED as also @@ -3439,6 +3531,7 @@ def report_document(fields, report_type, start=None, end=None, options=None, reason = detail or str(got.get("errorMessage") or "No error document was supplied by Amazon") error = AmazonReportError(f"Amazon ended the {report_type} report as {state} (reportId {rid})", reason, state) + error.report_id = rid error.has_error_document = bool(error_id or got.get("errorMessage")) raise error if once: @@ -3454,8 +3547,14 @@ def report_document(fields, report_type, start=None, end=None, options=None, meta = read(f"reports/2021-06-30/documents/{doc_id}") url = str(meta.get("url") or "") if not url: - raise AmazonError(f"the {report_type} report document carried no download URL") - text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True) + error = AmazonError(f"the {report_type} report document carried no download URL") + error.report_id = rid + raise error + try: + text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True) + except AmazonError as exc: + exc.report_id = rid + raise say(f" {report_type}: {len(text):,} chars") return text, {"reportId": rid, "reportDocumentId": doc_id, "processingStatus": state, "dataStartTime": got.get("dataStartTime"), diff --git a/platform/harness/amazon_listing_changes.py b/platform/harness/amazon_listing_changes.py new file mode 100644 index 0000000000000000000000000000000000000000..70a5598dfeef58d7a1085204fe8b942b523f9d31 --- /dev/null +++ b/platform/harness/amazon_listing_changes.py @@ -0,0 +1,230 @@ +"""Tenant-store listing changes shared by signed notifications and daily snapshots.""" +from __future__ import annotations + +import copy +import hashlib +import json +from contextlib import closing +from dataclasses import dataclass +from datetime import date, datetime, timezone + + +KEY = "amzn_listing_changes" +_ATTRIBUTE_NAMES = {"title": "item_name", "bullet_points": "bullet_point", + "description": "product_description"} + + +class _Replay(Exception): + pass + + +def normalize_attribute(attribute): + """Give both writers the same Amazon attribute identity.""" + if not isinstance(attribute, str) or not attribute.strip(): + raise ValueError("A listing attribute name is required.") + name = attribute.strip().lower() + return _ATTRIBUTE_NAMES.get(name, name) + + +def _utc(value): + if isinstance(value, datetime): + stamp = value + elif isinstance(value, date): + stamp = datetime.combine(value, datetime.min.time()) + elif isinstance(value, str): + stamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + else: + raise ValueError("A listing change timestamp is required.") + return (stamp.replace(tzinfo=timezone.utc) if stamp.tzinfo is None else stamp).astimezone(timezone.utc) + + +def _text(value): + return value.strip() if isinstance(value, str) else "" + + +def _runtime(rt): + if rt is None or not getattr(rt, "key", None): + raise ValueError("A tenant runtime is required.") + return rt + + +def record(rt, change=None, **fields): + """Atomically record tenant attributes and replay receipts, returning the inserted row count.""" + _runtime(rt) + change = {**(change or {}), **fields} + source = change.get("source") + if source not in {"notification", "snapshot"}: + raise ValueError("A listing change source is required.") + asin, sku = _text(change.get("asin")), _text(change.get("sku")) + marketplace = _text(change.get("marketplace_id") or change.get("marketplaceId")) + if not (asin or sku) or not marketplace or (source == "notification" and not asin): + raise ValueError("A listing identity and marketplace are required.") + notification_id = _text(change.get("notificationId")) + if source == "notification" and not notification_id: + raise ValueError("A notification ID is required.") + detected_at = _utc(change.get("publishedAt") if source == "notification" else change.get("detected_at")).isoformat() + batch = source == "snapshot" and "attributesChanged" in change + attributes = change.get("attributesChanged") if source == "notification" or batch else [change.get("attribute")] + if not isinstance(attributes, list) or not attributes: + raise ValueError("Changed listing attributes are required.") + attributes = list(dict.fromkeys(normalize_attribute(attribute) for attribute in attributes)) + values = {} + if batch: + # A snapshot batch expands to the same C4 rows as the single-attribute writer. + for key in ("before_values", "after_values"): + if not isinstance(change.get(key), dict): + raise ValueError("Snapshot attribute values are required.") + values[key] = {normalize_attribute(name): value for name, value in change[key].items()} + if any(attribute not in values[key] for attribute in attributes): + raise ValueError("Every snapshot attribute requires before and after values.") + incoming = [{"sku": sku or None, "asin": asin or None, "marketplace_id": marketplace, + "attribute": attribute, "before_value": (values["before_values"][attribute] if batch else + None if source == "notification" else change.get("before_value")), + "after_value": (values["after_values"][attribute] if batch else + None if source == "notification" else change.get("after_value")), + "detected_at": detected_at, "source": source, + "values_source": "snapshot" if source == "snapshot" else None, + "notificationId": notification_id or None, + "publishedAt": detected_at if source == "notification" else None, + "attributesChanged": attributes if source == "notification" or batch else [attribute]} + for attribute in attributes] + inserted = [0] + + def save(bucket): + inserted[0] = 0 + receipts = bucket.setdefault("notificationIds", {}) + if source == "notification" and notification_id in receipts: + # Abort the transaction so a replay performs no durable write. + raise _Replay + rows = bucket.setdefault("rows", {}) + for row in incoming: + # Different notification IDs remain separate changes, even within one day. + identity = [notification_id, row["attribute"]] if source == "notification" else row + row_id = hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + if row_id in rows: + continue + # Snapshots can fill values absent from a notification for the same attribute and UTC day. + counterpart = next((saved for saved in reversed(rows.values()) + if saved["source"] != source and asin and saved["asin"] == asin + and saved["marketplace_id"] == marketplace and saved["attribute"] == row["attribute"] + and saved["detected_at"][:10] == detected_at[:10] + and (not sku or not saved.get("sku") or saved["sku"] == sku) + and (source != "snapshot" or saved.get("values_source") is None + or (saved["before_value"], saved["after_value"]) + == (row["before_value"], row["after_value"]))), None) + if counterpart is not None: + if source == "snapshot": + counterpart.update({key: row[key] for key in ("sku", "before_value", "after_value", "values_source")}) + else: + counterpart.update({key: row[key] for key in + ("source", "notificationId", "publishedAt", "attributesChanged")}) + continue + rows[row_id] = row + inserted[0] += 1 + if source == "notification": + receipts[notification_id] = detected_at + return bucket + + try: + rt.update(KEY, save) + except _Replay: + return 0 + return inserted[0] + + +def _matching_changes(rows, asins, skus, since, mapped_asins=(), mapped_skus=()): + lower = _utc(since) if since is not None else None + return sorted((copy.deepcopy(row) for row in rows.values() + if (row.get("asin") in asins or row.get("sku") in skus + or (not row.get("sku") and row.get("asin") in mapped_asins) + or (not row.get("asin") and row.get("sku") in mapped_skus)) + and (lower is None or _utc(row["detected_at"]) >= lower)), + key=lambda row: row["detected_at"]) + + +def changes_for(rt, asins=None, since=None, *, skus=None): + """Read all matching tenant changes by ASIN or SKU, with an inclusive UTC lower bound.""" + _runtime(rt) + if asins is None and skus is None: + raise ValueError("An ASIN or SKU selector is required.") + wanted_asins = {_text(value) for value in ([asins] if isinstance(asins, str) else asins or [])} + wanted_skus = {_text(value) for value in ([skus] if isinstance(skus, str) else skus or [])} + rows = (rt.get(KEY, fresh=True) or {}).get("rows", {}) + return _matching_changes(rows, wanted_asins, wanted_skus, since) + + +def _target_selectors(targets): + if targets is None: + raise ValueError("Listing targets are required.") + selected = {"sku": set(), "asin": set(), "parent_asin": set()} + for target in [targets] if isinstance(targets, (str, dict)) else targets: + if not isinstance(target, (str, dict)): + raise ValueError("A listing target requires a grain and key.") + grain, key = ("sku", target) if isinstance(target, str) else (target.get("grain"), target.get("key")) + grain = "parent_asin" if grain == "parent" else grain + key = _text(key) + if grain not in selected or not key: + raise ValueError("A listing target requires a sku, asin or parent_asin grain and key.") + selected[grain].add(key) + return selected + + +def _resolve_targets(rt, selected): + from harness.amazon_store import _columns_of, _latest_account_snapshot_sql, asin_parents_sql + from harness.runtime import mirror_cursor + + asins, skus, mapped_asins = set(selected["asin"]), set(selected["sku"]), set() + mapped_skus, family_asins = set(), set() + parents = selected["parent_asin"] + if not asins and not skus and not parents: + return asins, skus, mapped_asins, mapped_skus + with closing(mirror_cursor(rt, require_ready=False)) as cur: + columns = _columns_of(cur, "amzn_listings") + current_listings = _latest_account_snapshot_sql(columns, "amzn_listings") + if skus and {"seller_sku", "asin1"} <= columns: + result = cur.execute("SELECT DISTINCT TRIM(CAST(asin1 AS VARCHAR)) " + f"FROM ({current_listings}) AS current_listings " + "WHERE TRIM(CAST(seller_sku AS VARCHAR)) IN (SELECT UNNEST(?)) " + "AND NULLIF(TRIM(CAST(asin1 AS VARCHAR)), '') IS NOT NULL", + [sorted(skus)]) + while batch := result.fetchmany(512): + mapped_asins.update(row[0] for row in batch) + if parents: + sql = asin_parents_sql(cur) + if sql is not None: + result = cur.execute(f"SELECT asin, parent_asin FROM ({sql}) AS families " + "WHERE parent_asin IN (SELECT UNNEST(?))", [sorted(parents)]) + while batch := result.fetchmany(512): + for asin, parent in batch: + family_asins.update((asin, parent)) + asins.update(family_asins) + if asins and {"seller_sku", "asin1"} <= columns: + result = cur.execute("SELECT DISTINCT TRIM(CAST(seller_sku AS VARCHAR)), TRIM(CAST(asin1 AS VARCHAR)) " + f"FROM ({current_listings}) AS current_listings " + "WHERE TRIM(CAST(asin1 AS VARCHAR)) IN (SELECT UNNEST(?)) " + "AND NULLIF(TRIM(CAST(seller_sku AS VARCHAR)), '') IS NOT NULL", [sorted(asins)]) + while batch := result.fetchmany(512): + for sku, asin in batch: + mapped_skus.add(sku) + if asin in family_asins: + skus.add(sku) + return asins, skus, mapped_asins, mapped_skus + + +@dataclass(frozen=True) +class _RuntimeReader: + _rt: object + + def changes_for(self, targets, since=None): + """Read project target changes from this tenant, closing the mirror cursor before returning.""" + selected = _target_selectors(targets) + rows = (self._rt.get(KEY, fresh=True) or {}).get("rows", {}) + if not rows or not any(selected.values()): + return _matching_changes({}, (), (), since) + asins, skus, mapped_asins, mapped_skus = _resolve_targets(self._rt, selected) + return _matching_changes(rows, asins, skus, since, mapped_asins, mapped_skus) + + +def for_runtime(rt): + """Bind project change reads to the exact tenant runtime without retaining a mirror cursor.""" + return _RuntimeReader(_runtime(rt)) diff --git a/platform/harness/amazon_schema.py b/platform/harness/amazon_schema.py index a0454467319de225d1ab7bb499906ca2dcc43365..147c30aa9a8f7ee9e9d33945047bfb6dd3dca28c 100644 --- a/platform/harness/amazon_schema.py +++ b/platform/harness/amazon_schema.py @@ -101,6 +101,10 @@ PROVENANCE = { PROVENANCE.update(dict.fromkeys(GROUP_A_TABLES, "measured_at_sync")) PROVENANCE.update(dict.fromkeys(DECLARED_DATASETS, "declared")) +REVIEW_UNAVAILABLE_NOTE = ( + "Amazon's API does not supply written reviews, average star rating or review count. " + "Read them on the product page or in Seller Central.") + #: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a #: screen and standing rule 2 governs it. PROVENANCE_COPY = { @@ -293,7 +297,8 @@ def catalogue(): label = declared["label"] native = ", ".join(declared.get("report_types") or declared.get("operations") or ()) else: - label, native = descriptors[key].label, descriptors[key].native + descriptor = descriptors[key] + label, native = descriptor.label, descriptor.native table[key] = {"label": label, "native": native} if provenance in {"derived", "apify_dataset"}: table["columns"] = [dict(field) for field in AMAZON_FIELDS[key] @@ -555,6 +560,10 @@ TABLE_SPECS.update({ for column, target, field, cardinality in _inbound_spec.links), grain="inbound_shipment_item"), "replenishment": _table_spec("replenishment", ("metric_id",), (("sku", "listings", "seller_sku", "many_to_one"),), grain="replenishment_metric"), + "review_topics": _table_spec("review_topics", ("asin", "marketplace_id", "sentiment", "topic"), + (("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic"), + "review_trends": _table_spec("review_trends", ("asin", "marketplace_id", "sentiment", "topic", "month"), + (("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic_month"), "sku_keywords": _table_spec("sku_keywords", ("keyword_id",), (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"), "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",), diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index e7a7338bce015b1ab2bceb7e895f7a4236b4d990..930f016dae0f5a9bc5c8172ab7421c67f6adf843 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -282,7 +282,7 @@ def _account_workspace(con, tenant_key, rt, entry): finally: con.execute(f"DETACH {alias}") root = duckdb.connect(str(workspace)) - scope = {"tenant": tenant_key, "entry": entry, "root": root, "path": path, + scope = {"tenant": tenant_key, "rt": rt, "entry": entry, "root": root, "path": path, "workspace": workspace, "changed": set()} previous = _account_scope() _ACCOUNT.scope = scope @@ -466,6 +466,31 @@ def read_database_enablement(rt): # Day length for per-table cadence. Manual sync keeps the report's own refresh floor. SCHEDULED_FLOOR_SECONDS = 86400 SCHEDULED_FLOOR_GRACE_SECONDS = 0 +# The resident cadence checks once a minute. +SCHEDULE_PHASE_GRACE_SECONDS = 60 +_SCHEDULE_OFFSET_READER = None + + +def register_schedule_offset(reader): + """Bind the application-owned slot offset without importing the API layer.""" + if not callable(reader): + raise TypeError("Amazon schedule offset requires a callable") + global _SCHEDULE_OFFSET_READER + _SCHEDULE_OFFSET_READER = reader + + +def _scheduled_phase_due(table, state, now, earliest): + """Keep scheduled pulls on deployment phases, including stale or absent boot ticks.""" + shift = int(_SCHEDULE_OFFSET_READER()) + interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"] + phase = ((now.timestamp() - shift) // interval) * interval + shift + stamp = _utc(state["updated_at"]).timestamp() if state and state.get("state") == "done" else None + if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS + or (stamp is not None and stamp >= phase)): + phase += interval + while earliest.timestamp() > phase + SCHEDULE_PHASE_GRACE_SECONDS: + phase += interval + return max(earliest, datetime.fromtimestamp(phase, timezone.utc)) def _cadence_seconds(table, state=None): @@ -574,18 +599,31 @@ def _readable_duration(seconds): def _interval_wait(table, state, now, floor_s=None): """Return the wait and deciding refresh floor, preserving the vendor floor on manual sync.""" + now = _utc(now) + vendor = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s") or 0 + seconds = max(int(vendor), int(floor_s or 0)) + phased = bool(floor_s and _SCHEDULE_OFFSET_READER is not None) + earliest = now if state and state.get("state") == "throttled" and state.get("detail"): due = _utc(state["detail"]) - wait = (due - _utc(now)).total_seconds() + wait = (due - now).total_seconds() if wait > 0: - return wait, due, int(wait), False - vendor = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s") or 0 - seconds = max(int(vendor), int(floor_s or 0)) + if not phased: + return wait, due, int(wait), False + earliest = due if seconds and state and state["state"] == "done": - due = _utc(state["updated_at"]) + timedelta(seconds=seconds) - wait = (due - _utc(now)).total_seconds() - if wait > 0: + # Daily cadence follows slots. Longer report periods and vendor floors keep their bounds. + minimum = (int(vendor) if phased and _client().CADENCE[table]["period"] == "day" else seconds) + due = _utc(state["updated_at"]) + timedelta(seconds=minimum) + earliest = max(earliest, due) + wait = (due - now).total_seconds() + if wait > 0 and not phased: return wait, due, seconds, seconds > int(vendor) + if phased: + due = _scheduled_phase_due(table, state, now, earliest) + wait = (due - now).total_seconds() + if wait > 0: + return wait, due, seconds, True return 0, None, seconds, False @@ -2250,6 +2288,16 @@ def _columns_of(con, table): "AND table_schema=current_schema() AND table_name=?", [table]).fetchall()} +def _catalog_walk_asins(con): + """Walk the listed catalogue plus every sales child ASIN without widening SQP's catalogue.""" + asins = set(_sqp_catalogue(con)) + if "child_asin" in _columns_of(con, "amzn_economics"): + asins.update(str(row[0]).strip() for row in con.execute( + "SELECT DISTINCT child_asin FROM amzn_economics WHERE child_asin IS NOT NULL").fetchall() + if str(row[0]).strip()) + return sorted(asins) + + _NEWEST_ORDER = ("start_date", "_period_start", "_snapshot_date", "_report_row_number") @@ -2373,11 +2421,22 @@ def _pricing_regrain(con, mirror, rows, log=print): return True +def _latest_account_snapshot_sql(columns, table): + """Select each connection and marketplace's latest snapshot, using available partition columns.""" + sql = f'SELECT * FROM "{table}" AS snapshot' + if "_snapshot_date" not in columns: + return sql + partitions = [name for name in ("_connection_id", "marketplace_id") if name in columns] + match = " AND ".join(f'latest."{name}" IS NOT DISTINCT FROM snapshot."{name}"' for name in partitions) + return (sql + f' WHERE snapshot."_snapshot_date" = (SELECT MAX(latest."_snapshot_date") ' + f'FROM "{table}" AS latest' + (" WHERE " + match if match else "") + ")") + + def asin_parents_sql(con): """S67-T29: ONE read-time resolver, one row per own ASIN: `(asin, parent_asin, parent_asin_source)`. - Own ASINs = DISTINCT listings `asin1` UNION the LATEST catalog `_snapshot_date`'s `asin` (the - snapshot tables keep one copy per sync day, D-3031, so only the newest day counts). Parent = + Own ASINs = DISTINCT listings `asin1` UNION each account/marketplace's latest catalog `asin`. + Legacy catalogs without account columns use their latest global date. Parent = that latest catalog row's `parent_asin` (source as stamped at sync), else pricing child_asin -> parent_asin when the pricing mirror carries child_asin (source `pricing`, O23), else economics child_asin -> parent_asin (newest start_date, source `economics`), else the ASIN itself @@ -2386,14 +2445,14 @@ def asin_parents_sql(con): Reusable verbatim by a SQL join (S67-T30): wrap it as a subquery. """ listings, catalog = _columns_of(con, "amzn_listings"), _columns_of(con, "amzn_catalog_items") - latest = ('(SELECT MAX("_snapshot_date") FROM amzn_catalog_items)') + current_catalog = _latest_account_snapshot_sql(catalog, "amzn_catalog_items") own = [] if "asin1" in listings: own.append("SELECT TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM amzn_listings WHERE asin1 IS NOT NULL") has_catalog = {"asin", "_snapshot_date"} <= catalog if has_catalog: - own.append("SELECT TRIM(CAST(asin AS VARCHAR)) AS asin FROM amzn_catalog_items " - f'WHERE asin IS NOT NULL AND "_snapshot_date" = {latest}') + own.append("SELECT TRIM(CAST(asin AS VARCHAR)) AS asin " + f"FROM ({current_catalog}) AS current_catalog WHERE asin IS NOT NULL") if not own: return None ctes = ["own AS (SELECT DISTINCT asin FROM (" + " UNION ".join(own) + ") WHERE asin <> '')"] @@ -2406,8 +2465,8 @@ def asin_parents_sql(con): "cat AS (SELECT asin, parent_asin, parent_asin_source FROM (SELECT TRIM(CAST(asin AS VARCHAR)) AS asin, " "TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, " f"COALESCE({source}, 'relationships') AS parent_asin_source, ROW_NUMBER() OVER (" - f"PARTITION BY TRIM(CAST(asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM amzn_catalog_items " - f"WHERE \"_snapshot_date\" = {latest} AND parent_asin IS NOT NULL " + f"PARTITION BY TRIM(CAST(asin AS VARCHAR)) ORDER BY {order}) AS _rn " + f"FROM ({current_catalog}) AS current_catalog WHERE parent_asin IS NOT NULL " "AND TRIM(CAST(parent_asin AS VARCHAR)) <> '') WHERE _rn = 1)") joins.append("LEFT JOIN cat ON cat.asin = own.asin") parents.append("cat.parent_asin") @@ -2989,7 +3048,11 @@ def _connect_tenant_mirror(tenant_key, rt=None): return scope["root"].cursor(), path candidate = datastore._candidate_connection(tenant_key, path) if candidate is not None: - return candidate, path + try: + return datastore.apply_limits(candidate), path + except BaseException: + candidate.close() + raise if Path(datastore.DB_PATH).resolve() != path.resolve(): datastore.use_path(path) if rt is not None: @@ -3670,7 +3733,7 @@ from core.connector_spec import mirror_for as _mirror_spec MANIFEST_VERSIONS = _mirrors.MANIFEST_VERSIONS MANIFEST_VERSION = _mirrors.MANIFEST_VERSION -MIRROR_PART_MAX_BYTES = _mirrors.MIRROR_PART_MAX_BYTES +MIRROR_PART_MAX_BYTES = min(_mirrors.MIRROR_PART_MAX_BYTES, 8_000_000) _MANIFEST_KEY = "manifest.json" _KEY_LEDGER = "keys.json" _TABLE_BLOB_RE = _mirrors._TABLE_BLOB_RE @@ -4788,7 +4851,7 @@ def _record_state(con, key, phase, rows=0, detail=None): "INSERT OR REPLACE INTO _sync_state " "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", [str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_", "amzn_resume_", - "amzn_resync_", "amzn_waiting_")) + "amzn_resync_", "amzn_waiting_", "amzn_listing_content_")) else str(detail)[:300]) if detail else None, int(rows or 0), _iso(datetime.now(timezone.utc))]) @@ -4896,6 +4959,22 @@ def _refused_reports(con): def plain_refusal_reason(reason, *, status="", table="", operation=""): """Project refusals to cause and action copy, retaining the fee API's vendor reason.""" text = str(reason or "") + feedback = table in ("review_topics", "review_trends") + subject = "review trends" if table == "review_trends" else "review topics" + if feedback and (str(status) in ("403", "NOT_OFFERED") + or re.search(r"answered\s+403\b|\(403\)", text)): + vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text) + detail = (vendor[1] if vendor and vendor[1] else "Amazon denied this seller or marketplace").strip() + detail = detail.replace("\u2014", ". ").replace("\u2013", " to ") + return (f"Amazon refused Customer Feedback {subject} (403): {detail}. " + "Ask an administrator to check seller eligibility, marketplace support, and either the Brand Analytics or Selling Partner Insights role, then reconnect Amazon.") + if feedback: + refusal = re.search(r"answered\s+(\d{3})(?:: (.*?))?(?:; fix:|$)", text) + if refusal and refusal[1] != "429": + detail = (refusal[2] or "Amazon refused this child ASIN or marketplace").strip() + detail = detail.replace("\u2014", ". ").replace("\u2013", " to ") + return (f"Amazon refused Customer Feedback {subject} ({refusal[1]}): {detail}. " + "Check the child ASIN and marketplace, then retry sync. Customer Feedback supports US, UK, France, Italy, Germany, Spain and Japan.") if table == "replenishment": if text.startswith("Amazon refused replenishment metrics:"): return text.replace("\u2014", ". ").replace("\u2013", " to ") @@ -5208,10 +5287,20 @@ FIRST_LOAD_POLL_SECONDS = 30 FIRST_LOAD_RESERVE_SECONDS = 1800 #: Generation records the document keeps. The job's history, never connector data. FIRST_LOAD_GENERATION_HISTORY = 200 -#: D-2643: each of these returns a row count or names why it is empty while the load runs. -EMPTY_REASON_TABLES = ("orders", "inventory", "listings") +#: D-2643: the codes an empty or refused Amazon table names. Inventory alone also keeps +#: S85-T107's `request_unknown` for missing request metadata. +EMPTY_REASON_CODES = ("not_requested", "refused_by_amazon", "first_load_pending", "no_source_rows") +#: D-2643: every declared database returns a row count or names why it is empty while the load runs. +#: Declared datasets and saved refusals join at run time through `empty_reason_tables`. +EMPTY_REASON_TABLES = DATABASE_TABLES +#: Retain the latest request after pending markers are cleared, within the inventory proof family. +INVENTORY_REQUEST_KEY = "reconcile_inventory_request" #: D-2642 and D-2649: these name why they are behind while the load runs. BEHIND_TABLES = ("orders", "order_items") +#: D-2642: the dated tables whose newest row is held against the table's cadence on every read. +LAG_DATE_COLUMNS = dict.fromkeys(BEHIND_TABLES, "purchase_date") +#: The keys `table_empty_status(lag=True)` adds. `lagCode` and `lag` appear only when a table trails. +LAG_KEYS = ("newestPurchaseDate", "cadence", "cadenceSeconds", "lastPullAt", "lagCode", "lag") #: S82-T02's C-status document (lane A). `connector_sync.STATUS_KEY` wins once it exists. SYNC_STATUS_DOC = "connector_sync_status" @@ -5524,80 +5613,119 @@ def _status_stamp(status, connector): def inventory_table_status(con): """Read inventory counts and account-scoped source proof without changing the mirror or requesting reports.""" rows = (int(con.execute("SELECT COUNT(*) FROM amzn_inventory").fetchone()[0]) - if _columns_of(con, "amzn_inventory") else 0) - out = {"rows": rows, "empty": rows == 0} - if rows: - return out - keys = ("amzn_inventory", "amzn_reconcile_inventory", "amzn_refused_inventory", "amzn_inventory_awd") + if _columns_of(con, "amzn_inventory") else None) + out = {"rows": rows, "empty": rows == 0 if rows is not None else None} + keys = ("amzn_inventory", "amzn_reconcile_inventory", "amzn_refused_inventory", "amzn_inventory_awd", + entity_key(INVENTORY_REQUEST_KEY)) clauses = " OR ".join("entity=? OR ends_with(entity, ?)" for _ in keys) saved = con.execute( "SELECT entity, phase, cursor_wd, rows, updated_at FROM _sync_state WHERE " + clauses + " OR contains(entity, 'amzn_pending_report_')", [value for key in keys for value in (key, "__" + key)]).fetchall() - accounts, pending, unknown_request = {}, [], False + accounts = {} + + def payload(state): + try: + value = json.loads(state.get("detail") or "{}") + except (ValueError, TypeError): + value = {} + return value if isinstance(value, dict) else {} + + def request_status(state): + detail = payload(state) + window = detail.get("window") if isinstance(detail.get("window"), dict) else {} + return {"type": detail.get("reportType") or _client().REPORT_TYPES["inventory"]["report"], + "state": state.get("state"), "reportId": detail.get("reportId"), + "processingStatus": detail.get("processingStatus"), + "documentId": detail.get("reportDocumentId") or detail.get("document"), + "start": detail.get("start") or window.get("start"), + "end": detail.get("end") or window.get("end"), "updatedAt": state.get("updated_at")} + + def refusal_sentence(reason): + cause, separator, action = str(reason).partition(". ") + return f"{cause}, so {action[:1].lower()}{action[1:]}" if separator else cause + for entity, phase, detail, count, stamp in saved: prefix, key = (str(entity).split("__", 1) if str(entity).startswith("amzn_account_") and "__" in str(entity) else ("", str(entity))) - if key.startswith("amzn_pending_report_"): - try: - request = json.loads(detail or "{}") - except (ValueError, TypeError): - request = {} - if not isinstance(request, dict): - request = {} + state = {"state": phase, "detail": detail, "rows": count, "updated_at": stamp} + if key.startswith("amzn_pending_report_") and payload(state).get("table") not in (None, "inventory"): + continue + accounts.setdefault(prefix, {})[key] = state + # Account markers own their proof once account scoping exists. + scoped = {key: state for key, state in accounts.items() if key} + accounts = scoped or accounts + reports, pending, unknown_request = [], [], False + for states in accounts.values(): + own_pending = [] + for key, state in states.items(): + if not key.startswith("amzn_pending_report_"): + continue + request = payload(state) if request.get("table") == "inventory": - pending.append({"type": request.get("reportType"), "state": phase, - "reportId": request.get("reportId"), - "processingStatus": request.get("processingStatus"), "updatedAt": stamp}) + own_pending.append(request_status(state)) elif not request.get("table"): unknown_request = True - continue - accounts.setdefault(prefix, {})[key] = {"state": phase, "detail": detail, - "rows": count, "updated_at": stamp} + latest = states.get(entity_key(INVENTORY_REQUEST_KEY)) or states.get("amzn_reconcile_inventory") + if latest: + own_pending = [row for row in own_pending + if str(row.get("updatedAt") or "") >= str(latest.get("updated_at") or "")] + pending.extend(own_pending) + if own_pending and (not latest or max(row.get("updatedAt") or "" for row in own_pending) + >= str(latest.get("updated_at") or "")): + reports.extend(own_pending) + elif latest: + reports.append(request_status(latest)) + if latest and latest.get("state") in ("pending", "throttled") and not own_pending: + pending.append(request_status(latest)) + out["reports"] = reports + if rows: + return out if pending: + throttled = any(report["state"] == "throttled" for report in pending) return {**out, "code": "first_load_pending", "reason": - "Amazon is preparing the inventory report, so keep sync enabled and retry after it completes.", - "reports": pending} - # Account markers own their proof once account scoping exists. - scoped = {key: state for key, state in accounts.items() if key} - accounts = scoped or accounts - reports = [] + "Amazon limited the inventory request, so keep sync enabled to resume after its waiting period." + if throttled else + "Amazon is preparing the inventory report, so keep sync enabled and retry after it completes."} + periods = set() for states in accounts.values(): marker = states.get("amzn_inventory") or {} refused = refused_reports(states) if refused: - return {**out, "code": "refused_by_amazon", "reason": refused[0]["reason"]} + return {**out, "code": "refused_by_amazon", "reason": refusal_sentence(refused[0]["reason"])} + request = states.get(entity_key(INVENTORY_REQUEST_KEY)) or {} + if request.get("state") in ("refused", "failed"): + return {**out, "code": "refused_by_amazon", "reason": refusal_sentence( + payload(request).get("reason") or plain_refusal_reason("", table="inventory"))} proof = states.get("amzn_reconcile_inventory") or {} - try: - detail = json.loads(proof.get("detail") or "{}") - except (ValueError, TypeError): - detail = {} - if not isinstance(detail, dict): - detail = {} + if not marker and not proof and not request: + if unknown_request: + return {**out, "code": "request_unknown", "reason": + "A saved Amazon request has no table identity, so retry sync to refresh the inventory report status."} + return {**out, "code": "request_unknown", "reason": + "Inventory request metadata is unavailable, so retry Amazon sync to record the FBA ledger request."} + detail = payload(proof) dates = detail.get("dates") valid_dates = (isinstance(dates, dict) and bool(dates) and all(isinstance(day, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", day) and type(count) is int and count == 0 for day, count in dates.items())) awd = states.get("amzn_inventory_awd") or {} + cancelled_empty = (detail.get("cancelled") is True and detail.get("processingStatus") == "CANCELLED" + and isinstance(detail.get("reportId"), str) and bool(detail["reportId"].strip()) + and not detail.get("document")) + source_document = isinstance(detail.get("document"), str) and bool(detail["document"].strip()) if (marker.get("state") != "done" or marker.get("rows") != 0 or proof.get("state") != "done" or type(detail.get("count")) is not int or detail["count"] != 0 - or not isinstance(detail.get("document"), str) or not detail["document"].strip() + or not (source_document or cancelled_empty) or detail.get("column") != "_snapshot_date" or not valid_dates or detail.get("start") != min(dates) or detail.get("end") != max(dates) or awd.get("state") != "done" or awd.get("rows") != 0): return {**out, "code": "first_load_pending", "reason": "Inventory source checks are incomplete, so retry Amazon sync to finish the FBA and AWD inventory checks."} - reports.append({"type": _client().REPORT_TYPES["inventory"]["report"], "state": proof["state"], - "reportId": detail.get("reportId"), "processingStatus": detail.get("processingStatus"), - "documentId": detail["document"], "start": detail["start"], "end": detail["end"], - "rows": detail["count"], "updatedAt": proof.get("updated_at")}) - if not reports: - if unknown_request: - return {**out, "code": "first_load_pending", "reason": - "A saved Amazon request has no table identity, so retry sync to refresh the inventory report status."} - return {**out, "code": "not_requested", "reason": - "No inventory report request is saved, so start Amazon sync to request the FBA ledger."} - periods = {(report["start"], report["end"]) for report in reports} + periods.add((detail["start"], detail["end"])) + if not periods: + return {**out, "code": "request_unknown", "reason": + "Inventory request metadata is unavailable, so retry Amazon sync to record the FBA ledger request."} start, end = next(iter(periods)) period = (start if start == end else f"{start} to {end}") if len(periods) == 1 else "the loaded dates" return {**out, "code": "no_source_rows", "reason": @@ -5605,6 +5733,277 @@ def inventory_table_status(con): "reports": reports} +_UNCOUNTED = object() +_INVENTORY_UNKNOWN = ("Inventory request metadata is unavailable, " + "so retry Amazon sync to record the FBA ledger request.") + + +def _reason_line(text): + """One screen line with no line break and no em or en dash.""" + return " ".join(str(text or "").replace("\u2014", ". ").replace("\u2013", " to ").split()) + + +def _marker_detail(state): + try: + value = json.loads((state or {}).get("detail") or "{}") + except (ValueError, TypeError): + value = {} + return value if isinstance(value, dict) else {} + + +def _reason_markers(con, table=None): + """Saved sync markers per account prefix (`""` for unscoped). Read only. + + With `table`, only that table's own, refusal and reconciliation markers are read. + """ + if not _columns_of(con, "_sync_state"): + return {} + sql, params = "SELECT entity, phase, cursor_wd, rows, updated_at FROM _sync_state", [] + if table is not None: + dated = str(table) in LAG_DATE_COLUMNS + keys = [entity_key(name + str(table)) + for name in ("", "refused_", "reconcile_", *((WATERMARK_PREFIX,) if dated else ()))] + keys += [entity_key(FIRST_LOAD_END_KEY)] if dated else [] + sql += " WHERE " + " OR ".join("entity=? OR ends_with(entity, ?)" for _ in keys) + params = [value for key in keys for value in (key, "__" + key)] + accounts = {} + for entity, phase, detail, count, stamp in con.execute(sql, params).fetchall(): + entity = str(entity) + prefix, key = (entity.split("__", 1) if entity.startswith("amzn_account_") and "__" in entity + else ("", entity)) + accounts.setdefault(prefix, {})[key] = {"state": phase, "detail": detail, "rows": count, + "updated_at": stamp} + return accounts + + +def _table_markers(accounts, table): + """Each account's markers for `table`. Account markers own a table once any account holds one.""" + keys = {entity_key(name + str(table)) for name in ("", "refused_", "reconcile_")} + scoped = [states for prefix, states in accounts.items() if prefix and keys & set(states)] + return scoped or ([accounts[""]] if "" in accounts else []) + + +def empty_reason_tables(con=None, *, accounts=None): + """D-2643: declared databases and datasets read now, plus every dataset with a saved refusal.""" + names = [*DATABASE_TABLES, *_client().DECLARED_DATASETS] + if con is not None or accounts is not None: + refused = entity_key("refused_") + for states in (_reason_markers(con) if accounts is None else accounts).values(): + names.extend(key[len(refused):] for key, state in states.items() + if key.startswith(refused) and state.get("state") == "refused") + return tuple(dict.fromkeys(names)) + + +def _stamp_or_none(value): + try: + return _utc(value) if value else None + except (ValueError, TypeError, AttributeError, OverflowError): + return None + + +def _lag_age(seconds): + """`259200` -> "3 days". Screen copy for a lag, rounded down to its largest whole unit.""" + seconds = max(0, int(seconds)) + for size, unit in ((86400, "day"), (3600, "hour")): + if seconds >= 2 * size: + return f"{seconds // size} {unit}s" + minutes = seconds // 60 + return f"{minutes} minute" + ("" if minutes == 1 else "s") + + +def _newest_row_at(con, table): + """The newest row date of a dated table in UTC, or None when no date can be read.""" + column, mirror = LAG_DATE_COLUMNS[table], table_name(table) + if column not in _columns_of(con, mirror): + return None + value = con.execute(f'SELECT epoch(MAX(TRY_CAST("{column}" AS TIMESTAMPTZ))) FROM {mirror}').fetchone()[0] + return None if value is None else datetime.fromtimestamp(float(value), timezone.utc) + + +def _last_pull_at(markers, table): + """The most behind account's newest finished pull of `table`. None when any account has none.""" + pulls = [] + for states in markers: + stamps = [stamp for key in (entity_key(WATERMARK_PREFIX + table), entity_key(table)) + if (states.get(key) or {}).get("state") == "done" + and (stamp := _stamp_or_none(states[key].get("updated_at"))) is not None] + if not stamps: + return None + pulls.append(max(stamps)) + return min(pulls) if pulls else None + + +def _lag_cause(table, markers, newest, last, now, cadence): + """D-2642: the code, cause and fix of a dated table that trails its cadence. Read only.""" + phases = {(states.get(entity_key(table)) or {}).get("state") for states in markers} + loads = {(states.get(entity_key(FIRST_LOAD_END_KEY)) or {}).get("state") for states in markers} + if "refused" in phases: + return "refused_by_amazon", "Amazon refused the last request", "follow the refusal reason and retry Amazon sync" + if "throttled" in phases: + return "throttled", "Amazon throttled the last pass", "the next pass resumes after Amazon's waiting period" + if "failed" in phases: + return "pass_failed", "the last Amazon pass did not finish", "retry Amazon sync" + if "pending" in phases: + return "report_pending", "Amazon is still preparing the last report", "the next pass collects it" + if last is None or (now - last).total_seconds() > cadence + SCHEDULE_PHASE_GRACE_SECONDS: + # A pull inside its cadence makes the newest row Amazon's answer. Only a missed pull is a stop. + if loads & {"running", "partial"}: + return ("first_load_pending", "the first Amazon load has not finished", + "newer orders arrive as it continues") + if last is None: + return "no_recorded_pull", "no finished Amazon pull is recorded", "run Amazon sync" + return ("sync_stopped", f"no Amazon pull has finished for {_lag_age((now - last).total_seconds())}", + "run Amazon sync and check its status in Connectors") + return ("no_newer_orders", f"Amazon reported no orders newer than {newest.date().isoformat()} in the last pull", + "check seller activity in Seller Central") + + +def unknown_lag(table): + """The lag keys of a dated table whose newest row date cannot be read. Never a fresh answer.""" + noun = str(table).replace("_", " ").removesuffix("s") + return {"lagCode": "unknown_timestamp", + "lag": f"The purchase date of the newest {noun} cannot be read, so run Amazon sync and check again."} + + +def _freshness(con, table, markers, now=None): + """D-2642: newest row date, cadence and last pull of a dated table, plus one lag line when it trails. + + The table trails when its newest row is older than one cadence. An unread date never reads fresh. + """ + now = _utc(now) + cadence = _cadence_seconds(table) + last = _last_pull_at(markers, table) + try: + newest = _newest_row_at(con, table) + except Exception: # noqa: BLE001 + newest = None + out = {"newestPurchaseDate": _iso(newest) if newest else None, + "cadence": "every " + _readable_duration(cadence).removeprefix("1 "), "cadenceSeconds": cadence, + "lastPullAt": _iso(last) if last else None} + if newest is None: + return {**out, **unknown_lag(table)} + age = (now - newest).total_seconds() + if age <= cadence: + return out + code, cause, fix = _lag_cause(table, markers, newest, last, now, cadence) + noun = str(table).replace("_", " ").removesuffix("s") + return {**out, "lagCode": code, + "lag": _reason_line(f"The newest {noun} is {_lag_age(age)} old because {cause}, so {fix}.")} + + +def table_empty_status(con, table, *, rows=_UNCOUNTED, enabled=None, accounts=None, lag=False, now=None): + """D-2643: one table's row count, or the code and one-line reason it answers with. + + The one per-table producer every door reads (Data grid, MCP, list_workspace). A saved refusal + names its reason beside any rows. Missing markers read first load pending, never a zero. + `enabled` is a readable enablement map. Pass None when enablement is unknown. + D-2642: with `lag`, a dated table that holds rows also answers `LAG_KEYS` from `_freshness`. + """ + table = str(table) + declared = _client().DECLARED_DATASETS.get(table) + if declared is not None: + return {"table": table, "rows": None, "empty": True, "code": "not_requested", + "reason": _reason_line(declared.get("reason"))} + mirror = table_name(table) + if rows is _UNCOUNTED: + rows = (int(con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]) + if _columns_of(con, mirror) else None) + out = {"table": table, "rows": rows, "empty": rows == 0 if rows is not None else None} + if table == "inventory": + if _columns_of(con, "_sync_state"): + status = inventory_table_status(con) + return {**out, **{key: status[key] for key in ("code", "reason") if status.get(key)}} + return out if rows else {**out, "code": "request_unknown", "reason": _INVENTORY_UNKNOWN} + accounts = _reason_markers(con, table) if accounts is None else accounts + markers = _table_markers(accounts, table) + refusals = [row["reason"] for states in markers for row in refused_reports(states) + if row["table"] == table] + if not refusals and table not in TABLES: + for states in markers: + saved = states.get(entity_key("refused_" + table)) or {} + if saved.get("state") == "refused": + detail = _marker_detail(saved) + refusals.append(plain_refusal_reason(detail.get("reason") or "", table=table, + status=detail.get("amazonStatus") or "")) + dated = (_freshness(con, table, markers, now) + if lag and rows and table in LAG_DATE_COLUMNS else {}) + if refusals: + return {**out, "code": "refused_by_amazon", "reason": _reason_line(refusals[0]), **dated} + if rows: + return {**out, **dated} + if enabled is not None and enabled.get(amazon_schema.DATABASE_PREFIX + table) is False: + return {**out, "code": "not_requested", "reason": + "This Amazon database is turned off in Amazon connection settings, so turn it on to request its rows."} + if table in DERIVED_TABLES: + return {**out, "code": "no_source_rows" if rows == 0 else "first_load_pending", "reason": + "This table is built from other saved Amazon data that holds no matching rows yet, " + "so sync Amazon and check again."} + landed = [states.get(entity_key(table)) or {} for states in markers] + phases = {marker.get("state") for marker in landed} + if "throttled" in phases: + return {**out, "code": "first_load_pending", "reason": + "Amazon is limiting requests for this table, so sync resumes after Amazon's waiting period."} + if "failed" in phases: + return {**out, "code": "first_load_pending", "reason": + "The last Amazon sync of this table did not finish, so retry Amazon sync."} + measured_zero = rows == 0 or (rows is None and all(marker.get("rows") == 0 for marker in landed)) + if landed and phases == {"done"} and measured_zero: + spans = [] + for states in markers: + proof = _marker_detail(states.get(entity_key("reconcile_" + table))) + span = " to ".join(dict.fromkeys(str(proof[key])[:10] for key in ("start", "end") if proof.get(key))) + if span and span not in spans: + spans.append(span) + period = spans[0] if len(spans) == 1 else "the loaded dates" if spans else "the last requested period" + return {**out, "code": "no_source_rows", "reason": + f"Amazon returned no rows for {period}, so check seller activity in Seller Central and retry Amazon sync."} + return {**out, "code": "first_load_pending", "reason": loading_reason(table)} + + +def loading_reason(table): + """The generic first-load sentence. It matches `amazon_relational.missing_source`'s sync sentence.""" + return (f"Amazon {str(table).replace('_', ' ')} is still loading from Amazon, " + "so ask again after the next Amazon sync finishes.") + + +def empty_reason_census(tenant_key, rt=None, *, con=None, counts=None, now=None): + """D-2643: every runtime-declared table and refused dataset with its count or reason. Read only. + + None when this tenant's mirror is absent or never held Amazon data. Opens no writer and issues no DDL. + `counts` holds row counts the caller already measured, such as a publication manifest's. + D-2642: a dated table with rows also answers its newest row date, cadence and any lag line. + """ + enablement = read_database_enablement(rt) if rt is not None else {} + enabled = enablement.get("databases") if enablement.get("available") else None + measured = {str(table): rows for table, rows in (counts or {}).items() if type(rows) is int} + + def build(cur): + accounts = _reason_markers(cur) + held = {row[0] for row in cur.execute( + "SELECT DISTINCT table_name FROM information_schema.columns WHERE table_catalog=current_database() " + "AND table_schema=current_schema() AND starts_with(table_name, ?)", [PREFIX]).fetchall()} + if not held and not any(key.startswith(PREFIX) for states in accounts.values() for key in states): + return None # This tenant's mirror has never held Amazon data. + return [table_empty_status(cur, table, enabled=enabled, accounts=accounts, lag=True, now=now, + **({"rows": measured[table]} if table in measured else {})) + for table in empty_reason_tables(accounts=accounts)] + + if con is not None: + return build(con) + path = _tenant_mirror_path(tenant_key, rt) + if not path.is_file(): + return None + if path.resolve() == Path(datastore.DB_PATH).resolve(): + cur = datastore.ro_cursor() + try: + return build(cur) + finally: + cur.close() + import duckdb + with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur: + return build(cur) + + def first_load_table_status(tasks, finished, failed, counts, *, table_status=None): """D-2643, D-2642, D-2649: a row count or a named reason per table, and why orders are behind.""" out = {} @@ -5619,6 +6018,9 @@ def first_load_table_status(tasks, finished, failed, counts, *, table_status=Non f"The first load is still loading this table. {len(left)} report windows remain." if left else "Amazon returned no rows for this seller in the loaded period." if own else "This table holds no rows and the first load owes it no windows.") + row["code"] = (("refused_by_amazon" if is_refusal_reason(reason) else "first_load_pending") + if reason else "first_load_pending" if left + else "no_source_rows" if own else "not_requested") if table == "inventory": row.update(empty=True, code="first_load_pending", reason= "Inventory source proof is not available in this job record, so retry Amazon sync to refresh its status.") @@ -5630,7 +6032,7 @@ def first_load_table_status(tasks, finished, failed, counts, *, table_status=Non else ", starting with the most recent month.")) if table_status and table in table_status: row.update(table_status[table]) - if not row.get("empty"): + if row.get("rows") is not None and not row.get("empty"): row.pop("reason", None) row.pop("code", None) out[table] = row @@ -6097,7 +6499,8 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log): held = all(r.strip() in cx.ROLES_TAKEN for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip()) held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I) - fix = ("review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else + fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in ("review_topics", "review_trends") else + "review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else "review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else f"needs the {role} role; this application already holds it, so Amazon does not " "offer this report to this seller account" if role and held else @@ -6170,9 +6573,31 @@ def _report_resume(con, table, report_type, **kwargs): def _report_document(con, table, fields, report_type, **kwargs): + from harness.amazon_client import AmazonError, AmazonReportPending, AmazonThrottled cx = _client() - with _report_resume(con, table, report_type, **kwargs) as resume_kwargs: - return cx.report_document(fields, report_type, **resume_kwargs) + resume_kwargs = {} + + def remember(state, meta): + if table == "inventory": + detail = {key: meta.get(key) for key in ("reportId", "processingStatus", "reportDocumentId")} + detail.update(reportType=report_type, start=kwargs.get("start"), end=kwargs.get("end")) + if meta.get("reason"): + detail["reason"] = meta["reason"] + _record_state(con, entity_key(INVENTORY_REQUEST_KEY), state, detail=json.dumps(detail)) + + try: + with _report_resume(con, table, report_type, **kwargs) as resume_kwargs: + result = cx.report_document(fields, report_type, **resume_kwargs) + except AmazonError as exc: + state = ("pending" if isinstance(exc, AmazonReportPending) else + "throttled" if isinstance(exc, AmazonThrottled) else "refused") + status = str(exc.status) if getattr(exc, "status", None) is not None else None + remember(state, {"reportId": getattr(exc, "report_id", None) or resume_kwargs.get("report_id"), + "processingStatus": status, + "reason": plain_refusal_reason(str(exc), status=status, table=table)}) + raise + remember("done", result[1]) + return result def _close_history_floor(con, table, exc, start, end): @@ -6214,6 +6639,22 @@ def _restatement_may_overlap(con, mirror, predicate, params, start, end, *, unsa [*params, end, end, start]).fetchone()) +def _validate_report_snapshot(con, table, marker): + """Reconcile one flat-file snapshot against its raw document count and source coordinates.""" + total, matched, distinct = con.execute( + f'SELECT COUNT(*), COUNT(*) FILTER (WHERE _snapshot_date IS NOT DISTINCT FROM ? ' + 'AND _report_document_id IS NOT DISTINCT FROM ?), ' + 'COUNT(DISTINCT CASE WHEN _report_row_number > 0 THEN _report_row_number END) ' + f'FROM {table_name(table)}', + [marker["start"], marker["document"]]).fetchone() + expected = marker["count"] + matches = total == matched == distinct == expected + return {"state": "ok" if matches else "refused", "expected": expected, "actual": total, + "source_rows": matched, "distinct": distinct, + "reason": (f"Amazon document has {expected} rows; stored snapshot has {total} rows, " + f"{matched} from that document and date, and {distinct} source row numbers")} + + def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False): """Atomic range/date replacement and proof; an empty restatement removes old rows too. @@ -6233,6 +6674,10 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N if table == "inbound_shipments": declared["quantity"] = "BIGINT" declared.update(shipment_confirmation_id="VARCHAR", quantity_received="BIGINT") + if table == "review_topics": + declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns}) + if table == "review_trends": + declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns}) types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True) predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?' if column == "posted_date" and end: @@ -6281,6 +6726,10 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N count = con.execute(f"SELECT COUNT(*) FROM {staged}").fetchone()[0] else: count = _replace(con, mirror, columns, types, rows) + if snapshot and marker is not None and marker.get("document"): + proof = _validate_report_snapshot(con, table, marker) + if proof["state"] != "ok": + raise ValueError(f"{table}: snapshot replacement refused: {proof['reason']}") if marker is not None: _record_reconciliation(con, table, marker) if table == "listing_items": @@ -6436,6 +6885,8 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count): marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date", "count": source_count, "dates": counts, "document": meta.get("reportDocumentId")} marker.update({key: meta[key] for key in ("reportId", "processingStatus") if meta.get(key)}) + if table == "inventory" and meta.get("cancelled") is True and meta.get("processingStatus") == "CANCELLED": + marker["cancelled"] = True # Every earlier date and any undated pre-migration copy goes; AWD rows are another leg. predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE" con.execute("BEGIN TRANSACTION") @@ -6744,10 +7195,328 @@ def _name_skipped_plans(con, skipped, operations, log): log(f"inbound_shipments: skipped plans {json.dumps(skipped)}") -def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations=None): +def _catalog_listing_content(item, marketplace): + """Select listing text and image links using Amazon's notification attribute identities.""" + attributes = item.get("attributes") or {} + + def local(values): + return [value for value in values if isinstance(value, dict) + and value.get("marketplace_id", value.get("marketplaceId", marketplace)) == marketplace] + + content = {name: [{key: value[key] for key in ("value", "language_tag") if key in value} + for value in local(attributes.get(name) or [])] + for name in ("item_name", "bullet_point", "product_description")} + if not content["item_name"]: + content["item_name"] = [{"value": summary["itemName"]} + for summary in local(item.get("summaries") or []) if summary.get("itemName")] + for name, values in attributes.items(): + if name.endswith("_image_locator") or re.fullmatch(r"other_product_image_locator_\d+", name): + content[name] = sorted({value["media_location"] for value in local(values or []) + if value.get("media_location")}) + pictures = {} + for group in local(item.get("images") or []): + for picture in group.get("images") or []: + variant, link = picture.get("variant"), picture.get("link") + if not link: + continue + name = ("main_product_image_locator" if variant == "MAIN" else + "swatch_product_image_locator" if variant == "SWCH" else + "other_product_image_locator_" + str(int(variant[2:])) + if re.fullmatch(r"PT\d+", variant or "") else "images") + pictures.setdefault(name, set()).add(link) + for name, links in pictures.items(): + content[name] = sorted(set(content.get(name, [])) | links) + return {name: value for name, value in content.items() if value} + + +def _catalog_listing_hash(content): + """Hash the selected content independently of snapshot dates and response key order.""" + return hashlib.sha256(json.dumps(content, sort_keys=True, separators=(",", ":"), + ensure_ascii=False).encode("utf-8")).hexdigest() + + +def _catalog_listing_changes(con, records, marketplace, rt=None): + """Compare every ASIN with its connection's durable content baseline and record C4 changes.""" + from harness import amazon_listing_changes + scope = _account_scope() + rt = _ledger_runtime(scope["tenant"] if scope else getattr(rt, "key", None), + rt if rt is not None else (scope or {}).get("rt")) + for item in records: + asin = str(item.get("asin") or "").strip() + if not asin: + continue + identity = json.dumps([marketplace, asin], separators=(",", ":")) + key = entity_key("listing_content_" + hashlib.sha256(identity.encode()).hexdigest()) + content = _catalog_listing_content(item, marketplace) + digest = _catalog_listing_hash(content) + state = _read_state(con, key) + saved = json.loads(state["detail"]) if state else None + + def deliver(previous): + pending = previous["pending"] + changed = sorted(name for name in previous["attributes"].keys() | pending["attributes"].keys() + if previous["attributes"].get(name, []) != pending["attributes"].get(name, [])) + amazon_listing_changes.record(rt, asin=asin, sku=None, marketplace_id=marketplace, + attributesChanged=changed, source="snapshot", + before_values={name: previous["attributes"].get(name, []) for name in changed}, + after_values={name: pending["attributes"].get(name, []) for name in changed}, + detected_at=pending["detected_at"]) + baseline = {"hash": pending["hash"], "attributes": pending["attributes"]} + _record_state(con, key, "done", detail=json.dumps(baseline, ensure_ascii=False)) + return baseline + + if saved and saved.get("pending"): + # A retry retains its timestamp so a recorded change cannot be inserted twice. + saved = deliver(saved) + if saved is None: + _record_state(con, key, "done", detail=json.dumps({"hash": digest, "attributes": content}, + ensure_ascii=False)) + continue + if saved["hash"] == digest: + continue + saved["pending"] = {"hash": digest, "attributes": content, + "detected_at": datetime.now(timezone.utc).isoformat()} + _record_state(con, key, "pending", detail=json.dumps(saved, ensure_ascii=False)) + deliver(saved) + + +REVIEW_TOPIC_METRICS = ( + ("asin_metrics_number_of_mentions", ("asinMetrics", "numberOfMentions")), + ("asin_metrics_occurrence_percentage", ("asinMetrics", "occurrencePercentage")), + ("asin_metrics_star_rating_impact", ("asinMetrics", "starRatingImpact")), + ("parent_asin_metrics_number_of_mentions", ("parentAsinMetrics", "numberOfMentions")), + ("parent_asin_metrics_occurrence_percentage", ("parentAsinMetrics", "occurrencePercentage")), + ("parent_asin_metrics_star_rating_impact", ("parentAsinMetrics", "starRatingImpact")), + ("browse_node_metrics_occurrence_percentage_all_products", ("browseNodeMetrics", "occurrencePercentage", "allProducts")), + ("browse_node_metrics_star_rating_impact_all_products", ("browseNodeMetrics", "starRatingImpact", "allProducts")), + ("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")), + ("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")), +) +REVIEW_TOPIC_TYPES = { + **dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "review_snippets", "subtopics", + "review_snippet_1", "review_snippet_2", "review_snippet_3"), "VARCHAR"), + **{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE" + for name, _path in REVIEW_TOPIC_METRICS}, +} + + +def _review_topic_evidence(records, asins, marketplace): + """Count raw vendor topics and retain their metrics before the row parser runs.""" + import math + expected, counts, metrics, absent, seen = set(asins), {}, {}, [], set() + for record in records: + asin = record.get("asin") + if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace: + raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace") + seen.add(asin) + no_content = record.get("_no_feedback_reason") + topics = record.get("topics") + if not no_content and not isinstance(topics, dict): + raise ValueError("Amazon getItemReviewTopics returned no valid topics object") + total = 0 + for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): + group = None if no_content else topics.get(key) + group = [] if group is None else group + if not isinstance(group, list): + raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list") + counts[json.dumps([asin, sentiment], separators=(",", ":"))] = len(group) + total += len(group) + for topic in group: + if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip() + or not isinstance(topic.get("asinMetrics"), dict)): + raise ValueError("Amazon review topic requires a topic name and asinMetrics object") + identity = json.dumps([asin, marketplace, sentiment, topic["topic"]], separators=(",", ":")) + if identity in metrics: + raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment") + values = [] + for name, path in REVIEW_TOPIC_METRICS: + value = topic + for segment in path: + if value is None: + break + if not isinstance(value, dict): + raise ValueError("Amazon review topic returned invalid nested metrics") + value = value.get(segment) + if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value)): + raise ValueError("Amazon review topic returned a nonnumeric metric") + if name.endswith("_number_of_mentions") and value is not None and (type(value) is not int or value < 0): + raise ValueError("Amazon review topic returned an invalid mention count") + values.append(value) + metrics[identity] = values + if not total: + absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TOPICS", + "reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."}) + if seen != expected: + raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN") + return {"count": sum(counts.values()), "source_counts": counts, "source_metrics": metrics, + "marketplace_id": marketplace, "absent_asins": absent} + + +def review_topic_rows(records, document): + """Flatten positive and negative topics while retaining null metrics and vendor percentage units.""" + rows = [] + for record in records: + for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): + for topic in (record.get("topics") or {}).get(key) or []: + row = dict.fromkeys(REVIEW_TOPIC_TYPES) + row.update(_flatten_row(topic)) + row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment) + row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record})) + snippets = topic.get("reviewSnippets") + if snippets is not None and (not isinstance(snippets, list) or len(snippets) > 3 + or any(not isinstance(snippet, str) for snippet in snippets)): + raise ValueError("Amazon review snippets must be null or a list of up to three strings") + for index, snippet in enumerate(snippets or [], 1): + row[f"review_snippet_{index}"] = snippet + rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics")) + return rows + + +def _sync_review_topics(con, fields): + """Replace the complete listing-ASIN topic snapshot with same-pass vendor reconciliation.""" + columns = _columns_of(con, "amzn_listings") + asins = ([row[0] for row in con.execute( + 'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()] + if "asin1" in columns else []) + marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID + records = _client().item_review_feedback(fields, asins) + evidence = _review_topic_evidence(records, asins, marketplace) + document = "operation:" + uuid.uuid4().hex + rows = review_topic_rows(records, document) + period = datetime.now(timezone.utc).date().isoformat() + for row in rows: + row["_snapshot_date"] = period + reason = ("Amazon Customer Feedback returned no review topics for this marketplace. " + "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly." + if asins else "No source rows: the Amazon listings catalogue has no child ASINs. Sync Amazon listings first.") + marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, + "empty_reason": reason if not rows else ""} + count = _replace_period(con, "review_topics", rows, + sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}), + "_snapshot_date", period, marker=marker, snapshot=True) + _record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({ + "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]})) + return count + + +REVIEW_TREND_TYPES = { + **dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month", + "date_range_start_date", "date_range_end_date"), "VARCHAR"), + "asin_metrics_occurrence_percentage": "DOUBLE", +} + + +def _review_trend_evidence(records, asins, marketplace): + """Count vendor trend points and retain source values before flattening or storing rows.""" + import math + expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, [] + for record in records: + asin = record.get("asin") + if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace: + raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace") + seen.add(asin) + no_content = record.get("_no_feedback_reason") + trends = record.get("reviewTrends") + if not no_content and not isinstance(trends, dict): + raise ValueError("Amazon getItemReviewTrends returned no valid reviewTrends object") + total = 0 + for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): + topics = [] if no_content else trends.get(key) + topics = [] if topics is None else topics + if not isinstance(topics, list): + raise ValueError("Amazon getItemReviewTrends returned an invalid sentiment topic list") + group = json.dumps([asin, sentiment], separators=(",", ":")) + counts[group] = 0 + topic_names = set() + for topic in topics: + if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) + or not topic["topic"].strip() or not isinstance(topic.get("trendMetrics"), list)): + raise ValueError("Amazon review trend requires a topic name and trendMetrics list") + if topic["topic"] in topic_names: + raise ValueError("Amazon returned duplicate trend topics for one ASIN and sentiment") + topic_names.add(topic["topic"]) + # Count raw points, independently of review_trend_rows. + counts[group] += len(topic["trendMetrics"]) + for point in topic["trendMetrics"]: + if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict) + or not isinstance(point.get("asinMetrics"), dict)): + raise ValueError("Amazon review trend requires dateRange and asinMetrics objects") + start, end = (point["dateRange"].get(name) for name in ("startDate", "endDate")) + if any(not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value) + for value in (start, end)): + raise ValueError("Amazon review trend requires ISO calendar dates") + first, last = datetime.strptime(start, "%Y-%m-%d"), datetime.strptime(end, "%Y-%m-%d") + if first > last or start[:7] != end[:7]: + raise ValueError("Amazon review trend dateRange must identify one calendar month") + identity = json.dumps([asin, marketplace, sentiment, topic["topic"], start[:7]], + separators=(",", ":")) + if identity in metrics: + raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month") + value = point["asinMetrics"].get("occurrencePercentage") + if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value) or not 0 <= value <= 100): + raise ValueError("Amazon review trend returned an invalid occurrence percentage") + metrics[identity] = [start, end, value] + total += counts[group] + if not total: + absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS", + "reason": no_content or "Amazon returned no positive or negative review trend points for this ASIN and marketplace."}) + if seen != expected: + raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN") + return {"count": sum(counts.values()), "source_counts": counts, "source_metrics": metrics, + "marketplace_id": marketplace, "absent_asins": absent} + + +def review_trend_rows(records, document): + """Flatten vendor occurrence percentages at ASIN, sentiment, topic and month grain.""" + rows = [] + for record in records: + for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): + for topic in (record.get("reviewTrends") or {}).get(key) or []: + for point in topic["trendMetrics"]: + dates = point["dateRange"] + row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"], + "sentiment": sentiment, "topic": topic["topic"], "month": dates["startDate"][:7], + "date_range_start_date": dates["startDate"], "date_range_end_date": dates["endDate"], + "asin_metrics_occurrence_percentage": point["asinMetrics"].get("occurrencePercentage")} + rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends")) + return rows + + +def _sync_review_trends(con, fields): + """Replace the full six-month review snapshot with independently counted source evidence.""" + columns = _columns_of(con, "amzn_listings") + asins = ([row[0] for row in con.execute( + 'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()] + if "asin1" in columns else []) + marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID + records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends") + evidence = _review_trend_evidence(records, asins, marketplace) + rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex) + period = datetime.now(timezone.utc).date().isoformat() + for row in rows: + row["_snapshot_date"] = period + reason = ("Amazon Customer Feedback returned no review trends for this marketplace. " + "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly." + if asins else "No source rows: the Amazon listings catalogue has no child ASINs. Sync Amazon listings first.") + marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, + "empty_reason": reason if not rows else ""} + count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES, + "_snapshot_date", period, marker=marker, snapshot=True) + _record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({ + "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]})) + return count + + +def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations=None, rt=None): """The five money tables share the outer sync's budget, errors and completion markers.""" cx = _client() spec = cx.REPORT_TYPES[table] + if table == "review_topics": + return _sync_review_topics(con, fields) + if table == "review_trends": + return _sync_review_trends(con, fields) if table == "economics": start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat() if start >= end: @@ -6786,7 +7555,7 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations return _replace_period(con, table, rows, sorted({k for r in rows for k in r}), "posted_date", start, end, marker) if spec.get("api"): - asins = _sqp_catalogue(con) if table == "catalog_items" else [] + asins = _catalog_walk_asins(con) if table == "catalog_items" else [] if table == "catalog_items": _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({ "reason": (f"Amazon Catalog: {len(asins)} ASINs in {(len(asins) + 19) // 20} batches. " @@ -6826,6 +7595,7 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations # The raw `images` JSON column stays; this names the MAIN link `amazon_pictures.fill` stores. row["main_image_url"] = (pictures.get(asin) or {}).get("url") row["main_image_full_url"] = (pictures.get(asin) or {}).get("full_url") + _catalog_listing_changes(con, records, mkt, rt=rt) column = "_snapshot_date" if spec.get("snapshot") else "_period_start" period = datetime.now(timezone.utc).date().isoformat() if spec.get("snapshot") else start for row in rows: @@ -6947,6 +7717,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations and spec["json_root"] not in json.loads(_document_text(text) or "{}"))) marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period, "column": column, "count": 0 if empty else document_row_count(text, roots)} + if spec.get("snapshot") and not roots and meta.get("reportDocumentId"): + marker["document"] = meta["reportDocumentId"] return _replace_period(con, table, rows, columns, column, period, marker=marker, snapshot=bool(spec.get("snapshot"))) @@ -7130,6 +7902,8 @@ def _validate_margin(con, table): try: marker = json.loads(state["detail"]) column = marker.get("column") or ("start_date" if table == "economics" else "posted_date") + if column == "_snapshot_date" and marker.get("document"): + return _validate_report_snapshot(con, table, marker) # Identifiers come only from this fixed table-kind selection, never stored JSON text. if column not in ("start_date", "posted_date", "_snapshot_date", "month", "_period_start"): raise ValueError("unrecognized reconciliation period column") @@ -7142,6 +7916,37 @@ def _validate_margin(con, table): result = con.execute(f"SELECT * FROM {table_name(table)} WHERE {predicate}", values) names = [c[0] for c in result.description] rows = [dict(zip(names, row)) for row in result.fetchall()] + if table == "review_trends": + counts = dict.fromkeys(marker["source_counts"], 0) + metrics = {} + for row in rows: + group = json.dumps([row.get("asin"), row.get("sentiment")], separators=(",", ":")) + counts[group] = counts.get(group, 0) + 1 + identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"), + row.get("topic"), row.get("month")], separators=(",", ":")) + metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"), + row.get("asin_metrics_occurrence_percentage")] + total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] + trends_match = (marker["count"] == len(rows) == len(metrics) == total + and counts == marker["source_counts"] and metrics == marker["source_metrics"]) + return {"state": "ok" if trends_match else "refused", "expected": marker["count"], "actual": len(rows), + "reason": (marker["empty_reason"] if trends_match and not rows else + f"Amazon returned {marker['count']} review trend points. Stored {len(rows)} points with per-ASIN, sentiment, topic and month reconciliation.")} + if table == "review_topics": + counts = dict.fromkeys(marker["source_counts"], 0) + metrics = {} + for row in rows: + group = json.dumps([row.get("asin"), row.get("sentiment")], separators=(",", ":")) + counts[group] = counts.get(group, 0) + 1 + identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"), row.get("topic")], + separators=(",", ":")) + metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS] + total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] + matches = (marker["count"] == len(rows) == len(metrics) == total + and counts == marker["source_counts"] and metrics == marker["source_metrics"]) + return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows), + "reason": (marker["empty_reason"] if matches and not rows else + f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")} if table == "fee_estimates_api": expected, actual = marker["count"], len(rows) identities = {row.get("estimate_id") for row in rows} @@ -7980,7 +8785,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if isinstance(report_budget, int) and documents >= max(0, report_budget): raise SQPBudgetDeferred("Search Query Performance report budget reached; remaining batches continue next pass.") documents += 1 - _sync_event("amazon.sync", tenant_key, SQP_TABLE, + _sync_event("amazon.access", tenant_key, SQP_TABLE, {"state": "access", "operation": "report_document"}, rt, report, log) return _report_document(con, SQP_TABLE, *args, **kwargs, max_wait=max_wait, log=log) @@ -8231,7 +9036,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri pricing_asins = contexts.setdefault(table, []) if table == "settlements": operation = "getReports" - _sync_event("amazon.sync", tenant_key, table, + _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": operation}, rt, report, log) discovered = cx.settlement_reports( fields, since=_iso(plan["start"]) if plan["incremental"] else None, @@ -8251,7 +9056,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri probe = _DocumentProbe(table, rtype) operation = "getReportDocument" documents += 1 - _sync_event("amazon.sync", tenant_key, table, + _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": operation}, rt, report, log) text = cx.settlement_document(fields, document_id) _DocumentProbe.downloaded(text) @@ -8354,10 +9159,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri # One detail window is one budget task, including its reconciliation query. probe = _DocumentProbe(table, spec.get("report") or operation) documents += 1 - _sync_event("amazon.sync", tenant_key, table, + _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": operation}, rt, report, log) count = _sync_margin_window(con, table, fields, w0, w1, - max_wait=max_wait, log=log, operations=operations) + max_wait=max_wait, log=log, operations=operations, rt=rt) if (w0, w1) in resync: # Per window, so a failure later in this tick never downloads a landed window again. resync_landed(con, table, (w0, w1)) @@ -8393,9 +9198,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri window = [_iso(w0), _iso(w1)] if window in completed_windows: continue - # W59-T43 / C2 has no amazon.access kind. Record each credential-backed - # vendor attempt as a sync ACCESS phase, distinct from the completion event. - # Lane F/combine own the vocabulary decision; no credential or row enters C2. + # Each vendor attempt records access before the call, without credentials or source rows. documents += 1 probe = _DocumentProbe(table, rtype) try: @@ -8406,7 +9209,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri **({"options": {**REPORT_OPTIONS.get(table, {}), "reportPeriod": cx.REPORT_TYPES[table]["period"]}} if cx.REPORT_TYPES[table].get("period") else {"options": dict(REPORT_OPTIONS[table])} if table in REPORT_OPTIONS else {})) as resume_kwargs: - _sync_event("amazon.sync", tenant_key, table, + _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": "report_document"}, rt, report, log) text, meta = cx.report_document(fields, rtype, **resume_kwargs) except cx.AmazonReportError as exc: @@ -8550,7 +9353,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri log(" pricing: getCompetitiveSummary UNPROVEN, no ASIN was returned") else: try: - _sync_event("amazon.sync", tenant_key, table, + _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": "getCompetitiveSummary"}, rt, report, log) responses = cx.competitive_summaries(fields, pricing_asins) @@ -8572,7 +9375,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if table == "inventory": state_key = ENTITY_STATE_KEYS[table] try: - _sync_event("amazon.sync", tenant_key, table, + _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": "listInventory"}, rt, report, log) inventory = cx.awd_inventory(fields) awd_rows = _awd_rows(inventory) @@ -8879,7 +9682,7 @@ def _validate_account(tenant_key="royal-imports", rt=None, log=print): out[table] = {"state": "unproven", "reason": "table has never synced", "reconciles": spec.get("validates_against")} continue - if table in ("fee_estimates_api", "replenishment", "inbound_shipments"): + if table in ("fee_estimates_api", "replenishment", "inbound_shipments", "review_topics", "review_trends"): out[table] = {"rows": n, "reconciles": spec.get("validates_against"), **_validate_margin(con, table)} continue @@ -9053,6 +9856,172 @@ def _fixture_damaged_parquet(data): return data[:4] + bytes([data[4] ^ 1]) + data[5:] +def inventory_status_selftest(log=print, check=None): + """Exercise request retention and empty-ledger evidence using memory tables and stubbed Amazon responses.""" + from unittest.mock import patch + import duckdb + cx = _client() + failures = [] + + def ck(name, ok): + if check: + check("[inventory status] " + name, bool(ok)) + else: + log(f" {'ok' if ok else 'FAIL'} {name}") + if not ok: + failures.append(name) + + report_type = cx.REPORT_TYPES["inventory"]["report"] + window = {"start": "2026-11-09T00:00:00Z", "end": "2026-11-09T23:59:59Z"} + header = "date\tmsku\tfnsku\tdisposition\tlocation\tending-warehouse-balance\n" + meta = {"reportId": "fixture-inventory-pending", "processingStatus": "DONE", + "reportDocumentId": "fixture-ledger-document", "dataEndTime": window["end"]} + con = duckdb.connect(":memory:") + try: + con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " + "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") + con.execute("CREATE TABLE amzn_orders AS SELECT i AS fixture_row FROM range(2) t(i)") + con.execute("CREATE TABLE amzn_listings AS SELECT i AS fixture_row FROM range(3) t(i)") + with patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("offline fixture")), \ + patch.object(cx, "access_token", return_value="fixture-access"), \ + patch.object(cx, "_create_report", return_value="fixture-inventory-terminal"): + unknown = inventory_table_status(con) + ck("missing metadata and an absent mirror remain unknown", unknown["code"] == "request_unknown" + and unknown["rows"] is None and unknown["empty"] is None) + projected = first_load_table_status([], set(), {}, {}, table_status={"inventory": unknown})["inventory"] + ck("job projection preserves an absent mirror's reason and unknown count", + projected["rows"] is None and projected["code"] == "request_unknown") + pending = cx.AmazonReportPending("fixture-inventory-pending", "IN_PROGRESS") + with patch.object(cx, "report_document", side_effect=pending): + try: + _report_document(con, "inventory", {}, report_type, **window) + except cx.AmazonReportPending: + pass + status = inventory_table_status(con) + ck("pending report keeps its actual ID and state", status["code"] == "first_load_pending" + and status["reports"][0]["reportId"] == pending.report_id + and status["reports"][0]["processingStatus"] == "IN_PROGRESS") + with patch.object(cx, "report_document", return_value=(header, meta)) as document: + text, response = _report_document(con, "inventory", {}, report_type, **window) + ck("completion resumes the saved ID and clears only the pending marker", + document.call_args.kwargs["report_id"] == pending.report_id + and not _read_state(con, _pending_report_key("inventory", report_type, **window)) + and json.loads(_read_state(con, entity_key(INVENTORY_REQUEST_KEY))["detail"])["reportId"] + == meta["reportId"]) + + def land_empty(response): + columns, records = parse_flat_file(header) + _replace_legacy_snapshot(con, "inventory", columns, records, response, document_row_count(header)) + _record_state(con, "amzn_inventory", "done", 0) + _record_state(con, "amzn_inventory_awd", "done", 0) + + land_empty(response) + status = inventory_table_status(con) + ck("empty source document names cause and request evidence", status["code"] == "no_source_rows" + and status["rows"] == 0 and "retry Amazon sync" in status["reason"] + and status["reports"][0]["reportId"] == meta["reportId"] + and status["reports"][0]["processingStatus"] == "DONE") + ck("an empty source does not certify a metric", _validate_snapshot(con, "inventory")["state"] == "unproven") + _record_state(con, "amzn_inventory_awd", "pending", 0) + ck("unfinished AWD cannot prove empty inventory", inventory_table_status(con)["code"] != "no_source_rows") + _record_state(con, "amzn_inventory_awd", "done", 0) + + with patch.object(cx, "_request", return_value={"processingStatus": "CANCELLED", + "dataEndTime": window["end"]}): + text, cancelled = _report_document(con, "inventory", {}, report_type, **window) + land_empty(cancelled) + status = inventory_table_status(con) + ck("confirmed no-data cancellation needs no document ID", text == "" + and status["code"] == "no_source_rows" + and status["reports"][0]["processingStatus"] == "CANCELLED" + and status["reports"][0]["reportId"] == "fixture-inventory-terminal") + saved = _read_state(con, "amzn_reconcile_inventory") + proof = json.loads(saved["detail"]) + proof.pop("cancelled") + _record_state(con, "amzn_reconcile_inventory", "done", detail=json.dumps(proof)) + ck("a cancellation label alone proves no empty source", inventory_table_status(con)["code"] != "no_source_rows") + _record_state(con, "amzn_reconcile_inventory", "done", detail=saved["detail"]) + + for terminal in ("FATAL", "CANCELLED"): + with patch.object(cx, "_request", return_value={"processingStatus": terminal, + "errorMessage": "Fixture source rejection"}): + try: + _report_document(con, "inventory", {}, report_type, **window) + except cx.AmazonReportError as exc: + ck(terminal + " error carries the report ID", exc.report_id == "fixture-inventory-terminal") + else: + ck(terminal + " error carries the report ID", False) + status = inventory_table_status(con) + ck(terminal + " refusal retains request state instead of claiming empty", + status["code"] == "refused_by_amazon" + and status["reports"][0]["processingStatus"] == terminal + and status["reports"][0]["reportId"] == "fixture-inventory-terminal" + and status["reason"].count(".") == 1 and ", so " in status["reason"]) + + with patch.object(cx, "_request", side_effect=cx.AmazonError("Fixture status read failed")): + try: + _report_document(con, "inventory", {}, report_type, **window) + except cx.AmazonError: + pass + status = inventory_table_status(con) + ck("a failed status read retains the created ID and unknown vendor state", + status["reports"][0]["reportId"] == "fixture-inventory-terminal" + and status["reports"][0]["processingStatus"] is None) + + with patch.object(cx, "report_document", side_effect=cx.AmazonThrottled("Fixture throttle")): + try: + _report_document(con, "inventory", {}, report_type, **window) + except cx.AmazonThrottled: + pass + status = inventory_table_status(con) + ck("creation throttle retains an unknown ID and an actionable cause", + status["code"] == "first_load_pending" and status["reports"][0]["reportId"] is None + and status["reports"][0]["state"] == "throttled" and "limited" in status["reason"]) + + with patch.object(cx, "report_document", return_value=(header, meta)): + _report_document(con, "inventory", {}, report_type, **window) + long_meta = {**meta, "reportId": "fixture-" + "r" * 160, + "reportDocumentId": "fixture-" + "d" * 160} + with patch.object(cx, "report_document", return_value=(header, long_meta)): + _report_document(con, "inventory", {}, report_type, **window) + ck("request metadata survives beyond the short-marker limit", + json.loads(_read_state(con, entity_key(INVENTORY_REQUEST_KEY))["detail"])["reportDocumentId"] + == long_meta["reportDocumentId"]) + with patch.object(cx, "report_document", return_value=(header, meta)): + _report_document(con, "inventory", {}, report_type, **window) + scoped = _account_prefix("fixture-seller") + con.execute("INSERT INTO _sync_state SELECT ? || entity, phase, last_id, cursor_wd, rows, updated_at " + "FROM _sync_state", [scoped]) + _record_state(con, "amzn_pending_report_stale", "pending", detail=json.dumps({ + "table": "inventory", "reportId": "fixture-stale-unscoped", "processingStatus": "IN_PROGRESS"})) + status = inventory_table_status(con) + ck("scoped proof ignores stale unscoped pending requests", status["code"] == "no_source_rows" + and all(row["reportId"] != "fixture-stale-unscoped" for row in status["reports"])) + _record_state(con, scoped + "amzn_pending_report_previous_day", "pending", detail=json.dumps({ + "table": "inventory", "reportId": "fixture-previous-day", "processingStatus": "IN_PROGRESS"})) + con.execute("UPDATE _sync_state SET updated_at='2026-11-08T00:00:00Z' WHERE entity=?", + [scoped + "amzn_pending_report_previous_day"]) + ck("a completed newer request supersedes an old pending window", + inventory_table_status(con)["code"] == "no_source_rows") + _record_state(con, _account_prefix("fixture-other") + "amzn_inventory", "pending") + ck("another incomplete seller prevents a combined empty claim", inventory_table_status(con)["code"] != "no_source_rows") + con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [_account_prefix("fixture-other")]) + _record_state(con, "amzn_pending_report_other", "pending", detail=json.dumps({"table": "orders"})) + con.execute("INSERT INTO amzn_inventory (_snapshot_date, inventory_source) VALUES ('2026-11-09', 'FBA')") + status = inventory_table_status(con) + ck("nonempty inventory still returns request metadata", status["rows"] == 1 and not status["empty"] + and status["reports"][0]["reportId"] == meta["reportId"]) + kept = _restored_state_rows(_sync_state_rows(con), ["amzn_inventory"], ["amzn_inventory"]) + ck("a refused restore clears the request with its inventory proof", + not any(row[0].endswith(entity_key(INVENTORY_REQUEST_KEY)) for row in kept)) + ck("inventory diagnostics preserve orders and listings", + con.execute("SELECT COUNT(*) FROM amzn_orders").fetchone()[0] == 2 + and con.execute("SELECT COUNT(*) FROM amzn_listings").fetchone()[0] == 3) + finally: + con.close() + return len(failures) + + def snapshot_selftest(log=print, check=None): """S66-T47: real sync with in-memory storage and fixture-only vendor transports.""" from unittest.mock import patch @@ -9232,6 +10201,7 @@ def snapshot_selftest(log=print, check=None): and con.execute("SELECT ending_warehouse_balance FROM amzn_inventory " "WHERE inventory_source='FBA'").fetchall() == [(9,)]) con.close() + ck("inventory request retention and empty-source controls", lambda: inventory_status_selftest(log, check) == 0) ck("S67-T32 saved mirror: one copy, content keys, kill-safe checkpoint, full purge", lambda: mirror_selftest(log, check) == 0) return len(failures) @@ -9263,6 +10233,7 @@ def snapshot_sync_selftest(log=print, check=None): disjoint fixture SKUs. Saved-mirror restore and checkpoint integrity are separate tests. """ sync = _sync_candidate + from contextlib import ExitStack from collections import Counter from copy import deepcopy from types import SimpleNamespace @@ -9301,7 +10272,7 @@ def snapshot_sync_selftest(log=print, check=None): {"databases": {amazon_schema.DATABASE_PREFIX + t: t in selected for t in TABLES}} if key == ENABLEMENT_KEY else None)) start = Clock.instant - timedelta(days=65) - calls, generation = [], [0] + calls, generation, feedback_asins = [], [0], [] report_tables = {spec["report"]: table for table, spec in cx.REPORT_TYPES.items() if table in snapshots and spec.get("report")} order_reports = {cx.REPORT_TYPES["orders"][key] for key in ("report", "report_incremental")} @@ -9380,6 +10351,23 @@ def snapshot_sync_selftest(log=print, check=None): "items": [{"msku": f"S{i}", "quantity": generation[0] + 2} for i in range(size("inbound_shipments"))]}] + def feedback_records(_fields, asins, *, operation="getItemReviewTopics"): + if operation == "getItemReviewTrends": + calls.append(("review_trends", {})) + return [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "reviewTrends": { + "positiveTopics": [{"topic": f"Fixture trend {generation[0]}", "trendMetrics": [{ + "dateRange": {"startDate": "2026-10-01", "endDate": "2026-10-31"}, + "asinMetrics": {"occurrencePercentage": 12.5}}]}], "negativeTopics": None}} + for asin in asins] + calls.append(("review_topics", {})) + feedback_asins.append(tuple(asins)) + if any(not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in asins): + raise AssertionError("Snapshot fixture supplied an invalid child ASIN") + return [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": { + "positiveTopics": [{"topic": f"Fixture topic {generation[0]}", "asinMetrics": { + "numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5, + "starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins] + def snapshot_counts(requests): return Counter(table for table, _kwargs in requests if table in snapshots) @@ -9401,25 +10389,30 @@ def snapshot_sync_selftest(log=print, check=None): "marketplaceId": cx.US_MARKETPLACE_ID, "start": "2026-08-01T00:00:00Z", "end": "2026-08-31T23:59:59Z"} try: - with patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_a: (con.cursor(), None)), \ - patch.object(module, "credential", return_value={"fixture": True}), \ - patch.object(module, "restore_mirror", return_value={"problems": []}), \ - patch.object(module, "persist_mirror"), \ - patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ - patch.object(module, "_sync_event"), \ - patch.object(module, "DEFAULT_BACKFILL_DAYS", 65), \ - patch.object(module, "datetime", Clock), \ - patch.object(cx, "report_document", side_effect=document), \ - patch.object(cx, "group_a_records", side_effect=records), \ - patch.object(cx, "transactions", side_effect=finances), \ - patch.object(module, "_fee_estimate_inputs", side_effect=fee_inputs), \ - patch.object(cx, "fee_estimates", side_effect=fee_records), \ - patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records), \ - patch.object(cx, "inbound_shipment_items", side_effect=inbound_records), \ - patch.object(cx, "replenishment_metrics", side_effect=replenishment_records), \ - patch.object(cx, "awd_inventory", return_value=[]), \ - patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")), \ - patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("offline snapshot sync fixture")): + with ExitStack() as stack: + for fixture in ( + patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_a: (con.cursor(), None)), + patch.object(module, "credential", return_value={"fixture": True}), + patch.object(module, "restore_mirror", return_value={"problems": []}), + patch.object(module, "persist_mirror"), + patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), + patch.object(module, "_sync_event"), + patch.object(module, "DEFAULT_BACKFILL_DAYS", 65), + patch.object(module, "datetime", Clock), + patch.object(cx, "report_document", side_effect=document), + patch.object(cx, "group_a_records", side_effect=records), + patch.object(cx, "transactions", side_effect=finances), + patch.object(module, "_fee_estimate_inputs", side_effect=fee_inputs), + patch.object(cx, "fee_estimates", side_effect=fee_records), + patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records), + patch.object(cx, "inbound_shipment_items", side_effect=inbound_records), + patch.object(cx, "item_review_feedback", side_effect=feedback_records), + patch.object(cx, "replenishment_metrics", side_effect=replenishment_records), + patch.object(cx, "awd_inventory", return_value=[]), + patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")), + patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("offline snapshot sync fixture")), + ): + stack.enter_context(fixture) first = sync(rt.key, tables=selected, rt=rt, first_load=True, report_budget=len(snapshots) + 2, log=lambda *_a: None) ck("first load requests every snapshot once alongside multiwindow history", lambda: @@ -9438,6 +10431,10 @@ def snapshot_sync_selftest(log=print, check=None): for table in snapshots: ck(table + " first load stores one document's rows", lambda: con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] == expected_rows(table)) + ck("review topics first load reads every current listing ASIN once", lambda: + feedback_asins == [("B000000001", "B000000002")] + and con.execute("SELECT asin, topic FROM amzn_review_topics ORDER BY asin").fetchall() + == [("B000000001", "Fixture topic 0"), ("B000000002", "Fixture topic 0")]) # Advance beyond every real refresh floor; do not weaken vendor cadence in the fixture. Clock.instant += timedelta(seconds=max(cx.REPORT_TYPES[t].get("min_interval_s", 0) @@ -9461,6 +10458,11 @@ def snapshot_sync_selftest(log=print, check=None): ck(table + " a new snapshot replaces older dates and removed rows", lambda: con.execute(f"SELECT COUNT(*), COUNT(DISTINCT _snapshot_date), MAX(_snapshot_date) " f"FROM {table_name(table)}").fetchone() == (expected_rows(table), 1, date)) + ck("review topics resync replaces removed ASINs and changed topics", lambda: + feedback_asins == [("B000000001", "B000000002"), ("B000000001",)] + and con.execute("SELECT asin, topic, asin_metrics_number_of_mentions " + "FROM amzn_review_topics ORDER BY asin").fetchall() + == [("B000000001", "Fixture topic 1", 3)]) ck("resync history totals still match the full fixture history", lambda: history_counts() == ((2, 2), (3, 6), (6, 3, 60))) # A watermark replay preserves the mirror, so this catches plain append behavior @@ -9727,7 +10729,8 @@ def mirror_selftest(log=print, check=None): _manifest_keys(json.loads(rt.blobs[manifest_path])))) # A refused restore (store unreachable at boot) must not publish its rebuild. - valid_manifest = rt.blobs[manifest_path] + valid_blobs = dict(rt.blobs) + valid_rows = dump() wipe() def unreachable(kind, path): @@ -9742,26 +10745,33 @@ def mirror_selftest(log=print, check=None): rt.uploads.clear() report = persist({"amzn_orders"}) ck("a refused restore does not publish a partial rebuild over the valid manifest", lambda: - refused["state"] == "refused" and rt.blobs[manifest_path] == valid_manifest + refused["state"] == "refused" and rt.blobs == valid_blobs and rt.uploads == [] and any("checkpoint held" in p for p in report["problems"])) _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z") report = persist({"amzn_orders"}) - ck("the refused process publishes once its own first load completes", lambda: - not report["problems"] and rt.blobs[manifest_path] != valid_manifest - and set(json.loads(rt.blobs[manifest_path])["tables"]) == {"amzn_orders"}) + ck("the refused process cannot publish after its own first load completes", lambda: + rt.blobs == valid_blobs and rt.uploads == [] + and any("checkpoint held" in p for p in report["problems"])) - # A state fault refuses the whole snapshot and keeps the existing rebuild policy. + # A corrupt state refuses publication until a new process restores valid saved bytes. wipe() corrupt = json.loads(rt.blobs[manifest_path]) corrupt["sync_state"]["rows"] += 1 rt.blobs[manifest_path] = _mirror_json(corrupt) + corrupt_blobs = dict(rt.blobs) + rt.uploads.clear() refused = restore() con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-03' AS _period_start, 'O3' AS id, 2.0 AS amount") _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z") report = persist({"amzn_orders"}) - ck("a corrupt stored state refuses whole and the rebuild may publish over it", lambda: - refused["state"] == "refused" and not report["problems"] - and json.loads(rt.blobs[manifest_path])["tables"]["amzn_orders"]["rows"] == 1) + ck("a corrupt stored state refuses whole and cannot publish its rebuild", lambda: + refused["state"] == "refused" and rt.blobs == corrupt_blobs and rt.uploads == [] + and any("checkpoint held" in p for p in report["problems"])) + rt.blobs = dict(valid_blobs) + wipe() + recovered = restore() + ck("a new process restores the valid snapshot before later checkpoints", lambda: + recovered["state"] == "restored" and not recovered["problems"] and dump() == valid_rows) # Purge reaches every version: legacy keys, named keys and a killed checkpoint's orphan. for path, data in legacy_blobs.items(): @@ -10695,6 +11705,17 @@ def group_a_selftest(log=print, check=None): "offerCount": 1, "marketplaceId": cx.US_MARKETPLACE_ID, "start": _iso(begin), "end": _iso(end)} inbound = [{"inboundPlanId": "plan-1", "planStatus": "ACTIVE", "shipmentId": "shipment-1", "status": "WORKING", "items": [{"msku": "SKU-1", "quantity": 2, "buyerEmail": "planted@example.invalid"}]}] + rt = _FixtureSyncRuntime("group-a-document-fixture", GROUP_A_TABLES) + con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") + con.execute("INSERT INTO amzn_listings VALUES ('B000000001')") + feedback = [{"asin": "B000000001", "marketplaceId": cx.US_MARKETPLACE_ID, "reviewTrends": { + "positiveTopics": [{"topic": "Fixture durability", "trendMetrics": [{ + "dateRange": {"startDate": "2026-10-01", "endDate": "2026-10-31"}, + "asinMetrics": {"occurrencePercentage": 12.5}}]}], "negativeTopics": None}, "topics": { + "positiveTopics": [{"topic": "Fixture durability", "asinMetrics": {"numberOfMentions": 4, + "occurrencePercentage": 12.5, "starRatingImpact": 0.2}}], + "negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2, + "occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}] try: with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \ @@ -10704,13 +11725,14 @@ def group_a_selftest(log=print, check=None): patch.object(cx, "outbound_fulfillment_orders", side_effect=lambda fields: records(fields, "listAllFulfillmentOrders")), \ patch.object(cx, "inbound_shipment_items", return_value=inbound), \ patch.object(cx, "replenishment_metrics", return_value=replenishment), \ + patch.object(cx, "item_review_feedback", return_value=feedback), \ patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \ patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)): for table in GROUP_A_TABLES: spec = cx.REPORT_TYPES[table] window = (None, None) if spec.get("snapshot") else (begin, end) def ingest(): - return _sync_margin_window(con, table, {}, *window, max_wait=1, log=lambda *_: None) + return _sync_margin_window(con, table, {}, *window, max_wait=1, log=lambda *_: None, rt=rt) ck(table + " without evidence is unproven", _validate_margin(con, table)["state"] == "unproven") count = ingest() # A real broken mirror must fail before the matching document count passes. @@ -11091,7 +12113,8 @@ def catalog_parent_selftest(log=print, check=None): with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")), \ patch.object(cx, "group_a_records", side_effect=records): - count = _sync_margin_window(con, "catalog_items", {}, None, None, max_wait=1, log=lambda *_: None) + count = _sync_margin_window(con, "catalog_items", {}, None, None, max_wait=1, log=lambda *_: None, + rt=_FixtureSyncRuntime("catalog-parent-fixture", ("catalog_items",))) con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) " "VALUES (?, 'B0STALEPAR', 'relationships', '2000-01-01')", [child]) latest = con.execute("SELECT MAX(_snapshot_date) FROM amzn_catalog_items").fetchone()[0] diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index d4fdf6bcb84cc32649f04a5031effa19065be5e5..37f9f271821fb5ab343e7e0be932ae8eca402ee8 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -928,6 +928,15 @@ def _mirror_measurement(started): "database_node_limit_bytes": 2 * 1024 ** 3} +def odoo_mirror_restored(tenant_key, rt): + """Check this process's successful manifest restore for the exact tenant and mirror path.""" + store = _odoo_mirror_store() + cache_key = store._mirror_cache_key(tenant_key, rt) + with store.lock: + result = store.cache.get(cache_key, {}) + return result.get("state") in ("restored", "local") and bool(result.get("manifest")) + + def restore_odoo_mirror(tenant_key, rt, log=print): """Restore only this tenant's saved Odoo tables and preserve a refusal until restart.""" if versioned_mode() and getattr(_SYNC_TLS, "candidate", None) is None: @@ -3195,6 +3204,8 @@ def ensure_seed(): import shutil rt = _resident_odoo_runtime() if rt is not None: + if odoo_mirror_restored(rt.key, rt): + return False result = restore_odoo_mirror(rt.key, rt) if result["state"] != "absent": return False diff --git a/platform/harness/filter_eval.py b/platform/harness/filter_eval.py index 987ed348e0f9ee2d74153fd58661328a0d0c20ee..d3999697239c242710987898a3ca5cd6cae50640 100644 --- a/platform/harness/filter_eval.py +++ b/platform/harness/filter_eval.py @@ -1,823 +1,6 @@ -"""harness/filter_eval.py — the filter tree evaluated against ROWS, in Python (wave 15, C-EVAL). +"""The filter evaluator lives in `core/filter_eval.py`, and this name stays for its callers.""" +import sys -THE THIRD ENGINE, and the one with a different job. There are now three readers of the same -filter tree and it is worth being precise about why: +from core import filter_eval as _impl - * `customer-grid/useVisibleRows.ts` — the CLIENT, over in-memory rows. The reference. - * `harness/filter_sql.py` — compiles the tree to SQL for the DuckDB semantic tables. - * **this module** — evaluates the tree against wire-shaped row dicts in the - API/host process, where there is no SQL and the rows are - already in hand. - -It exists because a PERMANENT (permission) filter has to be applied server-side, to rows the -server is about to send, on a surface whose columns are not a DuckDB table. Neither existing -engine can do that: the client one is TypeScript, and the SQL one needs a table. - -⛔ TWO ENTRY POINTS, OPPOSITE DEFAULTS, AND THE DIFFERENCE IS THE WHOLE POINT. - - matches(...) DISPLAY semantics. A faithful mirror of the client, including the places - where the client IGNORES a leaf. Refereed against the client by - `aios-web/verify_filter_parity.py` — if this disagrees with the browser about - one row, a saved view means two different things in two places. - - permits(...) PERMISSION semantics. Same evaluator, `strict=True`: anything the display - engine would IGNORE, this DENIES. - -Why they cannot be the same function (C-EVAL amendment 1, and the reason this module has a -docstring this long). The client engine is TRI-STATE — `True` / `False` / `None`, where `None` -means "this leaf is INACTIVE, parent ignores it entirely". A leaf naming a column that does not -exist returns `None` (`useVisibleRows.ts:318`), and a group whose children are all inactive is -itself inactive (:273). For a DISPLAY filter that is exactly right: a half-built condition must -not zero the grid, and a stale column reference must not either. - -For a PERMISSION filter it is a catastrophe. `agent is Tara` is the only thing standing between -a sales rep and the whole book; rename or delete the `agent` field and that leaf becomes -`None` — ignored — and the rep silently receives every customer. The wall would be one field -rename away from open, and nothing would go red. So `permits()` inverts every ignore into a -denial: an unanswerable permission filter is an OUTAGE, not a licence. - -The asymmetry is a PARAMETER (`strict`) rather than a second implementation, deliberately: a -mirror must copy the original's asymmetries, and the only way to be sure it still does next -month is for both behaviours to come out of the same walk. - -⚠ WHAT THIS MODULE DOES NOT RE-IMPLEMENT. Activeness (`is_rule_active`), the JS-`Number` -coercion (`to_num`), the op vocabularies, cohort-id parsing and the date-window/anchor resolvers -all already exist in `filter_sql.py` and `windows.py`, and are already held against the client by -`verify_filter_engine.py` (525 checks) and `verify_windows.py` (123). Importing them is not -laziness — a second copy of "what counts as an active rule" is precisely the drift those gates -exist to catch, and it would be a copy the gates do not cover. -""" -import json -import math -from collections import namedtuple - -from harness.filter_sql import ( # noqa: F401 (COHORT_OPS re-exported for callers) - COHORT_FIELD, COHORT_OPS, COHORT_OP_ALIASES, FILTER_OPS, MULTI_VALUE_OPS, RANK_OPS, - is_rule_active, parse_cohort_ids, parse_filter_values, to_num, - _resolve_anchor, _window_bounds, -) - -#: The CLIENT's numeric family, which is WIDER than `filter_sql.NUMERIC_TYPES`. `rating` (an -#: integer 1..max) and `formula` (computed in the browser and injected at the field's key) compare -#: as numbers here because this engine sees the same assembled row the client does. They are -#: absent from the SQL port for a good reason — they are client-only strata that can never reach a -#: DuckDB table — so the two lists differing is correct, not drift. Mirrors `isNumericType`. -NUMERIC_TYPES = frozenset({'currency', 'int', 'pct', 'rating', 'formula', 'autonumber'}) - -#: `created_time` is a DATETIME rendered read-only, and it filters as a DATE (truncated to the -#: day below). Mirrors `isDateType`. -DATE_TYPES = frozenset({'date', 'created_time', 'last_modified_time'}) - -#: The date ops that actually compare against an ANCHOR. Everything else falls to the -#: active-and-always-true default WITHOUT resolving one — ordering matters, see `_match_date`. -DATE_CMP_OPS = frozenset({'eq', 'neq', 'lt', 'lte', 'gt', 'gte'}) - - -class EvalCtx: - """Everything the engine needs besides the row and the columns. - - All four members share one property — each is an answer a single ROW cannot compute — and - therefore share one failure mode: absent, the condition matches NOTHING rather than - everything. That is the widening sin the cohort/measure/rank leaves are each written to - avoid, and it is why none of these defaults to a permissive value. - """ - - __slots__ = ('cohort_sets', 'measure_sets', 'rank_sets', 'today') - - def __init__(self, cohort_sets=None, measure_sets=None, rank_sets=None, today=None): - #: {cohort_id: set(pid)} — the cohorts THIS reader may see. - self.cohort_sets = cohort_sets or {} - #: {rule_id: set(pid)} — resolved measure conditions (CG-8). - self.measure_sets = measure_sets or {} - #: {id(leaf) or leaf-key: set(pid)} — resolved rank slices (C-OPS). - self.rank_sets = rank_sets or {} - #: The TENANT's today, ISO. Never the server clock at call time — a date filter that - #: means something different per process is not a filter. - self.today = today - - -_EMPTY_CTX = EvalCtx() - - -def _is_group(node): - return isinstance(node, dict) and isinstance(node.get('children'), list) - - -def _is_cohort(node): - return isinstance(node, dict) and node.get('colId') == COHORT_FIELD - - -def _is_measure(node): - """Mirrors `isMeasureRule`: a leaf carrying a `window` IS a measure condition.""" - return isinstance(node, dict) and node.get('window') is not None - - -def _rhs_col(rhs): - """Mirrors `rhsColId`: a STATISTIC names no column.""" - if not isinstance(rhs, dict) or rhs.get('kind') == 'stat': - return None - return rhs.get('colId') - - -def _as_text(v): - """The client's `String(raw ?? "").toLowerCase()`. `None` becomes '' (never 'None').""" - return ('' if v is None else str(v)).lower() - - -def _as_day(v): - """Day-granularity truncation. A `created_time` cell is a DATETIME and every comparand in - this vocabulary is a DATE — lexically the longer string sorts AFTER its own day, so without - this `on or before ` would silently exclude today's rows. Identity for a pure date.""" - return ('' if v is None else str(v))[:10] - - -def _cmp_num(op, a, b): - """`default: True` is the documented ACTIVE-and-always-true case — an operator that exists - but does not apply to this type (`contains` on a number) narrows nothing and asks nothing. - An operator in NO vocabulary never reaches here; `_eval` refuses it first.""" - if op == 'gt': - return a > b - if op == 'gte': - return a >= b - if op == 'lt': - return a < b - if op == 'lte': - return a <= b - if op == 'eq': - return a == b - if op == 'neq': - return a != b - return True - - -def _cmp_str(op, a, v): - if op == 'eq': - return a == v - if op == 'neq': - return a != v - if op == 'lt': - return a < v - if op == 'lte': - return a <= v - if op == 'gt': - return a > v - if op == 'gte': - return a >= v - return True - - -def _cmp_text(op, a, v): - if op == 'contains': - return v in a - if op == 'doesNotContain': - return v not in a - if op == 'eq': - return a == v - if op == 'neq': - return a != v - return True - - -def _as_choice_text(raw): - """The client's ``String(raw ?? "")`` over a CHOICE-SET cell, lowercased. - - ⚠ NOT ``_as_text``. A multi-select value can still arrive from the wire as a LIST (see - ``_is_blank``, which already carries that case), and Python's ``str(['a', 'b'])`` renders - ``"['a', 'b']"`` where JS's ``String(["a","b"])`` renders ``"a,b"`` — the comma-joined form - the storage layer uses and the one the membership test below has to see. - """ - if isinstance(raw, (list, tuple)): - return ','.join('' if x is None else str(x) for x in raw).lower() - return _as_text(raw) - - -def _cell_members(raw): - """The MEMBERS of one cell, mirroring `useVisibleRows.cellMembers`. - - A multi-select cell is a comma-joined set; a single-valued cell is a one-member set, so - ``hasAnyOf`` over a select column means exactly what ``isAnyOf`` does there. A BLANK cell has - NO members, which is the blank decision the three engines pin as a lock-step case. - """ - return [m for m in (part.strip() for part in _as_choice_text(raw).split(',')) if m] - - -def _match_multi_value(op, raw, values): - """⭐ W51-T07 — the five multi-value operators, COMPOSED from `_cmp_text`. - - ⛔ THIS FUNCTION IS THE POINT OF THE WHOLE AMENDMENT. This module imports ``FILTER_OPS`` from - ``filter_sql``, and its only vocabulary guard is ``op not in FILTER_OPS -> False``. The moment - the five ops joined that set, the guard began PASSING them straight into ``_cmp_text``, whose - tail returns True for an operator it does not know — so every row would have matched, in the - module that is BOTH the server-side display mirror (``matches``) and the PERMISSION wall - (``permits``). A bare ``return False`` would have been no better: the server would then - disagree with the browser about real rows, which is the drift this module exists to prevent. - - ``isAnyOf`` is the OR of ``_cmp_text('eq', ...)`` over the values, so it is the union of the - equivalent single-value leaves by construction rather than by testing. - """ - want = [v.strip().lower() for v in values] - # ⛔ NO VALUES = UNANSWERABLE = MATCHES NOTHING, uniformly. A SECOND layer: `_eval` already - # calls such a leaf inactive via `is_rule_active`. Written anyway because the vacuous cases - # go the WIDENING way — `all(... for v in [])` is True, so `hasAllOf []` would match every - # row, on a permission wall. `filter_sql` answers `1=0` for the identical input. - if not want: - return False - if op in ('isAnyOf', 'isNoneOf'): - a = _as_choice_text(raw) - hit = any(_cmp_text('eq', a, v) for v in want) - return hit if op == 'isAnyOf' else not hit - members = _cell_members(raw) - if op == 'hasAllOf': - return all(v in members for v in want) - hit = any(v in members for v in want) - return hit if op == 'hasAnyOf' else not hit - - -def _customer_tag_rules(nodes): - """Find every Customer type operand, including operands inside OR groups.""" - for node in nodes: - if _is_group(node): - yield from _customer_tag_rules(node['children']) - elif isinstance(node, dict) and node.get('colId') == 'customer_type': - yield node - - -def _match_customer_tags(rule, raw): - """Test exact JSON string-list membership, or return None when the rule is uncomputable.""" - if not isinstance(raw, str) or rule.get('rhs') is not None or _is_measure(rule): - return None - try: - tags = json.loads(raw) - except (ValueError, RecursionError): - return None - if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags): - return None - op = rule.get('op') - if op in ('isEmpty', 'isNotEmpty'): - return (not tags) if op == 'isEmpty' else bool(tags) - if op not in ('eq', 'neq', 'contains', 'doesNotContain') and op not in MULTI_VALUE_OPS: - return None - values = rule.get('values') if op in MULTI_VALUE_OPS else [rule.get('value')] - if (not isinstance(values, list) or not values - or any(not isinstance(value, str) or not value for value in values)): - return None - hits = [value in tags for value in values] - hit = all(hits) if op == 'hasAllOf' else any(hits) - return not hit if op in ('neq', 'doesNotContain', 'isNoneOf', 'hasNoneOf') else hit - - -def _is_blank(raw): - """The condition-builder blank contract: absent, empty choice set, or ``'(none)'``. - - Connector grids intentionally render a missing attribute as the visible ``'(none)'`` sentinel, - while multi-select wire values can arrive as a list before their comma-joined storage form. - Both are absence for ``isEmpty``; numeric 0 and false remain real values. - """ - if raw is None: - return True - if isinstance(raw, (list, tuple, set, frozenset)): - return not raw or all(_is_blank(value) for value in raw) - return isinstance(raw, str) and raw.strip() in ('', '(none)') - - -def _match_numeric(rule, raw, other, has_rhs): - a = to_num(raw) - op = rule.get('op') - if has_rhs: - return _cmp_num(op, a, to_num(other)) - if op == 'between': - v, v2 = rule.get('value'), rule.get('value2') - if v == '' or v2 is None or v2 == '': - return True # both bounds required - lo, hi = to_num(v), to_num(v2) - if lo > hi: - lo, hi = hi, lo - return lo <= a <= hi - if rule.get('value') == '': - return True # inactive until a value is typed - return _cmp_num(op, a, to_num(rule.get('value'))) - - -def _match_date(rule, raw, other, has_rhs, today): - a = _as_day(raw) - op = rule.get('op') - - # "is within [the past month]" — a RANGE test, and the only date op reading `dateWindow`. - if op == 'within': - if not today: - return False - bounds = _window_bounds(rule.get('dateWindow'), today) - if not bounds: - return False # unresolvable range -> matches nothing - if a == '': - # A blank date is not "within" any range. Without this, '' >= a null lower bound is - # vacuously true and every never-ordered customer lands inside every window. - return False - lo, hi = bounds - return (lo is None or a >= lo) and (hi is None or a <= hi) - - if has_rhs: - return _cmp_str(op, a, _as_day(other)) - if op == 'between': - v, v2 = rule.get('value'), rule.get('value2') - if v == '' or v2 is None or v2 == '': - return True - lo, hi = (v, v2) if v < v2 else (v2, v) - return lo <= a <= hi - - # An op that does not apply to a date (`contains`) is ACTIVE-and-always-true, decided BEFORE - # any anchor is resolved — there is no date in that question to resolve. Ordering this after - # the anchor would turn `contains` on a date from always-true into matches-nothing, which is - # a different answer under `or`. - if op not in DATE_CMP_OPS: - return True - if not rule.get('dateMode') and rule.get('value') == '': - return True # inactive until a value is typed - mode = rule.get('dateMode') or 'exact' - # `today` is required by a RELATIVE anchor and by nothing else, which is what keeps every - # pre-anchor saved view working in a payload that carries no today. - if mode != 'exact' and not today: - return False - v = _resolve_anchor(mode, rule.get('value'), today or '1970-01-01') - if v is None: - # Unresolvable (junk date, missing N) -> matches nothing. The honest answer to a - # question with no date in it. - return False - return _cmp_str(op, a, v) - - -def _match_filter(rule, row, columns, today): - """One LEAF against one row, dispatched on the field's declared type. - - `isEmpty`/`isNotEmpty` are handled FIRST and type-independently: they must never fall - through to the "blank value -> inactive" guards below, which would turn them into - always-true no-ops. - """ - field = columns.get(rule.get('colId')) or {} - ftype = field.get('type') - raw = row.get(rule.get('colId')) - op = rule.get('op') - - if op == 'isEmpty': - return _is_blank(raw) - if op == 'isNotEmpty': - return not _is_blank(raw) - - # A RANK op can never be answered from one row. `_eval` routes rank leaves to their resolved - # set and one should never arrive here; this is the SECOND layer, and it exists because of - # what the first layer's absence would do — `_cmp_num`'s `default: True` would make `topN` - # match EVERY row, a filter that widens under a count that agrees with it. - if op in RANK_OPS: - return False - - # ⭐ W51-T07 — the MULTI-VALUE family, ON THE OP and before the type dispatch. Every branch - # below ends in a blank-value short-circuit and these leaves carry a blank `value`; one - # fall-through and a permission wall reading `Agent is any of [...]` admits every row. - if op in MULTI_VALUE_OPS: - if ftype in DATE_TYPES: - return False - values = parse_filter_values(rule.get('values')) - if ftype in NUMERIC_TYPES: - if op not in ('isAnyOf', 'isNoneOf') or not values: - return False - number = to_num(raw) - hit = any(_cmp_num('eq', number, to_num(value)) for value in values) - return hit if op == 'isAnyOf' else not hit - return _match_multi_value(op, raw, values) - - rhs = rule.get('rhs') - has_rhs = rhs is not None - other = row.get(_rhs_col(rhs) or '') if has_rhs else None - - if ftype in NUMERIC_TYPES: - return _match_numeric(rule, raw, other, has_rhs) - if ftype in DATE_TYPES: - return _match_date(rule, raw, other, has_rhs, today) - - a = _as_text(raw) - if has_rhs: - return _cmp_text(op, a, _as_text(other)) - if rule.get('value') == '': - return True - return _cmp_text(op, a, _as_text(rule.get('value'))) - - -def _eval(node, row, columns, ctx, strict): - """One node against one row. `True` / `False` / `None`, where `None` = INACTIVE. - - `strict` is the permission mode: every `None` this function would return becomes `False` - instead. Read the two behaviours side by side rather than trusting either alone — the - difference between them IS the security property (see the module docstring). - """ - ignore = False if strict else None - - if _is_group(node): - saw_active = False - all_true = True - any_true = False - for child in node['children']: - v = _eval(child, row, columns, ctx, strict) - if v is None: - continue # inactive -> ignored entirely - saw_active = True - all_true = all_true and v - any_true = any_true or v - if not saw_active: - # A group with no active children is itself inactive. In strict mode an empty or - # wholly-unanswerable permission group DENIES: a wall that evaluates to "nothing to - # say" must not mean "let them through". - return ignore - return any_true if node.get('conj') == 'or' else all_true - - if not isinstance(node, dict): - return ignore - - # A COHORT leaf, BEFORE the column lookup: `__cohort__` is not a column, so the lookup below - # would call it INACTIVE — and inactive means every row on screen under a count nobody doubts. - if _is_cohort(node): - if not is_rule_active(node, columns): - return ignore - ids = parse_cohort_ids(node.get('value')) - sets = [ctx.cohort_sets.get(cid) for cid in ids] - # ANY named cohort we were given no membership for makes the WHOLE condition - # unanswerable, and unanswerable matches nothing — for `is none of` as well. Answering - # "everything" to `is none of [a cohort we cannot see]` is the widening sin wearing a - # plausible face, and PARTIAL evaluation is worse: `is none of [A, B]` quietly becoming - # `is none of [A]` would show every row in B under a count nobody would doubt. - if any(s is None for s in sets): - return False - pid = row.get('pid') - op = node.get('op') - op = op if op in COHORT_OPS else COHORT_OP_ALIASES.get(op, 'anyOf') - if op == 'allOf': - return all(pid in s for s in sets) - in_any = any(pid in s for s in sets) - return (not in_any) if op == 'noneOf' else in_any - - # A MEASURE leaf, also before the column lookup and for the same reason: a measure's colId - # ("revenue") is not a column of this table. An ACTIVE rule whose set has not been resolved - # is PENDING and matches NOTHING — never everything. - if _is_measure(node): - if not is_rule_active(node, columns): - return ignore - s = ctx.measure_sets.get(node.get('id')) if node.get('id') else None - if s is None: - return False - return row.get('pid') in s - - if strict and node.get('colId') == 'customer_type': - return ('customer_type' in columns - and _match_customer_tags(node, row.get('customer_type')) is True) - - # ── a column leaf ──────────────────────────────────────────────────────────────────────── - if node.get('colId') not in columns: - # THE LEAF THIS WHOLE MODULE'S `strict` FLAG EXISTS FOR. Display: ignore it. Permission: - # deny — see the module docstring's field-rename walkthrough. - return ignore - if node.get('rhs') is not None and _rhs_col(node['rhs']) not in columns: - # An rhs naming a column this table does not have is ignored by the SAME rule as an - # unknown LEFT column. Treating one side as "ignore" and the other as "match nothing" - # would make `a > deleted` behave differently from `deleted > a`. - return ignore - if not is_rule_active(node, columns): - return ignore - - # A RANK leaf is a MEMBERSHIP test, like a cohort or a measure: the set was computed in a - # prior pass over the other rows. No set = REFUSE, never "ignore" and never "true" — both of - # those SHOW rows. - if node.get('op') in RANK_OPS: - # ``_rank_key`` gained an explicit scope in Wave 43. Accept the old four-part key - # for callers which resolved their own set before that addition; a missing answer is - # still a refusal, never a permissive fallback. - s = ctx.rank_sets.get(_rank_key(node)) - if s is None: - s = ctx.rank_sets.get(_legacy_rank_key(node)) - return (row.get('pid') in s) if s is not None else False - - # An operator in NO vocabulary at all. The comparators end in `default: True`, right for an - # operator that exists but does not apply to this type; NOT right for one that exists - # nowhere (`inTertile` out of a stale saved view), which would match every row. - if node.get('op') not in FILTER_OPS: - return False - - return _match_filter(node, row, columns, ctx.today) - - -def _rank_key(node): - """The client keys `rankSets` by leaf IDENTITY (a Map keyed on the node object). Python - dicts cannot key on a dict, so the caller that resolves rank sets and this evaluator agree - on a derived key. Kept in ONE place so the two cannot drift apart.""" - return (_legacy_rank_key(node) + (_rank_scope_token(node),)) - - -def _legacy_rank_key(node): - """The Wave-42 four-part resolver key, retained for supplied membership sets.""" - return (node.get('id') or '', node.get('colId') or '', node.get('op') or '', - str(node.get('value') if node.get('value') is not None else '')) - - -def _rank_scope_token(node): - """A stable, serialisable identity for one rank leaf's scope configuration. - - ``rankScope`` is deliberately optional: old views keep their global meaning. A grouped - rank may name its group field explicitly (``{mode:'group', field:'agent'}``) or defer to - the view's ``groupBy`` through the concise ``'group'`` shape. The resolver below refuses - the latter when no group field was supplied, rather than silently widening it to global. - """ - raw = node.get('rankScope') if isinstance(node, dict) else None - if isinstance(raw, dict): - mode = raw.get('mode', raw.get('scope')) - field = raw.get('field', raw.get('groupBy')) - if mode == 'group': - return 'group:' + (str(field).strip()[:80] if field is not None else '') - return 'global' - return 'group:' if raw == 'group' else 'global' - - -def _rank_scope(node, group_by): - """``('global' | 'group', field | None)`` for a cleaned or directly-posted rank leaf.""" - raw = node.get('rankScope') if isinstance(node, dict) else None - if raw == 'group': - field = group_by - elif isinstance(raw, dict) and raw.get('mode', raw.get('scope')) == 'group': - field = raw.get('field', raw.get('groupBy', group_by)) - else: - return 'global', None - field = str(field or '').strip()[:80] - return ('group', field or None) - - -def _rank_leaves(nodes, columns): - """ACTIVE rank leaves in tree order; measure leaves are not rank questions.""" - out = [] - for node in nodes or (): - if _is_group(node): - out.extend(_rank_leaves(node.get('children'), columns)) - elif (isinstance(node, dict) and node.get('op') in RANK_OPS - and not _is_measure(node) and is_rule_active(node, columns)): - out.append(node) - return out - - -def _strip_rank_leaves(nodes): - """The residue that defines the ranking domain (the client performs the same pass).""" - out = [] - for node in nodes or (): - if _is_group(node): - children = _strip_rank_leaves(node.get('children')) - if children: - out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', - 'children': children}) - elif not (isinstance(node, dict) and node.get('op') in RANK_OPS - and not _is_measure(node)): - out.append(node) - return out - - -def _rank_number(raw): - """A rank value, where blank/unparseable is ABSENT rather than numeric zero.""" - if raw is None or raw == '': - return None - if isinstance(raw, bool): - return float(raw) # JavaScript ``Number(true)`` parity. - try: - value = float(raw) - except (TypeError, ValueError): - return None - return value if math.isfinite(value) else None - - -def _rank_bound(value, lo, hi): - """The exact whole-number bounds the rank builder writes; junk remains unanswerable.""" - try: - number = float(value) - except (TypeError, ValueError): - return None - if not math.isfinite(number) or int(number) != number: - return None - number = int(number) - return number if lo <= number <= hi else None - - -def _pid_order(pid): - """Stable ascending pid order even for mixed legacy string/integer wire ids.""" - try: - return 0, int(pid) - except (TypeError, ValueError): - return 1, str(pid) - - -def _group_value(row, key): - """A hashable, deterministic grouping value; list/dict cells remain one visible group.""" - value = row.get(key) - if isinstance(value, (list, dict, set)): - return repr(value) - try: - hash(value) - except TypeError: - return repr(value) - return value - - -RankResolution = namedtuple('RankResolution', 'sets unanswerable') - - -def resolve_rank_sets(tree, rows, fields, ctx=None, group_by=None): - """Resolve top/bottom/percentile/average slices over the server's filtered row domain. - - Rank filters cannot be compiled as a row-level SQL ``WHERE`` clause, but they *can* and must - be answered before an in-memory API/host payload is filtered. The residue (all non-rank - conditions) defines the domain, ties are selected by ascending ``pid`` in both directions, - and ``rankScope: 'group'`` independently ranks every ``group_by`` bucket. A grouped request - without an actual group key is unanswerable and gets an empty membership set; it never falls - back to a global rank. - """ - columns = _columns_map(fields) - nodes, conj = tree_parts(tree) - leaves = _rank_leaves(nodes, columns) - if not leaves: - return RankResolution({}, 0) - - base_ctx = ctx or _EMPTY_CTX - residue = _strip_rank_leaves(nodes) - residue_tree = {'nodes': residue, 'conj': conj} - domain = [row for row in (rows or ()) - if matches(residue_tree, row, columns, base_ctx)] - sets, unanswerable = {}, 0 - - for leaf in leaves: - key = _rank_key(leaf) - field = columns.get(leaf.get('colId')) or {} - if field.get('type') not in NUMERIC_TYPES: - sets[key] = set() - unanswerable += 1 - continue - scope, scoped_by = _rank_scope(leaf, group_by) - if scope == 'group' and scoped_by not in columns: - sets[key] = set() - unanswerable += 1 - continue - groups = {} - for row in domain: - group_key = _group_value(row, scoped_by) if scope == 'group' else None - groups.setdefault(group_key, []).append(row) - - selected, answered = set(), False - for group_rows in groups.values(): - ranked = [] - for row in group_rows: - value = _rank_number(row.get(leaf.get('colId'))) - if value is not None: - ranked.append((value, row.get('pid'))) - if not ranked: - continue - answered = True - asc = sorted(ranked, key=lambda item: (item[0], _pid_order(item[1]))) - desc = sorted(ranked, key=lambda item: (-item[0], _pid_order(item[1]))) - count = len(asc) - op = leaf.get('op') - keep = None - if op == 'topN': - bound = _rank_bound(leaf.get('value'), 1, 10000) - keep = desc[:bound] if bound is not None else None - elif op == 'bottomN': - bound = _rank_bound(leaf.get('value'), 1, 10000) - keep = asc[:bound] if bound is not None else None - elif op in ('inTopPct', 'inBottomPct'): - pct = _rank_bound(leaf.get('value'), 1, 100) - take = min(count, max(1, math.ceil(count * pct / 100))) if pct else None - order = desc if op == 'inTopPct' else asc - keep = order[:take] if take is not None else None - elif op in ('aboveAvg', 'belowAvg'): - mean = sum(value for value, _ in asc) / count - keep = [(value, pid) for value, pid in asc - if (value > mean if op == 'aboveAvg' else value < mean)] - elif op in ('inQuartile', 'inDecile'): - buckets = 4 if op == 'inQuartile' else 10 - wanted = _rank_bound(leaf.get('value'), 1, buckets) - if wanted is not None and count >= buckets: - keep = [item for index, item in enumerate(asc) - if min(buckets, (index * buckets) // count + 1) == wanted] - if keep is None: - selected = set() - answered = False - break - selected.update(pid for _, pid in keep) - sets[key] = selected - if domain and not answered: - unanswerable += 1 - return RankResolution(sets, unanswerable) - - -def _columns_map(fields): - """`[{key,type,…}]` -> `{key: field}`. Accepts a mapping unchanged, so callers holding - either shape need no adapter.""" - if isinstance(fields, dict): - return fields - out = {} - for f in fields or (): - if isinstance(f, dict) and f.get('key'): - out[f['key']] = f - return out - - -def evaluate(tree, row, fields, ctx=None): - """The TRI-STATE the client actually has: `True` / `False` / `None`, where `None` means the - tree is INACTIVE — it asks nothing, so it narrows nothing. - - Exposed because collapsing `None` into `True` (which `matches` does, correctly, for its own - purpose) destroys a distinction some callers must keep. The `memberPids` union is the one - that bit: the client's rule is `members.has(pid) || matchFilterTree(…)` and that union sits - INSIDE `if (filters.length)`, so an INACTIVE tree must show every row and ignore the member - list — whereas an ACTIVE tree that merely fails must still admit hand-picked members. A - caller holding only `matches` cannot tell those two apart and will narrow an inactive view - down to its hand-picked rows. (`filter_sql.compile_filter_tree` keeps the same distinction - by returning None rather than a predicate; this is the row-engine's equivalent.) - - Activeness is ROW-INDEPENDENT — every `None` here comes from `is_rule_active` or an empty - group, none of which reads the row — so a caller may compute it once per tree if it wishes. - """ - columns = _columns_map(fields) - nodes, conj = tree_parts(tree) - return _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, False) - - -def matches(tree, row, fields, ctx=None): - """DISPLAY semantics — does `row` survive this filter tree? Mirrors the client exactly. - - A wholly INACTIVE tree returns True: no narrowing, every row shows. That is what the browser - does with a half-built condition, and this function's whole job is to agree with the browser. - Use `evaluate()` instead when you need to tell "inactive" from "matched" (see its docstring). - """ - v = evaluate(tree, row, fields, ctx) - return True if v is None else bool(v) - - -def visible_pids(tree, rows, fields, ctx=None, member_pids=None, group_by=None): - """The client's whole row pipeline for one tree, member union included — the shape a caller - should reach for instead of assembling `matches` in a loop. - - Mirrors `useVisibleRows`: an INACTIVE tree yields every row (and IGNORES `member_pids`, - because the union lives inside `if (filters.length)`); an ACTIVE tree yields the rows it - matches UNION the hand-picked members. - """ - ctx = ctx or _EMPTY_CTX - # This is the whole-payload entry point, so it is the one place a server can answer a rank - # filter without trusting a browser to have done a prior pass. Preserve supplied cohort / - # measure answers, but replace rank memberships with this payload's authoritative answer. - # Direct ``evaluate`` intentionally stays a one-row primitive and therefore stays - # fail-closed when a caller has not supplied a membership set. - resolved = resolve_rank_sets(tree, rows, fields, ctx, group_by=group_by) - if resolved.sets: - rank_sets = dict(ctx.rank_sets) - rank_sets.update(resolved.sets) - ctx = EvalCtx(ctx.cohort_sets, ctx.measure_sets, rank_sets, ctx.today) - members = set(member_pids or ()) - out, active = [], False - verdicts = [] - for r in rows: - v = evaluate(tree, r, fields, ctx) - if v is not None: - active = True - verdicts.append((r, v)) - for r, v in verdicts: - if not active: - out.append(r.get('pid')) - elif v or (members and r.get('pid') in members): - out.append(r.get('pid')) - return out - - -def permits(tree, row, fields, ctx=None): - """PERMISSION semantics — may this reader SEE `row` under this permanent filter? - - Same evaluator, every ignore inverted into a denial. An empty/absent tree means "no - restriction declared" and admits (the caller decides whether a module with no entry is - reachable at all — that is `perm_scope`'s question, not this one); anything PRESENT but - unanswerable DENIES. - """ - nodes, conj = tree_parts(tree) - if not nodes: - return True # nothing declared -> nothing to enforce here - columns = _columns_map(fields) - tag_rules = list(_customer_tag_rules(nodes)) - # An uncomputable tag operand denies the whole wall, including an OR branch. - if tag_rules and ('customer_type' not in columns or any( - _match_customer_tags(rule, row.get('customer_type')) is None for rule in tag_rules)): - return False - v = _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, True) - return bool(v) - - -def tree_parts(tree): - """Read C-PERM amendment 2's `FilterTree` — `{conj?, nodes}` — into `(nodes, conj)`. - - Accepts a BARE LIST too, because that is what `ViewConfig.filters` is and what - `clean_filter_tree` returns; an absent conj is `'and'`, mirroring `ViewConfig.filterConj`. - One reader, so the absent-conj default lives here instead of at every call site. - """ - if isinstance(tree, dict): - nodes = tree.get('nodes') - conj = 'or' if tree.get('conj') == 'or' else 'and' - return (nodes if isinstance(nodes, list) else []), conj - return (tree if isinstance(tree, list) else []), 'and' +sys.modules[__name__] = _impl diff --git a/platform/harness/meta_store.py b/platform/harness/meta_store.py index 1bf8840d5feff4baf4124ecf87a2bd623e2ef3f6..92a432d35c7374bffff3876120327048102e81c5 100644 --- a/platform/harness/meta_store.py +++ b/platform/harness/meta_store.py @@ -34,8 +34,11 @@ larger than 2^53, so any float or JS-number path silently corrupts it. Same ruli (W26/R3), for the same reason, and it is why every `id` column here is VARCHAR. """ import argparse +import errno import json import os +import socket +import ssl import sys import time import urllib.error @@ -261,6 +264,30 @@ def token(rt=None, *, tenant_key=None, connection_id=None): return "" +def _transport_problem(reason): + """Name transport failures without disclosing request URLs or credential-bearing exception text.""" + text = str(reason).lower() + code = getattr(reason, "errno", None) + if isinstance(reason, ssl.SSLCertVerificationError) or any( + word in text for word in ("certificate verify failed", "certificate_verify_failed", + "certificate has expired", "certificate verification")): + return "Meta Ads could not verify the server certificate. Check the server's trusted certificates and clock." + if isinstance(reason, ssl.SSLError) or "ssl" in text or "tls" in text: + return "Meta Ads could not establish a secure connection. Check the server's TLS connection." + if isinstance(reason, socket.gaierror) or any(word in text for word in ( + "name or service not known", "name resolution", "getaddrinfo failed", "nodename nor servname")): + return "Meta Ads could not resolve the provider's host name. Check the server's DNS connection." + if isinstance(reason, TimeoutError) or code == errno.ETIMEDOUT or "timed out" in text: + return "The Meta Ads request timed out. Retry the sync after checking the server's network connection." + if isinstance(reason, ConnectionRefusedError) or code == errno.ECONNREFUSED or "connection refused" in text: + return "The connection to Meta Ads was refused. Check the server's firewall and proxy settings." + if isinstance(reason, ConnectionResetError) or code == errno.ECONNRESET or "connection reset" in text: + return "The connection to Meta Ads was reset. Retry the sync after checking the server's network connection." + if code in (errno.ENETUNREACH, errno.EHOSTUNREACH) or "unreachable" in text: + return "The server cannot reach the Meta Ads network. Check its network route and firewall settings." + return "The network request to Meta Ads failed. Check the server's network and proxy settings, then retry the sync." + + def _get(path, tok, **params): params["access_token"] = tok url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params) @@ -275,6 +302,10 @@ def _get(path, tok, **params): except Exception: msg = "" raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None + except urllib.error.URLError as e: + raise MetaError(_transport_problem(e.reason)) from None + except OSError as e: + raise MetaError(_transport_problem(e)) from None except Exception as e: raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None diff --git a/platform/harness/mirror_store.py b/platform/harness/mirror_store.py index d70df5f0b4d8e706cce20f8787e447c92ca07f8e..fcc87344a3de01c15a9f1dc5eb13081b02ec201d 100644 --- a/platform/harness/mirror_store.py +++ b/platform/harness/mirror_store.py @@ -14,6 +14,7 @@ import hashlib import json import os import re +import shutil import socket import tempfile import threading @@ -48,6 +49,57 @@ _RESTORE_LOCK = threading.Lock() _NEWEST_READERS = {} _PUBLICATION_CHECKS = {} _PUBLISH_LISTENERS = [] +_RESTORE_FOLDER_MAX_AGE = 60 * 60 + + +def _restore_folder_prefix(tenant_key, connector): + """Bind scratch cleanup to one tenant and connector without exposing tenant names.""" + identity = json.dumps(_restore_key(tenant_key, connector)).encode("utf-8") + return "mirror-restore-" + hashlib.sha256(identity).hexdigest() + "-" + + +def _lock_restore_folder(handle): + """Claim a folder without waiting, with a kernel lock released on process death.""" + if os.name == "nt": + import msvcrt + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + + +def _sweep_restore_folders(tenant_key, connector): + """Remove only old, unlocked restore folders owned by this tenant and connector.""" + cutoff = time.time() - _RESTORE_FOLDER_MAX_AGE + prefix = _restore_folder_prefix(tenant_key, connector) + removed = 0 + for folder in Path(tempfile.gettempdir()).glob(prefix + "*"): + try: + marker = folder / ".restore.lock" + if (folder.is_symlink() or not folder.is_dir() or folder.stat().st_mtime >= cutoff + or marker.is_symlink()): + continue + with marker.open("r+b") as handle: + _lock_restore_folder(handle) + # Restore folders are never reused. Closing first permits Windows deletion too. + shutil.rmtree(folder) + removed += 1 + except OSError: + # A locked, unmarked, inaccessible or concurrently removed folder stays untouched. + continue + return removed + + +@contextmanager +def _restore_directory(tenant_key, connector): + """Hold a restore folder's ownership lock until its work has finished.""" + with tempfile.TemporaryDirectory(prefix=_restore_folder_prefix(tenant_key, connector)) as tmp: + with (Path(tmp) / ".restore.lock").open("w+b") as handle: + handle.write(b"0") + handle.flush() + _lock_restore_folder(handle) + yield tmp def register_publish_listener(fn): @@ -822,6 +874,7 @@ class MirrorStore: if cache_key in self.cache: return self.cache[cache_key] restore.enter_context(_restore_entry(tenant_key, self.connector)) + _sweep_restore_folders(tenant_key, self.connector) result = {"state": "local", "problems": [], "manifest": {}, "dirty": set(), "runtime": rt, "refused_tables": []} con, _path = self._connect(tenant_key, rt) @@ -876,7 +929,7 @@ class MirrorStore: progress = _catch_up_source(rt, self.connector) progress["part_total"] = len(part_keys | progress["parts"]) self._before_restore(tenant_key, rt) - with tempfile.TemporaryDirectory(prefix="mirror-restore-") as tmp: + with _restore_directory(tenant_key, self.connector) as tmp: manifest, rows, paths, total, refused = self._validated_snapshot( tenant_key, rt, con, raw, tmp, allow_partial=True) self._install_snapshot(con, manifest, rows, paths) @@ -918,33 +971,8 @@ class MirrorStore: def _refused_may_publish(self, tenant_key, rt, con, cache): - """D-3481: a process that could not restore must not publish over a snapshot that is valid. - - It publishes once its own first load is complete (its rebuild is then whole), or once a - fresh re-validation proves the stored snapshot unrestorable (nothing valid is lost). A - store-access failure during that re-check proves nothing, so it holds and asks again later. - """ - if cache.get("publish_ok"): - return True - if self._first_load_complete(con): - cache["publish_ok"] = True - return True - if cache.get("stored_valid") is None: - try: - raw = self._download(rt, _MANIFEST_KEY) - if raw is None: - cache["stored_valid"] = False - else: - with tempfile.TemporaryDirectory(prefix="mirror-recheck-") as tmp: - self._validated_snapshot(tenant_key, rt, con, raw, tmp) - cache["stored_valid"] = True - except (ValueError, KeyError, TypeError, AttributeError): - cache["stored_valid"] = False - except Exception: # noqa: BLE001 - return False - if cache["stored_valid"] is False: - cache["publish_ok"] = True - return True + """Keep a refused process from checkpointing until a new process restores the saved copy.""" + # Local completion and a failed recheck cannot authorize replacing unread stored data. return False @@ -1112,8 +1140,8 @@ class MirrorStore: try: if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache): message = (f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot " - "at start, and that snapshot is still valid or unreachable. It is kept; restart to " - "restore it. This process saves only after its own first load completes.") + "at start. The saved copy is kept; restore tenant-store access and restart to " + "restore it before checkpointing.") report["problems"].append(message) log(message) return diff --git a/platform/harness/tools.py b/platform/harness/tools.py index 14922f0e1f9b3ab3df00b265a9d5ce8aebfdeff8..0c51c7cc4b399a6e1a7ea4378cfa5ed6ba54926d 100644 --- a/platform/harness/tools.py +++ b/platform/harness/tools.py @@ -378,9 +378,39 @@ def report_gap(kind, missing, question, workaround=None): # ------------------------------------------------------------------ workspace tools (OM-3/P11) def list_workspace(): - """Everything in the tenant workspace (uniform modular objects) + available templates.""" + """Everything in the tenant workspace (uniform modular objects) + available templates. + + D-2643: a tenant with an Amazon mirror also gets `amazonTables`, each declared table's row + count or its code and one-line reason, from the one producer the Data grid reads. + D-2642: a dated order table also answers its newest purchase date, cadence and any lag line. + """ import harness.workspace as W - return {"objects": W.items(), "templates": W.templates()} + out = {"objects": W.items(), "templates": W.templates()} + try: + tables = amazon_table_reasons() + except Exception as exc: # noqa: BLE001 + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + # The workspace list still answers. The Amazon part names why it is missing. + out["amazonTablesUnavailable"] = ("Amazon table reasons could not be read from the Amazon " + f"mirror ({type(exc).__name__}), so retry after the next Amazon sync.") + return out + if tables is not None: + out["amazonTables"] = tables + return out + + +def amazon_table_reasons(): + """`amazon_store.empty_reason_census` for the request's tenant, or None with no Amazon mirror.""" + from harness import amazon_store + runtime, _user = _REQUEST_CONTEXT.get() + if runtime is None: + from harness import runtime as runtimes + tenant = runtimes.tenant_for_store_path() + if not tenant: + return None + runtime = runtimes.get_runtime(tenant) + return amazon_store.empty_reason_census(runtime.key, runtime) def instantiate_template(filename, new_name=None): diff --git a/platform/modules/amazon_operator.py b/platform/modules/amazon_operator.py index 4c250cfb0f87e3e910ea586523f348b683d3cd6f..9f2d056265e1546659ad3d7cd97cf3e3d3aa2906 100644 --- a/platform/modules/amazon_operator.py +++ b/platform/modules/amazon_operator.py @@ -1573,8 +1573,8 @@ def set_status(tenant_key, user, item_id, status, *, note=None, st=None, now=Non PermissionError for a viewer, before any store change. ValueError on an unknown status. Only a `proposed` row moves. A refused write records its cause on its row and the rest of the batch goes on. Ads proposal ids and App inbox ids may share one call. An inbox id goes back to its App's on_decision. - `reviews` maps each removal inbox id to the `review.reviewed` value its person saw. Approving a removal - without it, or with a changed one, is refused before any store change. + `reviews` binds each removal's reviewed fingerprint and boolean `validateOnly` mode. Legacy fingerprints + retain the source mode. A missing or changed fingerprint is refused before a decision is recorded. """ st = _rt(tenant_key, st) who = _editor(st, user, tenant_key) @@ -1789,13 +1789,14 @@ def _review_of(raw): "sku": preview.get("sku"), "unitCost": preview.get("unitCost"), "currency": preview.get("currency"), "arithmetic": arithmetic, "shipTo": preview.get("shipTo") if isinstance(preview.get("shipTo"), dict) else None, - "validateOnly": True}) + "validateOnly": raw.get("validateOnly") is not False, "canSubmit": False}) if isinstance(rv, dict) and isinstance(rv.get("fingerprint"), str) and rv["fingerprint"].strip(): return _json_safe({"reviewed": rv["fingerprint"], "fate": rv.get("disposition"), "units": rv.get("quantity"), "sku": rv.get("sku"), "sellable": rv.get("sellable"), "marketplace": rv.get("marketplace"), "arithmetic": None, "shipTo": rv.get("ship_to") if isinstance(rv.get("ship_to"), dict) else None, - "validateOnly": True}) + "validateOnly": raw.get("validateOnly", rv.get("validateOnly", rv.get("validate_only"))) is not False, + "canSubmit": True}) return None @@ -1975,7 +1976,9 @@ def _hand_back(tenant_key, rec, decision, reason, *, user=None, review=None): return cause, None offered = {"user": user} if review is not None and decision == "approved": - offered.update(reviewed=copy.deepcopy(review["reviewed"]), validate_only=True) + offered.update(reviewed=copy.deepcopy(review["reviewed"]), validate_only=review["validateOnly"]) + if not review["validateOnly"] and "validate_only" not in _accepted(fn, offered): + return "Submission is unavailable for this removal.", None try: result = fn(tenant_key, rec["item_id"], decision, reason, **_accepted(fn, offered)) except Exception as exc: # noqa: BLE001 @@ -1990,7 +1993,7 @@ def _validated(result): def _checked_reviews(ids, inbox, status, reviews): - """{inbox id: reviewed} for each removal in ids. ValueError when one is missing or changed.""" + """Bind each removal's fingerprint and chosen mode before any decision is recorded.""" if status != "approved": return {} given = reviews if isinstance(reviews, dict) else {} @@ -2002,9 +2005,14 @@ def _checked_reviews(ids, inbox, status, reviews): continue if iid not in given or given[iid] in (None, "", {}): raise ValueError(NEED_REVIEW) - if _json_safe(given[iid]) != review.get("reviewed"): + supplied = given[iid] + chosen = supplied if isinstance(supplied, dict) and "reviewed" in supplied else { + "reviewed": supplied, "validateOnly": review.get("validateOnly", True)} + if type(chosen.get("validateOnly")) is not bool: + raise ValueError(NEED_REVIEW) + if _json_safe(chosen.get("reviewed")) != review.get("reviewed"): raise ValueError(REVIEW_CHANGED) - out[iid] = copy.deepcopy(review["reviewed"]) + out[iid] = {"reviewed": copy.deepcopy(review["reviewed"]), "validateOnly": chosen["validateOnly"]} return out @@ -2029,7 +2037,8 @@ def _decide_apps(st, tenant_key, ids, inbox, status, note, who, at, *, reviews=N "sources": {iid: {"module": inbox[iid].get("source_module"), "item_id": inbox[iid].get("item_id")} for iid in eligible}} if reviewed: - record["reviews"] = {iid: reviewed[iid] for iid in eligible if iid in reviewed} + record["reviews"] = {iid: reviewed[iid]["reviewed"] for iid in eligible if iid in reviewed} + record["review_modes"] = {iid: reviewed[iid]["validateOnly"] for iid in eligible if iid in reviewed} _put_decision(st, record) claimed = {} @@ -2043,7 +2052,7 @@ def _decide_apps(st, tenant_key, ids, inbox, status, note, who, at, *, reviews=N if rec.get("status") == "stale": continue current_review = (rec.get("review") or {}).get("reviewed") - if current_review != reviewed.get(iid): + if current_review != reviewed.get(iid, {}).get("reviewed"): raise ValueError(REVIEW_CHANGED) claimed[iid] = rec["status"] rec.update(status=status, decision_id=decision_id, decided_by=who, decided_at=at, note=note, @@ -2081,7 +2090,7 @@ def _decide_apps(st, tenant_key, ids, inbox, status, note, who, at, *, reviews=N for iid in single: rec = inbox[iid] cause, result = _hand_back(tenant_key, rec, status, reason, user=who, - review={"reviewed": reviewed[iid]} if iid in reviewed else None) + review=reviewed.get(iid)) if cause: _reopen(st, iid, cause) rows[iid] = {"id": iid, "status": "refused", "cause": cause, "app": rec.get("app")} diff --git a/platform/modules/amazon_profit.py b/platform/modules/amazon_profit.py index a4fb9a4e7f8436ae9ae75cf53316310af55a46d6..49ddb2714c3acf30c112c7b19c0f75228e23f6b2 100644 --- a/platform/modules/amazon_profit.py +++ b/platform/modules/amazon_profit.py @@ -45,6 +45,7 @@ PROJECTS_LABEL = "Projects" ASIN_KEY = "ut_amazon_asin_profit" ASIN_LABEL = "Child ASIN" PARENT_KEY = "ut_amazon_parent_profit" +PROFIT_MAX_FIELDS = 100 # A module table declares its field cap; a connector table stays exempt. PARENT_LABEL = "Parent ASIN" # The four tables are the module's, not a person's: they take no MAX_TABLES slot (D-3284 spirit). _registry.register_module_tables(TABLE_KEY, PROJECTS_KEY, ASIN_KEY, PARENT_KEY) @@ -142,6 +143,7 @@ _ECON_FIELDS = ( _field("sponsored_ads", "Sponsored ads", "currency", "Ad spend.", computed=True), _field("net_proceeds", "Net proceeds", "currency", "Net sales minus fees and ads.", computed=True), + _field("sales_note", "Sales note", "text", "Why sales, units, fees or proceeds are blank.", computed=True), _field("cogs_per_unit", "COGS per unit", "currency", "Receipt purchase and landed cost per Amazon unit.", computed=True), _field("cogs_total", "COGS total", "currency", "COGS per unit times units sold.", @@ -166,6 +168,9 @@ _ECON_FIELDS = ( _ECON_KEYS = tuple(f["key"] for f in _ECON_FIELDS) #: The row's main listing picture: an identity cell, set after the rollup and never summed. PICTURE_FIELD = _field("picture", "Picture", "image", "Main Amazon listing picture.", computed=True) +#: Why a health label, the score, the priority or a trend is blank. It sits after the trends. +HEALTH_NOTE_FIELD = _field("health_note", "Health note", "text", "Why health, score or a trend is blank.", + computed=True) #: Computed columns: a refresh owns them and the pre-set lock keeps people out of them. COMPETITOR_FIELDS = tuple(_field(k, label, t, d, computed=True, **dict(x)) for k, label, t, d, x in _rivals.ROW_COLUMNS) @@ -174,7 +179,7 @@ PARENT_COMPETITOR_FIELDS = tuple(_field(k, label, t, d, computed=True, **dict(x) COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS _SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.") COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (PICTURE_FIELD,) + _ECON_FIELDS[1:] + _metric_fields() - + _SKU_PROJECT_FIELDS[1:] + + (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:] + COMPETITOR_FIELDS) #: Owned columns: people edit them and a refresh never writes them. OWNED_FIELDS = ( @@ -240,15 +245,60 @@ GRAIN_COST_NOTE = "A SKU here has no cost. Check the SKU tab." NO_COST_NOTE = "No Odoo cost for this SKU." COST_FIGURES = ("cogs_per_unit", "cogs_total", "net_profit", "margin") ODOO_CHANNEL_FIGURES = ("odoo_channel_sales", "odoo_channel_profit", "odoo_channel_margin") +#: The period, sales and rollup count figures. A blank one comes only from a refusal or an absent row. +SALES_FIGURES = ("period", "sku_count", "skus", "units_sold", "net_sales", "fees", "sponsored_ads", "net_proceeds") +#: Health labels, score, priority and trends. A blank one comes only from a refusal or an absent row. +HEALTH_FIGURES = ("stock_health", "profit_health", "ads_health", "listing_health", "score", "priority", + "units_30d", "net_sales_30d", "ad_spend_30d") #: Every reason column and the figures it explains, on all three tables. -BLANK_REASONS = {**M.NOTE_GROUPS, "cost_note": COST_FIGURES, "odoo_margin_source": ODOO_CHANNEL_FIGURES} +BLANK_REASONS = {"sales_note": SALES_FIGURES, **M.NOTE_GROUPS, "cost_note": COST_FIGURES, + "odoo_margin_source": ODOO_CHANNEL_FIGURES, "health_note": HEALTH_FIGURES} +#: Every declared figure that holds a number or a reason (S85-T125). A blank Band is "neither Losing nor Thin" +#: when net profit is set; otherwise the cost or sales note explains it. +DECLARED_FIGURES = tuple(dict.fromkeys(f for figures in BLANK_REASONS.values() for f in figures)) + ("band",) +BLANK_CHECK = "every blank says why" +#: Every refusal sentence starts so. +REFUSED_PREFIX = "Not verified" def unexplained_blanks(row, keys=None): - """`[figure key]` blank in `row` with an empty reason column. `keys`: the table's field keys.""" + """`[figure key]` blank in `row` with an empty reason column. `keys`: the table's field keys. + A refusal in the Period cell is no reason: the rows route projects it to the period label.""" have = set(row if keys is None else keys) - return [figure for note, figures in BLANK_REASONS.items() if not str(row.get(note) or "").strip() - for figure in figures if figure in have and row.get(figure) in (None, "")] + out = [figure for note, figures in BLANK_REASONS.items() if not str(row.get(note) or "").strip() + for figure in figures if figure in have and row.get(figure) in (None, "")] + explained = any(str(row.get(k) or "").strip() for k in ("net_profit", "cost_note", "sales_note")) + if "band" in have and row.get("band") in (None, "") and not explained: + out.append("band") + return out + + +def declared_figures(keys): + """The declared figures a table with these field keys holds, in declaration order.""" + have = set(keys) + return [figure for figure in DECLARED_FIGURES if figure in have] + + +def row_active(row): + """A row with units sold or net sales in its period.""" + return M._active(row) + + +def blank_check(tables): + """`every blank says why` over `{table key: (field keys, rows)}`: ok when no declared figure is blank + without its reason. Counts only; it names figures, never row keys or values.""" + blanks = unexplained = rows = 0 + figures = {} + for keys, table_rows_ in tables.values(): + declared = declared_figures(keys) + for row in table_rows_: + rows += 1 + blanks += sum(1 for figure in declared if row.get(figure) in (None, "")) + for figure in unexplained_blanks(row, keys): + unexplained += 1 + figures[figure] = figures.get(figure, 0) + 1 + return {"check": BLANK_CHECK, "ok": rows > 0 and not unexplained, "rows": rows, "blanks": blanks, + "unexplained": unexplained, "figures": dict(sorted(figures.items()))} def cost_note(row, *, code=None, found=True): @@ -272,7 +322,7 @@ _PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPE #: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute. _ECON_ADDITIVE = ("units_sold", "net_sales", "fees", "sponsored_ads", "net_proceeds", "cogs_total", "net_profit") -_ROLLUP_ECON = ("period",) + _ECON_ADDITIVE + ("margin", "cost_note", "band") +_ROLLUP_ECON = ("period",) + _ECON_ADDITIVE[:5] + ("sales_note",) + _ECON_ADDITIVE[5:] + ("margin", "cost_note", "band") def _grain_fields(key_column, grain): @@ -286,7 +336,7 @@ def _grain_fields(key_column, grain): where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN" progress = _progress_fields(f"Average progress of the projects on {where}.") return (ids[:1] + progress[:1] + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON) - + _metric_fields(skip=("parent_asin",)) + + _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,) + progress[1:]) @@ -646,10 +696,23 @@ def settlements_total(con, start, end, currency=None): "currency": currency if scoped else None} -def settlement_components(con, start, end, currency=None): +def _source_drill(con, sql, params, expected): + """Read every private source row and return only its count, digest and replay query.""" + digest, count = hashlib.sha256(), 0 + cursor = con.execute(sql, params) + while rows := cursor.fetchmany(256): + for row in rows: + digest.update(json.dumps(row, default=str, ensure_ascii=True, separators=(",", ":")).encode()) + digest.update(b"\n") + count += 1 + return {"sql": sql, "params": params, "expectedRows": int(expected), "rows": count, + "countMatches": count == expected, "sha256": digest.hexdigest(), "executed": True} + + +def settlement_components(con, start, end, currency=None, *, drill=False): """Tie purchase-date order charges, service-month storage and daily ads to independent sources.""" month = dt.date.fromisoformat(start).strftime("%B %Y") - parts, problems = [], [] + parts, problems, drills = [], [], [] def problem(component, reason): problems.append(f"Not verified: {component} in {month}: {reason}.") @@ -741,18 +804,27 @@ def settlement_components(con, start, end, currency=None): WHEN NULLIF(trim(order_id), '') IS NOT NULL THEN bought ELSE posted END AS basis FROM classified) """ + window = """basis BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) + OR ((basis IS NULL OR dates <> 1 OR bad_dates <> 0) + AND (posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL))""" + source_params = [currency, LOAN_REPAYMENT, start, end, start, end] groups = con.execute(cte + """SELECT component, COALESCE(SUM(currency_amount), 0), count(*), count(*) FILTER (WHERE currency_amount IS NULL OR posted IS NULL), count(*) FILTER (WHERE NULLIF(trim(order_id), '') IS NOT NULL AND (bought IS NULL OR dates <> 1 OR bad_dates <> 0)) - FROM dated WHERE basis BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) - OR ((basis IS NULL OR dates <> 1 OR bad_dates <> 0) - AND (posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL)) - GROUP BY component ORDER BY component""", - [currency, LOAN_REPAYMENT, start, end, start, end]).fetchall() + FROM dated WHERE """ + window + " GROUP BY component ORDER BY component", source_params).fetchall() amounts = {name: total for name, total, *_ in groups} if not groups: problem("settlement components", "no settlement source rows on the purchase or service dates") + if drill: + counts = {row[0]: row[2] for row in groups} + for name in sorted(set(counts) | {"product sales", "order fees", "monthly storage", + "reimbursements", "account fees", "tax"}): + query = (cte + "SELECT settlement_id, order_id, posted_date, amount_type, amount_description, " + "amount, bought, dates, bad_dates, component, basis FROM dated WHERE (" + window + + ") AND component = ? ORDER BY ALL") + drills.append(dict(_source_drill(con, query, source_params + [name], counts.get(name, 0)), + component="sales tax" if name == "tax" else name, source=SETTLEMENTS)) for name, _total, _n, bad, unmatched in groups: if bad or unmatched: problem(name, "unmatched purchase date, unreadable posting date or unavailable amount") @@ -781,6 +853,27 @@ def settlement_components(con, start, end, currency=None): reimbursements = sum(values[c] for c in reimbursement_columns) account_fees = sum(values[c] for c in account_columns) ads = sum(values[c] for c in ads_columns(ec)) + if drill: + def expression(columns): + return " + ".join(f'COALESCE(TRY_CAST("{c}" AS DECIMAL(38, 2)), 0)' for c in columns) or "0" + + non_order = storage_columns + reimbursement_columns + account_columns + component_columns = {"product sales": ["sales_net_product_sales_amount"], + "order fees": ["fees_total_amount", *non_order], + "monthly storage": storage_columns, "reimbursements": reimbursement_columns, + "account fees": account_columns, "ads": ads_columns(ec)} + for name, columns in component_columns.items(): + amount = expression(columns) + if name == "order fees": + amount = expression(["fees_total_amount"]) + " - (" + expression(non_order) + ")" + invalid = " OR ".join(f'("{c}" IS NOT NULL AND TRY_CAST("{c}" AS DECIMAL(38, 2)) IS NULL)' + for c in columns) or "FALSE" + where = f"left(start_date, 10) BETWEEN ? AND ? AND (({amount}) <> 0 OR {invalid})" + count = con.execute(f"SELECT count(*) FROM {ECONOMICS} WHERE {where}", [start, end]).fetchone()[0] + selected = list(dict.fromkeys(["start_date", "msku", "currency_code", *columns])) + projection = ", ".join(f'"{c}"' for c in selected if c in ec) + query = f"SELECT {projection} FROM {ECONOMICS} WHERE {where} ORDER BY ALL" + drills.append(dict(_source_drill(con, query, [start, end], count), component=name, source=ECONOMICS)) tie("product sales", sales, amounts.get("product sales", 0.0)) tie("order fees", fees - storage - reimbursements - account_fees, -amounts.get("order fees", 0.0)) tie("monthly storage", storage, -amounts.get("monthly storage", 0.0)) @@ -800,6 +893,10 @@ def settlement_components(con, start, end, currency=None): f"SELECT SUM(TRY_CAST(cost AS DECIMAL(38, 2))), count(*), " f"count(TRY_CAST(cost AS DECIMAL(38, 2))) FROM {table} " "WHERE left(CAST(date AS VARCHAR), 10) BETWEEN ? AND ?", [start, end]).fetchone() + if drill: + query = (f"SELECT date, cost FROM {table} WHERE left(CAST(date AS VARCHAR), 10) " + "BETWEEN ? AND ? ORDER BY ALL") + drills.append(dict(_source_drill(con, query, [start, end], count), component="ads", source=table)) if not count or count != present or total is None or not math.isfinite(float(total)): problem("ads", f"unavailable daily amounts in {table}") ad_total = None @@ -812,6 +909,7 @@ def settlement_components(con, start, end, currency=None): ("product sales", "order fees", "monthly storage", "reimbursements", "account fees"))) - float(ad_total)) if ad_total is not None else None return {"total": target, "parts": parts, "reason": problems[0] if problems else None, + **({"drills": drills, "problems": problems} if drill else {}), "rows": sum(row[2] for row in groups), "excluded": {k: float(amounts[k]) for k in ("shipping and promotions", "ads billing") if k in amounts}} @@ -1730,7 +1828,9 @@ def ensure_table(st, username, *, table_key=None, label=None, fields=None, owned With no `table_key` it ensures every table in `table_specs` order and returns `TABLE_KEY`. Format, grain word, cost description and Link status (`_relabel`) repairs stay here.""" if table_key is None: - specs = [{"key": s["key"], "label": s["label"], "fields": _definition_fields(s["fields"], s["owned"])} + # module_tables persists the trusted maxFields; user_tables.field_limit reads it. + specs = [{"key": s["key"], "label": s["label"], "fields": _definition_fields(s["fields"], s["owned"]), + "maxFields": PROFIT_MAX_FIELDS} for s in table_specs()] else: specs = [{"key": table_key, "label": label or table_key, "fields": _definition_fields(fields, owned_keys)}] @@ -2039,8 +2139,14 @@ def ensure_views(st, owner, *, table_key=TABLE_KEY, template_key=TEMPLATE_KEY): #: Columns a seeded view already written gains once; a person who hides one later keeps it hidden. +#: The S85-T125 metric columns follow template order, so each lands after the column before it. _VIEW_COLUMNS_ADDED = ("picture", "odoo_product", "cost_note", "ads_note", "tacos_note", "stock_note", "returns_note", - "cogs_source") + "cogs_source", "fees", "sponsored_ads", "net_proceeds", "sales_note", "cogs_per_unit", + "cogs_total", "odoo_channel_sales", "odoo_channel_profit", "odoo_channel_margin", + "odoo_margin_source", "ad_impressions", "ad_clicks", "ad_spend", "ad_sales", "ad_orders", + "cvr", "cpc", "roas", "fba_on_hand", "inbound_units", "return_units", "sessions", + "unit_session_pct", "buy_box_pct", "listing_note", "units_30d", "net_sales_30d", + "ad_spend_30d", "health_note") #: Columns a seeded view already written loses once; a person who shows one again keeps it. _VIEW_COLUMNS_DROPPED = ("suggested_product", "suggested_units", "match_rule", "match_state") @@ -2305,6 +2411,7 @@ def merge_metrics(computed, metrics): for key in METRIC_KEYS + ("asin",): if key in raw: merged[key] = stored_cell(_FIELD_BY_KEY.get(key), raw[key]) + merged.setdefault("health_note", None) out[sku] = merged return out @@ -2411,6 +2518,187 @@ def rollup_checks(computed, by_grain): return checks +#: Grain figure families tied per row to an independent SQL aggregate of the SKUs' Amazon rows (rule 8): +#: (figures, independent source, figure words, what a failed tie withholds on that grain table). +_TIE_FAMILIES = ( + (("units_sold", "net_sales", "fees", "sponsored_ads", "net_proceeds"), "Amazon economics", "sales figures", + _UNVERIFIED_KEYS + ("return_rate", "unit_session_pct", "listing_health")), + (("ad_spend",), "Amazon ad charges", "ad spend figures", + ("ad_spend", "cpc", "acos", "roas", "tacos", "ads_health", "score", "priority", "ad_spend_30d")), + (("ad_impressions", "ad_clicks", "ad_sales", "ad_orders"), "the Amazon Ads reports", "ads figures", + M.ADS_FIGURES + ("ads_health", "score", "priority")), + (("fba_on_hand", "inbound_units"), "Amazon inventory planning", "stock figures", + ("fba_on_hand", "inbound_units", "days_of_cover", "stock_health", "score", "priority")), + (("return_units",), "the Amazon returns reports", "returned units", + ("return_units", "return_rate", "listing_health", "score", "priority")), + (("sessions",), "the Amazon traffic report", "sessions", + ("sessions", "unit_session_pct", "buy_box_pct", "listing_health", "score", "priority")), + (("cogs_total", "net_profit"), "the SKU costs", "cost figures", + ("cogs_total", "net_profit", "margin", "band", "profit_health", "score", "priority")), +) +_COUNT_FIGURES = frozenset(("units_sold", "ad_impressions", "ad_clicks", "ad_orders", "fba_on_hand", + "inbound_units", "return_units", "sessions")) + + +def _cent(value): + return None if value is None else float(f"{float(value):.2f}") + + +def _sub_cent(value): + """A raw sum with a part below one cent: two float sums of it may round to cents one cent apart.""" + return value is not None and abs(float(value) * 100 - round(float(value) * 100)) > 1e-6 + + +def _sku_figures(con, start, end, skus): + """`{figure: {sku: value or None}}` read straight from the mirror, rounded as a SKU cell stores it. + A family whose source table is absent maps to None.""" + from modules import ranking_diagnostic, restock_planner + out = {} + cols = _require(con, ECONOMICS, _ECONOMICS_REQUIRED) + ads = " + ".join(f'COALESCE(TRY_CAST("{c}" AS DOUBLE), 0)' for c in ads_columns(cols)) or "0" + rows = con.execute( + "SELECT msku, SUM(sales_net_units_sold), SUM(sales_net_product_sales_amount), SUM(fees_total_amount), " + f"SUM({ads}) FROM {ECONOMICS} WHERE left(CAST(start_date AS VARCHAR), 10) BETWEEN ? AND ? " + "AND msku IN (SELECT unnest(?)) GROUP BY msku", [start, end, skus]).fetchall() + econ = {r[0]: [_num(v) for v in r[1:]] for r in rows} + zero = [0.0, 0.0, 0.0, 0.0] + out["units_sold"] = {s: int(econ.get(s, zero)[0]) for s in skus} + out["net_sales"] = {s: _cent(econ.get(s, zero)[1]) for s in skus} + out["fees"] = {s: _cent(econ.get(s, zero)[2]) for s in skus} + out["sponsored_ads"] = {s: _cent(econ.get(s, zero)[3]) for s in skus} + out["net_proceeds"] = {s: _cent(e[1] - e[2] - e[3]) for s, e in ((s, econ.get(s, zero)) for s in skus)} + loose = {figure: {s for s in econ if _sub_cent(econ[s][index])} + for index, figure in ((1, "net_sales"), (2, "fees"), (3, "sponsored_ads"))} + loose["net_proceeds"] = {s for s in econ if _sub_cent(econ[s][1] - econ[s][2] - econ[s][3])} + loose["ad_spend"] = loose["sponsored_ads"] + union = M._ads_parts(con) + adrows = {} if union is None else {r[0]: r[1:] for r in con.execute( + f"SELECT sku, SUM(impressions), SUM(clicks), SUM(cost), SUM(sales), SUM(orders) FROM ({union}) a " + "WHERE day BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) AND sku IN (SELECT unnest(?)) GROUP BY sku", + [start, end, skus]).fetchall()} + for index, figure in ((0, "ad_impressions"), (1, "ad_clicks"), (3, "ad_sales"), (4, "ad_orders")): + out[figure] = None if union is None else { + s: (M._int(adrows[s][index]) if figure in _COUNT_FIGURES else _cent(M._f(adrows[s][index]))) + if s in adrows else None for s in skus} + loose["ad_sales"] = {s for s in adrows if _sub_cent(M._f(adrows[s][3]))} + if ads_columns(cols): + out["ad_spend"] = {s: _cent(econ.get(s, zero)[3]) for s in skus} + else: + out["ad_spend"] = None if union is None else {s: _cent(M._f(adrows[s][2])) if s in adrows else None + for s in skus} + loose["ad_spend"] = {s for s in adrows if _sub_cent(M._f(adrows[s][2]))} + out["_loose"] = loose + plan_cols = _columns(con, restock_planner.PLANNING) + if {"sku", "available"} <= plan_cols: + inbound = 'TRY_CAST("inbound_quantity" AS DOUBLE)' if "inbound_quantity" in plan_cols else "NULL" + fnsku = "max(NULLIF(TRIM(CAST(\"fnsku\" AS VARCHAR)), ''))" if "fnsku" in plan_cols else "NULL" + plan = {r[0]: r[1:] for r in con.execute( + f'SELECT TRIM(CAST("sku" AS VARCHAR)), {fnsku}, SUM(TRY_CAST("available" AS DOUBLE)), SUM({inbound}) ' + f'FROM {restock_planner.PLANNING} WHERE {restock_planner._newest(plan_cols, restock_planner.PLANNING)} ' + 'AND TRIM(CAST("sku" AS VARCHAR)) IN (SELECT unnest(?)) GROUP BY 1', [skus]).fetchall()} + out["_pool"] = {s: (plan[s][0] if s in plan else None) or s for s in skus} + out["fba_on_hand"] = {s: M._int(plan[s][1]) if s in plan else None for s in skus} + out["inbound_units"] = {s: M._int(plan[s][2]) if s in plan else None for s in skus} + else: + out["fba_on_hand"] = out["inbound_units"] = None + returns = M._returns_parts(con) + if returns is None: + out["return_units"] = None + else: + got = dict(con.execute(f"SELECT sku, SUM(units) FROM ({returns}) r WHERE day BETWEEN CAST(? AS DATE) " + "AND CAST(? AS DATE) AND sku IN (SELECT unnest(?)) GROUP BY sku", + [start, end, skus]).fetchall()) + out["return_units"] = {s: M._int(got.get(s) or 0) for s in skus} + pricing = ranking_diagnostic._pricing(con) + out["_traffic"] = None if pricing is None else {a: M._int((v or {}).get("sessions")) for a, v in pricing.items()} + return out + + +def grain_ties(con, start, end, computed, by_grain): + """Each ASIN and parent ASIN figure that holds a number ties per row to the sum of its SKUs' Amazon + rows, read again from the mirror in SQL (rule 8). A blank cell is withheld and carries its reason.""" + skus = sorted(str(s) for s in computed) + source = _sku_figures(con, start, end, skus) if skus else {} + checks = [] + for grain, label in (("asin", "ASIN"), ("parent_asin", "Parent ASIN")): + rows = by_grain.get(grain) or {} + members = {} + for sku in skus: + value = str((computed[sku] or {}).get(grain) or "").strip() + if value: + members.setdefault(value, []).append(sku) + for figures, words, what, _withheld in _TIE_FAMILIES: + for figure in figures: + compared = withheld = 0 + mismatched, worst = [], 0.0 + for key, cells_by_key in rows.items(): + got = M._f(cells_by_key.get(figure)) + if got is None: + withheld += 1 + continue + compared += 1 + want = _tie_expected(figure, members.get(key, []), computed, source) + gap = math.inf if want is None else abs(got - want) + # Half a cent for the row's own rounding, plus one cent per member SKU whose raw sum + # has a part below a cent: two float sums of it may round one cent apart. + loose = (source.get("_loose") or {}).get(figure) or set() + tolerance = (0.01 if figure in _COUNT_FIGURES else + 0.0051 + 0.0101 * sum(1 for sku in members.get(key, []) if sku in loose)) + if not gap <= tolerance: + mismatched.append(key) + worst = max(worst, gap) + ok = not mismatched + checks.append({"check": f"amazon_profit: {label} {_FIELD_BY_KEY[figure]['label'].lower()} tie to " + f"{words} per row", "ok": ok, "grain": grain, "figure": figure, + "compared": compared, "withheld": withheld, "mismatched": len(mismatched), + "keys": mismatched[:5], "rowsFailed": mismatched, + "gap": None if not compared else round(worst, 4), + "reason": None if ok else f"Not verified: {label} {what} do not tie to {words}."}) + return checks + + +def _tie_expected(figure, members, computed, source): + """The independent value of one grain figure from its member SKUs, or None when no source holds it.""" + if figure in ("cogs_total", "net_profit"): + values = [M._f((computed[s] or {}).get(figure)) for s in members] + active = [s for s in members if M._active(_raw_row(computed[s] or {}))] + if any(M._f((computed[s] or {}).get(figure)) is None for s in active): + return None + found = [v for v in values if v is not None] + return _cent(sum(found)) if found else None + if figure == "sessions": + traffic = source.get("_traffic") + if traffic is None: + return None + asins = dict.fromkeys(str((computed[s] or {}).get("asin") or "").strip() for s in members) + found = [traffic.get(a) for a in asins if a and traffic.get(a) is not None] + return float(sum(found)) if found else None + per_sku = source.get(figure) + if per_sku is None: + return None + if figure in ("fba_on_hand", "inbound_units"): + pools = {} + for sku in sorted(members): + pools.setdefault(source["_pool"].get(sku) or sku, per_sku.get(sku)) + found = [v for v in pools.values() if v is not None] + else: + found = [per_sku.get(s) for s in members if per_sku.get(s) is not None] + return float(sum(found)) if found else None + + +def _withhold_untied(by_grain, checks): + """On each row whose tie failed, blank what that tie withholds and write the failed sentence as the reason.""" + withheld = {figure: keys for figures, _w, _what, keys in _TIE_FAMILIES for figure in figures} + for check in checks: + if check.get("ok") or check.get("grain") not in by_grain: + continue + keys, cells = withheld[check["figure"]], by_grain[check["grain"]] + for row in check.get("rowsFailed") or (): + if row in cells: + cells[row] = _explain(_blank(cells[row], keys), keys, check["reason"]) + return by_grain + + def _grain_period_keys(fields, key_column): return tuple(f["key"] for f in fields if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label") @@ -2548,7 +2836,21 @@ def paint_project_cells(st, con): return counts -def update_projects(st, con, *, today, computed, period, period_label): +def _evaluate_project_filter_tasks(st, *, today, by): + """Evaluate saved task filters with their authors' permissions after project publication.""" + from importlib import import_module + try: + tasks = import_module("core.amazon_project_tasks") + filters = import_module("modules.amazon_project_filters") + except ModuleNotFoundError as exc: + if exc.name not in ("core.amazon_project_tasks", "modules.amazon_project_filters"): + raise + _log(f"Project filter tasks need {exc.name}. Combine prerequisite: merge lane D's task evaluator and matcher.") + return None + return tasks.evaluate_filter_tasks(st, filters.matching_keys, today=today, by=by) + + +def update_projects(st, con, *, today, computed, period, period_label, filter_task_evaluator=None): """Publish independently verified project period totals and Impact.""" from core import amazon_projects as AP from modules import sku_action_results as SR @@ -2566,9 +2868,12 @@ def update_projects(st, con, *, today, computed, period, period_label): except (SR.SourceUnavailable, ValueError): return {"state": "unavailable"} - return AP.update_projects(st, supplied, period=period, period_label=period_label, - today=today, impact_reader=impact_reader, - impact_state_words=IMPACT_STATE_WORDS) + result = AP.update_projects(st, supplied, period=period, period_label=period_label, + today=today, impact_reader=impact_reader, + impact_state_words=IMPACT_STATE_WORDS) + evaluator = _evaluate_project_filter_tasks if filter_task_evaluator is None else filter_task_evaluator + evaluator(st, today=today, by=WRITER) + return result def write_competitor_cells(st, plan): @@ -2722,13 +3027,16 @@ def refresh(st, con, *, period=None, thin=None, by="", cost_reader=None, today=N by_grain = {key: {k: _explain(_blank(v, _UNVERIFIED_KEYS, failed), _UNVERIFIED_KEYS, failed) for k, v in cells_by_key.items()} for key, cells_by_key in by_grain.items()} - report["metricValidation"] = metric_checks + tie + ties = grain_ties(con, bounds["start"], bounds["end"], computed, by_grain) + by_grain = _withhold_untied(by_grain, ties) + report["metricValidation"] = metric_checks + tie + ties report["metricRefusal"] = metric_refusal report["asin"] = write_rows(st, by_grain["asin"], table_key=ASIN_KEY, key_column="asin", label=ASIN_LABEL, period_keys=_grain_period_keys(ASIN_FIELDS, "asin")) report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY, key_column="parent_asin", label=PARENT_LABEL, period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin")) + report["blankReasons"] = blank_check(stored_tables(st)) bands = {sku: values.get("band") for sku, values in computed.items()} report["owned"] = apply_owned(st, bands, today=day, owner=owner_label) stamp = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") @@ -2822,7 +3130,7 @@ def recompute_sku(st, con, sku, *, by="", cost_reader=None, today=None): return cur ut.update_with_activity(TABLE_KEY, _set, st=st, by=by or WRITER, via=VIA, row_ids=None) - _recompute_stored_grains(st, after, settings) + _recompute_stored_grains(st, after, settings, con) _owner, owner_label = default_owner(st, getattr(st, "key", TENANT_ZERO)) apply_owned(st, {sku: after.get("band")}, today=today or _today(), owner=owner_label, skus={sku}) after = next((row for row in table_rows(st).values() if row.get("sku") == sku), after) @@ -2830,8 +3138,9 @@ def recompute_sku(st, con, sku, *, by="", cost_reader=None, today=None): return {"sku": sku, "written": True, "row": dict(after)} -def _recompute_stored_grains(st, changed, settings): - """Update the edited SKU's rollups from persisted cells after their sums tie.""" +def _recompute_stored_grains(st, changed, settings, con=None): + """Update the edited SKU's rollups from persisted cells after their sums tie. + With `con`, each rewritten row also ties to the mirror; a failed figure stays withheld with its reason.""" period_label = str(changed.get("period") or "") if not period_label: return @@ -2843,6 +3152,12 @@ def _recompute_stored_grains(st, changed, settings): if not all(check["ok"] for check in checks): _log("edited SKU rollups not verified") return + bounds = settings.get("lastPeriod") or {} + if con is not None and bounds.get("start") and bounds.get("end"): + plans = {grain: {key: cells for key, cells in plans[grain].items() + if key == str(changed.get(grain) or "").strip()} for grain in plans} + sku_cells = {str(r.get("sku")).strip(): r for r in source.values() if str(r.get("sku") or "").strip()} + plans = _withhold_untied(plans, grain_ties(con, bounds["start"], bounds["end"], sku_cells, plans)) for grain, table_key, fields in (("asin", ASIN_KEY, ASIN_FIELDS), ("parent_asin", PARENT_KEY, PARENT_FIELDS)): key = str(changed.get(grain) or "").strip() if key and key in plans[grain]: @@ -2871,8 +3186,10 @@ def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS return _SNAPSHOT[key].cursor() import duckdb from harness import runtime as R + from harness import mirror_store rt = R.get_runtime(tenant) - raw = rt.download_bytes("amazon_mirror/manifest.json") + mirror = mirror_store.for_connector("amazon") + raw = mirror._download(rt, "manifest.json") if raw is None: raise SourceUnavailable(f"{tenant} has no stored Amazon mirror.") manifest = json.loads(raw) @@ -2888,7 +3205,7 @@ def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS paths = [] for index, part in enumerate(meta.get("parts") or [meta]): blob_key = part.get("key") or f"{name}.parquet" - data = rt.download_bytes("amazon_mirror/" + blob_key) + data = mirror._download(rt, blob_key) if data is None or (part.get("sha256") and hashlib.sha256(data).hexdigest() != part["sha256"]): raise SourceUnavailable(f"The stored {name} snapshot failed its checksum.") path = folder / f"{name}.{index:04d}.parquet" @@ -2997,10 +3314,35 @@ def typed_cogs_checks(con, computed, typed, usage=None, bounds=None): "a": share, "b": stored, "gap": round(abs(share - stored), 4), "unstored": unstored}] +def stored_tables(st): + """`{table key: (field keys, rows)}` of the three stored Command Center tables.""" + import core.user_tables as ut + out = {} + for key in (TABLE_KEY, ASIN_KEY, PARENT_KEY): + table = ut.get_one(key, st) or {} + out[key] = ([f["key"] for f in table.get("fields") or []], list((table.get("rows") or {}).values())) + return out + + +def stored_grain_checks(st, con): + """The stored ASIN and parent tables tie per row to the mirror over the stored period, and every + stored blank says why. Reads only.""" + tables = stored_tables(st) + bounds = read_settings(st).get("lastPeriod") or {} + if not bounds.get("start") or not bounds.get("end"): + return [{"check": "amazon_profit: the stored tables name their period", "ok": False, "grain": None, + "reason": "Not verified: the stored Command Center has no period."}, blank_check(tables)] + sku_cells = {str(r.get("sku")).strip(): r for r in tables[TABLE_KEY][1] if str(r.get("sku") or "").strip()} + by_grain = {grain: {str(r.get(grain)).strip(): r for r in tables[key][1] if str(r.get(grain) or "").strip()} + for grain, key in (("asin", ASIN_KEY), ("parent_asin", PARENT_KEY))} + return grain_ties(con, bounds["start"], bounds["end"], sku_cells, by_grain) + [blank_check(tables)] + + def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None, - typed_values=None, typed_period=None): + typed_values=None, typed_period=None, st=None): """Tie net proceeds components to independent Amazon sources within 2% over settled days. - Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.""" + Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values. + With `st`, the stored ASIN and parent figures tie per row to the mirror and every blank says why.""" month = month or last_complete_month() bounds = period_bounds(month) start, end = bounds["start"], bounds["end"] @@ -3057,6 +3399,8 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No checks.extend(landed_cost_checks(con, cost_usage, cost_history)) if typed_rows is not None: checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds)) + if st is not None: + checks.extend(stored_grain_checks(st, con)) return checks diff --git a/platform/modules/amazon_project_filters.py b/platform/modules/amazon_project_filters.py new file mode 100644 index 0000000000000000000000000000000000000000..80755e4a987a4a76761ad355d27b99e04bd3044a --- /dev/null +++ b/platform/modules/amazon_project_filters.py @@ -0,0 +1,74 @@ +"""Resolve project target filters through the shared row permission and filter readers.""" + +from core import amazon_projects as projects +from core import periods, perm_scope, users +from harness import filter_eval +from modules import amazon_profit + +TABLES = {'sku': amazon_profit.TABLE_KEY, 'asin': amazon_profit.ASIN_KEY, + 'parent_asin': amazon_profit.PARENT_KEY} +MATCH_PAGE = 25 + + +def table_key(grain): + """Return the Command Center table for a validated project target grain.""" + if not isinstance(grain, str) or grain not in TABLES: + raise ValueError('Name a grain of sku, asin or parent_asin.') + return TABLES[grain] + + +def matching_keys(st, grain, tree, *, user=None): + """Return every distinct matching key from this principal's permitted grain rows.""" + key = table_key(grain) + if not (isinstance(tree, list) or isinstance(tree, dict) and isinstance(tree.get('nodes'), list)): + raise ValueError('Send filter as a filter tree or a list of conditions.') + task_author = isinstance(user, tuple) and len(user) == 2 + principal = user if user is not None else {} + if task_author: + tenant, username = user + account = users.registry().get(username) + # A record with no tenant is tenant #0's, as in users._tenant_accounts. A blank want refuses. + if (not isinstance(account, dict) + or str(account.get('tenant') or 'royal-imports').strip().lower() != tenant + or account.get('active') is False): + raise perm_scope.Denied(f'The saved task filter author cannot read {key} in this workspace.') + principal = dict(account, username=username) + fields = perm_scope.scoped_fields(principal, key, st=st) + if task_author: + validate_task_tree(tree, fields) + rows = perm_scope.scoped_table(principal, key, st=st) + ctx = filter_eval.EvalCtx(today=periods.today().isoformat()) + ctx.rank_sets.update(filter_eval.resolve_rank_sets(tree, rows, fields, ctx).sets) + keys = set() + for row in rows: + if filter_eval.matches(tree, row, fields, ctx): + value = row.get(grain) + if value is None or isinstance(value, str) and not value.strip(): + continue + keys.add(projects.target(grain, value)[1]) + return sorted(keys) + + +def validate_task_tree(tree, fields): + """Refuse inactive or missing conditions before they can complete a saved task.""" + nodes, _conj = filter_eval.tree_parts(tree) + columns = {field['key']: field for field in fields} + pending = list(nodes) + while pending: + node = pending.pop() + if not isinstance(node, dict): + raise ValueError('Each task filter condition must be an object.') + if isinstance(node.get('children'), list): + if not node['children']: + raise ValueError('Each task filter group needs a condition.') + pending.extend(node['children']) + elif (not isinstance(node.get('colId'), str) or not isinstance(node.get('op'), str) + or node.get('op') not in filter_eval.FILTER_OPS + or not filter_eval.is_rule_active(node, columns)): + raise ValueError('Each task filter condition needs a readable field and a supported operator.') + + +def check_task_filter(st, grain, tree, *, user): + """Validate a task tree against its author's permitted fields through the existing filter door.""" + validate_task_tree(tree, perm_scope.scoped_fields(user, table_key(grain), st=st)) + return matching_keys(st, grain, tree, user=user) diff --git a/platform/modules/payouts.py b/platform/modules/payouts.py index 06974b88cc22f3de1429cff05418ae3179b97e20..7d1a07d1127845cc299a5ced6f7e5c48b6d47509 100644 --- a/platform/modules/payouts.py +++ b/platform/modules/payouts.py @@ -244,8 +244,10 @@ def snapshot_cursor(tenant=TENANT_ZERO, tables=(SETTLEMENTS, ECONOMICS)): return held[0].cursor() import duckdb from harness import runtime as R + from harness import mirror_store rt = R.get_runtime(tenant) - raw = rt.download_bytes("amazon_mirror/manifest.json") + mirror = mirror_store.for_connector("amazon") + raw = mirror._download(rt, "manifest.json") if raw is None: raise SourceUnavailable("No Amazon settlements are stored for this workspace yet.") manifest = json.loads(raw) @@ -261,7 +263,7 @@ def snapshot_cursor(tenant=TENANT_ZERO, tables=(SETTLEMENTS, ECONOMICS)): paths = [] for index, part in enumerate(meta.get("parts") or [meta]): blob_key = part.get("key") or f"{name}.parquet" - data = rt.download_bytes("amazon_mirror/" + blob_key) + data = mirror._download(rt, blob_key) if data is None or (part.get("sha256") and hashlib.sha256(data).hexdigest() != part["sha256"]): raise SourceUnavailable(f"The stored {name} snapshot failed its checksum.") path = folder / f"{name}.{index:04d}.parquet" diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 6e75d9a3c95c4848ebb45e18a97ab28ca988c1aa..cb4821c48d01e45cdac381cb7c674e74566ad085 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -802,23 +802,14 @@ def _fba_by_code(cat): take the product grid down. The cause travels in `report` and `validate_fba()` fails its leg on it, so an outage is a NAMED refusal rather than an all-blank column that quietly passes. - ⚠ ONE ID PER CODE, AND THE LEFTOVERS ARE ACCOUNTED FOR RATHER THAN SUMMED AWAY. A code can own - more than one `product.product` (re-SKU and merge history), and `products.catalogue()` carries - ONE id per code, so this column reads only that id's units. The others land in - `validate_fba()`'s `outside_the_grid` residual, which is exactly why its total leg is - DECOMPOSED instead of a bare equality. The loop below is shaped to sum a code's ids so it - stays correct the day the catalogue carries more than one, but it is not what makes the - reconciliation tie today. MEASURED live 2026-09-02: 1 active duplicate code, holding no FBA - stock, and a zero residual. + Read only the oldest active record selected by the pool for each code. + Flagged siblings remain in `validate_fba()`'s `outside_the_grid` residual. """ try: import modules.warehouse as warehouse - ids_by_code = {} - for code, meta in (cat or {}).items(): - pid = (meta or {}).get('id') - if pid is not None: - ids_by_code.setdefault(code, []).append(pid) - wanted = [pid for ids in ids_by_code.values() for pid in ids] + ids_by_code = {code: meta['id'] for code, meta in (cat or {}).items() + if (meta or {}).get('id') is not None} + wanted = list(ids_by_code.values()) by_pid, report = warehouse.fba_qty_by_product_id(wanted) except Exception as e: # noqa: BLE001 return None, {'available': False, @@ -826,10 +817,9 @@ def _fba_by_code(cat): if by_pid is None: return None, report out = {} - for code, ids in ids_by_code.items(): - seen = [by_pid[pid] for pid in ids if pid in by_pid] - if seen: - out[code] = sum(seen) + for code, pid in ids_by_code.items(): + if pid in by_pid: + out[code] = by_pid[pid] return out, report @@ -875,7 +865,7 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None): # both degrade to `{}` on a read failure rather than taking the grid down: they are columns, # not the row set. try: - _prods = products.active_products(extra_fields=raw_fields) # ONE read, shared by both + _prods = products.active_products(extra_fields=tuple(dict.fromkeys((*raw_fields, 'create_date')))) except Exception: # noqa: BLE001 _prods = None tiers, _tier_report = products.tier_prices_by_code(_prods) if wants( @@ -885,7 +875,14 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None): tiers, _tier_report = products.scope_tier_prices(tiers, _tier_report, team_id=team_id) pre_book, open_backlog, _backlog_report = backorders.sku_backlog(t=t, team_id=team_id) \ if wants(['pre_book_qty', 'open_backlog_qty']) else ({}, {}, {}) - cat = _catalogue_by_code() + catalogue = _catalogue_by_code() + try: + cat, unselected = _canonical_sku_records(catalogue, _prods), set() + except ValueError: + # A failed active-record read blanks duplicate raw and FBA cells, never the row set. + cat, _ids = catalogue, _active_ids_by_code(_prods) + unselected = {code for code, meta in cat.items() + if meta.get('_active_product_count', 1) > 1 or len(_ids.get(code, [])) > 1} # ⭐ THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the # same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH # will sell. Degrades to `{}` on a read failure, matching every other column source here — a @@ -903,7 +900,9 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None): # because it is keyed off it, and UNSCOPED for the same reason `on_hand` is: a warehouse holds # units, not brands, and pro-rating them onto a BU would print a number no Odoo query could # reproduce. Blank, never zero, when the tenant has no FBA warehouse. - fba_by_code, _fba_report = _fba_by_code(cat) if inventory_needed or wants(['fba']) else ({}, {}) + fba_by_code, _fba_report = _fba_by_code( + {code: meta for code, meta in cat.items() if code not in unselected}) \ + if inventory_needed or wants(['fba']) else ({}, {}) rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} if wants( ['rev_ytd', 'rev_ly', 'yoy_pct', 'qty_ytd', 'orders_ytd']) else {} raw_by_code = {(p.get("default_code") or f"pid:{p.get('id')}"): p @@ -969,13 +968,17 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None): "qty_ytd": r.get("qty_ytd", 0.0) if r else None, "orders_ytd": r.get("orders_ytd", 0) if r else 0, } - raw = raw_by_id.get(meta.get("id")) or raw_by_code.get(code) or {} + raw = {} if code in unselected else (raw_by_id.get(meta.get("id")) + or raw_by_code.get(code) or {}) if raw_fields and raw: for field in raw_fields: if field in raw: row[field] = raw.get(field) if row["_active_product_count"] > 1: - _mark_duplicate_sku(row, ids_by_code.get(code) or [meta.get("id")]) + if code in unselected: + _mark_duplicate_sku(row, ids_by_code.get(code) or [meta.get("id")], selected=False) + else: + _mark_duplicate_sku(row, ids_by_code.get(code) or [meta.get("id")]) # ⭐⭐ W37-T14 / T15 — THE HONEST SETS. These answer "how many prices/units does this SKU # actually have" across every list. The numbered per-list cells below select Fisch, # Royal 1, and Royal 2 without rebuilding a legacy base-price payload. @@ -1051,7 +1054,9 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None): dos, bucket = None, None if not fba_known: row["_coverage_report"] = {**_fba_report, "available": False, - "cause": _fba_report.get("cause") + "cause": ("Duplicate SKU record selection unavailable" + if code in unselected else None) + or _fba_report.get("cause") or "FBA stock unavailable for this product"} row.update({ # UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf. @@ -1455,21 +1460,40 @@ DUPLICATE_SKU_MARKER = "duplicate in Odoo, owner to choose" def _active_ids_by_code(prods): - """`{code: [product ids]}` over active product rows, keyed the way `products.catalogue()` keys.""" + """Group active ids by catalogue code, ordered by create_date then id, with missing dates last.""" out = {} - for p in prods or []: - if p.get("id") is None: - continue + records = [p for p in prods or [] if p.get('id') is not None and p.get('active') is not False] + for p in sorted(records, key=lambda p: (str(p.get('create_date') or '9999'), p['id'])): code = str(p["default_code"]).strip() if p.get("default_code") else f"pid:{p['id']}" out.setdefault(code, []).append(p["id"]) return out -def _mark_duplicate_sku(row, product_ids): - """Name a collapsed SKU on its row: every active record id and the one record it serves.""" +def _canonical_sku_records(cat, prods): + """Select the oldest duplicate's id without changing catalogue rows, counts or aggregate cells.""" + ids_by_code = _active_ids_by_code(prods) + by_id = {p['id']: p for p in prods or [] if p.get('id') is not None} + out = {} + for code, meta in cat.items(): + ids = ids_by_code.get(code, []) + if meta.get('_active_product_count', 1) > 1 or len(ids) > 1: + if len(ids) < 2 or any(not by_id[pid].get('create_date') for pid in ids): + raise ValueError('Duplicate SKU selection requires active record ids and create_date.') + out[code] = {**meta, 'id': ids[0]} + else: + out[code] = meta + return out + + +def _mark_duplicate_sku(row, product_ids, selected=True): + """Name every active id, the served oldest record and the flagged newer records.""" + ids = sorted({i for i in product_ids if i is not None}) + # An unselected duplicate serves no record, so every id it knows is flagged. + served = row.get('product_id') if selected else None row["_duplicate_sku"] = {"marker": DUPLICATE_SKU_MARKER, - "product_ids": sorted(i for i in product_ids if i is not None), - "served_product_id": row.get("product_id")} + "product_ids": ids, + "served_product_id": served, + "flagged_product_ids": [i for i in ids if i != served]} def duplicate_sku_report(rows): @@ -1481,24 +1505,29 @@ def duplicate_sku_report(rows): out.append({"code": row["code"], "marker": mark["marker"], "product_ids": list(mark.get("product_ids") or []), "served_product_id": mark.get("served_product_id"), - "served": "FBA and raw fields read from record " - f"{mark.get('served_product_id')} only"}) + "flagged_product_ids": list(mark.get("flagged_product_ids") or []), + "served": ("FBA and raw fields read from record " + f"{mark.get('served_product_id')} only") + if mark.get("served_product_id") is not None + else "No record served because the oldest record could not be selected"}) return sorted(out, key=lambda e: e["code"]) def duplicate_sku_oracle(): - """`{code: [ids]}` for every code on 2+ active Odoo products, read live and apart from the pool.""" + """Read duplicate ids independently from Odoo, ordered oldest first by create_date then id.""" o = O.get_odoo() dom = [('active', '=', True), ('default_code', '!=', False)] - prods = o.search_read('product.product', dom, ['id', 'default_code']) + prods = o.search_read('product.product', dom, ['id', 'default_code', 'create_date']) n = o.search_count('product.product', dom) if len(prods) != n: raise ValueError(f"duplicate_sku_oracle: read {len(prods)} of {n} active coded products") - return {code: sorted(ids) for code, ids in _active_ids_by_code(prods).items() if len(ids) > 1} + if any(not p.get('create_date') for p in prods): + raise ValueError('duplicate_sku_oracle: active product create_date is unavailable') + return {code: ids for code, ids in _active_ids_by_code(prods).items() if len(ids) > 1} def validate_duplicate_skus(rows, oracle=None): - """Every SKU on 2+ active Odoo products is reported with the marker, never collapsed silently.""" + """Reconcile duplicate membership, oldest served ids and flagged newer ids to independent Odoo records.""" name = ("a duplicate SKU is reported, never collapsed: every SKU carried by 2+ active Odoo " f"products is marked '{DUPLICATE_SKU_MARKER}' with all its record ids") try: @@ -1508,19 +1537,41 @@ def validate_duplicate_skus(rows, oracle=None): return [{"check": name, "ours": None, "theirs": None, "ok": False, "detail": {"source_refusal": f"{type(e).__name__}: {str(e)[:160]}"}}] reported = {e["code"]: e for e in duplicate_sku_report(rows)} + pool_ids = {r.get('code'): r.get('product_id') for r in rows} in_pool = {r.get("code") for r in rows} unmarked = sorted(c for c in theirs if c in in_pool and c not in reported) wrong_ids = sorted(c for c, ids in theirs.items() - if c in reported and reported[c]["product_ids"] != list(ids)) + if c in reported and reported[c]["product_ids"] != sorted(ids)) + wrong_served = sorted(c for c, ids in theirs.items() if c in reported and + (reported[c]['served_product_id'] != ids[0] or pool_ids[c] != ids[0])) + wrong_flagged = sorted(c for c, ids in theirs.items() if c in reported and + reported[c]['flagged_product_ids'] != sorted(ids[1:])) absent = sorted(c for c in theirs if c not in in_pool) stray = sorted(c for c in reported if c not in theirs) return [{"check": name, "ours": len(reported), "theirs": len(theirs), - "ok": not (unmarked or wrong_ids or absent or stray), + "ok": not (unmarked or wrong_ids or wrong_served or wrong_flagged or absent or stray), "detail": {"duplicates": [reported[c] for c in sorted(reported)], "collapsed_without_marker": unmarked, "record_ids_differ": wrong_ids, + "served_record_differs": wrong_served, "flagged_records_differ": wrong_flagged, "absent_from_pool": absent, "marked_but_unique_in_odoo": stray}}] +def validate_population(rows, active_count=None): + """Reconcile grid rows plus flagged siblings to an independent active Odoo count.""" + n_active = products.catalogue_count() if active_count is None else active_count + duplicates = duplicate_code_report(rows) + flagged = sum(len(entry['flagged_product_ids']) for entry in duplicate_sku_report(rows)) + accounted = len(rows) + flagged + return [{ + "check": "the pool accounts for every ACTIVE product (R12): row count + flagged duplicates == an INDEPENDENT " + "Odoo search_count('product.product', active=True)", + "ours": accounted, "theirs": n_active, "ok": accounted == n_active, + "gap": n_active - len(rows), + "detail": {"rows": len(rows), "duplicate_gap": duplicates['gap'], "flagged_duplicates": flagged, + "duplicate_codes": duplicates['codes']}, + }] + + def validate(team_id=None, t=None): """Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number which does not tie to Odoo does not ship. @@ -1548,17 +1599,7 @@ def validate(team_id=None, t=None): # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active # products, asked fresh — not a `len()` over anything this module or `products.catalogue()` # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it. - n_active = products.catalogue_count() - duplicates = duplicate_code_report(rows) - accounted = len(rows) + duplicates['gap'] - checks = [{ - "check": "the pool accounts for every ACTIVE product (R12): row count + duplicate code gap == an INDEPENDENT " - "Odoo search_count('product.product', active=True)", - "ours": accounted, "theirs": n_active, "ok": accounted == n_active, - "gap": n_active - len(rows), - "detail": {"rows": len(rows), "duplicate_gap": duplicates['gap'], - "duplicate_codes": duplicates['codes']}, - }] + checks = validate_population(rows) checks.extend(validate_duplicate_skus(rows)) yf, yt = P.ytd(t) diff --git a/platform/modules/products.py b/platform/modules/products.py index 3d39ab2abc28635a1eac17b0e2487a34ad73bfc2..a16e4ca17a0cea53dca395abe461e86baae58eba 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -402,8 +402,7 @@ def catalogue(): prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name', 'create_date', 'product_tag_ids', 'incoming_qty', 'barcode', 'free_qty', - 'virtual_available', 'product_tmpl_id', 'volume'], - limit=50000) + 'virtual_available', 'product_tmpl_id', 'volume']) n = O.get_odoo().search_count('product.product', dom) if len(prods) != n: raise ValueError( @@ -695,7 +694,7 @@ def tier_cap_overflow(tiers): if name in declared and n > PRICELIST_TIER_DEPTH} -def active_products(limit=50000, extra_fields=()): +def active_products(limit=None, extra_fields=()): """`[{id, default_code, product_tmpl_id}]` for every ACTIVE product — read ONCE and shared. ⛔ RAISES ON A SHORT PULL, like `catalogue()` and for the same reason: a truncated product read diff --git a/platform/modules/ranking_diagnostic.py b/platform/modules/ranking_diagnostic.py index e572151eaf0f2b7ed747afef6deea838eebc8644..87e6ec761d11c7d325a67c253eedeaf2c4c79e8b 100644 --- a/platform/modules/ranking_diagnostic.py +++ b/platform/modules/ranking_diagnostic.py @@ -398,6 +398,25 @@ def _rank(raw): return None +def _product_title(raw): + """Read the first nonempty localized value without exposing Amazon attribute metadata.""" + if isinstance(raw, list): + return next((title for value in raw if (title := _product_title(value))), None) + if isinstance(raw, dict): + return _product_title(raw.get("value")) + if not isinstance(raw, str): + return None + title = raw.strip() + if not title: + return None + if title.startswith(("[", "{", '"')): + try: + return _product_title(json.loads(title)) + except ValueError: + return None + return title + + def _catalog(cur): cols = _columns(cur, CATALOG) if not {"asin"} <= cols: @@ -412,7 +431,7 @@ def _catalog(cur): rank = _rank(ranks) if rank is not None and (got["salesRank"] is None or rank < got["salesRank"]): got["salesRank"] = rank - got["title"] = got["title"] or (_text(title) or None) + got["title"] = got["title"] or _product_title(title) return out diff --git a/platform/modules/returns_refunds.py b/platform/modules/returns_refunds.py index d22de26b16bd7ef980a7d1a7370821e29e188be7..6e66abf400a35aafdbc24b947713feac58898452 100644 --- a/platform/modules/returns_refunds.py +++ b/platform/modules/returns_refunds.py @@ -36,6 +36,7 @@ RATE_BASIS = {"returnRate": "FBA customer returns plus merchant returns / shippe "refundRate": "Settlement and Finance refund event units, including returnless refunds / shipped units", "refundCost": "Refunded item principal, excluding tax, fees and claim reimbursements"} FINANCE_PRINCIPAL = "item_sub_breakdown_product_charges__principal_amount" +FINANCE_OUR_PRICE = "item_sub_breakdown_product_charges__our_price_principal_amount" KEY_SETTINGS = "returns_refunds_settings" KEY_ITEMS = "returns_refunds_items" KEY_HISTORY = "returns_refunds_history" @@ -1928,7 +1929,19 @@ def _source_ready(cur, table, required, identities=()): return bool(_baseline_markers(cur, table)) -def _rate_availability(cur): +def _merchant_shipment_bounds(cur): + """Project possible shipment intervals without treating purchase dates as shipment dates.""" + cols = _columns(cur, 'amzn_order_items') + if not {'fulfillment_channel', 'item_status', 'quantity'} <= cols: + return 'SELECT NULL::DATE AS purchase_day,NULL::DATE AS ship_by_day,1 AS unplaced WHERE false' + purchase = _sql_day(_text(cols, 'purchase_date')) + ship_by = _sql_day(_text(cols, _first(cols, 'latest_ship_date', 'ship_by_date', 'ship_by'))) + return (f'SELECT {purchase} AS purchase_day,{ship_by} AS ship_by_day,1 AS unplaced FROM amzn_order_items ' + "WHERE (fulfillment_channel IS NULL OR upper(trim(fulfillment_channel)) NOT IN ('AFN','AMAZON')) " + "AND (item_status IS NULL OR lower(trim(item_status)) NOT IN ('cancelled','canceled','unshipped'))") + + +def _rate_availability(cur, *, windowed=False): """An empty schema needs completed source evidence before it can mean zero.""" ready = { "returns": _source_ready(cur, RETURNS_TABLE, ("quantity", "return_date"), ("sku", "asin")), @@ -1943,12 +1956,10 @@ def _rate_availability(cur): "item_contexts", "item_related_identifiers", "related_identifiers", "sku", "asin", FINANCE_PRINCIPAL, FINANCE_PRINCIPAL + "_currency_code")), "merchantShipped": _source_ready(cur, "amzn_order_items", ("fulfillment_channel", "item_status", "quantity"))} - # Purchase and last-updated dates cannot place merchant shipments in a shipment-date window. - if ready["merchantShipped"]: - ready["merchantShipped"] = not bool(cur.execute("SELECT EXISTS(SELECT 1 FROM amzn_order_items " - "WHERE fulfillment_channel IS NULL OR upper(trim(fulfillment_channel)) NOT IN ('AFN','AMAZON') " - "AND (item_status IS NULL OR lower(trim(item_status)) NOT IN ('cancelled','canceled','unshipped')))" - ).fetchone()[0]) + # Windowed dashboards check possible merchant shipments inside their grouped source query. + if ready["merchantShipped"] and not windowed: + ready["merchantShipped"] = not bool(cur.execute('SELECT EXISTS(SELECT 1 FROM (' + + _merchant_shipment_bounds(cur) + ') merchant_shipments)').fetchone()[0]) for field, table, date, quantity in (("returns", RETURNS_TABLE, "return_date", "quantity"), ("shipped", SHIPPED_TABLE, "shipment_date", "quantity"), ("merchantReturns", MFN_TABLE, "return_request_date", "return_quantity")): @@ -1998,24 +2009,31 @@ def _refund_overlap_relations(): f"b.{field}=CASE WHEN (a.mask & m.mask & {1 << i})!=0 THEN a.{field} ELSE '' END" for i, field in enumerate(fields)) identities = ' AND '.join(f'a.{field} IS NOT DISTINCT FROM o.{field}' for field in fields) + # A refund matched in both reports has a proven identity. Only two unmatched captures can overlap. return f"""refund_identities AS ( SELECT DISTINCT source,account,order_id,sku,settlement_id,{mask} AS mask FROM refund_candidates), refund_overlap_keys AS ( SELECT DISTINCT source,account,i.mask AS identity_mask,p.mask AS projection_mask,{projection} - FROM refund_identities i CROSS JOIN range(8) p(mask) WHERE (i.mask & p.mask)=p.mask), + FROM refund_identities i CROSS JOIN range(8) p(mask) + WHERE (i.mask & p.mask)=p.mask AND i.source IS DISTINCT FROM 'both'), refund_overlaps AS ( SELECT a.source,a.account,a.order_id,a.sku,a.settlement_id,bool_or(b.source IS NOT NULL) AS overlap FROM refund_identities a CROSS JOIN range(8) m(mask) LEFT JOIN refund_overlap_keys b ON a.source!=b.source AND a.account=b.account AND b.identity_mask=m.mask AND b.projection_mask=(a.mask & m.mask) AND {matches} GROUP BY a.source,a.account,a.order_id,a.sku,a.settlement_id), - refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key,o.overlap + refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key, + o.overlap AND a.source IS DISTINCT FROM 'both' AS overlap FROM refund_candidates a JOIN refund_overlaps o ON a.source=o.source AND a.account=o.account AND {identities})""" def _refund_relations(cur): - """Keep source identities, capture deduplication and ambiguous overlap inside SQL.""" + """Merge settlement and Finance captures on one refund event identity inside SQL. + + The event is the account, refund id and adjustment item: settlement adjustment_id and order_item_code + equal Finance REFUND_ID and ORDER_ADJUSTMENT_ITEM_ID. A deferred Finance original yields to its release. + """ sc, tc = (_columns(cur, table) for table in FINANCE_TABLES) def txt(cols, name): return f"NULLIF(trim({_text(cols, name)}),'')" @@ -2026,10 +2044,19 @@ def _refund_relations(cur): f"WHERE json_extract_string(j.value,'$.{prefix}Name')='{name}')") principal = (f"lower(trim({_text(sc, 'amount_type')}))='itemprice' " f"AND lower(trim({_text(sc, 'amount_description')}))='principal'") + # Header rows carry the settlement currency. A dotted date is day first only when its settlement proves it. + stamps = ','.join(f'CAST({_text(sc, name)} AS VARCHAR)' for name in ( + 'posted_date', 'settlement_start_date', 'settlement_end_date', 'deposit_date')) + header = (f"SELECT COALESCE({txt(sc, '_connection_id')},'legacy') AS account,{txt(sc, 'settlement_id')} AS settlement_id," + f"CASE WHEN {txt(sc, 'transaction_type')} IS NULL THEN upper({txt(sc, 'currency')}) END AS currency," + f"split_part(trim(unnest([{stamps}])),' ',1) AS stamp FROM amzn_settlements" if sc else + "SELECT NULL::VARCHAR AS account,NULL::VARCHAR AS settlement_id,NULL::VARCHAR AS currency," + "NULL::VARCHAR AS stamp WHERE false") settlement = (f"SELECT COALESCE({txt(sc, '_connection_id')},'legacy') AS account," f"{txt(sc, 'settlement_id')} AS settlement_id,{txt(sc, 'adjustment_id')} AS event_id," f"{txt(sc, 'order_item_code')} AS item_id,{txt(sc, 'order_id')} AS order_id," f"{txt(sc, 'sku')} AS sku,NULL::VARCHAR AS asin,{_sql_day(_text(sc, 'posted_date'))} AS day," + f"split_part({txt(sc, 'posted_date')},' ',1) AS stamp,{txt(sc, '_report_row_number')} AS capture," f"({principal}) AS principal,CASE WHEN {principal} THEN {_num(sc, 'quantity_purchased')} END AS units," f"CASE WHEN {principal} THEN -{_num(sc, 'amount')} END AS money," f"CASE WHEN {principal} THEN upper({txt(sc, 'currency')}) END AS currency" @@ -2038,55 +2065,155 @@ def _refund_relations(cur): context = f"TRY_CAST({_text(tc, 'item_contexts')} AS JSON)" quantity = (f"(SELECT sum(TRY_CAST(json_extract_string(j.value,'$.quantityShipped') AS DOUBLE)) " f"FROM json_each({context}) j WHERE json_extract_string(j.value,'$.contextType')='ProductContext')") + # Finance names the item principal Principal or OurPricePrincipal. Two different amounts are a conflict. + named, priced = _num(tc, FINANCE_PRINCIPAL), _num(tc, FINANCE_OUR_PRICE) finance = (f"SELECT COALESCE({txt(tc, '_connection_id')},'legacy') AS account," f"{txt(tc, 'transaction_id')} AS event_id,{_num(tc, 'item_position')} AS position," + f"upper({txt(tc, 'transaction_status')}) AS status," f"{identifier('related_identifiers', 'SETTLEMENT_ID')} AS settlement_id," f"{identifier('related_identifiers', 'REFUND_ID')} AS refund_id," f"{identifier('item_related_identifiers', 'ORDER_ADJUSTMENT_ITEM_ID', True)} AS item_id," f"{txt(tc, 'amazon_order_id')} AS order_id,{txt(tc, 'sku')} AS sku,{txt(tc, 'asin')} AS asin," f"{_sql_day(_text(tc, 'posted_date'))} AS day,{quantity} AS context_units," - f"-{_num(tc, FINANCE_PRINCIPAL)} AS money," - f"upper({txt(tc, FINANCE_PRINCIPAL + '_currency_code')}) AS currency" + f"-COALESCE({named},{priced}) AS money," + f"upper(CASE WHEN {named} IS NOT NULL THEN {txt(tc, FINANCE_PRINCIPAL + '_currency_code')} " + f"ELSE {txt(tc, FINANCE_OUR_PRICE + '_currency_code')} END) AS currency," + f"COALESCE({named}!={priced},false) AS principal_conflict" + (" FROM amzn_transactions" if tc else " WHERE false") + (f" WHERE lower(trim({_text(tc, 'transaction_type')}))='refund'" if tc else "")) - return f"""settlement_capture AS (SELECT DISTINCT * FROM ({settlement}) source), + deferred = "COALESCE(status IN ('DEFERRED','DEFERRED_RELEASED'),false)" + return f"""settlement_header AS (SELECT account,settlement_id, + CASE WHEN min(currency)=max(currency) THEN min(currency) END AS currency, + bool_or(TRY_CAST(split_part(stamp,'.',1) AS INTEGER)>12) FILTER (WHERE dotted) + AND NOT COALESCE(bool_or(TRY_CAST(split_part(stamp,'.',2) AS INTEGER)>12) FILTER (WHERE dotted),false) + AS day_first + FROM (SELECT *,COALESCE(regexp_full_match(stamp,'[0-9]{{1,2}}[.][0-9]{{1,2}}[.][0-9]{{4}}'),false) AS dotted + FROM ({header}) h) stamps GROUP BY account,settlement_id), + settlement_capture AS (SELECT DISTINCT s.account,s.settlement_id,s.event_id,s.item_id,s.order_id,s.sku,s.asin, + COALESCE(s.day,CASE WHEN h.day_first THEN CAST(TRY_STRPTIME(s.stamp,'%d.%m.%Y') AS DATE) END) AS day, + s.capture,s.principal,s.units,s.money, + CASE WHEN s.principal AND (s.currency IS NULL OR h.currency IS NULL OR s.currency=h.currency) + THEN COALESCE(s.currency,h.currency) END AS currency + FROM ({settlement}) s LEFT JOIN settlement_header h ON h.account=s.account AND h.settlement_id=s.settlement_id), settlement_events AS (SELECT account,settlement_id,event_id,item_id,min(order_id) AS order_id, min(sku) AS sku,min(asin) AS asin, - CASE WHEN count(*) FILTER (WHERE principal)>1 THEN NULL - ELSE COALESCE(min(day) FILTER (WHERE principal),min(day)) END AS day, - min(units) AS units,min(money) AS money,min(currency) AS currency, - count(*) FILTER (WHERE principal)>1 AS conflict FROM settlement_capture - GROUP BY account,settlement_id,event_id,item_id), + CASE WHEN count(*) FILTER (WHERE principal)>count(DISTINCT COALESCE(capture,'')) FILTER (WHERE principal) + OR min(day) FILTER (WHERE principal)!=max(day) FILTER (WHERE principal) + OR count(day) FILTER (WHERE principal)0 THEN min(day) FILTER (WHERE principal) + WHEN min(day)=max(day) AND count(day)=count(*) THEN min(day) END AS day, + CASE WHEN count(day)=count(*) THEN min(day) END AS day_low, + CASE WHEN count(day)=count(*) THEN max(day) END AS day_high, + sum(units) FILTER (WHERE principal) AS units, + CASE WHEN count(money) FILTER (WHERE principal)=count(*) FILTER (WHERE principal) + THEN sum(money) FILTER (WHERE principal) END AS money, + CASE WHEN min(currency) FILTER (WHERE principal)=max(currency) FILTER (WHERE principal) + AND count(currency) FILTER (WHERE principal)=count(*) FILTER (WHERE principal) + THEN min(currency) FILTER (WHERE principal) END AS currency, + count(*) FILTER (WHERE principal)=0 AS principal_absent, + COALESCE(count(*) FILTER (WHERE principal)>count(DISTINCT COALESCE(capture,'')) FILTER (WHERE principal) + OR min(day) FILTER (WHERE principal)!=max(day) FILTER (WHERE principal),false) AS conflict, + count(*) FILTER (WHERE principal)=0 OR bool_or(units IS NULL OR NOT isfinite(units) OR units<=0 + OR units!=trunc(units)) FILTER (WHERE principal) AS unit_bad + FROM settlement_capture GROUP BY account,settlement_id,event_id,item_id), finance_capture AS (SELECT DISTINCT * FROM ({finance}) source), - finance_events AS (SELECT account,event_id,position,min(settlement_id) AS settlement_id, + finance_items AS (SELECT account,event_id,position,min(status) AS status,min(settlement_id) AS settlement_id, min(refund_id) AS refund_id,min(item_id) AS item_id,min(order_id) AS order_id, min(sku) AS sku,min(asin) AS asin,CASE WHEN count(*)>1 THEN NULL ELSE min(day) END AS day, min(context_units) AS context_units, - min(money) AS money,min(currency) AS currency,count(*)>1 AS conflict + min(money) AS money,min(currency) AS currency,count(*)>1 OR bool_or(principal_conflict) AS conflict, + event_id IS NULL OR position IS NULL OR NOT isfinite(position) OR position<0 OR position!=trunc(position) + OR min(sku) IS NULL AND min(asin) IS NULL OR min(refund_id) IS NULL OR min(item_id) IS NULL AS identity_bad FROM finance_capture GROUP BY account,event_id,position), - refund_candidates AS ( - SELECT 'settlement' AS source,account,settlement_id,event_id,item_id,order_id,sku,asin,day,units,money,currency, - conflict,event_id IS NULL OR item_id IS NULL OR settlement_id IS NULL OR sku IS NULL AS identity_bad, - units IS NULL OR NOT isfinite(units) OR units<=0 OR units!=trunc(units) AS unit_bad + finance_groups AS (SELECT *,COALESCE(event_key,'transaction'||chr(31)||COALESCE(event_id,'')||chr(31) + ||COALESCE(CAST(position AS VARCHAR),'')) AS group_key, + event_key IS NOT NULL AND {deferred} AND bool_or(NOT {deferred}) OVER (PARTITION BY account,event_key) AS superseded + FROM (SELECT *,CASE WHEN refund_id IS NOT NULL AND item_id IS NOT NULL + THEN 'refund'||chr(31)||refund_id||chr(31)||item_id END AS event_key FROM finance_items) keyed), + finance_deferrals AS (SELECT account,group_key, + COALESCE(min(round(total,2))!=max(round(total,2)) OR count(total) NOT IN (0,count(*)),false) AS disagree + FROM (SELECT account,group_key,event_id,sum(money) AS total FROM finance_groups + GROUP BY account,group_key,event_id) transactions GROUP BY account,group_key), + finance_events AS (SELECT g.account,min(event_id) AS event_id,min(position) AS position, + min(settlement_id) AS settlement_id,min(refund_id) AS refund_id,min(item_id) AS item_id, + min(order_id) AS order_id,min(sku) AS sku,min(asin) AS asin, + CASE WHEN count(day)=count(*) AND min(day)=max(day) THEN min(day) END AS day, + CASE WHEN count(day)=count(*) THEN min(day) END AS day_low, + CASE WHEN count(day)=count(*) THEN max(day) END AS day_high, + sum(context_units) AS context_units,sum(money) AS money, + CASE WHEN min(currency)=max(currency) AND count(currency)=count(money) THEN min(currency) END AS currency, + count(money)=0 AS principal_absent, + COALESCE(bool_or(conflict) OR min(event_id)!=max(event_id) OR min(settlement_id)!=max(settlement_id) + OR min(order_id)!=max(order_id) OR min(sku)!=max(sku) OR min(day)!=max(day) + OR min(currency)!=max(currency) OR bool_or(d.disagree),false) AS conflict, + bool_or(identity_bad) AS identity_bad, + COALESCE('refund '||min(refund_id)||' item '||min(item_id), + 'Finance transaction '||min(event_id)||' position '||CAST(min(position) AS BIGINT), + 'an unnamed Finance refund') AS label + FROM finance_groups g JOIN finance_deferrals d USING(account,group_key) + WHERE NOT superseded GROUP BY g.account,g.group_key), + refund_sources AS ( + SELECT 'settlement' AS source,account,settlement_id,event_id,item_id,order_id,sku,asin,day_low,day_high, + day,units,money,currency,principal_absent,conflict,event_id IS NULL OR item_id IS NULL OR settlement_id IS NULL OR sku IS NULL + AS identity_bad, + COALESCE('refund '||event_id||COALESCE(' item '||item_id,''),'order '||order_id||COALESCE(' item '||item_id,''), + 'settlement '||settlement_id,'an unnamed settlement refund') AS label, + unit_bad FROM settlement_events UNION ALL - SELECT 'finance',account,settlement_id,event_id,item_id,order_id,sku,asin,day,NULL,money,currency, - conflict,event_id IS NULL OR position IS NULL OR NOT isfinite(position) OR position<0 - OR position!=trunc(position) OR sku IS NULL AND asin IS NULL, true FROM finance_events), + SELECT 'finance',account,settlement_id,refund_id,item_id,order_id,sku,asin,day_low,day_high, + day,NULL,money,currency,principal_absent,conflict,identity_bad,label,true FROM finance_events), + refund_keyed AS (SELECT *,CASE WHEN event_id IS NOT NULL AND item_id IS NOT NULL + THEN event_id||chr(31)||item_id END AS event_key,row_number() OVER () AS serial FROM refund_sources), + refund_candidates AS ( + SELECT CASE WHEN min(source)!=max(source) THEN 'both' ELSE min(source) END AS source,account, + min(settlement_id) AS settlement_id,min(event_id) AS event_id,min(item_id) AS item_id, + min(order_id) AS order_id,min(sku) AS sku,min(asin) AS asin, + CASE WHEN count(day)=count(*) AND min(day)=max(day) THEN min(day) END AS day, + CASE WHEN count(day_low)=count(*) THEN min(day_low) END AS day_low, + CASE WHEN count(day_high)=count(*) THEN max(day_high) END AS day_high, + COALESCE(min(units) FILTER (WHERE source='settlement'),min(units)) AS units, + min(money) AS money,min(currency) AS currency,bool_and(principal_absent) AS principal_absent, + COALESCE(bool_or(conflict) OR count(*) FILTER (WHERE source='settlement')>1 + OR count(*) FILTER (WHERE source='finance')>1 OR min(settlement_id)!=max(settlement_id) + OR min(order_id)!=max(order_id) OR min(sku)!=max(sku) OR min(day)!=max(day) + OR min(round(money,2))!=max(round(money,2)) OR min(currency)!=max(currency) + OR bool_or(principal_absent)!=bool_and(principal_absent),false) AS conflict, + bool_or(identity_bad) AS identity_bad, + COALESCE(bool_or(unit_bad) FILTER (WHERE source='settlement'),bool_or(unit_bad)) AS unit_bad, + min(label) AS label + FROM refund_keyed GROUP BY account,COALESCE(event_key,'capture'||serial)), {_refund_overlap_relations()}""" +# A refund without a principal component adds no item principal. A principal without money or currency is bad. +_REFUND_MONEY_BAD = ("NOT principal_absent AND (currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') " + "OR money IS NULL OR NOT isfinite(money) OR money<0)") + + +def _refund_fold(): + """Fold settlement refunds plus Finance refunds absent from that report, without the event merge.""" + return """refund_fold AS (SELECT day,money FROM settlement_events + UNION ALL SELECT day,money FROM finance_events f WHERE NOT EXISTS (SELECT 1 FROM settlement_events s + WHERE s.account=f.account AND s.event_id=f.refund_id AND s.item_id=f.item_id))""" + + def _refund_totals_sql(first, last, group=False, *, windows=False): """Aggregate only event-grain refunds after source capture deduplication.""" return ("SELECT " + ("window_id," if windows else "key," if group else "") + "COALESCE(sum(units),0) AS refunded," "sum(money) AS refund_cost,min(currency) AS refund_currency,count(DISTINCT currency) AS currencies," - "count(*) FILTER (WHERE currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') " - "OR money IS NULL OR NOT isfinite(money) OR money<0) AS money_bad," + "count(*) FILTER (WHERE " + _REFUND_MONEY_BAD + ") AS money_bad," "count(*) FILTER (WHERE identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad," "count(*) FILTER (WHERE unit_bad) AS unit_bad,count(*) FILTER (WHERE source='finance') AS finance_units," - "count(*) FILTER (WHERE overlap) AS overlaps FROM refund_events " - + ("JOIN windows ON day BETWEEN first AND last OR day IS NULL GROUP BY window_id" if windows else - f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL" + (" GROUP BY key" if group else ""))) + "count(*) FILTER (WHERE overlap) AS overlaps," + "(list(label ORDER BY label) FILTER (WHERE identity_bad OR conflict OR overlap OR day IS NULL))[1:2] " + "AS identity_refunds,(list(label ORDER BY label) FILTER (WHERE " + _REFUND_MONEY_BAD + "))[1:2] " + "AS money_refunds,list(DISTINCT currency ORDER BY currency) FILTER (WHERE currency IS NOT NULL) " + "AS refund_currencies FROM refund_events " + + ("JOIN windows ON day BETWEEN first AND last OR day IS NULL AND (day_low IS NULL " + "OR day_low<=last AND day_high>=first) GROUP BY window_id" if windows else + f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL AND (day_low IS NULL " + f"OR day_low<=DATE '{last}' AND day_high>=DATE '{first}')" + (" GROUP BY key" if group else ""))) def _rate_values(row, ready): @@ -2442,10 +2569,13 @@ def validate(tenant_key, *, start=None, end=None, st=None, now=None, user=None, "sum(CASE WHEN unit_bad THEN 1 ELSE 0 END) AS unit_bad," "sum(CASE WHEN source='finance' THEN 1 ELSE 0 END) AS finance_units," "sum(CASE WHEN overlap THEN 1 ELSE 0 END) AS overlaps," - "sum(CASE WHEN currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') " + "sum(CASE WHEN principal_absent THEN 0 WHEN currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') " "OR money IS NULL OR NOT isfinite(money) OR money<0 THEN 1 ELSE 0 END) AS money_bad ") - expected_money = _mirror_one(cur, "WITH " + _refund_relations(cur) + " SELECT " + refund_projection - + "FROM refund_events WHERE day BETWEEN ? AND ? OR day IS NULL", [s, e]) + expected_money = _mirror_one(cur, "WITH " + _refund_relations(cur) + "," + _refund_fold() + " SELECT " + + refund_projection.replace("sum(money) AS refund_cost,", "", 1) + + ",(SELECT sum(money) FROM refund_fold WHERE day BETWEEN ? AND ?) AS refund_cost " + "FROM refund_events WHERE day BETWEEN ? AND ? OR day IS NULL AND (day_low IS NULL " + "OR day_low<=? AND day_high>=?)", [s, e, s, e, e, s]) ready = _rate_availability(cur) returned, shipped = independent[0] + independent[1], independent[2] expected_facts = {"returned": returned, "shipped": shipped, **expected_money} @@ -2457,14 +2587,15 @@ def validate(tenant_key, *, start=None, end=None, st=None, now=None, user=None, if keys: independent_base = ("WITH ir AS (" + sources[0] + " UNION ALL " + sources[1] + "), ish AS (" + sources[2] + "), " + _refund_relations(cur) + ", money AS (SELECT key," + refund_projection - + "FROM refund_events WHERE (day BETWEEN ? AND ? OR day IS NULL) AND key=ANY(?) GROUP BY key), " + + "FROM refund_events WHERE (day BETWEEN ? AND ? OR day IS NULL AND (day_low IS NULL " + "OR day_low<=? AND day_high>=?)) AND key=ANY(?) GROUP BY key), " "ret AS (SELECT COALESCE(sku,asin,'No SKU') AS key,sum(units) AS returned FROM ir " "WHERE day BETWEEN ? AND ? GROUP BY 1), ship AS (SELECT COALESCE(sku,asin,'No SKU') AS key," "sum(units) AS shipped FROM ish WHERE day BETWEEN ? AND ? GROUP BY 1) " "SELECT k.key,COALESCE(ret.returned,0) AS returned,COALESCE(ship.shipped,0) AS shipped," "money.* EXCLUDE(key) FROM (SELECT unnest(?) AS key) k LEFT JOIN ret USING(key) " "LEFT JOIN ship USING(key) LEFT JOIN money USING(key) ORDER BY k.key") - raw = _mirror_page(cur, independent_base, [s, e, keys, s, e, s, e, keys], 0, len(keys)) + raw = _mirror_page(cur, independent_base, [s, e, e, s, keys, s, e, s, e, keys], 0, len(keys)) actual_rows = {row["key"]: row for row in page["rows"]} for row in raw: wanted = _independent_rate_values(row, ready) @@ -3899,6 +4030,49 @@ def _dashboard_rates(cur, first, last, independent): return _dashboard_rate_windows(cur, [(first, last)], independent)[0] +def _refund_names(names, count): + """Name the first two unproved refunds and count the rest.""" + shown = [name for name in names or () if name][:2] + more = int(count or 0) - len(shown) + return (' for ' + ', '.join(shown) + (f' and {more} more' if more > 0 else '')) if shown else '' + + +def _dashboard_rate_reasons(row, ready): + """Name each missing operand without treating an unproved quantity as zero.""" + labels = {'returns': 'FBA returns', 'shipped': 'FBA shipments', 'merchantReturns': 'merchant returns', + 'merchantShipped': 'merchant shipments', 'settlements': 'settlement refunds', 'transactions': 'Finance refunds'} + missing = {name: 'Unknown: ' + label + ' source is missing or incomplete.' + for name, label in labels.items() if not ready[name]} + shipment = [missing[name] for name in ('shipped', 'merchantShipped') if name in missing] + if ready['merchantShipped'] and row.get('merchant_unplaced'): + shipment.append('Saved orders do not prove merchant shipment dates and quantities for this period.') + returns = [missing[name] for name in ('returns', 'merchantReturns') if name in missing] + refunds = [missing[name] for name in ('settlements', 'transactions') if name in missing] + identity = [] + named = _refund_names(row.get('identity_refunds'), row.get('identity_bad')) + if row.get('overlaps'): + identity.append('Refund reports overlap without a proven shared event identity' + named + '.') + elif row.get('identity_bad'): + identity.append('Refund event identity is missing or conflicting' + named + '.') + units = [] + if row.get('finance_units'): + units.append('Finance records do not prove refunded quantities.') + elif row.get('unit_bad'): + units.append('Settlement refund quantities are missing or invalid.') + money = [] + if row.get('money_bad'): + money.append('Refund money or currency is missing or invalid' + + _refund_names(row.get('money_refunds'), row.get('money_bad')) + '.') + elif (row.get('currencies') or 0) > 1: + money.append('Refunds in this period use more than one currency: ' + + ', '.join(row.get('refund_currencies') or ()) + '.') + denominator = [] if shipment or row.get('shipped') else ['No shipped units are recorded for this period.'] + return {'returnRate': ' '.join(returns + shipment + denominator), + 'refundRate': ' '.join(refunds + shipment + identity + units + denominator), + 'refundCost': ' '.join(refunds + identity + money) or 'No refund amount and currency are established for this period.', + 'returns': ' '.join(returns)} + + def _dashboard_rate_windows(cur, windows, independent=False, *, ready=None): """Fold total, comparison and monthly windows in one query over complete source captures.""" bounds = ','.join(f"({i},DATE '{a.isoformat()}',DATE '{b.isoformat()}')" for i, (a, b) in enumerate(windows)) @@ -3907,22 +4081,31 @@ def _dashboard_rate_windows(cur, windows, independent=False, *, ready=None): cur, MFN_TABLE, 'return_request_date', 'return_quantity', merchant_sku, 'return_reason') shipments = _return_source(cur, SHIPPED_TABLE, 'shipment_date', 'quantity') prefix = f"""WITH windows(window_id,first,last) AS (VALUES {bounds}), - returns AS ({physical}),shipments AS ({shipments}),{_refund_relations(cur)},""" + returns AS ({physical}),shipments AS ({shipments}),{_refund_relations(cur)}, + merchant_shipments AS ({_merchant_shipment_bounds(cur)}), + merchant_scope AS (SELECT window_id,count(unplaced) AS merchant_unplaced + FROM windows LEFT JOIN merchant_shipments ON + (purchase_day IS NULL OR purchase_day<=last) AND (ship_by_day IS NULL OR ship_by_day>=first) + GROUP BY window_id),""" if independent: # Separate scalar source folds do not reuse the serving aggregate or formatter. - sql = prefix + """r AS (SELECT window_id,COALESCE(sum(units),0) AS returned + sql = prefix + _refund_fold() + """,r AS (SELECT window_id,COALESCE(sum(units),0) AS returned FROM windows LEFT JOIN returns ON day>=first AND day<=last GROUP BY window_id), s AS (SELECT window_id,COALESCE(sum(units),0) AS shipped FROM windows LEFT JOIN shipments ON day>=first AND day<=last GROUP BY window_id), - f AS (SELECT window_id,COALESCE(sum(units),0) AS refunded,sum(money) AS refund_cost, + fold AS (SELECT window_id,sum(money) AS refund_cost FROM windows JOIN refund_fold + ON day>=first AND day<=last GROUP BY window_id), + f AS (SELECT window_id,COALESCE(sum(units),0) AS refunded, min(currency) AS refund_currency,count(DISTINCT currency) AS currencies, - count(*) FILTER (WHERE money IS NULL OR money<0 OR NOT isfinite(money) - OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}')) AS money_bad, + count(*) FILTER (WHERE NOT principal_absent AND (money IS NULL OR money<0 OR NOT isfinite(money) + OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}'))) AS money_bad, bool_or(identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad, bool_or(unit_bad) AS unit_bad FROM refund_events JOIN windows - ON day>=first AND day<=last OR day IS NULL GROUP BY window_id) - SELECT r.*,s.shipped,f.* EXCLUDE(window_id) FROM r JOIN s USING(window_id) - LEFT JOIN f USING(window_id) ORDER BY window_id""" + ON day>=first AND day<=last OR day IS NULL AND (day_low IS NULL OR NOT (day_highlast)) + GROUP BY window_id) + SELECT r.*,s.shipped,f.* EXCLUDE(window_id),fold.refund_cost,merchant_unplaced FROM r JOIN s USING(window_id) + LEFT JOIN f USING(window_id) LEFT JOIN fold USING(window_id) JOIN merchant_scope USING(window_id) + ORDER BY window_id""" else: sql = prefix + """events AS ( SELECT day,units AS returned,0.0 AS shipped FROM returns @@ -3931,16 +4114,18 @@ def _dashboard_rate_windows(cur, windows, independent=False, *, ready=None): COALESCE(sum(shipped),0) AS shipped FROM windows LEFT JOIN events ON day BETWEEN first AND last GROUP BY window_id), refunds AS (""" + _refund_totals_sql(None, None, windows=True) + """) - SELECT quantities.*,refunds.* EXCLUDE(window_id) FROM quantities - LEFT JOIN refunds USING(window_id) ORDER BY window_id""" - ready = _rate_availability(cur) if ready is None else ready + SELECT quantities.*,refunds.* EXCLUDE(window_id),merchant_unplaced FROM quantities + LEFT JOIN refunds USING(window_id) JOIN merchant_scope USING(window_id) ORDER BY window_id""" + ready = _rate_availability(cur, windowed=True) if ready is None else ready result = cur.execute(sql) names = [field[0] for field in result.description] rows = [] while batch := result.fetchmany(MAX_PAGE_SIZE): rows.extend(dict(zip(names, row)) for row in batch) formatter = _independent_rate_values if independent else _rate_values - return [formatter(row, ready) for row in rows] + return [{**formatter(row, {**ready, 'merchantShipped': ready['merchantShipped'] and not row['merchant_unplaced']}), + **({} if independent else {'figureReasons': _dashboard_rate_reasons(row, ready)})} + for row in rows] def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independent=False): @@ -3950,7 +4135,7 @@ def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independen total, previous, points, reasons, reasons_value = {}, {}, [], [], None if allowed: with _screen_reader(tenant_key, st) as (cur, _present): - ready = _rate_availability(cur) + ready = _rate_availability(cur, windowed=True) months = list(_dashboard_windows(first, last)) windows = [(first, last), (first - (last - first) - dt.timedelta(days=1), first - dt.timedelta(days=1)), *months] @@ -3983,9 +4168,7 @@ def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independen for _, values in points): value, series = None, [] reason = ('Current reader access needed' if not allowed else - 'Unknown: refund amount or currency is missing, mixed or unproved' if key == 'refundCost' and value is None else - 'Unknown: refunded units or shipment-date denominator is unproved' if key == 'refundRate' and value is None else - 'Unknown: return source or shipment-date denominator is unproved' if value is None else None) + total.get('figureReasons', {}).get(key or 'returns') if value is None else None) prior = previous.get(key) if key else None if key == 'refundCost' and previous.get('currency') != unit: prior = None diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index 39b589c0e910b9cf19daaa89a20abe89f8cda117..4b24785fbe0a37cce0ddab6a99acc85e4b7d3292 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -628aafcc10a75e7c0c3c83fd3e1713b2ca6c61192848508870843a2fe127afda +3c55ff309f3750e257b4bda0b2f42e7ebbeac02633a94e0c4477a7ed5fccd9f7 diff --git a/web/s71_frontend_contracts.py b/web/s71_frontend_contracts.py index 60117b9e22e0430bb4c9bfa0badd60a91006f130..a990a73d8bd57b1e2851bbb2849e6493f76f473e 100644 --- a/web/s71_frontend_contracts.py +++ b/web/s71_frontend_contracts.py @@ -5,52 +5,55 @@ import re WEB = Path(__file__).resolve().parent -def catalog_host_faults(grid: str, loader: str, map_view: str, catalog: str) -> list[str]: +def catalog_host_faults(grid: str, loader: str, map_view: str, catalog: str, host: str) -> list[str]: + # S85-T53: Application resolution lives in customer-grid/applicationHost.ts (`host`). checks = { - "registry discriminator": 'applicationTemplateSurface(addressedDefinition) === "catalog"' in grid - and 'applicationTemplateSurface(resolvedApplication) === "catalog"' in grid, + "registry discriminator": 'applicationTemplateSurface(addressedDefinition) === "catalog"' in host + and 'applicationTemplateSurface(resolvedApplication) === "catalog"' in host, "Map controls withheld before and after Catalog resolution": '(!applicationRef || (resolvedApplicationMatches && applicationTemplateSurface(resolvedApplication) !== "catalog"))' in grid - and '(!isApplicationSurface || !!addressedDefinition)' in grid, + and '(!hostOwnsDisplay || !!addressedDefinition)' in grid, "only registry Routing owns Routes": 'resolvedApplicationMatches && applicationTemplateSurface(resolvedApplication) === "route"' in grid, "Catalog has no route event listener": bool(re.search(r'useEffect\(\(\) => \{\s*if \(!routeApplicationSurface\) return;\s*const openRoute', grid)), - "cold App lookup precedes row admission": grid.find('const addressedDefinition') < grid.find('} = useCustomerData(scope,') + "cold App lookup precedes row admission": -1 < grid.find('useApplicationHost(') < grid.find('} = useCustomerData(scope,') + and grid.find('const addressedDefinition') < grid.find('} = useCustomerData(scope,') # S78-T17 (C4): a Competitor watch App also reads no source rows. and bool(re.search(r'const legacyRowsEnabled = !isPagedOdooApplication && ' r'(?:!isCompetitorWatchApplication &&\s+)?' - r'\(!isApplicationSurface \|\| !!addressedDefinition\)', grid)) - and 'rowsEnabled: legacyRowsEnabled' in grid, + r'\(!isApplicationSurface \|\| !!addressedDefinition\)', host)) + and 'rowsEnabled: legacyRowsEnabled' in host and 'rowsEnabled: applicationHost.rowsEnabled' in grid, "Catalog disables legacy rows and subscriptions": 'options.rowsEnabled === false ? EMPTY_APPLICATION_PAYLOAD' in loader and 'const suppliedPayload = embeddedPayload !== undefined' in loader and 'if (suppliedPayload || STREAMLIT_COMPONENT) return; // the caller owns refresh' in loader, - "Catalog mount independent of a legacy field": '(isPagedOdooApplication || fieldByKey.get(lockedKey))' in grid + "Catalog mount independent of a legacy field": '(applicationHost.paged || fieldByKey.get(lockedKey))' in grid and 'if (!props.field)' in map_view, "typed page supplies the real drawer": 'onCatalogData={acceptCatalogData}' in grid - and 'if (isPagedOdooApplication) return catalogData?.rows ?? []' in grid + and 'if (applicationHost.paged) return applicationHost.pagedRead?.rows ?? []' in grid and 'onData={props.onCatalogData}' in map_view and 'onData?.(data.fields, data.allRows)' in catalog, - "Routing shares the paged source admission": 'const isPagedOdooApplication = isCatalogApplication || isRoutingApplication' in grid - and '(isRoutingApplication || !serverWindowed)' in grid, + "Routing shares the paged source admission": 'const isPagedOdooApplication = isCatalogApplication || isRoutingApplication' in host + and '(applicationHost.routing || !serverWindowed)' in grid, } return [name for name, ok in checks.items() if not ok] def catalog_host_scan() -> int: paths = ['src/customer-grid/CustomerGrid.tsx', 'src/customer-grid/useCustomerData.ts', - 'src/customer-grid/MapView.tsx', 'src/apps/catalog/CatalogApp.tsx'] + 'src/customer-grid/MapView.tsx', 'src/apps/catalog/CatalogApp.tsx', 'src/customer-grid/applicationHost.ts'] sources = [(WEB / path).read_text(encoding='utf-8') for path in paths] faults = catalog_host_faults(*sources) for name in faults: print('FAIL S71 Catalog host: ' + name) controls = [ (0, 'applicationTemplateSurface(resolvedApplication) !== "catalog"', 'true'), + (0, '(!hostOwnsDisplay || !!addressedDefinition)', 'true'), (0, 'resolvedApplicationMatches && applicationTemplateSurface(resolvedApplication) === "route"', 'true'), (0, 'if (!routeApplicationSurface) return;\n const openRoute', 'const openRoute'), - (0, 'rowsEnabled: legacyRowsEnabled', 'rowsEnabled: true'), - (0, '(isPagedOdooApplication || fieldByKey.get(lockedKey))', 'fieldByKey.get(lockedKey)'), + (0, 'rowsEnabled: applicationHost.rowsEnabled', 'rowsEnabled: true'), + (0, '(applicationHost.paged || fieldByKey.get(lockedKey))', 'fieldByKey.get(lockedKey)'), (0, 'onCatalogData={acceptCatalogData}', ''), (1, 'options.rowsEnabled === false ? EMPTY_APPLICATION_PAYLOAD', 'false ? EMPTY_APPLICATION_PAYLOAD'), (3, 'onData?.(data.fields, data.allRows)', 'onData?.(data.fields, data.rows)'), - (0, 'const isPagedOdooApplication = isCatalogApplication || isRoutingApplication', + (4, 'const isPagedOdooApplication = isCatalogApplication || isRoutingApplication', 'const isPagedOdooApplication = isCatalogApplication'), ] for index, old, new in controls: @@ -63,7 +66,7 @@ def catalog_host_scan() -> int: else: print('ok NC S71 Catalog host: ' + old) if not faults: - print('PASS S71 Catalog host: 9 contracts, 9 removal controls') + print('PASS S71 Catalog host: 9 contracts, 10 removal controls') return int(bool(faults)) @@ -165,7 +168,7 @@ def mounted_catalog_host(base_url: str, out_dir: Path, negative_controls: bool = applied = [] if case in ('settings-discriminator', 'route-listener', 'legacy-loader'): old, new = ('applicationTemplateSurface(resolvedApplication) !== "catalog"', 'true') if case == 'settings-discriminator' \ - else ('rowsEnabled: legacyRowsEnabled', 'rowsEnabled: true') if case == 'legacy-loader' \ + else ('rowsEnabled: applicationHost.rowsEnabled', 'rowsEnabled: true') if case == 'legacy-loader' \ else (r'if \(!routeApplicationSurface\) return;\s*const openRoute', 'const openRoute') def mutate(route): response = route.fetch() diff --git a/web/src/App.tsx b/web/src/App.tsx index 0dcffc3f4790d9b8a0842285b64f4ff08b9d5da4..a2d52c368511ddddb139001e212311c048eb9752 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -11,6 +11,7 @@ const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); import { OverlayProvider } from "./customer-grid/OverlaySurface"; import { isStreamlitComponent } from "./customer-grid/hostBridge"; import Shell from "./shell/Shell"; +import { installViewAs } from "./settings/viewAs"; // HOSTED = either Streamlit path (the component iframe, or the legacy injected // payload). There the surrounding app owns all chrome and this tree renders the @@ -25,6 +26,11 @@ export const HOSTED = isStreamlitComponent() || typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; +// S85-T157: the member view transport. Module scope runs before `main.tsx` installs edge retry +// and the activity log, so both wrap it, and before Shell's first `me()`. Entering or leaving the +// view reloads the whole frame: no cache, route or request outlives the identity that made it. +if (!HOSTED) installViewAs(() => window.location.reload()); + export default function App() { if (HOSTED) { return ( diff --git a/web/src/account/UsagePage.tsx b/web/src/account/UsagePage.tsx index 65ecffce73264c8f627c38c57a3aeb07e224cd1f..6cec5bc03b34d65319fd9c2ca60fddd3ee8b2fdf 100644 --- a/web/src/account/UsagePage.tsx +++ b/web/src/account/UsagePage.tsx @@ -121,6 +121,30 @@ export function UsageSurfaceRows({ usage }: { usage: Usage }) { ); } +/** Both account and Settings render the server's allowance and over flag here. */ +export function AiUsageCard({ usage }: { usage: Usage }) { + return <> +
+
+ {usage.scope === "tenant" ? "This workspace" : "You"}, this week +
+
+ {fmt.int(usage.total)} + of {fmt.int(usage.allowance)} tokens +
+ +
+ Week {usage.week}. Resets {fmt.date(usage.resets)}. + {fmt.int(usage.calls)} {usage.calls === 1 ? "call" : "calls"} +
+
+ {usage.over ?

+ Over allowance. AI remains available. +

: null} + {usage.note ?

{usage.note}

: null} + ; +} + export function UsageView({ load, scope, @@ -175,52 +199,7 @@ export function UsageView({ ) : ( <> -
- {/* ⛔ THE METER SAYS WHOSE USAGE IT IS, IN WORDS, and that is not decoration. The - allowance is a WORKSPACE number; the total beside it is this account's own - unless the scope says otherwise. A meter that shows a personal figure against a - shared limit without saying so is a ratio nobody can read correctly, and it is - the one way this page can be wrong while every number on it is right. */} -
- - {load.usage.scope === "tenant" ? "This workspace" : "You"}, this week - -
-
- - {fmt.int(load.usage.total)} - - - of {fmt.int(load.usage.allowance)} tokens - -
- -
- {/* Both facts are the SERVER'S: the week it counted and the date it rolls over. - A week boundary computed in the browser can disagree with the bucketing that - produced the number above it, and then the page states a period nobody - measured. */} - - Week {load.usage.week}. Resets {fmt.date(load.usage.resets)}. - - - {fmt.int(load.usage.calls)} {load.usage.calls === 1 ? "call" : "calls"} - -
-
- - {/* ⛔ R9's whole point, in one line: over the allowance it STILL WORKS. The bar is - red, nothing is cut off, and the sentence says which of those two is true so - nobody has to infer it from a colour. */} - {load.usage.over ? ( -

- This workspace is over its weekly allowance. Nothing is cut off; the AI surfaces - keep working. -

- ) : null} - - {/* The server's own sentence, present only when something really was unmeasured. */} - {load.usage.note ?

{load.usage.note}

: null} +

By surface

diff --git a/web/src/apiContract.ts b/web/src/apiContract.ts index 53b759478f974ae06249c6afffa7bef0c2d2db34..52a0bce1e2042faf43a25cd6f3262db9ed97af8f 100644 --- a/web/src/apiContract.ts +++ b/web/src/apiContract.ts @@ -73,6 +73,11 @@ export const TENANT_HEADER = "X-AIOS-Tenant"; * An opaque per-person fingerprint; must match `api/deps.py::_PRINCIPAL_HEADER`. */ export const PRINCIPAL_HEADER = "X-AIOS-Principal"; +/** S85-T157: an administrator's read-only member view. `settings/viewAs.ts` sends it on every API + * call while the tab's mode is on, and on none after exit. Must match `api/deps.py`, which refuses + * every method but GET and HEAD while it is present. */ +export const VIEW_AS_HEADER = "X-View-As"; + /** The tenant this frame BOOTED for; `null` until the first authenticated * answer names one. Module-level on purpose — it must outlive every component * that could be unmounted by the very reset it triggers. */ diff --git a/web/src/apps/AppDashboard.tsx b/web/src/apps/AppDashboard.tsx index bdbb5638de02359ef334f79eff00fd93952f583e..52408bb36106361cc2843a7d3101ea2a61671175 100644 --- a/web/src/apps/AppDashboard.tsx +++ b/web/src/apps/AppDashboard.tsx @@ -1,12 +1,11 @@ import { useEffect, useState } from "react"; import type { ReactNode } from "react"; import { LogoLoader } from "../ui/LogoLoader"; -import { DashboardPeriodPicker } from "../customer-grid/DashboardPeriodPicker"; +import { DashboardPeriodPicker } from "../customer-grid/seam"; import { API_V1, CREDENTIALS, checkTenant, currentTenant, UNAUTHORIZED_EVENT, signal } from "../apiContract"; import { fetchWithDeadline } from "../net/deadline"; import { Kpi, KpiStrip, TrendChart, validSeries, finite, formatMeasure } from "../viz/kit"; import type { TrendChartProps, ChartKind, KpiProps, ChartPoint, ChartSeries } from "../viz/kit"; -import "../customer-grid/amazonPeriodPicker.css"; import "../styles/sdlc81/dashboard-kit.css"; export interface DashboardFigure { diff --git a/web/src/apps/AppGrid.tsx b/web/src/apps/AppGrid.tsx index 8c52b9ad4ac419bb7ed8d290ccaf9dcae5ca98ae..89fafbb4dcce6d22a7c63a45dd7f9e148662b3a8 100644 --- a/web/src/apps/AppGrid.tsx +++ b/web/src/apps/AppGrid.tsx @@ -1,49 +1,24 @@ -import { lazy, Suspense, useEffect, useState } from "react"; import type { ReactNode } from "react"; -import { fetchWorkspace } from "../customer-grid/apiBridge"; -import type { SurfaceScope } from "../customer-grid/apiBridge"; -import { OverlayProvider } from "../customer-grid/OverlaySurface"; -import type { Row } from "../customer-grid/types"; -import { LogoLoader } from "../ui/LogoLoader"; +import { AppTableGrid, OverlayProvider } from "../customer-grid/seam"; +import type { AppTableHost, Row } from "../customer-grid/seam"; import "../styles/sdlc82/app-grid.css"; -const CustomerGrid = lazy(() => import("../customer-grid/CustomerGrid")); - interface AppGridProps { table: string | null | undefined; onOpenRow: (row: Row) => void; empty?: ReactNode; } -function TableGrid({ table, onOpenRow }: { table: string; onOpenRow: (row: Row) => void }) { - const [admission, setAdmission] = useState<{ ready: boolean; message?: string }>({ ready: false }); - - useEffect(() => { - let current = true; - let failure: string | undefined; - // Read metadata through the shared identity guard before mounting a source-backed grid. - void fetchWorkspace(table as SurfaceScope, { fieldsOnly: true, onStatus: (status) => { - if (status.state === "error") failure = status.message; - } }).then((workspace) => { - if (!current) return; - setAdmission(workspace ? { ready: true } - : { ready: false, message: failure ?? "This table is unavailable." }); - }); - return () => { current = false; }; - }, [table]); - - if (admission.message) return

{admission.message}

; - if (!admission.ready) return ; - return }> - { if (row) onOpenRow(row); }} /> - ; +/** One App table through the C2 seam. The grid draws History in its toolbar slot. */ +function HostedTable({ table, onOpenRecord }: { table: string; onOpenRecord?: AppTableHost["data"]["onOpenRecord"] }) { + return ; } export function AppGrid({ table, onOpenRow, empty = "No records yet." }: AppGridProps) { return
- {table ? + {table ? { if (row) onOpenRow(row); }} /> :
{empty}
}
; diff --git a/web/src/apps/AppRail.tsx b/web/src/apps/AppRail.tsx index 52622d18674a4c32802b7cb6847816d981547207..78069588a36c18d8acc5d79d9613b8bbb70ebc80 100644 --- a/web/src/apps/AppRail.tsx +++ b/web/src/apps/AppRail.tsx @@ -1,6 +1,6 @@ import { useEffect, useRef, useState } from "react"; import { API_V1, CREDENTIALS, UNAUTHORIZED_EVENT, checkTenant, currentTenant, signal } from "../apiContract"; -import { appPathHash } from "../customer-grid/dataPath"; +import { appPathHash } from "../customer-grid/seam"; import { fetchWithDeadline } from "../net/deadline"; import PanelToggleIcon from "../shell/PanelToggleIcon"; import { useNarrowRailFold } from "../shell/railHeader"; diff --git a/web/src/apps/CollectionPage.tsx b/web/src/apps/CollectionPage.tsx index 66518abf311e763ca73ffd9931013548991b1b37..2704f22bb74127a7fef61afd9f3ab599a0547e48 100644 --- a/web/src/apps/CollectionPage.tsx +++ b/web/src/apps/CollectionPage.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { API_V1, CREDENTIALS, UNAUTHORIZED_EVENT, checkTenant, signal as raiseEvent } from "../apiContract"; -import { BodyPortal, useOverlayLayer } from "../customer-grid/OverlaySurface"; +import { BodyPortal, useOverlayLayer } from "../customer-grid/seam"; import { useAppFullscreen } from "../ui/useAppFullscreen"; import "./collection.css"; import "../styles/sdlc72/app-fullscreen.css"; diff --git a/web/src/apps/account-health/AppView.tsx b/web/src/apps/account-health/AppView.tsx index 2249c123dff1fe8b9465188230e9442cecead53f..3acef98b168bdf09547f1352e151d97a12ed5dd3 100644 --- a/web/src/apps/account-health/AppView.tsx +++ b/web/src/apps/account-health/AppView.tsx @@ -1,6 +1,6 @@ import { useCallback, useEffect, useId, useRef, useState } from "react"; import { API_V1, CREDENTIALS } from "../../apiContract"; -import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; +import { BodyPortal, useOverlayLayer } from "../../customer-grid/seam"; import { useEscapeClose } from "../../ui/useEscapeClose"; import { AppTabs } from "../AppTabs"; import { LineChart } from "../../viz/kit"; @@ -385,68 +385,71 @@ function CategoryPopup({ metric, appId, onClose }: { metric: Metric; appId: stri {category && {count}} -

Amazon sends no record list for this category

- {load.phase === "loading" && } - {load.phase === "error" && { - panel.current?.querySelector("[data-category-close]")?.focus(); - setAttempt((n) => n + 1); - }} />} - {load.phase === "refused" &&

{load.message}

} - {category && <> -
+
+

Amazon sends no record list for this category

+ {category && <> +
+
Status
+
Amazon's target
{category.target || "No target"}
+
+
+
Reporting window + {category.window.from && category.window.to + ? <>{" to "} + + : "No data"} +
+ {category.sellerCentralUrl && Open in Seller Central} +
+ } +
+
+ {load.phase === "loading" && } + {load.phase === "error" && { + panel.current?.querySelector("[data-category-close]")?.focus(); + setAttempt((n) => n + 1); + }} />} + {load.phase === "refused" &&

{load.message}

} + {category && <>

{category.count === 0 ? "No violations reported" : "Record list unavailable"}

-
-
-
Status
-
Amazon's target
{category.target || "No target"}
-
-
-
Reporting window - {category.window.from && category.window.to - ? <>{" to "} - - : "No data"} -
- {category.sellerCentralUrl && Open in Seller Central} -
- } - - {category && -
} +
+ {category &&
-
-

Flagged listings

Related -
-

{category.sourceNote}

- {category.related.length ?
    - {category.related.map((listing, index) => { - const severity = listing.severity.toLowerCase().replace(/_/g, " "); - const severityTone = /error|critical|fatal/.test(severity) ? "is-risk" - : /warn/.test(severity) ? "is-near" : "is-quiet"; - return
  • -
    -
    - {listing.sku && Sku: {listing.sku}} - {listing.asin && Asin: {listing.asin}} - {!listing.sku && !listing.asin && Listing} +
    +

    Flagged listings

    Related +
    +

    {category.sourceNote}

    + {category.related.length ?
      + {category.related.map((listing, index) => { + const severity = listing.severity.toLowerCase().replace(/_/g, " "); + const severityTone = /error|critical|fatal/.test(severity) ? "is-risk" + : /warn/.test(severity) ? "is-near" : "is-quiet"; + return
    • +
      +
      + {listing.sku && Sku: {listing.sku}} + {listing.asin && Asin: {listing.asin}} + {!listing.sku && !listing.asin && Listing} +
      + {severity && {severity[0].toUpperCase() + severity.slice(1)}}
      - {severity && {severity[0].toUpperCase() + severity.slice(1)}} -
    -

    {listing.issue || "No issue description"}

    -
  • ; - })} -
:
- {relatedIncomplete ? "Related listings unavailable" : "No flagged listings"} -
} -
- } +

{listing.issue || "No issue description"}

+ ; + })} + :
+ {relatedIncomplete ? "Related listings unavailable" : "No flagged listings"} +
} + } + diff --git a/web/src/apps/actions/ActionCadence.tsx b/web/src/apps/actions/ActionCadence.tsx index dd6c8d53703ddd94774238cdc78ef4bb48f261a5..0f3103bf612484ea808e59fcb47a30b7a1ad04f6 100644 --- a/web/src/apps/actions/ActionCadence.tsx +++ b/web/src/apps/actions/ActionCadence.tsx @@ -1,6 +1,6 @@ import { Component, useCallback, useEffect, useId, useRef, useState } from "react"; import type { ComponentType, ReactNode } from "react"; -import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; +import { BodyPortal, useOverlayLayer } from "../../customer-grid/seam"; import { ActionsError, listActionCadence, subscribeActions } from "./actionsApi"; import type { ActionCadencePage, CadenceAction, SkuAction } from "./actionsApi"; import "../../styles/sdlc81/action-cadence.css"; diff --git a/web/src/apps/actions/SkuActions.tsx b/web/src/apps/actions/SkuActions.tsx index 8458c51f1f574f269965d1a4319fa4a2a465ff31..a458c6b16e83e88aadcb9c84da2a53aab062f5c3 100644 --- a/web/src/apps/actions/SkuActions.tsx +++ b/web/src/apps/actions/SkuActions.tsx @@ -1,6 +1,6 @@ import { useEffect, useId, useRef, useState } from "react"; import type { FormEvent, ReactNode } from "react"; -import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; +import { BodyPortal, useOverlayLayer } from "../../customer-grid/seam"; import { Busy } from "../../ui/Busy"; import { ActionList } from "./ActionList"; import { ActionResults } from "./ActionResults"; diff --git a/web/src/apps/aged-stock/AppView.tsx b/web/src/apps/aged-stock/AppView.tsx index 36c88ca8b54f21fa24ba10c673e633fdd82cc8c2..e404e90baa1a247321015ae490e56d28f0cee8a3 100644 --- a/web/src/apps/aged-stock/AppView.tsx +++ b/web/src/apps/aged-stock/AppView.tsx @@ -5,6 +5,8 @@ import { AppGrid } from "../AppGrid"; import { BarChart, Kpi, KpiStrip } from "../../viz/kit"; import type { ChartPoint, KpiProps } from "../../viz/kit"; import { SkuActions } from "../actions/SkuActions"; +import { ActionList } from "../actions/ActionList"; +import type { SkuAction } from "../actions/actionsApi"; import { ActionCadence } from "../actions/ActionCadence"; import Dashboard, { AgedStockTabs, DashboardLoading } from "./Dashboard"; import "../../styles/sdlc79/aged-stock.css"; @@ -699,12 +701,14 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView const [refreshError, setRefreshError] = useState(""); const [opened, setOpened] = useState(null); const [gridVersion, setGridVersion] = useState(0); + const [listedAction, setListedAction] = useState(null); const changed = useRef(false); const alive = useRef(true); useEffect(() => { alive.current = true; return () => { alive.current = false; }; }, []); useEffect(() => { setTab(SKUS); setOpened(null); + setListedAction(null); }, [appId]); const summaryUrl = url("/summary", { offset: String(monthOffset) }); @@ -788,7 +792,7 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView /** A Grid row click opens the drawer, which reads the item by id. */ const openRow = useCallback((raw: Record) => { const row = gridRow(raw); - if (row) setOpened(row); + if (row) { setListedAction(null); setOpened(row); } }, []); const closeRow = useCallback(() => { setOpened(null); @@ -806,6 +810,7 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView const pickTab = (next: string) => { setTab(next); setOpened(null); + setListedAction(null); }; return
@@ -897,6 +902,9 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView empty={EMPTY_GRID} />
+ { + setOpened(null); setListedAction(action); + }} /> } } @@ -904,5 +912,8 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView {opened && } + {listedAction && setListedAction(null)} />} ; } diff --git a/web/src/apps/alerts/AppView.tsx b/web/src/apps/alerts/AppView.tsx index ea3b60722eccea1011e22dc2bf231d4f57245784..719591c8350b617be97fb3035c189caab83387b3 100644 --- a/web/src/apps/alerts/AppView.tsx +++ b/web/src/apps/alerts/AppView.tsx @@ -1,4 +1,4 @@ -import type { AppViewProps } from "../../customer-grid/viewModes"; +import type { AppViewProps } from "../../customer-grid/seam"; import Alerts from "./Alerts"; export default function AppView({ application, canEdit }: AppViewProps) { diff --git a/web/src/apps/amazon-dashboard/AppView.tsx b/web/src/apps/amazon-dashboard/AppView.tsx index c0b6acd6e9650893f609362f0fddfe37ac615385..45e983b311c6c93ba77fe1e0496d47c76f4470b3 100644 --- a/web/src/apps/amazon-dashboard/AppView.tsx +++ b/web/src/apps/amazon-dashboard/AppView.tsx @@ -1,9 +1,8 @@ import { Component, useEffect, useId, useRef, useState } from "react"; import type { ComponentType, ReactNode } from "react"; import { API_V1, CREDENTIALS, UNAUTHORIZED_EVENT, checkTenant, signal } from "../../apiContract"; -import type { AppViewProps } from "../../customer-grid/viewModes"; -import { dataPathHash } from "../../customer-grid/dataPath"; -import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; +import type { AppViewProps } from "../../customer-grid/seam"; +import { dataPathHash, BodyPortal, useOverlayLayer } from "../../customer-grid/seam"; import { ActionCadence } from "../actions/ActionCadence"; import "../../styles/sdlc81/amazon-dashboard.css"; diff --git a/web/src/apps/amazon-operator/AdsOverview.tsx b/web/src/apps/amazon-operator/AdsOverview.tsx index b1cd04c56fa852d48af2b707af3d65b53f24bc1a..3584c1c2f01a61ec2e3e1448854dfee654f0f04e 100644 --- a/web/src/apps/amazon-operator/AdsOverview.tsx +++ b/web/src/apps/amazon-operator/AdsOverview.tsx @@ -1,6 +1,9 @@ import { createContext, useContext, useEffect, useRef, useState } from "react"; import type { ComponentType, KeyboardEvent } from "react"; -import { ModeIcon } from "../../customer-grid/icons"; +import { SortAscending } from "@phosphor-icons/react"; +import { ModeIcon } from "../../customer-grid/seam"; +import { Popover, SortBuilder, runPipeline } from "../../filter-kit"; +import type { Field, FilterTree, SortSpec } from "../../filter-kit"; import { API_V1, CREDENTIALS } from "../../apiContract"; import ByHour from "./ByHour"; import WeeklySummary from "./WeeklySummary"; @@ -51,6 +54,14 @@ type Comparison = "prior" | "year"; type Sort = "spend" | "sales" | "orders" | "acos" | "campaign"; const CAMPAIGN_COLUMNS: [Sort, string][] = [["campaign", "Campaign"], ["spend", "Spend"], ["sales", "Ad sales"], ["orders", "Orders"], ["acos", "ACoS"]]; +const CAMPAIGN_FIELDS: Field[] = CAMPAIGN_COLUMNS.map(([key, label]) => ({ key, label, + source: "overlay", type: key === "campaign" ? "text" : key === "orders" ? "int" : key === "acos" ? "pct" : "currency" })); + +/** The first sort retains the overview endpoint's existing ordering and cursor contract. */ +export function campaignSortRequest(sorts: SortSpec): { sort: Sort; descending: boolean } { + const first = sorts.find(rule => CAMPAIGN_COLUMNS.some(([key]) => key === rule.colId)); + return { sort: (first?.colId as Sort | undefined) ?? "spend", descending: first ? first.dir === "desc" : true }; +} const NO_ADS = "The Amazon Ads connector is not installed."; const number = (value: unknown) => typeof value === "number" && Number.isFinite(value) ? value : null; const words = (value: unknown) => typeof value === "string" ? value : ""; @@ -140,12 +151,18 @@ function Failure({ state, retry }: { state: Extract; r ; } -function CampaignGrid({ page, sort, descending, busy, onSort, onPage, previous, onHour }: { +function CampaignGrid({ page, sort, descending, sorts, onSorts, busy, onSort, onPage, previous, onHour }: { page: Overview["campaigns"]; sort: Sort; descending: boolean; busy: boolean; + sorts: SortSpec; onSorts: (sorts: SortSpec) => void; onSort: (value: Sort) => void; onPage: (cursor: string | null) => void; previous: string | null | undefined; onHour: (campaign: Campaign) => void; }) { const [selected, setSelected] = useState(""); + const [filters, setFilters] = useState({ conj: "and", nodes: [] }); + const rows = runPipeline({ rawRows: page.items.map((row, index) => ({ pid: index + 1, campaign: row.campaign, + spend: row.spend, sales: row.sales, orders: row.orders, acos: row.acos })), fields: CAMPAIGN_FIELDS, + sorts, filters: filters.nodes, filterConj: filters.conj, search: "", groupBy: null, + collapsed: new Set() }).visibleRows.flatMap(row => row.kind === "data" ? [page.items[row.record.pid - 1]] : []); const ref = useRef(null); const navigate = (event: KeyboardEvent) => { const target = event.target as HTMLElement; @@ -165,7 +182,13 @@ function CampaignGrid({ page, sort, descending, busy, onSort, onPage, previous, : value.toLocaleString(undefined, { maximumFractionDigits: key === "orders" ? 0 : 2 }) + (key === "acos" ? "%" : ""); return
Grid view -

Campaigns

+

Campaigns

+ +
{CAMPAIGN_COLUMNS.map(([key, label]) => @@ -173,7 +196,7 @@ function CampaignGrid({ page, sort, descending, busy, onSort, onPage, previous, aria-sort={sort === key ? descending ? "descending" : "ascending" : "none"}> )} - {page.items.map((row, index) => { + {rows.map((row, index) => { const id = JSON.stringify([row.ad_type, row.connection_id, row.profile_id, row.campaign_id]); return @@ -184,7 +207,7 @@ function CampaignGrid({ page, sort, descending, busy, onSort, onPage, previous, )} ; })} -
#
{index + 1}{format(row[key], key)}
{!page.items.length &&

No campaigns in this period

}
+ {!rows.length &&

No campaigns in this period

}