diff --git a/RELEASES.json b/RELEASES.json index 81ab60306e081379e2d3c62152a259bd9b3eb769..c016502df01c4233937531ee3bacb28410b8ed8c 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,12 @@ { - "current": "v74 (40dceba)", + "current": "v75 (2c51469)", "releases": [ + { + "version": "v75", + "sha": "2c51469", + "date": "2026-09-19", + "subject": "release v75" + }, { "version": "v74", "sha": "40dceba", diff --git a/VERSION b/VERSION index 86c64d99f4dec766401a6d295a87d91df6d320cd..1b32611de31cb8a356b660771150e69fb2ed4a22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v74 (40dceba) +v75 (2c51469) diff --git a/api/ai_review.py b/api/ai_review.py index 03f68bd48c54f4543883f221926d84a4cd135c1f..022952fbd541ea9f04cf8623fc1441ffb3287089 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -174,8 +174,9 @@ def _call_openai(p, model, system, user, timeout): #: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. -#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say -#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. +#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so an administrator can ask WHICH path ran +#: rather than infer it from a requirements file: `GET /api/v1/automations/ai/transport` +#: (`routes_automation.ai_transport`, D-384) serves it beside the installed SDK version. LAST_ANTHROPIC_TRANSPORT = "" diff --git a/api/automation_engine.py b/api/automation_engine.py index 12e563752eefb872c876b18ee09e70ef3ab7040d..7d7e986f3580900519a7523825e5a3e77d946fd8 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -14553,6 +14553,74 @@ def relation_cells(rt, table_key, tables=None): return changes, limits +def related_metric_values(rt, table_key, rows, fields, offer, related, *, team_id=None, + today=None): + """Read-only linked cells at customer, product or declared entity grain. + + The host supplies the same resolved membership as its rows door and its walled catalogue. + A request-local memo never survives a parent, account, permission or tenant change. + """ + from core import measure_resolve, relations + from harness import semantic as sem, windows + import time + + rt.assert_datastore_matches() + today = today or time.strftime('%Y-%m-%d') + by_key = {m['key']: m for m in offer} + out = relations.MeasureAnswer({r['pid']: {f['key']: None for f in fields} for r in rows}) + memo = {} + for field in fields: + spec = field.get('measure') or {} + key = spec.get('key') + if key not in by_key: + continue # The host removes unavailable catalogue entries before serialization. + try: + bounds = windows.resolve(spec.get('window'), today) + if bounds is None: + raise ValueError('The metric date window cannot be resolved. Choose a valid period.') + if table_key == 'customer_data': + errors = [] + values = measure_resolve.column_values( + [field], team_id, [r['pid'] for r in rows], today, today, memo, + on_error=lambda _where, exc: errors.append(exc), related=related) + if errors: + raise ValueError('This metric could not be read for the linked record. Try again after the database is ready.') + out.refusals.extend(getattr(values, 'refusals', ())) + for pid, cells in values.items(): + out[pid].update(cells) + continue + topic = sem.topic_for_grid(table_key) + cache_key = (topic, key, bounds, team_id) + if cache_key not in memo: + memo[cache_key] = sem.entity_measure_values( + topic, [key], date_from=bounds[0], date_to=bounds[1], team_id=team_id, + offer=offer, related=related) + values = memo[cache_key] + markers = getattr(values, 'refusals', ()) + if markers: + out.refusals.extend(dict(marker, field=field['key']) for marker in markers) + continue # Never zero-fill a refused answer or expose its unscoped values. + for row in rows: + identity = row.get('code') if table_key == 'product_data' else row['pid'] + if identity in (None, ''): + raise ValueError('The metric identity cannot be read for this record.') + cell = sem.entity_zero_fill(values.get(identity), [key], offer) + out[row['pid']][field['key']] = cell.get(key) + except Exception as exc: + message = (str(exc) if isinstance(exc, (relations.PinRefusal, sem._EntityMeasureError)) else + 'This metric cannot be read for the linked record. Check its period and source binding, then try again.') + out.refusals.append(dict(code='metric_scope_unavailable', message=message, + path=related.path, field=field['key'], scoped=False)) + # Customer resolvers deliberately retain diagnostics even when their low-level pin refuses. + for marker in out.refusals: + marker['message'] = marker['message'].replace('Values are unscoped.', 'Values are unavailable.').replace( + 'Metric values are unscoped.', 'Metric values are unavailable.') + for cells in out.values(): + if marker.get('field') in cells: + cells[marker['field']] = None + return out + + def related_product_metric_values(rt, customer_id, products, fields, offer, *, team_id=None, agent=None, today=None): """Read-only customer × real product metrics; the caller supplies walled server rows. diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index 7d340dd7df920ddac75041612c607103275f5ac3..c6303734a0a2887e154bbebabe30991ef1df36fd 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -94,6 +94,137 @@ def assert_safe_status(payload): return payload +def first_load_status(tenant_key, rt=None): + """Writer-side progress helper; request status uses sync_state's read-only cursor.""" + from harness import amazon_store + return assert_safe_status(amazon_store.first_load_status(tenant_key, rt=rt)) + + +def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): + """Each Amazon table's sync state, or None when this mirror cannot be read. + + Read only: never _connect_tenant_mirror or first_load_status, which issue DDL. + Missing bookkeeping is unknown, not a measured zero. First-load numbers describe the + whole connection, because that is the scope of the producer's durable progress marker. + """ + import json + from datetime import datetime, timezone + import duckdb + from harness import amazon_store as az + from routes_odoo_tables import _sync_entities, stuck_verdict + + def stamp(value): + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + return (parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)).astimezone(timezone.utc) + except (ValueError, TypeError, OverflowError): + return None + + def iso(value): + return value.isoformat().replace("+00:00", "Z") if value else None + + cadence = cadence_seconds if cadence_seconds is not None else 1800 + try: + path = az._tenant_mirror_path(tenant_key, rt=rt) + if not path.is_file(): + return assert_safe_status(None) + # ⛔⛔ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES + # `ConnectionException: Can't open a connection to same database file with a different + # configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global + # analytical mirror (`datastore.DB_PATH`), so the swallowed refusal below turned this whole + # screen into a permanent "sync status is unknown" on exactly the deployment the ticket is + # for. Measured on staging 2026-09-14, W60-T35. `datastore.ro_con()` is the shipped + # per-thread READ cursor on that same instance, and it takes no lock against the writer. + from harness import datastore as _ds + if path.resolve() == _ds.DB_PATH.resolve(): + # `ro_cursor()` is a cursor on the instance this process ALREADY holds: no second + # connection, no lock contention with the writer, and no DDL. Deliberately NOT + # `ro_con()`, which refuses until `ready()` and would report the whole screen as + # unknown whenever the Odoo backfill is mid-flight, and NOT `datastore.connect()`, + # which issues `CREATE TABLE IF NOT EXISTS`. + entities = _sync_entities(_ds.ro_cursor(), cadence) + else: + with duckdb.connect(str(path), read_only=True) as cur: + entities = _sync_entities(cur, cadence) + except Exception: # noqa: BLE001 + # No exception text: paths, provider payloads and credentials never become status copy. + return assert_safe_status(None) + if entities is None: + return assert_safe_status(None) + + clock = stamp(now) if now is not None else datetime.now(timezone.utc) + states, bad_ticks = {}, set() + for table in az.TABLES: + tick = entities.get(az.entity_key("sync_tick_" + table)) + if tick: + updated = stamp(tick.get("updatedAt")) + if updated is None or tick.get("phase") not in ("attempted", "done", "running"): + bad_ticks.add(table) + else: + states[table] = {"state": tick["phase"], "updated_at": iso(updated)} + # An unreadable input cannot support a cadence promise for any table in the rotation. + plan = az.sync_plan(states, now=clock) if not bad_ticks and clock is not None else None + first_load, load_unknown = None, "" + load = entities.get(az.entity_key(az.FIRST_LOAD_STATUS_KEY)) + if load: + try: + progress = json.loads(load.get("cursor") or "") + remaining, eta = progress["remaining"], progress["etaSeconds"] + if (type(remaining) is not int or not 0 <= remaining <= 2**53 - 1 + or type(eta) is not int or not 0 <= eta <= 2**53 - 1 + or (load.get("phase") == "done" and remaining > 0)): + raise ValueError("Invalid first-load progress") + if remaining > 0: + first_load = {"remaining": remaining, "etaSeconds": eta, + "cause": az.first_load_eta_cause(remaining), + "needsAttention": load.get("phase") == "failed"} + except (ValueError, TypeError, KeyError, OverflowError, RecursionError): + load_unknown = "First-load progress is unknown because its saved marker could not be read. Retry sync to refresh it." + elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"): + load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass." + + answer = {} + for table in az.TABLES: + landed = entities.get(az.entity_key(table)) or {} + tick = entities.get(az.entity_key("sync_tick_" + table)) or {} + watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {} + rows = landed.get("rows") + if type(rows) is not int or rows < 0: + rows = None + successes = [when for marker in (landed, tick, watermark) + if marker.get("phase") == "done" + and (when := stamp(marker.get("updatedAt"))) is not None] + stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or "" + if tick.get("phase") == "attempted": + # The shared stall helper calls an active phase 'running'; Amazon calls it 'attempted'. + updated = stamp(tick.get("updatedAt")) + stuck, stuck_cause = stuck_verdict( + "running", iso(updated), cadence, + now=clock.replace(tzinfo=None) if clock else None) + reason = plan["deferred"].get(table) if plan else None + # Only the planner's explicit due stamp is a due date. Minimum stagger spacing is not + # a promise about when a later resident tick will select this table. + next_due = stamp(reason.partition("next due ")[2]) if reason else None + unknown = [load_unknown] if load_unknown else [] + if rows is None: + unknown.append("This container has no readable row count for this table.") + if plan is None: + unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.") + answer[table] = { + "rowsLoaded": rows, + "lastSuccess": iso(max(successes)) if successes else None, + "inProgress": tick.get("phase") == "attempted" and stuck is False, + "deferred": {"reason": reason} if reason else None, + "nextDue": iso(next_due), + "firstLoad": first_load, + "stuck": stuck, "stuckCause": stuck_cause, + "unknownCause": " ".join(unknown), + } + return assert_safe_status(answer) + + #: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's #: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and #: its seven databases are enabled at the provider boundary rather than masquerading as Odoo @@ -340,7 +471,7 @@ def main(argv): return 1 print("\nreport types this connector will pull (owner ruling R4):") for table, spec in REPORT_TYPES.items(): - rep = spec.get("report") or "NO CONFIRMED REPORT FLAVOR (see O5)" + rep = spec.get("report") or spec.get("report_discover") or "NO CONFIRMED REPORT FLAVOR (see O5)" print(f" {table:<16} {rep}") if "--probe-report" in argv: diff --git a/api/main.py b/api/main.py index 12d7d9584cda8402443c941fa63690a4996c1fd5..9d16bc42d09383f9b5d6cb869d91433de969fc85 100644 --- a/api/main.py +++ b/api/main.py @@ -20,6 +20,7 @@ session's tenant claim resolves through `harness.runtime.get_runtime`, which is is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. """ +import hmac import os import sys from pathlib import Path @@ -273,7 +274,7 @@ def _view_revision_unverifiable(request: Request, exc: view_audit.RevisionUnveri @app.get("/api/health") -def health(): +def health(challenge: str = ""): """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not reveal anything about the deployment beyond "the process is up". No version, no tenant list, no config: a health endpoint is the one URL every scanner finds first. @@ -282,7 +283,17 @@ def health(): release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind a session — and the authoritative copy is the `VERSION` file in the Space repo, which - `deploy_web.space_version()` reads without needing the app to be up at all.""" + `deploy_web.space_version()` reads without needing the app to be up at all. + + W-265 adds a possession challenge for the deployer. It reveals only whether this process was + born after one high-entropy, single-deploy nonce was stamped; it never returns the nonce, + version, Space id, or configuration. A normal liveness request keeps the exact + ``{"ok": true}`` response shape. + """ + if challenge: + expected = os.environ.get("AIOS_DEPLOY_NONCE") or "" + return {"ok": True, + "deployReady": bool(expected and hmac.compare_digest(challenge, expected))} return {"ok": True} @@ -398,6 +409,7 @@ app.include_router(routes_charts.router) # and that router, so this is the one mount in this block that crosses no fence. It is still # written HERE rather than at the end, for the reason the block below spells out at length. app.include_router(routes_sheet.router) +import routes_module_index; app.include_router(routes_module_index.router) # W61-T20 / C6: App and Agents folder index # ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the # FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses. # ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own @@ -766,6 +778,158 @@ def _pull_amazon(why): return False +class _SyncNowRefused(Exception): + """An application-authored, credential-free sentence for a Sync now receipt.""" + + +def _sync_now_failure(cause): + """Classify actual provider failures without returning vendor text or exception reprs.""" + import re + text = str(cause).lower() + causes = ( + (("reduce the amount of data", "response too large"), + "The provider refused the pull because the response contains too much data."), + (("no ad accounts",), "The Meta token cannot access any ad accounts."), + (("max_pages", "page cap"), "The pull stopped at the provider page limit."), + (("sandbox",), "Amazon sync requires a production credential; this credential is for the sandbox."), + (("deletion", "retention"), "Amazon sync is blocked by an outstanding data deletion requirement."), + (("no stored", "no odoo credentials", "access_token is not set"), + "This tenant has no usable credential for the connector."), + (("enablement document", "capability unavailable"), + "The connector database settings could not be read."), + (("throttl", "rate limit", "too many requests"), + "The provider rate limit stopped the sync. Retry after the provider delay."), + (("timed out", "timeout"), "The sync failed because the provider request timed out."), + (("permission", "access denied", "unauthorized", "invalid oauth", "invalid token", + "expired", "authentication"), "The provider refused the connector credential or its permissions."), + (("duplicate key", "primary key", "constraint"), + "The mirror rejected rows because they violate a database constraint."), + (("connection", "network", "name resolution"), + "The sync could not connect to the provider or tenant database."), + # A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises + # DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error": + # "the loader is broken" and "this server has never held these tables" are different + # operator actions, and only the second one is true here. + (("does not exist", "no such table", "catalog error"), + "This server's mirror does not hold this connector's tables yet, so there was nothing " + "to update. Run the connector's first load on the server that owns this tenant."), + ) + for needles, sentence in causes: + if any(needle in text for needle in needles): + return sentence + status = re.search(r"\bhttp\s+(\d{3})\b", text) + if status: + return f"The provider refused the sync request with HTTP status {status.group(1)}." + if isinstance(cause, TypeError): + return "The connector failed because its provider call used incompatible arguments." + if isinstance(cause, (KeyError, ValueError)): + return "The connector could not read an expected value in the sync response." + return "The connector encountered an unexpected error while processing the sync response." + + +def _run_sync_requests(): + """Drain Sync now intents on the resident resync thread, the mirror's single writer. + + D-29 / intake F6: HTTP handlers only queue intents. Provider calls stay here, before the + resident projection pass. A failed request cannot discard another tenant's queued work. + """ + from core import connector_sync + odoo_requested = False + for receipt in connector_sync.take_sync_requests(): + provider, tenant = receipt["provider"], receipt["tenant"] + ok, detail = False, "" + first_load_progress, continuing = None, False + try: + from harness import runtime, datastore + rt = runtime.get_runtime(tenant) + if rt.key != tenant: + raise _SyncNowRefused("The sync runtime does not match the requesting tenant.") + # These two loaders use a process-global mirror. Never repoint it under readers. + # Amazon's writer opens an explicit tenant file and does not need this restriction. + if provider in ("odoo", "meta_ads") and ( + rt.datastore_path.resolve() != datastore.DB_PATH.resolve() or + datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()): + raise _SyncNowRefused( + "This tenant's mirror is not resident on this server. Sync requires its tenant worker.") + if provider == "odoo": + import core.odoo as odoo + import odoo_relational as rel + if rt.odoo_paused() or rel.frozen(rt): + raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.") + source = rt.odoo_source() + if source is None: + raise _SyncNowRefused("This tenant has no Odoo credential. Connect Odoo before syncing.") + odoo_requested = True + previous_client = getattr(odoo._tlocal, "client", None) + previous_context = getattr(odoo._tlocal, "tenant_ctx", None) + try: + # Raw mirror pulls use the resolved tenant credential and a dedicated + # transport. Do not borrow a request's cached XML-RPC client. + odoo._tlocal.client = odoo.OdooClient(creds=source._creds) + odoo._tlocal.tenant_ctx = None + report = datastore.sync_all(log=lambda *a, **k: None) + finally: + odoo._tlocal.client = previous_client + odoo._tlocal.tenant_ctx = previous_context + if not report: + raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.") + if any(row.get("phase") != "live" for row in report.values()): + raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.") + problems = [] + elif provider == "amazon_sp": + from harness import amazon_store as amazon + if not amazon.credential(tenant, rt=rt): + raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.") + first_load = amazon.needs_first_load(tenant, rt=rt) + options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {} + report = amazon.sync(tenant, rt=rt, scheduled=False, **options, log=lambda *a, **k: None) + from connectors_amazon import assert_safe_status + first_load_progress = assert_safe_status(report.get("firstLoad")) + continuing = report.get("state") == "partial" + problems = list(report.get("problems") or []) + list(report.get("audit_problems") or []) + tables = report.get("tables") or {} + if not problems and not tables and not continuing: + raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.") + if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()): + raise _SyncNowRefused("All Amazon databases are disabled. Enable a database before syncing.") + elif provider == "meta_ads": + from harness import meta_store as meta + if not meta.token(rt): + raise _SyncNowRefused("Meta Ads has no access token. Connect Meta Ads before syncing.") + report = meta.sync(tenant, log=lambda *a, **k: None, insights_days=7) + problems = list(report.get("problems") or []) + if any(row.get("capped") for row in (report.get("tables") or {}).values()): + problems.append("The pull hit the page cap.") + if not problems and not report.get("tables"): + raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.") + else: + raise _SyncNowRefused("This connector has no Sync now runner.") + if continuing: + ok = True + detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; " + f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.") + if problems: + detail += " " + _sync_now_failure(problems[0]) + elif problems: + detail = _sync_now_failure(problems[0]) + else: + ok, detail = True, "Sync finished." + except _SyncNowRefused as exc: + detail = str(exc) + except Exception as exc: # noqa: BLE001 + detail = _sync_now_failure(exc) + if first_load_progress is not None: + connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, + first_load=first_load_progress) + else: + connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail) + if continuing and ok: + # Finish first: the desk refuses a second in-flight intent. Return to the resident + # loop so its Odoo mirror pass runs before this continuation is claimed. + connector_sync.request_sync("amazon_sp", tenant, requested_by="first load") + return odoo_requested + + def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False): """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. @@ -1320,6 +1484,7 @@ def _store_resync_loop(): Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a refreshed pool re-reads the freshly synced store.""" import time as _t + from core import connector_sync passes = 0 last_relational_success_at = None last_meta_fingerprint = None @@ -1388,10 +1553,13 @@ def _store_resync_loop(): # stored fields such as customer_rank can change without partner write_date advancing, # so staging served a provably stale customer population for its first hour. Subsequent # passes still sleep at the freshly resolved cadence, and manual still skips only Odoo. + # The wait is now interruptible so an administrator's Sync now wakes this writer. if passes: - _t.sleep(_every) + connector_sync.sync_wake_event().wait(_every) + connector_sync.sync_wake_event().clear() passes += 1 RESYNC_STATE["passes"] = passes + _odoo_requested = _run_sync_requests() sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes) RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()) RESYNC_STATE["lastPassOk"] = _err is None @@ -1444,6 +1612,10 @@ def _store_resync_loop(): _amzn_previous = last_amzn_fingerprint if _meta_reason is None else None _reason = _resync_rebuild_reason(sync_report, delete_report, last_relational_success_at, _meta_now) + # The scheduled pass may now be quiet because Sync now just advanced this mirror. + # Its zero-change report cannot suppress projection of the explicit Odoo pull. + if _odoo_requested: + _reason = "sync now" _meta_state = {} if _reason is not None: print(f"[aios-api] odoo relational rebuild required ({_reason})") diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 7e225b28ec364e4d1dce3b04a61056898441670b..45ea700983de24e92d12bcdef4c0681448e9c936 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -1876,6 +1876,9 @@ def authoritative_schema_snapshot(): MODEL_CATALOG_KEY = "odoo_readable_model_catalog" MODEL_CATALOG_VERSION = 1 MODEL_READ_PAGE = 2000 +# Inline rows rewrite the whole store document, so batch publications after the first page. +MODEL_PUBLISH_PAGES = 5 +MODEL_PUBLISH_SECONDS = 10 MODEL_CATALOG_TTL = _dt.timedelta(hours=6) _MODEL_FIELD_TYPES = { "boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency", @@ -3776,7 +3779,7 @@ def _model_cell(value): return value -def _read_model_rows(client, catalogue_row): +def _read_model_rows(client, catalogue_row, *, on_page=None): """Read one explicitly enabled model in stable pages; every call is a read method.""" model = str(catalogue_row.get("model") or "") metadata = catalogue_row.get("fields") or {} @@ -3792,19 +3795,76 @@ def _read_model_rows(client, catalogue_row): continue rows.append({"_id": str(rid), **{name: _model_cell((item or {}).get(name)) for name in fields}}) - if len(page or ()) < MODEL_READ_PAGE: + final = len(page or ()) < MODEL_READ_PAGE + if on_page is not None: + on_page(rows, final) + if final: break offset += len(page) return rows +def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, today, + on_progress): + """Publish a single job's growing prefix; source reads and progress have no row ceiling.""" + ut = _ut() + key, model = choice["key"], choice["model"] + fields, stamp = _model_field_specs(catalogue_row), today or _iso_today() + # A bounded census is never a denominator. Failure to count does not prevent row reads. + try: + total = client.search_count(model, []) + except Exception: # noqa: BLE001 + total = None + if isinstance(total, bool) or not isinstance(total, int) or total < 0: + total = None + + stored_n = len(((rt.get(ut.STORE_KEY) or {}).get(key) or {}).get("rows") or {}) + if total is not None and stored_n and total < stored_n * MAX_SHRINK: + raise Refused(f"{model}: source count is {total} rows against {stored_n} stored. " + "The prior database was preserved. Check Odoo read access and retry sync.") + + written, published_rows, pages_since_publish = {}, 0, 0 + last_publish = _time.monotonic() + + def on_page(rows, final): + nonlocal published_rows, pages_since_publish, last_publish + pages_since_publish += 1 + if (not written or final or pages_since_publish >= MODEL_PUBLISH_PAGES + or _time.monotonic() - last_publish >= MODEL_PUBLISH_SECONDS): + def apply(current): + current = current if isinstance(current, dict) else {} + written[key] = _ensure_table_inplace( + current, key, choice["label"], fields, rows, username, stamp) + from core import connector_sync + current[key]["connector"] = { + **connector_sync.database_identity( + "odoo", connection_for_table_key(key, rt), key, model), + "key": "odoo", "readOnly": True} + return current + + # The caller supplies its fenced runtime. A superseded write must propagate. + rt.update(ut.STORE_KEY, apply, flush="sync") + ut.register_connected(key) + published_rows = written[key]["rows"] + pages_since_publish, last_publish = 0, _time.monotonic() + on_progress(published_rows, total) + + values = _read_model_rows(client, catalogue_row, on_page=on_page) + if total is not None and len(values) < total: + raise Refused("Odoo returned fewer rows than its count. " + "Retry sync to load the remaining rows.") + return {"tables": written, "errors": []} + + def sync_discovered_models(rt, source_connection_id=None, namespace=None, - username="automation", today=None, table_keys=None): - """Materialise only explicitly enabled long-tail models, atomically and tenant-bound. + username="automation", today=None, table_keys=None, on_progress=None): + """Materialise explicitly enabled long-tail models through the tenant's store. Disabled models are absent from this pass rather than represented by an empty row set. That distinction is the preservation guarantee: no updater touches their definition, rows, fields, grants, ownership, or tenant rename, and re-enabling resumes against the same stable key. + A single-database job supplies on_progress and publishes growing prefixes. Other callers + retain the all-model atomic publication and its shrink guards. """ requested = set(table_keys) if table_keys is not None else None if requested is not None and not requested: @@ -3834,6 +3894,10 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None, from core.odoo import get_odoo with source._bound(): # tenant credential + scope client = get_odoo() + if requested is not None and len(requested) == 1 and on_progress is not None: + return _publish_discovered_model( + rt, client, selected[0], by_model[str(selected[0]["model"])], + username=username, today=today, on_progress=on_progress) for choice, key in zip(selected, keys): model = str(choice["model"]) row = by_model[model] @@ -3936,7 +4000,7 @@ def available(rt=None, tenant_key=None, connection_id=None): def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None, - all_connections=False, table_keys=None): + all_connections=False, table_keys=None, on_apply=None): """THE entry point — the store-resync path and the route both call this. ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row @@ -4046,6 +4110,11 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_ # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no # longer built) SAYS SO, with the keys. plan_report = {} + if on_apply is not None: + try: + on_apply() + except Exception: # noqa: BLE001 + pass # Optional status reporting cannot fail a valid plan's publication. written = apply_plan(rt, built, username=username, today=today, report=plan_report, connection_id=ns, retire=requested is None) # The fixed mirror roster above and the discovered long tail deliberately use separate diff --git a/api/providers.py b/api/providers.py index d294400d33431e3b8c3bda9253357544cf451875..587061579818fd20c626e2af424b219a3d5bb8a5 100644 --- a/api/providers.py +++ b/api/providers.py @@ -641,7 +641,8 @@ def anthropic_send(req, timeout=None, use_sdk=None): ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader who cannot tell which path ran cannot tell whether the requirements pin reached the container [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on - `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a + `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for the admin view + `GET /api/v1/automations/ai/transport` (D-384), and a check keyed to it passes on a stale value written by an earlier check in the same process. `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — diff --git a/api/routes_admin.py b/api/routes_admin.py index e13219b90fbc0aac5e00916ac74377ce138fc7cf..3eeeeb6d3cfa051df9c7134f49647a3adb7efcc0 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -1956,10 +1956,15 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non # client, or a copy of a record written before this ticket, must not read as a # revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so # the validator and the wall cannot disagree about what an absent key means. + # D-465: type checked like `hiddenFields`. `bool("false")` is True, so a coerced string + # stored a GRANT the administrator meant as a denial. Absence still grants. + metrics = raw.get("metrics", True) + if not isinstance(metrics, bool): + raise err(400, "bad_perms", f"{key}: metrics must be true or false") out[key] = {"access": bool(raw.get("access", True)), "filter": clean_tree, "hiddenFields": sorted(submitted), - "metrics": bool(raw.get("metrics", True))} + "metrics": metrics} _refuse_unshapeable_bu(out) return out diff --git a/api/routes_automation.py b/api/routes_automation.py index a8d1d21cbc93de2404e89d42e22c3f3697e8ecc1..37a2fe43754480705da8bcaa46e7ff4bf37428b7 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -126,10 +126,27 @@ def _voice_options(session): tables.append(option) except Exception: # noqa: BLE001 - permission/absence. pass - # User tables are permissioned by their own object wall inside automation_tables(). - for table in automation_tables(session).get("tables") or []: - option = _voice_option(table.get("key"), table.get("label") or table.get("key"), - table.get("fields"), table.get("views")) + # User tables are permissioned by their own object wall inside _automation_table_rows(). + # + # ⭐⭐ W62-T17 — THE VIEW LIST IS BOUGHT PER DATABASE AND `_voice_option` THROWS MOST OF IT + # AWAY. That helper returns None unless a database has BOTH a phone-type field AND a saved + # View, so a database with no phone field is dropped whatever its Views say — and resolving + # them cost a `shares.grants` read plus a whole workspace-document read EACH. MEASURED on + # staging: `_table_views` x24 = 6,810 ms of this pass + # (`AutoWiki/waves/wave62/proto/T17-profile.md`). The phone test is free (the field list is + # already in hand), so it is asked FIRST and the view read is spent only where it can matter. + # + # ⚠ THE ANSWER IS IDENTICAL, not merely similar: the skipped databases are exactly the ones + # `_voice_option` returned None for, and `_table_views` is still the single source of the + # view list for every database that survives — ABSENT (None) still reads as "nobody looked" + # and still drops the option, exactly as it did when the list rode on the row. + for table in _automation_table_rows(session, with_views=False): + if not any(isinstance(field, dict) and field.get("key") and field.get("type") == "phone" + for field in (table.get("fields") or [])): + continue + key = table.get("key") + option = _voice_option(key, table.get("label") or key, + table.get("fields"), _table_views(session, key)) if option: tables.append(option) return {"tables": tables} @@ -210,8 +227,23 @@ def _voice_view_input(session, audience): @router.get("/voice-agents") def list_voice_agents(session: Session = Depends(_GATE)): - options = _voice_options(session) + """Every voice agent, each carrying the audience choices its editor offers. + + ⭐⭐ W62-T17 (R12, D-739) — **THE OPTIONS ARE RESOLVED ONLY IF A ROW WILL CARRY THEM**, and on + the store that produced this ticket that is the whole fix. `_voice_options` is the most + expensive read pass in this file (MEASURED at 14,705 ms of this route's 14,748 ms on staging's + Royal Imports, `AutoWiki/waves/wave62/proto/T17-profile.md`), and Royal Imports has NO voice + agents, so every one of those seconds was assembled and then thrown away with the empty list. + The payload is unchanged by construction: with no rows the loop below never ran, so the + options never reached a caller. + + ⛔ THE ORDER IS THE FIX, not a tidy-up — read the rows first, then decide. Reinstating the + eager call "because it is simpler" restores a 14 second answer to an 18 byte payload. This is + the same shape this file already carries twice (W29-T01 and W30-T12, both on `list_automations` + below): work whose result is discarded is still paid for. + """ rows = voice.list_browser_dtos(session.runtime) + options = _voice_options(session) if rows else None for row in rows: row["audienceOptions"] = options return {"voiceAgents": rows} @@ -1081,6 +1113,49 @@ def _discover_default(session, tables): lambda: engine.discover_default_table(session.runtime)) +def automation_ids(session): + """Every automation id `GET /automations` would list, WITHOUT wiring a single DTO. + + ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR `routes_module_index._readable_ids` ASKED FOR IN WRITING: + *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until it + has an id-only accessor."* That route needs the id SET and nothing else — it uses it to drop a + folder placement whose item the caller cannot read — and it was building the whole Agents rail + payload to get it. MEASURED on staging: `GET /api/v1/module-index/agents` spent 1,101 ms of its + 1,180 ms inside `list_automations`, whose `_wire` reads `agent_contracts` and + `automation_agent_actions` once PER AUTOMATION (`AutoWiki/waves/wave62/proto/T17-profile.md`). + + ⛔⛔ THE SET MUST BE `list_automations`' SET EXACTLY, BECAUSE IT IS A WALL. An id this omits is + a tile that silently stops rendering — the worst failure shape on a folder home, because the + placement is still stored and the page simply looks emptier. So all three strata stay: the + stored definitions, the synthetic field agents (C3), and the derived Odoo sync row (C4). Add a + stratum to `list_automations` and it belongs here in the same commit. + + ⛔ AND THE STATEMENTS SEED STAYS, for a reason that is not obvious: `seed_statements_agent` + MINTS a row that `all_definitions` then lists (W35-T37 / R10), once per tenant per process. If + only `list_automations` seeded it, this set would depend on WHICH ROUTE the process happened to + serve first — the Agents home would hide the statements agent's placement until somebody + opened the rail. Same guard, same idempotency key, same never-take-the-rail-down posture. + + ⚠ WHAT IT DELIBERATELY DOES NOT DO is the board-retirement scan. That is hygiene over stored + state (`all_definitions` strips retired board state on every read anyway), it changes no id, + and `list_automations` still runs it on its own first call per tenant. + """ + if session.tenant not in _STATEMENTS_SEEDED: + try: + engine.seed_statements_agent(session.runtime, username=session.uname or "system") + except Exception: # noqa: BLE001 + pass + _STATEMENTS_SEEDED.add(session.tenant) + ids = {str(defn.get("id")) for defn in engine.all_definitions(session.runtime).values() + if isinstance(defn, dict) and defn.get("id")} + ids |= {str(row.get("id")) for row in _field_agent_rows(session) + if isinstance(row, dict) and row.get("id")} + odoo = _odoo_sync_row(session) + if isinstance(odoo, dict) and odoo.get("id"): + ids.add(str(odoo["id"])) + return ids + + def warm_default(rt): """⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT. @@ -1396,7 +1471,22 @@ def _table_views(session, key): @router.get("/automations/tables") def automation_tables(session: Session = Depends(_GATE)): - """The blank databases an automation can target, with their fields. + """The blank databases an automation can target, with their fields. See the helper below.""" + return {"tables": _automation_table_rows(session)} + + +def _automation_table_rows(session, with_views=True): + """The blank databases an automation can target, with their fields — the rows behind + `GET /automations/tables`, as a plain list. + + ⭐⭐ W62-T17 — SPLIT OUT OF THE ROUTE SO ONE IN-PROCESS CALLER CAN DECLINE THE EXPENSIVE HALF. + The route's payload is unchanged (`with_views=True` is the default and the route passes + nothing else); `_voice_options` above is the caller that asks for `with_views=False`, because + it filters on phone fields first and most databases never reach the view list at all. See that + call site for the measurement. + + ⛔ A PLAIN FUNCTION, NOT A SECOND ROUTE. The wall, the C8 migration and the lent document all + stay exactly where they were; only the `views` read became optional. ⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the two can never collide. It exists because the automation editor needs the table+field list to @@ -1442,11 +1532,12 @@ def automation_tables(session: Session = Depends(_GATE)): "type": f.get("type") or "text", "automation": f.get("automation") or None} for f in (t.get("fields") or [])]} - views = _table_views(session, key) - if views is not None: - row["views"] = views # absent when the bucket did not answer — never [] + if with_views: + views = _table_views(session, key) + if views is not None: + row["views"] = views # absent when the bucket did not answer — never [] out.append(row) - return {"tables": out} + return out @router.get("/automations/presets") @@ -1953,6 +2044,26 @@ def agent_author_action(auto_id: str, body: dict = Body(default=None), "actionId": action_id, "agent": agent_id, "notes": notes} +@router.get("/automations/ai/transport") +def ai_transport(session: Session = Depends(_GATE)): + """D-384: which Anthropic path THIS process ran, read-only, administrators only. + + A verifier could prove the SDK path on its own box and never in the container, because the + installed version and `ai_review.LAST_ANTHROPIC_TRANSPORT` had no reader. Two segments under + `/automations/`, so the one-segment `/automations/{auto_id}` below cannot shadow it. + ⚠ `anthropic` is imported HERE, lazily: `verify_no_streamlit.py` boots the API with optional + packages blocked, and it drives this handler with the SDK blocked too.""" + if not session.admin: + raise err(403, "admin_only", "AI transport diagnostics are an admin view") + try: + import anthropic # noqa: PLC0415 + version = str(getattr(anthropic, "__version__", "") or "") + except ImportError: + version = "" + return {"anthropicVersion": version or None, "sdkInstalled": bool(version), + "lastTransport": ai_review.LAST_ANTHROPIC_TRANSPORT or None} + + @router.get("/automations/{auto_id}") def get_automation(auto_id: str, session: Session = Depends(_GATE)): """One automation, by id. diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 6a8733ddb1fb4b886a5b322f89396e26510b4649..c9f462aaf85aa84210b2a98a620bf503a07a4165 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -23,6 +23,8 @@ from __future__ import annotations import os import time import threading +import xmlrpc.client +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from fastapi import APIRouter, Body, Depends @@ -1201,13 +1203,12 @@ def _sync_payload(connector_key, connection_id, row=None): def _odoo_sync_runner(session, connection_id): """Run Odoo through C6 without moving provider imports into the data layer.""" connector_sync.record(session.runtime, "odoo", connection_id, "syncing") - # The current Odoo entry point plans and applies in one call, so the closest truthful seam is - # immediately before it starts applying the tenant mirror into governed database definitions. - connector_sync.record(session.runtime, "odoo", connection_id, "materializing") import odoo_relational result = odoo_relational.refresh(session.runtime, session.tenant, username=session.uname or "automation", - connection_id=connection_id) + connection_id=connection_id, + on_apply=lambda: connector_sync.record( + session.runtime, "odoo", connection_id, "materializing")) tables = result.get("tables") if isinstance(result, dict) else {} databases = sorted(str(key) for key in (tables or {}).keys()) sync_errors = [] @@ -1235,6 +1236,11 @@ def _odoo_sync_runner(session, connection_id): _SYNC_RUNNERS = {"odoo": _odoo_sync_runner} +ODOO_DISCONNECTED_CAUSE = ( + "Odoo is disconnected, so rows cannot load. " + "Reconnect Odoo in the Keychain and this resumes on its own.") + + class _SupersededMaterialization(Exception): pass @@ -1296,15 +1302,29 @@ class _MaterializationRuntime: marker = [self.job["id"], self.job["leaseToken"]] def apply(current): self.valid() - if name == ut.STORE_KEY and (current.get(key) or {}).get("_connectorJob") != marker: + missing = name == ut.STORE_KEY and key not in current + if (name == ut.STORE_KEY and not missing + and (current.get(key) or {}).get("_connectorJob") != marker): raise _SupersededMaterialization() answer = fn(current) + if missing and key in answer: + # A reconciled receipt may precede its definition. Its first schema write + # installs the marker under the same lease and configuration fence. + answer[key]["_connectorJob"] = marker # Lease expiry during a long merge cannot acknowledge a stale write either. self.valid() return answer return self.rt.update(name, apply, flush=flush, **kwargs) +def _odoo_read_access_refused(exc): + """Match only XML-RPC faults with explicit access/read refusal markers.""" + if not isinstance(exc, xmlrpc.client.Fault): + return False + message = str(exc).casefold() + return "security restrictions" in message or "operation: read" in message + + def _run_database_job(rt, candidate): """Run one persisted receipt. Repeating a completed publication reconciles the same rows.""" import core.user_tables as ut @@ -1312,13 +1332,14 @@ def _run_database_job(rt, candidate): # A disconnected source is paused work, not a failed attempt. Preserve the receipt so # reconnect can resume it; the publication fence still rechecks a mid-run disconnect. if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"): + connector_sync.stall(rt, candidate, ODOO_DISCONNECTED_CAUSE) return job = connector_sync.claim(rt, candidate) if job is None: return fenced = _MaterializationRuntime(rt, job) key = job["databaseKey"] - def finish(success=False): + def finish(success=False, cause=""): # The pre-claim check cannot cover a disconnect during source I/O or publication. # Only the current lease may requeue; disable and a newer Save still win. interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen")) @@ -1326,7 +1347,13 @@ def _run_database_job(rt, candidate): # A newer field Save can persist after rows commit but before this acknowledgement. # The old rows remain valid history; they cannot establish the new settings ready. fenced.valid() - return connector_sync.finish(rt, job, success=success, interrupted=interrupted) + # Only provider-authored sentences reach this seam, never raw exception messages. + cause = str(cause or "").replace("\u2014", ",").replace("\u2013", "-") + result = connector_sync.finish( + rt, job, success=success, interrupted=interrupted, cause=cause) + if interrupted: + connector_sync.stall(rt, job, ODOO_DISCONNECTED_CAUSE) + return result stopped = threading.Event() def heartbeat(): @@ -1339,9 +1366,8 @@ def _run_database_job(rt, candidate): try: def install(current): fenced.valid() - if key not in (current or {}): - raise _SupersededMaterialization() - current[key]["_connectorJob"] = [job["id"], job["leaseToken"]] + if key in (current or {}): + current[key]["_connectorJob"] = [job["id"], job["leaseToken"]] return current rt.update(ut.STORE_KEY, install, flush="sync") heartbeat_thread = threading.Thread(target=heartbeat, daemon=True, @@ -1349,6 +1375,15 @@ def _run_database_job(rt, candidate): heartbeat_thread.start() cid = job["connectionId"] source_id = None if cid == "default" else cid + if key not in (rt.get(ut.STORE_KEY, fresh=True) or {}): + schema = rel.publish_database_schema(fenced, key, connection_id=source_id, + username="automation") + if not job["model"]: + # Reconciliation cannot infer a missing definition's model. Now the provider + # has resolved it, queue its real field fingerprint before reading any rows. + fenced.valid() + _queue_odoo_database(rt, key, model=schema["identity"]["model"], changed=True) + return if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS: result = rel.refresh( fenced, getattr(rt, "tenant", None), username="automation", @@ -1358,26 +1393,58 @@ def _run_database_job(rt, candidate): _cid, namespace = rel.connection_namespace(rt, source_id) result = {"discovered": rel.sync_discovered_models( fenced, source_connection_id=source_id, namespace=namespace, - username="automation", table_keys={key})} + username="automation", table_keys={key}, + on_progress=lambda rows, total: connector_sync.progress( + fenced, job, rows=rows, total=total))} discovered = result.get("discovered") or {} published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})} success = (key in published and not result.get("failed") and not result.get("frozen") and not result.get("limits") and not discovered.get("errors")) - finish(success=success) + finish(success=success, cause=result.get("note") or discovered.get("note") or "") except _SupersededMaterialization: # Disable/new Save already invalidated the table marker; never restore it here. finish() - except Exception: + except rel.Refused as exc: + finish(cause=str(exc)) + except Exception as exc: # Raw provider exceptions may carry credentials or source values. Store only safe copy. - finish() + cause = "" + if _odoo_read_access_refused(exc): + cause = (f"Odoo refused to read {job['model']} because the connected Odoo user has no " + "read access, so retrying cannot load rows. " + "Give that user read access in Odoo, or switch this database off.") + finish(cause=cause) finally: stopped.set() if heartbeat_thread: heartbeat_thread.join(timeout=1) +def _reconcile_database_jobs(rt): + """Repair enabled Odoo grids with no receipt using only tenant-local definitions.""" + import core.user_tables as ut + import odoo_relational as rel + grids = (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("grids") or {} + receipts = {job["databaseKey"] for job in connector_sync.jobs(rt, "odoo")} + missing = [key for key, enabled in grids.items() if enabled is True and key not in receipts] + if not missing: + return + definitions = rt.get_projection(ut.STORE_KEY, drop=("rows",)) or {} + for key in missing: + try: + model = str(((definitions.get(key) or {}).get("connector") or {}).get("model") or "") + # The queue's identity validator also decides whether an unknown model is legal. + _queue_odoo_database(rt, key, model=model) + except Exception: + continue # One unresolvable definition cannot starve other queued work. + + def _resume_database_jobs(rt): """One serial pass, also usable by isolated synthetic fixtures.""" + try: + _reconcile_database_jobs(rt) + except Exception: + pass # A reconciliation failure must still fall through to the receipt scan. for candidate in connector_sync.jobs(rt): if candidate.get("provider") == "odoo" and ( candidate.get("state") == "queued" or @@ -1394,6 +1461,52 @@ _materialization_wake = threading.Event() _materialization_stop = threading.Event() _materialization_thread = None _materialization_lifecycle = threading.Lock() +# 3 is a starting point, not tuned: each worker holds an Odoo read and a growing store document in a small Space container. +MATERIALIZATION_WORKERS = 3 +_materialization_pool = None +_materialization_inflight = set() +_materialization_inflight_lock = threading.Lock() + + +def _release_database_job(key): + # Completion and cancellation both release the reservation, including failed claims. + with _materialization_inflight_lock: + _materialization_inflight.discard(key) + + +def _dispatch_database_jobs(rt): + """Submit a resident scan without waiting for source reads or row publication.""" + global _materialization_pool + try: + _reconcile_database_jobs(rt) + except Exception: + pass # Match the serial fixture pass even when reconciliation is unavailable. + for candidate in connector_sync.jobs(rt): + if candidate.get("provider") == "odoo" and ( + candidate.get("state") == "queued" or + candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()): + key = (rt.key, candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) + with _materialization_inflight_lock: + if key in _materialization_inflight: + continue + _materialization_inflight.add(key) + try: + with _materialization_lifecycle: + if _materialization_stop.is_set(): + _release_database_job(key) + return + if _materialization_pool is None: + _materialization_pool = ThreadPoolExecutor( + max_workers=MATERIALIZATION_WORKERS, + thread_name_prefix="connector-materialization-job") + pool = _materialization_pool + future = pool.submit(_run_database_job, rt, candidate) + # Register outside both locks: an already finished future calls back inline. + future.add_done_callback(lambda _future, key=key: _release_database_job(key)) + except Exception: + _release_database_job(key) + # A concurrent shutdown or failed submit leaves the receipt for the next pass. + continue @router.on_event("startup") @@ -1417,7 +1530,7 @@ def _start_database_jobs(): if _materialization_stop.is_set(): break try: - _resume_database_jobs(runtime.get_runtime(tenant)) + _dispatch_database_jobs(runtime.get_runtime(tenant)) except Exception: continue # Durable queued/leased work is retried by the next scan. except Exception: @@ -1435,9 +1548,14 @@ def _start_database_jobs(): @router.on_event("shutdown") def _stop_database_jobs(): + global _materialization_pool with _materialization_lifecycle: _materialization_stop.set() _materialization_wake.set() + pool, _materialization_pool = _materialization_pool, None + if pool is not None: + # Running reads retain their reservation until done; lease expiry permits recovery. + pool.shutdown(wait=False, cancel_futures=True) def _sync_refusals(): @@ -1575,6 +1693,30 @@ def connector_sync_status(connector_key: str, connection_id: str = None, return _sync_payload(connector_key, connection_id, row) +@router.post("/connectors/{connector_key}/sync/now") +def connector_sync_now(connector_key: str, session: Session = Depends(require_session)): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") + try: + return connector_sync.request_sync(connector_key, session.runtime.key, + requested_by=session.uname) + except ValueError: + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None + + +@router.get("/connectors/{connector_key}/sync/now") +def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") + return connector_sync.sync_request(connector_key, session.runtime.key) or { + "provider": connector_key, "tenant": session.runtime.key, "state": "idle", + "requestedAt": None, "startedAt": None, "finishedAt": None, + "requestedBy": "", "detail": "", "accepted": False, + } + + @router.post("/connectors/{connector_key}/sync/retry") def connector_sync_retry(connector_key: str, connection_id: str = None, session: Session = Depends(require_session), database_key: str = None): @@ -1809,6 +1951,7 @@ def _enable_odoo_database(table_key, session, connection_id=None): # materialisation remains separate work. if failure is not None: raise err(*failure) + model = str(choice.get("model") or "").strip() if not exists: reason = _materialisation_failure_reason(result) message = "The setting was enabled, but the database was not created." @@ -1816,11 +1959,14 @@ def _enable_odoo_database(table_key, session, connection_id=None): message = f"{message} {reason}" else: message = f"{message} The field catalogue could not be published." - raise err(409, "database_not_created", message) + try: + _queue_odoo_database(session.runtime, key, model=model) + finally: + # Preserve the original refusal envelope even if the queue store is unavailable. + raise err(409, "database_not_created", message) # Every permission-visible field is available from schema alone. Existing per-database # choices survive a retry; population counts are optional background observations. - model = str(choice.get("model") or "").strip() try: schema_keys = {_odoo_field_key(session.runtime, key, model, name): True for name in result["sourceFields"]} @@ -1832,9 +1978,12 @@ def _enable_odoo_database(table_key, session, connection_id=None): }, flush="sync") except Exception: # noqa: BLE001 - raise err(503, "field_auto_enable_failed", - "The database was created, but its field choices could not be saved. " - "Try again.") + try: + _queue_odoo_database(session.runtime, key, model=model) + finally: + raise err(503, "field_auto_enable_failed", + "The database was created, but its field choices could not be saved. " + "Try again.") _evict_odoo_product_pools(session.runtime) # W56-T36: there is deliberately no navigation projection eviction here. Data's database @@ -2578,11 +2727,25 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None) return _named_databases(rt, is_admin, uname, out) +def _amazon_report_name(table): + """The report type this Amazon table reads, whichever key declares it. + + ⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on + its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter + and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an + empty Source name for settlements on the connector screen, which reads as "this table has no + report" when the truth is "this table is not requested, it is listed". + """ + import connectors_amazon as cx # noqa: PLC0415 - provider owns state + spec = cx.REPORT_TYPES.get(table) or {} + return str(spec.get("report") or spec.get("report_discover") or "") + + def _amazon_databases(rt, is_admin, spec=None): """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM. - Fills the SAME `ConnectorDatabase` shape Odoo does, so the Databases tab needs no client - change: `key`, `label`, `enabled`, `fields`, `excluded`, `sharesFieldsWith`. + Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync` + block. Members retain the shared shape without the field catalogue or sync bookkeeping. ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five of the seven tables have NO published schema, because Amazon publishes artifacts only for its @@ -2611,12 +2774,13 @@ def _amazon_databases(rt, is_admin, spec=None): enabled = enablement.get("databases") or {} cat = amazon_schema.catalogue() tables = cat.get("tables") or {} + states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None # Which OTHER declared tables are filled from the SAME report document. Derived from the # report map so a table added to `REPORT_TYPES` gets this disclosure for free. by_report = {} for name in amazon_schema.PROVENANCE: - rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "") + rep = _amazon_report_name(name) if rep: by_report.setdefault(rep, []).append("ut_amzn_" + name) @@ -2625,7 +2789,7 @@ def _amazon_databases(rt, is_admin, spec=None): key = "ut_amzn_" + name spec = tables.get(name) or {} cols = spec.get("columns") or [] - rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "") + rep = _amazon_report_name(name) fields = [{"key": str(c.get("name") or ""), "label": str(c.get("name") or "").replace("_", " ").capitalize(), "type": "text" if str(c.get("type")) == "VARCHAR" else "number", @@ -2656,6 +2820,7 @@ def _amazon_databases(rt, is_admin, spec=None): # read from Odoo's grid config, and disabling it leaves every stored row and # definition untouched: only this provider config document changes. "enabled": enabled.get(key) is True, + **({"sync": (states or {}).get(name)} if is_admin else {}), "fields": fields, "excluded": excluded, "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]}) @@ -2876,9 +3041,11 @@ def connector_detail(rt, user, uname, is_admin, connector_key): return _safe_connector_status(key, {"overview": overview, "keychain": keychain, "databases": databases, + "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"} + if is_admin and key in connector_sync.providers() else None), "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs", "retryPath": f"/connectors/{key}/sync/retry"} - if is_admin and key in connector_sync.providers() else None), + if is_admin and key == "odoo" else None), # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints. "enablement": enablement, diff --git a/api/routes_customers.py b/api/routes_customers.py index 357010727f17ff95ff21c3912c1caf17efe0a92c..0fca2b2ed7e4cd951fecfa34d28437157f9e2dd6 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -513,6 +513,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = if _ov: ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden} for pid, cells in _ov.items()} + # D-464: and the saved views are a FOURTH wire, served by `/workspace` from `workspace`. + workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden) today = time.strftime("%Y-%m-%d") fields = aios_grid.project_date_recurrences(fields, today=today) @@ -603,7 +605,7 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "team_id": team_id, "routes": normalized_routes, "zones": normalized_zones} -def _payload(session: Session): +def _payload(session: Session, related=None): """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py` referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by construction); see `grid_assembly` for what that fixed. @@ -617,11 +619,18 @@ def _payload(session: Session): """ import aios_grid - g = grid_assembly(session) + g = grid_assembly(session, consume_corrections=not bool(related)) rows = aios_grid.rows_from_pool( g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) from core import route_system rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"]) + from routes_odoo_tables import narrow_related_rows + rows = narrow_related_rows(session, MODULE, rows, related) + from routes_tables import scope_linked_metrics, linked_metric_limits + rows, g['fields'], metric_refusals = scope_linked_metrics( + session, MODULE, rows, g['fields'], g['measures'], related) + if related: + g["pids"] = frozenset(row["pid"] for row in rows) # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key. @@ -636,6 +645,8 @@ def _payload(session: Session): # "every document in the tenant" mode to reach for, deliberately. from core import grid_events as _ge return {"fields": g["fields"], "rows": rows, + "refusals": metric_refusals, + "limits": linked_metric_limits(metric_refusals), # `today` rides the payload because every relative date condition must resolve against # the TENANT's day, never the browser's — a client that falls back to its own clock # disagrees with the server for everyone west of it. @@ -718,8 +729,15 @@ def allowed_pids(session: Session): @router.get("/customers") -def customers(session: Session = Depends(module_gate(MODULE))): - return _payload(session) +def customers(session: Session = Depends(module_gate(MODULE)), related: str = None): + return _payload(session, related=related) + + +@router.post("/customers/aggregates") +def customer_aggregates(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from routes_odoo_tables import odoo_table_aggregates + return odoo_table_aggregates(MODULE, body=body, session=session) @router.get("/customers/assigned/{view_id}/rows") diff --git a/api/routes_dependents.py b/api/routes_dependents.py index fcbe5a98b89db0c2f1576017b6de36b780e7ba3f..dec7248ae75082666949344cdb58e205da7ba49f 100644 --- a/api/routes_dependents.py +++ b/api/routes_dependents.py @@ -42,6 +42,8 @@ docstring names the cost of publishing it. This door returns names and a count, "Shared with Anna and 2 others" consumes. """ +from urllib.parse import parse_qs + from fastapi import APIRouter, Depends, Query import core.dependents as dependents_registry @@ -148,6 +150,23 @@ def read_dependents(kind: str = Query(...), key: str = Query(...), target = {"kind": target_kind, "key": target_key, "database": resolved} report = dependents_registry.dependents_report(session.runtime, target) + seen = {resolved: True} + for dependent in report['dependents']: + # An address is not an authorisation. Inbound fields can live on a different + # database, so reuse the same wall before offering either C3 address shape. + for item in [dependent, *dependent.get('items', [])]: + href = item.get('href') + if not isinstance(href, str): + continue + if not href.startswith(('#/data?', '#/sheet?')): + item['href'] = None + item['reason'] = 'This dependent has no supported address available.' + continue + destinations = parse_qs(href.partition('?')[2]).get('database', []) + if (href.startswith('#/data?') and not destinations) or any( + not _may_open(session, destination, seen) for destination in destinations): + item['href'] = None + item['reason'] = 'You do not have access to open this dependent\'s database.' label = _label_for(session, resolved, target_kind, target_key) body = { "target": target, diff --git a/api/routes_external.py b/api/routes_external.py index 0c98858f01e4ec3fff3d938f1cc8db2434f8e158..7eb5fe13b6c922f46fe9de3cb2b230f6a69048df 100644 --- a/api/routes_external.py +++ b/api/routes_external.py @@ -1,13 +1,16 @@ -"""Per-user credential management and the bearer-authenticated external read door.""" +"""Per-user credentials, governed reads and explicitly allowed bearer build tools.""" +import copy import json +import uuid from urllib.parse import quote from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response +from starlette.concurrency import run_in_threadpool import deps from deps import Session, err, require_session -from core import external_access +from core import external_access, external_tools, sheets router = APIRouter(prefix="/api/v1/external", tags=["external"]) @@ -15,6 +18,53 @@ router = APIRouter(prefix="/api/v1/external", tags=["external"]) _CREDENTIAL_PREFIX = "lpb1." +def _key_scope(session): + """(access, databases) for THIS request. A session without the claims is an in-app human.""" + claims = session.claims or {} + return external_access.scope_of({ + "access": claims.get("external_access"), + "databases": claims.get("external_databases"), + }) + + +def _assert_database_scope(session, database): + if external_access.database_allowed(_key_scope(session)[1], database): + return + # A binding can carry no database at all. Name what was asked for when there is a name, and + # never interpolate a bare None into a sentence a caller reads. + named = str(database or "").strip() + raise err(403, "key_database_out_of_scope", + f"Database {named} is outside this credential's scope." if named + else "This credential is limited to named databases, and none was named.") + + +def _assert_write_scope(session, tool_name): + if not external_access.may_write(_key_scope(session)[0]): + raise err(403, "key_read_only", f"{tool_name} writes, and this credential is read only.") + + +def _databases_in_arguments(value): + """Find database references, including bound cell documents, without unbounded recursion.""" + found = [] + + def scan(node, depth): + if depth > 12: + return + if isinstance(node, dict): + for key, item in node.items(): + if key in ("database", "sourceDatabase") and isinstance(item, str): + found.append(item) + elif key == "compatibleDatabases" and isinstance(item, list): + found.extend(database for database in item if isinstance(database, str)) + scan(item, depth + 1) + elif isinstance(node, (list, tuple)): + for item in node: + scan(item, depth + 1) + + scan(value, 0) + return found + + def _auth_refusal(): raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.") @@ -40,6 +90,10 @@ def require_external_session( "u": verified.user["username"], "external": True, "external_key_id": verified.key_id, + "external_access": verified.access, + "external_databases": (external_access.ALL_DATABASES + if verified.databases == external_access.ALL_DATABASES + else list(verified.databases)), }, runtime=verified.runtime, ) @@ -64,13 +118,17 @@ def _can_use_analyst(session: Session) -> bool: def _management_refusal(exc: external_access.ExternalAccessError): + if exc.code == "external_access_scope_invalid": + raise err(400, exc.code, "Choose Read only or Read and write.") + if exc.code == "external_databases_invalid": + raise err(400, exc.code, "Choose all databases, or name at least one database.") if exc.code == "external_label_invalid": raise err(400, exc.code, "Enter a label between 1 and 80 characters.") if exc.code == "external_key_not_found": raise err(404, exc.code, "That credential was not found.") if exc.code in {"external_identity_invalid", "external_account_unavailable"}: raise err(409, exc.code, "This account cannot create an external credential.") - raise err(503, "external_access_unavailable", "External access is unavailable right now.") + raise err(503, "external_access_unavailable", "The AI API is unavailable right now.") def _resolve_external_scope( @@ -128,6 +186,8 @@ def _resolve_external_scope( else: row["label"] = fallback_label(database) row["fields"] = [] + else: + row["label"] = fallback_label(database) resolved.append(row) return resolved @@ -150,13 +210,14 @@ def _portable_skill_document(session: Session, base_url: str) -> str: lines = [ "---", "name: loopable-external-data", - "description: Read this user's governed Loopable data through the external API.", + "description: Read governed data and build Sheet workbooks, saved Views and Apps as the key's maker.", "---", "", "# Loopable external data", "", - ("This surface is a read-only view of the Loopable data this user can already see. " - "It applies the same database, field, and row permissions as the Loopable app."), + ("This surface reads governed data and writes only through the named build tools, " + "as the key's maker with the maker's own grants. It applies the same database, " + "field, row and workbook permissions as the Loopable app."), "", "## Authentication", "", @@ -268,12 +329,67 @@ def _portable_skill_document(session: Session, base_url: str) -> str: ]) lines.append("") + example_call = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { + "name": "sheet_create_workbook", "arguments": {"name": "Gateway workbook"}}} + lines.extend([ + "## Remote MCP tools", + "", + "Endpoint: `POST /api/v1/external/mcp`", + "", + ("Send one JSON-RPC 2.0 object per request. Methods: initialize, tools/list, tools/call " + "and ping. Initialize echoes your protocolVersion and advertises tools. Requests " + "without an id, or with a null id, receive HTTP 204."), + "", + "```json", + json.dumps(example_call, indent=2), + "```", + "", + ("A tools/call response has jsonrpc, id and result. The result contains " + "content: [{type: 'text', text: ''}] and isError. " + "A tool refusal uses HTTP 200 with ok: false and isError: true; its error names " + "the tool, code, status and reason. Invalid or revoked credentials receive HTTP 401."), + "", + "```sh", + "curl --request POST \\", + " --header 'Authorization: Bearer ' \\", + " --header 'Content-Type: application/json' \\", + f" --data '{json.dumps(example_call, separators=(',', ':'))}' \\", + f" '{surface_url}/mcp'", + "```", + "", + ("Sheet tools require analyst and sheets grants. Geometry patches carry the revision " + "read by the server; pass rev to also protect the revision you read. Cell writes " + "return accepted and refused addresses, retaining the app's partial-write behavior."), + "", + ("View tools use the grid's native config: filters, sorts, groupBy, visible and display. " + "Fields omitted from visible are hidden. The grid validates the config. App tools use " + "the Application registry's typed config. app_delete requires the exact current " + "App name in confirmName, including after a rename."), + "", + ]) + for name, tool in external_tools.TOOLS.items(): + lines.extend([ + f"### `{name}`", + "", + _markdown_inline(tool["description"]), + "", + "Writes: " + ("yes" if tool["write"] else "no") + ".", + "", + "```json", + json.dumps(tool["inputSchema"], indent=2, ensure_ascii=True), + "```", + "", + ]) lines.extend([ "## Limits", "", - ("This surface performs no writes. It returns no data the user cannot already see in " - "Loopable, and it provides no export beyond what the user could already export from " - "the app."), + ("This surface writes only through the named tools, as the key's maker with the maker's " + "own grants. Workbook roles and database restrictions still apply. Each credential also " + "carries its own access level and database scope, set when it is created and shown in " + "Loopable Settings. It returns no data " + "the maker cannot already see in Loopable. Sheet cell history records the maker and " + "key label. Saved Views and Apps keep the UI's username attribution; their writers " + "have no author display-name field. No tool writes to Odoo."), "", ]) document = "\n".join(lines) @@ -287,8 +403,11 @@ def create_key(body=Body(default=None), session: Session = Depends(require_sessi """Mint a credential for the signed-in human. Bearer auth is not accepted here.""" session.require("analyst") label = body.get("label") if isinstance(body, dict) else None + access = body.get("access") if isinstance(body, dict) else None + databases = body.get("databases") if isinstance(body, dict) else None try: - key, token = external_access.mint(session.runtime, session.uname, session.tenant, label) + key, token = external_access.mint(session.runtime, session.uname, session.tenant, label, + access=access, databases=databases) except external_access.ExternalAccessError as exc: _management_refusal(exc) return {"key": key, "secret": token} @@ -316,7 +435,7 @@ def revoke_key(key_id: str, session: Session = Depends(require_session)): @router.get("/skill.md") def portable_skill(request: Request, session: Session = Depends(require_session)): - """Download the signed-in human's current governed external read instructions.""" + """Download current governed external read and build instructions.""" session.require("analyst") document = _portable_skill_document(session, str(request.base_url)) return Response( @@ -328,21 +447,39 @@ def portable_skill(request: Request, session: Session = Depends(require_session) @router.get("/whoami") def whoami(session: Session = Depends(require_external_analyst)): - """The one observable bearer route in this ticket; later tickets add governed data reads.""" + """Name the authenticated maker and the explicit limits on bearer writes.""" + access, databases = _key_scope(session) return { "username": session.uname, "name": session.user.get("name") or session.uname, "tenant": session.tenant, "key_id": session.claims["external_key_id"], - "read_only": True, + "access": access, + "databases": (external_access.ALL_DATABASES + if databases == external_access.ALL_DATABASES else list(databases)), + "write_scope": "Named tools only, as the key's maker with the maker's own grants.", } +@router.get("/scope/databases") +def scope_databases(session: Session = Depends(require_session)): + """List the databases this signed-in maker could put inside a new credential's scope.""" + session.require("analyst") + return {"databases": [ + {key: row[key] for key in ("database", "label", "permitted", "answerable")} + for row in _resolve_external_scope(session) + if row.get("permitted") or row.get("visible") + ]} + + @router.get("/databases") def list_databases(session: Session = Depends(require_external_analyst)): """List only sources this user may read or already sees elsewhere in the product.""" databases = [] + key_databases = _key_scope(session)[1] for row in _resolve_external_scope(session): + if not external_access.database_allowed(key_databases, row["database"]): + continue if not (row.get("permitted") or row.get("visible")): continue public = { @@ -353,12 +490,14 @@ def list_databases(session: Session = Depends(require_external_analyst)): if not public["answerable"]: public["reason"] = row.get("reason") or "" databases.append(public) - return {"databases": databases, "read_only": True} + return {"databases": databases, + "write_scope": "Named tools only, as the key's maker with the maker's own grants."} @router.get("/databases/{database}/schema") def database_schema(database: str, session: Session = Depends(require_external_analyst)): """Return the field contract after resolving the ordinary strict Assistant read wall.""" + _assert_database_scope(session, database) resolved = _resolve_external_scope( session, databases=[database], include_schema=True, strict_schema=True, ) @@ -372,6 +511,7 @@ def query_database( session: Session = Depends(require_external_analyst), ): """Read one detached Assistant snapshot; FastAPI rejects non-object request bodies.""" + _assert_database_scope(session, database) payload = body or {} return deps.assistant_read_scope( session, @@ -381,19 +521,342 @@ def query_database( ) -# C7 / R3: one import-time chokepoint makes the external bearer surface structurally read only. -# POST remains available for bounded query bodies and for the cookie-only credential lifecycle -# above. No bearer dependency appears on those lifecycle routes. -_READ_ONLY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS"}) +def _view_context(session, database, event): + import routes_grid + scope = routes_grid._scope_or_400( + {"customer_data": "customer", "product_data": "product"}.get(database, database)) + try: + if scope.startswith("ut_"): + from routes_tables import ut_write_ctx + + g = ut_write_ctx(session, scope) + if g.get("limits") and (event.get("pid") is not None or event.get("pids")): + limit = g["limits"][0] + raise err(409, "pid_scope_unresolved", + "This connector mirror's record scope is unresolved, so a change " + "addressed to particular records cannot be admitted. " + f"{limit.get('cause') or 'The row set is unresolved'}. " + f"{limit.get('recommendation') or ''}".strip()) + elif scope == "product": + from routes_products import MODULE, product_assembly + + session.require(MODULE) + g = product_assembly(session, consume_corrections=False) + else: + from routes_customers import grid_assembly -def _assert_read_only_router() -> None: - for route in router.routes: - methods = set(getattr(route, "methods", set()) or set()) - refused = methods - _READ_ONLY_METHODS - if refused: - joined = ", ".join(sorted(refused)) - raise RuntimeError(f"external route {route.path} advertises refused method {joined}") + session.require(routes_grid.MODULE) + g = grid_assembly(session, scope=scope, consume_corrections=False) + except HTTPException as exc: + info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} + raise err(exc.status_code, info.get("code", "view_refused"), + f"Database {database}: {info.get('message') or 'The app refused access.'}") from exc + ctx = routes_grid._ctx( + session, g["fields"], g["pids"], scope_key=scope, + measure_keys=frozenset(m["key"] for m in g["measures"]), + resolved_ids=frozenset(g["measure_sets"]), + cohort_ids=frozenset(c["id"] for c in g["lists"]), + measure_offer=tuple(g["measures"]), visible_views=tuple(g["views"]), + recurrence_validator=lambda raw: routes_grid.govern_date_recurrence( + session, scope, g["fields"], raw)) + if scope.startswith("ut_"): + ctx.hidden_keys = frozenset(g["hidden"]) + return ctx + + +def _application_access(session, databases, *, strict=False): + import routes_assets + + if not strict: + return routes_assets.application_access_context(session, databases) + allowed, records = set(), {} + for database in databases or (): + try: + permitted, semantic = routes_assets.application_access_context( + session, [database], strict=True) + except PermissionError as exc: + # Name only the submitted database, never a hidden registry object's source. + raise err(403, "forbidden", f"Database {database}: {exc}") from exc + allowed |= permitted + records.update(semantic) + return allowed, records + + +def _view_event(session, kind, args): + from core import grid_events + + database = args["database"] + _assert_database_scope(session, database) + event = {"id": uuid.uuid4().hex, "type": kind} + if kind == "view_upsert": + raw = args["view"] + if (not isinstance(raw.get("config"), dict) + or not isinstance(raw.get("id"), str) or not raw["id"].strip() + or not isinstance(raw.get("name"), str) or not raw["name"].strip()): + raise err(400, "tool_arguments_invalid", "A View needs an id, a name and an object config.") + view_id = raw["id"] + event["view"] = raw + # A membership config also addresses records, even though the payload nests its ids. + if raw["config"].get("memberPids"): + event["pids"] = raw["config"]["memberPids"] + else: + view_id = args["view_id"] + event["viewId"] = view_id + if not view_id.strip() or len(view_id) > 120 or view_id != view_id.strip(): + raise err(400, "tool_arguments_invalid", "A View id must have 1 to 120 characters without surrounding spaces.") + ctx = _view_context(session, database, event) + try: + before = ctx.table.find_view(view_id, prefer_owner=session.uname) + rerender = grid_events.handle_one(event, ctx) + except grid_events.StoreUnavailable as exc: + raise err(503, "store_unavailable", f"View {view_id}: the tenant store is unavailable.") from exc + if ctx.out.refusals: + refusal = ctx.out.refusals[0] + raise err(400, refusal.get("code", "view_refused"), + f"View {view_id}: {refusal.get('message') or 'The app refused this change.'}") + if ctx.out.toast: + raise err(403, "view_refused", f"View {view_id}: {ctx.out.toast}") + after = ctx.table.find_view(view_id, prefer_owner=session.uname) + changed = before != after + if not changed: + raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.") + # ⛔⛔ `handle_one`'s BOOL IS A RERENDER FLAG, NOT A SUCCESS FLAG, and reading it as success is + # the mistake this comment exists to stop the next reader making. Its own contract block + # (`grid_events.py:1182`) says so in as many words: "view_delete - False (the deleted view + # simply stops being served). view_upsert - unchanged: False, except an unresolved measure + # condition." So an ordinary successful view write returns False, and refusing on it would + # refuse every save that worked. + # ⭐ THE SILENT NO-OP IS CAUGHT BY `changed` ABOVE INSTEAD, which is the scar this tool was + # written against: `view_upsert` answers 200 and writes nothing when the view id belongs to + # another user ([[a-view-id-another-user-holds-refuses-silently]]), and a store read before and + # after is the only thing that can tell that apart from a save that landed + # ([[lost-write-looks-like-failed-read]]). `rerender` rides out as information, never as the + # verdict. + return {"database": database, "view_id": view_id, "rerender": bool(rerender)} + + +def _application_upsert(session, args): + import application_registry as applications + import routes_assets + + for database in args.get("compatibleDatabases") or []: + _assert_database_scope(session, database) + if args.get("sourceDatabase") is not None: + _assert_database_scope(session, args["sourceDatabase"]) + payload = {key: value for key, value in args.items() if key != "app_id"} + app_id = args.get("app_id") + if "app_id" in args and not str(app_id or "").strip(): + raise err(400, "bad_application", "app_id must name an App to update.") + if not session.runtime.available(): + raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was saved.") + try: + if not app_id: + databases = payload.get("compatibleDatabases") or [] + if payload.get("sourceDatabase"): + databases = [*databases, payload["sourceDatabase"]] + allowed, records = _application_access(session, databases, strict=True) + return {"application": applications.create( + session.runtime, session.tenant, session.uname, payload, + semantic_records=records, allowed_databases=allowed)} + current = routes_assets._application_databases(session.runtime, app_id) + allowed, _ = _application_access(session, current) + if not current or not allowed: + raise applications.ApplicationError("application does not exist") + proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current + proposed_allowed, records = _application_access(session, proposed, strict=True) + return {"application": applications.mutate( + session.runtime, app_id, session.uname, session.admin, payload, + semantic_records=records, allowed_databases=allowed | proposed_allowed, + target_database=next((database for database in proposed + if database in proposed_allowed), None))} + except (applications.ApplicationError, PermissionError) as exc: + routes_assets._application_error(exc) + + +def _tool_context(session: Session) -> external_tools.ToolContext: + # Import within the API layer. Core receives callbacks, never an upward dependency. + import routes_sheet + import formula_runtime + import routes_assets + from modules import sheet as sheet_mod + + def project(book): + served = routes_sheet._read_for(session, book) + databases = _key_scope(session)[1] + if databases != external_access.ALL_DATABASES: + # The owner projection can share cells with its input; key scope is response-only. + served = copy.deepcopy(served) + for tab in served.get("sheets", []): + for cell in tab.get("cells", {}).values(): + if not isinstance(cell, dict) or cell.get("kind") != "bound": + continue + binding = cell.get("binding") + database = binding.get("database") if isinstance(binding, dict) else None + if not external_access.database_allowed(databases, database): + cell["value"] = None + cell["display"] = "" + routes_sheet._recompute_formulas(served, formula_runtime, sheet_mod) + return served + + def resolve_binding(binding, book, tab): + if isinstance(binding, dict): + _assert_database_scope(session, binding.get("database")) + return routes_sheet._resolve_cell_binding(session, binding, book, tab) + + def cache_binding(workbook_id, tab, addr, binding, resolved): + # Same best-effort cache fill as the UI's compute_cell. A fresh value must replace + # this scope's old slot before an editor's read projection can use that slot. + try: + sheets.fill_scope_cache(session.runtime, workbook_id, tab + "!" + addr, + routes_sheet._binding_scope_key(session, resolved["database"], binding), + resolved["value"], routes_sheet._now_iso(), resolved["database"]) + except Exception: + pass + + label = external_access.label_for(session.runtime, session.claims["external_key_id"]) + maker = session.user.get("name") or session.uname + actor_name = f"{maker} via key {label}" + if _CREDENTIAL_PREFIX in actor_name: + raise sheets.SheetRefusal("credential_in_attribution", "Maker names and key labels must not contain credential values.") + return external_tools.ToolContext( + session=session, actor_name=actor_name, + require_reader=lambda book: routes_sheet._require_reader(session, book), + require_editor=lambda book: routes_sheet._require_editor(session, book), + project_book=project, + resolve_binding=resolve_binding, + cache_binding=cache_binding, + list_databases=lambda: list_databases(session), + describe_database=lambda database: database_schema(database, session), + view_event=lambda kind, args: _view_event(session, kind, args), + app_upsert=lambda args: _application_upsert(session, args), + # Reuse the UI's atomic NameConfirmedDelete wrapper as well as the registry delete. + app_delete=lambda args: routes_assets.delete_application( + args["app_id"], {"confirmName": args["confirmName"]}, session), + ) -_assert_read_only_router() +def _tool_refusal(name, code, message, status=400, **extra): + # Never echo arbitrary method/tool names or raw exception text across the bearer seam. + tool_name = name if isinstance(name, str) and name in external_tools.TOOLS else "tools/call" + sentence = f"{tool_name}: {_markdown_inline(message)}" + if _CREDENTIAL_PREFIX in sentence: + sentence = f"{tool_name}: The result contained a credential value and was withheld." + return {"ok": False, "error": {"tool": tool_name, "code": code, + "message": sentence, "status": status, **extra}} + + +def _mcp_content(rid, out): + encoded = json.dumps(out, default=str, ensure_ascii=True, allow_nan=False) + if _CREDENTIAL_PREFIX in encoded: + out = _tool_refusal(None, "credential_in_output", "The result contained a credential value and was withheld.") + encoded = json.dumps(out) + return {"jsonrpc": "2.0", "id": rid, "result": { + "content": [{"type": "text", "text": encoded}], "isError": not out.get("ok", False)}} + + +def _handle_mcp(req, session: Session): + """One HTTP request object; the stdio precedent's JSON-RPC and content envelopes.""" + def protocol_error(rid, code, message): + return {"jsonrpc": "2.0", "id": rid, "error": {"code": code, "message": message}} + + if not isinstance(req, dict): + return protocol_error(None, -32600, "An MCP request must be a JSON-RPC object.") + rid, method = req.get("id"), req.get("method") + if (rid is not None and (type(rid) not in (str, int) or _CREDENTIAL_PREFIX in str(rid))): + return protocol_error(None, -32600, "The request id must be a number or a safe string.") + if rid is None: + # Notifications never dispatch writes and never return a JSON-RPC body. + return None + if req.get("jsonrpc") != "2.0" or not isinstance(method, str): + return protocol_error(rid, -32600, "An MCP request needs jsonrpc 2.0 and a method.") + params = req.get("params", {}) + if not isinstance(params, dict): + return protocol_error(rid, -32602, "MCP params must be an object.") + name = params.get("name") + try: + # Same Analyst wall as the read doors, inside the dispatcher so a grant refusal + # remains MCP content. Only the unchanged bearer dependency produces an HTTP 401. + require_external_analyst(session) + if method == "initialize": + protocol = params.get("protocolVersion") or "2025-06-18" + if not isinstance(protocol, str) or _CREDENTIAL_PREFIX in protocol: + return protocol_error(rid, -32602, "The protocol version must be a safe string.") + return {"jsonrpc": "2.0", "id": rid, "result": { + "protocolVersion": protocol, "capabilities": {"tools": {}}, + "serverInfo": {"name": "loopable-external", "version": "1.0.0"}}} + if method == "ping": + return {"jsonrpc": "2.0", "id": rid, "result": {}} + if method == "tools/list": + return {"jsonrpc": "2.0", "id": rid, "result": {"tools": [ + {"name": key, "description": tool["description"], "inputSchema": tool["inputSchema"]} + for key, tool in external_tools.TOOLS.items()]}} + if method != "tools/call": + return protocol_error(rid, -32601, "The requested MCP method was not found.") + if isinstance(name, str) and name in _WRITE_TOOLS: + _assert_write_scope(session, name) + for database in _databases_in_arguments(params.get("arguments", {})): + _assert_database_scope(session, database) + data = external_tools.dispatch(_tool_context(session), name, params.get("arguments", {})) + if isinstance(data, dict) and data.get("refused"): + first = data["refused"][0] + out = _tool_refusal(name, first["code"], first["sentence"]) + out["error"]["data"] = data # Accepted edits remain visible even on partial refusal. + else: + out = {"ok": True, "data": data} + except external_access.ExternalAuthError: + _auth_refusal() + except external_access.ExternalAccessError: + out = _tool_refusal(name, "external_access_unavailable", "The AI API is unavailable right now.", 503) + except sheets.SheetRefusal as exc: + import routes_sheet + translated = routes_sheet._refuse(exc) + info = translated.detail["error"] + out = _tool_refusal(name, info["code"], info["message"], translated.status_code, + **{k: v for k, v in info.items() if k not in ("code", "message")}) + except HTTPException as exc: + if exc.status_code == 401: + _auth_refusal() + info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} + out = _tool_refusal(name, info.get("code", "tool_refused"), + info.get("message", "The app refused this operation."), exc.status_code) + except (TypeError, ValueError, KeyError, AttributeError): + out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.") + except Exception: + out = _tool_refusal(name, "tool_unavailable", "The tool could not complete this operation.", 503) + return _mcp_content(rid, out) + + +@router.post("/mcp") +async def external_mcp(request: Request, session: Session = Depends(require_external_session)): + """Remote MCP without a browser session, with the read doors' maker and Analyst wall.""" + try: + req = await request.json() + except (ValueError, UnicodeError): + return {"jsonrpc": "2.0", "id": None, + "error": {"code": -32700, "message": "The request body must be valid JSON."}} + reply = await run_in_threadpool(_handle_mcp, req, session) + return Response(status_code=204) if reply is None else reply + + +# R8 (SDLC 61) revokes the read-only surface. Every write must be named here deliberately. +# A new write in the registry without a matching literal raises at import, before serving. +_WRITE_TOOLS = frozenset({ + "sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding", + "sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze", + "sheet_add_rule", "sheet_set_chart", "sheet_structure", + "sheet_save_template", + "view_upsert", "view_delete", "app_upsert", "app_delete", +}) + + +def _assert_write_allow_list() -> None: + declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")} + undeclared = declared - _WRITE_TOOLS + stale = _WRITE_TOOLS - declared + if undeclared or stale: + raise RuntimeError("External write allow-list mismatch: undeclared=" + + repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale))) + + +_assert_write_allow_list() diff --git a/api/routes_grid.py b/api/routes_grid.py index 85e47ea32129d69c6a470da4714b2349b553ec14..a80984f40ec24149142fbe06fe76f6c4b07a8eae 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -2491,9 +2491,11 @@ def _topic_rt(st, module): return st -def _customer_rows(table_key, user, st): +def _customer_rows(table_key, user, st, want_rows=True): """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses.""" import aios_grid + if not want_rows: + return list(aios_grid.FIELDS), [] import core.perm_scope as perm_scope from routes_customers import _pool_for @@ -2502,15 +2504,16 @@ def _customer_rows(table_key, user, st): return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent) -def _product_rows(table_key, user, st): +def _product_rows(table_key, user, st, want_rows=True): """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a BU-pinned reader gets that BU's field contract rather than the consolidated one.""" import core.perm_scope as perm_scope from routes_products import _pool_for, pd_fields - rt = _topic_rt(st, table_key) + rt = _topic_rt(st, table_key) if want_rows else None team_id, _agent = perm_scope.derive_pool_scope(user, table_key) - return pd_fields(consolidated=team_id is None, st=st), _pool_for(rt, team_id) + return pd_fields(consolidated=team_id is None, st=st), ( + _pool_for(rt, team_id) if want_rows else []) def _register_topic_rows(): @@ -2747,7 +2750,8 @@ def _may_open(session: Session, key: str, st=None, memo: dict = None): return _walled_fields(session, key, st=st, memo=memo) is not None -def _source_fields(session: Session, table: str, st=None, memo: dict = None): +def _source_fields(session: Session, table: str, st=None, memo: dict = None, + read_status: dict = None): """The link-bearing field list for `table`, under this session's field wall. ⛔⛔ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS @@ -2769,7 +2773,8 @@ def _source_fields(session: Session, table: str, st=None, memo: dict = None): import odoo_relational fields = fields + list(odoo_relational.module_link_fields(table) or ()) except Exception: # noqa: BLE001 - pass + if read_status is not None: + read_status["cacheable"] = False return fields @@ -2826,7 +2831,7 @@ def _ensure_odoo_spec(): return False -def _unwalled_fields(defs: dict, key: str): +def _unwalled_fields(defs: dict, key: str, read_status: dict = None): """`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only. ⛔⛔ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of @@ -2849,7 +2854,8 @@ def _unwalled_fields(defs: dict, key: str): import odoo_relational fields = fields + list(odoo_relational.module_link_fields(key) or ()) except Exception: # noqa: BLE001 - pass + if read_status is not None: + read_status["cacheable"] = False return fields @@ -2884,7 +2890,7 @@ def _relation_context(session: Session, table: str, memo: dict = None): than a silently empty grid — and `scoped_fields` is a `want_rows=False` door by construction. Nothing here is handed to a mutation; `grid_related` is a GET. ⛔ IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves - only `user_tables` and (lazily, once) `object_shares`, every other store key falls through to + `user_tables`, (lazily, once) `object_shares` and read-only `__shared` overlays; other keys fall through to the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list EQUAL, key for key and value for value, to the one it returned under the bare runtime. @@ -2895,16 +2901,19 @@ def _relation_context(session: Session, table: str, memo: dict = None): """ import core.user_tables as user_tables + read_status = {"cacheable": True} try: defs = dict(user_tables.all_defs(st=session.runtime) or {}) except Exception: # noqa: BLE001 defs = {} + read_status["cacheable"] = False try: lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs}) except Exception: # noqa: BLE001 # A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback # is: the pass simply goes back to paying a round trip per wall resolution. lent = session.runtime + read_status["cacheable"] = False # ⭐ ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo` # below still memoise their own composition (the grant gate, the pre-set link append); this is @@ -2919,24 +2928,30 @@ def _relation_context(session: Session, table: str, memo: dict = None): def may_open(key): if key not in open_memo: open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo) + if not open_memo[key]: + read_status["cacheable"] = False return open_memo[key] def fields_for(key): if key not in field_memo: - field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo) + field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo, + read_status=read_status) + if wall_memo.get(key) is None: + read_status["cacheable"] = False return field_memo[key] def open_fields_for(key): if key not in wide_memo: walled = list(fields_for(key) or ()) seen = {str(f.get("key")) for f in walled if isinstance(f, dict)} - extra = [f for f in _unwalled_fields(defs, key) + extra = [f for f in _unwalled_fields(defs, key, read_status=read_status) if isinstance(f, dict) and str(f.get("key")) not in seen] wide_memo[key] = walled + extra return wide_memo[key] - return {"defs": defs, "may_open": may_open, "fields_for": fields_for, - "open_fields_for": open_fields_for} + read_status.update(defs=defs, may_open=may_open, fields_for=fields_for, + open_fields_for=open_fields_for) + return read_status def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None): @@ -2952,14 +2967,120 @@ def related_path_entries(session: Session, table: str, depth: int, ctx: dict = N """ import core.relations as relations - _ensure_odoo_spec() + spec_ready = _ensure_odoo_spec() table = str(table or "") ctx = ctx or _relation_context(session, table) + if not spec_ready: + ctx["cacheable"] = False return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"], depth=depth, may_open=ctx["may_open"], fields_for=ctx["fields_for"]) +def related_measure_pin(session, target, related, requested): + """Compile metric membership with the rows door's exact path and leg resolvers. + + SQL bindings are host declarations. No SQL, callbacks or row identities are trusted from + the client beyond the origin id, named edge and requested subset of resolved target ids. + Retain through rows as well as explicit hops so a metric cannot expand behind a row wall. + """ + from core import perm_scope, relations + import routes_odoo_tables as door + + if isinstance(related, str): + related = door._json_arg(related, 'related') + allowed = door._relation_target_ids(session, target, related) + if set(requested) - allowed: + raise err(403, 'metric_record_refused', 'A requested record is outside this linked grid.') + source, via = str(related['table']), related.get('via') + path = door._via_path(via) + entries = related_path_entries(session, source, len(path) if path else 1) + entry = relations.pick_path(entries, path) if path else relations.pick(entries, via) + legs = [(source, entry)] + if len(entry.get('path') or ()) > 1: + first, middle, last = door._related_path_legs(session, source, entry, entries) + legs = [(source, first), (middle, last)] + + bindings = { + 'sale_order_line': [ + dict(table='customer_data', column='pid', sql='{alias}.order_partner_id'), + dict(table='product_data', column='product_id', sql='{alias}.product_id'), + dict(table='ut_odoo_orders', column='pid', sql='{alias}.order_id'), + dict(table='ut_odoo_order_lines', column='pid', sql='{alias}.id'), + dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'), + ], + 'sale_order': [ + dict(table='customer_data', column='pid', sql='{alias}.partner_id'), + dict(table='ut_odoo_orders', column='pid', sql='{alias}.id'), + dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'), + dict(table='ut_odoo_order_lines', column='order_id', sql='{alias}.id'), + ], + } + members = {} + def read(table, column, values, where): + columns = {'pid', column} + if table == 'product_data': + columns.add('product_id') + if table == 'ut_odoo_order_lines': + columns.add('order_id') + return door._relation_rows(session, table, columns, match=(column, values), where=where) + + def retain(table, ids): + rows = read(table, 'pid', ids, None) + if table in members: + existing = {r['pid'] for r in members[table]} + if {r['pid'] for r in rows} != existing: + raise err(409, 'metric_scope_unsupported', + 'This metric cannot distinguish repeated databases in the record path. ' + 'Use a direct relation or add a metric binding for these separate roles.') + rows = [r for r in rows if r['pid'] in existing] + members[table] = rows + + with relations.measure_pin_context(read=read, bindings=bindings, + may_open=lambda key: _may_open(session, key)): + ids = {int(related['record'])} + retain(source, ids) + if not members[source]: + raise err(403, 'metric_record_refused', 'The origin record is unavailable to this account.') + for table, leg in legs: + link, via = leg.get('link') or {}, leg.get('via') or {} + through = link.get('through') + if through and link.get('selectionMode') != 'manual': + reverse = via.get('direction') == 'reverse' + left = str(link['throughTo'] if reverse else link['throughOn']) + frm = str(via.get('from') or 'pid') + origins = door._relation_rows(session, table, [frm], ids=ids) + bridge = read(str(through), left, + [r[frm] for r in origins if r.get(frm) is not None], None) + retain(str(through), {r['pid'] for r in bridge}) + ids = door._relation_leg_ids(session, table, ids, leg) + retain(str(leg['target']), ids) + required = tuple(members) + # A manual link need not name the sales bridge. Its metrics still obey that + # database's row wall, intersected with the already resolved record membership. + if 'ut_odoo_order_lines' not in members: + for table, column, join in [('customer_data', 'pid', 'partner_id'), + ('product_data', 'product_id', 'product_id'), + ('ut_odoo_orders', 'pid', 'order_id')]: + if table in members: + bridge = read('ut_odoo_order_lines', join, + [r[column] for r in members[table]], None) + # This extra wall belongs to line metrics. Header metrics use their own + # traversed order membership, including legitimate orders with no lines. + retain('ut_odoo_order_lines', {r['pid'] for r in bridge}) + bindings['sale_order'] = [binding for binding in bindings['sale_order'] + if binding['table'] != 'ut_odoo_order_lines'] + break + # Required path tables must all bind. Binding only the destination would return globals. + pin = relations.MeasurePin(entry['path'], members, bindings, + required=required) + scopes = [perm_scope.derive_pool_scope(session.user, table) for table in required] + teams = {team for team, _agent in scopes if team is not None} + if len(teams) > 1: + raise err(403, 'metric_scope_conflict', 'The linked databases have incompatible business unit scopes.') + return pin, next(iter(teams), None) + + def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None): """W53-T18 / R13 / C3 — WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`. @@ -3031,6 +3152,33 @@ def _refusal_error(diag: dict): return exc +_RELATED_CACHE_LIMIT = 128 + + +def _related_cache_key(session: Session, table: str, depth: int, via): + """Names-only answers follow this user's schema, grant, sync and registrar revisions. + + Tokens are read-through dict lookups, not document downloads. A runtime without revision + support can still serve an uncached answer. Never reuse an answer with unknown revisions. + """ + import core.store as store + import core.shares as shares + import core.user_tables as user_tables + import odoo_relational + + try: + if not session.uname: + return None + return (session.uname, table, depth, via, + session.runtime.revision(user_tables.STORE_KEY)["token"], + session.runtime.revision(shares.SHARES_KEY)["token"], + store.revision("users")["token"], + session.runtime.revision(odoo_relational.CONFIG_KEY)["token"], + user_tables.register_read_through(), user_tables.register_connected_prefix()) + except Exception: # noqa: BLE001 + return None + + @router.get("/grid/{table}/related") def grid_related(table: str, depth: int = Query(default=2), @@ -3102,12 +3250,27 @@ def grid_related(table: str, f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and " f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}") + wanted = tuple(str(v).strip() for v in via if str(v or "").strip()) if via else None + if wanted is not None and len(wanted) > relations.MAX_DEPTH: + raise err(400, "over_depth", relations.over_depth_refusal(table, list(wanted))) + + # This registration must run even on a hit; an import order repair cannot be skipped by + # memoisation. Failed registration or reads are served as before, but never remembered. + spec_ready = _ensure_odoo_spec() + cache = getattr(session.runtime, "related_cache", None) + cache_key = (_related_cache_key(session, table, depth, wanted) + if spec_ready and cache is not None else None) + if spec_ready and cache_key is not None: + from copy import deepcopy + with session.runtime._lock: + cached = cache.get(cache_key) + if cached is not None: + return deepcopy(cached) + ctx = _relation_context(session, table, memo=wall_memo) entries = related_path_entries(session, table, depth, ctx) - if via: - wanted = [str(v) for v in via if str(v or "").strip()] - if len(wanted) > relations.MAX_DEPTH: - raise err(400, "over_depth", relations.over_depth_refusal(table, wanted)) + if wanted is not None: + wanted = list(wanted) hit = relations.pick_path(entries, wanted) if hit is None: # ⭐⭐ W53-T18 / R13 / C3 — THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED @@ -3127,7 +3290,19 @@ def grid_related(table: str, # ⚠ A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other # tabs the wall removed are the LISTING's answer, not this one, and repeating them here # would attach an unasked-for permission report to every successful read. - return {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []} - return {"ok": True, "table": table, "depth": depth, "related": entries, - "refusals": related_refusals(session, table, depth, entries, ctx)} - + answer = {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []} + else: + answer = {"ok": True, "table": table, "depth": depth, "related": entries, + "refusals": related_refusals(session, table, depth, entries, ctx)} + if (spec_ready and cache_key is not None and ctx.get("cacheable", False) and ctx["defs"] + and not answer["refusals"] + and cache_key == _related_cache_key(session, table, depth, cache_key[3])): + from copy import deepcopy + # The Amazon purge's pool-cache tuple does not include this memo. That is safe because + # every lookup checks the current tokens; old entries cannot match changed metadata. + # Limit memory, not the number of related databases: eviction just rebuilds next open. + with session.runtime._lock: + if len(cache) >= _RELATED_CACHE_LIMIT: + cache.clear() + cache[cache_key] = deepcopy(answer) + return answer diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 1e186678eee9165932bc0c465255ff1d02aefb15..c557a66edd40d8f0c54876d788498bf0ed548c13 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -920,6 +920,14 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_ _rel_reconnect(session.runtime) except Exception: # noqa: BLE001 pass + elif etype == "amazon_sp": + # D-29 / intake F6: the HTTP thread only records intent. The resident resync thread + # remains the mirror's single writer. A queue failure must not undo a stored credential. + try: + from core import connector_sync + connector_sync.request_sync("amazon_sp", session.runtime.key, requested_by=session.uname) + except Exception: # noqa: BLE001 + pass if scope != DEFAULT_SCOPE: try: _write_scope(session.runtime, row["id"], scope, session.uname) diff --git a/api/routes_module_index.py b/api/routes_module_index.py new file mode 100644 index 0000000000000000000000000000000000000000..fe9ea335582f43f40d172eba42c92c634275fd64 --- /dev/null +++ b/api/routes_module_index.py @@ -0,0 +1,132 @@ +"""W61-T20 / C6: personal App and Agents folder trees in the tenant store.""" +import copy + +from fastapi import APIRouter, Body, Depends, HTTPException + +from core.sheets import SheetFolderRefusal, validate_folder_tree +from deps import Session, err, module_gate, require_session +import routes_assets +import routes_automation +import voice_control_plane as voice + +router = APIRouter(prefix="/api/v1/module-index") +_AGENTS_GATE = module_gate("automation") + + +def _sentence(value): + return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split()) + + +def _refuse(exc): + status = {"module_index_unknown": 404, "module_index_user_forbidden": 403, + "module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400) + return err(status, exc.code, _sentence(exc.message)) + + +def _access(module: str, session: Session = Depends(require_session)): + if module not in {"app", "agents"}: + raise _refuse(SheetFolderRefusal( + "module_index_unknown", f'Folder index "{module}" is not supported; use app or agents.')) + # App has the same session-only door as GET /applications; its rows have their own wall. + if module == "agents": + try: + _AGENTS_GATE(session) + except HTTPException as exc: + if exc.status_code != 403: + raise + raise err(403, "forbidden", "Your account or workspace does not have access to Agents (automation).") + return session + + +def _owner(session, user): + owner = user or session.uname + if owner != session.uname and not session.admin: + raise SheetFolderRefusal( + "module_index_user_forbidden", f'Only an administrator may read the folder tree for "{owner}".') + return owner + + +def _readable_ids(module, session): + # The projection is hygiene; the per-item list door is the wall. The client renders + # only listed items, so an unverifiable placement renders no tile. Use local voice + # records, never audience-option assembly, just to verify placement ids. + if module == "app": + return {row["id"] for row in routes_assets.application_index(session=session)["applications"]} + # ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read + # *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until + # it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a + # membership question. `routes_automation.automation_ids` keeps all three strata (stored + # definitions, field agents, the derived Odoo sync row) and the statements seed, so the SET is + # unchanged; what it drops is `_wire` per automation, which read `agent_contracts` and + # `automation_agent_actions` once EACH TIME (MEASURED: 1,101 ms of this route's 1,180 ms on + # staging, `AutoWiki/waves/wave62/proto/T17-profile.md`). + return ({"agent:" + aid for aid in routes_automation.automation_ids(session)} + | {"voice:" + row["id"] for row in + voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)}) + + +def _project(bucket, ids): + folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders")) + return {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids}, + "rev": bucket.get("rev", 0)} + + +def _available(session): + if not session.runtime.available(): + raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.") + + +@router.get("/{module}") +def read_module_index(module: str, user: str = "", session: Session = Depends(_access)): + try: + owner = _owner(session, user) + _available(session) + doc = session.runtime.get("module_index_" + module) or {} + bucket = (doc.get("users") or {}).get(owner) or {} + ids = _readable_ids(module, session) if bucket.get("itemFolders") else set() + return {"index": _project(bucket, ids)} + except SheetFolderRefusal as exc: + raise _refuse(exc) + + +@router.put("/{module}") +def write_module_index(module: str, body: dict = Body(default=None), user: str = "", + session: Session = Depends(_access)): + try: + owner = _owner(session, user) + # The admin override is for reading another user's tree, never for replacing it. + if owner != session.uname: + raise SheetFolderRefusal("module_index_user_forbidden", + f'The folder tree for "{owner}" can only be changed by its owner.') + _available(session) + raw = body or {} + expect_rev = raw.get("rev") + if type(expect_rev) is not int or expect_rev < 0: + raise SheetFolderRefusal("index_revision_required", "Reload folders before saving a folder change.") + placements = raw.get("itemFolders") + if not isinstance(raw.get("folders"), list) or not isinstance(placements, dict): + raise SheetFolderRefusal("module_index_invalid", "Send folders and itemFolders together to save the folder tree.") + ids = _readable_ids(module, session) if placements else set() + if any(item_id not in ids for item_id in placements): + raise SheetFolderRefusal("module_index_item_forbidden", + "A folder placement names an item you cannot read in this module.") + folders, placed, _ = validate_folder_tree(raw["folders"], placements) + saved = {} + + def apply(current): + state = copy.deepcopy(current) if isinstance(current, dict) else {} + users = state.setdefault("users", {}) + rev = (users.get(owner) or {}).get("rev", 0) + # Compare inside the store transaction, before touching any user's bucket. + if expect_rev != rev: + raise SheetFolderRefusal("index_revision_stale", + "Folders changed somewhere else. Reload folders and try again.") + saved.update(folders=folders, itemFolders=placed, rev=rev + 1) + users[owner] = dict(saved) + state["schema"] = 1 + return state + + session.runtime.update("module_index_" + module, apply, flush="sync") + return {"index": _project(saved, ids)} + except SheetFolderRefusal as exc: + raise _refuse(exc) diff --git a/api/routes_nav.py b/api/routes_nav.py index 13162190d3882c910b5a4e657df93f7f218306d6..6613020e5468ad25e844cf4a1c348fea72ae0f2a 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -81,6 +81,36 @@ def _now() -> int: return int(time.time()) +def _clean_linked_layouts(raw): + """R8 personal presentation only. Never accept a SavedView or a field definition.""" + if not isinstance(raw, dict): + return {} + layouts = {} + for key, value in raw.items(): + if not isinstance(key, str) or not key or not isinstance(value, dict): + continue + layout = {} + for name in ("order", "visible"): + if isinstance(value.get(name), list): + layout[name] = list(dict.fromkeys(k for k in value[name] if isinstance(k, str))) + if isinstance(value.get("widths"), dict): + layout["widths"] = {k: w for k, w in value["widths"].items() + if isinstance(k, str) and type(w) in (int, float) and w > 0} + if isinstance(value.get("sorts"), list): + layout["sorts"] = [{"colId": s["colId"], "dir": s["dir"]} + for s in value["sorts"] if isinstance(s, dict) + and isinstance(s.get("colId"), str) and s.get("dir") in ("asc", "desc")] + if isinstance(value.get("filters"), list): + layout["filters"] = [n for n in value["filters"] if isinstance(n, dict)] + if value.get("filterConj") in ("and", "or"): + layout["filterConj"] = value["filterConj"] + if "groupBy" in value and (value["groupBy"] is None or isinstance(value["groupBy"], str)): + layout["groupBy"] = value["groupBy"] + if layout: + layouts[key] = layout + return layouts + + def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. @@ -118,7 +148,35 @@ def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): placement[k] = v elif keep_unknown_ut and k.startswith("ut_"): placement[k] = v - return {"folders": folders, "placement": placement} + clean = {"folders": folders, "placement": placement} + layouts = _clean_linked_layouts(raw.get("linkedLayouts")) + if layouts: + clean["linkedLayouts"] = layouts + return clean + + +def _merge_nav_prefs(stored, body, page_keys, *, keep_unknown_ut=False): + """Replace navigation only when supplied; patch each personal link independently. + + Called inside runtime.update so concurrent link and folder writes see the latest entry. + Folder clients may echo a stale linkedLayouts from GET; only a layout-only request owns it. + """ + stored = stored if isinstance(stored, dict) else {} + body = body if isinstance(body, dict) else {} + navigation = "folders" in body or "placement" in body + merged = {**stored} + if navigation: + merged.update(folders=body.get("folders", []), placement=body.get("placement", {})) + else: + layouts = _clean_linked_layouts(stored.get("linkedLayouts")) + layouts.update(_clean_linked_layouts(body.get("linkedLayouts"))) + merged["linkedLayouts"] = layouts + # A layout write does not own navigation pruning either. + clean = _clean_nav_prefs(merged, page_keys, keep_unknown_ut=keep_unknown_ut) + if not navigation: + for name, default in (("folders", []), ("placement", {})): + clean[name] = stored.get(name, default) + return clean def _placeable_top_keys(session): @@ -397,6 +455,28 @@ def _visible_database_keys(session): return keys, enumerated +def _granted_view_sources(session): + """`{view_id: owner}` for grants this caller may read, from one registry read. + + `owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed + source lookup for that case; modern grants point directly at the canonical personal stratum. + """ + import core.shares as shares + try: + out = {} + for raw_id, record in shares.grants_bulk("view", st=session.runtime).items(): + owner = str(record.get("owner") or "").strip().lower() or None + if owner == session.uname: + continue + # Match the historical `shared_with` + `may_see` path: being an admin alone did not + # add a View to "shared with me"; a valid named or everyone grant did. + if shares.role_from(record, session.uname) is not None: + out[str(raw_id)] = owner + return out + except Exception: + return {} + + def _granted_view_ids(session): """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. @@ -406,21 +486,13 @@ def _granted_view_ids(session): answer "which views has this person starred", so the derivation moves here beside the reader it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). - ⚠ THE READ-PREDICATE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry - naming me"; `may_see` answers whether that entry currently grants this caller visibility. - Keeping those questions separate preserves dynamic all-eligible-user grants and fails closed - for a stale or malformed grant. One tenant-wide bucket is read once per call. + ⚠ THE READ PREDICATE STILL HAS ONE OWNER. `_granted_view_sources` obtains the registry once + through `grants_bulk` and delegates each held record to `shares.role_from`, the same pure + evaluator `may_see` uses. Named and dynamic-everyone grants therefore keep identical semantics + without one registry round trip per id. ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. """ - import core.shares as shares - try: - return {str(v) for v in - ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) - .get("view") or []) - if shares.may_see("view", str(v), session.uname, - st=session.runtime)} - except Exception: - return set() + return set(_granted_view_sources(session)) #: ⭐⭐ W54-T48 / OWNER RULING R4 / CONTRACT C4 — ONE CONNECTION, ONE FOLDER IN THE RAIL. @@ -959,7 +1031,7 @@ def nav_prefs(session: Session = Depends(require_session)): try: stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} except Exception: - stored = {} + raise err(503, "store_unavailable", "Your saved preferences could not be loaded.") keys, enumerated = _placeable_top_keys(session) return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} @@ -967,17 +1039,19 @@ def nav_prefs(session: Session = Depends(require_session)): @router.post("/nav/prefs") def save_nav_prefs(body: dict = Body(default=None), session: Session = Depends(require_session)): - """Wholesale replace, like the table folder stratum — the client sent its complete - picture, validated here; a partial merge would resurrect deleted folders forever.""" + """Replace the supplied navigation picture or patch personal linked-grid layouts.""" keys, enumerated = _placeable_top_keys(session) - clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) + clean = {} if not session.runtime.available(): raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.") def _up(data): + nonlocal clean data = data if isinstance(data, dict) else {} - if clean["folders"] or clean["placement"]: + clean = _merge_nav_prefs(data.get(session.uname), body, keys, + keep_unknown_ut=not enumerated) + if clean["folders"] or clean["placement"] or clean.get("linkedLayouts"): data[session.uname] = clean else: data.pop(session.uname, None) @@ -987,7 +1061,7 @@ def save_nav_prefs(body: dict = Body(default=None), session.runtime.update(_NAV_PREFS_KEY, _up) except Exception: raise err(503, "store_unavailable", - "the folder change was not saved: the store refused the write.") + "the preference change was not saved: the store refused the write.") return {"prefs": clean} diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index 025d65118a21cdce083f070823f0f2dd54ac028c..b5bd776e3ced4bff1200670c8dea7385c352c018 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -23,6 +23,7 @@ import hashlib import hmac import json import time +from contextlib import contextmanager from fastapi import Body, Depends, HTTPException from fastapi import APIRouter, Query @@ -1516,6 +1517,189 @@ def _related_source_value(session, entry, record): return None +def _materialised_table(table_key): + """Storage dispatch, independent of the browser's fallible status roster.""" + return module_cache_source(table_key) is not None or table_key not in _sources() + + +def _traversal_defn_or_refuse(session, table_key): + """R50 / C4: admit an intermediate after the caller has established both end grants. + + The session may already open both ends of a declared two-hop link. Its intermediate + is read only to find target ids, and only join values leave this traversal. The compiled + read still applies the row wall, hidden-field closure, BU scope and mirror guard. + """ + ut = _ut() + st = ut.lend_defs(session.runtime) + defn = ut.get(table_key, st=st) + if not defn: + raise err(404, "unknown_table", "that database does not exist") + return defn + + +def _relation_rows(session, table, columns, *, ids=None, match=None, where=None, + traversal_of=None): + """Internal join values, after the ordinary row wall, with no row-count ceiling. + + Join columns may be hidden on the wire. Only identities leave the resolver; the native + payload builders still apply their complete hidden-field closure before returning cells. + With traversal_of, the caller must already have established both ends of the declared + edge named by that key. Only join values leave this intermediate read, never a cell. + """ + import core.perm_scope as perm_scope + import core.relations as relations + + columns = set(columns) | {relations.IDENTITY_COL} + if not _materialised_table(table): + read = (_compiled_read(table, session, traversal_intermediate=True) + if traversal_of is not None else _compiled_read(table, session)) + spec = read["spec"] + bindings = dict(spec["cols"], pid=(spec["id"], int)) + if columns - set(bindings): + raise err(409, "relation_target_unavailable", "A required relation column is unavailable.") + parts = [p for p in (read["where"], where) if p] + params = list(read["params"]) + for key, values in (("pid", ids), match or ("pid", None)): + if values is None: + continue + expr, cast = bindings[key] + values = list(values) + parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0") + params.extend(cast(value) for value in values) + names = sorted(columns) + projection = ", ".join(bindings[key][0] for key in names) + source = spec.get("from_sql") or spec["table"] + predicate = " AND ".join(f"({part})" for part in parts) or "1=1" + return [dict(zip(names, row)) for row in read["cur"].execute( + f"SELECT {projection} FROM {source} WHERE {predicate}", params).fetchall()] + + # This is an internal join, never a cell response. Apply the same permanent wall on the + # complete contract before taking any join value, including a join on a hidden column. + if traversal_of is None and not perm_scope.may_read(session.user, table, st=session.runtime): + raise err(403, "forbidden_table", "This account cannot open the linked database.") + if module_cache_source(table) is not None: + session.require(table) + fields, rows = perm_scope._read(table, session.user, session.runtime) + rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime) + if where: + raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.") + if ids is not None: + wanted = {str(value) for value in ids} + rows = [row for row in rows if str(row.get("pid")) in wanted] + if match is not None: + key, values = match + wanted = {str(value) for value in values} + rows = [row for row in rows if str(row.get(key)) in wanted] + return [{key: row.get(key) for key in columns} for row in rows] + + +def _relation_cell_ids(value): + """Stored link cells contain comma-separated identities, not a scalar join value.""" + parts = value if isinstance(value, (list, tuple, set)) else str(value or "").split(",") + return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0} + + +def _relation_leg_ids(session, source, source_ids, entry): + """Follow one declared leg, including manual, inverse and through links.""" + import core.relations as relations + + target = str(entry["target"]) + via = entry.get("via") or {} + link = entry.get("link") or {} + reverse = via.get("direction") == "reverse" + frm, on = str(via.get("from") or "pid"), str(via.get("on") or "pid") + field = str(via.get("field") or "") + manual = bool(field and link and not link.get("inverse") + and (link.get("selectionMode") == "manual" or not link.get("on"))) + inverse = str(link.get("inverse") or "") + source_col, target_col = frm, on + if manual: + source_col, target_col = ("pid", field) if reverse else (field, "pid") + elif inverse: + source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse) + source_rows = _relation_rows(session, source, [source_col], ids=source_ids) + values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")} + # A column-keyed edge cannot fall back to the wire record id. In particular, + # product_data.pid is a stable hash while sale_order_line.product_id is an Odoo + # identity. Treating a missing/hidden/blank source value as an empty match would + # turn "the join input is unavailable" into the business claim "no related rows". + if source_col != relations.IDENTITY_COL and not values: + raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry)) + if (manual and not reverse) or (inverse and reverse): + values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows)) + target_col = "pid" + through = relations.declared_through(link) if not manual else None + if through: + left = str(link["throughTo"] if reverse else link["throughOn"]) + right = str(link["throughOn"] if reverse else link["throughTo"]) + middle = _relation_rows(session, str(through), [left, right], match=(left, values), + traversal_of=str(entry.get("key") or via.get("key") or "")) + values = {str(row[right]) for row in middle if row.get(right) not in (None, "")} + if (manual and reverse) or (inverse and not reverse): + rows = _relation_rows(session, target, [target_col]) + rows = [row for row in rows if _relation_cell_ids(row.get(target_col)) & values] + else: + rows = _relation_rows(session, target, [target_col], match=(target_col, values), + where=entry.get("where")) + return {int(row["pid"]) for row in rows} + + +def _relation_label(key): + """Prefer the database's display label in relation refusals.""" + source = module_cache_source(key) + if source and source.get("label"): + return source["label"] + return next((label for _bucket, table, label, _fields in _rel().TABLES + if table == key and label), str(key)) + + +def _relation_target_ids(session, target, related): + """One pin resolver for SQL, pool and tenant rows, selected from the scoped path list.""" + import core.relations as relations + from routes_grid import _may_open, related_path_entries + + if isinstance(related, str): + related = _json_arg(related, "related") + if not isinstance(related, dict) or not related.get("table") or related.get("record") in (None, ""): + raise err(400, "bad_relation", "A pinned read needs {table, record, via}.") + source = str(related["table"]) + via = related.get("via") + path = _via_path(via) + if path and len(path) > relations.MAX_DEPTH: + raise err(400, "over_depth", relations.over_depth_refusal(source, path)) + if not _may_open(session, source): + raise err(403, "forbidden_table", + f"the relation starts at '{_relation_label(source)}', which your account cannot open") + entries = related_path_entries(session, source, len(path) if path else 1) + entry = relations.pick_path(entries, path) if path else relations.pick(entries, via) + if entry is None or str(entry.get("target")) != str(target): + raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via)) + try: + ids = {int(related["record"])} + except (TypeError, ValueError): + raise err(400, "bad_relation", "The source record must have an integer identity.") + legs = [(source, entry)] + if len(entry.get("path") or ()) > 1: + first, middle, last = _related_path_legs(session, source, entry, entries) + legs = [(source, first), (middle, last)] + for table, leg in legs: + if not leg.get("served"): + raise err(409, "relation_not_served", relations.unserved_target_refusal(leg)) + if not _may_open(session, leg["target"]): + raise err(403, "forbidden_table", + f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open") + ids = _relation_leg_ids(session, table, ids, leg) + return ids + + +def narrow_related_rows(session, table_key, rows, related): + """Intersect an already walled and stripped payload with one resolved identity set.""" + if related in (None, "", {}): + return rows + ids = _relation_target_ids(session, table_key, related) + return [row for row in rows if int(row["pid"]) in ids] + + def _cache_relation_plan(session, source, via, target): """The offered module link and its explicit bridge, or a cause-specific refusal. @@ -1538,9 +1722,8 @@ def _cache_relation_plan(session, source, via, target): raise err(404, "relation_target_missing", f"'{target}' is not a current module/cache relation target in this workspace") if not _may_open(session, source): - raise err(403, "not_permitted", - f"'{source}' could not be opened for your account, so this relation cannot " - "be followed") + raise err(403, "forbidden_table", + f"the relation starts at '{_relation_label(source)}', which your account cannot open") offered = next((entry for entry in related_entries(session, source) if str(entry.get("key") or "") == via @@ -1557,7 +1740,7 @@ def _cache_relation_plan(session, source, via, target): if not isinstance(link, dict) or str(link.get("table") or "") != target: raise err(404, "relation_target_missing", f"the relation '{via}' no longer has a current target in this workspace") - through = str(link.get("through") or "") + through = relations.declared_through(link) through_on = str(link.get("throughOn") or "") through_to = str(link.get("throughTo") or "") on = str(link.get("on") or relations.IDENTITY_COL) @@ -1567,13 +1750,11 @@ def _cache_relation_plan(session, source, via, target): if relations.is_retired_table(through): raise err(404, "relation_target_retired", f"'{through}' is retired, so the relation to '{target}' is no longer available") - if not _may_open(session, through): - raise err(403, "forbidden_table", - f"the relation to '{target}' crosses '{through}', which your account cannot " - "open") + # R50: both end grants suffice. Traversal admission in _compiled_read replaces the + # intermediate grant check and still applies the intermediate's row wall. if not _may_open(session, target): raise err(403, "forbidden_table", - f"the relation ends at '{target}', which your account cannot open") + f"the relation ends at '{_relation_label(target)}', which your account cannot open") return dict(offered, __source__=source), link, through, through_on, through_to, on @@ -1601,15 +1782,23 @@ def _cache_relation_source_value(session, entry, record): def _cache_relation_target_ids(session, source, record, via, target): - """Resolve one explicit source -> mirror bridge -> module-cache path to target row pids.""" + """Resolve any offered path; retain the legacy Products ordered string transport.""" + via_key = str(via.get("key") or "") if isinstance(via, dict) else str(via or "") + if not (source == "customer_data" and target == "product_data" + and via_key == "products_ordered"): + ids = _relation_target_ids(session, target, {"table": source, "record": record, "via": via}) + return {"ok": True, "target": str(target), "ids": sorted(ids)} entry, _link, through, through_on, through_to, on = _cache_relation_plan( - session, source, via, target) + session, source, via_key, target) source_value = _cache_relation_source_value(session, entry, record) if source_value in (None, ""): raise err(400, "unknown_relation", - f"record {record} has no readable source value for relation '{via}'") + f"record {record} has no readable source value for relation '{via_key}'") - compiled = _compiled_read(through, session) + # R50 / C4: the intermediate of a DECLARED two-hop link is read to find target ids + # only, so it is admitted on both ends' grants. The compiler still applies its row + # wall, hidden-field closure and mirror guard to it. + compiled = _compiled_read(through, session, traversal_intermediate=True) spec = compiled["spec"] left = (spec.get("cols") or {}).get(through_on) right = (spec.get("cols") or {}).get(through_to) @@ -1658,7 +1847,11 @@ def odoo_relation_targets(source: str = Query(...), record: int = Query(..., ge= via: str = Query(...), target: str = Query(...), session: Session = Depends(require_session)): """Target pids for a declared relation whose destination is a runtime-cache database.""" - return _cache_relation_target_ids(session, source, record, via, target) + parsed_via = _json_arg(via, "via") if via.startswith(("[", "{")) else via + answer = _cache_relation_target_ids(session, source, record, parsed_via, target) + # Internal cache diagnostics (sourceKind/path) stay internal; the public door has one stable + # response shape for generic, legacy-string and structured-bag transports. + return {"ok": True, "target": str(target), "ids": list(answer.get("ids") or ())} def _via_path(via): @@ -1721,308 +1914,24 @@ def _related_path_legs(session, source, entry, entries): return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key) -def _related_hop_values(session, source, record, first, mid_key, last, notes): - """The values hop 2 joins on, resolved from the INTERMEDIATE database's own rows. - - ⭐⭐ W54-T40 — THIS IS THE HALF THAT MAKES A TWO-HOP TAB ANSWER ROWS INSTEAD OF A SENTENCE. - Hop 1 turns the source record into a set of intermediate rows; this returns the value each of - those rows carries in the column hop 2 is keyed on, and the caller pins THIS database by them. - - ⛔⛔ THE INTERMEDIATE IS READ THROUGH `_compiled_read`, WHICH IS THE SAME COMPILER THIS DOOR - USES FOR AN ORDINARY WINDOW, AND THAT IS THE WHOLE PERMISSION ARGUMENT. Its grant gate, its - administrator row wall and its hidden-field closure all run before a single id is produced, so - a traversal cannot be used to learn about rows of the middle database a reader may not see. A - second, lighter read here would be a second permission wall, and this file already carries the - rule that a second copy of that block does not drift towards refusing. - ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ REFUSES BY NAME. `product_data` is a compiled - registry module with no read-through binding, so the one enumerated path that passes through it - answers a sentence carrying the cause and a way round it, rather than an empty grid. - - ⚠ NO CAP, AND THE COST IS REPORTED INSTEAD (standing rule 1). Every distinct value comes back, - however many there are, and the reader is told through `limits` that this tab resolves a second - database on every page. - """ - import core.relations as relations - from fastapi import HTTPException # noqa: PLC0415 - - value = _related_source_value(session, first, record) - if value in (None, ""): - raise err(400, "unknown_relation", - relations.unresolvable_source_refusal(source, first)) - try: - mid = _compiled_read(mid_key, session) - except HTTPException as exc: - why = "" - if isinstance(getattr(exc, "detail", None), dict): - why = str((exc.detail.get("error") or {}).get("message") or "") - raise err(409, "relation_not_served", - f"this relation reaches '{last.get('target')}' through '{mid_key}', and that " - f"database could not be read here to follow it" - + (f" ({why})" if why else "") - + f". Open '{mid_key}' on its own, find the records this one is linked to, and " - f"follow the next relation from there") - mid_spec = mid["spec"] - on1 = str((first.get("via") or {}).get("on") or "") - step = (mid_spec.get("cols") or {}).get(on1) - if not step: - raise err(409, "relation_not_served", - f"'{mid_key}' has no live expression for '{on1}', which is the column the first " - f"hop of this relation joins on, so the traversal cannot be resolved. Open " - f"'{mid_key}' on its own, or ask for '{on1}' to be added to its live connection") - # ⭐ THE COLUMN HOP 2 IS KEYED ON, READ OFF THE INTERMEDIATE. `pid` is the row IDENTITY and is - # not a column of the mirror spec, so it resolves to the spec's own identity expression, which - # is the same one this route trusts everywhere else and is per table rather than a bare `id`. - frm2 = str((last.get("via") or {}).get("from") or relations.IDENTITY_COL) - take = (mid_spec["id"] if frm2 == relations.IDENTITY_COL - else ((mid_spec.get("cols") or {}).get(frm2) or (None,))[0]) - if not take: - raise err(409, "relation_not_served", - f"'{mid_key}' has no live expression for '{frm2}', which is the value the second " - f"hop of this relation is keyed on, so the traversal cannot be resolved. Open " - f"'{mid_key}' on its own, or ask for '{frm2}' to be added to its live connection") - try: - value = step[1](value) - except Exception: # noqa: BLE001 - value = str(value) - mid_from = mid_spec["from_sql"] if mid_spec.get("from_sql") else mid_spec["table"] - parts = [p for p in (mid["base_where"], first.get("where")) if p] - parts.append(f"{step[0]} = ?") - pred = " AND ".join(f"({p})" for p in parts) - rows = mid["cur"].execute( - f"SELECT DISTINCT {take} FROM {mid_from} WHERE {pred}", - list(mid["wall_params"]) + [value]).fetchall() - values = [r[0] for r in rows if r and r[0] is not None] - # ⭐⭐ STANDING RULE 1'S SECOND SENTENCE, AND `effect` IS CHOSEN SO THE READER GETS **THIS** - # SENTENCE RATHER THAN A NEIGHBOUR'S. `counts.ts::plainLimit` maps a known `effect` to one - # fixed client phrase and falls through to the SERVER's own first sentence for an effect it has - # not learned; `slow` maps to *"Rows this far down load slowly, each page is reached by walking - # every row above it"*, which is a true statement about a deep page and a FALSE one about a - # seven-row traversal. `traversal` is deliberately not in that table, so the strip prints the - # cause below, which is what actually happened. - # ⚠ THE FIRST CLAUSE IS THE ONE THAT REACHES THE STRIP: `firstSentence` cuts at the first full - # stop and truncates past 120 characters, so the mechanism is stated before the count. - notes.append({ - "subject": "related", "effect": "traversal", - "cause": f"these rows are reached through '{mid_key}', so every page resolves that " - f"database first. This record matched {len(values):,} of its records, and this " - f"database is then narrowed to the rows those reach", - "recommendation": f"open '{mid_key}' on its own if you are paging through a large set, or " - f"filter this database beside the relation to ask a smaller question"}) - return values def _related_pin(table_key, session, spec, base_where, wall_params, cur, related): - """`(sql, params, notes)` for C6's pinned read, or `(None, [], [])` when nothing is pinned. - - ⛔⛔ WHY THIS DOES NOT CALL `compile_filter_tree(member_ids=)`, WHICH IS WHAT C6'S SENTENCE - SAYS. Measured against `platform/harness/filter_sql.py` on 2026-09-03, that call cannot express - a pinned read and the failure is silent in the widening direction, twice over: - - 1. `:583` returns None the moment the tree is INACTIVE, and the `member_ids` branch is BELOW - that return. A pin with no client filter is exactly the inactive case, so the ids would be - dropped and the whole table served. - 2. `:588` emits `(tree OR id_sql IN (...))`. Its own docstring: *"`member_ids` reproduces - `members.has(r.pid) || matchFilterTree(...)`"*. That is a UNION. ORing a pin onto a - filter WIDENS; a pinned read needs an INTERSECTION. - - So the tail is compiled by `core.relations.pin_sql`, in `filter_sql`'s OWN spelling (proven - byte for byte by `verify_pivot.py`, not by this comment), and ANDed in. `filter_sql.py` is lane - B's file this wave; a narrowing member mode there is an AMENDMENT and is booked in the handoff, - not edited in from here. - - ⭐ `id_sql` IS PASSED EXPLICITLY, AS `spec["id"]`, AND IS NEVER DEFAULTED. `harness/semantic.py` - defaults it to `f"{alias}.id"` inside its OWN compile calls; a call from the route layer never - reaches that line, and `filter_sql:586` RAISES without it. It matters that it is per table - rather than a literal `id`: `ut_odoo_order_lines` is join-shaped and its identity is `sol.id`, - so a bare `id` would be ambiguous across `sale_order_line` and `sale_order`. It is the same - expression this route already trusts as `tiebreak_sql` and as the fallback ORDER BY. - - ⭐⭐ W54-T40 / INSTRUCTION 22 / R12 — **A `via` MAY NOW BE A PATH, AND THE PATH IS SERVED.** - The owner's second quoted error was this function's own `bad_relation` sentence, and it fired - on 18 of the 25 rail tabs of `customer_data`: R12 says the client pins a traversal by naming - each edge, so a two-hop tab sends `via` as the ARRAY `["invoices","order_link"]`, and the - guard below accepted only the one-hop bag. MEASURED 2026-09-06, 6 of 25 tabs answered 200 and - 18 answered 400 on the first click. - - ⛔⛔ ACCEPTING THE ARRAY IS HALF A FIX AND WOULD HAVE MOVED THE ERROR RATHER THAN CLOSED IT. - `relations.pick` matches `via.direction` against `DIRECTIONS`, and a two-hop entry's direction - is the literal `path`, deliberately outside that tuple so nothing serves half a traversal. So - a path resolves through `relations.pick_path` against the DEPTH-AWARE listing, and the read is - then answered in TWO steps: hop 1 resolves the intermediate database's own rows to the values - hop 2 joins on, and hop 2 pins this database by them. Sending the object form alone answered - `400 unknown_relation` and changed nothing the reader could see. - - ⭐ EVERY HOP IS STILL PERMISSION CHECKED, AND AT PIN TIME, NOT ONLY AT LISTING TIME (R13). The - intermediate is resolved through `_compiled_read`, i.e. THE SAME compiler this door uses for - an ordinary read, so its grant, its row wall and its hidden-field closure all apply before a - single id crosses into this table's predicate. A second, laxer read of the intermediate would - be exactly the permission wall with a hole in it that `related_entries` exists to prevent. - ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ IS A **REPORTED LIMIT**, never a silent empty: - `product_data` is a registry module with no read-through binding, so the one enumerated path - that passes through it refuses by name with a cause and a recommendation. - """ + """Compile the same resolved identity set on both mirror transports, as an intersection.""" import core.relations as relations if isinstance(related, str): related = _json_arg(related, "related") if related in (None, "", {}): return None, [], [] - if not isinstance(related, dict): - raise err(400, "bad_relation", - "`related` must be an object naming the record to pin these rows to: " - "{table, record, via}") - - source = str(related.get("table") or "") - record = related.get("record") - via = related.get("via") - # ⭐⭐ W54-T40 — `via` IS EITHER ONE EDGE'S BAG OR A PATH OF EDGE KEYS, and `_via_path` is the - # one place that decides which. It answers None for the bag, which is what keeps the guard - # below spelling the SAME refusal for a genuinely incomplete pin. - path = _via_path(via) - if not source or record in (None, "") or (path is None and not isinstance(via, dict)): - raise err(400, "bad_relation", - "a pinned read needs the record's database, the record id and the relation to " - "follow, as {table, record, via}, where `via` is one relation or the array of " - "edge keys naming each hop. Ask GET /api/v1/grid/" + (source or "{table}") - + "/related for the relations this account may follow") - - from routes_grid import _may_open, related_entries, related_path_entries - - if not _may_open(session, source): - # ⚠ THE CAUSE IS NAMED AS A PAIR, ON PURPOSE. `_may_open` is fail-closed and answers False - # both for a genuine permission refusal and for a database it could not read at all, so a - # sentence claiming only the first would tell a reader their access was revoked when the - # store had merely hiccupped. Naming both is honest and still refuses. - raise err(403, "not_permitted", - f"'{source}' could not be opened for your account, either because it is not " - f"granted to you or because it could not be read just now, so rows here cannot " - f"be pinned to one of its records. Ask an administrator for access to that " - f"database, or try again") - - # ⛔ RESOLVED AGAINST THE **SCOPED** LISTING, which is what makes "an edge you cannot be - # offered is an edge you cannot pin" structural rather than remembered. `related_entries` is - # the same function `GET /grid/{table}/related` answers with; there is no second resolution. - # ⭐ W54-T40 — AND A PATH RESOLVES AGAINST THE DEPTH-AWARE LISTING, WHICH IS THE SAME RULE ONE - # LEVEL DEEPER: `related_path_entries` is what `GET /grid/{table}/related?depth=2` answers - # with, so a traversal this account is not offered is a traversal it cannot pin either. - # ⚠ THE DEPTH ASKED FOR IS THE LENGTH OF THE PATH, never a flat 2. A one-element path costs - # the depth-1 enumeration it needs and not the depth-2 one, which on `customer_data` is the - # difference between roughly half a second and roughly two. - if path is None: - entries = related_entries(session, source) - entry = relations.pick(entries, via) - else: - if len(path) > relations.MAX_DEPTH: - raise err(400, "over_depth", relations.over_depth_refusal(source, path)) - entries = related_path_entries(session, source, len(path)) - entry = relations.pick_path(entries, path) - if entry is None: - raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via)) - entry = dict(entry, __source__=source) - - if str(entry.get("target") or "") != str(table_key): - raise err(400, "unknown_relation", - f"the relation '{entry.get('key')}' on '{source}' points at " - f"'{entry.get('target')}', not at '{table_key}', so it cannot narrow this " - f"database. Read the database the relation names, or ask for the relation that " - f"reaches this one") - if not entry.get("served", True): - raise err(409, "relation_not_served", relations.unserved_target_refusal(entry)) - - # ⭐⭐ W54-T40 — THE **LAST** HOP IS WHAT JOINS THIS DATABASE, and on a two-hop entry that is - # not `entry`. A composite carries `via = {direction: 'path', key, path}` and NO `on`, no - # `from` and no `where`: `relations._two_hop_entry` refuses to flatten the two hops' predicates - # onto the pair, because one of them may carry a predicate the other does not (the open-invoice - # bridge is exactly that edge). So the legs are resolved, each from its OWN table's scoped - # listing, and the last one is what this block below reads. + ids = _relation_target_ids(session, table_key, related) + sql, params = relations.pin_sql(spec["id"], sorted(ids)) notes = [] - first, mid_key, last = entry, "", entry - if len(entry.get("path") or ()) > 1: - first, mid_key, last = _related_path_legs(session, source, entry, entries) - - on = str((last.get("via") or {}).get("on") or "") - # ⚠ `spec["cols"]`, NOT the caller's narrowed `cols`. The join column is a JOIN, not a value - # this response returns: a relation must keep working when an administrator has hidden the id - # column it runs over, and no cell of it is ever projected. The hidden-field closure still - # governs everything that reaches the wire, one layer up. - # ⭐⭐ W54-T40 — AND THE ROW IDENTITY IS NOT ONE OF THOSE COLUMNS. `relations` defaults an edge - # with no declared `on` to `pid`, which is the wire's name for the row identity and is never a - # key of a mirror spec's `cols` (measured across all eight bound grains on 2026-09-06: not one - # carries it). Every such edge therefore refused with "has no live expression for 'pid'" while - # the expression was in hand the whole time, as `spec["id"]` - the same per-table identity this - # route already uses for the pin itself, for the tiebreak and for the fallback ORDER BY, and - # deliberately not a bare `id`, which is ambiguous on a join-shaped grain. - if on == relations.IDENTITY_COL: - binding = (spec["id"], int) - else: - binding = (spec.get("cols") or {}).get(on) - if not binding: - raise err(409, "relation_not_served", - f"'{table_key}' has no live expression for '{on}', which is the column this " - f"relation joins on, so the rows cannot be narrowed to one record here. Open " - f"this database directly, or ask for '{on}' to be added to its live connection") - on_expr, on_cast = binding[0], binding[1] - - if mid_key: - # ⭐ HOP 1, ANSWERED BY THE INTERMEDIATE DATABASE ITSELF, UNDER ITS OWN WALL. See - # `_related_hop_values`. An empty answer is an ANSWER (this record reaches nothing through - # that database) and compiles to `1=0` two blocks down, never to "no pin". - values = _related_hop_values(session, source, record, first, mid_key, last, notes) - else: - value = _related_source_value(session, entry, record) - if value in (None, ""): - raise err(400, "unknown_relation", - relations.unresolvable_source_refusal(source, entry)) - values = [value] - cast = [] - for one in values: - try: - cast.append(on_cast(one)) - except Exception: # noqa: BLE001 - cast.append(str(one)) - - # ── the member ids, in ONE SQL over the bridge column (the ticket's `how:`) ──────────────── - # ⛔ NOT THROUGH `datastore.window`: that clamps at `WINDOW_MAX` (5,000) rows, so a customer - # with more order lines than that would be pinned to the first 5,000 of them and the tab would - # show a smaller, entirely plausible number with nothing red anywhere. Standing rule 1: no cap - # on data from a connected source. - # ⚠ THE WALL RIDES ALONG. Resolving ids under `base_where` means the id SET a scoped reader can - # produce is already narrowed to rows they may see, so the pin cannot be used to learn how many - # rows exist beyond their boundary. - # ⭐ W54-T40 — `= ?` FOR ONE VALUE AND `IN (...)` FOR MANY, and the one-value spelling is kept - # BYTE FOR BYTE so a one-hop pin compiles exactly the SQL it compiled before this ticket. The - # many-value arm is only ever reached by the second hop of a traversal. - frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"] - ids = [] - if cast: - parts = [p for p in (base_where, last.get("where")) if p] - parts.append(f"{on_expr} = ?" if len(cast) == 1 - else f"{on_expr} IN ({','.join('?' for _ in cast)})") - pred = " AND ".join(f"({p})" for p in parts) - ids = [int(r[0]) for r in cur.execute( - f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {pred}', - list(wall_params) + list(cast)).fetchall() if r and r[0] is not None] - - # ⭐ STANDING RULE 1'S SECOND SENTENCE, ON A COST RATHER THAN ON A CAP. Nothing is limited - # here; the reader is told that this particular relation is answered by reading the source - # database once per page, because it is keyed on a column rather than on the record id. - if str((first.get("via") or {}).get("from") or "") != relations.IDENTITY_COL: + if len(ids) > _PIN_REPORT_AT: notes.append({ "subject": "related", "effect": "slow", - "cause": f"this relation is keyed on the '{first['via']['from']}' column of " - f"'{source}' rather than on its record id, so every page of these rows is " - f"resolved by reading that database first", - "recommendation": "read this database on its own and filter it on " - f"'{on}' if you are paging through a large set"}) - if len(ids) >= _PIN_REPORT_AT: - notes.append({ - "subject": "related", "effect": "slow", - "cause": f"this record is linked to {len(ids):,} rows, and each page names every one " - f"of them in the query that narrows this database to them", - "recommendation": "add a filter beside the relation to ask a smaller question, or " - "read this database on its own"}) - - sql, params = relations.pin_sql(spec["id"], ids) + "cause": f"This relation matched {len(ids):,} records, all of which are included.", + "recommendation": "Add a filter to reduce the result while continuing to page through every match."}) return sql, params, notes @@ -2300,7 +2209,7 @@ def _may_see_view(session, vid, store): def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None, - search=None, tree=_TREE_UNSET, related=None): + search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False): """EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once. Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits, @@ -2360,7 +2269,10 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= # tenant #0's eight databases is read-through. Left un-applied, an administrator could set a # filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows # used to be. So the wall is APPLIED here, in SQL, three paragraphs down. - defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + if traversal_intermediate: + defn = _traversal_defn_or_refuse(session, table_key) + else: + defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) fields = None for _bucket, key, _label, mk_fields in rel.TABLES: if key == table_key: @@ -2615,6 +2527,87 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= "pred": pred} +@contextmanager +def _materialised_read(table_key, session, query, related): + """One compiler for native rows and their aggregate sibling, over the walled payload. + + The temporary connection contains only this request's already stripped, pinned rows and is + closed before returning. It never registers data on the tenant's shared mirror connection. + """ + import duckdb + + if table_key == "customer_data": + from routes_customers import _payload + session.require(table_key) + payload = _payload(session, related=related) + elif table_key == "product_data": + from routes_products import products + session.require(table_key) + payload = products(session=session, related=related) + else: + from routes_tables import table_rows + payload = table_rows(table_key, session=session, related=related) + fields = payload["fields"] + with duckdb.connect(":memory:") as cur: + cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)", + [json.dumps(payload["rows"], default=str)]) + bindings = {} + for field in fields: + key = str(field["key"]) + pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''") + expr = f"json_extract_string(payload, '{pointer}')" + numeric = field.get("type") in ("int", "currency", "pct", "float", "number") + bindings[key] = (f"TRY_CAST({expr} AS DOUBLE)" if numeric else expr, + float if numeric else str) + spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)", + "cols": bindings, "where": None} + cols = _filter_columns(spec, fields) + limits = list(payload.get("limits") or []) + tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols, + None, [], cur, limits) + missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols) + if missing: + raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.") + try: + pred = _fs().compile_filter_tree(tree, conj=query["filterConj"], columns=cols, + id_sql=spec["id"], today=payload.get("today") or _today(), + cohort_sets=cohorts) + except ValueError as exc: + raise err(400, "filter_unsupported", str(exc)) + if pred is not None and pred.uses_aggregate: + raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.") + where, params = (pred.sql, list(pred.params)) if pred is not None else (None, []) + search = _fs().compile_search(query["search"], cols) if query["search"] else None + if search is not None: + where = f"({where}) AND {search.sql}" if where else search.sql + params.extend(search.params) + yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params, + "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload} + + +def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related): + from harness import datastore + + query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj, + "sorts": _json_arg(sorts, "sorts"), "search": search}) + with _materialised_read(table_key, session, query, related) as read: + spec = read["spec"] + order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"]) + win = datastore.window(select=f'{spec["id"]} AS pid', where=read["where"], + params=tuple(read["params"]), order_by=order or spec["id"], + offset=offset, limit=limit or None, cur=read["cur"], **read["frm"]) + payload = read["payload"] + by_id = {int(row["pid"]): row for row in payload["rows"]} + rows = [by_id[int(row[0])] for row in win["rows"]] + if win["clamped"]: + read["limits"].append({"subject": "limit", "effect": "window_clamped", + "cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.", + "recommendation": "Continue paging with offset. Every matching row remains available."}) + return {**payload, "ok": True, "rows": rows, "total": win["total"], + "totalUnfiltered": len(by_id), "offset": win["offset"], "limit": win["limit"], + "limits": read["limits"], "scope": {"table": table_key, "readThrough": True}} + + @router.get("/odoo-tables/{table_key}/rows") def odoo_table_rows(table_key: str, offset: int = Query(default=0, ge=0), @@ -2652,6 +2645,10 @@ def odoo_table_rows(table_key: str, import core.perm_scope as perm_scope from harness import datastore + if _materialised_table(table_key): + return _materialised_window(table_key, session, offset, limit, filters, filterConj, + sorts, search, related) + # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the # hidden-field closure, the user predicate and the search all live in `_compiled_read` above, # because C1 promises that `/aggregates` answers over *the same compiled predicate this window @@ -3041,6 +3038,13 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None), if group_by is not None and not isinstance(group_by, str): raise err(400, "bad_selection", "groupBy must be a column id") + if _materialised_table(table_key): + with _materialised_read(table_key, session, query, payload.get("related")) as read: + out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"], + read["params"], read["frm"], aggs, group_by=group_by, + limits=read["limits"]) + return {"ok": True, **out, "scope": {"table": table_key, "readThrough": True}} + # ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3). # `/rows` takes `related` as a JSON query STRING and this door takes it as a BODY KEY; a worker # who assumed both were bodies would have wired the rows and left the summaries answering over diff --git a/api/routes_products.py b/api/routes_products.py index 9c0537a9c7905d102c332a189180243de1f38fca..de358b42b63cdf0d52303c3d8adf805d6b2c4b6a 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -498,6 +498,8 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str str(pid): {key: value for key, value in (row or {}).items() if key not in hidden} for pid, row in (ws.get("overlays") or {}).items() } + # D-464: the saved views' column keys are the next wire `/workspace` serves. + workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden) today = time.strftime("%Y-%m-%d") fields = aios_grid.project_date_recurrences(fields, today=today) @@ -561,7 +563,7 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str @router.get("/products") -def products(session: Session = Depends(module_gate(MODULE))): +def products(session: Session = Depends(module_gate(MODULE)), related: str = None): """The product table for this session's scope — the /customers envelope byte-for-byte (`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`). @@ -573,13 +575,21 @@ def products(session: Session = Depends(module_gate(MODULE))): import aios_grid import modules.product_data as pd - g = product_assembly(session) + g = product_assembly(session, consume_corrections=not bool(related)) rows = aios_grid.rows_from_pool( g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) rows = _seed_image(_seed_shared(rows, g["rows_src"], g["fields"]), g["fields"]) from core import user_tables as _zones rows = _zones.zone_cells("product_data", rows, allowed_row_ids=g["pids"], st=session.runtime) + from routes_odoo_tables import narrow_related_rows + rows = narrow_related_rows(session, MODULE, rows, related) + from routes_tables import scope_linked_metrics, linked_metric_limits + rows, g['fields'], metric_refusals = scope_linked_metrics( + session, MODULE, rows, g['fields'], g['measures'], related) + if related: + pinned = {row["pid"] for row in rows} + g["rows_src"] = [row for row in g["rows_src"] if row["pid"] in pinned] # ⭐⭐ W41-T17 (owner instruction 25) — THE IDENTITY REPORT RIDES THE ENVELOPE, because a # report nothing reads is not a report. `product_data.identity_report` is standing rule 1's # second sentence as data for this grid's two identity columns: `product_id` (the Odoo @@ -599,12 +609,21 @@ def products(session: Session = Depends(module_gate(MODULE))): # The envelope check beside it filters to the four `/customers` keys, so an additive key is # the shape this route already established with `identity` and `scope`. return {"fields": g["fields"], "rows": rows, + "refusals": metric_refusals, + "limits": linked_metric_limits(metric_refusals), "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"), "identity": {"pid": "pid", "businessKey": "code"}, "identity_report": pd.identity_report(g["rows_src"]), "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}} +@router.post("/products/aggregates") +def product_aggregates(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from routes_odoo_tables import odoo_table_aggregates + return odoo_table_aggregates(MODULE, body=body, session=session) + + @router.patch("/products/{pid}") def patch_product(pid: int, body: dict = Body(default=None), session: Session = Depends(module_gate(MODULE))): @@ -769,7 +788,8 @@ def pd_fields(consolidated=True, st=None): # `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a # column `modules.product_data.pool()` reads off a SKU — and this one is neither read from # Odoo nor written to it. Injected after the JSON so the canonical file stays the answer to - # "what does Odoo give us", which is the question `verify_fields_contract` referees. + # "what does Odoo give us". `verify_fields_contract` does NOT referee the product contract + # (D-400); `platform/verify_product_pool.py::section_product_field_contract` does. if st is not None: image = pd.image_field_definition(st=st) else: diff --git a/api/routes_query.py b/api/routes_query.py index 83149a0e9004e4968162542e7343e93fbbebfeb6..75dc5d74852681e27f9d9a58907514ea35331e28 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -74,6 +74,19 @@ MAX_HISTORY_CHARS = 6000 MAX_RATING_REASON = 200 QUERY_RATINGS = ("up", "down") MODEL_AUTO = "auto" +QUERY_NO_SOURCE_MATCHED = ( + "None of the databases you can read cover that question. Pick one from the list, " + "or ask about a database you can open." +) +QUERY_DRAFT_ADMIN = "This database has no approved fields yet. Pick the ones this answer should use and they will be approved for everyone." +QUERY_DRAFT_MEMBER = "This database has no approved fields yet. Pick the ones you need and your administrators will be asked to approve them." +QUERY_DRAFT_SENT = "Sent to your administrators. Ask again once one of them approves." + + +def _draft_approved_line(name, date): + return f"Approved by {_no_dashes(name)} on {str(date)[:10]}." + + NO_CITED_SOURCES_REASON = ( "No authorized application data or retrieved knowledge was cited for this answer." ) @@ -166,6 +179,11 @@ def _answer_provenance(artifact=None): def _public_message(row, allowed_citations): """Project a stored answer through its current citation wall before it reaches the browser.""" out = _safe(row) if isinstance(row, dict) else {} + if isinstance(out.get("draft"), dict): + # Historical cards cannot replay samples admitted under an older sensitivity or row + # grant. GET /semantics/{database}/draft computes their fresh permissioned profiles. + for field in out["draft"].get("fields") or (): + field["sample"] = semantic_rt.redacted_profile() raw_citations = out.get("citationIds") citation_ids = [str(citation_id) for citation_id in (raw_citations if isinstance(raw_citations, list) else []) if str(citation_id) in allowed_citations] @@ -1114,6 +1132,8 @@ def _public_state(state, session): messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", "")) views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True) messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids] + messages = [row for row in messages if not isinstance(row.get("draft"), dict) + or row["draft"].get("database") in permitted] messages = [_public_message(row, allowed_citations) for row in messages] return {"threads": _safe(threads), "messages": messages, "views": views, "citations": _safe([row for key, row in state["citations"].items() @@ -1344,14 +1364,21 @@ def _submit(body, session, chat=None): raise err(400, "bad_request", "no question was asked") if len(question) > MAX_QUESTION: raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters") - if not target: - raise err(400, "bad_request", "one target database must be selected") if selected_model not in model_choices(): raise err(400, "unknown_model", "that model is not available in Query") + routed = _route(session, question) if not target else { + "database": target, "auto": False, "alternatives": []} + target = routed["database"] + route_refusal = (QUERY_NO_SOURCE_MATCHED if not target else routed.get("refusal")) + # This call happens before model selection and is the only permitted source-data read. - snapshot = assistant_read_scope(session, target, fields=None, filters=body.get("filters")) - sources = _sources(body, target, session) + snapshot = (assistant_read_scope(session, target, fields=None, filters=body.get("filters")) + if not route_refusal else None) + sources = _sources(body, target, session) if not route_refusal else ([target] if target else []) + if snapshot is not None and not routed["auto"]: + routed.update({"label": _no_dashes(snapshot["label"]), + "reason": f"You selected {_no_dashes(snapshot['label'])}."}) state = _state(session) thread_id = str(body.get("threadId") or "").strip() if thread_id and thread_id not in state["threads"]: @@ -1365,9 +1392,25 @@ def _submit(body, session, chat=None): # turn's own messages exist, so the replay is strictly the prior conversation. # ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s # docstring for why it is an argument and not ambient state (PRD amendment A1). - spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat, - history=_history(state, thread_id), session=session) - view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason) + draft_card = None + if not route_refusal: + _semantic_authorize(session, target) + records = _semantic_records(session, target) + if not semantic_rt.approved_fields(records, target): + draft_card = _draft_card(session, target, question, records) + else: + snapshot = _semantic_answer_snapshot(snapshot, records) + if route_refusal: + said, provider = route_refusal, None + view, refusal, refusal_code = (None, route_refusal, + "query_source_unavailable" if target else "query_no_source_matched") + elif draft_card: + said, provider = draft_card["alignmentQuestion"]["question"], None + view, refusal, refusal_code = None, said, "query_semantics_drafting" + else: + spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat, + history=_history(state, thread_id), session=session) + view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason) artifact = None citation = None @@ -1404,12 +1447,17 @@ def _submit(body, session, chat=None): assistant_message = { "id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now, "content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"), + # D-745: the Assistant restores routing from targetDatabase; keep it equal to routed["database"]. "targetDatabase": target, "requestedModel": selected_model, "model": provider, + "routedTo": routed, **_model_identity(provider), "reason": refusal_code, "viewId": artifact["id"] if artifact else None, "provenance": _answer_provenance(artifact), "citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None, } + if draft_card: + assistant_message.update({key: draft_card[key] for key in + ("status", "draft", "alignmentQuestion", "resumeToken")}) user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now, "content": question, "sources": sources, "targetDatabase": target, "requestedModel": selected_model} @@ -1451,18 +1499,147 @@ def submit_chat(body: dict = Body(default=None), session: Session = Depends(requ # Governed Query contract (Wave 45 C2). The existing chat workspace remains the presentation # surface; this provider-neutral door is the shared execution seam for Assistant, Agents, voice, # Slack and future adapters. -def _semantic_baseline(database): +def _semantic_baseline(database, session=None): from harness import semantic as sem topic = next((row for row in sem.topics().values() if isinstance(row, dict) and row.get("grid") == database), None) - if not topic: - raise err(404, "semantic_source_unknown", - "that database does not have a semantic contract") - return semantic_rt.topic_records(database, topic) + if topic: + return semantic_rt.topic_records(database, topic) + if session is not None: + try: + import odoo_relational + base_key, connection_id = odoo_relational.split_table_key(database) + if base_key is not None: + namespace = (odoo_relational.connection_namespace(session.runtime, connection_id)[1] + if connection_id else None) + choice = next((row for row in odoo_relational.grid_choices( + session.runtime, connection_id) if row.get("key") == database), None) + model = (choice or {}).get("model") + catalog = odoo_relational.readable_model_catalog(session.runtime, namespace) + row = next((row for row in catalog.get("models") or () + if model and row.get("model") == model), None) + if row is not None: + return semantic_rt.generated_records(database, row, model=model) + except Exception: + # An unavailable cached catalogue is an unknown schema, not a server failure. + pass + # A custom database's own declared schema supplies draft rows. This is metadata only; + # routing must not read records from every candidate just to recognize its label. + projection = getattr(session.runtime, "get_projection", None) + definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {} + definition = definitions.get(database) + if (database.startswith("ut_") and not database.startswith("ut_odoo_") + and isinstance(definition, dict)): + baseline = semantic_rt.topic_records(database, {"fields": definition.get("fields") or []}) + for field in definition.get("fields") or (): + row = baseline.get(semantic_rt.field_key(database, field.get("key"))) + if row is None: + continue + row["relation"] = str((field.get("link") or {}).get("table") or "") + if row["relation"] or row["dataType"] == "link": + row["role"] = "link" + elif row["dataType"] in ("date", "datetime"): + row["role"] = "date" + # Creating a source column does not itself grant tenant-wide approval. + row.update({"state": "draft", "approvedBy": None, "approvedAt": None}) + return baseline + raise err(404, "semantic_source_unknown", + "that database does not have a semantic contract") def _semantic_records(session, database): - return semantic_rt.load(session.runtime, _semantic_baseline(database)) + return semantic_rt.load(session.runtime, _semantic_baseline(database, session)) + + +def _route_candidates(session): + """The status wall is the sole source of candidate keys; read semantic metadata only.""" + from harness import semantic as sem + statuses = assistant_source_status(session, None) + candidates = [] + definitions = None + for database, status in statuses.items(): + if not status.get("permitted"): + continue + try: + topic = next((row for row in sem.topics().values() + if row.get("grid") == database), {}) + records = _semantic_records(session, database) + approved = semantic_rt.approved_fields(records, database) + if definitions is None: + projection = getattr(session.runtime, "get_projection", None) + definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {} + definition = definitions.get(database) or {} + label = _said(status.get("label") or definition.get("label") or + topic.get("label") or database.removeprefix("ut_").replace("_", " ")) or database + terms = [] + + def words(value): + if isinstance(value, str) and value.strip(): + terms.append(value) + terms.extend(re.findall(r"[^\W_]+", value)) + + words(label) + words(topic.get("label")) + words(topic.get("grain")) + scope = topic.get("scope") + for value in (scope.values() if isinstance(scope, dict) else [scope]): + words(value) + words((topic.get("semantic") or {}).get("scope")) + # ⭐⭐ ONLY A CURATED TOPIC LENDS ITS FIELDS TO THE VOCABULARY, AND THIS WAS MEASURED + # ON A RUNNING TENANT RATHER THAN REASONED ABOUT. W61-T28 gives every topic-less Odoo + # database a pre-approved map generated from Odoo's own `fields_get`, so + # `ut_odoo_order_lines` arrived carrying every raw field name `sale.order.line` has. + # Asked "invoiced sales last week", the router picked ODOO ORDER LINES: raw names + # matched both "invoiced" and "sale", and a generated map has dozens of fields where a + # curated topic has a dozen, so the long tail outvoted all eight curated topics. A raw + # column name is not a description of what a person is asking about. A generated + # database therefore routes on WHAT IT IS (its label, below) and never on the field + # names nobody wrote for this purpose; it stays fully readable once chosen, which is + # what R27 pre-approved it for. + if topic: + for field in approved.values(): + words(field.get("label")) + terms.extend(field.get("synonyms") or ()) + # Head words describe the field; explanatory tails often name OTHER databases. + words(" ".join(str(field.get("purpose") or "").split()[:4])) + candidates.append({"database": database, "label": label, + "topic": topic.get("label") or "", "terms": terms, + # What this database IS, as opposed to what it HOLDS. See + # `route_question`: a subject word outweighs a field word, which is + # what stops an orders question landing in the customer book. + "subject": [label, topic.get("label") or ""], + "curated": bool(topic), + "answerable": bool(status.get("answerable")), + "reason": _said(status.get("reason")) or ""}) + except Exception: + # One unavailable semantic contract must not hide the other permitted databases. + continue + return candidates + + +def _route(session, question): + from harness import semantic as sem + candidates = _route_candidates(session) + result = sem.route_question(question, candidates) + if not result["database"]: + return {"database": "", "label": "", "reason": QUERY_NO_SOURCE_MATCHED, + "auto": True, "alternatives": []} + readable = sem.route_question(question, [row for row in candidates if row["answerable"]]) + # Availability breaks equally strong matches, never substitutes a worse semantic match. + strength = lambda row: (row["score"], max( + (len(" ".join(sem._route_tokens(term))) for term in row["matched"]), default=0)) + chosen = readable if readable["database"] and strength(readable) == strength(result) else result + candidate = next(row for row in candidates if row["database"] == chosen["database"]) + matched = " and ".join(" ".join(re.findall(r"[^\W_]+", term)) + for term in chosen["matched"][:2]) + routed = {"database": candidate["database"], "label": candidate["label"], + "reason": f"{candidate['label']} matched {matched}.", "auto": True, + "alternatives": [{"database": row["database"], "label": _no_dashes(row["label"])} + for row in result["ranked"] if row["database"] != candidate["database"]][:2]} + if not candidate["answerable"]: + cause = candidate["reason"] or "this database cannot answer Query yet" + routed["refusal"] = f"{candidate['label']} cannot answer because {cause.rstrip('.')}." + return routed def _semantic_authorize(session, database, *, require_answerable=False): @@ -1536,7 +1713,7 @@ def update_semantic_field(database: str, field: str, body: dict = Body(default=N session: Session = Depends(require_session)): _semantic_authorize(session, database) try: - row = semantic_rt.update(session.runtime, _semantic_baseline(database), database, field, + row = semantic_rt.update(session.runtime, _semantic_baseline(database, session), database, field, body, actor=session.uname, is_admin=session.admin) except semantic_rt.SemanticError as exc: code = "forbidden" if "only a tenant admin" in str(exc) else "bad_semantic_contract" @@ -1550,7 +1727,7 @@ def profile_semantic_field(database: str, field: str, session: Session = Depends(require_session)): """Profile values through Query's permissioned snapshot and persist only a bounded summary.""" _semantic_authorize(session, database, require_answerable=True) - baseline = _semantic_baseline(database) + baseline = _semantic_baseline(database, session) records = semantic_rt.load(session.runtime, baseline) try: current = records[semantic_rt.field_key(database, field)] @@ -1574,6 +1751,115 @@ def profile_semantic_field(database: str, field: str, return {"field": row} +def _semantic_answer_snapshot(snapshot, records): + """Custom approvals govern model input too; sensitive values never become answer rows.""" + database = snapshot.get("database") or "" + custom = (database.startswith("ut_") and not database.startswith("ut_odoo_") + and not any(row.get("topic") for row in records.values())) + hidden = {row["field"] for row in records.values() + if row.get("sensitivity") in ("sensitive", "restricted")} + if not hidden and not custom: + return snapshot + out = copy.deepcopy(snapshot) + if custom: + approved = semantic_rt.approved_fields(records, database) + out["fields"] = [field for field in out.get("fields") or () if field.get("key") in approved] + out["records"] = [{key: value for key, value in row.items() if key == "pid" or key in approved} + for row in out.get("records") or ()] + for row in out.get("records") or (): + for field in hidden.intersection(row): + row[field] = None + return out + + +def _draft_card(session, database, question, records): + snapshot = assistant_read_scope(session, database, fields=None, filters=None) + fields = semantic_rt.draft_fields(records, snapshot) + statuses = assistant_source_status(session, None) + projection = getattr(session.runtime, "get_projection", None) + definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {} + from harness import semantic as sem + topic_labels = {row.get("grid"): row.get("label") for row in sem.topics().values()} + for field in fields: + record = records[semantic_rt.field_key(database, field["field"])] + relation = record.get("relation") or "" + relation = {"customer": "customer_data", "product": "product_data"}.get(relation, relation) + status = statuses.get(relation) or {} + if status.get("permitted"): + label = (status.get("label") or (definitions.get(relation) or {}).get("label") + or topic_labels.get(relation)) + if label: + field["why"] = field["why"].rstrip(".") + f", linked to {_no_dashes(label)}." + field["label"] = _no_dashes(field["label"]) + # Profiles are projections for display, never an authority for field keys or approval. + for key in ("values", "sample"): + field["sample"][key] = [_no_dashes(value) for value in field["sample"].get(key) or ()] + request = semantic_rt.draft_request(session.runtime, database) or {} + approved = semantic_rt.approved_fields(records, database) + approval = request if request.get("state") == "approved" else next(iter(approved.values()), {}) + state = "approved" if approved else "pending" + draft = {"database": database, "label": _no_dashes(snapshot.get("label") or database), + "canApprove": bool(session.admin), "fields": fields, + "sentToAdmins": not session.admin, "state": state, + "requestedBy": request.get("requestedBy") or session.uname, + "approvedBy": approval.get("approvedBy"), "approvedAt": approval.get("approvedAt"), + "sentMessage": QUERY_DRAFT_SENT} + if draft["approvedBy"] and draft["approvedAt"]: + draft["approvalLine"] = _draft_approved_line(draft["approvedBy"], draft["approvedAt"]) + return {"status": "clarification", "alignmentQuestion": { + "id": "draft_fields", "question": QUERY_DRAFT_ADMIN if session.admin else QUERY_DRAFT_MEMBER, + "choices": [{"field": field["field"], "label": field["label"]} for field in fields], + "maxSelections": 8}, "draft": draft, "resumeToken": ""} + + +@router.post("/query/semantics/{database}/approve") +def approve_semantic_draft(database: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _semantic_authorize(session, database) + baseline = _semantic_baseline(database, session) + body = body if isinstance(body, dict) else {} + fields = body.get("fields") + if (not isinstance(fields, list) or not fields or len(fields) > 8 + or any(not isinstance(field, str) for field in fields)): + raise err(400, "bad_semantic_contract", "pick between one and eight database fields") + fields = list(dict.fromkeys(fields)) + question = " ".join(str(body.get("question") or "").split()) + if not question or len(question) > MAX_QUESTION: + raise err(400, "bad_request", "a question of up to 500 characters is required") + try: + # Validate the whole selection before the first write, including member requests. + if any(semantic_rt.field_key(database, field) not in baseline for field in fields): + raise semantic_rt.SemanticError("that field is not in the database contract") + now = _dt.datetime.now(_dt.timezone.utc) + if not session.admin: + request = semantic_rt.record_draft_request( + session.runtime, database, question, fields, actor=session.uname, now=now) + return {"approved": [], "state": "pending", "sentToAdmins": True, + "requestedBy": request["requestedBy"], "sentMessage": QUERY_DRAFT_SENT} + for field in fields: + semantic_rt.update(session.runtime, baseline, database, field, {"state": "approved"}, + actor=session.uname, is_admin=session.admin, now=now) + if not semantic_rt.draft_request(session.runtime, database): + semantic_rt.record_draft_request( + session.runtime, database, question, fields, actor=session.uname, now=now) + request = semantic_rt.approve_draft_request( + session.runtime, database, fields, actor=session.uname, is_admin=session.admin, now=now) + except semantic_rt.SemanticError as exc: + raise err(400, "bad_semantic_contract", str(exc)) + return {"approved": fields, "approvedBy": request["approvedBy"], + "approvedAt": request["approvedAt"], "state": "approved", + "approvalLine": _draft_approved_line(request["approvedBy"], request["approvedAt"])} + + +@router.get("/query/semantics/{database}/draft") +def semantic_draft(database: str, session: Session = Depends(require_session)): + _semantic_authorize(session, database) + records = _semantic_records(session, database) + request = semantic_rt.draft_request(session.runtime, database) + return {**_draft_card(session, database, (request or {}).get("question") or "", records), + "request": request} + + def _resume_b64(raw): return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") @@ -1713,12 +1999,10 @@ def query_metrics(database: str = "", session: Session = Depends(require_session _semantic_authorize_schema_only(session, database) metrics, reason = _metric_catalogue(session, database) # ⛔⛔ THE GOVERNED SOURCE WALL STAYS, BUT IT CANNOT BE THE FIRST THING ASKED. - # `_semantic_records` resolves `_semantic_baseline`, which raises 404 `semantic_source_unknown` - # for a database no topic declares — and `ut_odoo_order_lines` / `ut_odoo_gl_lines` are exactly - # that: read-through grains with no entity topic. A 404 carries no body the picker can read, - # so the reader was told nothing at all where C7 promises a sentence. The wall is therefore - # asked only once a topic is known to exist, which is the only case where it has a contract to - # enforce; a topic-less database answers 200 with `reason: no_topic`, and that IS its answer. + # Cached Odoo fields can now supply a baseline without an entity topic, including order and + # GL lines. That schema does not create a validated entity-measure binding: keep returning + # `reason: no_topic` for this picker until a topic owns the measure offer. Other callers of + # `_semantic_records` still resolve the generated schema through their existing record wall. if reason != "no_topic": _semantic_records(session, database) return {"metrics": metrics, "reason": reason} @@ -1838,22 +2122,34 @@ def governed_query(body: dict = Body(default=None), body = body if isinstance(body, dict) else {} database = str(body.get("database") or "").strip() question = " ".join(str(body.get("prompt") or body.get("question") or "").split()) - if not database or not question: - raise err(400, "bad_request", "a database and prompt are required") + if not question: + raise err(400, "bad_request", "a prompt is required") if len(question) > MAX_QUESTION: raise err(400, "question_too_long", f"a prompt must be under {MAX_QUESTION} characters") - _semantic_authorize(session, database, require_answerable=True) + routed = _route(session, question) if not database else { + "database": database, "label": "", "reason": "", "auto": False, "alternatives": []} + database = routed["database"] + if database: + _semantic_authorize(session, database, require_answerable=not routed["auto"]) + route_refusal = QUERY_NO_SOURCE_MATCHED if not database else routed.get("refusal") + if route_refusal: + return {"status": "refused", "wording": route_refusal, "speakable": route_refusal, + "artifacts": None, "routedTo": routed, "model": _model_identity(None)} records = _semantic_records(session, database) + if not routed["auto"]: + from harness import semantic as sem + label = next((row.get("label") for row in sem.topics().values() + if row.get("grid") == database), database) + routed.update({"label": _no_dashes(label), "reason": f"You selected {_no_dashes(label)}."}) approved = semantic_rt.approved_fields(records, database) if not approved: - raise err(409, "query_semantics_unconfigured", - "this database has no approved semantic fields yet") + return {**_draft_card(session, database, question, records), "routedTo": routed} raw_requested = body.get("fieldKeys") if raw_requested is not None and not isinstance(raw_requested, list): raise err(400, "bad_request", "fieldKeys must be a list") requested = list(dict.fromkeys(str(x) for x in (raw_requested or ()) if str(x))) if not requested: - return _clarification(session, database, question, approved) + return {**_clarification(session, database, question, approved), "routedTo": routed} _verify_resume_token(body.get("resumeToken"), session, database, question, approved, requested) try: @@ -1861,12 +2157,13 @@ def governed_query(body: dict = Body(default=None), except semantic_rt.SemanticError as exc: raise err(403, "semantic_field_unapproved", str(exc)) snapshot = assistant_read_scope(session, database, fields=requested, filters=None) + snapshot = _semantic_answer_snapshot(snapshot, records) snapshot, formulas = _tool_formula_snapshot(body, snapshot, requested) spec, planned_wording, provider = _tool_plan(question, body, snapshot, session) if spec is None: return {"status": "refused", "wording": planned_wording, "speakable": planned_wording, "artifacts": None, - "model": _model_identity(provider)} + "model": _model_identity(provider), "routedTo": routed} view, refusal, refusal_code = _validate(spec, snapshot.get("fields") or selected) if view is None: raise err(400, refusal_code or "bad_query_artifact", refusal or "that Query is invalid") @@ -1891,7 +2188,7 @@ def governed_query(body: dict = Body(default=None), "rows": [{key: row.get(key) for key in view["visible"]} for row in rows], "groups": groups, "numeric": numeric, "artifacts": artifacts, "semanticRouter": semantic_rt.router_projection(records, database), - "model": _model_identity(provider)} + "model": _model_identity(provider), "routedTo": routed} @router.post("/query/formulas/parse") diff --git a/api/routes_sheet.py b/api/routes_sheet.py index fb6e7d8c9fb3477d656367473a775f45dfe05e29..02f11d4cccf6a79ee7da0edba19b0a76f147304b 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -221,6 +221,11 @@ def _registry_assembly(session: Session, table_key: str): if table_key in memo: return memo[table_key] try: + # W62 C2: a placed ut_* view needs its own field vocabulary, without fetching its rows. + if table_key.startswith("ut_"): + from routes_tables import ut_assembly + memo[table_key] = ut_assembly(session, table_key, consume_corrections=False, with_rows=False) + return memo[table_key] if table_key == "product_data": from routes_products import product_assembly memo[table_key] = product_assembly(session, scope="product", @@ -248,6 +253,159 @@ def _registry_assembly(session: Session, table_key: str): "shown rather than shown from an older set of columns. Try again in a moment.") +def _linked_customer_bridges(database): + """Only declared reverse customer edges with an unconditional, served relation.""" + from core.relations import BRIDGES + return [bridge for bridge in BRIDGES if bridge.get("target") == database + and bridge.get("source") == "customer_data" and bridge.get("served") + and bridge.get("cardinality") == "one_to_many" and not bridge.get("where")] + + +def _linked_customer_fields(session, bridge): + from core import perm_scope + _require_database(session, bridge["source"]) + g = _registry_assembly(session, bridge["source"]) + fields = list(g.get("fields") or []) + hidden = perm_scope.hidden_keys(session.user, bridge["source"], fields, st=session.runtime) + # Only values the server can read per record. A browser formula or linked rollup needs + # another evaluator; offering it here would make a missing value look like a blank value. + fields = [f for f in fields if f.get("key") not in hidden + and f.get("type") not in {"formula", "link", "lookup", "rollup", "metric"} + and not f.get("measure")] + return g, fields + + +def _linked_relation_column(session, bridge, cur, spec, columns): + from core import perm_scope + from modules import sheet as sheet_mod + key = str(bridge.get("target_col") or "") + subject = bridge["key"] + ":" + (key or "(missing column)") + hidden = perm_scope.hidden_keys(session.user, bridge["target"], spec.get("fields") or [], + st=session.runtime) + if key in hidden: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the relation column is hidden from this account.") + # The catalogue owns expressions such as sol.order_partner_id and aml.partner_id. + # The declaration supplies the key, and is a fallback only when the catalogue has none. + if (columns.get(key) or {}).get("sql"): + return key + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the link does not declare a readable relation column.") + source = spec.get("from_sql") or spec.get("table") + try: + cur.execute(f'SELECT {key} FROM {source} LIMIT 0') + except Exception: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the declared relation column is unavailable on this database.") + columns[key] = {"sql": key, "type": "int", "aggregate": False} + return key + + +@router.get("/sheets/filter-fields/{database}") +def linked_filter_fields(database: str, session: Session = Depends(require_session)): + """The linked Filter offer, through the same field wall as the related grid.""" + from modules import sheet as sheet_mod + database = _require_database(session, database) + bridges = _linked_customer_bridges(database) + if not bridges: + return {"fields": []} + cur, spec, columns = _mirror(session)(database) + columns = dict(columns) + offered = [] + for bridge in bridges: + try: + _linked_relation_column(session, bridge, cur, spec, columns) + _g, fields = _linked_customer_fields(session, bridge) + except HTTPException as exc: + if exc.status_code == 403: + continue + raise + except sheet_mod.SheetRefusal: + continue + link_label = f'{bridge.get("source_label") or bridge["source"]} via {bridge.get("label") or bridge["key"]}' + for field in fields: + # Copy the existing field vocabulary, including options. No customer values or + # hidden definitions travel on the picker wire. + offered.append({**field, + "key": f'{sheet_mod.LINKED_FILTER_PREFIX}{bridge["key"]}:{field["key"]}', + "label": f'{link_label}: {field.get("label") or field["key"]}'}) + return {"fields": offered} + + +def _linked_filter_scope(session, database, nodes, cur, spec, columns, *, today=None): + """Read linked values in Python, then give the relation-blind compiler flat predicates.""" + from modules import sheet as sheet_mod + if not sheet_mod.linked_filter_ids(nodes): + return nodes + import aios_grid + from core import perm_scope, shared_overlay, view_templates + from harness import filter_eval, filter_sql + + bridges = {b["key"]: b for b in _linked_customer_bridges(database)} + related = {} + + def resolve(leaf): + subject = leaf["colId"] + bridge_key, field_key = sheet_mod.linked_field_parts(subject) + bridge = bridges.get(bridge_key) + if bridge is None: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "this database no longer has that available customer link.") + try: + relation_column = _linked_relation_column(session, bridge, cur, spec, columns) + if bridge_key not in related: + g, fields = _linked_customer_fields(session, bridge) + # Use the rows door's wall, including its shared-field enrichment. The pool's + # identity is pid; it must never be confused with the orders' partner_id. + rows = perm_scope.apply_row_scope(g["rows_src"], session.user, bridge["source"], + aios_grid.FIELDS, st=session.runtime) + related[bridge_key] = (g, fields, rows) + g, fields, source_rows = related[bridge_key] + except HTTPException as exc: + if exc.status_code != 403: + raise + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "this account cannot read the linked customer database.") + field = next((f for f in fields if f.get("key") == field_key), None) + if field is None: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the field is hidden, removed, or has no readable per-record value.") + local = {**leaf, "colId": field_key} + # Validate before evaluating even an empty pool, so malformed/stale leaves refuse + # rather than looking like a genuine zero. Cell slots were resolved by W60-T06 above. + probe_columns = {field_key: {"sql": "linked_value", "type": sheet_mod.mirror_field_type(field)}} + try: + compiled = sheet_mod.compile_scope([local], "and", probe_columns, today=today) + if (local.get("op") not in filter_sql.FILTER_OPS or compiled is None + or field_key not in compiled.columns_used): + raise ValueError("unusable linked condition") + except (ValueError, sheet_mod.SheetRefusal): + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "its value or operator cannot be applied to this field.") + storage_key = view_templates.workspace_key(bridge["source"]) + try: + definitions = shared_overlay.fields(storage_key, st=session.runtime) + rows = aios_grid.rows_from_pool(source_rows, fields, (g.get("ws") or {}).get("overlays"), + derived=g.get("derived")) + if field_key in definitions: + cells = shared_overlay.cells(storage_key, [r["pid"] for r in source_rows], st=session.runtime) + rows = [{**row, field_key: (cells.get(str(int(row["pid"]))) or {}).get(field_key, "")} + for row in rows] + ctx = filter_eval.EvalCtx(today=today) + ids = [row[bridge["source_col"]] for row in rows + if filter_eval.permits({"conj": "and", "nodes": [local]}, row, fields, ctx)] + return sheet_mod.linked_id_predicate(relation_column, ids) + except Exception: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the linked customer values could not be read. Retry after the data is available.") + + try: + return sheet_mod.rewrite_linked_filters(nodes, resolve) + except sheet_mod.SheetRefusal as exc: + raise err(409, exc.code, exc.message) + + def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None): """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period. @@ -616,7 +774,7 @@ _CAP_CODES = ("sheet_too_many_cells", "workbook_too_large") #: The refusals that mean "that thing is not here", answered 404 so a client can tell a missing #: workbook from a malformed one without reading the sentence. -_MISSING_CODES = ("workbook_missing", "tab_missing") +_MISSING_CODES = ("workbook_missing", "folder_missing", "tab_missing", "unknown_sheet_template") #: ⛔⛔ C5's TWO SHARING REFUSALS ANSWER 403, AND THE TUPLE EXISTS BECAUSE THE FALL-THROUGH BELOW #: IS 400 (wave 51, W51-T25). Without this branch `workbook_read_only` would arrive as a NAMED 400 @@ -642,10 +800,8 @@ def _refuse(exc): nothing else, so the 409 and the 413 are raised here directly rather than by bending `err`, which would change the error shape of every route in the app. - ⚠ `workbook_revision_stale` (the PUT door's code) ALSO ENDS `_stale` and still lands here. It - carries no `rev_current`, so its body is unchanged and the client's existing PUT branch keeps - working. ⭐ D-639's `index_revision_stale` lands here too and DOES carry one, so the rail's - new guard needed no branch of its own: matching the family's spelling was the whole cost. + Workbook, retained cell history and folder index refusals all carry the current revision + through the same existing envelope. """ if exc.code in _CAP_CODES: return HTTPException(status_code=413, @@ -739,13 +895,23 @@ def _role_on_workbook(session: Session, book): round because a workbook that has never been shared has no registry record at all. Asking `shares.role_for` alone would answer `None` for the person who created it. """ - from core import shares - if session.admin: - return "owner" - me = str(session.uname or "").strip().lower() - if me and _workbook_owner(book) == me: - return "owner" - return shares.role_for("workbook", book.get("id"), me, st=_store(session)) + from core import sheets + return sheets.workbook_role(_store(session), book, session.uname, is_admin=session.admin) + + +def _require_reader(session: Session, book): + """R28: use the same predicate as the index; hidden and missing ids answer identically.""" + from core import sheets + if not sheets.may_read_workbook(book, _role_on_workbook(session, book)): + raise _refuse(sheets.SheetRefusal("workbook_missing", "That workbook no longer exists.")) + + +def _index_for(session: Session): + """C2: every list response uses the index's sole visibility filter and role helper.""" + from core import sheets + # The index already owns the bulk grant join. Passing the per-object route helper here + # converts that one read back into N full registry reads and made 50-row lists miss budget. + return sheets.read_index(_store(session), viewer=session.uname, is_admin=session.admin) def _require_editor(session: Session, book): @@ -778,6 +944,8 @@ def _wire_shares(rows): out = [] for row in rows or (): row = dict(row) + if "role" in row: + row["role"] = "owner" if row["role"] == "owner" else wire_role(row["role"]) row["shares"] = [{"who": s.get("who"), "role": wire_role(s.get("role"))} for s in (row.get("shares") or []) if wire_role(s.get("role"))] @@ -809,7 +977,24 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str: return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}) -def _reresolved_for_reader(session: Session, book): +def _has_ending(binding): + return (isinstance(binding, dict) and isinstance(binding.get("period"), dict) + and "ending" in binding["period"]) + + +def _has_cell_inputs(binding): + from modules import sheet as sheet_mod + if _has_ending(binding): + return True + scope = binding.get("scope") if isinstance(binding, dict) else None + nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None) + # Linked values and related-table grants can change independently of the orders' cache key. + if sheet_mod.linked_filter_ids(nodes): + return True + return any(sheet_mod.filter_cell_nodes(nodes)) + + +def _reresolved_for_reader(session: Session, book, *, endings_only=False): """A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5). ⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with @@ -850,6 +1035,8 @@ def _reresolved_for_reader(session: Session, book): for addr, cell in (tab.get("cells") or {}).items(): if not isinstance(cell, dict) or cell.get("kind") != "bound": continue + if endings_only and not _has_cell_inputs(cell.get("binding")): + continue touched += 1 binding = cell.get("binding") or {} db = str(binding.get("database") or "") @@ -861,18 +1048,14 @@ def _reresolved_for_reader(session: Session, book): seen_db[db] = exc failure = seen_db.get(db) if failure is not None: - cell["state"] = "refused" - cell["value"] = None - cell["display"] = "" - cell["refusal"] = {"code": "database_unreadable", - "message": f"You cannot read {db}."} + cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}.")) cell.pop("limits", None) continue cell_key = tab_ref + "!" + addr scope_key = _binding_scope_key(session, db, binding) hit = None - if wid and scope_key: + if wid and scope_key and not _has_cell_inputs(binding): try: hit = sheets.cached_scope_value(store, wid, cell_key, scope_key) except Exception: # noqa: BLE001 @@ -890,17 +1073,15 @@ def _reresolved_for_reader(session: Session, book): continue try: - out = _resolve_binding(session, binding, view_memo=view_memo) + out = _resolve_cell_binding(session, binding, fresh, tab_ref, view_memo=view_memo) except HTTPException as exc: detail = getattr(exc, "detail", None) or {} info = detail.get("error") if isinstance(detail, dict) else None - cell["state"] = "refused" - cell["value"] = None - cell["display"] = "" - cell["refusal"] = { - "code": (info or {}).get("code", "sheet_unresolved"), - "message": (info or {}).get("message", "This cell could not be refreshed."), - } + cell.update(_binding_refusal( + (info or {}).get("code", "sheet_unresolved"), + (info or {}).get("message", "This cell could not be refreshed."), + )) + cell.pop("limits", None) continue cell["state"] = "fresh" cell["value"] = out["value"] @@ -932,15 +1113,24 @@ def _reresolved_for_reader(session: Session, book): return fresh +def _response_workbook(book): + """Omit private write history from response tabs without changing the stored document.""" + return {**book, "sheets": [{key: value for key, value in tab.items() + if key not in ("writes", "writesFrom")} + for tab in book.get("sheets") or []]} + + def _read_for(session: Session, book): """The workbook this session is entitled to see: stored for the owner, re-resolved otherwise. - ⚠ A STRANGER IS A NON-OWNER AND TAKES THE SAME PATH, deliberately. Nothing in this file - refuses a stranger's READ today (see the module note the measurement is booked under), so the - fail-closed answer to "what should they be shown" is: whatever THEIR OWN scope computes, - never the owner's cached numbers. + W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers + and grantees still receive their own scoped numbers, never the owner's cached numbers. """ + book = _response_workbook(book) if _role_on_workbook(session, book) == "owner": + if any(_has_cell_inputs(cell.get("binding")) + for tab in book.get("sheets") or [] for cell in (tab.get("cells") or {}).values()): + return _reresolved_for_reader(session, book, endings_only=True) return book return _reresolved_for_reader(session, book) @@ -951,8 +1141,7 @@ def list_sheets(session: Session = Depends(require_session)): ⚠ C5's `shares` rides here as well as on `/sheets/index`, because the client reads both. """ - from core import sheets - rows = sheets.project_shares(_store(session), sheets.list_workbooks(_store(session))) + rows = _index_for(session)["workbooks"] return {"workbooks": _wire_shares(rows)} @@ -964,10 +1153,33 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req try: book = sheets.create_workbook(_store(session), name=raw.get("name") or "", - created_by=session.uname) + created_by=session.uname, + on_duplicate="refuse") except sheets.SheetRefusal as exc: raise _refuse(exc) - return {"workbook": book} + return {"workbook": _response_workbook(book)} + + +@router.post("/sheets/from-template/{key}", status_code=201) +def create_sheet_from_template(key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create a private workbook from a tenant-saved Sheet template.""" + from core import sheets, sheet_templates + book = None + try: + record = sheet_templates.saved_template(_store(session), key) + for database in (record or {}).get("databases", []): + _require_database(session, database) + book = sheet_templates.apply_saved_template( + _store(session), key, name=(body or {}).get("name") or "", created_by=session.uname) + # Owners read persisted results, so refresh under this maker before their first open. + return {"workbook": _response_workbook(refresh_sheet(book["id"], session=session)["workbook"])} + except Exception as exc: + if book is not None: + sheets.delete_workbook(_store(session), book["id"]) + if isinstance(exc, sheets.SheetRefusal): + raise _refuse(exc) + raise # ── the rail's own document (wave 50, W50-T17) ─────────────────────────────────────────────── @@ -982,17 +1194,10 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req # ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the # Sheet is granted to everyone, and the rail's folders name no database. # -# ⛔⛔ MEASURED IN W51-T25 AND BOOKED, NOT FIXED: THAT SENTENCE MEANS EVERY SIGNED IN ACCOUNT CAN -# ALREADY READ EVERY WORKBOOK. Nothing in `core.sheets.list_workbooks` or `::read_index` filters by -# owner, and `GET /sheets/{id}` walls on the session alone, so an account holding NO share reads a -# stranger's workbook with **200**. Driven in `verify_sheet.py` section 5c with a third identity. -# ⇒ `sharedWithMe` is therefore a PARTITION of a list everyone can already see, and C5's -# viewer/editor wall guards the WRITE doors rather than closing a read that was never shut. -# Refusing a stranger's read is not in W51-T25's `done-when` and widening it here would change the -# behaviour of a door six other tickets are built on, so it is an amendment finding for the root. -# ⚠ WHAT THIS TICKET DOES GUARANTEE MEANWHILE is the half that leaks data rather than existence: -# a non-owner never receives the owner's stored numbers, because every cell-bearing read -# re-resolves under the caller's own scope first. +# W60-T09, 2026-09-14: R28 supersedes the W51-T25 amendment finding. Both lists and all three +# workbook read shapes use core.sheets.may_read_workbook with _role_on_workbook. New workbooks +# are private to their creator and grantees; administrators see all. Unmarked or creatorless +# documents retain legacy visibility. Admitted non-owners still re-resolve under their own scope. @router.get("/sheets/index") @@ -1007,14 +1212,13 @@ def read_sheet_index(session: Session = Depends(require_session)): can reach: a client cannot send a revision it was never given, and a server that never checks one is a client stamping a number nobody reads. """ - from core import sheets - return {"index": _wire_index(sheets.read_index(_store(session), viewer=session.uname))} + return {"index": _wire_index(_index_for(session))} @router.put("/sheets/index") def write_sheet_index(body: dict = Body(default=None), session: Session = Depends(require_session)): - """Replace the folder, placement and star strata wholesale, or refuse by name. + """D-639 / W62-T25: replace the rail wholesale or merge a placements-only delta. ⭐ `repairs` RIDES BACK OUT. `core.sheets.save_folders` records every change it made to the payload, and the rail says so in a sentence rather than letting a workbook reappear at the @@ -1029,13 +1233,102 @@ def write_sheet_index(body: dict = Body(default=None), """ from core import sheets raw = dict(body or {}) + # This document is tenant-shared while its workbook rows are object-scoped. A response-only + # projection is not a write wall: without carrying these ids into the mutation, a stranger can + # move a hidden workbook by guessing its id, or save their empty projection wholesale and wipe + # the owner's folders/stars. Read the same projected index the GET returns, then make the core + # transaction preserve every stratum outside that projection. + visible = _index_for(session) + visible_ids = {str(row.get("id")) for row in (visible.get("workbooks") or []) + if isinstance(row, dict) and row.get("id")} + visible_folder_ids = {str(row.get("id")) for row in (visible.get("folders") or []) + if isinstance(row, dict) and row.get("id")} try: - out = sheets.save_folders(_store(session), raw.get("folders"), - raw.get("itemFolders"), starred=raw.get("starred"), - expect_rev=raw.get("rev")) + if "placements" in raw: + if set(raw) != {"placements"}: + raise sheets.SheetRefusal( + "index_request_mixed", + "Send placements alone, or send the full Sheet rail with its revision.") + if (not session.admin and isinstance(raw.get("placements"), dict) + and any(str(wid) not in visible_ids for wid in raw["placements"])): + # Hidden and nonexistent ids are deliberately identical: this write door is not + # an oracle for private workbook existence. + raise sheets.SheetRefusal("workbook_missing", + "That workbook no longer exists.") + out = sheets.merge_placements(_store(session), raw["placements"]) + else: + folders = raw.get("folders") + item_folders = raw.get("itemFolders") + if not session.admin: + submitted_workbooks = (set(str(wid) for wid in item_folders) + if isinstance(item_folders, dict) else set()) + submitted_stars = (set(str(wid) for wid in raw.get("starred")) + if isinstance(raw.get("starred"), list) else set()) + if (submitted_workbooks | submitted_stars) - visible_ids: + raise sheets.SheetRefusal("workbook_missing", + "That workbook no longer exists.") + # A wholesale snapshot cannot otherwise distinguish "create this client id" from + # "rename the private folder whose id I guessed". Current clients declare their + # newly-created temporary ids. Every created id is replaced server-side, whether + # it collides with a hidden folder or with nothing, so the response is not an + # existence oracle. + # + # Older clients predate `createdFolderIds`. Preserve their one safe, observable + # creation shape: at least one visible workbook is filed into the submitted new + # tree. In that legacy shape every unknown submitted id is treated as temporary + # and replaced, including empty siblings and ancestors. A bare undeclared id is + # still refused identically whether it is hidden or nonexistent, so the repair does + # not re-open the hidden-folder oracle while the old D-639 save door keeps working. + declared_creates = "createdFolderIds" in raw + created = raw.get("createdFolderIds", []) + if (not isinstance(created, list) + or any(not isinstance(fid, str) or not fid for fid in created)): + raise sheets.SheetRefusal( + "folder_create_ids_invalid", + "New folders must declare their temporary identifiers.") + created_ids = set(created) + incoming = [dict(row) if isinstance(row, dict) else row + for row in folders] if isinstance(folders, list) else [] + incoming_ids = {str(row.get("id")) for row in incoming + if isinstance(row, dict) and row.get("id")} + unknown = incoming_ids - visible_folder_ids + if declared_creates: + if unknown - created_ids or created_ids - unknown: + raise sheets.SheetRefusal("folder_missing", + "That folder no longer exists.") + elif unknown: + filed_into_unknown = { + str(fid) for wid, fid in (item_folders.items() + if isinstance(item_folders, dict) else []) + if str(wid) in visible_ids and fid is not None and str(fid) in unknown + } + if not filed_into_unknown: + raise sheets.SheetRefusal("folder_missing", + "That folder no longer exists.") + created_ids = unknown + replacements = {fid: "f" + uuid.uuid4().hex for fid in created_ids} + if replacements: + for row in incoming: + if not isinstance(row, dict): + continue + row["id"] = replacements.get(str(row.get("id")), row.get("id")) + if row.get("parent") is not None: + row["parent"] = replacements.get(str(row["parent"]), row["parent"]) + if isinstance(item_folders, dict): + item_folders = { + wid: replacements.get(str(fid), fid) + for wid, fid in item_folders.items() + } + folders = incoming + out = sheets.save_folders(_store(session), folders, + item_folders, starred=raw.get("starred"), + expect_rev=raw.get("rev"), + writable_ids=None if session.admin else visible_ids, + writable_folder_ids=(None if session.admin + else visible_folder_ids)) except sheets.SheetRefusal as exc: raise _refuse(exc) - return {"index": _wire_index(out)} + return {"index": _wire_index({**_index_for(session), "repairs": out.get("repairs", [])})} # ── C4: the .xlsx IMPORT, which is a JOB and not a request (wave 51, W51-T24) ──────────────── @@ -1131,9 +1424,11 @@ def _run_import(job_id, path, filename, runtime, uname): _job_set(job_id, report=report) stamp = _now_iso() - made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname) + made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname, + on_duplicate="suffix") saved = sheets.put_workbook(runtime, made["id"], - dict(book, refreshed_at=stamp, rev=made["rev"])) + dict(book, refreshed_at=stamp, rev=made["rev"]), actor=uname, + lossless=True) _job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"], sentence=_import_sentence(report)) except sheet_import.ImportRefusal as exc: @@ -1258,6 +1553,7 @@ def read_sheet(workbook_id: str, tabs: str = "", from core import sheets try: book = sheets.get_workbook(_store(session), workbook_id) + _require_reader(session, book) except sheets.SheetRefusal as exc: raise _refuse(exc) if str(tabs or "").strip().lower() == "meta": @@ -1294,6 +1590,7 @@ def read_sheet_tab(workbook_id: str, tab: str, from core import sheets try: book = sheets.get_workbook(_store(session), workbook_id) + _require_reader(session, book) served = _read_for(session, book) found = sheets.find_tab(served, tab) except sheets.SheetRefusal as exc: @@ -1319,9 +1616,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None same patch has landed. A CAP breach is the opposite: it is a whole request 413, because the document as a whole cannot be stored and there is no partial version of that. - ⚠ `rev` IN THE BODY MAKES THE WRITE CONDITIONAL, exactly as it does on the PUT door. Omit it - and the last writer wins; send the `rev` the tab read and a concurrent change answers 409 - `sheet_revision_stale` carrying `rev_current`, which is the revision to refetch. + With retained history, a stale revision merges by address and the later value wins; + `conflicts` names other people's overlapping writes, and only missing history refuses. ⛔⛔ C5's READ-ONLY CHECK RUNS BEFORE THAT COMPARISON AND THE ORDER IS THE CONTRACT. A viewer is refused 403 `workbook_read_only`; a viewer holding a stale rev is ALSO refused @@ -1347,7 +1643,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None expect = raw.get("rev") try: out = sheets.patch_cells(_store(session), workbook_id, tab, raw.get("cells"), - expect_rev=expect) + expect_rev=expect, actor=session.uname, + actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) accepted = out.pop("accepted", []) @@ -1396,11 +1693,32 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c (found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes} _recompute_formulas(book, fr, sheet_mod, cache_key=(session.tenant, session.uname, workbook_id), changes=edits) + # Date and filter-cell edits also change database totals. Recompute all readers, then formulas. + ending_cells = set() + for source in book.get("sheets") or []: + for addr, cell in (source.get("cells") or {}).items(): + binding = cell.get("binding") or {} + if cell.get("kind") != "bound" or not _has_cell_inputs(binding): + continue + try: + result = _resolve_cell_binding(session, binding, book, source.get("id")) + cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh") + cell.pop("refusal", None) + except HTTPException as exc: + info = ((exc.detail or {}).get("error") or {}) + cell.update(_binding_refusal( + info.get("code", "ending_cell_unreadable"), + info.get("message", "The Ending date could not be read. Choose a valid date."), + )) + ending_cells.add((source.get("id"), addr)) + if ending_cells: + _recompute_formulas(book, fr, sheet_mod) found = sheets.find_tab(book, tab_ref) out = {} for addr, cell in (found.get("cells") or {}).items(): if not isinstance(cell, dict) or (cell.get("kind") != "formula" - and "spill" not in cell and "spillParent" not in cell): + and "spill" not in cell and "spillParent" not in cell + and (found.get("id"), addr) not in ending_cells): continue row = {"value": cell.get("value"), "display": "" if cell.get("value") is None else str(cell.get("display") or ""), @@ -1435,10 +1753,12 @@ def write_sheet(workbook_id: str, body: dict = Body(default=None), # `rev` inside the body IS the expectation; `core.sheets.put_workbook` reads it. This door # does not translate it into a second argument, because two ways to say the same thing is # how one of them ends up ignored. - book = sheets.put_workbook(_store(session), workbook_id, raw) + book = sheets.put_workbook(_store(session), workbook_id, raw, + actor=session.uname, + actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) - return {"workbook": book} + return {"workbook": _response_workbook(book)} @router.delete("/sheets/{workbook_id}") @@ -1509,6 +1829,14 @@ def share_sheet(workbook_id: str, body: dict = Body(default=None), rec = shares.set_grants("workbook", book["id"], entries, owner=(None if held.get("owner") else _workbook_owner(book)), granter=session.uname, st=_store(session), expect=prior) + # A successful explicit share narrows legacy visibility; ordinary saves never do. + if "access" not in book: + # W60-T09, 2026-09-14: the first share decides VISIBILITY, making the owner gate load bearing. + # A non-owner must not reach this narrowing on an ownerless legacy workbook + # and lock themselves out by sharing it. + sheets.narrow_workbook(_store(session), book["id"]) + except sheets.SheetRefusal as exc: + raise _refuse(exc) except shares.GrantsChanged as exc: # ⚠ `grants_changed`, THE SAME CODE `routes_shares.py` ALREADY ANSWERS, and a plain 409 # rather than the `_stale` shape: there is no revision to hand back here, and a @@ -1601,6 +1929,8 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) for addr, cell in (tab.get("cells") or {}).items(): if not isinstance(cell, dict) or cell.get("kind") != "bound": continue + if _is_view_binding(cell): + continue # C2: a placed view refreshes through its own door binding = cell.get("binding") or {} db = str(binding.get("database") or "") if db not in seen_db: @@ -1611,22 +1941,19 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) seen_db[db] = exc failure = seen_db.get(db) if failure is not None: - cell["state"] = "refused" - cell["value"] = None - cell["refusal"] = {"code": "database_unreadable", - "message": f"You cannot read {db}."} + cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}.")) + cell.pop("limits", None) continue try: - out = _resolve_binding(session, binding, view_memo=view_memo) + out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo) except HTTPException as exc: detail = getattr(exc, "detail", None) or {} info = detail.get("error") if isinstance(detail, dict) else None - cell["state"] = "refused" - cell["value"] = None - cell["refusal"] = { - "code": (info or {}).get("code", "sheet_unresolved"), - "message": (info or {}).get("message", "This cell could not be refreshed."), - } + cell.update(_binding_refusal( + (info or {}).get("code", "sheet_unresolved"), + (info or {}).get("message", "This cell could not be refreshed."), + )) + cell.pop("limits", None) continue cell["state"] = "fresh" cell["value"] = out["value"] @@ -1652,16 +1979,20 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) _recompute_formulas(book, fr, sheet_mod) recalc_ms = round((perf_counter() - started) * 1000.0, 1) try: + # R19: book came from get_workbook; clamping on refresh would destroy imported widths. saved = sheets.put_workbook(_store(session), workbook_id, {"name": book.get("name"), "sheets": book.get("sheets"), - "refreshed_at": stamp, "rev": book.get("rev")}) + "refreshed_at": stamp, "rev": book.get("rev")}, + actor=session.uname, + actor_name=session.user.get("name") or session.uname, + lossless=True) except sheets.SheetRefusal as exc: raise _refuse(exc) # ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE # VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET # recomputes once, under their own scope, and fills its own slot again from there. sheets.clear_scope_cache(_store(session), workbook_id) - return {"workbook": saved, "recalc_ms": recalc_ms} + return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms} def _now_iso(): @@ -1916,7 +2247,150 @@ def _metric_column_aggregate(session, database, aggregate, window): return column(raw) -def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None): +def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()): + """Resolve a period from authored cells. Submitted ending.date is never trusted for a ref.""" + from modules import sheet as sheet_mod + from harness import windows as W + period = binding.get("period") + if not isinstance(period, dict) or (period.get("ending") or {}).get("kind") != "cell": + return sheet_mod.period_window(period, today) + ref = period.get("ref") or "(blank)" + value = _read_binding_cell(session, ref, book=book, tab=tab, today=today, seen=seen) + date = W.ending_cell_date(value, book.get("dateSystem", "1900")) + if date is None: + raise sheet_mod.SheetRefusal("ending_cell_unreadable", ref, "its value is not a real date.") + return sheet_mod.period_window(period, today, cell_date=date) + + +def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(), + code="ending_cell_unreadable"): + """Shared cell read for Ending and Filter, including scoped bound and formula inputs.""" + from modules import sheet as sheet_mod + import formula_runtime as fr + + def refuse(reason): + raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason) + + if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref): + refuse("its reference is invalid.") + if book is None: + refuse("the source workbook is unavailable.") + tabs = book.get("sheets") or [] + names = _tab_scope_names(tabs) + home = _resolve_home_tab(book, tabs, names, tab) + key = _asked_key(ref, home, fr) + if key in seen: + refuse("the reference is circular.") + cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names) + for addr, cell in (source.get("cells") or {}).items()} + cell = cells.get(key) + if not isinstance(cell, dict) or cell.get("kind") == "empty": + refuse("the cell or its sheet is missing or blank.") + if cell.get("kind") == "bound": + try: + value = _resolve_binding(session, cell.get("binding"), today=today, + book=book, tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"] + except HTTPException as exc: + reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.") + refuse(reason) + elif cell.get("kind") == "formula": + # The existing formula graph discovers actual inputs, including dynamic references. + # A database input starts refused, so a cached value can never bypass this reader's wall. + bound = {k: c for k, c in cells.items() if c.get("kind") == "bound"} + specs = {k: ({"refused": "ending_input_pending"} if k in bound + else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()} + engine = fr.SheetCalculation(specs, sheets=names, today=today, + named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900")) + attempted, failures = set(), {} + while True: + dependencies, pending = set(), [key] + while pending: + dependency = pending.pop() + if dependency in dependencies: + continue + dependencies.add(dependency) + pending.extend(engine.consumed.get(dependency, ())) + needed = (dependencies & set(bound)) - attempted + if not needed: + break + changes = {} + for dependency in needed: + attempted.add(dependency) + try: + result = _resolve_binding(session, bound[dependency].get("binding"), today=today, + book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency)) + changes[dependency] = {"value": result["value"]} + except HTTPException as exc: + info = ((exc.detail or {}).get("error") or {}) + failures[dependency] = info.get("message", "a database input is refused.") + changes[dependency] = {"refused": info.get("code", "ending_cell_unreadable")} + # Each pass resolves previously unread database cells, so the workbook bounds the loop. + engine.update(changes, revision=engine.revision + 1, today=today) + if engine.errors.get(key): + error_code = engine.errors[key] + reason = next((failures[k] for k in dependencies if k in failures), None) + if not reason: + reason = (sheet_mod.SheetRefusal(error_code, ref, "a formula input is refused.").message + if error_code in sheet_mod.REFUSALS else fr._sheet_error_message(error_code)) + refuse(reason) + value = engine.values.get(key) + else: + if cell.get("state") == "refused": + refuse((cell.get("refusal") or {}).get("message") or "the cell is refused.") + if cell.get("state") == "stale": + refuse("the cell needs to be refreshed.") + value = cell.get("value", cell.get("text")) + return value + + +def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen=()): + from modules import sheet as sheet_mod + try: + return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell( + session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable")) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) + + +# ⛔⛔ W62-T12 / C2 / R29 — A `mode: "view"` BINDING IS SKIPPED BY EVERY METRIC RESOLVER, AND +# SKIPPING IS NOT THE SAME AS REFUSING. A placed view has no scalar value: its rows are read through +# `POST /sheets/{id}/view-rows` and PLACED by the client, because the placement is a structure op +# only the client can take. Before this guard the refresh door handed a view binding to +# `modules/sheet.py::clean_binding`, which REQUIRES an `aggregate`, so `Refresh All` marked every +# placed view `refused` and painted "That cell's source is not set up correctly." over a view that +# was perfectly well formed. Caught by this lane's own screenshot, not by a gate. +def _is_view_binding(cell) -> bool: + """True for a C2 placed view, which the metric resolvers must leave exactly as they found it.""" + binding = (cell or {}).get("binding") if isinstance(cell, dict) else None + return isinstance(binding, dict) and binding.get("mode") == "view" + + +# R48: the persisted code is short; the original cause stays intact in the document. +_BINDING_REFUSAL_DISPLAY = { + "database_unreadable": "#NO ACCESS", + "field_unknown": "#FIELD?", +} + + +def _binding_refusal(code, message): + """One refusal shape for compute, refresh and caller-scoped reads of metric cells.""" + return { + "state": "refused", + "value": None, + "display": _BINDING_REFUSAL_DISPLAY.get(code, "#ERROR!"), + "refusal": {"code": code, "message": message}, + } + + +def _resolve_cell_binding(session, binding, book, tab, **kwargs): + """Keep the legacy resolver calling convention for bindings without workbook inputs.""" + if _has_cell_inputs(binding): + return _resolve_binding(session, binding, book=book, tab=tab, **kwargs) + return _resolve_binding(session, binding, **kwargs) + + +def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None, + book=None, tab=None, ending_seen=()): """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`. The order below is the contract, not a preference: @@ -1944,7 +2418,10 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non raise err(400, exc.code, exc.message) database = _require_database(session, binding["database"]) - window = sheet_mod.period_window(binding.get("period")) + try: + window = _ending_window(session, binding, book=book, tab=tab, today=today, seen=ending_seen) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) # W59-T14 / R10: the per-database wall is contract step 1. A metric key must not be looked # up or acted on before the caller passes it; the ordinary column guards below still apply. if binding["aggregate"].get("metric_key"): @@ -1966,12 +2443,16 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non scope.get("view_name", ""), memo=view_memo) else: scope_nodes, scope_conj = _fe_parts(scope.get("filter")) + scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab, + today=today, seen=ending_seen) needed = (sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes)) if binding["aggregate"].get("field"): needed.add(str(binding["aggregate"]["field"])) cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed) + cols = dict(cols) + scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today) # ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a # real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder @@ -2069,7 +2550,15 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None), field through the database's ordinary field writer; reusing a column needs no field write. """ raw = dict(body or {}) - out = _resolve_binding(session, raw.get("binding")) + binding = raw.get("binding") or {} + book = None + if _has_cell_inputs(binding): + from core import sheets + try: + book = sheets.get_workbook(_store(session), workbook_id) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + out = _resolve_cell_binding(session, binding, book, raw.get("tab")) return {"value": out["value"], "display": "" if out["value"] is None else str(out["value"]), "limits": out["limits"]} @@ -2196,6 +2685,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None), raise err(404, "cell_missing", f"Cell {key} is not filled in.") if cell.get("kind") != "bound": raise err(409, "cell_not_bound", f"Cell {key} is not bound to a database.") + if _is_view_binding(cell): + # C2: a placed view has rows, not a value. Refusing BY NAME beats resolving it as a metric, + # which is what painted "That cell's source is not set up correctly." over a good view. + raise err(409, "cell_is_a_view", + f"Cell {key} holds a view, so it is read through its own rows door.") tab_ref = str((tab or {}).get("id") or (tab or {}).get("name") or "") binding = cell.get("binding") or {} @@ -2203,20 +2697,17 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None), saved = dict(cell) limits, resolved = [], None try: - resolved = _resolve_binding(session, binding) + resolved = _resolve_cell_binding(session, binding, book, tab_ref) except HTTPException as exc: #: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door #: and a cell refused by that one read identically. It is PERSISTED rather than raised: #: a cell left `stale` would shimmer forever, and the reason belongs on the cell. detail = getattr(exc, "detail", None) or {} info = detail.get("error") if isinstance(detail, dict) else None - saved["state"] = "refused" - saved["value"] = None - saved["display"] = "" - saved["refusal"] = { - "code": (info or {}).get("code", "sheet_unresolved"), - "message": (info or {}).get("message", "This cell could not be refreshed."), - } + saved.update(_binding_refusal( + (info or {}).get("code", "sheet_unresolved"), + (info or {}).get("message", "This cell could not be refreshed."), + )) else: limits = resolved["limits"] saved["state"] = "fresh" @@ -2226,7 +2717,9 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None), saved.pop("refusal", None) try: - landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved}) + landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved}, + actor=session.uname, + actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) @@ -2250,6 +2743,104 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None), return {"cell": wire, "rev": landed.get("rev"), "tab": tab_ref, "addr": key} +@router.post("/sheets/{workbook_id}/view-rows") +def view_rows(workbook_id: str, body: object = Body(default=None), + session: Session = Depends(require_session)): + """W62 C2 / R29: read the live rows of a placed view under this caller's walls.""" + from datetime import date + from fastapi.responses import JSONResponse + from core import perm_scope, sheets, user_tables + from modules import sheet as sheet_mod + + try: + binding = sheets.clean_view_binding(body.get("binding") if isinstance(body, dict) else None) + except sheets.SheetRefusal as exc: + return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message}) + try: + book = sheets.get_workbook(_store(session), workbook_id) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + _require_reader(session, book) + database = _require_database(session, binding["database"]) + fields = list(_registry_assembly(session, database).get("fields") or []) + hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime) + visible = {field["key"]: field for field in fields if field.get("key") not in hidden} + requested = binding["fields"] + sorted_keys = {item["field"] for item in binding["sort"]} + for key in dict.fromkeys([*requested, *sorted(sorted_keys)]): + if key not in visible: + return JSONResponse(status_code=400, content={ + "error": "view_field_unknown", "message": f"Field {key} is not available in this database."}) + + # W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row." + wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter") + if perm_scope.row_scope_applies(session.user, database) else None) + wall_nodes, wall_conj = _fe_parts(wall_tree) + scope_nodes, scope_conj = _fe_parts(binding.get("filter")) + # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall. + for key in sorted(sheet_mod.filter_columns_named(scope_nodes)): + if not key.startswith(sheet_mod.LINKED_FILTER_PREFIX) and key not in visible: + return JSONResponse(status_code=400, content={ + "error": "view_field_unknown", "message": f"Field {key} is not available in this database."}) + today = date.today() + try: + scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today) + window = _ending_window(session, binding, book=book, today=today) + needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes) + | sheet_mod.filter_columns_named(wall_nodes)) + cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed) + cols = dict(cols) + scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today) + for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]): + if key in hidden or not (cols.get(key) or {}).get("sql"): + return JSONResponse(status_code=400, content={ + "error": "view_field_unknown", "message": f"Field {key} is not available in this database."}) + # W62 C2: an unanswerable row wall must refuse before either SQL read can widen it. + wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today) + if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled): + raise ValueError("unapplied row wall") + count_sql, params, limits = sheet_mod.build_aggregate_query( + spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes, + scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today) + if limits: + raise ValueError("unapplied view filter") + except sheet_mod.SheetRefusal as exc: + return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message}) + except ValueError: + raise err(409, "scope_not_applicable", + "This view uses a restriction that cannot be applied to this database's live connection.") + + # W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited. + total = (cur.execute(count_sql, params).fetchone() or [0])[0] + # ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD. + # A placed Grid over one of our own tables writes a record back, and the write door is Data's + # own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess + # from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or + # refreshes. So the id rides beside the values under the key `pid` while `fields` still names + # only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query` + # uses, so the identity here and the identity the drill lists can never be two different things. + id_expr = str(spec.get("id") or "").strip() + projection = ", ".join([*(cols[key]["sql"] for key in requested), + *([id_expr] if id_expr else [])]) + rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1) + if binding["sort"]: + # W62 C2 / R30: expressions come only from resolved columns; directions are validated. + order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC") + for item in binding["sort"]) + rows_sql += " ORDER BY " + order + rows_sql += f" LIMIT {binding['limit']}" + keys = [*requested, *(["pid"] if id_expr else [])] + rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()] + # W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus + # W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply. + editable = user_tables.records_mutable(database, st=session.runtime) + return {"fields": [{"key": key, "label": visible[key].get("label") or key, + "kind": visible[key].get("type") or visible[key].get("kind") or "text"} + for key in requested], + "rows": rows, "total": int(total or 0), "limit": binding["limit"], + "viewType": binding["viewType"], "editable": bool(editable)} + + @router.get("/sheets/{workbook_id}/cells/{addr}/drill") def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)): """`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17). @@ -2285,6 +2876,9 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s if cell.get("kind") != "bound": raise err(409, "literal_no_drill", "A hand typed value has no rows behind it.") + if _is_view_binding(cell): + raise err(409, "cell_is_a_view", + f"Cell {key} holds a view, so its rows are already on the sheet.") try: binding = sheet_mod.clean_binding(cell.get("binding")) @@ -2317,14 +2911,24 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s # `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the # window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from # the column set instead, where the time actually is. + try: + scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab.get("id")) + window = _ending_window(session, binding, book=book, tab=tab.get("id")) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) cur, spec, cols = _mirror(session)( - database, metric_window=sheet_mod.period_window(binding.get("period")), + database, metric_window=window, needed=(sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes))) + cols = dict(cols) + scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols) - count_sql, count_params, limits = sheet_mod.build_aggregate_query( - spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes, - scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj) + try: + count_sql, count_params, limits = sheet_mod.build_aggregate_query( + spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes, + scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) total = (cur.execute(count_sql, count_params).fetchone() or [0])[0] ids_sql, ids_params, _ = sheet_mod.build_ids_query( @@ -2388,7 +2992,7 @@ def _asked_key(asked, home, fr): @router.post("/sheets/{workbook_id}/resolve-bindings") def resolve_bindings(workbook_id: str, body: dict = Body(default=None), session: Session = Depends(require_session)): - """`{addr: value}` for the NAMED bound cells only (contract C1, ruling R2). + """Values or named refusals for the NAMED bound cells only (contract C1, rulings R2 and R48). ⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL. Under R2 a formula evaluates in the browser the instant it is committed, so the only thing the @@ -2404,20 +3008,13 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None), by THE EXACT STRING THE CALLER ASKED WITH, so a client looks its answer up under the reference it already holds and never has to re-derive this file's key spelling. - ⛔⛔ A KEY IS ABSENT WHEN NO BOUND CELL LIVES AT THAT ADDRESS, AND THAT IS THE ONE - DISTINCTION THE SHAPE CAN CARRY. An address that is empty, holds a typed literal or holds a - formula is not this door's business - lane A's evaluator already has those values in hand - - so asking for one costs nothing and answers nothing. - - ⛔⛔ AND HERE IS WHAT THE SHAPE CANNOT CARRY, STATED RATHER THAN HIDDEN: a bound cell whose - binding was REFUSED just now (a database wall, a row wall, a scope condition this database - cannot compile) comes back as `null`, which is the same thing an aggregate over zero matching - rows returns. The cause is real and this door knows it; `{addr: value}` has nowhere to put it. - `POST /sheets/{id}/refresh` and `GET /sheets/{id}` both still carry the full - `{code, message}` refusal on the cell, so nothing is lost from the document - what is lost is - the client's ability to paint THIS cell's refusal at commit time. Raised as a dated PRD - amendment rather than fixed by widening the shape on one lane's authority, because lane A is - the consumer and a second shape agreed by nobody is worse than a known gap. + A KEY IS ABSENT WHEN NO BOUND CELL LIVES AT THAT ADDRESS. An address that is empty, holds a + typed literal or holds a formula is not this door's business - lane A's evaluator already + has those values in hand - so asking for one costs nothing and answers nothing. + + R48: a refused binding answers `{value: null, refusal: {code, message}}`, so it cannot be + mistaken for a legitimate empty aggregate. The flat shape is kept for every address that + has an answer, including a genuine null, so no existing reader of those answers changes. ⛔ ONE BAD CELL NEVER FAILS THE BATCH, exactly as in `refresh_sheet`'s bound-cell loop: a refusal is caught per cell and the siblings still answer. @@ -2479,16 +3076,21 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None), if cell is None: continue binding = cell.get("binding") or {} - slot = json.dumps(binding, sort_keys=True, default=str) + own_tab = _asked_key(label, home, fr).rpartition("!")[0] + slot = json.dumps([own_tab, binding], sort_keys=True, default=str) if slot not in by_binding: try: - by_binding[slot] = _resolve_binding(session, binding, - view_memo=view_memo)["value"] - except HTTPException: - # ⛔ SWALLOWED HERE AND NOWHERE ELSE. See the docstring: the flat body has no - # room for a cause, and failing the whole batch on one walled cell would take - # every sibling's number down with it. The same refusal still travels in full on - # the refresh and read doors, which is where the cell's own sentence comes from. - by_binding[slot] = None + by_binding[slot] = _resolve_cell_binding(session, binding, book, own_tab, + view_memo=view_memo)["value"] + except HTTPException as exc: + detail = getattr(exc, "detail", None) or {} + info = detail.get("error") if isinstance(detail, dict) else None + by_binding[slot] = { + "value": None, + "refusal": { + "code": (info or {}).get("code", "sheet_unresolved"), + "message": (info or {}).get("message", "This cell could not be refreshed."), + }, + } answer[label] = by_binding[slot] return answer diff --git a/api/routes_starred.py b/api/routes_starred.py index e362f508289c23604ab1fa5ea252acfda7b47247..30d48707c737cdc3736102a95cb0dd4a8d2e3a11 100644 --- a/api/routes_starred.py +++ b/api/routes_starred.py @@ -28,9 +28,11 @@ disagree — a view deleted, a view unmarked from inside the database, a store w and not the other — the product has two answers to one question and no way to tell which is right. The scan is the price of one flag, and it is paid off the critical path. """ +import copy import hashlib import json import time +from concurrent.futures import ThreadPoolExecutor from fastapi import APIRouter, Body, Depends @@ -39,17 +41,17 @@ from deps import Session, err, require_session router = APIRouter(prefix="/api/v1") -#: The per-user bucket holding the three ID-LIST kinds. `{username: {databases, agents, queries}}` +#: The per-user bucket holding ID-LIST kinds. `{username: {databases, agents, queries, sheets}}` #: #: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about #: the object, so a tenant-wide star would put one person's shortlist on everybody's Home. _STARRED_KEY = "starred" -#: The four kinds C2 declares. `view` is the odd one out — see the header. -KINDS = ("database", "agent", "query", "view", "application") +#: Accepted star kinds. Views and applications keep their own record-backed write paths. +KINDS = ("database", "agent", "query", "view", "application", "sheet") -#: The three kinds this file's own bucket holds, and the payload key each answers under. -_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries"} +#: The kinds this file's own bucket holds, and the payload key each answers under. +_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries", "sheet": "sheets"} #: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with #: the cause and the action (unstar something), rather than being accepted and truncated. A star @@ -65,6 +67,88 @@ _MAX_ID = 120 #: container starts; warm it is not reached. VIEW_SCAN_BUDGET_S = 2.5 +# Four projected reads fit beneath the store pool's eight-connection ceiling while removing the +# serial N-read wall from Home. This is deliberately bounded: a tenant with many databases must +# not turn one Starred request into an unbounded connection burst. +VIEW_SCAN_WORKERS = 4 + +# One bounded process-wide pool, rather than one executor per authenticated request. A timed-out +# Python thread cannot be killed safely; a per-request executor therefore let repeated slow store +# calls grow one live worker apiece until the backend returned. The shared pool keeps that failure +# bounded to VIEW_SCAN_WORKERS and `future.cancel()` removes work that has not started yet. +_BATCH_READ_POOL = ThreadPoolExecutor( + max_workers=VIEW_SCAN_WORKERS, + thread_name_prefix="starred-batch", +) + + +class _RequestReadMemo: + """Read-through snapshot for one Starred payload assembly, never a cross-request cache.""" + + def __init__(self, runtime): + self._runtime = runtime + self._docs = {} + self._projections = {} + + def preload(self, names): + reader = getattr(self._runtime, "get_many", None) + if not callable(reader): + return + try: + answer = reader(names) + except Exception: + return + if isinstance(answer, dict): + self._docs.update(answer) + + def get(self, name, fresh=False): + key = str(name) + if not fresh and key in self._docs: + return copy.deepcopy(self._docs[key]) + answer = self._runtime.get(name, fresh=fresh) + if not fresh: + self._docs[key] = copy.deepcopy(answer) + return answer + + def get_projection(self, name, drop=()): + stamp = (str(name), tuple(str(key) for key in drop)) + if stamp not in self._projections: + self._projections[stamp] = self._runtime.get_projection(name, drop=drop) + return copy.deepcopy(self._projections[stamp]) + + def get_projections(self, names, drop=(), timeout_s=None): + keys = [str(name) for name in names or ()] + stamps = [(key, tuple(str(field) for field in drop)) for key in keys] + if all(stamp in self._projections for stamp in stamps): + return {key: copy.deepcopy(self._projections[stamp]) + for key, stamp in zip(keys, stamps)} + reader = getattr(self._runtime, "get_projections", None) + if not callable(reader): + return None + answer = reader(keys, drop=drop, timeout_s=timeout_s) + if answer is None: + return None + for key, stamp in zip(keys, stamps): + self._projections[stamp] = copy.deepcopy(answer.get(key, {})) + return {key: copy.deepcopy(self._projections[stamp]) + for key, stamp in zip(keys, stamps)} + + def __getattr__(self, name): + return getattr(self._runtime, name) + + +def _memoized_session(session): + """Preload the four small registries Starred rereads while assembling one response.""" + import application_registry + from core import shares + + runtime = _RequestReadMemo(session.runtime) + runtime.preload((_STARRED_KEY, routes_nav._NAV_RECENTS_KEY, + shares.SHARES_KEY, application_registry._KEY)) + wrapped = copy.copy(session) + wrapped.runtime = runtime + return wrapped + def _clean_id(raw): """One id, or "" — the only shape this file stores.""" @@ -90,7 +174,7 @@ def _clean_list(raw): def _read_lists(session): - """This caller's three id lists. A store blip answers empty rather than taking the page down.""" + """This caller's id lists. A store blip answers empty rather than taking the page down.""" try: mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {} except Exception: @@ -144,18 +228,56 @@ def _view_docs(session, order): import core.view_templates as view_templates docs, unread = {}, [] started = time.perf_counter() + readable = [] for key in order: ws_key = view_templates.workspace_key(key) if not ws_key: continue # a module surface or a folder head, not a table at all - if time.perf_counter() - started > VIEW_SCAN_BUDGET_S: - unread.append(key) - continue + readable.append((key, ws_key)) + + batch_reader = getattr(session.runtime, "get_projections", None) + if readable and callable(batch_reader): + # The Postgres implementation also installs a statement timeout, but the route keeps its + # own wall-clock guard: an optional store capability must not silently defeat the scan + # budget merely by ignoring that hint. The process-wide pool is deliberately bounded; + # creating one executor here would let repeated timeouts amplify live background workers. + remaining = max(0.001, VIEW_SCAN_BUDGET_S - (time.perf_counter() - started)) + future = None try: - docs[key] = session.runtime.get_projection(ws_key, - drop=("overlays", "fields")) or {} + future = _BATCH_READ_POOL.submit( + batch_reader, + [ws_key for _key, ws_key in readable], + drop=("overlays", "fields"), + timeout_s=remaining, + ) + batch = future.result(timeout=remaining) except Exception: - unread.append(key) + # One batch has one availability answer. Reporting every key as unread is honest; + # falling through to N retries would recreate the latency wall on an unhealthy store. + if future is not None: + future.cancel() + return {}, [key for key, _ws_key in readable] + if batch is not None: + for key, ws_key in readable: + docs[key] = batch.get(ws_key) or {} + return docs, unread + + for offset in range(0, len(readable), VIEW_SCAN_WORKERS): + if time.perf_counter() - started > VIEW_SCAN_BUDGET_S: + unread.extend(key for key, _ in readable[offset:]) + break + chunk = readable[offset:offset + VIEW_SCAN_WORKERS] + with ThreadPoolExecutor(max_workers=max(1, len(chunk))) as pool: + futures = [pool.submit(session.runtime.get_projection, ws_key, + drop=("overlays", "fields")) + for _key, ws_key in chunk] + # Consume in request order so recents-first remains the response order even when a + # later store read finishes first. + for (key, _ws_key), future in zip(chunk, futures): + try: + docs[key] = future.result() or {} + except Exception: + unread.append(key) return docs, unread @@ -207,8 +329,25 @@ def _scan_starred_views(session, keys=None, cache=None): import core.table_store as table_store import core.view_templates as view_templates - granted = routes_nav._granted_view_ids(session) - out = [] + granted = routes_nav._granted_view_sources(session) + # ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is + # where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases + # (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was + # 3,075 ms — `object_shares` read 28 times (1,117 ms) and `TableStore.find_view` 26 times + # (2,072 ms), for ONE granted id. Neither figure is about a slow read: the grant registry is + # 8 KB and each read is a single ~40 ms round trip. They are about the same question being + # asked once per database. + # + # ⭐ A VIEW LIVES IN EXACTLY ONE DATABASE AND ITS GRANT NAMES ITS OWNER. The modern fast path + # resolves that owner directly in the projected document this scan already paid for; it never + # re-reads the grant registry or calls `find_view` for every database. `resolved` still removes + # a found id from every later database's lookup work. + # + # ⛔ THIS IS NOT A LENT `object_shares` SNAPSHOT. `TableStore.find_view` reaches the legacy + # migration, which WRITES grants and reads them back; lending that key across the migration + # would serve a pre-write value and raise. The branch below detects retired shapes in the + # document already held, runs the unchanged strict migration, and refreshes that one document. + resolved, out = set(), [] for key in order: doc = docs.get(key) if doc is None: @@ -217,8 +356,43 @@ def _scan_starred_views(session, keys=None, cache=None): if not ws_key: continue views = table_store.make(ws_key, st=session.runtime) - for vid, view in routes_nav._visible_views(doc, session.uname, session.admin, - granted, views).items(): + pending = set(granted) - resolved + + # The old path called `find_view` once for every pending grant in every database. Each + # call strictly re-read this same workspace document and the whole share registry. The + # projected document above already contains every personal View, so modern grants can + # resolve directly through their canonical owner. A legacy document still takes the old + # strict migration door and is re-read afterwards; the optimization never skips a write. + if pending and views.prepare_view_projection(doc): + doc = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {} + docs[key] = doc + + found = {} + own = doc.get(session.uname) + for vid, view in ((own.get("views") or {}) if isinstance(own, dict) else {}).items(): + if isinstance(view, dict): + found[str(vid)] = view + + for vid in pending: + owner = granted.get(vid) + view = None + if owner: + owner_ws = doc.get(owner) + view = ((owner_ws.get("views") or {}).get(vid) + if isinstance(owner_ws, dict) else None) + else: + # Preserve the historical first-source lookup for a legacy ownerless grant. + for candidate_owner, workspace in doc.items(): + if candidate_owner in table_store.RESERVED_MEMBERS or not isinstance(workspace, dict): + continue + candidate = (workspace.get("views") or {}).get(vid) + if isinstance(candidate, dict): + view = candidate + break + if isinstance(view, dict): + found.setdefault(vid, view) + resolved.add(vid) + for vid, view in found.items(): cfg = view.get("config") or {} if cfg.get("important") is True: out.append({"id": vid, "database": key, @@ -289,13 +463,14 @@ def _query_names(session, qids): def _payload(session, keys=None, cache=None): - """The C2 answer: the three id lists, the starred views, and what could not be read. + """The star answer: the caller's id lists, starred records, and what could not be read. ⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s `omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a consumer forgets to test for. ⚠ `cache` is a scan an earlier pass in THIS request already paid for — see `_scan_starred_views`. """ + session = _memoized_session(session) lists = _read_lists(session) if keys is None: keys, _ = _database_keys(session) @@ -307,6 +482,8 @@ def _payload(session, keys=None, cache=None): # validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can # disclose nothing they did not already know, and B/C render only objects they can already see # (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap. + # Sheets are also the CALLER'S OWN writes. R28 filters the sheet index; C2 forbids a second + # access check here, so sheet ids pass through unchanged. lists["databases"] = [k for k in lists["databases"] if k in keys] # ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers # "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a @@ -327,7 +504,7 @@ def _payload(session, keys=None, cache=None): except Exception: application_rows = [] out = {"databases": lists["databases"], "agents": lists["agents"], - "queries": lists["queries"], "queryNames": qnames, + "queries": lists["queries"], "sheets": lists["sheets"], "queryNames": qnames, "views": views, "applications": application_rows, "degraded": unread} if unread: # R6's second sentence, in the payload: the CAUSE and what it means, not a bare list. @@ -503,7 +680,7 @@ def set_starred(body: dict = Body(default=None), session: Session = Depends(require_session)): """C2 — star or unstar ONE object, and answer with the caller's whole new list. - Body: `{"kind": "database"|"agent"|"query"|"view", "id": str, "on": bool}`, plus an OPTIONAL + Body: `{"kind": "database"|"agent"|"query"|"view"|"application"|"sheet", "id": str, "on": bool}`, plus an OPTIONAL `"database"` when `kind` is `view` (the fast path — see `_find_view`). ⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled diff --git a/api/routes_tables.py b/api/routes_tables.py index c6b1d7cd39880750563dcff17fc96d5158bae101..b73d44dee2b8b64a9a5cff72f2697fd7ad6fd985 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -68,33 +68,45 @@ class _MetricReadRuntime: raise AttributeError(name) -def _related_product_fields(session, rows): +def _related_product_fields(session, rows, table_key='product_data'): """Resolve authored Metric definitions through the existing workspace and field wall.""" import aios_grid from core import grid_events, perm_scope, shared_overlay from modules import product_data as pd + import routes_customers + + scope = {'product_data': 'product', 'customer_data': 'customer'}.get(table_key, table_key) + if table_key == 'product_data': + table, base = pd.table_ops(session.runtime), aios_grid.product_fields() + shared_key = pd.TABLE_KEY + elif table_key == 'customer_data': + table, base = routes_customers._customer_table(session), aios_grid.FIELDS + shared_key = routes_customers._shared_key() + else: + table = _ops(session, table_key) + base = perm_scope.scoped_fields(session.user, table_key, st=session.runtime) + shared_key = table_key pids = frozenset(r["pid"] for r in rows) ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=set(), admin=session.admin, fallback_ws=None, - seen_ids={}, st=session.runtime, scope_key="product", - table=pd.table_ops(session.runtime)) + seen_ids={}, st=session.runtime, scope_key=scope, table=table) ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) _, fields, _, _ = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key="product", with_cohorts=False, - fields_base=aios_grid.product_fields(), st=session.runtime) + ws, session.uname, set(pids), defs={}, scope_key=scope, with_cohorts=False, + fields_base=base, st=session.runtime) have = {f["key"] for f in fields} fields = list(fields) + [dict(f, source="overlay", shared=True) - for key, f in shared_overlay.fields(pd.TABLE_KEY, st=session.runtime).items() + for key, f in shared_overlay.fields(shared_key, st=session.runtime).items() if key not in have] - hidden = perm_scope.hidden_keys(session.user, "product_data", fields, st=session.runtime) + hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime) return [f for f in fields if f.get("measure") and f["key"] not in hidden] @router.post("/tables/{table_key}/related-metrics") def related_metrics(table_key: str, body: dict = Body(default=None), session: Session = Depends(require_session)): - """C6: a visible Products ordered edge, origin customer and real destination ids. + """R5: any offered relation, with the rows door's exact walled membership. The request supplies only grid identities and a named relation. Fields, periods, source identities and values are read from this tenant; nothing is authored by this endpoint. @@ -106,66 +118,79 @@ def related_metrics(table_key: str, body: dict = Body(default=None), import automation_engine body = body if isinstance(body, dict) else {} - pin = body.get("related") or {} + pin = body.get("related") pids = body.get("pids") - if (table_key != "product_data" or not isinstance(pin, dict) - or pin.get("table") != "customer_data" - or type(pin.get("record")) is not int or pin["record"] <= 0 - or not isinstance(pids, list) or not pids + if (not isinstance(pin, dict) or not isinstance(pids, list) or any(type(pid) is not int or pid <= 0 for pid in pids)): - raise err(400, "bad_metric_scope", "Name the origin customer and related product records.") + raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.") viewer = copy.copy(session) viewer.runtime = _MetricReadRuntime(session.runtime) try: - # Both endpoints and the actual bridge are permission-bearing databases. - for key in ("customer_data", "product_data", "ut_odoo_order_lines"): - viewer.require(key) - entries = routes_grid.related_path_entries(viewer, "customer_data", 1) - via = pin.get("via") - edge = relations.pick_path(entries, via) if isinstance(via, list) else relations.pick(entries, via) - if (edge is None or edge.get("target") != "product_data" - or edge.get("via", {}).get("field") != "products_ordered"): - raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.") - source_fields = routes_grid._source_fields(viewer, "customer_data") - hidden = perm_scope.hidden_keys(viewer.user, "customer_data", source_fields, st=viewer.runtime) - if "products_ordered" in hidden: - raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.") - customers = perm_scope.scoped_table(viewer.user, "customer_data", st=viewer.runtime) - if not any(r.get("pid") == pin["record"] for r in customers): - raise err(403, "metric_record_refused", "The origin record is not available to this account.") - products = perm_scope.scoped_table(viewer.user, "product_data", st=viewer.runtime) + compiled, team_id = routes_grid.related_measure_pin(viewer, table_key, pin, pids) + # Read actual rows, not the fields-only door introduced by T24. + products = perm_scope.scoped_table(viewer.user, table_key, st=viewer.runtime) requested = set(pids) selected = [r for r in products if r.get("pid") in requested] - if (len(selected) != len(requested) - or any(type(r.get("product_id")) is not int or r["product_id"] <= 0 for r in selected)): + if len(selected) != len(requested): raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.") - # A bridge restriction that cannot be translated must never be silently dropped. - bridge_fields = perm_scope.scoped_fields(viewer.user, "ut_odoo_order_lines", st=viewer.runtime) - declared = perm_scope.entry(viewer.user, "ut_odoo_order_lines") or {} - if (perm_scope.row_scope_applies(viewer.user, "ut_odoo_order_lines") - or (not viewer.admin and declared.get("hiddenFields")) - or not {"partner_id", "product_id"}.issubset({f["key"] for f in bridge_fields})): - raise err(409, "metric_scope_unsupported", "Order-line permission restrictions cannot be applied to these Metrics yet.") - scopes = [perm_scope.derive_pool_scope(viewer.user, key) - for key in ("customer_data", "product_data", "ut_odoo_order_lines")] - teams = {team for team, _ in scopes if team is not None} - agents = {agent for _, agent in scopes if agent is not None} - if len(teams) > 1 or len(agents) > 1: - raise err(403, "metric_scope_conflict", "The related databases have incompatible permission scopes.") - fields = _related_product_fields(viewer, products) - values, reasons = automation_engine.related_product_metric_values( - viewer.runtime, pin["record"], selected, fields, routes_products.product_measures(viewer), - team_id=next(iter(teams), None), agent=next(iter(agents), None)) + fields = _related_product_fields(viewer, products, table_key) + from core import measure_resolve + if not perm_scope.may_metrics(viewer.user, table_key): + offer = [] + elif table_key == 'customer_data': + offer = measure_resolve.offer(team_id) + elif table_key == 'product_data': + offer = routes_products.product_measures(viewer) + else: + offer = ut_measures(table_key, viewer) + admitted = {m['key'] for m in offer} + fields = [f for f in fields if f['measure'].get('key') in admitted] + values = automation_engine.related_metric_values( + viewer.runtime, table_key, selected, fields, offer, compiled, team_id=team_id) return {"ok": True, "values": values, "fields": [f["key"] for f in fields], - "scope": {"customer": pin["record"], - "products": {r["pid"]: r["product_id"] for r in selected}}, - "reason": " ".join(reasons)} + "refusals": values.refusals, + "scope": {"table": table_key, "related": pin, "pids": sorted(requested), + "customer": pin.get('record'), + "products": {r['pid']: r.get('product_id') for r in selected}}, + "reason": ' '.join(dict.fromkeys(r['message'] for r in values.refusals))} except (perm_scope.Denied, perm_scope.UnknownTable): raise err(403, "metric_record_refused", "The related records are not available to this account.") except (perm_scope.Unresolvable, RuntimeError, ValueError): raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.") +def scope_linked_metrics(session, table_key, rows, fields, offer, related): + """Replace metric cells before a pinned payload can be filtered or aggregated.""" + if not related or not any(f.get('measure') for f in fields): + return rows, fields, [] + from core import perm_scope + import routes_grid + import automation_engine + + admitted = {m['key'] for m in offer} if perm_scope.may_metrics(session.user, table_key) else set() + hidden = {f['key'] for f in fields if f.get('measure') + and f['measure'].get('key') not in admitted} + fields = [f for f in fields if f['key'] not in hidden] + rows = [{k: v for k, v in row.items() if k not in hidden} for row in rows] + metrics = [f for f in fields if f.get('measure')] + if not metrics: + return rows, fields, [] + pin, team_id = routes_grid.related_measure_pin(session, table_key, related, + [row['pid'] for row in rows]) + answer = automation_engine.related_metric_values(session.runtime, table_key, rows, metrics, + offer, pin, team_id=team_id) + return [{**row, **answer[row['pid']]} for row in rows], fields, answer.refusals + + +def linked_metric_limits(refusals): + """The same refusal accompanies rows, filters and aggregate siblings.""" + if not refusals: + return [] + return [dict(subject='Linked metrics', + cause=' '.join(dict.fromkeys(marker['message'] for marker in refusals)), + recommendation='Check the metric source binding and period, then reopen this tab.')] + + def _ut(): import core.user_tables as user_tables return user_tables @@ -1537,6 +1562,9 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict. shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden} for pid, cells in shared_cells.items()} + # D-464: the saved views' column keys, on the `/workspace` wire only; `views` stays whole + # for `_ut_measure_sets` below. + workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden) # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real @@ -2277,7 +2305,8 @@ def _thin_json(fields, merged, table_key=""): @router.get("/tables/{table_key}/rows") -def table_rows(table_key: str, session: Session = Depends(require_session)): +def table_rows(table_key: str, session: Session = Depends(require_session), + related: str = None): """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, identity}` — so the client's generic topic fetch consumes it with zero new parsing. @@ -2299,7 +2328,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the # value it replaced (contract C5). _lent = _ut().lend(session.runtime) - g = ut_assembly(session, table_key, st=_lent) + g = ut_assembly(session, table_key, st=_lent, consume_corrections=not bool(related)) # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge # this line does not perform since the route was written (owner item 3, 2026-08-09: # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). @@ -2361,6 +2390,12 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. _stamp_ai_states(table_key, g["fields"], rows, st=_lent) + from routes_odoo_tables import narrow_related_rows + rows = narrow_related_rows(session, table_key, rows, related) + rows, g['fields'], metric_refusals = scope_linked_metrics( + session, table_key, rows, g['fields'], g['measures'], related) + if related: + g["pids"] = frozenset(row["pid"] for row in rows) # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this # database says so here, with its cause and the recommendation, rather than waiting to be @@ -2382,16 +2417,24 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): if str(pid) in _zone_pids]) for zone in _ut().zones(table_key, st=_lent)] return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"], + "refusals": metric_refusals, "zones": _zones, "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, admin=session.admin, st=session.runtime), "pulled_at": time.strftime("%Y-%m-%d %H:%M"), "identity": {"pid": "pid"}, "scope": {"table": table_key, "rowCount": len(rows)}, - "limits": [_report] if _report else [], + "limits": ([_report] if _report else []) + linked_metric_limits(metric_refusals), "recordsMutable": _ut().records_mutable(table_key, st=_lent)} +@router.post("/tables/{table_key}/aggregates") +def table_aggregates(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + from routes_odoo_tables import odoo_table_aggregates + return odoo_table_aggregates(table_key, body=body, session=session) + + #: The per-cell provenance a row carries on the wire, one key per enrichment column. #: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. #: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new diff --git a/api/routes_templates.py b/api/routes_templates.py index ca5b6745664c79131a426c452d9c595caaca3361..4ed2048f15fa809750663c9e4a0f0d9aa7a1ed0c 100644 --- a/api/routes_templates.py +++ b/api/routes_templates.py @@ -72,13 +72,17 @@ def _target_or_refuse(session, table_key): @router.get("/templates") -def list_templates(table: str = "", session: Session = Depends(require_session)): +def list_templates(table: str = "", session: Session = Depends(require_session), module: str = ""): """What can be applied to THIS database. `{table, templates: [...]}`. Filtered by the target's real columns, so the picker cannot offer something the apply door would refuse — one predicate, two callers (`view_templates.offer` wraps `missing_columns`, which is the same function the POST below re-runs). """ + if module: + if module not in ("sheet", "data", "app"): + raise err(400, "unknown_module", "Choose Sheet, Data or App.") + return _module_templates(session) fields, source = _target_or_refuse(session, table) vt = _templates() return {"table": table, "templates": vt.offer(fields, source or None)} @@ -172,6 +176,134 @@ def _store(session: Session): return session.runtime +def _saved_sheet_templates(st): + """C4 metadata only. An absent or unrecognised document offers no saved templates.""" + raw = st.get("sheet_templates_saved") + if not isinstance(raw, dict) or raw.get("schema") != 1: + return [] + rows = raw.get("templates") + if not isinstance(rows, dict): + return [] + return [row for key, row in rows.items() + if isinstance(row, dict) and row.get("key") == key + and isinstance(row.get("name"), str) + and isinstance(row.get("description"), str) + and isinstance(row.get("workbook"), dict) + and isinstance(row.get("databases"), list) + and all(isinstance(db, str) and db.strip() for db in row["databases"])] + + +def _database_states(session, definitions, databases): + """Resolve switches once per request without rereading the stored table document.""" + from core import registry, user_tables + + # With no arguments the registrar returns its current keys without changing them. + # Match is_connected's three legs using the definitions projection already in hand. + registered = user_tables.register_connected() + prefixes = user_tables.connected_prefixes() + states = {} + for database in set(databases): + key = str(database or "").strip() + definition = definitions.get(key) or {} + if registry.BY_KEY.get(database): + states[database] = perms.tenant_has_module(session.runtime, database) + else: + # Process-wide registration alone must not admit another tenant's database. + states[database] = database in definitions and ( + key in registered or definition.get("connected") is True + or (any(key.startswith(prefix) for prefix in prefixes) + and definition.get("source") == user_tables.AUTOMATION_SOURCE + and definition.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE)) + return states + + +def _requirement(database, definitions, have, database_states, connector=""): + from core import registry + from routes_connectors import _module_connection_key, _table_connection_key + + definition = definitions.get(database) or {} + module = registry.BY_KEY.get(database) or {} + connector = (connector or _table_connection_key(database, definition) + or _module_connection_key(module.get("subject"))) + # These two compiled connector databases replaced the retired ut_odoo_* copies. + # Their switch is tenant admission; stored databases use the connector registrar. + switched_on = database_states[database] if module or connector else bool(definition) + label = definition.get("label") or module.get("label") + if not label: + parts = database.removeprefix("ut_").split("_") + brand = {"odoo": "Odoo", "amzn": "Amazon", "ig": "Instagram"}.get(parts[0], parts[0].capitalize()) + label = brand + " " + " ".join(parts[1:]) + return {"connector": connector, "database": database, + "label": str(label).replace(chr(0x2014), "-").replace(chr(0x2013), "-"), + "state": "disconnected" if connector and connector not in have + else "ready" if switched_on else "off"} + + +def _catalog_row(key, name, description, requires): + # Prefer the missing connection even when an earlier database is merely off. + blocked = next((row for row in requires if row["state"] == "disconnected"), None) + blocked = blocked or next((row for row in requires if row["state"] != "ready"), None) + return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"), + "description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"), + "requires": requires, "creatable": all(row["state"] == "ready" for row in requires), + "action": None if blocked is None else { + "kind": "connect" if blocked["state"] == "disconnected" else "switch_on", + "connector": blocked["connector"], "database": blocked["database"]}} + + +def _data_template_database(template, definitions): + fixed = {"odoo_customer": "customer_data", "odoo_product": "product_data"} + if template["source"] in fixed: + return fixed[template["source"]] + from routes_connectors import _table_connection_key + for key, definition in definitions.items(): + fields = [field.get("key") for field in definition.get("fields", [])] + if (_table_connection_key(key, definition) == template["source"] + and not _templates().missing_columns(template, fields)): + return key + return "" + + +def _module_templates(session): + from core import sheet_templates, user_tables + from routes_connectors import _entry_types + from routes_keychain import env_odoo_available + from routes_nav import application_templates + + definitions = user_tables.all_defs(session.runtime) + have, _scopes, _failure = _entry_types(session.runtime, session.uname) + have = set(have) + if env_odoo_available(session.runtime): + have.add("odoo") + saved = _saved_sheet_templates(session.runtime) + data_templates = [(row, _data_template_database(row, definitions)) + for row in _templates().TEMPLATES] + databases = {row["database_hint"] for row in sheet_templates.TEMPLATES} + databases.update(db for row in saved for db in row["databases"]) + databases.update(database for _row, database in data_templates if database) + database_states = _database_states(session, definitions, databases) + requirement = lambda database, connector="": _requirement( + database, definitions, have, database_states, connector) + sheet = [_catalog_row(row["key"], row["title"], row["answers"], + [requirement(row["database_hint"], row["source"])]) + for row in sheet_templates.TEMPLATES] + sheet.extend(_catalog_row(row["key"], row["name"], row["description"], + [requirement(db) for db in row["databases"]]) + for row in saved) + data = [] + for row, database in data_templates: + if database: + connector = "odoo" if row["source"].startswith("odoo_") else row["source"] + data.append(_catalog_row(row["key"], row["label"], row["desc"], + [requirement(database, connector)])) + # These App contracts map fields in the existing wizard and require no fixed connector. + app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", []) + for row in application_templates(session=session)["templates"]] + return {"sections": [{"module": "sheet", "templates": sheet}, + {"module": "data", "templates": data}, + {"module": "app", "templates": app}]} + + @router.get("/sheet-templates") def list_sheet_templates(session: Session = Depends(require_session)): """The workbook gallery, including sources this tenant has not connected.""" @@ -210,28 +342,35 @@ def apply_sheet_template(key: str, body: dict = Body(default=None), from routes_sheet import _require_database, refresh_sheet descriptor = sheet_templates.template(key) - if descriptor is None: + saved = (None if descriptor else + next((row for row in _saved_sheet_templates(_store(session)) if row["key"] == key), None)) + if descriptor is None and saved is None: raise err(404, "unknown_sheet_template", "That sheet template does not exist.") body = body if isinstance(body, dict) else {} database_key = body.get("database") name = body.get("name", "") - if not isinstance(database_key, str) or not database_key.strip(): + if saved is None and (not isinstance(database_key, str) or not database_key.strip()): raise err(404, "unknown_database", "That database does not exist.") if not isinstance(name, str): raise err(400, "bad_request", "The workbook name must be text.") - database_key = database_key.strip() + database_key = database_key.strip() if isinstance(database_key, str) else "" st = _store(session) - if database_key not in user_tables.all_defs(st): + if saved is None and database_key not in user_tables.all_defs(st): raise err(404, "unknown_database", "That database does not exist.") - _require_database(session, database_key) - if sheet_templates.source_of_database(database_key) != descriptor["source"]: + for required in saved["databases"] if saved else [database_key]: + _require_database(session, required) + if descriptor and sheet_templates.source_of_database(database_key) != descriptor["source"]: raise err(400, "template_source_mismatch", "That database uses a different connector from this template.") book = None try: - book = sheet_templates.apply_template(st, key, database_key, name=name, - created_by=session.uname) + if saved: + # Lane G owns C4 materialisation; never reconstruct its workbook here. + book = sheet_templates.apply_saved_template(st, key, name=name, created_by=session.uname) + else: + book = sheet_templates.apply_template(st, key, database_key, name=name, + created_by=session.uname) # Owners read their persisted values. Use the same permission-scoped # refresh as Sheet so the first open already contains computed cells. refreshed = refresh_sheet(book["id"], session=session)["workbook"] @@ -256,3 +395,34 @@ def apply_sheet_template(key: str, body: dict = Body(default=None), if isinstance(exc, (sheets.SheetRefusal, BindingRefusal)): raise err(409, exc.code, exc.message) from exc raise + + +# Declaration order is load bearing: keep this AFTER /templates/{key}/apply. +# Both paths have three segments; reversing them steals /templates/x/apply. +@router.post("/templates/{module}/{key}") +def create_module_template(module: str, key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + from routes_nav import create_nav_application + from urllib.parse import quote + + body = body if isinstance(body, dict) else {} + section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None) + template = next((row for row in section["templates"] if row["key"] == key), None) if section else None + if template is None: + raise err(404, "unknown_template", "That template does not exist.") + if not template["creatable"]: + raise err(409, "template_unavailable", "Connect the required source and switch on its databases.") + if module == "sheet": + database = template["requires"][0]["database"] if template["requires"] else "" + result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session) + return {"address": "#/sheet?wb=" + quote(result["workbook"]["id"], safe=""), **result} + if module == "data": + database = template["requires"][0]["database"] + result = apply_template(key, {"table": database}, session) + return {"address": "#/" + quote(database, safe=""), **result} + # The existing wizard still owns its POST-then-PATCH share sequence. API clients may + # dispatch creation here; the existing App writer retains schema and permission walls. + result = create_nav_application({**body, "templateKind": key}, session=session) + app = result["application"] + return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""), + **result} diff --git a/api/semantic_registry.py b/api/semantic_registry.py index 73d8ba247edaeb82107dcfc9fbdbcaae4d6092e8..9104cc77860c18810c9e665bac587e230097027f 100644 --- a/api/semantic_registry.py +++ b/api/semantic_registry.py @@ -9,6 +9,7 @@ import copy import hashlib import json from collections import Counter +from datetime import datetime, timezone SEMANTIC_STATES = ("draft", "approved", "disabled") @@ -17,6 +18,19 @@ VALUE_PROFILE_CAP = 50 VALUE_SAMPLE_CAP = 12 _KEY = "semantic_registry_v1" +# The catalogue fields_get attribute list belongs to odoo_relational.py (lane N); add selection there to replace profile-derived choice values with Odoo choices. +ODOO_SEMANTIC_TYPES = { + "char": "text", "text": "text", "html": "text", "selection": "select", + "boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency", + "date": "date", "datetime": "datetime", "many2one": "link", "one2many": "link", + "many2many": "link", "binary": "text", +} +_ODOO_PII_NAMES = ( + "email", "phone", "mobile", "fax", "vat", "street", "zip", "bank", "acc_number", + "iban", "password", "passwd", "token", "api_key", "secret", "signature", "ssn", + "birthday", "private", +) + class SemanticError(ValueError): """A semantic contract was refused before persistence or retrieval.""" @@ -42,36 +56,48 @@ def field_key(database, field): def model_record(database, topic, field): - """Build the governed baseline from a topic field declaration. - - Existing topic files predate approval. They therefore enter as ``draft`` unless the - declaration explicitly says otherwise; merely having a label never grants retrieval. - """ + """Build a baseline from topic curation, with per-field declarations taking precedence.""" field = field if isinstance(field, dict) else {} - semantic = field.get("semantic") if isinstance(field.get("semantic"), dict) else {} + key = field.get("key") + curated = (topic or {}).get("semantic") or {} + defaults = { + "state": "approved" if key in (curated.get("approved") or ()) else "draft", + "sensitivity": "sensitive" if key in (curated.get("sensitive") or ()) else "internal", + "purpose": (curated.get("purposes") or {}).get(key, field.get("means") or ""), + "synonyms": (curated.get("synonyms") or {}).get(key, []), + "choices": (curated.get("choices") or {}).get(key, []), + } + field_meta = field.get("semantic") if isinstance(field.get("semantic"), dict) else {} + semantic = {**defaults, **field_meta} # Per-field curation wins. state = str(semantic.get("state") or "draft").lower() sensitivity = str(semantic.get("sensitivity") or "internal").lower() if state not in SEMANTIC_STATES: raise SemanticError(f"unknown semantic state {state!r}") if sensitivity not in SENSITIVITY: raise SemanticError(f"unknown sensitivity {sensitivity!r}") - return { + return enforce_redaction({ "database": _text(database, 120), "topic": _text((topic or {}).get("key"), 120), "field": _text(field.get("key"), 120), "label": _text(field.get("label") or field.get("key"), 120), - "purpose": _text(semantic.get("purpose") or field.get("means"), 500), + "purpose": _text(semantic.get("purpose"), 500), "dataType": _text(field.get("type") or "text", 40), "role": _text(semantic.get("role") or "attribute", 60), "synonyms": _strings(semantic.get("synonyms")), + "relation": "", + "choices": copy.deepcopy(semantic.get("choices") or []), + "choicesFrom": "curated" if key in (curated.get("choices") or {}) or + "choices" in field_meta else ("profile" if field.get("type") == "select" else ""), "sensitivity": sensitivity, "state": state, "valueProfile": None, "revision": int(semantic.get("revision") or 1), - "approvedBy": _text(semantic.get("approvedBy"), 120) or None, + "approvedBy": (_text(semantic.get("approvedBy"), 120) or "platform baseline") + if state == "approved" else None, + "approvedAt": None, "disabledBy": _text(semantic.get("disabledBy"), 120) or None, "profiledBy": _text(semantic.get("profiledBy"), 120) or None, - } + }) def topic_records(database, topic): @@ -82,7 +108,39 @@ def topic_records(database, topic): } -def _clean_override(raw, baseline, *, actor, is_admin): +def generated_records(database, model_row, *, model=""): + """Pre-approved Odoo schema from the cached model's own fields, never inferred fields.""" + out = {} + model = model or model_row.get("model") or "" + for name, metadata in (model_row.get("fields") or {}).items(): + if not isinstance(metadata, dict) or not name: + continue + odoo_type = metadata.get("type") + relation = str(metadata.get("relation") or "") + sensitive = any(part in str(name).casefold() for part in _ODOO_PII_NAMES) + choices = metadata.get("selection") + choices = copy.deepcopy(list(choices)) if isinstance(choices, (list, tuple)) else [] + role = ("link" if relation else "measure" if odoo_type in ("monetary", "float") else + "date" if odoo_type in ("date", "datetime") else "attribute") + row = model_record(database, {}, { + "key": name, "label": metadata.get("string") or name, + "type": ODOO_SEMANTIC_TYPES.get(odoo_type, "text"), + "semantic": {"state": "approved", "approvedBy": "odoo connector baseline", + "sensitivity": "sensitive" if sensitive else "internal", "role": role, + "purpose": f"The {name} field on Odoo model {model}."}, + }) + row.update({"relation": relation, "choices": choices, + "choicesFrom": "odoo" if choices else + ("profile" if odoo_type == "selection" else "")}) + out[field_key(database, name)] = row + return out + + +def _approval_time(now=None): + return (now or datetime.now(timezone.utc)).astimezone(timezone.utc).isoformat() + + +def _clean_override(raw, baseline, *, actor, is_admin, now=None): if not is_admin: raise SemanticError("only a tenant admin can change semantic approval") raw = raw if isinstance(raw, dict) else {} @@ -111,6 +169,7 @@ def _clean_override(raw, baseline, *, actor, is_admin): raise SemanticError("semantic state is not recognized") out["state"] = state out["approvedBy"] = _text(actor, 120) if state == "approved" else None + out["approvedAt"] = _approval_time(now) if state == "approved" else None out["disabledBy"] = _text(actor, 120) if state == "disabled" else None out["revision"] = int(baseline.get("revision") or 0) + 1 return out @@ -127,7 +186,7 @@ def merge_records(baseline, persisted): if key not in out or not isinstance(raw, dict): continue for name in ("purpose", "role", "synonyms", "sensitivity", "state", "valueProfile", - "revision", "approvedBy", "disabledBy", "profiledBy"): + "revision", "approvedBy", "approvedAt", "disabledBy", "profiledBy"): if name in raw: out[key][name] = copy.deepcopy(raw[name]) out[key] = enforce_redaction(out[key]) @@ -142,12 +201,12 @@ def load(runtime, baseline): return merge_records(baseline, raw) -def update(runtime, baseline, database, field, patch, *, actor, is_admin): +def update(runtime, baseline, database, field, patch, *, actor, is_admin, now=None): key = field_key(database, field) if key not in baseline: raise SemanticError("that field is not in the database contract") current = load(runtime, baseline)[key] - changed = _clean_override(patch, current, actor=actor, is_admin=is_admin) + changed = _clean_override(patch, current, actor=actor, is_admin=is_admin, now=now) def apply(raw): doc = copy.deepcopy(raw) if isinstance(raw, dict) else {} @@ -240,6 +299,85 @@ def value_profile(values, *, sensitivity="internal", distinct_cap=VALUE_PROFILE_ return {"redacted": False, "distinctCount": count, "values": [], "sample": sample} +def draft_fields(records, snapshot, *, limit=8): + """Rank only source-declared, visible fields using the caller's real scoped rows.""" + rows = [row for row in snapshot.get("records") or () if isinstance(row, dict)] + visible = {row["key"] for row in snapshot.get("fields") or () if row.get("key")} + ranked = [] + for record in (records or {}).values(): + field = record.get("field") + if (record.get("database") != snapshot.get("database") or field not in visible + or record.get("state") == "disabled"): + continue + values = [row.get(field) for row in rows] + populated = [value for value in values if value is not None and value != "" + and value != [] and value != {}] + role, kind = record.get("role") or "attribute", record.get("dataType") or "text" + linked = bool(record.get("relation")) or role == "link" or kind == "link" + dated = role == "date" or kind in ("date", "datetime", "created_time", "modified_time") + identifier = field.lower() in ("id", "pid", "uuid") or field.lower().endswith("_id") + evidence = f"Filled on {len(populated)} of {len(rows)} rows" + detail = (", and it links this database to related records" if linked else + ", and it is a date this database can be scoped by" if dated else + ", and it identifies a record" if identifier else "") + rank = len(populated) / max(1, len(rows)) + (0.4 if linked or dated else 0) + rank -= 1 if identifier and not linked else 0 + sensitivity = record.get("sensitivity") or "internal" + ranked.append((rank, field, { + "field": field, "label": record.get("label") or field, + "why": evidence + detail + ".", "role": "link" if linked else "date" if dated else role, + "dataType": kind, "sensitivity": sensitivity, + "sample": value_profile(populated, sensitivity=sensitivity), + })) + ranked.sort(key=lambda item: (-item[0], item[1])) + return [item[2] for item in ranked[:max(0, min(int(limit), 8))]] + + +def pending_approval_key(): + return "semantic_draft_requests_v1" + + +def record_draft_request(runtime, database, question, fields, *, actor, now=None): + """Queue field keys only; shared requests never persist samples from a member's rows.""" + request = {"database": database, "question": _text(question, 500), + "fields": list(dict.fromkeys(fields))[:8], "requestedBy": _text(actor, 120), + "requestedAt": _approval_time(now), "state": "pending", + "approvedBy": None, "approvedAt": None} + + def apply(raw): + doc = copy.deepcopy(raw) if isinstance(raw, dict) else {} + doc["schema"] = 1 + doc.setdefault("requests", {})[database] = request + return doc + + runtime.update(pending_approval_key(), apply) + return copy.deepcopy(request) + + +def draft_request(runtime, database): + raw = runtime.get(pending_approval_key()) or {} + row = (raw.get("requests") or {}).get(database) + return copy.deepcopy(row) if isinstance(row, dict) else None + + +def approve_draft_request(runtime, database, fields, *, actor, is_admin, now=None): + if not is_admin: + raise SemanticError("only a tenant admin can change semantic approval") + landed = {} + + def apply(raw): + doc = copy.deepcopy(raw) if isinstance(raw, dict) else {"schema": 1, "requests": {}} + row = doc.setdefault("requests", {}).get(database) + if row: + row.update({"fields": list(dict.fromkeys(fields))[:8], "state": "approved", + "approvedBy": _text(actor, 120), "approvedAt": _approval_time(now)}) + landed.update(row) + return doc + + runtime.update(pending_approval_key(), apply) + return copy.deepcopy(landed) + + def generated_markdown(record): """A deterministic projection. Sensitive profiles carry no values by construction.""" row = enforce_redaction(record) diff --git a/api/test_odoo_rank_refresh.py b/api/test_odoo_rank_refresh.py index 4215d9296dd890ccf90a3cba957f99a14dda04a3..a6310e88ba64cbd18779408b944590b07d055cfd 100644 --- a/api/test_odoo_rank_refresh.py +++ b/api/test_odoo_rank_refresh.py @@ -5,7 +5,9 @@ import copy import io import os from pathlib import Path +import builtins import socket +import symtable import sys import tempfile from types import SimpleNamespace @@ -17,6 +19,59 @@ import duckdb from harness import datastore as ds +class FixtureGap(AssertionError): + """The synthetic namespace is missing a collaborator. A fixture defect, never a loop failure.""" + + +def _free_globals(node): + """Global names `node` reads, from Python's own symbol table rather than a name list.""" + names = set() + def walk(table): + for sym in table.get_symbols(): + if sym.is_referenced() and ( + table.get_type() == 'module' and not (sym.is_assigned() or sym.is_imported()) + or table.get_type() != 'module' and sym.is_global()): + names.add(sym.get_name()) + for child in table.get_children(): + walk(child) + walk(symtable.symtable(ast.unparse(node), 'main.py', 'exec')) + return names - set(dir(builtins)) + + +def _with_collaborators(source, nodes, ns): + """`nodes` plus every top-level main.py def/class/assign they transitively call. + + W-228: the namespace below was hand-maintained, so `_run_sync_requests` arrived as a bare + NameError that named the resident loop. Explicit stubs in `ns` still win. A name that is + neither stubbed nor a top-level def/class/assign (an import, say) raises a FixtureGap. + """ + top = {} + for node in source.body: + if isinstance(node, (ast.FunctionDef, ast.ClassDef)): + top.setdefault(node.name, node) + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + top.setdefault(target.id, node) + have = list(nodes) + defined = {name for name, node in top.items() if node in have} + todo = [name for node in have for name in _free_globals(node)] + while todo: + name = todo.pop() + if name in ns or name in defined: + continue + node = top.get(name) + if node is None: + raise FixtureGap( + f'FIXTURE GAP, not a resident-loop failure: main.py:_store_resync_loop reads ' + f'{name!r}, which this test neither stubs nor can extract as a top-level ' + f'def/class/assign of main.py. Stub it in observe_resident_recovery.') + have.append(node) + defined |= {n for n, candidate in top.items() if candidate is node} + todo.extend(_free_globals(node)) + return have + + class RankRefreshTests(unittest.TestCase): def setUp(self): self.stack = ExitStack() @@ -146,16 +201,26 @@ class RankRefreshTests(unittest.TestCase): # tests died on it -- in the MERGED tree only: lane G does not run this # gate and lane D's run predates lane G. The real `main.py` defines the # function at module scope and the deployed Space boots fine. + # W-228: no longer a bare NameError. Unstubbed collaborators are extracted + # from main.py by `_with_collaborators`, or fail as a named FixtureGap. _resync_mirror_pass=sync, _pull_meta=lambda *_: True, _pull_amazon=lambda *_: True, _rebuild_odoo_relational=rebuild, _rebuild_meta_relational=lambda *_: {}, _rebuild_amazon_relational=lambda *_: {}) + # W-228: the real helpers the loop calls (`_run_sync_requests` and what it needs) are + # extracted too; anything unresolvable fails as a named FixtureGap. + nodes = _with_collaborators(source, nodes, ns) exec(compile(ast.Module(body=nodes, type_ignores=[]), 'main.py:resident-loop', 'exec'), ns) runtime = SimpleNamespace(get_runtime=lambda *_: object()) + from core import connector_sync + # The cadence wait is now an interruptible event, so it is routed to the synthetic clock. + wake = SimpleNamespace(wait=sleep, clear=lambda: None) with patch.dict(sys.modules, {'time': clock, 'harness.runtime': runtime}), \ patch.object(harness, 'runtime', runtime, create=True), \ + patch.object(connector_sync, 'sync_wake_event', return_value=wake), \ + patch.object(connector_sync, 'take_sync_requests', return_value=[]), \ redirect_stdout(io.StringIO()): with self.assertRaisesRegex(StopIteration, 'bounded synthetic'): ns['_store_resync_loop']() diff --git a/platform/aios_grid.py b/platform/aios_grid.py index d9906fc0a4c6fa055d3e7edc1d9e331e8785877b..1b53f612e77d281eb1ebc2129ba19db907934b33 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -471,6 +471,11 @@ def _clean_permissions(raw): #: # $ % TEXT number formats already accepted by the client; `$` is also a regex anchor. #: #: `;` stays refused: no legal formula token needs a statement separator. +#: +#: ⛔ D-547: A CHARACTER FILTER, NOT A SAFETY BOUNDARY. `__import__("os")` passes it (word +#: characters, parens, quotes). Such a source is inert because the ENGINE refuses it: the server +#: parser door in `aios-web/api/formula_runtime.py` is an AST interpreter over a function allowlist +#: with no Python eval. `api/verify_formula_storage.py` records both halves; never read this as the wall. _FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"'\[\]|?:\\#$%@]*$") _FORMULA_REF = re.compile(r"\{([^{}]*)\}") @@ -3783,6 +3788,49 @@ def views_from_defs(defs, saved_views, fields, system_name="All customers", lock return views +def narrow_view_keys(views, hidden): + """D-464: a WIRE COPY of `views` whose column-bearing config names no key in `hidden`. + + `views_from_defs` merges saved configs verbatim and runs before the field wall exists, so a + hidden field's KEY rode `/workspace` in `order`/`visible`/`widths`/`sorts`/`groupBy`/`colorBy` + (and `stackField`, `summary`, `display.*Field`) to the very reader the wall targets. Each + assembly calls this beside its overlay narrowing, + on `workspace["views"]` only: `g["views"]` stays whole for server-side work (exports, measure + sets), and a walled reader's own autosave is already cleaned against their walled keys. + ⚠ FILTER LEAVES (and `secondaryView`) ARE LEFT ALONE: dropping a leaf would change which rows + a view shows, so a hidden key named only in a filter still rides this wire.""" + hidden = set(hidden or ()) + if not hidden: + return views + out = [] + for view in views or []: + cfg = view.get("config") if isinstance(view, dict) else None + if not isinstance(cfg, dict): + out.append(view) + continue + cfg = dict(cfg) + for name in ("order", "visible"): + if isinstance(cfg.get(name), list): + cfg[name] = [key for key in cfg[name] if key not in hidden] + if isinstance(cfg.get("widths"), dict): + cfg["widths"] = {key: w for key, w in cfg["widths"].items() if key not in hidden} + if isinstance(cfg.get("sorts"), list): + cfg["sorts"] = [s for s in cfg["sorts"] + if not (isinstance(s, dict) and s.get("colId") in hidden)] + for name in ("groupBy", "colorBy", "stackField"): + if isinstance(cfg.get(name), str) and cfg[name] in hidden: + cfg[name] = None + if isinstance(cfg.get("summary"), dict): + cfg["summary"] = {key: s for key, s in cfg["summary"].items() if key not in hidden} + if isinstance(cfg.get("display"), dict): + # `DisplaySpec`'s `dateField`/`stackField`/`titleField`/`colorField`/... members. + cfg["display"] = {name: value for name, value in cfg["display"].items() + if not (name.endswith("Field") and isinstance(value, str) + and value in hidden)} + out.append(dict(view, config=cfg)) + return out + + def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None, fields_base=None, with_cohorts=True, st=None): """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE diff --git a/platform/core/connector_sync.py b/platform/core/connector_sync.py index 457a3ae1539d6ab509e32769934148a2fa7d4db6..01afa95a5b5f0cf16ea3f3f977f64d767de3e1fe 100644 --- a/platform/core/connector_sync.py +++ b/platform/core/connector_sync.py @@ -8,6 +8,7 @@ from __future__ import annotations import datetime as _dt import hashlib import json +import threading import time import uuid @@ -20,6 +21,77 @@ _ERROR_LIMIT = 50 _DEFAULT_CONNECTION = "default" _PROVIDERS = {} +# One in-flight Sync now per (provider, tenant). Only the resident resync thread drains this +# intent desk. Nothing here imports a provider, calls a runner, or writes to the mirror/store. +_SYNC_REQUESTS = {} +_SYNC_REQUEST_LOCK = threading.Lock() +_SYNC_WAKE = threading.Event() + + +def request_sync(provider, tenant, *, requested_by="", now=None): + """Queue an intent. A duplicate queued/running request returns accepted=False.""" + if provider not in _PROVIDERS: + raise ValueError("This connector is not registered for sync.") + tenant = str(tenant or "").strip() + if not tenant: + raise ValueError("A tenant is required for sync.") + with _SYNC_REQUEST_LOCK: + previous = _SYNC_REQUESTS.get((provider, tenant)) + if previous and previous["state"] in ("queued", "running"): + return {**previous, "accepted": False} + receipt = {"provider": provider, "tenant": tenant, "state": "queued", + "requestedAt": time.time() if now is None else now, + "startedAt": None, "finishedAt": None, "detail": "", + "requestedBy": str(requested_by or ""), "accepted": True} + if requested_by == "first load" and previous and previous.get("firstLoad"): + # A continuation keeps the safe numeric progress visible while waiting for the writer. + receipt.update(detail=previous["detail"], firstLoad=dict(previous["firstLoad"])) + _SYNC_REQUESTS[(provider, tenant)] = receipt + _SYNC_WAKE.set() + return dict(receipt) + + +def sync_request(provider, tenant): + """Return a detached current receipt, or None.""" + with _SYNC_REQUEST_LOCK: + receipt = _SYNC_REQUESTS.get((provider, tenant)) + if receipt is None: + return None + result = {**receipt, "accepted": False} + if "firstLoad" in result: + result["firstLoad"] = dict(result["firstLoad"]) + return result + + +def take_sync_requests(): + """Claim every queued intent atomically, on the resident resync thread.""" + with _SYNC_REQUEST_LOCK: + _SYNC_WAKE.clear() + claimed = [] + for receipt in _SYNC_REQUESTS.values(): + if receipt["state"] == "queued": + receipt.update(state="running", startedAt=time.time(), accepted=False) + claimed.append(dict(receipt)) + return claimed + + +def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load=None): + """Finish a claimed intent. The API runner supplies a safe cause sentence, never raw errors.""" + with _SYNC_REQUEST_LOCK: + receipt = _SYNC_REQUESTS.get((provider, tenant)) + if receipt is None or receipt["state"] != "running": + return None + receipt.update(state="done" if ok else "failed", detail=str(detail or "")[:300], + finishedAt=time.time() if now is None else now, accepted=False) + if first_load is not None: + receipt["firstLoad"] = dict(first_load) + return dict(receipt) + + +def sync_wake_event(): + """The event the resident thread waits on between passes.""" + return _SYNC_WAKE + # One receipt per immutable database identity. No credentials, session, or source values. JOBS_KEY = "connector_materialization_jobs" MAX_ATTEMPTS = 3 @@ -34,6 +106,9 @@ def job_key(provider, connection_id, database_key): def _job_public(row): answer = {k: v for k, v in row.items() if k != "leaseToken"} + # Older status readers only know error. Keep one persisted cause and project it there. + if row.get("state") == "queued" and row.get("stall"): + answer["error"] = row["stall"]["cause"] answer["retryable"] = (row.get("state") == "failed" and int(row.get("attempts", 0)) < MAX_ATTEMPTS) answer["maxAttempts"] = MAX_ATTEMPTS @@ -104,6 +179,36 @@ def enqueue(rt, provider, connection_id, database_key, *, model="", retry=False, return _job_public(landed[key]) +def stall(rt, candidate, cause, *, now=None): + """Record WHY queued work cannot start, without consuming an attempt.""" + key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) + stamp = time.time() if now is None else now + sentence = str(cause or "")[:300] + + def eligible(row): + return (row.get("id") == candidate.get("id") and + (row.get("state") == "queued" or + row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp)) + + previous = (rt.get(JOBS_KEY, fresh=True) or {}).get(key) or {} + # Avoid even submitting a store update on the unchanged 15-second drain pass. + if not eligible(previous) or (previous.get("stall") or {}).get("cause") == sentence: + return _job_public(previous) + + def apply(current): + doc = dict(current or {}) + row = dict(doc.get(key) or {}) + if not eligible(row) or (row.get("stall") or {}).get("cause") == sentence: + return doc + row.update(state="queued", leaseUntil=0, leaseToken="", + stall={"at": stamp, "cause": sentence}) + doc[key] = row + return doc + + landed = rt.update(JOBS_KEY, apply, flush="sync") + return _job_public(landed.get(key) or {}) + + def claim(rt, candidate, *, now=None): """Atomically claim queued or abandoned work; replay callbacks have no side effects.""" key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) @@ -117,6 +222,7 @@ def claim(rt, candidate, *, now=None): if row.get("state") != "queued" and not ( row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp): return doc + row.pop("stall", None) if int(row.get("attempts", 0)) >= MAX_ATTEMPTS: row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0, error="Sync attempts exhausted. Check the connection and update its settings.") @@ -139,7 +245,27 @@ def owns(rt, job, *, now=None): and row.get("leaseUntil", 0) > (time.time() if now is None else now)) -def finish(rt, job, *, success=False, renew=False, interrupted=False, now=None): +def progress(rt, job, *, rows, total): + """Record published rows only while this worker still owns its unexpired lease.""" + key = job_key(job["provider"], job["connectionId"], job["databaseKey"]) + stamp = time.time() + + def apply(current): + doc = dict(current or {}) + row = dict(doc.get(key) or {}) + if (row.get("id") != job.get("id") or row.get("state") != "syncing" + or row.get("leaseToken") != job.get("leaseToken") + or row.get("leaseUntil", 0) <= stamp): + return doc + row["progress"] = {"rows": rows, "total": total, "at": stamp} + doc[key] = row + return doc + + landed = rt.update(JOBS_KEY, apply, flush="sync") + return _job_public(landed.get(key) or {}) + + +def finish(rt, job, *, success=False, renew=False, interrupted=False, cause="", now=None): """A stale worker cannot acknowledge, renew, or overwrite a newer receipt.""" key = job_key(job["provider"], job["connectionId"], job["databaseKey"]) stamp = time.time() if now is None else now @@ -160,7 +286,8 @@ def finish(rt, job, *, success=False, renew=False, interrupted=False, now=None): else: row.update(state="ready" if success else "failed", at=stamp, leaseUntil=0, leaseToken="", workspaceChanged=bool(success), - error="" if success else "Rows could not be synced. Check the connection and retry.") + error="" if success else (str(cause or "")[:300] or + "Rows could not be synced. Check the connection and retry.")) doc[key] = row return doc @@ -178,6 +305,8 @@ def cancel(rt, provider, connection_id, database_key): doc[key] = {**doc[key], "state": "configured", "at": stamp, "leaseToken": "", "leaseUntil": 0, "workspaceChanged": False, "error": ""} + doc[key].pop("stall", None) + doc[key].pop("progress", None) return doc rt.update(JOBS_KEY, apply, flush="sync") @@ -363,3 +492,5 @@ register_provider( label="Amazon", retry_hint="No sync runner is wired for this connector yet.", ) +register_provider("meta_ads", label="Meta Ads", + retry_hint="Use Sync now to retry the Meta Ads pull.") diff --git a/platform/core/dependents.py b/platform/core/dependents.py index fac771e19254ff547fd5d9f3561d2627651da995..5086758b133a06605dda03b02a0dd0a9cd19b246 100644 --- a/platform/core/dependents.py +++ b/platform/core/dependents.py @@ -47,7 +47,13 @@ walked directly instead, exactly as the shape allows. FIELD KEY (a string) of the link the rollup traverses. A rollup may also carry no `link` at all (the `{'source': {...}}` shape, which joins through a topic dim). Every walker below branches on `field['type']` first and on which key is present second. + +W60-T31 / C3: grouped kind/label/count/ids retain their meanings. Additive `items` names each +id in order and gives its address or a reason it cannot be opened. Data addresses use only +the existing #/data? prefix; workbook addresses belong to the Sheet provider (C2). """ +from urllib.parse import parse_qs, urlencode + import core.chart_system as chart_system import core.route_system as route_system import core.shares as shares @@ -60,6 +66,13 @@ from core.table_store import ASSIGNED_KEY, SHARED_KEY #: happened to register first (W49-T11 registers `sheet_cell` after the merge). KIND_ORDER = ('view', 'route', 'link', 'rollup', 'grant', 'chart', 'sheet_cell') +#: Built-in Default views are not dependents, including their saved owner copies. Their ids are +#: shared by EVERY database, so counting them charges one database's Default view to another. +#: `aios_grid.py` mints `all-customers` on every topic; its `IG_OVERVIEW_ID` is `tpl_overview`. +#: Hardcoded here because `core/` may not reach UP into `aios_grid`; `verify_shared_views.py` +#: pins this pair against that module so the declarations cannot drift. +PINNED_VIEW_IDS = frozenset({'all-customers', 'tpl_overview'}) + #: Singular / plural noun phrase per kind. ⛔ THE CASING IS R23's OWN — lowercase `views`, capital #: `Route`, capital `Sheet cells` — because the ruling quotes the sentence the user reads. _NOUNS = { @@ -75,6 +88,36 @@ _NOUNS = { _PROVIDERS = {} +DATA_HASH_PREFIX = '#/data?' + + +def data_hash(database, **params): + """Build a #/data?database=... address. The only hash builder in this module.""" + database = str(database or '').strip() + if not database: + return None + query = {'database': database} + query.update({key: str(value) for key, value in params.items() + if key != 'database' and value is not None and str(value) != ''}) + return DATA_HASH_PREFIX + urlencode(query) + + +def _dependent_item(item_id, label, href=None, reason=''): + """A named thing with an existing hash address or an explicit explanation.""" + item_id = str(item_id or '') + label = str(label or item_id) + href = href if isinstance(href, str) and href.startswith((DATA_HASH_PREFIX, '#/sheet?')) else None + return {'id': item_id, 'label': label, 'href': href, + 'reason': str(reason or '') if href else + str(reason or f'{label} has no direct address available.')} + + +def _field_item(table_key, field): + key = str(field.get('key') or '') + return _dependent_item(f'{table_key}:{key}', field.get('label') or key, + data_hash(table_key, field=key) if key else None) + + def noun(kind, count): """The noun phrase for `count` of `kind`, agreeing in number.""" pair = _NOUNS.get(str(kind or '')) @@ -120,7 +163,7 @@ def clean_target(target): return {'kind': kind, 'key': key, 'database': database} -def _dependent(kind, ids): +def _dependent(kind, ids, items=None): """A Dependent, or None when nothing depends. ⛔ NONE IS THE ZERO CASE — never `count: 0`.""" seen, out = set(), [] for raw in ids or (): @@ -130,7 +173,13 @@ def _dependent(kind, ids): out.append(value) if not out: return None - return {'kind': kind, 'label': noun(kind, len(out)), 'count': len(out), 'ids': out} + result = {'kind': kind, 'label': noun(kind, len(out)), 'count': len(out), 'ids': out} + if items is not None: + by_id = {} + for item in items: + by_id.setdefault(str(item.get('id') or ''), item) + result['items'] = [by_id.get(value) or _dependent_item(value, value) for value in out] + return result def dependents(st, target): @@ -162,10 +211,36 @@ def dependents_report(st, target): count = int(item.get('count') or len(ids) or 0) if count <= 0: continue - found.append({'kind': str(item.get('kind') or kind), + normalized = {'kind': str(item.get('kind') or kind), 'label': str(item.get('label') or noun(kind, count)), 'count': count, - 'ids': [str(i) for i in ids]}) + 'ids': [str(i) for i in ids]} + # Lane A may supply a single Sheet address on the dependent itself. Preserve + # both that shape and per-id items; neither changes the legacy count or label. + if 'href' in item: + normalized['href'] = item['href'] if isinstance(item['href'], str) else None + if 'reason' in item: + normalized['reason'] = str(item.get('reason') or '') + # W60-T12 (combine-owned), 2026-09-14: `refusal` is the OTHER half of the same fact + # and was being dropped at this seam. A provider says "no address" with `href: None` + # AND says WHY in `refusal`; carrying the null while dropping the cause turns an + # explained dead end into an unexplained one, which is the defect R23's sentence + # exists to prevent. Both keys are optional and neither changes count or label. + if 'refusal' in item: + normalized['refusal'] = str(item.get('refusal') or '') + if isinstance(item.get('items'), list): + normalized['items'] = [] + for entry in item['items']: + entry = entry if isinstance(entry, dict) else {'id': entry} + href = entry.get('href') if isinstance(entry.get('href'), str) else None + normalized['items'].append({ + 'id': str(entry.get('id') or ''), + 'label': str(entry.get('label') or entry.get('id') or ''), + 'href': href, + 'reason': str(entry.get('reason') or ('' if href else + 'This dependent has no direct address available.')), + }) + found.append(normalized) order = {kind: index for index, kind in enumerate(KIND_ORDER)} found.sort(key=lambda item: (order.get(item['kind'], len(order)), item['kind'])) return {'dependents': found, 'errors': errors} @@ -334,16 +409,27 @@ def view_dependents(st, target): """ kind, key, database = target['kind'], target['key'], target['database'] if kind == 'table': - return _dependent('view', [vid for _owner, vid, _v in walk_views(st, key)]) + items = [_dependent_item(vid, view.get('name') or vid, data_hash(key, view=vid)) + for _owner, vid, view in walk_views(st, key) if vid not in PINNED_VIEW_IDS] + return _dependent('view', [item['id'] for item in items], items=items) if kind == 'view': if not database: return None - return _dependent('view', _assigned_ids(st, database, key)) - hits = [] + ids = _assigned_ids(st, database, key) + assignments = _table_doc(st, database).get(ASSIGNED_KEY) or {} + items = [] + for item_id in ids: + recipient, _, view_id = item_id.partition(':') + view = assignments[recipient]['views'][view_id] + label = f'{recipient}: {view.get("name") or view_id}' + items.append(_dependent_item(item_id, label, data_hash(database, view=view_id))) + return _dependent('view', ids, items=items) + items = [] for _owner, view_id, view in walk_views(st, database): - if _view_names_field(view, key): - hits.append(view_id) - return _dependent('view', hits) + if view_id not in PINNED_VIEW_IDS and _view_names_field(view, key): + items.append(_dependent_item(view_id, view.get('name') or view_id, + data_hash(database, view=view_id))) + return _dependent('view', [item['id'] for item in items], items=items) def _view_names_field(view, field_key): @@ -383,21 +469,31 @@ def route_dependents(st, target): a `geographicField`, which is what makes them a FIELD dependent too. """ kind, key, database = target['kind'], target['key'], target['database'] - routes, hits = _routes(st), [] + routes, matched_routes, matched_zones = _routes(st), [], [] if kind == 'view': - hits += [str(r.get('id')) for r in routes if str(r.get('sourceViewId') or '') == key] - hits += [str(z.get('id')) for z in _zones(st, database or None) - if str(z.get('sourceViewId') or '') == key] - return _dependent('route', hits) - if kind == 'table': - view_ids = {vid for _owner, vid, _v in walk_views(st, key)} - hits += [str(r.get('id')) for r in routes - if str(r.get('sourceViewId') or '') in view_ids] - hits += [str(z.get('id')) for z in _zones(st, key)] - return _dependent('route', hits) - hits += [str(z.get('id')) for z in _zones(st, database) - if str(z.get('geographicField') or '') == key] - return _dependent('route', hits) + matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') == key] + matched_zones = [z for z in _zones(st, database or None) + if str(z.get('sourceViewId') or '') == key] + elif kind == 'table': + view_ids = {vid for _owner, vid, _v in walk_views(st, key) + if vid not in PINNED_VIEW_IDS} + # A-QI7: Routes store no source database and are created only from customer_data. + # The database test must therefore use the target table key, not the tenant-wide Route + # list or a shared view id. Zones already carry their own table scope. + if str(key) == route_system.DEFAULT_ZONE_TABLE: + matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') in view_ids] + matched_zones = _zones(st, key) + else: + matched_zones = [z for z in _zones(st, database) + if str(z.get('geographicField') or '') == key] + items = [_dependent_item(str(r.get('id')), r.get('name') or r.get('id'), + data_hash(route_system.DEFAULT_ZONE_TABLE, route=r.get('id'))) + for r in matched_routes] + for zone in matched_zones: + name = zone.get('name') or zone.get('id') + items.append(_dependent_item(str(zone.get('id')), name, reason= + f'{name} has no direct address. Edit this zone in the Map view Zones menu.')) + return _dependent('route', [item['id'] for item in items], items=items) def link_dependents(st, target): @@ -410,18 +506,18 @@ def link_dependents(st, target): kind, key = target['kind'], target['key'] if kind == 'view': return None - hits = [] + items = [] for table_key, definition in _defs(st).items(): for field in _fields(definition): bag = _link_bag(field) if bag is None: continue if kind == 'table' and str(bag.get('table') or '') == key and str(table_key) != key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) elif kind == 'field' and str(bag.get('inverse') or '') == key \ and str(bag.get('table') or '') == str(target['database']): - hits.append(f'{table_key}:{field.get("key")}') - return _dependent('link', hits) + items.append(_field_item(table_key, field)) + return _dependent('link', [item['id'] for item in items], items=items) def rollup_dependents(st, target): @@ -435,7 +531,7 @@ def rollup_dependents(st, target): kind, key = target['kind'], target['key'] if kind == 'view': return None - hits = [] + items = [] for table_key, definition in _defs(st).items(): fields = _fields(definition) by_key = {str(f.get('key') or ''): f for f in fields} @@ -449,18 +545,18 @@ def rollup_dependents(st, target): hop_bag = _link_bag(hop_field) if hop_field else None if kind == 'table': if source and str(source.get('topic') or '') == key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) elif hop_bag is not None and str(hop_bag.get('table') or '') == key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) continue if str(target['database']) and str(table_key) != str(target['database']): continue if hop == key or str(bag.get('field') or '') == key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) elif source and key in (str(source.get('on') or ''), str(source.get('groupBy') or ''), str(source.get('measure') or '')): - hits.append(f'{table_key}:{field.get("key")}') - return _dependent('rollup', hits) + items.append(_field_item(table_key, field)) + return _dependent('rollup', [item['id'] for item in items], items=items) def grant_dependents(st, target): @@ -471,13 +567,21 @@ def grant_dependents(st, target): * field -> the `field` grant on `:` """ kind, key, database = target['kind'], target['key'], target['database'] + + def named_grants(names): + subject = {'view': 'view', 'table': 'database', 'field': 'field'}[kind] + items = [_dependent_item(name, name, reason= + f'{name} has no direct access address. Change their access from this ' + f'{subject}\'s share dialog.') for name in names] + return _dependent('grant', names, items=items) + if kind == 'view': - return _dependent('grant', share_audience(st, key)['names']) + return named_grants(share_audience(st, key)['names']) if kind == 'field': if not database: return None record = shares.grants('field', shares.field_oid(database, key), st=st) - return _dependent('grant', [str(e.get('user') or '') for e in (record.get('entries') or ())]) + return named_grants([str(e.get('user') or '') for e in (record.get('entries') or ())]) holders = [str(e.get('user') or '') for e in (shares.grants('database', key, st=st).get('entries') or ())] try: @@ -490,21 +594,110 @@ def grant_dependents(st, target): continue for entry in (shares.grants('field', oid, st=st).get('entries') or ()): holders.append(str(entry.get('user') or '')) - return _dependent('grant', holders) + return named_grants(holders) + + +def _sheet_chart_items(st, target): + """Sheet charts drawn over a cell that depends on `target` (W60-T12, contract C2). + + ⛔⛔ THE DEPENDENCY DECISION IS NOT REMADE HERE, AND THAT IS THE WHOLE DESIGN. Whether a CELL + depends on the target is `modules/sheet.py`'s answer, and `core/` may not reach UP into + `modules/` to ask it directly -- the same layering rule `PINNED_VIEW_IDS` is hardcoded for. So + this reads the ALREADY-REGISTERED `sheet_cell` provider's own rows and takes the cells it + named. One source of truth for the predicate, so a chart can never disagree with the cell it + is drawn over; and if nothing registered `sheet_cell` there are no sheet charts to report and + this is silent, exactly as an absent provider is everywhere else in this file. + + ⚠ A CELL WITH NO ADDRESS CONTRIBUTES NO CHART ADDRESS. `sheet_cell` answers `href: None` plus a + cause when the destination is hidden or unreadable, and a link built anyway would open some + OTHER cell -- which is the defect W60-T12 exists to close, not one to reintroduce one layer up. + """ + provider = _PROVIDERS.get('sheet_cell') + if provider is None: + return [] + try: + rows = provider(st, dict(target)) + except Exception: + return [] # a provider that raises omits its kind; so does this + wanted = {} + for row in rows if isinstance(rows, (list, tuple)) else (): + if not isinstance(row, dict) or row.get('kind') != 'sheet_cell': + continue + href = row.get('href') + if not isinstance(href, str) or not href.startswith('#/sheet?'): + continue + params = parse_qs(href.split('?', 1)[1]) + workbook_id = (params.get('wb') or [''])[0] + tab_id = (params.get('tab') or [''])[0] + addr = (params.get('cell') or [''])[0] + if workbook_id and tab_id and addr: + wanted.setdefault((workbook_id, tab_id), []).append(addr) + if not wanted: + return [] + + from core import sheets as sheet_store + items, seen = [], set() + for workbook_id in dict.fromkeys(book for book, _ in wanted): + try: + book = sheet_store.get_workbook(st, workbook_id) + except Exception: + continue # an unreadable workbook is `sheet_cell`'s to report + for tab in book.get('sheets') or (): + tab_id = str(tab.get('id') or '') + addrs = wanted.get((workbook_id, tab_id)) or [] + if not addrs: + continue + # ⚠ THE ADDRESS MATHS IS `core/sheets.py`'s, not a second copy. `_clean_range` accepts + # both `"B4"` and the chart's `{col0,row0,col1,row1}` and answers the same zero-based + # rectangle for each, so a cell and the chart over it cannot drift apart. + cells = [rect for rect in (sheet_store._clean_range(a) for a in addrs) if rect] + for chart in tab.get('charts') or (): + if not isinstance(chart, dict): + continue + chart_id = str(chart.get('id') or '') + rect = sheet_store._clean_range(chart.get('range')) + if not chart_id or chart_id in seen or not rect: + continue + if not any(rect['col0'] <= c['col0'] <= rect['col1'] + and rect['row0'] <= c['row0'] <= rect['row1'] for c in cells): + continue + seen.add(chart_id) + label = str(chart.get('name') or chart.get('title') or '').strip() or ( + f"Chart in {book.get('name') or workbook_id} / {tab.get('name') or tab_id}") + items.append(_dependent_item(chart_id, label, '#/sheet?' + urlencode( + {'wb': workbook_id, 'tab': tab_id, 'chart': chart_id}))) + return items def chart_dependents(st, target): - """Saved charts filed against the target DATABASE. + """Saved charts filed against the target DATABASE, and Sheet charts drawn over its cells. ⛔ A CHART NEVER DEPENDS ON A VIEW, and that is `chart_system`'s stated contract rather than an omission here: a chart's `config` is OPAQUE and this file must not start reading keys out of it to guess at a reference. Deleting a view cannot delete a chart, so it must not disclose one. + + ⭐ W60-T12 (combine-owned) ADDED THE SECOND SOURCE. A Sheet chart is not reachable from + `sheet_dependents` at all -- it is a sibling of the cells, not one of them -- so before this a + database could be switched off while a chart still drew from it, and R23's refusal would not + have said so. Charts are not read from the sheet document here; see `_sheet_chart_items`. """ kind, key = target['kind'], target['key'] if kind != 'table': return None - return _dependent('chart', [str(c.get('id')) for c in _charts(st) - if str(c.get('database') or '') == key]) + items = [_dependent_item(str(c.get('id')), c.get('name') or c.get('id'), c.get('href'), + '' if c.get('href') else + 'This chart has no direct address. Open it from Charts.') + for c in _charts(st) if str(c.get('database') or '') == key] + items += _sheet_chart_items(st, target) + row = _dependent('chart', [item['id'] for item in items], items=items) + # ⭐ WHEN EXACTLY ONE CHART DEPENDS ON THE TARGET, THE ROW ADDRESSES ONE THING, so it carries + # that address itself -- the single-address-on-the-dependent shape `dependents_report` already + # preserves. ⛔ With several it carries none: a row-level link would have to pick one chart and + # would silently mean the others, which is the naming-by-display defect W60-T12 closed. + if row and len(items) == 1: + row['href'] = items[0]['href'] + row['reason'] = items[0]['reason'] + return row register('view', view_dependents) diff --git a/platform/core/external_access.py b/platform/core/external_access.py index cf2cbb6dcb6fa95c14255033e30ab72fe502b0b0..2312af7687425a10c923e79c08c44f4b4ff69e87 100644 --- a/platform/core/external_access.py +++ b/platform/core/external_access.py @@ -1,8 +1,9 @@ -"""Per-user credentials for the governed external read surface. +"""Per-user credentials carrying an access level and database subset for governed reads and +builds, with the maker's live grants still applying on top. -The credential is authentication only. Verification reconstructs the same PUBLIC user record -the normal API session carries, so every later database read still goes through ``perm_scope`` -and the existing Assistant mirror. This module never reads product rows and never caches a +Verification reconstructs the same PUBLIC user record the normal API session carries, so every +later database read still goes through ``perm_scope`` and the existing Assistant mirror. +This module never reads product rows and never caches a credential, user, bucket, or tenant runtime. """ import calendar @@ -17,6 +18,11 @@ from harness import runtime BUCKET = "external_access" +ACCESS_READ = "read" +ACCESS_READWRITE = "readwrite" +ACCESS_LEVELS = (ACCESS_READ, ACCESS_READWRITE) +ALL_DATABASES = "all" +MAX_SCOPE_DATABASES = 200 ALGORITHM = "sha256-salt-v1" AUTH_ERROR = "invalid_external_credential" _DUMMY_SALT = bytes(32) @@ -48,6 +54,50 @@ class ExternalAuthError(ExternalAccessError): """A uniform authentication refusal that reveals no credential or tenant detail.""" +def normalise_access(value) -> str: + if value is None or value == "": + return ACCESS_READWRITE + if value in ACCESS_LEVELS: + return value + raise ExternalAccessError("external_access_scope_invalid") + + +def normalise_databases(value): + if value is None or (isinstance(value, str) and value.strip().lower() == ALL_DATABASES): + return ALL_DATABASES + if not isinstance(value, (list, tuple)) or not 1 <= len(value) <= MAX_SCOPE_DATABASES: + raise ExternalAccessError("external_databases_invalid") + if any(not isinstance(item, str) or not item.strip() or len(item.strip()) > 200 + for item in value): + raise ExternalAccessError("external_databases_invalid") + return tuple(dict.fromkeys(item.strip() for item in value)) + + +def scope_of(record): + """Read legacy defaults, but preserve a stored empty subset as no database access.""" + record = record if isinstance(record, dict) else {} + access = record.get("access") + if access not in ACCESS_LEVELS: + access = ACCESS_READWRITE + databases = record.get("databases") + if isinstance(databases, (list, tuple)): + databases = tuple(item.strip() for item in databases + if isinstance(item, str) and item.strip()) + else: + databases = ALL_DATABASES + return access, databases + + +def may_write(access) -> bool: + return access == ACCESS_READWRITE + + +def database_allowed(databases, database) -> bool: + if databases == ALL_DATABASES: + return True + return isinstance(database, str) and bool(database.strip()) and database.strip() in databases + + @dataclass(frozen=True) class VerifiedCredential: """The authenticated public principal plus the request-scoped tenant runtime.""" @@ -56,6 +106,8 @@ class VerifiedCredential: key_id: str user: dict runtime: object + access: str = ACCESS_READWRITE + databases: object = ALL_DATABASES def _now() -> str: @@ -130,17 +182,33 @@ def _active_public_user(username: str, tenant: str) -> dict: def _metadata(record: dict) -> dict: """Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam.""" + access, databases = scope_of(record) return { "id": record.get("id"), "label": record.get("label"), "created": record.get("created"), "last_used": record.get("last_used"), "revoked_at": record.get("revoked_at"), + "access": access, + "databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases), } -def mint(st, username: str, tenant: str, label: str) -> tuple[dict, str]: +def label_for(st, key_id: str) -> str: + """Read only a verified key's label from its tenant runtime, never credential material.""" + record = _book(st).get(key_id) + if not isinstance(record, dict) or record.get("revoked_at"): + raise ExternalAuthError(AUTH_ERROR) + label = record.get("label") + if not isinstance(label, str) or not label: + raise ExternalAccessError("external_access_unavailable") + return label + + +def mint(st, username: str, tenant: str, label: str, access=None, databases=None) -> tuple[dict, str]: """Create one credential and return ``(metadata, token)``; the token is never persisted.""" + access = normalise_access(access) + databases = normalise_databases(databases) owner, tenant_key = _normalise_owner(username, tenant) _assert_bound_runtime(st, tenant_key) _active_public_user(owner, tenant_key) @@ -168,6 +236,8 @@ def mint(st, username: str, tenant: str, label: str) -> tuple[dict, str]: "username": owner, "tenant": tenant_key, "label": clean_label, + "access": access, + "databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases), "salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM, @@ -295,8 +365,10 @@ def verify(token: str) -> VerifiedCredential: except ExternalAccessError as exc: raise ExternalAuthError(AUTH_ERROR) from exc + access, databases = scope_of(record) if not _stamp_is_due(record.get("last_used"), time.time()): - return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st) + return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st, + access=access, databases=databases) used_at = _now() @@ -321,7 +393,8 @@ def verify(token: str) -> VerifiedCredential: raise except Exception as exc: raise ExternalAuthError(AUTH_ERROR) from exc - return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st) + return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st, + access=access, databases=databases) except ExternalAuthError: raise except Exception as exc: diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..57cfe7c52be5565876268cc3c0f09106753b813b --- /dev/null +++ b/platform/core/external_tools.py @@ -0,0 +1,372 @@ +"""External build tools over the app's Sheet, saved View and Application writers. + +The API supplies its existing permission, read-projection and binding-resolution functions as +request-scoped callbacks. Core never imports API modules or chooses another tenant runtime. +Adding a tool means adding a registry entry; writes also require the router's literal allow-list. +""" +import copy +import json +import uuid +from dataclasses import dataclass +from typing import Callable + +from core import sheets, sheet_structure, sheet_templates + + +@dataclass(frozen=True) +class ToolContext: + session: object + actor_name: str + require_reader: Callable + require_editor: Callable + project_book: Callable + resolve_binding: Callable + cache_binding: Callable + list_databases: Callable + describe_database: Callable + view_event: Callable + app_upsert: Callable + app_delete: Callable + + +def _object(properties=None, required=(), **extra): + return {"type": "object", "properties": properties or {}, "required": list(required), **extra} + + +_STRING = {"type": "string"} +_ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."} +_FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"}, + "currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",)) +_STYLE = _object({ + **{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")}, + **{key: _STRING for key in ("fill", "color", "font", "align", "valign")}, + "size": {"type": ["number", "string"]}, "indent": {"type": "integer"}, + "textRotation": {"type": "number"}, "format": _FORMAT, + "border": _object({side: {"type": "boolean"} for side in ("top", "right", "bottom", "left")}), +}) +_VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",)) +_FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({ + "colId": _STRING, "op": _STRING, "value": {}, + "valueRef": _object({"ref": _ADDRESS}, ("ref",)), + "conj": _STRING, "children": {"type": "array", "items": {"type": "object"}}, +})}}) +_BINDING = _object({ + "database": _STRING, + "aggregate": _object({"fn": _STRING, "field": _STRING, "metric_key": _STRING}), + "scope": _object({"filter": _FILTER, "view_id": _STRING, "view_name": _STRING}), + "period": _object({"unit": _STRING, "start": _STRING}), + "ending": {"type": "object"}, +}, ("database", "aggregate", "scope")) +_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]}, + "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING, + "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",)) +_RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0} + for key in ("col0", "row0", "col1", "row1")}, + ("col0", "row0", "col1", "row1"))]} +_RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING, + "value": {}, "value2": {}, "formula": _STRING, "style": _STYLE, + "scale": {"type": "object"}}, ("range", "op")) +_CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING, + "stacking": _STRING, "source": {"type": "object"}, + "series": {"type": "array", "items": {"type": "object"}}, + "geometry": {"type": "object"}, "anchor": {"type": "object"}, + "settings": {"type": "object"}}, ("kind", "range")) +_TARGET = {"workbook_id": _STRING, + "tab": {"type": "string", "description": "Tab id or tab name."}, + "rev": {"type": "integer", "description": "Optional expected workbook revision."}} +_VIEW_CONFIG = _object({ + "filters": {"oneOf": [_FILTER, {"type": "array", "items": {"type": "object"}}]}, + "filterConj": {"enum": ["and", "or"]}, + "sorts": {"type": "array", "items": _object({ + "colId": _STRING, "dir": {"enum": ["asc", "desc"]}, + "locked": {"type": "boolean"}}, ("colId", "dir"))}, + "groupBy": {"type": ["string", "null"]}, + "visible": {"type": "array", "items": _STRING, + "description": "Fields to show. Fields omitted from this list are hidden."}, + "order": {"type": "array", "items": _STRING}, + "display": _object({"mode": _STRING}, ("mode",)), +}, description="The grid's native config, passed unchanged to its saved View validator.") +_APPLICATION = { + "app_id": {"type": "string", "minLength": 1, + "description": "Omit to create. Supply an id to update mutable properties."}, + "name": _STRING, + "type": {"enum": ["chart", "route", "map", "timeseries"]}, + "compatibleDatabases": {"type": "array", "items": _STRING}, + "sourceDatabase": {"type": "string", "description": "Source for a template Application only."}, + "templateKind": {"enum": ["route", "catalog"]}, + "fieldMap": {"type": "object", "additionalProperties": _STRING}, + "navPlacement": _object({"section": {"const": "app"}, + "order": {"type": "integer", "minimum": 0, "maximum": 999}}, + ("section", "order"), additionalProperties=False), + "semanticRoleBindings": {"type": "object"}, + "config": _object({"schemaVersion": {"const": 1}, + "display": _object({"mode": _STRING}, ("mode",))}, + ("schemaVersion", "display"), additionalProperties=False), + "share": _object({"mode": {"enum": ["private", "users", "tenant"]}, + "users": {"type": "array", "items": _STRING}}, ("mode",)), + "starred": {"type": "boolean"}, +} + + +def _schema(properties, required=(), *, tab=True): + return _object({**_TARGET, **properties}, + ("workbook_id", *(("tab",) if tab else ()), *required), + additionalProperties=False) + + +def _book(ctx, args, *, write=False): + book = sheets.get_workbook(ctx.session.runtime, args["workbook_id"]) + (ctx.require_editor if write else ctx.require_reader)(book) + return book + + +def _saved(book, tab=None): + out = {"id": book["id"], "rev": book["rev"], + "tabs": [{"id": row["id"], "name": row["name"]} for row in book["sheets"]]} + if tab is not None: + out["tab"] = tab + return out + + +def _put(ctx, args, book, tab=None): + # Carry the revision read for this document patch even when the client omitted one. + # Never overwrite an intervening UI save with the older snapshot we just patched. + if "rev" in args: + book["rev"] = args["rev"] + return _saved(sheets.put_workbook(ctx.session.runtime, book["id"], book, + actor=ctx.session.uname, actor_name=ctx.actor_name), tab) + + +def _create(ctx, args): + return _saved(sheets.create_workbook(ctx.session.runtime, name=args["name"], + created_by=ctx.session.uname, + on_duplicate="refuse")) + + +def _add_tab(ctx, args): + book = _book(ctx, args, write=True) + #: ⛔ A DUPLICATE TAB NAME IS REFUSED, because every other tool addresses a tab by `tab`, + #: which `sheets.find_tab` resolves BY NAME to the FIRST match. Two tabs sharing a name make + #: every later write land on one of them and the other unreachable, with nothing reported. + #: Measured at SDLC 61's C7 build: a retried `sheet_add_tab` produced a second + #: `Summary (All)` and the workbook's cells split silently across the pair. + wanted = str(args["name"]).strip() + if any(str(sheet.get("name") or "").strip().casefold() == wanted.casefold() + for sheet in (book.get("sheets") or ())): + raise sheets.SheetRefusal( + "tab_name_taken", + "Tab " + wanted + " is already in this workbook. Choose another name.") + tab = {"id": uuid.uuid4().hex[:12], "name": args["name"], "cells": {}} + book["sheets"].append(tab) + return _put(ctx, args, book, tab["id"]) + + +def _save_template(ctx, args): + book = ctx.project_book(_book(ctx, args)) + record = sheet_templates.save_sheet_template( + ctx.session.runtime, book, args["key"], name=args["name"], + description=args["description"], created_by=ctx.session.uname, actor_name=ctx.actor_name) + return {"template": record} + + +def _read_cells(ctx, args): + book = ctx.project_book(_book(ctx, args)) + tab = sheets.find_tab(book, args["tab"]) + if "addresses" in args: + wanted = {addr.strip().upper() for addr in args["addresses"]} + tab = {**tab, "cells": {addr: cell for addr, cell in tab["cells"].items() if addr in wanted}} + return {"id": book["id"], "rev": book["rev"], "tab": tab} + + +def _patch(ctx, args, cells, *, resolved=None): + out = sheets.patch_cells(ctx.session.runtime, args["workbook_id"], args["tab"], cells, + expect_rev=args.get("rev"), actor=ctx.session.uname, + actor_name=ctx.actor_name) + if resolved is not None: + for addr in out["accepted"]: + cell = cells[addr] + ctx.cache_binding(args["workbook_id"], out["tab"], addr, cell["binding"], resolved) + # The UI computes after its patch too. Project before computing so a shared workbook + # never returns its owner's cached database numbers through an editor's write response. + book = ctx.project_book(_book(ctx, args)) + tab = sheets.find_tab(book, args["tab"]) + out["computed"] = {addr: cell for addr, cell in tab["cells"].items() + if cell.get("kind") in ("formula", "bound") or "spillParent" in cell} + return out + + +def _write_cells(ctx, args): + _book(ctx, args, write=True) + return _patch(ctx, args, args["cells"]) + + +def _set_binding(ctx, args): + book = _book(ctx, args, write=True) + tab = sheets.find_tab(book, args["tab"]) + resolved = ctx.resolve_binding(args["binding"], book, tab["id"]) + addr = args["addr"].strip().upper() + before = tab["cells"].get(addr) or {} + cell = {key: copy.deepcopy(before[key]) for key in ("style", "validation", "link", "note") + if key in before} + cell.update(kind="bound", binding=copy.deepcopy(args["binding"]), state="fresh", + value=resolved["value"], + display="" if resolved["value"] is None else str(resolved["value"])) + out = _patch(ctx, args, {addr: cell}, resolved=resolved) + out["limits"] = resolved.get("limits") or [] + return out + + +def _cell_property(ctx, args, property_name, values): + book = _book(ctx, args, write=True) + tab = sheets.find_tab(book, args["tab"]) + cells = {} + for addr, value in values.items(): + cell = copy.deepcopy(tab["cells"].get(addr.strip().upper()) or {"kind": "empty"}) + cell[property_name] = value + cells[addr] = cell + return _patch(ctx, args, cells) + + +def _dimensions(ctx, args): + book = _book(ctx, args, write=True) + tab = sheets.find_tab(book, args["tab"]) + tab.update(copy.deepcopy(args["dimensions"])) + return _put(ctx, args, book, tab["id"]) + + +def _freeze(ctx, args): + book = _book(ctx, args, write=True) + tab = sheets.find_tab(book, args["tab"]) + tab["freeze"] = args["freeze"] + return _put(ctx, args, book, tab["id"]) + + +def _rule(ctx, args): + book = _book(ctx, args, write=True) + tab = sheets.find_tab(book, args["tab"]) + tab["rules"] = [*(tab.get("rules") or []), args["rule"]] + return _put(ctx, args, book, tab["id"]) + + +def _chart(ctx, args): + book = _book(ctx, args, write=True) + tab = sheets.find_tab(book, args["tab"]) + chart = copy.deepcopy(args["chart"]) + chart.setdefault("id", uuid.uuid4().hex[:12]) + tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart] + return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]} + + +def _structure(ctx, args): + book = _book(ctx, args, write=True) + tab = sheets.find_tab(book, args["tab"]) + shifted = sheet_structure.apply_structure(book, tab["id"], args["op"]) + return _put(ctx, args, shifted, tab["id"]) + + +def _tool(description, schema, handler, *, write=False, grants=("analyst", "sheets")): + return {"description": description, "inputSchema": schema, "write": write, + "grants": grants, "handler": handler} + + +TOOLS = { + "sheet_create_workbook": _tool("Create a private workbook with one initial tab.", + _object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True), + "sheet_add_tab": _tool("Add a blank tab to a workbook.", + _schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True), + "sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.", + _schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells), + "sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.", + _schema({"cells": {"type": "object", "additionalProperties": {"anyOf": [_CELL, {"type": "null"}]}}}, + ("cells",)), _write_cells, write=True), + "sheet_set_binding": _tool("Set and resolve a Cell database binding, including filter valueRef cells.", + _schema({"addr": _ADDRESS, "binding": _BINDING}, ("addr", "binding")), _set_binding, write=True), + "sheet_set_styles": _tool("Replace styles by address, preserving cell content; null clears a style.", + _schema({"styles": {"type": "object", "additionalProperties": {"anyOf": [_STYLE, {"type": "null"}]}}}, + ("styles",)), lambda ctx, args: _cell_property(ctx, args, "style", args["styles"]), write=True), + "sheet_set_dimensions": _tool("Replace supplied geometry fields; widths use letters and row indices start at one.", + _schema({"dimensions": _object({ + "cols": {"type": "integer"}, "rows": {"type": "integer"}, + "colWidths": {"type": "object", "additionalProperties": {"type": "number"}}, + "rowHeight": {"type": "number"}, + "rowHeights": {"type": "object", "additionalProperties": {"type": "number"}}, + "hiddenRows": {"type": "array", "items": {"type": "integer"}}, + "hiddenCols": {"type": "array", "items": {"type": "integer"}}, + }, additionalProperties=False)}, ("dimensions",)), _dimensions, write=True), + "sheet_set_validation": _tool("Replace dropdown lists by address, preserving content; null clears validation.", + _schema({"validations": {"type": "object", "additionalProperties": { + "anyOf": [_VALIDATION, {"type": "null"}]}}}, ("validations",)), + lambda ctx, args: _cell_property(ctx, args, "validation", args["validations"]), write=True), + "sheet_set_freeze": _tool("Set frozen row and column counts, using zero to unfreeze.", + _schema({"freeze": _object({"rows": {"type": "integer"}, "cols": {"type": "integer"}})}, + ("freeze",)), _freeze, write=True), + "sheet_add_rule": _tool("Append a conditional format rule accepted by the Sheet document cleaner.", + _schema({"rule": _RULE}, ("rule",)), _rule, write=True), + "sheet_set_chart": _tool("Add a chart or replace a chart with the same id using the app's chart document.", + _schema({"chart": _CHART}, ("chart",)), _chart, write=True), + "sheet_structure": _tool("Insert or delete rows or columns and reanchor references; positive delta inserts.", + _schema({"op": _object({"axis": {"enum": ["row", "col"]}, + "at": {"type": "integer", "minimum": 1}, "delta": {"type": "integer"}}, + ("axis", "at", "delta"))}, ("op",)), _structure, write=True), + "sheet_save_template": _tool("Save a readable workbook as a tenant Sheet template with its Cell databases.", + _object({key: _STRING for key in ("workbook_id", "key", "name", "description")}, + ("workbook_id", "key", "name", "description"), additionalProperties=False), + _save_template, write=True), + "list_databases": _tool("List the databases permitted or visible to the key's maker.", + _object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("analyst",)), + "describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.", + _object({"database": _STRING}, ("database",), additionalProperties=False), + lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)), + "view_upsert": _tool( + "Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.", + _object({"database": _STRING, "view": _object({ + "id": {"type": "string", "minLength": 1, "maxLength": 120}, + "name": {"type": "string", "minLength": 1, "maxLength": 120}, + "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"))}, + ("database", "view"), additionalProperties=False), + lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)), + "view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.", + _object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"), + additionalProperties=False), + lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("analyst",)), + "app_upsert": _tool( + "Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.", + _object(_APPLICATION, additionalProperties=False, + anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]}, + {"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]), + lambda ctx, args: ctx.app_upsert(args), write=True, grants=("analyst",)), + "app_delete": _tool("Delete an App. confirmName must match its current name exactly.", + _object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"), + additionalProperties=False), + lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)), +} + + +def dispatch(ctx, name, arguments): + """Dispatch one declared tool. The API translates core and grant refusals to MCP content.""" + tool = TOOLS.get(name) if isinstance(name, str) else None + if tool is None: + raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.") + for grant in tool["grants"]: + ctx.session.require(grant) + schema = tool["inputSchema"] + if not isinstance(arguments, dict): + raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.") + missing = set(schema["required"]) - arguments.keys() + unknown = arguments.keys() - schema["properties"].keys() + if missing or (unknown and schema.get("additionalProperties") is False): + raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.") + # Validate only the transport shape here. Document semantics belong to the app writers. + for key, value in arguments.items(): + kind = schema["properties"].get(key, {}).get("type") + types = {"object": dict, "array": list, "string": str, "integer": int} + if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))): + raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.") + # A dimension patch must never become an arbitrary tab-document replacement. + if name == "sheet_set_dimensions": + allowed = schema["properties"]["dimensions"]["properties"] + if arguments["dimensions"].keys() - allowed.keys(): + raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.") + if "lpb1." in json.dumps(arguments, ensure_ascii=False, allow_nan=False): + raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.") + return tool["handler"](ctx, arguments) diff --git a/platform/core/measure_resolve.py b/platform/core/measure_resolve.py index 53633d1226c3177a498bfec750894b5cd1a6a3b2..bb65aa676899b4f42a373e866b4a8c28f4ab3597 100644 --- a/platform/core/measure_resolve.py +++ b/platform/core/measure_resolve.py @@ -152,8 +152,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No `related` is the row-window {table, record, via} pin. The host binds its scoped readers with relations.measure_pin_context and publishes the returned mapping's `refusals` beside - its cells. A refused pin retains unscoped values plus a per-field named marker, on memo - hits too. Every measure and its conditions use the same query scope and caller's pool. + its cells. A refused pin publishes a per-field marker and no values, on memo hits too. + Every measure and its conditions use the same query scope and caller's pool. """ mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)] if not mfields: @@ -197,6 +197,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No if related is not None: out.refusals.extend(dict(marker, field=fkey) for marker in getattr(vals, 'refusals', ())) + if getattr(vals, 'refusals', ()): + continue for pid, v in vals.items(): out.setdefault(pid, {})[fkey] = v if len(memo) > 100: diff --git a/platform/core/odoo.py b/platform/core/odoo.py index 0e709a1b0e7ef30acf467898d8986dabf9330bbf..5e1506a7fa513a5818576107b578885d649cbbc9 100644 --- a/platform/core/odoo.py +++ b/platform/core/odoo.py @@ -103,7 +103,9 @@ def tenant_scope(slug, creds=None, team_ids=None, team_names=None, exclude_partn Unbound (the default) = tenant #0 via env, which is every existing Streamlit/API caller. The connector layer (harness/connectors/odoo.py) wraps each query in this, so a second tenant's keychain credentials reach the wire without any module knowing. Scope constants - are PUSHED here from Tenant.config because core/ may not import harness/ (layering).""" + are PUSHED here from Tenant.config because core/ may not import harness/ (layering; enforced + forward by platform/verify_layering.py, which grandfathers the older core -> harness edges + by name, D-460).""" prev = getattr(_tlocal, 'tenant_ctx', None) _tlocal.tenant_ctx = { 'slug': str(slug or _DEFAULT_SLUG), diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index de0a35a7606287201446a71f6e90e21c10b35c06..f7494bdcc99bff23462183cbbd6818f34fab7ef5 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -40,6 +40,7 @@ twice in this codebase (`modules: []` and `bus: []` both read as UNR perms block; without this clause an explicit-marker scheme locks the owner out of their own product at exactly the moment the store is broken. """ +import inspect import re import core.perms as perms @@ -1400,7 +1401,7 @@ class Unresolvable(RuntimeError): #: Row readers DECLARED by the app layer, keyed by EXACT database key. -#: `reader(table_key, user, st) -> (fields, rows)`. +#: `reader(table_key, user, st[, want_rows=True]) -> (fields, rows)`. #: #: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and #: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache @@ -1409,6 +1410,7 @@ class Unresolvable(RuntimeError): #: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being #: interrogated by it."* _ROW_SOURCES = {} +_ROW_SOURCE_WANT_ROWS = {} #: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror. #: `reader(table_key, field_keys, st) -> rows`. @@ -1423,10 +1425,18 @@ def register_rows(reader, *table_keys): says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write door's return keeps one construction site of the set instead of two. """ + # Decide the calling convention once. Never retry a reader's own TypeError as an old + # signature: doing so can silently replace a failed read with a different answer. + parameters = inspect.signature(reader).parameters + accepts_want_rows = any(p.kind == inspect.Parameter.VAR_KEYWORD + for p in parameters.values()) or ( + 'want_rows' in parameters and parameters['want_rows'].kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)) for key in table_keys: k = str(key or '').strip() if k: _ROW_SOURCES[k] = reader + _ROW_SOURCE_WANT_ROWS[k] = accepts_want_rows return frozenset(_ROW_SOURCES) @@ -1568,8 +1578,8 @@ def scoped_fields(user, table_key, st=None): second path to the store (W1). So both come from here, off one wall. ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a - registry topic it goes through the registered reader, which builds that topic's pool; the - pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`. + registry topic it asks the registered reader for fields only when supported. Legacy + three-argument readers retain their existing call contract. """ fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False) return fields @@ -1613,7 +1623,10 @@ def _read(table_key, user, st, want_rows=True): """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`.""" reader = _ROW_SOURCES.get(table_key) if reader is not None: - fields, rows = reader(table_key, user, st) + if _ROW_SOURCE_WANT_ROWS.get(table_key, False): + fields, rows = reader(table_key, user, st, want_rows=want_rows) + else: + fields, rows = reader(table_key, user, st) return list(fields or ()), list(rows or ()) ut = _ut() if not table_key.startswith(ut.KEY_PREFIX): diff --git a/platform/core/relations.py b/platform/core/relations.py index 301c18461cfe04f98d3f886d6870b5a31b9a925c..a8c95a4b9ec8bae8ce24a52dd10aa78a3faa8727 100644 --- a/platform/core/relations.py +++ b/platform/core/relations.py @@ -265,6 +265,20 @@ def _link_bag(field): return bag if isinstance(bag, dict) else None +def declared_through(link): + """R50 / C4: return the intermediate of a complete declared two-hop link, or ''. + + The intermediate is a join, not a destination. R50 says grants on both ends suffice; + intermediate fields are never projected, returned or rendered. An incomplete through + bag is not a declared two-hop link and earns no traversal admission. Manual links do + not declare a traversal either. + """ + if (isinstance(link, dict) and link.get('through') and link.get('throughOn') + and link.get('throughTo') and link.get('selectionMode') != 'manual'): + return str(link['through']) + return '' + + def _entry(key, label, target, direction, field=None, on=None, frm=None, served=True, where=None, cardinality=None): """ONE `/related` entry, in C6's shape. @@ -745,6 +759,10 @@ def measure_pin_key(related): if related is None: return None ctx = _MEASURE_PIN_CONTEXT.get() or {} + if isinstance(related, MeasurePin): + # Compiled pins contain the server's walled membership, not just a wire path. + related = dict(path=related.path, rows=related.rows, bindings=related.bindings, + refusal=related.refusal, required=related.required) return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key') @@ -754,11 +772,17 @@ class MeasurePin: rows: dict bindings: dict refusal: dict = None + required: tuple = () def predicate(self, store): """AND every declared, traversed member set into this metric's raw rows.""" if self.refusal is not None: return None, [] + bound = {b['table'] for b in self.bindings.get(store['table'], ())} + if set(self.required) - bound: + raise PinRefusal('relation_binding_unavailable', + 'This metric cannot be restricted to every database in this record path. ' + 'Ask an administrator to add the missing metric binding.', self.path) parts, params = [], [] for binding in self.bindings.get(store['table'], ()): table, column = binding['table'], binding['column'] @@ -1126,6 +1150,15 @@ def one_hop(table, fields=None, defs=None, include_retired=False): out = [] for entry in base: entry = dict(entry) + # Keep related()'s legacy shape. Depth-aware pins also need the declared manual, + # inverse or through configuration, which remains schema-only cache input. + via = entry.get('via') or {} + owner_fields = (((defs or {}).get(entry['target']) or {}).get('fields') or [] + if via.get('direction') == 'reverse' else fields or []) + declared = next((field for field in owner_fields + if field.get('key') == via.get('field')), None) + if declared and _link_bag(declared): + entry['link'] = dict(_link_bag(declared)) got = cards.get(entry['key']) or bridge_cards.get(entry['key']) entry['cardinality'] = str(got or 'many_to_many') out.append(entry) diff --git a/platform/core/shares.py b/platform/core/shares.py index 7219234a6cf300afaa20ff1daa6c0f068dd2c50b..2d6de7f85d1d01c56f1b61107695c261b65885ac 100644 --- a/platform/core/shares.py +++ b/platform/core/shares.py @@ -553,11 +553,28 @@ def role_for(kind, oid, user, is_admin=False, st=None): user = str(user or '').strip().lower() if not user: return None - rec = grants(kind, oid, st=st) - if is_admin or (rec['owner'] and rec['owner'] == user): + return role_from(grants(kind, oid, st=st), user, is_admin=is_admin) + + +def role_from(rec, user, is_admin=False): + """The SAME answer as `role_for`, from a record the caller already holds. PURE. + + ⭐⭐ W61-T17 — ONE EVALUATOR, TWO READ PATHS. `grants_bulk` exists so a list of N objects + costs ONE bucket read instead of N, and the moment a batched caller re-implements "which role + does this grant record give?" the two answers can drift — and the one that drifts is a + PERMISSION answer. So the loop lives here, `role_for` is now a thin read-then-call, and a + batched caller passes the record it already has. [[one-evaluator-per-question]] + + ⚠ `rec` may be None (an object with no grant record at all), which reads as no role. + """ + user = str(user or '').strip().lower() + if not user: + return None + rec = rec or {'owner': None, 'entries': []} + if is_admin or (rec.get('owner') and rec['owner'] == user): return 'owner' best = None - for e in rec['entries']: + for e in rec.get('entries') or (): if e['user'] == user or e['user'] == EVERYONE: # The STRONGER of the two wins when both a personal and an everyone grant exist: # naming somebody explicitly is how you RAISE them above the room, so an @@ -568,6 +585,37 @@ def role_for(kind, oid, user, is_admin=False, st=None): return best +def grants_bulk(kind, st=None): + """`{oid: {'owner', 'entries'}}` for every object of one `kind` — in ONE bucket read. + + ⭐⭐ W61-T17 — THE SAME SHAPE `granted_oids` EXISTS FOR, AND THE SAME REASON, stated there in + full: `grants()` re-reads and re-copies the WHOLE registry on every call, so a list of N + objects that asks per object pays N full reads of the tenant's entire grant registry. This + reads it once. `granted_oids` answers "does it have any grants"; this answers "what are they", + which is what a list joining `shares` onto rows actually needs. + + ⛔ IT NORMALISES THROUGH `_clean_entries`, exactly as `grants` does, so a junk entry cannot + reach a caller by the batched door that the single door would have dropped. An owner husk left + by a full revoke keeps its owner and an EMPTY entry list, which is what `grants` answers too. + ⛔ FAIL-CLOSED: an unreadable bucket answers `{}` — no roles, no shares — never a default. + An unknown KIND still RAISES, as everywhere else in this module. + """ + kind = _check_kind(kind) + try: + by_kind = (_st(st).get(SHARES_KEY) or {}).get(kind) or {} + rows = list(by_kind.items()) + except Exception: + return {} + out = {} + for oid, rec in rows: + if not isinstance(rec, dict): + continue + out[str(oid)] = {'owner': (str(rec.get('owner')).strip().lower() + if rec.get('owner') else None), + 'entries': _clean_entries(rec.get('entries'))} + return out + + def may_see(kind, oid, user, is_admin=False, st=None): return role_for(kind, oid, user, is_admin=is_admin, st=st) is not None diff --git a/platform/core/sheet_binding.py b/platform/core/sheet_binding.py new file mode 100644 index 0000000000000000000000000000000000000000..e1842a0fb86d34e4788f0a7b0926ce073b11208c --- /dev/null +++ b/platform/core/sheet_binding.py @@ -0,0 +1,180 @@ +"""Pure document cleaners shared by Sheet storage and Sheet query resolution. + +These functions decide only what binding metadata may be stored. They do not resolve data, +permissions, or SQL. Keeping them in core lets the workbook document validate view bindings +without importing the higher ``modules.sheet`` layer. +""" + +from __future__ import annotations + +import datetime +import re + +from core import windows as W + + +PERIOD_UNITS = ("day", "week", "month", "quarter", "year") +PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$") +PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$") +PERIOD_REF = re.compile(r"^[A-Z]{1,3}[1-9][0-9]{0,6}$") +DRAG_ACROSS = ("period", "none") +DRAG_DOWN_NONE = "none" + +_MESSAGES = { + "period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.", + "ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.", + "ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.", + "ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.", +} + + +class BindingMetadataRefusal(Exception): + """A structured metadata refusal which each caller translates to its route exception.""" + + def __init__(self, code: str, subject: str = "", detail: str = ""): + super().__init__(code) + self.code = code + self.subject = str(subject or "") + self.detail = str(detail or "") + self.message = _MESSAGES.get(code, "{subject} could not be answered.").format( + subject=self.subject or "this database") + if code in ("ending_invalid", "ending_cell_unreadable") and self.detail: + verb = "used" if code == "ending_invalid" else "read" + self.message = self.message.replace( + f"cannot be {verb}.", f"cannot be {verb} because {self.detail}") + + +def clean_period(raw): + """Return one canonical Sheet period, or ``None`` when no window was named.""" + if not isinstance(raw, dict): + return None + out = {} + + if "ending" in raw: + window = W.normalize(raw.get("window")) + kind = (window or {}).get("kind") + if kind not in W.TO_DATE_KINDS: + raise BindingMetadataRefusal("ending_not_supported", kind or "This period") + ending = raw["ending"] + mode = ending.get("kind") if isinstance(ending, dict) else None + if mode not in W.ENDING_KINDS: + raise BindingMetadataRefusal("ending_invalid", str(mode or "selection"), + "the choice is unknown.") + step = raw.get("endingStep", W.ending_step_default(kind)) + offset = raw.get("endingOffset", 0) + if (step not in W.ENDING_STEPS or isinstance(offset, bool) + or not isinstance(offset, int) or offset < 0): + raise BindingMetadataRefusal("ending_invalid", "step", + "its size or copy offset is invalid.") + out = {"window": window, "ending": {"kind": mode}, "endingStep": step} + if offset: + out["endingOffset"] = offset + if raw.get("locked") is True: + out["locked"] = True + if mode == "fixed": + day = W.ending_date(ending.get("date")) + if day is None: + raise BindingMetadataRefusal("ending_invalid", "fixed date", + "it is not a real ISO date.") + out["ending"]["date"] = day + if mode == "n_before_today": + n, unit = ending.get("n"), ending.get("unit") + if isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= W.MAX_N: + raise BindingMetadataRefusal( + "ending_invalid", "number", f"it must be a whole number from 1 to {W.MAX_N}.") + if unit not in W.ENDING_STEPS: + raise BindingMetadataRefusal("ending_invalid", "unit", "the unit is unknown.") + out["ending"].update({"n": n, "unit": unit}) + if mode == "cell": + ref = str(raw.get("ref") or "").strip() + if not re.fullmatch( + r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref): + raise BindingMetadataRefusal("ending_cell_unreadable", ref or "(blank)", + "its reference is invalid.") + out["ref"] = ref + return out + + unit = str(raw.get("unit") or "").strip().lower() + if unit: + if unit not in PERIOD_UNITS: + raise BindingMetadataRefusal("period_invalid", unit) + out["unit"] = unit + + start = str(raw.get("start") or "").strip() + if start: + pattern = PERIOD_DAY if unit in ("day", "week") else PERIOD_START + if not pattern.fullmatch(start): + raise BindingMetadataRefusal("period_invalid", start) + if unit in ("day", "week"): + try: + day = datetime.date.fromisoformat(start) + except ValueError as exc: + raise BindingMetadataRefusal("period_invalid", start) from exc + if unit == "week": + start = (day - datetime.timedelta(days=day.weekday())).isoformat() + out["start"] = start + + if raw.get("frozen") is True: + dates = {} + for key in ("from", "to"): + value = str(raw.get(key) or "").strip() + if not PERIOD_DAY.fullmatch(value): + raise BindingMetadataRefusal("period_invalid", value or key) + dates[key] = value + if dates["to"] < dates["from"]: + raise BindingMetadataRefusal("period_invalid", dates["to"]) + out.update({"frozen": True, **dates}) + + if raw.get("locked") is True: + out["locked"] = True + ref = str(raw.get("ref") or "").strip().upper().replace("$", "") + if ref: + if not PERIOD_REF.match(ref): + raise BindingMetadataRefusal("period_invalid", ref) + if not out.get("frozen"): + out["ref"] = ref + if (raw.get("window") is not None and "unit" not in out and "start" not in out + and not out.get("frozen")): + out["window"] = raw["window"] + if not ({"start", "window", "ref", "frozen"} & set(out)): + return None + return out + + +def clean_on_drag(raw): + """Return canonical two-axis drag metadata, or ``None`` when absent.""" + if not isinstance(raw, dict): + return None + across = str(raw.get("across") or "").strip().lower() + if across not in DRAG_ACROSS: + across = "period" + down = raw.get("down") + if isinstance(down, dict) and str(down.get("field") or "").strip(): + out_down = {"field": str(down["field"]).strip()} + if down.get("mode") in ("value", "presence", "member"): + out_down["mode"] = down["mode"] + else: + out_down = DRAG_DOWN_NONE + if isinstance(raw.get("steps"), list): + steps = [] + for step in raw["steps"]: + if not isinstance(step, dict): + continue + field = step.get("field") + if not isinstance(field, str) or not field.strip() or step.get("axis") not in ("across", "down"): + continue + if "size" in step and step["size"] not in ("day", "week", "month", "quarter", "year"): + continue + cleaned = {"field": field.strip(), "axis": step["axis"]} + if step.get("mode") in ("value", "presence", "member"): + cleaned["mode"] = step["mode"] + if "size" in step: + cleaned["size"] = step["size"] + steps.append(cleaned) + first_down = next((step for step in steps if step["axis"] == "down"), None) + out_down = ({key: first_down[key] for key in ("field", "mode") if key in first_down} + if first_down else DRAG_DOWN_NONE) + across = ("period" if any(step["axis"] == "across" and step["field"] + in ("__period", "__ending") for step in steps) else "none") + return {"across": across, "down": out_down, "steps": steps} + return {"across": across, "down": out_down} diff --git a/platform/core/sheet_import.py b/platform/core/sheet_import.py index 502ba719f5b6c8ee6c37127c9281f188f715d62b..c1c433f81775ceec046d623622b5e4e39ed5e3d3 100644 --- a/platform/core/sheet_import.py +++ b/platform/core/sheet_import.py @@ -71,6 +71,13 @@ PALETTE = { "--lp-ink": "#202433", "--lp-muted": "#6b7280", "--lp-line": "#e6e9ef", + # ⭐ ADDED AT THE WAVE-62 COMBINE. W62-T31 / D-684 declares `--lp-line-strong` in + # `index.css` as a darker step than `--lp-line` for stronger separators, and + # `verify_sheet.py` C4's other direction went red the moment lane H's branch merged: + # a colour `:root` declares that the importer cannot quantize onto is a colour an + # imported cell can never keep. `--lp-line` was already here, so a line colour belongs + # in this table by the same rule; the size tokens the head note excludes are unaffected. + "--lp-line-strong": "#cbd1dc", "--lp-wash": "#fafbfc", "--lp-blue": "#9dbff2", "--lp-blue-tint": "#edf3fd", @@ -89,6 +96,23 @@ PALETTE = { "--lp-purple": "#c3b3ee", "--lp-purple-tint": "#f1eefb", "--lp-purple-deep": "#6b57a8", + # ⭐ W61-T08 / THE COMBINE — THE THIRD SEAM. W61-T08 gives chart series eight distinct brand + # hues and declares each in the two seams its `done-when` names, `index.css` `:root` and + # `customer-grid/theme.ts`. This copy is a THIRD, and nothing pointed lane A at it: it lives in + # `platform/core`, which may not read the web tree at all, so only `verify_sheet.py` section 4 + # relates the two — and it is an API gate, not one of that ticket's. It caught the drift on the + # merged tree exactly as designed ("a colour ADDED to :root cannot sit unreachable to an + # import"), which is why these are ADDED here rather than the leg being narrowed: an .xlsx + # painted one of the new hues must quantize onto it, not onto the nearest older colour. + "--lp-orange": "#edbf91", + "--lp-orange-tint": "#fcf1e6", + "--lp-orange-deep": "#ba6c27", + "--lp-pink": "#e6afcc", + "--lp-pink-tint": "#f9edf4", + "--lp-pink-deep": "#a54c78", + "--lp-teal": "#9bcfce", + "--lp-teal-tint": "#eaf5f4", + "--lp-teal-deep": "#00938d", "--lp-brand": "#6e56cf", "--lp-primary": "#5433cc", "--lp-primary-hover": "#472bad", diff --git a/platform/core/sheet_structure.py b/platform/core/sheet_structure.py new file mode 100644 index 0000000000000000000000000000000000000000..102ae0bc304a82b55a749e6d809746f34d6e5b7f --- /dev/null +++ b/platform/core/sheet_structure.py @@ -0,0 +1,316 @@ +"""Pure workbook structure edits using Sheet's A1 scanner and the Wave 61 C1 rules. + +Addresses, hidden lines and row-height keys are 1-based; rectangles are 0-based. +Structural edits move absolute references too. The dollar marks only pin a copy/fill. +The caller owns document validation, persistence and recalculation. +""" +from __future__ import annotations + +import copy +import math +import re +from collections.abc import Callable + +from core.sheets import SheetRefusal + + +REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})" +_REF = re.compile(REF_SOURCE) +_BARE_SHEET = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,63}") +_PREVIOUS = re.compile(r"[A-Za-z0-9_$]") +_START = re.compile(r"[A-Za-z$']") +_FUNCTION = re.compile(r"\s*\(") +BINDING_REF_KEYS = frozenset({"ref", "valueRef", "cellRef", "endingRef"}) + + +def col_to_index(letters: str) -> int: + """Spreadsheet base-26: A -> 1, Z -> 26, AA -> 27.""" + index = 0 + for letter in letters.upper(): + index = index * 26 + ord(letter) - 64 + return index + + +def index_to_col(index: int) -> str: + """The inverse of col_to_index; an index below 1 yields an empty string.""" + index = math.floor(index) + letters = [] + while index > 0: + index, remainder = divmod(index - 1, 26) + letters.append(chr(65 + remainder)) + return "".join(reversed(letters)) + + +def parse_ref(text: str) -> dict | None: + """Parse one A1 reference, retaining the client's CellRef dictionary keys.""" + match = _REF.fullmatch(str(text or "").strip()) + if not match: + return None + sheet_quoted, sheet_bare, col_abs, col, row_abs, row = match.groups() + return {"sheet": sheet_quoted or sheet_bare or "", "colAbs": col_abs == "$", + "col": col.upper(), "rowAbs": row_abs == "$", "row": int(row)} + + +def format_ref(ref: dict | str) -> str: + """Format a CellRef, preserving dollar marks and quoting non-bare sheet names.""" + if isinstance(ref, str): + return ref + body = (f"{'$' if ref['colAbs'] else ''}{ref['col']}" + f"{'$' if ref['rowAbs'] else ''}{ref['row']}") + sheet = ref["sheet"] + if not sheet: + return body + name = sheet if _BARE_SHEET.fullmatch(sheet) else "'" + sheet.replace("'", "") + "'" + return name + "!" + body + + +def rewrite_refs(text: str, rewrite: Callable[[dict], dict | str], + rewrite_range: Callable[[dict, dict, str], str] | None = None) -> str: + """Walk text exactly as a1.ts::rewriteRefs; strings and function names survive.""" + src = str(text or "") + out = [] + i = 0 + quote = None + while i < len(src): + char = src[i] + if quote: + out.append(char) + if char == quote: + quote = None + i += 1 + continue + if char in ('"', "'"): + close = src.find(char, i + 1) + is_sheet = char == "'" and close > i and src[close + 1:close + 2] == "!" + if not is_sheet: + quote = char + out.append(char) + i += 1 + continue + previous = src[i - 1] if i else "" + if not _PREVIOUS.search(previous) and _START.fullmatch(char): + match = _REF.match(src, i) + if match and not _FUNCTION.match(src, match.end()): + ref = parse_ref(match[0]) + if ref: + separator = re.match(r"\s*:\s*", src[match.end():]) if rewrite_range else None + if separator: + end = _REF.match(src, match.end() + separator.end()) + if end and not _FUNCTION.match(src, end.end()): + last = parse_ref(end[0]) + if last: + out.append(rewrite_range(ref, last, separator[0])) + i = end.end() + continue + out.append(format_ref(rewrite(ref))) + i = match.end() + continue + out.append(char) + i += 1 + return "".join(out) + + +def shift_index(value: int, at: int, delta: int) -> int | None: + """R49 retires the clamp: both grammars already parse #REF! as an error token. + + The feared bad_formula cannot occur. Deleted indices return None; only a range + with surviving cells may clamp its deleted endpoint and shrink. + """ + return _moved_index(value, at, delta) + + +def _same_name(left: str, right: str) -> bool: + return str(left or "").strip().lower() == str(right or "").strip().lower() + + +def shift_for_structure(text: str, shift: dict, own_sheet: str, *, bare: bool = False) -> str: + """Re-anchor formula text or, with bare=True, a single reference field. + + shift names the edited sheet and contains axis (row/col), at and delta. + A bare reference belongs to own_sheet. Invalid bare fields are left verbatim. + """ + if not shift["delta"] or (bare and parse_ref(text) is None): + return str(text or "") + + def index(ref): + return ref["row"] if shift["axis"] == "row" else col_to_index(ref["col"]) + + def at_index(ref, value): + return {**ref, shift["axis"]: value if shift["axis"] == "row" else index_to_col(value)} + + def rewrite(ref): + if not _same_name(ref["sheet"] or own_sheet, shift["sheet"]): + return ref + moved = shift_index(index(ref), shift["at"], shift["delta"]) + return "#REF!" if moved is None else at_index(ref, moved) + + def rewrite_range(first, last, separator): + sheet = first["sheet"] or own_sheet + if last["sheet"] and not _same_name(sheet, last["sheet"]): + return format_ref(rewrite(first)) + separator + format_ref(rewrite(last)) + # A bare right endpoint inherits the left endpoint's sheet, not the formula's tab. + if not _same_name(sheet, shift["sheet"]): + return format_ref(first) + separator + format_ref(last) + start, end = index(first), index(last) + low, high = min(start, end), max(start, end) + if low >= shift["at"] and high < shift["at"] - shift["delta"]: + return "#REF!" + + def move(value): + moved = shift_index(value, shift["at"], shift["delta"]) + return moved if moved is not None else shift["at"] if value == low else shift["at"] - 1 + + return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end))) + + # Insertions keep the existing per-reference scan; range handling is deletion-only. + return rewrite_refs(text, rewrite, rewrite_range if shift["delta"] < 0 else None) + + +def _moved_index(value: int, at: int, delta: int) -> int | None: + if value < at: + return value + if delta > 0: + return value + delta + return None if value < at - delta else value + delta + + +def _reanchor_binding(value, shift: dict, own_sheet: str) -> bool: + """Rewrite only parseable ref-bearing fields in the copied binding's dict/list tree.""" + changed = False + if isinstance(value, list): + for child in value: + changed = _reanchor_binding(child, shift, own_sheet) or changed + elif isinstance(value, dict): + for key, child in value.items(): + if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child): + value[key] = shift_for_structure(child, shift, own_sheet, bare=True) + changed = value[key] != child or changed + elif isinstance(child, (dict, list)): + changed = _reanchor_binding(child, shift, own_sheet) or changed + return changed + + +def _reanchor_cells(tab: dict, shift: dict) -> None: + for cell in tab.get("cells", {}).values(): + changed = False + if cell.get("kind") == "formula" and isinstance(cell.get("formula"), str): + previous = cell["formula"] + cell["formula"] = shift_for_structure(previous, shift, tab["name"]) + changed = cell["formula"] != previous + if changed and cell.get("display") == previous: + cell["display"] = cell["formula"] + if cell.get("kind") == "bound": + changed = _reanchor_binding(cell.get("binding"), shift, tab["name"]) or changed + link = cell.get("link") + if isinstance(link, dict) and link.get("kind") in ("cell", "sheet"): + href = link.get("href") + if isinstance(href, str) and parse_ref(href): + link["href"] = shift_for_structure(href, shift, tab["name"], bare=True) + changed = link["href"] != href or changed + if changed and cell.get("kind") in ("formula", "bound"): + cell["state"] = "stale" + + +def _move_range(rect: dict, op: dict, *, drop_deleted: bool = False) -> dict | None: + """Keep surviving rectangle cells; collapse a fully deleted chart at the gap.""" + start, end = op["axis"] + "0", op["axis"] + "1" + lo, hi = rect[start], rect[end] + point, delta = op["at"] - 1, op["delta"] + if delta > 0: + lo += delta if point <= lo else 0 + hi += delta if point <= hi else 0 + elif delta < 0: + past = point - delta + before, after = lo < point, hi >= past + if not before and not after and drop_deleted: + return None + lo = lo if before else max(lo, past) + delta if after else point + hi = hi + delta if after else min(hi, point - 1) if before else point + return {**rect, start: lo, end: hi} + + +def _move_tab(tab: dict, op: dict) -> None: + axis, at, delta = op["axis"], op["at"], op["delta"] + cells = {} + for address, cell in tab.get("cells", {}).items(): + ref = parse_ref(address) + if not ref: + cells[address] = cell + continue + moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta) + if moved is not None: + ref[axis] = moved if axis == "row" else index_to_col(moved) + cells[format_ref(ref)] = cell + tab["cells"] = cells + dimension = "rowHeights" if axis == "row" else "colWidths" + if dimension in tab: + sizes = {} + for key, size in tab[dimension].items(): + moved = _moved_index(int(key) if axis == "row" else col_to_index(key), at, delta) + if moved is not None: + sizes[str(moved) if axis == "row" else index_to_col(moved)] = size + tab[dimension] = sizes + hidden = "hiddenRows" if axis == "row" else "hiddenCols" + if hidden in tab: + tab[hidden] = [moved for value in tab[hidden] + if (moved := _moved_index(value, at, delta)) is not None] + frozen = tab.get("freeze", {}) + key = "rows" if axis == "row" else "cols" + if key in frozen and at <= frozen[key]: + # Only deleted frozen lines shrink the count, even when the band extends beyond it. + frozen[key] = max(0, frozen[key] + (delta if delta > 0 else -min(-delta, frozen[key] - at + 1))) + if "rules" in tab: + rules = [] + for rule in tab["rules"]: + rect = _move_range(rule["range"], op, drop_deleted=True) + if rect is not None: + rules.append({**rule, "range": rect}) + tab["rules"] = rules + + +def _move_charts(tab: dict, tab_id: str, op: dict) -> None: + for chart in tab.get("charts", []): + source = chart.get("source") or {} + if source.get("sheetId", tab["id"]) == tab_id: + chart["range"] = _move_range(chart["range"], op) + if "ranges" in source: + source["ranges"] = [_move_range(rect, op) for rect in source["ranges"]] + if tab["id"] == tab_id: + # Stored geometry anchors are zero-based too; pixel offsets and sizes stay intact. + for key in ("geometry", "anchor"): + geometry = chart.get(key) + if isinstance(geometry, dict) and op["axis"] in geometry: + moved = shift_index(geometry[op["axis"]] + 1, op["at"], op["delta"]) + geometry[op["axis"]] = (op["at"] if moved is None else moved) - 1 + + +def apply_structure(book: dict, tab_id: str, op: dict) -> dict: + """Return a new document after an insert/delete on tab_id; never access persistence. + + op requires axis (row/col), a positive integer at and an integer delta. Zero delta + returns an independent unchanged copy. An optional sheet name must match tab_id. + Sheet dimensions are viewport extents, retained as in the client applyStructure. + """ + if not isinstance(op, dict) or op.get("axis") not in ("row", "col"): + raise SheetRefusal("sheet_structure_invalid", "A structure change needs a row or column axis.") + if type(op.get("at")) is not int or op["at"] < 1: + raise SheetRefusal("sheet_structure_invalid", "A structure change needs a positive whole starting index.") + if type(op.get("delta")) is not int: + raise SheetRefusal("sheet_structure_invalid", "A structure change needs a whole insert or delete count.") + if not isinstance(book, dict) or not isinstance(book.get("sheets"), list): + raise SheetRefusal("sheet_structure_invalid", "A structure change needs a workbook with sheets.") + target = next((tab for tab in book["sheets"] if tab.get("id") == tab_id), None) + if target is None: + raise SheetRefusal("sheet_structure_tab_missing", "The sheet to insert or delete on does not exist.") + if "sheet" in op and not _same_name(op["sheet"], target["name"]): + raise SheetRefusal("sheet_structure_invalid", "The structure change names a different sheet.") + result = copy.deepcopy(book) + if not op["delta"]: + return result + shift = {**op, "sheet": target["name"]} + for tab in result["sheets"]: + _reanchor_cells(tab, shift) + _move_charts(tab, tab_id, op) + if tab["id"] == tab_id: + _move_tab(tab, op) + return result diff --git a/platform/core/sheet_templates.py b/platform/core/sheet_templates.py index b393ff560a24262bb0f339bf3260995bfdaaed7f..9e375e0d35b55b2888cda0e52635a2726bd8e324 100644 --- a/platform/core/sheet_templates.py +++ b/platform/core/sheet_templates.py @@ -1,4 +1,4 @@ -"""Loopable's prebuilt connector workbooks (W59-T29, contract C4). +"""Loopable's built-in and tenant-saved connector workbooks. Descriptors are data; applying them uses the existing workbook persistence engine. The API owns permission checks and calculation under the calling user's row scope. @@ -6,7 +6,7 @@ The API owns permission checks and calculation under the calling user's row scop from __future__ import annotations from copy import deepcopy -from datetime import date, timedelta +from datetime import date, datetime, timedelta, timezone import re from core import sheets, user_tables @@ -154,6 +154,103 @@ def templates_for_sources(sources): return [row for row in TEMPLATES if row["source"] in allowed] +SAVED_TEMPLATES_KEY = "sheet_templates_saved" + + +def saved_templates(st): + """Pure, detached C4 document reader; absent storage is an empty schema-1 document.""" + return deepcopy(st.get(SAVED_TEMPLATES_KEY) or {"schema": 1, "templates": {}}) + + +def saved_template(st, key): + """One detached saved record, or None. Built-in descriptors remain separate.""" + return saved_templates(st)["templates"].get(key) + + +def _template_workbook(book): + clean = deepcopy(book) + for field in ("id", "rev", "writes", "writesFrom"): + clean.pop(field, None) + for tab in clean.get("sheets", []): + tab.pop("writes", None) + tab.pop("writesFrom", None) + return clean + + +def save_sheet_template(st, workbook, key, *, name="", description="", created_by="", actor_name=""): + """Save a workbook already admitted by the caller's reader wall, using C4's tenant document.""" + if not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", key): + raise sheets.SheetRefusal("sheet_template_key_invalid", + "A Sheet template key must use letters, numbers, underscores or hyphens.") + if template(key) is not None: + raise sheets.SheetRefusal("sheet_template_key_exists", + f"Sheet template {key} already names a built-in template.") + if saved_template(st, key) is not None: + raise sheets.SheetRefusal("sheet_template_key_exists", + f"Sheet template {key} already exists. Choose another key.") + if not isinstance(name, str) or not isinstance(description, str): + raise sheets.SheetRefusal("sheet_template_text_invalid", + f"Sheet template {key} needs a text name and description.") + checked = sheets.migrate_workbook(workbook) + book = _template_workbook(checked) + databases = set() + for tab in book["sheets"]: + for addr, cell in tab["cells"].items(): + if cell.get("kind") != "bound": + continue + database = (cell.get("binding") or {}).get("database") + if not isinstance(database, str) or not database.strip(): + raise sheets.SheetRefusal("sheet_template_database_missing", + f"Sheet template {key} has no Cell database at {addr}.") + databases.add(database) + record = {"key": key, "name": name.strip() or book.get("name") or key, + "description": description, "databases": sorted(databases), "workbook": book, + "created_by": created_by, "created_at": datetime.now(timezone.utc).isoformat(), + "byName": actor_name or created_by} + + def save(state): + document = deepcopy(state or {"schema": 1, "templates": {}}) + if document.get("schema") != 1 or not isinstance(document.get("templates"), dict): + raise sheets.SheetRefusal("sheet_templates_schema_unknown", + "Saved Sheet templates use a document version this app cannot save.") + if key in document["templates"]: + raise sheets.SheetRefusal("sheet_template_key_exists", + f"Sheet template {key} already exists. Choose another key.") + document["templates"][key] = deepcopy(record) + return document + + st.update(SAVED_TEMPLATES_KEY, save, flush="sync") + return deepcopy(record) + + +def apply_saved_template(st, key, *, name="", created_by=""): + """Materialise a saved workbook through the same create/put/patch engine as built-ins.""" + record = saved_template(st, key) + if record is None: + # Keys from a URL can contain user text; refusal prose follows standing rule 2. + label = str(key).replace(chr(0x2014), "-").replace(chr(0x2013), "-") + raise sheets.SheetRefusal("unknown_sheet_template", f"Sheet template {label} does not exist.") + # Validate the whole document before the first write, including tabs after the first. + checked = sheets.migrate_workbook(_template_workbook(record["workbook"])) + book = sheets.create_workbook(st, name=name or record["name"], created_by=created_by, + on_duplicate="suffix") + try: + tabs = [{**tab, "cells": {}} for tab in checked["sheets"]] + book = sheets.put_workbook(st, book["id"], { + **checked, "name": book["name"], "sheets": tabs, "rev": book["rev"]}, actor=created_by) + for index, tab in enumerate(checked["sheets"]): + result = sheets.patch_cells(st, book["id"], book["sheets"][index]["id"], + tab["cells"], actor=created_by) + if result.get("refused"): + addresses = [str(item["addr"]) for item in result["refused"]] + raise sheets.SheetRefusal("template_cells_refused", + "The template could not apply cells: " + ", ".join(addresses) + ".") + return sheets.get_workbook(st, book["id"]) + except Exception: + sheets.delete_workbook(st, book["id"]) + raise + + class TemplateRefusal(Exception): def __init__(self, code, *, missing=(), detail=""): self.code = code @@ -205,8 +302,15 @@ def build_cells(template, resolved, database_key): for entry in template["bindings"]: cells, addr = target(entry) + if set(entry) - {"sheet", "addr", "fn", "field", "metric_key", "scope", "period"}: + raise TemplateRefusal("template_invalid", detail="The template binding has an unknown setting.") + aggregate = ({"metric_key": entry["metric_key"]} if "metric_key" in entry else + {"fn": entry["fn"], "field": resolved[entry["field"]]}) + if "metric_key" in entry and (not entry["metric_key"] or "field" in entry or "fn" in entry + or not entry.get("period")): + raise TemplateRefusal("template_invalid", detail="A template metric needs a period and no field aggregate.") binding = {"database": database_key, - "aggregate": {"fn": entry["fn"], "field": resolved[entry["field"]]}, + "aggregate": aggregate, "scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)} if "period" in entry: binding["period"] = deepcopy(entry["period"]) @@ -233,12 +337,14 @@ def apply_template(st, key, database_key, *, name="", created_by=""): raise TemplateRefusal("unknown_database", detail="That database does not exist.") if source_of_database(database_key) != descriptor["source"]: raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.") - resolved = resolve_fields(descriptor, (field["key"] for field in definition.get("fields", []))) + field_keys = [field["key"] for field in definition.get("fields", [])] + resolved = resolve_fields(descriptor, field_keys) prepared = build_cells(descriptor, resolved, database_key) # Validate tab names, extents and every cell before the first store write. checked = sheets.migrate_workbook({"sheets": [ {"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]}) - book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by) + book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by, + on_duplicate="suffix") try: tabs = [{**tab, "cells": {}} for tab in checked["sheets"]] book = sheets.put_workbook(st, book["id"], {"name": book["name"], "sheets": tabs, "rev": book["rev"]}) diff --git a/platform/core/sheets.py b/platform/core/sheets.py index c80931acb68d3fdd2763f09116f14a64006dcdba..1740e147df692032d8aa59f92c0a53d4b577d059 100644 --- a/platform/core/sheets.py +++ b/platform/core/sheets.py @@ -63,6 +63,12 @@ import re import uuid from datetime import datetime, timezone +from core.sheet_binding import ( + BindingMetadataRefusal, + clean_on_drag as clean_binding_on_drag, + clean_period as clean_binding_period, +) + #: C1's document key. `` is the workbook id. WORKBOOK_KEY_PREFIX = "sheet_workbook_" @@ -81,6 +87,7 @@ AMAZON_DATA_ORIGIN_NAME = "Amazon Services API" SHEET_SCHEMA_VERSION = 1 DOCUMENT_MAX_COLS = 16384 DOCUMENT_MAX_ROWS = 1048576 +WRITE_LOG_REVS = 200 #: ⛔ THE FROZEN VOCABULARY (contract C1, amended by C2). `verify_api.py` and `verify_sheet.py` #: assert `web/src/sheet/types.ts` carries exactly these, name for name. @@ -92,6 +99,11 @@ DOCUMENT_MAX_ROWS = 1048576 CELL_KINDS = ("literal", "formula", "bound", "empty") CELL_STATES = ("fresh", "stale", "typed", "refused") +# W62 C2 / R31: "every view type can be placed" inside the binding. +VIEW_BINDING_TYPES = ("grid", "kanban", "map", "gallery", "timeline", "list", "calendar", "catalog", "swipe") +VIEW_BINDING_LIMIT_DEFAULT = 100 +VIEW_BINDING_LIMIT_MAX = 10000 + #: A literal is ALWAYS this state (R19). Stated once, here, so no caller has to remember it. #: ⚠ AN `empty` CELL TAKES THE SAME STATE, deliberately: it holds no binding, so `stale` would say #: a refresh is owed for a value that does not exist, and a reader counting stale cells would @@ -192,7 +204,9 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains", #: ⭐ `cell` IS STORED AS AN ABSENCE, exactly as `wrap: false` and `indent: 0` are: it is what #: every rule written before this ticket already is, so writing the word onto them would make a #: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`. -SHEET_RULE_KINDS = ("cell", "scale", "bar") +SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula") +# The client compiler accepts 500 characters per condition. +MAX_RULE_FORMULA_LENGTH = 500 #: The style keys a conditional format rule may repaint. A rule is a PAINT, never a format: it #: cannot change how a number reads, so it carries no `format`. @@ -212,7 +226,11 @@ _RULE_STYLE_COLOURS = ("fill", "color") #: but not stored here, is the defect, and it is what widening this tuple prevents. SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", "combo", "waterfall", "histogram", "bubble", "bullet", "funnel", "gauge", - "polar", "radar", "treemap", "stock", "gantt") + "polar", "radar", "treemap", "stock", "gantt", "sunburst", "boxWhisker") + +# C5 / R21: a spec-level sub-type, never a presentation setting or another chart kind. +SHEET_CHART_STACKING_KINDS = ("column", "bar", "line", "area") +SHEET_CHART_STACKING = ("clustered", "stacked", "percent") #: ⛔⛔ D-640's MECHANISM, STATED. THIS IS EVERY KEY A CLEANED TAB MAY CARRY, and a key that is #: not named here is DELETED on the next save, in silence, with nothing going red: `_clean_sheet` @@ -230,7 +248,7 @@ SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", #: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata. SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze", "rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights", - "visibility", "print", "excel") + "visibility", "view", "print", "excel") #: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of #: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab @@ -254,6 +272,7 @@ _MAX_DECIMALS = 10 #: constants, which is why nothing could grow a sheet or remember a dragged column: there was #: nowhere to put the answer. These are the DEFAULTS a sheet is created with and the ceiling it may #: grow to, and `web/src/sheet/types.ts` mirrors the three defaults. +#: W62 C1 / R14 also mirrors the width bounds there; the client half lands in W62-T05. #: #: ⚠ THE CEILING IS A RENDERING BUDGET, NOT A GRAMMAR LIMIT. `_ADDR` below already admits three #: column letters and seven row digits, and `a1.ts` agrees with it; 702 is the last TWO letter @@ -266,7 +285,7 @@ MIN_SHEET_COLS = 1 MIN_SHEET_ROWS = 1 MAX_SHEET_COLS = 702 MAX_SHEET_ROWS = 100000 -MIN_COL_WIDTH = 40 +MIN_COL_WIDTH = 0 MAX_COL_WIDTH = 1000 #: A column key in `colWidths`. Letters only: a WIDTH belongs to a column, and a row number there @@ -683,6 +702,54 @@ def _clean_sparkline(raw): return {"kind": kind, "range": rect} +def clean_view_binding(raw): + """W62-T10 / C2: normalize a PLACED VIEW binding, or raise `SheetRefusal` with a named code. + + `{database, mode: "view", viewType, fields: [key], filter?, sort: [{field, dir}], period?,` + `limit (1..10000, default 100), onDrag?, block?: {cols, rows}}`. + + ⛔ IT IS A SEPARATE VALIDATOR FROM `modules/sheet.py::clean_binding` RATHER THAN A WIDENING OF + IT. That one REQUIRES an `aggregate`, because a metric binding IS an aggregate; a view has + none. Widening it would make `aggregate` optional for every caller, and the day a metric + binding arrives without one it would resolve to nothing instead of refusing by name. + """ + if not isinstance(raw, dict) or not isinstance(raw.get("database"), str) or not raw["database"].strip(): + raise SheetRefusal("view_binding_invalid", "A view binding must be an object naming a database.") + if raw.get("mode", raw.get("kind")) != "view": + raise SheetRefusal("view_binding_mode_unknown", "A view binding must choose view mode.") + if raw.get("viewType") not in VIEW_BINDING_TYPES: + raise SheetRefusal("view_binding_type_unknown", "That view display type is not supported.") + fields = raw.get("fields") + if not isinstance(fields, list) or not fields or any( + not isinstance(key, str) or not key.strip() for key in fields): + raise SheetRefusal("view_binding_fields_invalid", "A view needs a non-empty list of field keys.") + sort = raw.get("sort", []) + if not isinstance(sort, list) or any( + not isinstance(item, dict) or not isinstance(item.get("field"), str) + or not item["field"].strip() or item.get("dir") not in ("asc", "desc") for item in sort): + raise SheetRefusal("view_binding_sort_invalid", "A view sort needs a field key and asc or desc direction.") + limit = raw.get("limit", VIEW_BINDING_LIMIT_DEFAULT) + if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= VIEW_BINDING_LIMIT_MAX: + raise SheetRefusal("view_binding_limit_invalid", "A view limit must be a whole number from 1 to 10000.") + out = {"database": raw["database"].strip(), "mode": "view", "viewType": raw["viewType"], + "fields": list(fields), "sort": [{"field": item["field"], "dir": item["dir"]} for item in sort], + "limit": limit} + for key in ("filter", "block"): + if key in raw: + out[key] = _document_object(raw[key], "view_binding_" + key) + # W62 C2: one lower-layer metadata cleaner serves both view and metric bindings. + if "period" in raw or "onDrag" in raw: + for key, cleaner in (("period", clean_binding_period), ("onDrag", clean_binding_on_drag)): + if key in raw: + try: + value = cleaner(_json_safe(copy.deepcopy(raw[key]), "view binding " + key)) + except BindingMetadataRefusal as exc: + raise SheetRefusal(exc.code, exc.message) from exc + if value is not None: + out[key] = value + return out + + def clean_cell(raw, addr=""): """Normalize ONE cell, or raise `SheetRefusal` with a named code. @@ -728,7 +795,17 @@ def clean_cell(raw, addr=""): if meta and meta.get("kind", "normal") not in ("normal", "array", "shared", "dataTable"): raise SheetRefusal("cell_formula_kind_unknown", "That formula representation is not supported.") binding = raw.get("binding") - if isinstance(binding, dict): + # ⭐⭐ W62-T10 / C2 — A PLACED VIEW LIVES INSIDE `kind: "bound"`, AND `CELL_KINDS` STAYS + # FROZEN. Wave 49's cell-kind contract is the reason: the vocabulary is pinned here and two + # gates prove the client agrees with it, so a fifth kind would be a change to a frozen + # contract for a feature that does not need one. The discriminator is `binding.mode == "view"`, + # and a view binding carries no `aggregate` at all, which is why it cannot go through the + # metric validator below. + if isinstance(binding, dict) and (binding.get("mode") == "view" or binding.get("kind") == "view"): + if kind != "bound": + raise SheetRefusal("view_binding_invalid", "A view binding needs a bound cell.") + binding = clean_view_binding(binding) + elif isinstance(binding, dict): for tag in ("kind", "mode"): if tag in binding and binding[tag] not in ("number", "column"): raise SheetRefusal("cell_binding_kind_unknown", "A cell binding must choose Number or Column.") @@ -778,7 +855,7 @@ def clean_cell(raw, addr=""): cell["value"] = _json_safe(raw.get("value"), "cell value") for key in ("binding", "source", "refusal"): - blob = raw.get(key) + blob = binding if key == "binding" else raw.get(key) if blob is None: continue if not isinstance(blob, dict): @@ -881,12 +958,19 @@ def _clean_col_widths(raw, cols, *, lossless=False): "Column " + (letters or str(key)) + " is not a column letter.") if isinstance(value, bool) or not isinstance(value, (int, float)): if lossless: - raise SheetRefusal("col_width_invalid", "A column width must be a finite positive size.") + raise SheetRefusal("col_width_invalid", "A column width must be a finite size.") continue if not math.isfinite(value): raise SheetRefusal("col_width_invalid", "A column width must be a finite size.") - if lossless and value <= 0: - raise SheetRefusal("col_width_invalid", "A column width must be a finite positive size.") + # ⭐ W62-T04 / R14 — A ZERO IS NOW A WIDTH, AND A NEGATIVE ONE STILL IS NOT. Before this + # ticket the lossless door refused every `value <= 0`, which is the same test for both; R14 + # made 0 Excel's own hidden column, so an xlsx carrying one must import rather than refuse. + # A negative width is still nothing a file or a person can mean, so it keeps its refusal on + # the lossless door and is clamped to the floor on a live save. + if lossless and value < 0: + raise SheetRefusal("col_width_invalid", "A column width cannot be negative.") + #: W62 C1 / R14 / R19: reads retain hidden zero widths and exact imported values; + #: only a live save applies the bounds a person can set. width = value if lossless else max(MIN_COL_WIDTH, min(MAX_COL_WIDTH, int(value))) if lossless or width != DEFAULT_COL_WIDTH: out[letters] = width @@ -1045,6 +1129,18 @@ def _clean_rules(raw, *, lossless=False): raise SheetRefusal( "sheet_rule_kind_unknown", "A sheet rule kind must be one of " + ", ".join(SHEET_RULE_KINDS) + ".") + if kind == "formula": + condition = row.get("value") + if not isinstance(condition, str) or not condition.lstrip("= \t\r\n"): + raise SheetRefusal("sheet_rule_formula_missing", + "A formula rule needs a condition.") + if len(condition) > MAX_RULE_FORMULA_LENGTH: + raise SheetRefusal("sheet_rule_formula_too_long", + "A formula rule condition can hold at most " + + str(MAX_RULE_FORMULA_LENGTH) + " characters.") + # A formula has one operand. Normalize even an imported empty/notEmpty op so + # the existing value persistence path cannot silently drop its condition. + op = "eq" rect = _clean_range(row.get("range")) if rect is None: continue @@ -1113,6 +1209,12 @@ def _clean_charts(raw): rect = _clean_range(row.get("range")) if kind not in SHEET_CHART_KINDS: raise SheetRefusal("sheet_chart_kind_unknown", "That chart kind is not supported.") + if "stacking" in row: + if row["stacking"] not in SHEET_CHART_STACKING: + raise SheetRefusal("sheet_chart_stacking_unknown", "That chart sub-type is not supported.") + if kind not in SHEET_CHART_STACKING_KINDS: # Includes sunburst and boxWhisker (C5). + raise SheetRefusal("sheet_chart_stacking_kind_invalid", + "Chart sub-types need a column, bar, line or area chart.") if rect is None: raise SheetRefusal("sheet_chart_range_invalid", "A chart needs a valid source range.") # Settings and source/series bags are authored data. Validate the discriminants we @@ -1120,6 +1222,11 @@ def _clean_charts(raw): chart = _document_object(row, "sheet_chart") chart.update({"id": _text(row.get("id"), _MAX_ID) or _migration_id("chart", index, row), "range": rect, "kind": kind}) + # Keep the C5 spec field explicitly across cleaning (D-640). Clustered is an absence, + # including when an external writer sends the default by name. + chart.pop("stacking", None) + if row.get("stacking") in ("stacked", "percent"): + chart["stacking"] = row["stacking"] for key in ("geometry", "anchor"): if chart.get(key) is not None: box = _document_object(chart[key], "sheet_chart_" + key) @@ -1174,15 +1281,18 @@ def _clean_base_row_height(raw): return raw -def _clean_row_heights(raw): +def _clean_row_heights(raw, *, lossless=False): if raw is None: return {} out = _document_object(raw, "sheet_row_heights") for row, height in out.items(): if not re.fullmatch(r"[1-9][0-9]*", str(row)) or int(row) > DOCUMENT_MAX_ROWS \ or isinstance(height, bool) or not isinstance(height, (int, float)) \ - or not math.isfinite(height) or height <= 0: + or not math.isfinite(height) or (lossless and height <= 0): raise SheetRefusal("sheet_row_height_invalid", "A row height needs a row number and a positive size.") + #: W62 C1 / R19: a live size is clamped, while imported positive sizes stay exact. + if not lossless: + out[row] = max(18, min(200, height)) return out @@ -1197,6 +1307,38 @@ def _clean_print(raw): return out +#: Wave 60 base kit: `SheetTabView` in `sheet/types.ts`. Display settings, so a bad value is +#: DROPPED rather than refused: a sheet must never fail to save over a zoom level. +_VIEW_ZOOM_MIN, _VIEW_ZOOM_MAX = 10, 400 +_VIEW_PAGE_VIEWS = ("normal", "pageBreak", "pageLayout") + + +def _clean_view(raw): + """A tab's display settings: gridlines, headings, zoom, pageView and split, nothing else. + + Booleans stay booleans, zoom clamps to 10..400 (a non-number is dropped), pageView keeps only + the three words, split keeps non-negative integer rows and cols or is dropped. An empty + result is None, so the tab stores no empty stratum. + """ + if not isinstance(raw, dict): + return None + out = {} + for key in ("gridlines", "headings"): + if isinstance(raw.get(key), bool): + out[key] = raw[key] + zoom = raw.get("zoom") + if not isinstance(zoom, bool) and isinstance(zoom, (int, float)) and math.isfinite(zoom): + out["zoom"] = max(_VIEW_ZOOM_MIN, min(_VIEW_ZOOM_MAX, zoom)) + if raw.get("pageView") in _VIEW_PAGE_VIEWS: + out["pageView"] = raw["pageView"] + split = raw.get("split") + if isinstance(split, dict): + parts = [split.get("rows"), split.get("cols")] + if all(isinstance(n, int) and not isinstance(n, bool) and n >= 0 for n in parts): + out["split"] = {"rows": parts[0], "cols": parts[1]} + return out or None + + def _clean_imported(raw): """C4's import record: when, from what, and WHAT DID NOT SURVIVE. @@ -1267,8 +1409,9 @@ def _clean_sheet(raw, index=0, *, lossless=False): "charts": _clean_charts(raw.get("charts")), "imported": _clean_imported(raw.get("imported")), "rowHeight": _clean_base_row_height(raw.get("rowHeight")), - "rowHeights": _clean_row_heights(raw.get("rowHeights")), + "rowHeights": _clean_row_heights(raw.get("rowHeights"), lossless=lossless), "visibility": visibility, + "view": _clean_view(raw.get("view")), "print": _clean_print(raw.get("print")), "excel": excel, } @@ -1287,7 +1430,7 @@ def _clean_sheet(raw, index=0, *, lossless=False): if value: sheet[key] = value extensions = {key: copy.deepcopy(value) for key, value in raw.items() - if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount"}} + if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount", "writes", "writesFrom"}} sheet.update(_json_safe(extensions, "sheet extensions")) if lossless: sheet["cols"] = max(cols, (sheet.get("freeze") or {}).get("cols", 0)) @@ -1308,7 +1451,7 @@ def _clean_sheets(raw, *, lossless=False): # ── the workbook ───────────────────────────────────────────────────────────────────────────── -def migrate_workbook(raw): +def migrate_workbook(raw, *, lossless=True): """Pure unversioned/v0 -> v1 migration; validate current documents without recalculating. A read never writes or changes `rev`, timestamps or cached results. Unknown versions @@ -1332,7 +1475,7 @@ def migrate_workbook(raw): if book.get("print") is not None: book["print"] = _clean_print(book["print"]) book["schemaVersion"] = SHEET_SCHEMA_VERSION - book["sheets"] = _clean_sheets(raw.get("sheets"), lossless=True) + book["sheets"] = _clean_sheets(raw.get("sheets"), lossless=lossless) return _json_safe(book, "workbook") @@ -1346,10 +1489,11 @@ def new_workbook(name="", created_by="", workbook_id=""): "name": _text(name, _MAX_NAME) or "Untitled workbook", "sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1", "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {}, - "usedExtent": None}], + "usedExtent": None, "writesFrom": 0}], "rev": 1, "refreshed_at": None, "created_by": _text(created_by, _MAX_ID), + "access": "private", "created_at": stamp, "updated_at": stamp, } @@ -1416,8 +1560,45 @@ def _merge_data_origins(doc, *, sheets=(), databases=()): return origins -def create_workbook(st, name="", created_by=""): - """Write a new `sheet_workbook_` document and register it in the index.""" +def create_workbook(st, name="", created_by="", on_duplicate="allow"): + """Write a new `sheet_workbook_` document and register it in the index. + + `on_duplicate` decides what a name already in this workspace means: + `allow` (the default) writes it anyway, `refuse` raises `workbook_name_taken`, and `suffix` + writes `Name (2)`, `Name (3)` instead. + + ⛔ THE DUPLICATE-NAME RULE BELONGS TO THE DOORS, NOT TO THIS MECHANISM (owner ruling + 2026-09-16, shaped by what the gate then measured). A workbook's identity is its id, so + duplicates were always legal and nothing ever checked; the rail shows N identical rows with no + date, size or owner and no reader can tell which is theirs. SDLC 61's C7 build left five + workbooks named `Daily Sales Performance`, which is what made it visible. + ⚠ **Where the rule lives matters.** A person names a workbook at exactly two doors, and both + pass `refuse`: `POST /sheets` (Create new) and the gateway's `sheet_create_workbook`. Every + other caller creates a workbook as a MECHANISM, not as a naming act: applying a template and + importing a file both INHERIT a name, so they pass `suffix` and a template stays usable more + than once. Putting `refuse` on this function instead made a template single use, made an + import single use, and broke thirteen gate fixtures whose names are incidental. A rule that + fires on a mechanism fires on everything that uses it. + """ + wanted = str(name or "").strip() + if wanted and on_duplicate != "allow": + taken = {str((row or {}).get("name") or "").strip().casefold() + for row in list_workbooks(st)} + if wanted.casefold() in taken: + if on_duplicate != "suffix": + raise SheetRefusal( + "workbook_name_taken", + "A workbook named " + wanted + " is already in this workspace. " + "Choose another name.") + for nth in range(2, 1000): + candidate = wanted + " (" + str(nth) + ")" + if candidate.casefold() not in taken: + name = candidate + break + else: + raise SheetRefusal( + "workbook_name_taken", + "Too many workbooks are already named " + wanted + ".") book = new_workbook(name=name, created_by=created_by) _json_safe(book, "workbook") @@ -1429,6 +1610,30 @@ def create_workbook(st, name="", created_by=""): return book +def _carry_write_logs(book, prior): + """Only stored tabs may supply write history; client metadata never owns it.""" + held = {tab.get("id"): tab for tab in prior.get("sheets") or []} + for tab in book.get("sheets") or []: + source = held.get(tab.get("id")) or {} + tab.pop("writes", None) + tab["writesFrom"] = int(source.get("writesFrom") or 0) + if "writes" in source: + tab["writes"] = copy.deepcopy(source["writes"]) + return book + + +def _append_write_log(tab, rev, addrs, *, actor="", actor_name=""): + """Record accepted addresses and the oldest revision with complete retained history.""" + if not addrs: + return + writes = [*(tab.get("writes") or []), {"rev": rev, "by": actor, + "byName": actor_name or actor, "addrs": list(addrs)}] + dropped = writes[:-WRITE_LOG_REVS] + if dropped: + tab["writesFrom"] = max(int(tab.get("writesFrom") or 0), int(dropped[-1]["rev"])) + tab["writes"] = writes[-WRITE_LOG_REVS:] + + def get_workbook(st, workbook_id): """One workbook, or a named refusal when the document is not there.""" wid = _slug_id(workbook_id) @@ -1436,13 +1641,13 @@ def get_workbook(st, workbook_id): if not doc or not doc.get("id"): raise SheetRefusal("workbook_missing", "That workbook no longer exists.") - return migrate_workbook(doc) + return _carry_write_logs(migrate_workbook(doc), doc) -def put_workbook(st, workbook_id, raw, expect_rev=None): +def put_workbook(st, workbook_id, raw, expect_rev=None, *, actor="", actor_name="", lossless=False): """Replace a workbook's name and sheets, bumping `rev`. - ⛔ `id`, `created_by` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by + ⛔ `id`, `created_by`, `access` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by sending a different id, and a client that never read the document cannot blank its provenance. ⚠ `expect_rev` makes the write CONDITIONAL, and when it is not passed explicitly the `rev` @@ -1461,7 +1666,8 @@ def put_workbook(st, workbook_id, raw, expect_rev=None): if expect_rev is False: expect_rev = None - incoming = migrate_workbook(raw) + #: W62 C1 / R19: app and AI saves share bounded defaults; import and refresh opt out. + incoming = migrate_workbook(raw, lossless=lossless) sheets = incoming["sheets"] name = _text(raw.get("name"), _MAX_NAME) refreshed_at = raw.get("refreshed_at") @@ -1472,9 +1678,10 @@ def put_workbook(st, workbook_id, raw, expect_rev=None): if prior is None: raise SheetRefusal("workbook_missing", "That workbook no longer exists.") if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev): - raise SheetRefusal( + raise SheetStaleRefusal( "workbook_revision_stale", - "That workbook changed somewhere else, so this save was not applied.") + "That workbook changed somewhere else, so this save was not applied.", + rev_current=int(prior.get("rev") or 0)) book = migrate_workbook(prior) # Immutable ownership and server read projections cannot be reassigned by a PUT. # ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may @@ -1483,16 +1690,24 @@ def put_workbook(st, workbook_id, raw, expect_rev=None): # ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an # unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned # cells by ``_merge_data_origins`` immediately below. - protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at", + protected = {"id", "created_by", "access", "created_at", "updated_at", "rev", "refreshed_at", "sheets", "name", "refreshedFor", "shares", "role", "permissions", "dataOrigins"} book.update({key: value for key, value in incoming.items() if key not in protected}) book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook" - book["sheets"] = sheets + book["sheets"] = copy.deepcopy(sheets) + _carry_write_logs(book, prior) # DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is # ignored, so a client cannot erase or replace the server-maintained attribution list. _merge_data_origins(book, sheets=sheets) book["rev"] = int(prior.get("rev") or 0) + 1 + prior_tabs = {tab.get("id"): tab for tab in prior.get("sheets") or []} + for tab in book["sheets"]: + before = (prior_tabs.get(tab.get("id")) or {}).get("cells") or {} + after = tab.get("cells") or {} + changed = sorted(addr for addr in before.keys() | after.keys() + if before.get(addr) != after.get(addr)) + _append_write_log(tab, book["rev"], changed, actor=actor, actor_name=actor_name) if refreshed_at is not None: book["refreshed_at"] = _text(refreshed_at, 120) or None book["updated_at"] = _now() @@ -1574,7 +1789,7 @@ def workbook_meta(book): return meta -def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): +def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None, *, actor="", actor_name=""): """Merge a RANGE of cells into ONE tab. Returns `{rev, tab, refused}` (contract C3). ⭐⭐ ONE BAD CELL DOES NOT THROW THE COMMIT AWAY. Every address is cleaned on its own and a @@ -1640,13 +1855,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): if prior is None: raise SheetRefusal("workbook_missing", "That workbook no longer exists.") current = int(prior.get("rev") or 0) - if expect_rev is not None and current != int(expect_rev): - raise SheetStaleRefusal( - "sheet_revision_stale", - "This sheet changed somewhere else while you had it open, so this save was not " - "applied. Reload the tab and try again.", - rev_current=current) - book = migrate_workbook(prior) + book = _carry_write_logs(migrate_workbook(prior), prior) tabs = book.get("sheets") or [] at = _tab_index(tabs, tab_ref) if at is None: @@ -1654,6 +1863,21 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): "tab_missing", "Tab " + (str(tab_ref) or "(unnamed)") + " is not in this workbook.") tab = tabs[at] + writes = tab.get("writes") or [] + touched = list(dict.fromkeys([*merged, *deleted])) + conflicts = {} + if touched and expect_rev is not None and int(expect_rev) < current: + revision = int(expect_rev) + if revision < int(tab.get("writesFrom") or 0): + raise SheetStaleRefusal( + "sheet_revision_stale", + "This tab's retained change history was truncated after your revision. Reload the tab and try again.", + rev_current=current) + for write in writes: + if int(write["rev"]) > revision and write["by"] != actor: + for addr in sorted(set(touched).intersection(write["addrs"])): + conflicts[addr] = {"addr": addr, "by": write["by"], + "byName": write.get("byName") or write["by"]} cells = dict(tab.get("cells") or {}) for key in deleted: cells.pop(key, None) @@ -1670,6 +1894,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): book["sheets"] = tabs _merge_data_origins(book, sheets=tabs) book["rev"] = current + 1 + _append_write_log(tab, book["rev"], touched, actor=actor, actor_name=actor_name) book["updated_at"] = _now() _json_safe(book, "workbook") out["rev"] = book["rev"] @@ -1677,6 +1902,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): out["cells"] = len(cells) out["schemaVersion"] = SHEET_SCHEMA_VERSION out["usedExtent"] = tab["usedExtent"] + out["conflicts"] = list(conflicts.values()) # Server-cleaned keys only. The route consumes this delta for calculation context. out["accepted"] = list(dict.fromkeys([*merged, *deleted])) saved.update(book) @@ -1832,6 +2058,7 @@ def _index_row(book): "id": book.get("id") or "", "name": book.get("name") or "", "created_by": book.get("created_by") or "", + "has_access": "access" in book, "created_at": book.get("created_at") or "", "updated_at": book.get("updated_at") or "", "rev": int(book.get("rev") or 0), @@ -1883,6 +2110,21 @@ def _index_upsert(st, book): #: registry's words, so a second spelling cannot be introduced here by accident. +def narrow_workbook(st, workbook_id): + """Persist explicit sharing's one way visibility decision on the latest document.""" + wid = _slug_id(workbook_id) + + def _apply(state): + if not state.get("id"): + raise SheetRefusal("workbook_missing", "That workbook no longer exists.") + state.setdefault("access", "private") + return state, state + + # R28 / W62-T25: explicit sharing changes the fact used by the index's visibility filter. + book = _persist(st, workbook_key(wid), _apply) + _index_upsert(st, book) + + def workbook_shares(st, workbook_id): """C5's `shares` list for ONE workbook, computed from `core.shares` and never stored. @@ -1901,24 +2143,112 @@ def workbook_shares(st, workbook_id): rec = shares.grants("workbook", str(workbook_id), st=st) except Exception: return [] - return [{"who": e["user"], "role": e["role"]} for e in (rec.get("entries") or [])] + return shares_from_record(rec) + +def shares_from_record(rec): + """C5's `shares` list from a grant record the caller already read. PURE. -def project_shares(st, rows): + W61-T17: `project_shares` reads the whole registry once and calls this per row, so the list + a batched caller builds and the list `workbook_shares` builds are the same list by + construction rather than by two copies of one comprehension. + """ + return [{"who": e["user"], "role": e["role"]} for e in ((rec or {}).get("entries") or [])] + + +def workbook_role(st, book, viewer, *, is_admin=False, rec=None): + """Document ownership plus the existing grant reader, shared with the route's role helper. + + ⭐ W61-T17 — `book` NEED ONLY ANSWER `id` AND `created_by`, and that is what lets the index + stop reading whole workbook documents to build a list. An index row already carries both + (`_index_row` rebuilds them off the document on every save), so the caller may pass the ROW. + ⭐ `rec` is an already-read grant record, from `shares.grants_bulk`. Passing it skips a + registry read; omitting it reads exactly as before. The role logic itself is + `shares.role_from` either way, so the two paths cannot answer differently. + """ + from core import shares + me = str(viewer or "").strip().lower() + if me and str(book.get("created_by") or "").strip().lower() == me: + return "owner" + if rec is not None: + return shares.role_from(rec, me, is_admin=is_admin) + return shares.role_for("workbook", book.get("id"), me, is_admin=is_admin, st=st) + + +def may_read_workbook(book, role): + """R28: a role admits a reader; unmarked or creatorless documents remain legacy-visible. + + R28 / W62-T25: creation and explicit sharing stamp access. The index records its presence; + workbook reads still use the document, and PUT must preserve marker absence too. + Legacy visibility grants no ownership or editing role. + """ + return bool(role) or "access" not in book or not book.get("created_by") + + +def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=None, + grant_records=None): """Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched. - ⚠ Called by `read_index`, `save_folders` and `routes_sheet.list_sheets` — the three doors a - client reads the rail through. `list_workbooks` itself stays a bare pointer list on purpose: + W62-T25: a named viewer filters rows using document facts and receives their role. + The route supplies its existing role helper; core callers use workbook_role directly. + `list_workbooks` itself stays a bare pointer list on purpose: `iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry read per workbook there would put a permission lookup inside a scan that asks no permission question at all. + + ⭐⭐ W61-T17 — IT NO LONGER READS EVERY WORKBOOK DOCUMENT TO BUILD A LIST, and that was the + whole cost of opening the Sheet home. Measured on staging's Royal Imports with 50 workbooks: + 12,485 ms and 269,946,170 characters read, to produce a 12 KB answer. Every one of those + documents was pulled in full — all its tabs, all its cells — so that three fields could be + consulted, and the list got slower with every cell anybody typed. + + ⭐ THE ROW ALREADY ANSWERS TWO OF THE THREE. `_index_row` rebuilds `id` and `created_by` off + the document on every save, and `workbook_role` reads nothing else; so does the route's + injected helper. The third is `may_read_workbook`'s legacy-visibility clause, `"access" not in + book`, and it is consulted ONLY when the viewer has no role at all AND the row names a + creator - the rows that are otherwise about to be REFUSED. W62-T25 records that third fact + as `has_access`; only rows written before that fact existed still read their document. + + R28 / W62-T25: admission uses the document facts rebuilt on save and explicit sharing. + `_index_drop` removes a deleted document's row. Older rows still check document existence + when taking the fallback; opening any workbook always reads its document for real. """ out = [] + bulk = grant_records + if bulk is None and rows: + try: + from core import shares as _shares + bulk = _shares.grants_bulk("workbook", st=st) + except Exception: + bulk = {} + bulk = bulk or {} for row in rows or (): if not isinstance(row, dict) or not row.get("id"): continue joined = dict(row) - joined["shares"] = workbook_shares(st, row["id"]) + # ⛔ NEVER None: an object with NO grants has no record in the bucket, and passing + # None to `workbook_role` would read "caller supplied nothing" and send it back to + # `shares.role_for` for a per-row read of the whole registry — which is one of the + # two costs this change exists to remove. An empty record IS the answer. + rec = bulk.get(str(row["id"])) or {"owner": None, "entries": []} + if viewer: + # The two fields the role question actually asks of a "book". + light = {"id": row["id"], "created_by": row.get("created_by") or ""} + role = (role_on_workbook(light) if role_on_workbook is not None + else workbook_role(st, light, viewer, is_admin=is_admin, rec=rec)) + if not role and light["created_by"]: + if "has_access" in row: + if row["has_access"]: + continue + else: + # R28 / W62-T25: old rows lack this fact, so retain their document check. + # R28: this fallback set is expected to be empty after every workbook + # has been saved once; guessing would change R28's legacy admission. + book = _read(st, workbook_key(row["id"])) + if not book or not book.get("id") or not may_read_workbook(book, role): + continue + joined["role"] = role or "view" + joined["shares"] = shares_from_record(rec) out.append(joined) return out @@ -2374,7 +2704,31 @@ def _index_rev(doc): return 0 -def read_index(st, viewer=None): +def _visible_rail(folders, placed, starred, ids): + """Prune hidden placements, stars and folders occupied only by hidden workbooks. + + Keep visible folders' ancestors and intentionally empty folders. Otherwise creating an + empty folder would immediately hide it from the person who needs to put a workbook in it. + """ + parents = {f["id"]: f.get("parent") for f in folders} + + def ancestors(folder_ids): + found = set() + for fid in folder_ids: + while fid in parents and fid not in found: + found.add(fid) + fid = parents[fid] + return found + + visible = {wid: fid for wid, fid in placed.items() if wid in ids} + occupied = ancestors(placed.values()) + needed = ancestors(set(visible.values()) | (parents.keys() - occupied)) + hidden_only = occupied - needed + return ([f for f in folders if f["id"] not in hidden_only], visible, + _clean_starred(starred, ids, [])) + + +def read_index(st, viewer=None, *, is_admin=False, role_on_workbook=None): """The whole rail document: `{workbooks, folders, itemFolders, starred, rev}`, plus `sharedWithMe` when a `viewer` is named. @@ -2388,8 +2742,8 @@ def read_index(st, viewer=None): so a `rev` here without the check there (or the check without this) is a guard nothing can reach ([[a-rev-guard-inert-on-the-events-channel]]). The pair lands together or not at all. - ⚠ WHICH WRITES MOVE IT IS PART OF THE CONTRACT: `save_folders` and `_index_drop`, which are - the two that rewrite the folder, placement and star strata this guard protects. + D-639 / W62-T25: `save_folders`, `merge_placements` and `_index_drop` move the revision + because they write the folder, placement and star strata this guard protects. `_index_upsert` deliberately does NOT, because it is called by `create_workbook`, `put_workbook` AND `patch_cells` — bumping there would make every cell edit 409 a folder drag that touched nothing the guard is about. @@ -2399,6 +2753,20 @@ def read_index(st, viewer=None): doc = _read(st, INDEX_KEY) rows = [dict(r) for r in (doc.get("workbooks") or []) if isinstance(r, dict) and r.get("id")] + # W62 release repair: both the role join and Shared with me ask the SAME grant-registry + # question. Read it once and hand the normalized records to both consumers; otherwise a + # 50-row index pays one bulk read here, N more through an injected route helper, then one + # final read through `shared_with`. + grant_records = {} + if rows: + try: + from core import shares as _shares + grant_records = _shares.grants_bulk("workbook", st=st) + except Exception: + grant_records = {} + rows = project_shares(st, rows, viewer, is_admin=is_admin, + role_on_workbook=role_on_workbook, + grant_records=grant_records) ids = {r["id"] for r in rows} placements = doc.get("itemFolders") try: @@ -2407,37 +2775,45 @@ def read_index(st, viewer=None): flat = [dict(f, parent=None) for f in (doc.get("folders") or []) if isinstance(f, dict) and f.get("id")] folders, placed, _repairs = validate_folder_tree(flat, placements) + starred = _clean_starred(doc.get("starred"), ids, []) + if viewer: + folders, placed, starred = _visible_rail(folders, placed, doc.get("starred"), ids) out = { - "workbooks": project_shares(st, rows), + "workbooks": rows, "folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids}, - "starred": _clean_starred(doc.get("starred"), ids, []), + "starred": starred, "rev": _index_rev(doc), } if viewer: - # C5's `sharedWithMe`. ⚠ FREE, and that is the point of using the registry: `shared_with` - # already answers "what was granted TO me" across kinds and already EXCLUDES what the - # caller owns, so the "Shared with me" partition of the rail needs no second store and no - # second definition of what being shared means. + # C5's `sharedWithMe`, derived from the SAME normalized grant records as the role join. + # Match `shares.shared_with`: exclude owned objects, admit personal/everyone grants, sort, + # and finally intersect with the already-visible index ids. No second store read. try: import core.shares as shares - granted = (shares.shared_with(viewer, kind="workbook", st=st) or {}).get("workbook") + me = str(viewer or "").strip().lower() + granted = sorted( + oid for oid, rec in grant_records.items() + if str(rec.get("owner") or "").strip().lower() != me + and shares.role_from(rec, me) is not None + ) except Exception: granted = [] out["sharedWithMe"] = [w for w in (granted or []) if w in ids] return out -def save_folders(st, folders, item_folders, starred=None, expect_rev=None): +def save_folders(st, folders, item_folders, starred=None, expect_rev=None, *, + writable_ids=None, writable_folder_ids=None): """Replace the folder, placement and star strata WHOLESALE, once the tree is proven legal. Wholesale rather than per folder because the three are interdependent: a placement is only legal while its folder exists, so committing them separately would leave a window where a reader sees a workbook filed into a folder that is not there yet. One write, one state. - ⛔ VALIDATION RUNS BEFORE THE UPDATE, NEVER INSIDE ITS CALLBACK. A raise from inside the - callback would abandon a half touched transaction; raising first means a refused tree leaves - the store exactly as it was. The callback below only PRUNES, and cannot raise. + ⛔ STRUCTURAL VALIDATION RUNS BEFORE THE UPDATE. A permission-scoped save also reconciles + the submitted projection with hidden stored strata inside the callback, under the store lock; + any refusal still abandons the callback atomically, so no half-touched rail can persist. RETURNS `{workbooks, folders, itemFolders, starred, rev, repairs}`. RAISES `SheetFolderRefusal` on a cycle or an over deep chain, having written nothing. @@ -2455,9 +2831,16 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None): impossible. """ tree, placed, repairs = validate_folder_tree(folders, item_folders) + writable = None if writable_ids is None else {str(wid) for wid in writable_ids} + writable_folders = (None if writable_folder_ids is None + else {str(fid) for fid in writable_folder_ids}) out = {} def _apply(state): + tree_now = [dict(row) for row in tree] + placed_now = dict(placed) + repairs_now = list(repairs) + starred_now = starred if expect_rev is not None and _index_rev(state) != int(expect_rev): raise SheetStaleRefusal( "index_revision_stale", @@ -2467,25 +2850,76 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None): rows = [r for r in (state.get("workbooks") or []) if isinstance(r, dict) and r.get("id")] ids = {r["id"] for r in rows} + saved_tree, saved_placed, _saved_repairs = validate_folder_tree( + state.get("folders"), state.get("itemFolders")) + + if writable is not None: + # An id absent from this caller's projection is indistinguishable from a missing one. + # Refuse explicit guesses, then preserve every hidden placement/star against omission + # from the caller's necessarily partial document. + if any(item_id not in writable for item_id in placed_now): + raise SheetRefusal("workbook_missing", "That workbook no longer exists.") + incoming_stars = (_clean_starred(starred_now, None, []) + if starred_now is not None else []) + if any(item_id not in writable for item_id in incoming_stars): + raise SheetRefusal("workbook_missing", "That workbook no longer exists.") + + saved_by_id = {row["id"]: row for row in saved_tree} + incoming_ids = {row["id"] for row in tree_now} + if writable_folders is not None: + hidden_folder_ids = set(saved_by_id) - writable_folders + if incoming_ids & hidden_folder_ids: + raise SheetRefusal("folder_missing", "That folder no longer exists.") + + hidden_ids = ids - writable + hidden_placed = {wid: fid for wid, fid in saved_placed.items() + if wid in hidden_ids} + parents = {row["id"]: row.get("parent") for row in saved_tree} + required = set() + for fid in hidden_placed.values(): + while fid in parents and fid not in required: + required.add(fid) + fid = parents[fid] + # Missing hidden-only branches are restored from the stored tree. A shared folder the + # caller edited keeps the submitted row; deleting one that still holds a hidden book + # preserves the stored row only for that hidden stratum. + combined_tree = list(tree_now) + # Preserve the stored order of every restored hidden branch. Iterating the set + # difference made an honest partial save reorder another user's hidden folders by + # Python hash seed: the contents survived, but their rail order did not. + combined_tree.extend( + dict(row) for row in saved_tree + if row["id"] in required and row["id"] not in incoming_ids) + combined_placed = {**hidden_placed, **placed_now} + tree_now, placed_now, combined_repairs = validate_folder_tree( + combined_tree, combined_placed) + repairs_now.extend(combined_repairs) + + if starred_now is not None: + hidden_stars = [wid for wid in _clean_starred(state.get("starred"), ids, []) + if wid in hidden_ids] + starred_now = hidden_stars + [wid for wid in incoming_stars if wid in writable] + kept = {} - for item_id, fid in placed.items(): + for item_id, fid in placed_now.items(): if item_id in ids: kept[item_id] = fid else: - repairs.append({"reason": "placement_missing_workbook", "item_id": item_id}) - state["folders"] = tree + repairs_now.append({"reason": "placement_missing_workbook", "item_id": item_id}) + state["folders"] = tree_now state["itemFolders"] = kept - if starred is not None: - state["starred"] = _clean_starred(starred, ids, repairs) + if starred_now is not None: + state["starred"] = _clean_starred(starred_now, ids, repairs_now) # D-639: this write IS the one the guard protects, so it moves the revision. state["rev"] = _index_rev(state) + 1 out["value"] = { "workbooks": [dict(r) for r in rows], - "folders": [dict(f) for f in tree], + "folders": [dict(f) for f in tree_now], "itemFolders": dict(kept), "starred": list(state.get("starred") or []), "rev": _index_rev(state), } + out["repairs"] = repairs_now return state, None _persist(st, INDEX_KEY, _apply, seed={"workbooks": []}) @@ -2495,5 +2929,54 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None): # store, and reading one document from inside another document's update callback puts a second # store call under a lock this one already holds. result["workbooks"] = project_shares(st, result.get("workbooks")) - result["repairs"] = repairs + result["repairs"] = list(out.get("repairs") or repairs) + return result + + +def merge_placements(st, placements, *, starred=None): + """D-639 / W62-T25: apply only named moves to the current rail under the store's lock. + + None unfiles a workbook. Disjoint moves need no expected revision; every accepted merge + advances it so a stale wholesale save still refuses. Stars change only if explicitly supplied. + """ + if not isinstance(placements, dict): + raise SheetRefusal("placements_not_a_map", "Sheet placements must be a map of workbook ids.") + + def _apply(state): + rows = [r for r in (state.get("workbooks") or []) + if isinstance(r, dict) and r.get("id")] + ids = {r["id"] for r in rows} + merged = dict(state.get("itemFolders") or {}) + for wid, fid in placements.items(): + if wid not in ids: + raise SheetRefusal("placement_missing_workbook", + 'The workbook "' + str(wid) + '" no longer exists in the Sheet rail.') + if fid is None: + merged.pop(wid, None) + else: + merged[wid] = fid + # D-639: validate the merged tree inside the same update, before changing any stratum. + # D-639: use wholesale repair names, but refuse a malformed delta instead of losing it. + _tree, placed, repairs = validate_folder_tree(state.get("folders"), merged) + if repairs: + repair = repairs[0] + raise SheetFolderRefusal( + repair["reason"], + 'The placement for workbook "' + str(repair.get("item_id") or "") + + '" cannot be saved. Use an existing folder id or null to unfile it.', + folder_id=str(repair.get("folder_id") or ""), reason=repair["reason"]) + for wid, fid in placements.items(): + if fid is not None: + merged[wid] = placed[wid] + state["itemFolders"] = merged + if starred is not None: + state["starred"] = _clean_starred(starred, ids, repairs) + state["rev"] = _index_rev(state) + 1 + return state, {"workbooks": rows, "folders": state.get("folders") or [], + "itemFolders": merged, "starred": state.get("starred") or [], + "rev": _index_rev(state), "repairs": repairs} + + result = _persist(st, INDEX_KEY, _apply, seed={"workbooks": []}) + # D-639: grant reads happen after the store releases its update lock, as in save_folders. + result["workbooks"] = project_shares(st, result["workbooks"]) return result diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index 2368a3d45df9dbb98d44d8ec55cb4eea21108f47..3f6ff9037465c132ee61fd9353ba017aaf6da960 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -347,6 +347,66 @@ def get_projection(name, drop=(), tenant_slug=None): return _hf._project(data, frozenset(drops)) +def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): + """Several projected documents in one round trip, keyed by their requested store key. + + Private on purpose: the public module interface is the backend-parity surface, while this is + an optimization exposed only by a bound ``PgStore`` handle. Callers must retain a per-key + fallback for stores that do not offer it. + """ + import core.store as _hf # noqa: PLC0415 — projection shape lives with Store + + keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name))) + if not keys: + return {} + drops = tuple(str(d) for d in drop) + with _pool().connection() as con: + if timeout_s is not None: + timeout_ms = max(1, int(float(timeout_s) * 1000)) + # Transaction-local: a pooled connection must not lend this caller's budget to the + # next request. PgStore connections are non-autocommit, so this setting covers the + # following projection and is discarded when the context commits or rolls back. + con.execute("SELECT set_config('statement_timeout', %s, true)", + (str(timeout_ms),)) + rows = con.execute( + _q('SELECT kv.key, CASE ' + "WHEN jsonb_typeof(kv.value) = 'object' THEN " + "COALESCE((SELECT jsonb_object_agg(e.entry_key, " + "CASE WHEN jsonb_typeof(e.entry_value) = 'object' " + 'THEN e.entry_value - %s::text[] ELSE e.entry_value END) ' + 'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), ' + "'{{}}'::jsonb) ELSE kv.value END FROM {tbl} AS kv " + 'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug), + (list(drops), keys)).fetchall() + found = {str(key): value for key, value in rows} + dropped = frozenset(drops) + return { + key: _hf._project( + dict(found[key]) if isinstance(found.get(key), dict) + else (found[key] if key in found else {}), + dropped) + for key in keys + } + + +def _get_many(names, tenant_slug=None): + """Several ordinary documents in one round trip, keyed by requested store key.""" + keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name))) + if not keys: + return {} + with _pool().connection() as con: + rows = con.execute( + _q('SELECT key, value FROM {tbl} WHERE key = ANY(%s::text[])', + 'store_kv', tenant_slug), + (keys,)).fetchall() + found = {str(key): value for key, value in rows} + return { + key: (dict(found[key]) if isinstance(found.get(key), dict) + else (found[key] if key in found else {})) + for key in keys + } + + def exists(name, tenant_slug=None): """True/False — and unlike the HF backend this is NEVER "True on uncertainty". @@ -610,6 +670,15 @@ class PgStore: """ return get_projection(name, drop=drop, tenant_slug=self.tenant_slug) + def get_projections(self, names, drop=(), timeout_s=None): + """Bound batch projection; absent keys keep the single-read empty-document shape.""" + return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug, + timeout_s=timeout_s) + + def get_many(self, names): + """Bound multi-key read used by request-scoped read snapshots.""" + return _get_many(names, tenant_slug=self.tenant_slug) + def _read_strict(self, name): """Interface parity with the HF Store. There, `get` is lenient (swallows a transient failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead diff --git a/platform/core/table_store.py b/platform/core/table_store.py index f70e2130c4cc7e1ae15acffd775f20d81e1f5779..0d1f47d6b62494a6e04642286bf4d0ede7a7f7bf 100644 --- a/platform/core/table_store.py +++ b/platform/core/table_store.py @@ -1331,6 +1331,34 @@ class TableStore: }) return plans + def legacy_view_migration_needed(self, source=None): + """Whether a preloaded workspace document still carries either retired View shape. + + This is the read-only half of :meth:`migrate_legacy_shared_views`. A caller that already + paid for a rows-free workspace projection can avoid a second store read for the common + clean document while still routing every legacy document through the full strict, + write-and-read-back migration below. Malformed legacy data raises here exactly as it does + there; it is never interpreted as clean. + """ + data = self._strict_document() if source is None else source + if not isinstance(data, dict): + raise LegacySharedViewMigrationError( + 'legacy shared-view migration found a non-object workspace document') + has_blank_legacy, _blank = self._blank_legacy_view_bucket(data) + return bool(has_blank_legacy or self._legacy_shared_plans(data) + or self._personal_permission_plans(data)) + + def prepare_view_projection(self, source): + """Migrate a preloaded projection when needed; return whether its caller must refresh. + + Keeping this decision beside the migration prevents read routes from acquiring a second, + partial definition of which retired shapes are write-bearing. + """ + if not self.legacy_view_migration_needed(source): + return False + self.migrate_legacy_shared_views() + return True + def migrate_legacy_shared_views(self): """Convert ``__shared__.views`` to personal source records plus explicit grants. diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 897cc75b80971f01362bbaefb841b26b3b1626e7..a94ea82dfc3be527d10db159ac9b0a7332d9d708 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -218,7 +218,20 @@ def is_connected(table_key, st=None): key = str(table_key or '').strip() if key in _CONNECTED: return True - table = get(key, st) or {} + # ⭐⭐ W62-T17 (D-739) — `all_defs`, NOT `get`, AND THE THREE KEYS BELOW ARE THE WHOLE ARGUMENT. + # This function asks `connected`, `source` and `recordMode`. Every one of them is a DEFINITION + # key; not one of them is a row. `get` reaches `all_tables`, which is the whole tenant + # document — MEASURED at 97,859,052 bytes and 2,274-2,816 ms PER CALL on staging's Royal + # Imports (`AutoWiki/waves/wave62/proto/T17-profile.md`), against 106,779 bytes for the + # rows-free projection that answers identically. D-739 called this "a platform-wide hazard, + # not a templates one: anything that calls it in a loop has this defect", and the loop callers + # were never the only victims — one call was already over the whole 2,000 ms module budget. + # + # ⚠ THE PROJECTION IS SAFE HERE FOR THE SAME REASON IT IS SAFE IN `/nav`: `all_defs` drops + # exactly `rows`, raises loudly (or falls back to the whole read) rather than answering empty, + # and a lent handle already serves whichever document the pass is holding. A caller that lends + # the WHOLE document gets MORE than asked for, which is the one direction this may be wrong in. + table = all_defs(st).get(key) or {} if not table: return False if table.get('connected') is True: @@ -927,7 +940,7 @@ def _slug(label): return (s or 'table')[:40] -def _ag_formula(raw): +def _ag_formula(raw, valid_keys=None): """`aios_grid._clean_formula`, reached the way this module reaches everything one layer up. ⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and @@ -936,12 +949,11 @@ def _ag_formula(raw): cost twice. A third and fourth copy of `import aios_grid as _agX` is how one of them ends up calling a different validator. - ⚠ `valid_keys` IS DELIBERATELY NOT PASSED. See the note at `_clean_field`'s formula branch: - neither door has reliable sibling-field context, and a second, weaker copy of the ref rule is - worse than leaving refs to the read-time behaviour `_clean_formula` already documents. + ⚠ `valid_keys` is forwarded as given, `None` by default. D-540: only `add_field`/`patch_field` + supply it; see the note at `_clean_field`'s formula branch. """ import aios_grid as _agf - return _agf._clean_formula(raw) + return _agf._clean_formula(raw, valid_keys) def _ag_geocode(raw): @@ -2874,9 +2886,13 @@ def note_human_edit(table_key, values, row_id, st=None): return sorted(touched) -def _clean_field(raw, previous=None, recurrence_validator=None): +def _clean_field(raw, previous=None, recurrence_validator=None, valid_keys=None): """One field dict → the stored shape, or None. The single validator for create AND patch, so - a column cannot be typed one way on the way in and another on the way back.""" + a column cannot be typed one way on the way in and another on the way back. + + `valid_keys` (D-540): the table's other column keys, handed in ONLY by the two write doors + that have table context (`add_field`, and `patch_field` when the caller sends a formula). + Every other caller keeps `None`, which is the read-time/list-construction posture.""" prev = previous or {} raw = raw if isinstance(raw, dict) else {} label = ' '.join(str(raw.get('label') or prev.get('label') or '').split())[:80] @@ -3149,21 +3165,21 @@ def _clean_field(raw, previous=None, recurrence_validator=None): # formula column with no expression is a column that can never show a value, so the pairing is # enforced BOTH WAYS and either half alone refuses the field. # - # ⚠ `valid_keys` IS DELIBERATELY OMITTED, and it is the one real asymmetry with the overlay - # door. `grid_events` passes the workspace's own key set, so a `{ref}` naming no column is - # refused at write time there. This function is handed ONE field with no sibling context — - # `add_field`/`patch_field` could supply the table's keys, but `clean_fields` validates a list - # mid-construction where half the referenced columns do not exist yet, and `clean_machine_ - # fields` judges a list that has no table at all. A second, weaker copy of the ref rule in one - # of the three would be the divergence this module keeps paying for. STRUCTURE is checked here - # (charset, length, balanced parens and quotes, well-formed refs); an unresolvable ref blanks - # the CELL client-side, which is `_clean_formula`'s own documented read-time behaviour and the - # same answer a later-deleted column already produces. + # ⛔ D-540 (2026-09-17): `valid_keys` IS PASSED WHEN THE CALLER HAS TABLE CONTEXT, and only + # then. `grid_events` passes the workspace's own key set, and `formulaEngine.ts:: + # validateFormula` refuses a ref outside `knownKeys`, so the `ut_*` write door was the one + # door that stored `{notacolumn}`. `add_field` and `patch_field` (when the caller sends a + # formula) now hand in the table's other column keys and an unknown ref is refused there. + # `clean_fields` (a list mid-construction, where half the referenced columns do not exist + # yet), `clean_machine_fields` (a list with no table at all) and the link reciprocal keep the + # `None` default: STRUCTURE only, and an unresolvable ref blanks the CELL client-side, which + # is `_clean_formula`'s documented read-time behaviour. A patch that does not send a formula + # keeps its stored one unjudged, so deleting a referenced column never vaporises a column. # ⚠ Reached through `_ag_formula` so BOTH doors call one validator through one import. formula_raw = (raw.get('formula') if 'formula' in raw else (prev.get('formula') if prev_type == ftype else None)) if formula_raw is not None: - fx = _ag_formula(formula_raw) + fx = _ag_formula(formula_raw, valid_keys) if fx is None or ftype != 'formula': return None out['formula'] = fx @@ -3780,7 +3796,8 @@ def add_field(table_key, raw, st=None, recurrence_validator=None): have = [f for f in (defn.get('fields') or [])] if len(have) >= MAX_FIELDS: return None - field = _clean_field(raw, recurrence_validator=recurrence_validator) + field = _clean_field(raw, recurrence_validator=recurrence_validator, + valid_keys={str(f.get('key')) for f in have if f.get('key')}) if not field or any(f.get('key') == field['key'] for f in have): return None if not dynamic_link_filter_valid(field, have): @@ -3988,7 +4005,13 @@ def patch_field(table_key, fkey, raw, st=None, recurrence_validator=None): if 'description' in merged and isinstance(fields[idx].get('automation'), dict) \ and fields[idx]['automation'].get('preset') is True: merged.pop('description') - field = _clean_field(merged, fields[idx], recurrence_validator=recurrence_validator) + # D-540: a patch that SENDS a formula is judged against the table's other columns; one that + # does not keeps its stored formula unjudged (a later-deleted ref blanks cells, not the column). + field = _clean_field( + merged, fields[idx], recurrence_validator=recurrence_validator, + valid_keys=({str(f.get('key')) for f in fields + if f.get('key') and str(f.get('key')) != str(fkey)} + if isinstance(raw, dict) and 'formula' in raw else None)) if not field: return None if not dynamic_link_filter_valid(field, fields): @@ -4473,10 +4496,13 @@ def may_edit_database(table_key, viewer, is_admin=False, st=None): #: #: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through #: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the -#: runtime through `__getattr__` — so this can serve a stale value for exactly two keys, for +#: runtime through `__getattr__` — so this can serve a stale value for the two named keys and +#: dynamically named `__shared` overlay buckets, for #: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level #: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS -#: `createdBy`). +#: `createdBy`). Shared overlays are also read-only on this path: `_shared_value_keys` only +#: reads their field keys, never mutates the document. Their first read is lent for one pass; +#: `fresh=True` and write verbs still reach the runtime, never the snapshot. _LENDABLE = (STORE_KEY, 'object_shares') @@ -4487,7 +4513,7 @@ class _Lent: self._runtime, self._docs = runtime, dict(docs or {}) def get(self, name, fresh=False): - if fresh or name not in _LENDABLE: + if fresh or (name not in _LENDABLE and not str(name).endswith('__shared')): return self._runtime.get(name, fresh) if fresh else self._runtime.get(name) if name not in self._docs: self._docs[name] = self._runtime.get(name) # lazily, once, for this pass @@ -4507,7 +4533,7 @@ class _Lent: A re-projection would cost a second copy to convert the first case and change nothing about the second. """ - if name not in _LENDABLE: + if name not in _LENDABLE and not str(name).endswith('__shared'): return self._runtime.get_projection(name, drop=drop) if name not in self._docs: self._docs[name] = self._runtime.get_projection(name, drop=drop) @@ -4518,7 +4544,7 @@ class _Lent: def lend(st=None, **docs): - """A handle that serves `docs` for `_LENDABLE` keys and defers everything else to `st`. + """Lend `_LENDABLE` keys and `__shared` suffix buckets; defer other keys to `st`. `lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and hand the result to every predicate in that pass; see the note above for what makes it safe. diff --git a/platform/core/windows.py b/platform/core/windows.py new file mode 100644 index 0000000000000000000000000000000000000000..f65b63bf8191823723d836baf9e6c09c016cc265 --- /dev/null +++ b/platform/core/windows.py @@ -0,0 +1,492 @@ +"""core/windows.py — the DATE WINDOW vocabulary (CG-7, owner item 2, 2026-07-26). + +A filter condition on a measure reads: + + Where [Sales] [in the last 90 days] [>] [5,000] + +The middle bracket is this module. It turns a window SPEC — a small JSON object a saved view can +persist — into a concrete `(date_from, date_to)` pair of ISO dates. + +WHY IT IS ITS OWN MODULE, AND WHY IT TAKES `today` AS AN ARGUMENT +----------------------------------------------------------------------------------------------- +Two engines must agree on what "this quarter" means: this one (which compiles the SQL) and +`customer-grid/windows.ts` (which renders the label the user reads and the bounds the client +engine would use). If they disagree by one day, the grid shows a number the label denies, and +nothing errors. `aios-web/verify_windows.py` holds them in lock-step over a fixed set of probe +dates, exactly as verify_filter_engine.py does for the filter tree. + +`today` is a PARAMETER, never `date.today()` read inside. Three reasons, all learned: + - a gate cannot compare two engines on "now" — the two runs are milliseconds apart and land on + different sides of midnight roughly once every few thousand runs, which is the worst kind of + flake because it looks like a real divergence + - a window resolved during a request must not shift between the count query and the row query + - the tenant's day boundary is the tenant's, not the server's + +DESIGN NOTES THAT ARE EASY TO GET WRONG +----------------------------------------------------------------------------------------------- + - Bounds are INCLUSIVE at both ends and are DATES, not timestamps. `store_query` casts both + sides to TIMESTAMP and appends 23:59:59 to the upper bound, so a bare date is correct there; + the client compares the 10-char ISO prefix, so a bare date is correct there too. + - "Last N days" INCLUDES today. Airtable's "the past week" means the last 7 days up to and + including now, not the 7 days before yesterday. Off-by-one here silently drops today's + orders from every "recent" filter — invisible until someone asks why a sale they just + entered is missing. + - Week starts MONDAY (ISO 8601). Stated rather than defaulted: Python's weekday() is + Monday=0 and JavaScript's getDay() is Sunday=0, so the two engines disagree unless one of + them is explicitly corrected. That correction is the single most likely divergence in this + file, and the gate probes a Sunday and a Monday for exactly that reason. + - A `custom` window with only one bound is legal and means open-ended on the other side. + - An UNRESOLVABLE window returns None rather than a default. A window that quietly becomes + "all time" would widen a filter while still reporting an authoritative count. +The historical ``harness.windows`` import remains a compatibility re-export. The vocabulary +lives in core because workbook document validation also consumes it and core must never import +upward from harness. +""" +import datetime as _dt +import re +import math + +#: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts; +#: verify_windows.py asserts the two lists are identical, because a kind the client can emit and +#: the server cannot resolve is a filter that silently stops narrowing. +WINDOW_KINDS = ( + "all_time", + "today", + "yesterday", + "this_week", + "last_week", + "this_month", + "last_month", + "this_quarter", + "last_quarter", + "this_year", + "last_year", + "ytd", + "ytd_last_year", + "ltm", + "past_week", + "past_month", + "past_year", + "last_n_days", + "next_n_days", + "custom", +) + +#: Kinds that carry an integer `n`. +N_KINDS = frozenset({"last_n_days", "next_n_days"}) + +#: How each kind reads in a sentence. The UI renders these; they are here so the label and the +#: arithmetic cannot drift apart in the one place a user would never think to check. +WINDOW_LABELS = { + "all_time": "all time", + "today": "today", + "yesterday": "yesterday", + "this_week": "this week", + "last_week": "last week", + "this_month": "this month", + "last_month": "last month", + "this_quarter": "this quarter", + "last_quarter": "last quarter", + "this_year": "this year", + "last_year": "last year", + "ytd": "year to date", + # "last year to date" (LYTD), not "year to date, last year": the longer phrasing CLIPPED to + # "year to date, last ye" in the 146px window select — and this vocabulary is CLOSED, so + # the rule for it is that it stays readable rather than that it fits. Seen in a live + # screenshot of the owner's own comparison; every assertion in that run was green. + "ytd_last_year": "last year to date", + "ltm": "the last 12 months", + "past_week": "the past week", + "past_month": "the past month", + "past_year": "the past year", + "last_n_days": "the last {n} days", + "next_n_days": "the next {n} days", + "custom": "a custom range", +} + +MAX_N = 3650 # ten years; a bound, not a business rule + +# --- the DATE-VALUE anchors (owner item 3, 2026-07-26) ---------------------------------------- +# The second half of a date CONDITION, as distinct from a measure's window: +# +# Where [Last order] [is before] [one month ago] +# ^ op ^ THIS +# +# A window answers "over what period do I sum"; an anchor answers "which single date am I +# comparing against". Both live in this module for one reason: they are the same two-engine +# contract, resolved from the same `today`, and a gate that holds one in lock-step and not the +# other would leave half the sentence free to drift. + +#: Anchor modes, in the order the picker offers them. `exact` is the historical behaviour — a +#: rule with NO mode is an `exact` rule whose value is an ISO date, which is what every view +#: saved before this change carries. +ANCHOR_MODES = ( + "today", + "yesterday", + "one_week_ago", + "one_month_ago", + "n_days_ago", + "exact", +) + +#: Modes that carry NO value. ⚠ This set is load-bearing far outside this module: a rule whose +#: value is blank normally reads as INACTIVE, and an inactive rule is IGNORED — which WIDENS the +#: result under a count nobody would doubt. Every activeness check (`isRuleActive` in TS, +#: `filter_sql.is_rule_active`, and the validator's value handling) has to know these four are +#: active with an empty value. +ANCHOR_VALUE_FREE = frozenset({"today", "yesterday", "one_week_ago", "one_month_ago"}) + +ANCHOR_LABELS = { + "today": "today", + "yesterday": "yesterday", + "one_week_ago": "one week ago", + "one_month_ago": "one month ago", + "n_days_ago": "{n} days ago", + "exact": "an exact date", +} + + +def _iso(d): + return d.isoformat() + + +def _month_start(d): + return d.replace(day=1) + + +def _month_end(d): + return _next_month(d.replace(day=1)) - _dt.timedelta(days=1) + + +def _next_month(d): + return (d.replace(day=28) + _dt.timedelta(days=4)).replace(day=1) + + +def _quarter_start(d): + return _dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1) + + +def _days_in_month(y, m): + return (_dt.date(y + (m == 12), 1 if m == 12 else m + 1, 1) - _dt.timedelta(days=1)).day + + +def _shift_months(d, n): + """`d` moved `n` months, CLAMPING the day to the target month's length. + + Jan 31 back one month is Dec 31, but Mar 31 back one month is Feb 28 (or 29) — there is no + Feb 31 to land on. Both engines must clamp identically or "the past month" differs by up to + three days for a third of the calendar; the gate probes a 31st and a leap day for exactly it. + """ + total = (d.year * 12 + (d.month - 1)) + n + y, m = divmod(total, 12) + m += 1 + return _dt.date(y, m, min(d.day, _days_in_month(y, m))) + + +def _parse_date(v): + """Accept 'YYYY-MM-DD' (and tolerate a longer ISO timestamp by taking its date part).""" + if isinstance(v, _dt.date): + return v + s = str(v or "").strip()[:10] + if not s: + return None + try: + return _dt.date.fromisoformat(s) + except ValueError: + return None + + +# Sheet Ending metadata belongs to the binding period, never to a window spec. +ENDING_KINDS = ("today", "yesterday", "last_week", "last_month", "last_quarter", + "last_year", "fixed", "cell", "n_before_today") +ENDING_LABELS = dict(zip(ENDING_KINDS, ("Today (live)", "Yesterday", "End of last week", + "End of last month", "End of last quarter", "End of last year", "A fixed date", "From a cell", + "N periods before today"))) +TO_DATE_KINDS = ("ytd", "ytd_last_year", "this_month", "this_quarter") +ENDING_STEPS = ("day", "week", "month", "quarter", "year") + + +def ending_step_default(kind): + return "quarter" if kind == "this_quarter" else "month" + + +def ending_date(value): + """A real ISO day only. No truncation, calendar rollover, or blank fallback.""" + if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value.strip()): + return None + d = _parse_date(value) + return _iso(d) if d else None + + +def resolve_ending(mode, value, today, step="month", offset=0, n=0, unit="month"): + """Inclusive ending date; cell values are supplied by the caller's trusted read.""" + d = _parse_date(today) + if d is None: + raise ValueError("Ending needs a valid today date") + if mode not in ENDING_KINDS or step not in ENDING_STEPS: + return None + if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0: + return None + if mode in ("fixed", "cell"): + parsed = ending_date(value) + if parsed is None: + return None + end = _parse_date(parsed) + elif mode == "n_before_today": + if (isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= MAX_N + or unit not in ENDING_STEPS): + return None + try: + if unit in ("day", "week"): + end = d - _dt.timedelta(days=n * (7 if unit == "week" else 1)) + else: + end = _shift_months(d, -n * {"month": 1, "quarter": 3, "year": 12}[unit]) + except (ValueError, OverflowError): + return None + elif mode == "today": + end = d + elif mode == "yesterday": + end = d - _dt.timedelta(days=1) + elif mode == "last_week": + end = d - _dt.timedelta(days=d.weekday() + 1) + elif mode == "last_month": + end = _month_start(d) - _dt.timedelta(days=1) + elif mode == "last_quarter": + end = _quarter_start(d) - _dt.timedelta(days=1) + else: + end = _dt.date(d.year - 1, 12, 31) + try: + if step in ("day", "week"): + end += _dt.timedelta(days=offset * (7 if step == "week" else 1)) + else: + end = _shift_months(end, offset * {"month": 1, "quarter": 3, "year": 12}[step]) + return _iso(end) + except (ValueError, OverflowError): + return None + + +def ending_cell_date(value, date_system="1900"): + """Sheet dates may be ISO days or Excel serials in the workbook's declared epoch.""" + if not isinstance(value, (int, float)) or isinstance(value, bool): + return ending_date(value) + if not math.isfinite(value) or value < (0 if date_system == "1904" else 1): + return None + days = math.floor(value) + if date_system != "1904" and days == 60: + return None # Excel's fictional February 29, 1900 is not a calendar date. + epoch = (_dt.date(1904, 1, 1) if date_system == "1904" else + _dt.date(1899, 12, 31) if days < 60 else _dt.date(1899, 12, 30)) + try: + return _iso(epoch + _dt.timedelta(days=days)) + except (ValueError, OverflowError): + return None + + +def ending_bounds(kind, mode, value, today, step="month", offset=0, n=0, unit="month"): + """Keep the kind's own start. An ending before it refuses instead of reversing bounds.""" + if kind not in TO_DATE_KINDS: + return None + end = resolve_ending(mode, value, today, step, offset, n, unit) + if end is None: + return None + start = resolve({"kind": kind}, today)[0] + return (start, end) if start <= end else None + + +def normalize(spec): + """Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None. + + Fail-closed on the KIND (an unknown kind is not a window), tolerant on the rest — the same + split `clean_filter_tree` uses, so one malformed window cannot cost a user their saved view. + """ + if not isinstance(spec, dict): + return None + kind = spec.get("kind") + if kind not in WINDOW_KINDS: + return None + out = {"kind": kind} + if kind in N_KINDS: + try: + n = int(spec.get("n")) + except (TypeError, ValueError): + return None + if n < 1 or n > MAX_N: + return None + out["n"] = n + if kind == "custom": + f, t = _parse_date(spec.get("from")), _parse_date(spec.get("to")) + if f is None and t is None: + return None # a custom range with no bounds is not a window + if f is not None and t is not None and f > t: + f, t = t, f # the builder cannot enforce order; the engine can + if f is not None: + out["from"] = _iso(f) + if t is not None: + out["to"] = _iso(t) + return out + + +def resolve(spec, today): + """Window spec + the tenant's today -> `(date_from, date_to)`, both inclusive ISO dates. + + Either side may be None, meaning open-ended. Returns None when the spec is not a window at + all — callers MUST treat that as "this condition cannot be evaluated" and refuse, never as + "no window", which would silently widen the result to all time. + """ + spec = normalize(spec) + if spec is None: + return None + kind = spec["kind"] + d = _parse_date(today) + if d is None: + raise ValueError("resolve() needs an explicit `today`; see this module's docstring") + + if kind == "all_time": + return (None, None) + if kind == "today": + return (_iso(d), _iso(d)) + if kind == "yesterday": + y = d - _dt.timedelta(days=1) + return (_iso(y), _iso(y)) + + # ISO 8601: the week starts MONDAY. Python's weekday() is already Monday=0; the TS mirror + # has to correct getDay(), which is Sunday=0. The gate probes both a Sunday and a Monday. + if kind == "this_week": + start = d - _dt.timedelta(days=d.weekday()) + return (_iso(start), _iso(start + _dt.timedelta(days=6))) + if kind == "last_week": + start = d - _dt.timedelta(days=d.weekday() + 7) + return (_iso(start), _iso(start + _dt.timedelta(days=6))) + + if kind == "this_month": + return (_iso(_month_start(d)), _iso(_month_end(d))) + if kind == "last_month": + prev = _month_start(d) - _dt.timedelta(days=1) + return (_iso(_month_start(prev)), _iso(prev)) + + if kind == "this_quarter": + qs = _quarter_start(d) + qe = _month_end(_dt.date(qs.year, qs.month + 2, 1)) + return (_iso(qs), _iso(qe)) + if kind == "last_quarter": + prev_end = _quarter_start(d) - _dt.timedelta(days=1) + qs = _quarter_start(prev_end) + return (_iso(qs), _iso(prev_end)) + + if kind == "this_year": + return (_iso(_dt.date(d.year, 1, 1)), _iso(_dt.date(d.year, 12, 31))) + if kind == "last_year": + return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_dt.date(d.year - 1, 12, 31))) + if kind == "ytd": + # year-to-date ENDS TODAY, unlike this_year which runs to Dec 31. The distinction is the + # whole reason both exist: comparing "this year" against last year double-counts the + # months that have not happened yet. + return (_iso(_dt.date(d.year, 1, 1)), _iso(d)) + if kind == "ytd_last_year": + # SAME PERIOD last year — Jan 1 LY through today's month/day LY. This is the honest + # partner of `ytd`, and it is not a nicety: it mirrors `core.periods.ytd_last_year` + # EXACTLY (including the Feb 29 -> Feb 28 clamp), which is what `pool()` computes the + # `revenue_ly` column from. `Sales[ytd] < Sales[ytd_last_year]` therefore reproduces the + # retired `at_risk > 0` condition rather than approximating it. Comparing `ytd` against + # `last_year` instead would pit seven months against twelve. + return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_shift_months(d, -12))) + if kind == "ltm": + # the last twelve months INCLUDING today: 365 days back, both ends inclusive + return (_iso(d - _dt.timedelta(days=364)), _iso(d)) + + # "The past week/month/year" are ROLLING and end TODAY — Airtable's wording, and NOT the + # calendar kinds above: `last_week` is the previous Monday-Sunday, `past_week` is the seven + # days ending today. Both are offered because a user means different things by them, and + # picking one to serve both would silently answer the other question. + # past_week = the same rule as last_n_days(7) — one period back, PLUS ONE DAY, so today + # is included and the span is exactly 7 days rather than 8. + # past_year agrees with `ltm` except across a leap day, where the calendar shift keeps the + # same month/day and the 364-day subtraction cannot. + if kind == "past_week": + return (_iso(d - _dt.timedelta(days=6)), _iso(d)) + if kind == "past_month": + return (_iso(_shift_months(d, -1) + _dt.timedelta(days=1)), _iso(d)) + if kind == "past_year": + return (_iso(_shift_months(d, -12) + _dt.timedelta(days=1)), _iso(d)) + + if kind == "last_n_days": + # INCLUSIVE of today — "the last 7 days" is today and the 6 before it, not the 7 before + # yesterday. Getting this wrong drops today's orders from every recent-activity filter. + return (_iso(d - _dt.timedelta(days=spec["n"] - 1)), _iso(d)) + if kind == "next_n_days": + return (_iso(d), _iso(d + _dt.timedelta(days=spec["n"] - 1))) + + if kind == "custom": + return (spec.get("from"), spec.get("to")) + raise AssertionError(f"unhandled window kind {kind!r}") # unreachable; WINDOW_KINDS is closed + + +def label(spec): + """How a window reads in the condition sentence ('in the last 90 days').""" + spec = normalize(spec) + if spec is None: + return "an invalid range" + return WINDOW_LABELS[spec["kind"]].format(n=spec.get("n")) + + +def resolve_anchor(mode, value, today): + """A date condition's right-hand side -> ONE inclusive ISO date, or None. + + `mode` is an entry of ANCHOR_MODES; None/'' means `exact`, which is what every view saved + before anchors existed carries (op + an ISO date in `value`). `value` supplies the number for + `n_days_ago` and the date for `exact`, and is ignored by the four value-free modes. + + Returns None when the anchor cannot be resolved — an unknown mode, a non-numeric N, a value + that is not a date. The caller must then treat the CONDITION as unanswerable and match + nothing. It must NOT fall through to "no condition": that widens the result while the count + beside it still looks authoritative, which is the whole reason this returns None rather than + a best guess (same contract as `resolve()` above). + + `n_days_ago` accepts n = 0 where a WINDOW requires n >= 1. A zero-day window is empty and + could only be a mistake; "0 days ago" is today, which is a date a person can mean. + """ + m = mode or "exact" + if m not in ANCHOR_MODES: + return None + d = _parse_date(today) + if d is None: + raise ValueError("resolve_anchor() needs an explicit `today`; see this module's " + "docstring on why the clock is never read here") + if m == "today": + return _iso(d) + if m == "yesterday": + return _iso(d - _dt.timedelta(days=1)) + if m == "one_week_ago": + # A week ago is a DATE (today minus 7), not the past-week RANGE. The two read almost + # identically in English and mean different things in a comparison. + return _iso(d - _dt.timedelta(days=7)) + if m == "one_month_ago": + return _iso(_shift_months(d, -1)) + if m == "n_days_ago": + s = str("" if value is None else value).strip() + # `[0-9]` not `\d`: a Python \d matches every Unicode decimal digit, so a fullwidth + # '30' would resolve here and be rejected by the TS mirror's Number() — the exact + # two-engine divergence filter_sql.to_num was rewritten to avoid. + if not s or not all("0" <= ch <= "9" for ch in s): + return None + n = int(s) + # No `n < 0` guard: the digit test above already excludes a sign, so it would be + # unreachable. The negative control proved that — removing it changed nothing, which is + # how dead code hides in a defensive-looking line. The CAP is the live guard. + if n > MAX_N: + return None + return _iso(d - _dt.timedelta(days=n)) + parsed = _parse_date(value) # exact + return None if parsed is None else _iso(parsed) + + +def _anchor_label(mode, value): + """How an anchor reads in the condition sentence ('one month ago', '30 days ago').""" + m = mode or "exact" + if m not in ANCHOR_MODES: + return "an invalid date" + if m == "n_days_ago": + return ANCHOR_LABELS[m].format(n=str("" if value is None else value).strip() or "N") + if m == "exact": + return str("" if value is None else value).strip() or ANCHOR_LABELS[m] + return ANCHOR_LABELS[m] diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index 895ecb0265731657da7d212c8384c18481b30b15..ca274b336b7f9e8a162d0a32df236aff23c29506 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -245,8 +245,10 @@ REPORT_TYPES = { # reads whatever V2 actually sends and nothing here needs to know the column list -- which is # exactly the property that makes swapping the report type a one-line change. # · https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement + # Amazon creates settlements on its own schedule and refuses createReport for them. + # This type is a getReports discovery filter, never a requestable report flavor. "settlements": { - "report": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2", + "report_discover": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2", "validates_against": "settlement summary vs settlement lines", "role": "Finance and Accounting", }, @@ -324,6 +326,13 @@ OPERATIONS = { "name": "createReport", "roles": _BULK_REPORT_ROLES, "method": "POST", "path": "reports/2021-06-30/reports", "report_types": _APPROVED_REPORT_TYPES, }, + # Discovery is approved only for Amazon-issued settlements and their finance role. + # Sharing the Reports path grants neither the other bulk types nor their roles. + "getReports": { + "name": "getReports", "roles": ("Finance and Accounting",), "method": "GET", + "path": "reports/2021-06-30/reports", + "report_types": (REPORT_TYPES["settlements"]["report_discover"],), + }, "getReport": { "name": "getReport", "roles": _BULK_REPORT_ROLES, "method": "GET", "path": "reports/2021-06-30/reports/{reportId}", @@ -967,6 +976,69 @@ def report_document(fields, report_type, start=None, end=None, options=None, "compression": meta.get("compressionAlgorithm")} +SETTLEMENT_REPORT_PAGE_CAP = 1000 + + +def settlement_reports(fields, *, since=None, timeout=_TIMEOUT, log=None): + """List settlements Amazon has already created, oldest first. + + Amazon issues these on its own fortnightly cycle, so this lists rather than requests. + Only safe, named metadata leaves this function; vendor extras and tokens stay here. + """ + require_operation("GET", _REPORTS_CREATE_PATH, + report_type=REPORT_TYPES["settlements"]["report_discover"]) + import re + + rtype = REPORT_TYPES["settlements"]["report_discover"] + token = access_token(fields, timeout=timeout) + region = str((fields or {}).get("region") or DEFAULT_REGION) + params = {"reportTypes": rtype, "processingStatuses": "DONE", "pageSize": 100} + if since is not None: + params["createdSince"] = str(since) + reports = [] + for _page in range(SETTLEMENT_REPORT_PAGE_CAP): + # _request owns the same _retry policy as the existing report/document reads. + got = _request(_REPORTS_CREATE_PATH, token, region=region, timeout=timeout, + sandbox=is_sandbox(fields), **params) + for raw in got.get("reports") or (): + row = {key: str(raw[key]) if raw.get(key) is not None else "" for key in ( + "reportId", "reportDocumentId", "reportType", "dataStartTime", + "dataEndTime", "createdTime")} + # Match the API status door without importing upward into the API layer. + # Refuse unsafe values with a fixed sentence that cannot echo the credential. + if any(len(value) > 512 or "Atzr|" in value + or re.search(r"\bbearer\s+\S+", value, flags=re.IGNORECASE) + or any(ch in value for ch in ("\u2014", "\u2013")) + for value in row.values()): + raise AmazonError("Settlement discovery returned unsafe metadata; no status was emitted.") + if row["reportType"] != rtype or not row["reportDocumentId"]: + raise AmazonError("Settlement discovery returned an unexpected type or no document ID.") + reports.append(row) + next_token = got.get("nextToken") + if not next_token: + if log: + log(f"getReports discovered {len(reports)} settlement reports") + return sorted(reports, key=lambda row: (row["createdTime"], row["reportId"])) + # Amazon continuation tokens replace the original filters, which the token carries. + params = {"nextToken": str(next_token)} + raise AmazonError( + f"Settlement discovery hit the {SETTLEMENT_REPORT_PAGE_CAP}-page cap because Amazon kept " + "returning nextToken; retry with a narrower createdSince window.") + + +def settlement_document(fields, document_id, *, timeout=_TIMEOUT): + """One settlement document's text through getReportDocument and the presigned URL.""" + require_operation("GET", f"reports/2021-06-30/documents/{document_id}") + token = access_token(fields, timeout=timeout) + region = str((fields or {}).get("region") or DEFAULT_REGION) + meta = _request(f"reports/2021-06-30/documents/{document_id}", token, region=region, + timeout=timeout, sandbox=is_sandbox(fields)) + url = str(meta.get("url") or "") + if not url: + raise AmazonError("The settlement document carried no download URL.") + return _download_document(url, str(meta.get("compressionAlgorithm") or "")) + + def _download_document(url, compression="", timeout=120): """Fetch one presigned report document and decode it. -> text. diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index 775cc8e841a70fe41fcba74d981ac9d13fba7353..d20c9c2d4f4f5f49b07a0126c9ca00467eeb3cad 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -70,6 +70,9 @@ PREFIX = "amzn_" TABLES = ("orders", "order_items", "listings", "settlements", "inventory", "pricing", "brand_analytics") +#: Head -> lines: both tables read the same document; a second report buys no new rows. +SHARED_REPORT_SIBLINGS = {"orders": "order_items"} + # C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity. ENABLEMENT_KEY = "amazon_connector_enablement" # Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing. @@ -149,8 +152,20 @@ def entity_key(table): #: How far back a first sync reaches when the caller names no window. Amazon's flat-file order #: reports accept at most ~31 days per request, so a backfill is sliced (`_windows`). DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730) +#: Per-table marker whose detail is the ISO UTC end of the last successfully landed window. +WATERMARK_PREFIX = "sync_mark_" #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one. MAX_WINDOW_DAYS = 30 +#: R18: the most recent month lands before any history, so a newly connected account has usable +#: rows in minutes instead of hours. One window, because Amazon's own ceiling is 30 days. +FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS +#: F4: Amazon paces createReport (burst 15, one per 60 s, 5 s gap), so 7 tables x 25 windows is +#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident +#: thread for hours and the Odoo resync behind it would never run. +FIRST_LOAD_REPORTS_PER_PASS = 15 +BACKFILL_FLOOR_PREFIX = "sync_floor_" +FIRST_LOAD_END_KEY = "sync_first_load_end" +FIRST_LOAD_STATUS_KEY = "sync_first_load_status" # Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores; # JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata. @@ -447,7 +462,7 @@ def _cell(value, kind): return None -def _ensure(con, table, cols, rows=None, declared=None): +def _ensure(con, table, cols, rows=None, declared=None, *, preserve_rows=False): """Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched. ⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data @@ -479,7 +494,15 @@ def _ensure(con, table, cols, rows=None, declared=None): clash = [c for c, ty in want.items() if c in have and have[c] != ty and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")] - if clash: + if clash and preserve_rows: + # Settlement document markers preclude replay: widening must retain prior lines. + # The measured types are VARCHAR, BIGINT and DOUBLE; never narrow historical data. + for c in clash: + widened = "VARCHAR" if "VARCHAR" in (have[c], want[c]) else "DOUBLE" + if have[c] != widened: + con.execute(f'ALTER TABLE "{table}" ALTER COLUMN "{c}" TYPE {widened}') + have[c] = want[c] = widened + elif clash: con.execute(f"DROP TABLE {table}") have = {} if not have: @@ -508,6 +531,47 @@ def _replace(con, table, cols, types, rows): return len(rows) +def replace_keys_for(table): + """Replace on declared identity excluding lineage, which changes on every fetch.""" + return tuple(c for c in (amazon_schema.TABLE_SPECS.get(table) or {}).get("row_identity", ()) + if c not in _ORIGIN_TYPES) + + +def replace_window(con, table, cols, types, rows, keys): + """Replace incoming key groups atomically, or report why only insertion is possible.""" + missing = [c for c in keys if c not in cols] + if missing: + note = (f"{table}: window replacement unavailable; missing identity columns: " + f"{', '.join(missing)}; rows inserted; use a report carrying these columns") + return _replace(con, table, cols, types, rows), note + if not keys or not rows: + return _replace(con, table, cols, types, rows), None + quote = lambda name: '"' + str(name).replace('"', '""') + '"' + target = quote(table) + temporary = quote("amazon_window_keys_" + uuid.uuid4().hex) + columns = ", ".join(quote(c) for c in keys) + # Use the mirror's types and the insert's cell conversion so NULL and numeric keys agree. + incoming = list(dict.fromkeys(tuple(_cell(row.get(c), types.get(c, "VARCHAR")) + for c in keys) for row in rows)) + con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE") + try: + con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})", + incoming) + predicate = " AND ".join( + f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys) + con.execute("BEGIN TRANSACTION") + try: + con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})') + count = _replace(con, table, cols, types, rows) + con.execute("COMMIT") + except Exception: + con.execute("ROLLBACK") + raise + return count, None + finally: + con.execute(f"DROP TABLE {temporary}") + + def _windows(start, end, size=MAX_WINDOW_DAYS): """Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden.""" out, cur = [], start @@ -518,6 +582,49 @@ def _windows(start, end, size=MAX_WINDOW_DAYS): return out +def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS): + """R18's request order: every recent month, then each table's history, oldest last. + + Settlements have one discovery slot (None, None), never a createReport window. + This planner is pure; durable floors filter its fixed windows when a pass resumes. + """ + start, end = _utc(start), _utc(end) + if start >= end: + return [] + size = min(float(size), MAX_WINDOW_DAYS) + recent_days = FIRST_LOAD_RECENT_DAYS if recent_days is None else recent_days + recent_days = min(float(recent_days), size) + if size <= 0 or recent_days <= 0: + raise ValueError("Amazon report windows must have a positive size.") + recent_start = max(start, end - timedelta(days=recent_days)) + tables = list(dict.fromkeys(tables)) + recent = [(table, None, None) if table == "settlements" else (table, recent_start, end) + for table in tables] + history = [(table, w0, w1) for table in tables if table != "settlements" + for w0, w1 in reversed(_windows(start, recent_start, size))] + return recent + history + + +def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None): + """Amazon createReport pacing floor in seconds; polling and other work can add time.""" + from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS + if report_count <= 0: + return 0 + stagger = REPORT_STAGGER_SECONDS if stagger_seconds is None else stagger_seconds + tokens = REPORT_BURST if burst is None else burst + rate = 60 / REPORT_SUSTAINED_SECONDS if per_minute is None else per_minute + seconds = max(0, (report_count - 1) * stagger, (report_count - tokens) * 60 / rate) + return int(seconds) + (seconds > int(seconds)) + + +def first_load_eta_cause(report_count): + """Screen-safe explanation of the vendor ceiling and the automatic continuation.""" + from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS + return (f"Amazon's report rate allows a burst of {REPORT_BURST}, then one report every " + f"{REPORT_SUSTAINED_SECONDS:g} seconds. Keep the connection enabled while history " + "loads automatically. Report processing and other syncs can add time.") + + def _iso(dt): return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1071,6 +1178,94 @@ def _read_state(con, key): return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]} +def settlement_is_new(con, document_id): + """A completed document is durable even when it contained no mirror lines.""" + state = _read_state(con, entity_key("settlement_doc_" + str(document_id))) + return not state or state["state"] != "done" + + +def read_watermark(con, table): + """Missing or malformed detail means no successful window, never the current time.""" + state = _read_state(con, entity_key(WATERMARK_PREFIX + table)) + detail = (state or {}).get("detail") + if not detail: + return None + try: + return _utc(detail) + except (ValueError, TypeError, AttributeError, OverflowError): + return None + + +def record_watermark(con, table, when, rows=0): + """Persist only after this table's pass succeeds.""" + _record_state(con, entity_key(WATERMARK_PREFIX + table), "done", rows, _iso(_utc(when))) + + +def read_backfill_floor(con, table): + """Oldest contiguous landed window; missing or corrupt markers restart safely.""" + state = _read_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table)) + try: + return _utc(state["detail"]) if state and state.get("detail") else None + except (ValueError, TypeError, AttributeError, OverflowError): + return None + + +def record_backfill_floor(con, table, when): + _record_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table), "done", detail=_iso(_utc(when))) + + +def needs_first_load(tenant_key, rt=None): + """A fresh account or an unfinished durable load uses bounded passes, even after one table finishes.""" + con, _path = _connect_tenant_mirror(tenant_key, rt) + try: + load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) + return bool(load and load["state"] != "done") or not any( + read_watermark(con, table) for table in TABLES) + finally: + con.close() + + +def first_load_status(tenant_key, rt=None): + """Durable numeric progress for the API's safe-status projection; no vendor payloads.""" + con, _path = _connect_tenant_mirror(tenant_key, rt) + try: + state = _read_state(con, entity_key(FIRST_LOAD_STATUS_KEY)) + if not state: + return None + values = json.loads(state["detail"]) + return {"state": state["state"], "remaining": int(values["remaining"]), + "etaSeconds": int(values["etaSeconds"]), "backfillDays": DEFAULT_BACKFILL_DAYS, + "cause": first_load_eta_cause(values["remaining"])} + finally: + con.close() + + +def sync_window(table, mark, *, now, since=None, backfill_days=None): + """Pure request decision for one table: start, end, report, incremental and cause.""" + end = _utc(now) + days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days + try: + mark = _utc(mark) if mark else None + except (ValueError, TypeError, AttributeError, OverflowError): + mark = None + if since is not None: + start, incremental = _utc(since), False + cause = f"caller named the window starting {_iso(start)}" + elif mark is not None: + start, incremental = mark, True + cause = f"incremental window since watermark {_iso(mark)}" + else: + start, incremental = end - timedelta(days=days), False + cause = f"first full load: {days} days of backfill" + spec = _client().REPORT_TYPES.get(table) or {} + rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report") + if table == "settlements": + # The watermark filters issued reports by creation time; it never requests one. + rtype = spec.get("report_discover") + return {"start": start, "end": end, "report": rtype, + "incremental": incremental, "cause": cause} + + def _sync_event(kind, tenant, table, detail, rt, report, log): """Sync is nondestructive: preserve C2's loud failure, but continue the data pull.""" try: @@ -1108,7 +1303,7 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log): def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, - max_wait=900, *, scheduled=False): + max_wait=900, *, scheduled=False, first_load=False, report_budget=None): """Pull every declared table into the tenant's mirror. -> a report dict. Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the @@ -1142,7 +1337,6 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None want = [t for t in (TABLES if tables is None else tables) if t in TABLES] end = datetime.now(timezone.utc) - start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS)) report = {"tenant": tenant_key, "tables": {}, "started": _iso(end), "problems": [], "audit_problems": []} enablement = read_database_enablement(rt) @@ -1171,6 +1365,16 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None if obligation["earliestTriggerAt"]: con.close() raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data") + load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) + if scheduled and load and load["state"] != "done": + # The pinned inline producer still calls sync(scheduled=True). Do not let it start an + # unbounded history pull between the resident writer's bounded first-load passes. + for table in want: + report["tables"][table] = {"state": "deferred", "reason": + "The first load is continuing in bounded passes on the resident sync thread."} + report["finished"] = _iso(datetime.now(timezone.utc)) + con.close() + return report if scheduled: plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}, want, now=end) @@ -1179,9 +1383,60 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None report["tables"][table] = {"state": "deferred", "reason": reason} log(f"{table}: DEFERRED: {reason}") - for table in want: - spec = cx.REPORT_TYPES.get(table) or {} - rtype = spec.get("report") + # Heads go first even when the caller listed lines first. Scheduled plans still select one. + pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items() + if head in want and sibling in want] + want)) + handled, noted = set(), set() + + def write_window(table, cols, types, rows): + count, note = replace_window(con, table_name(table), cols, types, rows, + replace_keys_for(table)) + if note and table not in noted: + noted.add(table) + report["problems"].append(note) + log(note) + return count + + work, plans, paired, contexts = [], {}, {}, {} + documents, settlement_remaining = 0, 0 + failed, landed = set(), set() + if first_load: + # Freeze the cohort end across passes. Recomputing it would move every window boundary. + end = _utc(load["detail"]) if load else _utc(_iso(end)) + _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end)) + plans = {table: sync_window(table, None, now=end) for table in pending} + floors = {table: read_backfill_floor(con, table) for table in pending} + wanted = [table for table in pending if not read_watermark(con, table) + or read_watermark(con, table) < end] + for task in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end): + table, w0, w1 = task + if w1 is None or floors[table] is None or w1 <= floors[table]: + work.append(task) + # A failed enrichment after the final report retries completion without fetching again. + for table in wanted: + if not any(task[0] == table for task in work): + work.append((table, None, None)) + # Reuse a document for head and lines only when BOTH durable cursors agree. + for head, sibling in SHARED_REPORT_SIBLINGS.items(): + head_tasks = [(w0, w1) for table, w0, w1 in work if table == head] + line_tasks = [(w0, w1) for table, w0, w1 in work if table == sibling] + if head_tasks and head_tasks == line_tasks: + paired[head] = sibling + work = [task for task in work if task[0] != sibling] + else: + work = [(table, None, None) for table in pending] + + for index, (table, task_start, task_end) in enumerate(work): + if first_load and table in failed: + continue + if (first_load and isinstance(report_budget, int) and documents >= max(0, report_budget) + and (task_end is not None or table == "settlements")): + break + if table in handled: + continue + plan = plans[table] if first_load else sync_window( + table, read_watermark(con, table), now=end, since=since) + rtype = plan["report"] key, mirror = entity_key(table), table_name(table) if not rtype: report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"} @@ -1189,15 +1444,89 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None continue prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync") root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1] + log(f"{table}: {plan['cause']}") + sibling = SHARED_REPORT_SIBLINGS.get(table) + sibling_plan = None + if first_load: + sibling = paired.get(table) + sibling_plan = plans.get(sibling) + elif sibling in want and sibling not in handled: + sibling_plan = sync_window(sibling, read_watermark(con, sibling), now=end, since=since) + differs = [part for part in ("start", "end", "report") + if plan[part] != sibling_plan[part]] + if differs: + log(f"{table} and {sibling}: cannot share this pass; differing {', '.join(differs)}") + sibling = None + else: + handled.add(sibling) + log(f"{sibling}: {sibling_plan['cause']}") + else: + sibling = None + operation = "report_document" try: # Attempt first, so a refusal or process interruption cannot starve the other tables. - _record_state(con, entity_key("sync_tick_" + table), "attempted") + if not first_load: + _record_state(con, entity_key("sync_tick_" + table), "attempted") + if sibling and not first_load: + _record_state(con, entity_key("sync_tick_" + sibling), "attempted") + sibling_written, sibling_cleared = 0, False written, cols, cleared = 0, [], False pricing_asins = [] - for w0, w1 in _windows(start, end): + operations = {} + if first_load: + cleared = floors[table] is not None + sibling_cleared = bool(sibling and floors[sibling] is not None) + pricing_asins = contexts.setdefault(table, []) + if table == "settlements": + operation = "getReports" + _sync_event("amazon.sync", tenant_key, table, + {"state": "access", "operation": operation}, rt, report, log) + discovered = cx.settlement_reports( + fields, since=_iso(plan["start"]) if plan["incremental"] else None, + log=lambda message: log(f"settlements: {message}")) + downloaded, skipped = 0, 0 + for issued_index, issued in enumerate(discovered): + document_id = str(issued.get("reportDocumentId") or "") + if not document_id: + raise cx.AmazonError("Settlement discovery returned no document ID.") + if not settlement_is_new(con, document_id): + skipped += 1 + continue + if first_load and isinstance(report_budget, int) and documents >= max(0, report_budget): + settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or "")) + for item in discovered[issued_index:]) + break + operation = "getReportDocument" + documents += 1 + _sync_event("amazon.sync", tenant_key, table, + {"state": "access", "operation": operation}, rt, report, log) + text = cx.settlement_document(fields, document_id) + cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype) + count = 0 + if cols and rows: + types = _ensure(con, mirror, cols, rows, preserve_rows=True) + # Never clear settlement history: durable markers prevent its replay. + # A newly issued document still replaces its own settlement's lines. + count = write_window(table, cols, types, rows) + written += count + # Record completion after the write, including header-only and blank files. + _record_state(con, entity_key("settlement_doc_" + document_id), "done", + count, issued.get("dataEndTime")) + downloaded += 1 + operations["getReports"] = { + "state": "done", "reports": len(discovered), + "downloaded": downloaded, "skipped": skipped} + if first_load and settlement_remaining: + break + windows = () + if table != "settlements": + windows = (([(task_start, task_end)] if task_end is not None else []) if first_load + else _windows(plan["start"], plan["end"])) + for w0, w1 in (() if table == "settlements" else windows): # W59-T43 / C2 has no amazon.access kind. Record each credential-backed # vendor attempt as a sync ACCESS phase, distinct from the completion event. # Lane F/combine own the vocabulary decision; no credential or row enters C2. + documents += 1 _sync_event("amazon.sync", tenant_key, table, {"state": "access", "operation": "report_document"}, rt, report, log) text, meta = cx.report_document( @@ -1237,7 +1566,25 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None text, document_id=meta.get("reportDocumentId"), report_type=rtype) declared = {} if not cols: - continue + if first_load: + # Even an empty recent window clears old rows exactly once. + for empty_table in (table, sibling): + if empty_table and floors[empty_table] is None: + if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?", + [table_name(empty_table)]).fetchone(): + con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1") + cleared = True + else: + continue + if sibling: + sibling_mirror = table_name(sibling) + sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared, + preserve_rows=first_load) if cols else {} + if cols and not sibling_cleared and not sibling_plan["incremental"]: + con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1") + sibling_cleared = True + if cols: + sibling_written += write_window(sibling, cols, sibling_types, rows) if table == "orders": rows = _dedupe_orders(rows) if table == "inventory": @@ -1246,16 +1593,32 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None cols.append("inventory_source") if table == "pricing": pricing_asins.extend(_pricing_asins(rows)) - types = _ensure(con, mirror, cols, rows, declared=declared) - # Clear once, after the table exists. The old multi-window branch deleted nothing, - # so every backfill appended a second copy of every historical row. - if not cleared: + types = _ensure(con, mirror, cols, rows, declared=declared, + preserve_rows=first_load) if cols else {} + # Explicit windows and first loads replace the mirror; incremental passes keep history. + if not cleared and not plan["incremental"]: con.execute(f"DELETE FROM {mirror} WHERE 1=1") cleared = True - written += _replace(con, mirror, cols, types, rows) + if cols: + written += write_window(table, cols, types, rows) + if first_load: + for reached in (table, sibling): + if reached: + record_backfill_floor(con, reached, w0) + floors[reached] = w0 + + if first_load: + landed.add((table, task_start, task_end)) + if any(next_table == table for next_table, _w0, _w1 in work[index + 1:]): + continue - operations = {} if table == "pricing": + if first_load: + # Completion can be in another process/pass than the recent ASIN report. + available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()} + pricing_asins = list(dict.fromkeys(pricing_asins + [str(row[0]) + for col in ("child_asin", "parent_asin") if col in available + for row in con.execute(f'SELECT DISTINCT "{col}" FROM {mirror} WHERE "{col}" IS NOT NULL').fetchall()])) state_key = ENTITY_STATE_KEYS[table] if not pricing_asins: _record_state(con, state_key, "unproven", 0, @@ -1291,11 +1654,11 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None awd_rows = _awd_rows(inventory) if awd_rows: awd_cols = sorted({c for row in awd_rows for c in row}) - awd_types = _ensure(con, mirror, awd_cols, awd_rows) - if not cleared: + awd_types = _ensure(con, mirror, awd_cols, awd_rows, preserve_rows=first_load) + if not cleared and not plan["incremental"]: con.execute(f"DELETE FROM {mirror} WHERE 1=1") cleared = True - written += _replace(con, mirror, awd_cols, awd_types, awd_rows) + written += write_window(table, awd_cols, awd_types, awd_rows) _record_state(con, state_key, "done", len(awd_rows)) operations["listInventory"] = { "state": "done", "rows": len(awd_rows)} @@ -1312,21 +1675,67 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None column_count, total_rows = len(cols), written report["tables"][table] = { "rows": total_rows, "columns": column_count, "provenance": prov, - "report": rtype, "operations": operations} + "report": rtype, "window": plan["cause"], "incremental": plan["incremental"], + "operations": operations} _record_state(con, key, "done", total_rows) _record_state(con, entity_key("sync_tick_" + table), "done", total_rows) + if not any(v["state"] in ("refused", "throttled") for v in operations.values()): + record_watermark(con, table, plan["end"], total_rows) _sync_event("amazon.sync", tenant_key, table, {"state": "done", "rows": total_rows, "operations_ok": all(v["state"] == "done" for v in operations.values())}, rt, report, log) log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}") + if sibling: + sibling_mirror = table_name(sibling) + try: + sibling_columns = len(con.execute(f"PRAGMA table_info('{sibling_mirror}')").fetchall()) + sibling_total = int(con.execute(f"SELECT COUNT(*) FROM {sibling_mirror}").fetchone()[0]) + except Exception: + sibling_columns, sibling_total = len(cols), sibling_written + sibling_prov = amazon_schema.PROVENANCE.get(sibling, "measured_at_sync") + report["tables"][sibling] = { + "rows": sibling_total, "columns": sibling_columns, "provenance": sibling_prov, + "report": rtype, "window": sibling_plan["cause"], + "incremental": sibling_plan["incremental"], "operations": {}} + _record_state(con, entity_key(sibling), "done", sibling_total) + _record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total) + record_watermark(con, sibling, sibling_plan["end"], sibling_total) + _sync_event("amazon.sync", tenant_key, sibling, + {"state": "done", "rows": sibling_total, "operations_ok": True}, + rt, report, log) + log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}") except Exception as e: # noqa: BLE001 + # A failed pass retains its previous watermark so its window will be retried. # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN # ticket answering itself, and it must not cost the six tables that did work. - problem = _sync_problem(e, table, "report_document", tenant_key, rt, report, log) + problem = _sync_problem(e, table, operation, tenant_key, rt, report, log) + failed.add(table) report["tables"][table] = {"rows": 0, **problem} _sync_event("amazon.sync", tenant_key, table, {"state": problem["state"], "rows": 0}, rt, report, log) + if sibling: + problem = _sync_problem(e, sibling, "report_document", tenant_key, rt, report, log) + report["tables"][sibling] = {"rows": 0, **problem} + _sync_event("amazon.sync", tenant_key, sibling, + {"state": problem["state"], "rows": 0}, rt, report, log) + if first_load: + remaining = sum(task not in landed for task in work) + if settlement_remaining: + remaining += settlement_remaining - 1 # Replace the discovery slot with known documents. + # Keep healthy tables moving after an isolated refusal. Once only failed work remains, + # stop automatic continuation instead of endlessly retrying the same vendor refusal. + continuable = any(task not in landed and task[0] not in failed for task in work) + state = ("partial" if remaining and continuable else + "failed" if remaining or report["problems"] or report["audit_problems"] else "done") + progress = {"state": state, "remaining": remaining, + "etaSeconds": first_load_eta(remaining, burst=0), + "cause": first_load_eta_cause(remaining), "backfillDays": DEFAULT_BACKFILL_DAYS} + report.update(progress) + report["firstLoad"] = dict(progress) + _record_state(con, entity_key(FIRST_LOAD_END_KEY), state, detail=_iso(end)) + _record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), state, + detail=json.dumps({key: progress[key] for key in ("remaining", "etaSeconds")})) report["finished"] = _iso(datetime.now(timezone.utc)) con.close() return report diff --git a/platform/harness/measure_filter.py b/platform/harness/measure_filter.py index 06389b14bb79590da1d076eccccbd7b4062ad1d9..0941b4e474057e9527cc84c92c64a56949078f17 100644 --- a/platform/harness/measure_filter.py +++ b/platform/harness/measure_filter.py @@ -302,7 +302,7 @@ def _base_topic(key): def _related_scope(related, keys): """Prepare all bases before querying: a refused composite must be wholly unscoped.""" - if related is None or isinstance(related, _relations.MeasurePin): + if related is None or (isinstance(related, _relations.MeasurePin) and related.refusal): return related try: pin = _relations.measure_pin(related) @@ -673,6 +673,10 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related= con = _sem._store_con() try: totals = _totals(con, key, window, today, team_id, related) + guard = None + metric = _sem.metrics().get(key, {}) + if related is not None and not related.refusal and metric.get('agg') == 'ratio': + guard = _totals(con, metric['denominator'], window, today, team_id, related) finally: con.close() scale = ADMITTED.get(key, {}).get('input_scale') or 1.0 @@ -682,7 +686,8 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related= # round(x, None) returns an int — exactly the pool's `_round` convention for money/counts. # The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction. return _related_answer( - {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}, related) + {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe + if guard is None or guard.get(p)}, related) #: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to diff --git a/platform/harness/runtime.py b/platform/harness/runtime.py index 9e66f9a23c1793fc387e358dca9567931bb482cf..96990ab0f21c65573a3e107e976e8c9f6d46eaaf 100644 --- a/platform/harness/runtime.py +++ b/platform/harness/runtime.py @@ -126,6 +126,8 @@ class TenantRuntime: tenant: object # harness.base.Tenant store_namespace: str = "" pool_cache: dict = field(default_factory=dict) + # Names/edges only, bounded and revision-keyed by routes_grid; no row pools live here. + related_cache: dict = field(default_factory=dict) #: EXIT-5 — the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps #: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on #: (stamp, scope, pool identity, question) so nothing user-shaped lives in them. @@ -179,6 +181,29 @@ class TenantRuntime: """ return self._store().get_projection(self.store_key(name), drop=drop) + def get_projections(self, names, drop=(), timeout_s=None): + """Batch projected reads when the active bound store has a native batch operation. + + ``None`` means the backend has no batch primitive; callers retain their ordinary per-key + path. That keeps this an optional optimization rather than widening the store contract. + """ + reader = getattr(self._store(), "get_projections", None) + if not callable(reader): + return None + keyed = [(str(name), self.store_key(name)) for name in (names or ())] + answer = reader([store_key for _name, store_key in keyed], drop=drop, + timeout_s=timeout_s) + return {name: answer.get(store_key, {}) for name, store_key in keyed} + + def get_many(self, names): + """Batch ordinary reads when the active bound store offers that optimization.""" + reader = getattr(self._store(), "get_many", None) + if not callable(reader): + return None + keyed = [(str(name), self.store_key(name)) for name in (names or ())] + answer = reader([store_key for _name, store_key in keyed]) + return {name: answer.get(store_key, {}) for name, store_key in keyed} + def revision(self, name): """Return the tiny durable change token for this tenant's addressed bucket.""" return self._store().revision(self.store_key(name)) diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py index 1de3e2bcfb725c3525cef7d5a6beb8b84031063e..f67713fc8e40aa5c35b6cf3288ac057cbd908ef8 100644 --- a/platform/harness/semantic.py +++ b/platform/harness/semantic.py @@ -18,8 +18,10 @@ Design constraints honored (from /odoo-api): - READ-ONLY on Odoo, always. """ import ast +import copy import functools import operator +import re from pathlib import Path import yaml @@ -28,6 +30,62 @@ import core.odoo as O MODEL_DIR = Path(__file__).resolve().parents[1] / "model" +_ROUTE_STOP_WORDS = frozenset( + "the a an of for in on at to from by with and or what which who when how much many " + "was were is are do did show me my our us last this that total sum count list give tell".split() +) + + +def _route_tokens(text): + words = re.findall(r"[^\W_]+", str(text or "").casefold()) + return tuple(word[:-1] if len(word) > 2 and word.endswith("s") else word + for word in words if len(word) >= 2 and word not in _ROUTE_STOP_WORDS) + + +def route_question(question, candidates): + """Rank semantic vocabulary using strings alone, without loading the model or any data. + + ⭐⭐ A SUBJECT WORD OUTWEIGHS A FIELD WORD, AND THE REASON WAS MEASURED, NOT ASSUMED. + Without it, "How many orders did we confirm in January?" routed to the CUSTOMERS registry: + `odoo_customers` approves a field labelled "Last order", "last" is a stop word, so that term + normalises to the single token "order" and ties exactly with `odoo_orders`' own subject. The + tie then fell to the alphabetical database key, and `customer_data` sorts before + `ut_odoo_orders`. A question about orders reached the customer book because of a sort order. + A candidate's `subject` is what the database IS (its label, its topic's label); a term drawn + entirely from it scores one more than the same term drawn from a field. `subject` is OPTIONAL: + a candidate that declares none scores exactly as it did before, which is what keeps every + hand-built routing fixture meaning what it says. + """ + words = _route_tokens(question) + scored = [] + for candidate in candidates: + subject = set() + for term in candidate.get("subject") or (): + subject.update(_route_tokens(term)) + terms = {} + for term in candidate.get("terms") or (): + tokens = _route_tokens(term) + if tokens and any(words[start:start + len(tokens)] == tokens + for start in range(len(words) - len(tokens) + 1)): + # Normalised duplicates count once, including singular/plural spellings. + label = str(term).strip().casefold() + terms[tokens] = min(terms.get(tokens, label), label) + matched = sorted(terms, key=lambda term: (-len(" ".join(term)), term)) + score = sum((2 if len(term) > 1 else 1) + (1 if subject.issuperset(term) else 0) + for term in matched) + if score: + scored.append({"database": candidate["database"], + "label": candidate.get("label") or candidate["database"], + "score": score, "matched": [terms[term] for term in matched], + "longest": len(" ".join(matched[0]))}) + scored.sort(key=lambda row: (-row["score"], -row["longest"], row["database"])) + best = scored[0] if scored else {} + return {"database": best.get("database", ""), "score": best.get("score", 0), + "matched": best.get("matched", []), + "ranked": [{key: row[key] for key in ("database", "label", "score")} + for row in scored]} + + # Topic scope binds to a NAMED, reviewed domain builder — never a YAML-assembled domain (one # source of truth for scope until OM-2 makes topics executable against the store). def _order_domain(*a, **kw): @@ -140,6 +198,8 @@ def _model(): if m["topic"] not in topics: raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}") metrics[m["key"]] = m + for topic in topics.values(): + _validate_topic_semantic(topic, metrics) return {"topics": topics, "metrics": metrics} @@ -160,21 +220,132 @@ def metrics(): return dict(_model()["metrics"]) -def semantic_fields(topic_key): - """Structured semantic suggestions for one topic, before tenant governance overrides. +def _validate_topic_semantic(topic, metric_rows): + """Validate a topic dict without re-entering the model loader.""" + key = topic["key"] + raw = topic.get("semantic") + + def fail(message): + raise ModelError(f"topic {key!r}: {message}") + + if not isinstance(raw, dict): + fail("semantic must be a curated block") + if ("measure" in raw) == ("measure_not_yet" in raw): + fail("declare exactly one of measure or measure_not_yet") + block = {"approved": [], "sensitive": [], "purposes": {}, "synonyms": {}, + "choices": {}, "date_field": "", "date_col_note": "", "scope": "", + "measure": None, "measure_not_yet": []} + block.update(copy.deepcopy(raw)) + fields = {field["key"]: field for field in topic.get("fields") or ()} + for name in ("approved", "sensitive", "purposes", "synonyms", "choices"): + expected = list if name in ("approved", "sensitive") else dict + if not isinstance(block[name], expected): + fail(f"semantic.{name} must be a {expected.__name__}") + for field in block[name]: + if not isinstance(field, str) or field not in fields: + fail(f"semantic.{name} names unknown field {field!r}") + for field, purpose in block["purposes"].items(): + if not isinstance(purpose, str) or not purpose.strip(): + fail(f"semantic.purposes field {field!r} needs a purpose") + for name in ("synonyms", "choices"): + for field, values in block[name].items(): + if not isinstance(values, list) or any(not isinstance(v, str) for v in values): + fail(f"semantic.{name} field {field!r} must contain strings") + for name in ("date_field", "date_col_note", "scope"): + if not isinstance(block[name], str): + fail(f"semantic.{name} must be a string") + date_field = block["date_field"] + if date_field and (date_field not in fields or + fields[date_field].get("type") not in ("date", "datetime")): + fail(f"semantic.date_field {date_field!r} must name a date or datetime field") + if not topic.get("grid") and not fields: + if raw.get("approved") != [] or raw.get("date_field") != "": + fail("a fact topic must declare approved: [] and date_field: an empty string") + if block["date_col_note"] != (topic.get("store") or {}).get("date_col"): + fail("semantic.date_col_note must document store.date_col") + if not block["scope"].strip(): + fail("semantic.scope must explain the topic scope") + # Every core topic has to say WHICH column a dated question lands on. A registry such as + # odoo_customers or odoo_products has no business date of its own, so it documents the date + # column of the fact topic its measure is bound to instead of inventing a field it does not + # have; `verify_w45_semantic_apps.py` is what holds that note to the real source column. + if not (date_field or block["date_col_note"].strip()): + fail("semantic needs a date_field or a date_col_note naming the dated column") + if "measure" in raw: + measure = block["measure"] + if not isinstance(measure, dict): + fail("semantic.measure must name a validated metric") + measure.setdefault("source", key) + for name in ("key", "source", "validate"): + if not isinstance(measure.get(name), str) or not measure[name].strip(): + fail(f"semantic.measure needs {name}") + metric = metric_rows.get(measure["key"]) or {} + if metric.get("topic") != measure["source"]: + fail(f"semantic.measure {measure['key']!r} does not belong to {measure['source']!r}") + if (measure["validate"] not in _VALIDATORS or + (metric.get("validate") or {}).get("method") != measure["validate"]): + fail(f"semantic.measure {measure['key']!r} needs a matching independent validator") + if measure["source"] != key: + bindings = topic.get("measures") or [] + bindings = [bindings] if isinstance(bindings, dict) else bindings + if not any(binding.get("source") == measure["source"] and + measure["key"] in (binding.get("keys") or ()) for binding in bindings): + fail(f"semantic.measure {measure['key']!r} has no source binding") + else: + pending = block["measure_not_yet"] + if not isinstance(pending, list) or not pending: + fail("semantic.measure_not_yet must report the missing measure") + for row in pending: + if not isinstance(row, dict) or any( + not isinstance(row.get(name), str) or not row[name].strip() + for name in ("key", "cause", "fix")): + fail("semantic.measure_not_yet needs non-empty key, cause and fix strings") + return block + + +def topic_semantic(topic_key): + """The validated platform curation for one topic, with all optional keys defaulted.""" + loaded = _model() + topic = loaded["topics"].get(str(topic_key)) + if not topic: + raise ModelError(f"unknown topic {topic_key!r}") + return _validate_topic_semantic(topic, loaded["metrics"]) - ``means`` remains accepted as the historical purpose source, but this projection is the one - consumers use: every field has all governance axes and enters as draft unless the model - explicitly disables it. Presence in a topic is never approval for retrieval. - """ + +def curated_topic_report(): + """Report declared reconciliation coverage without reading any Odoo records.""" + metric_rows = metrics() + out = [] + for key, topic in sorted(topics().items()): + block = topic_semantic(key) + measure = block["measure"] + validated = bool(measure and measure["validate"] in _VALIDATORS and + (metric_rows[measure["key"]].get("validate") or {}).get("method") + == measure["validate"]) + out.append({"topic": key, "grid": topic.get("grid") or "", + "approved": len(block["approved"]), "date_field": block["date_field"], + "scope": block["scope"], "measure": measure, + "measure_not_yet": block["measure_not_yet"], "validated": validated}) + return out + + +def semantic_fields(topic_key): + """Structured platform baseline, before tenant governance overrides.""" topic = _model()["topics"].get(str(topic_key)) if not topic: raise ModelError(f"unknown topic {topic_key!r}") + curated = topic_semantic(topic_key) out = {} for field in topic.get("fields") or (): if not isinstance(field, dict) or not field.get("key"): continue - meta = field.get("semantic") if isinstance(field.get("semantic"), dict) else {} + key = field["key"] + defaults = {"state": "approved" if key in curated["approved"] else "draft", + "sensitivity": "sensitive" if key in curated["sensitive"] else "internal", + "purpose": curated["purposes"].get(key, field.get("means") or ""), + "synonyms": curated["synonyms"].get(key, [])} + field_meta = field.get("semantic") if isinstance(field.get("semantic"), dict) else {} + meta = {**defaults, **field_meta} # Per-field curation wins. state = str(meta.get("state") or "draft").lower() sensitivity = str(meta.get("sensitivity") or "internal").lower() if state not in ("draft", "approved", "disabled"): @@ -183,7 +354,7 @@ def semantic_fields(topic_key): raise ModelError(f"topic {topic_key!r} field {field['key']!r}: bad sensitivity") out[field["key"]] = { "field": field["key"], "label": field.get("label") or field["key"], - "purpose": meta.get("purpose") or field.get("means") or "", + "purpose": meta.get("purpose") or "", "dataType": field.get("type") or "text", "role": meta.get("role") or "attribute", "synonyms": list(meta.get("synonyms") or ()), @@ -436,12 +607,39 @@ def validate(pre=None): def _store_con(): import harness.datastore as DS if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently - raise ModelError("the data cache is still warming up after a restart (a one-time first " - "sync). The dashboards work now from live data; ask me again in a " - "moment and I'll have it") + raise ModelError(_store_unready_cause(DS)) return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention +def _store_unready_cause(DS): + """D-513: the condition `ready()` actually hit. The old sentence always blamed a warm-up after + a restart, which is rarely the cause (another process holding the file is the usual one) and + stopped three sessions from looking further. Client-facing copy: no em or en dash.""" + live = "The dashboards work now from live data" + if not DS.DB_PATH.exists(): + return ("the data cache file does not exist on this server yet, so it has never synced. " + f"{live}; ask me again once the first sync has run") + try: + cur = DS.ro_cursor() + try: + phases = dict(cur.execute("SELECT entity, phase FROM _sync_state").fetchall()) + finally: + cur.close() + except Exception as exc: # noqa: BLE001 (the TYPE is named, never its message) + if "another process" in str(exc) or "lock" in str(exc).lower(): + return ("the data cache is held by another process right now, so it cannot be read. " + f"{live}; ask me again in a moment") + return (f"the data cache exists but could not be read ({type(exc).__name__}). " + f"{live}; ask me again in a moment") + required = [k for k, v in DS.ENTITIES.items() if not v.get("optional")] + pending = [k for k in required if phases.get(k) != "live"] + if not pending: + return ("the data cache was busy a moment ago and is readable again. " + f"{live}; ask me again now") + return (f"the data cache has not finished syncing: {len(required) - len(pending)} of " + f"{len(required)} required sources are ready. {live}; ask me again once it finishes") + + def _measure_sql(m, alias): """SQL for a base measure. Supports FILTERED measures (`store_filter_sql` — the Omni filtered-measure concept: sum(CASE WHEN … )) and `negate: true` (e.g. GL income, stored @@ -827,13 +1025,24 @@ def _scope_for_binding(store, scope_variant=None): def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None, team_id=None, filters=None, sort=None, limit=1000, exclude_services=False, - filter_tree=None, filter_conj="and", today=None, scope_variant=None): + filter_tree=None, filter_conj="and", today=None, scope_variant=None, related=None): """The semantic query over the tenant store — the engine behind the Analyst's run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model.""" t = _model()["topics"].get(topic) if not t or "store" not in t: raise ModelError(f"topic {topic!r} has no store binding") s = t["store"] + pin_predicate, pin_params = None, [] + if related is not None: + from core import relations + try: + pin = relations.measure_pin(related) + if pin.refusal: + raise relations.PinRefusal(pin.refusal['code'], pin.refusal['message'], pin.path) + pin_predicate, pin_params = pin.predicate(s) + except relations.PinRefusal as exc: + return {"topic": topic, "rows": [], "row_count": 0, "truncated": False, + "refusals": [exc.marker]} alias = s["alias"] dims = s.get("dims") or {} # ⭐ A GROUPED QUERY IS BOUNDED BY GROUPS, NOT BY ROWS — and the 5,000 row ceiling applied to @@ -869,7 +1078,9 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date foreign_vals = {} for k in foreign: sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from, - date_to=date_to, team_id=team_id) + date_to=date_to, team_id=team_id, related=related) + if sub.get("refusals"): + return sub foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0 if not base_keys: # purely foreign scalar request return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)", @@ -895,6 +1106,9 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}") where = [_scope_for_binding(s, scope_variant)] + if pin_predicate is not None: + where.append(f'({pin_predicate})') + params.extend(pin_params) # TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes — # datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date). # Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' < @@ -911,7 +1125,7 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date # intersecting either with a `team_id` restates the same dimension a second time and the two # statements can disagree. Concretely, `channel_amazon` AND `team_id = 5` returns an empty # cell that reads as "this SKU sold nothing on Amazon" — a FALSE number, not a narrower one. - if team_id and scope_variant is None: + if team_id and (scope_variant is None or related is not None): if not s.get("team_col"): raise ModelError(f"topic {topic!r} is company-level: it has no business-unit filter") where.append(f"{s['team_col']} = ?"); params.append(int(team_id)) @@ -1505,7 +1719,7 @@ def _one_binding_offer(topic, b, strict=False, ready=None): def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None, - exclude_services=False, offer=None): + exclude_services=False, offer=None, related=None): """`{group key: {measure key: value}}` for an entity topic's lookback columns. ONE grouped query for all `keys` together — they share a topic, a scope and a window, so @@ -1548,11 +1762,15 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non groups.setdefault((by_key[k]["topic"], by_key[k]["dim"], by_key[k].get("scope_variant")), []).append(k) if len(groups) > 1: - merged = {} + from core import relations + merged = relations.MeasureAnswer() if related is not None else {} for (_src, _dim, _variant), ks in groups.items(): - for gk, cell in entity_measure_values( + answer = entity_measure_values( topic, ks, date_from=date_from, date_to=date_to, team_id=team_id, - exclude_services=exclude_services, offer=offer).items(): + exclude_services=exclude_services, offer=offer, related=related) + if related is not None: + merged.refusals.extend(getattr(answer, 'refusals', ())) + for gk, cell in answer.items(): merged.setdefault(gk, {}).update(cell) return merged (src, dim, scope_variant), want = next(iter(groups.items())) @@ -1568,7 +1786,11 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non base_of = {k: (by_key[k].get("base_key") or k) for k in want} res = store_query(src, [base_of[k] for k in want], group_by=[dim], date_from=date_from, date_to=date_to, team_id=team_id, exclude_services=exclude_services, - limit=MAX_GROUPS, scope_variant=scope_variant) + limit=MAX_GROUPS, scope_variant=scope_variant, related=related) + if res.get("refusals"): + from core import relations + return relations.MeasureAnswer(refusals=[dict(marker, measure=k) + for marker in res['refusals'] for k in want]) # ⛔ A TRUNCATED GROUP SET IS A WRONG ANSWER PER ROW, not a short list — the same refusal # `rollup_sql.group_values` makes, for the same reason. Never write cells from one. if res.get("truncated"): diff --git a/platform/harness/windows.py b/platform/harness/windows.py index e0e615021db71e86d57d87284ac66e72468eca05..ff180f5c7bbbcf0d0c3a5bc880c9fa5526ab6b20 100644 --- a/platform/harness/windows.py +++ b/platform/harness/windows.py @@ -1,390 +1,11 @@ -"""harness/windows.py — the DATE WINDOW vocabulary (CG-7, owner item 2, 2026-07-26). +"""Compatibility import for the date-window vocabulary now owned by :mod:`core.windows`. -A filter condition on a measure reads: - - Where [Sales] [in the last 90 days] [>] [5,000] - -The middle bracket is this module. It turns a window SPEC — a small JSON object a saved view can -persist — into a concrete `(date_from, date_to)` pair of ISO dates. - -WHY IT IS ITS OWN MODULE, AND WHY IT TAKES `today` AS AN ARGUMENT ------------------------------------------------------------------------------------------------ -Two engines must agree on what "this quarter" means: this one (which compiles the SQL) and -`customer-grid/windows.ts` (which renders the label the user reads and the bounds the client -engine would use). If they disagree by one day, the grid shows a number the label denies, and -nothing errors. `aios-web/verify_windows.py` holds them in lock-step over a fixed set of probe -dates, exactly as verify_filter_engine.py does for the filter tree. - -`today` is a PARAMETER, never `date.today()` read inside. Three reasons, all learned: - - a gate cannot compare two engines on "now" — the two runs are milliseconds apart and land on - different sides of midnight roughly once every few thousand runs, which is the worst kind of - flake because it looks like a real divergence - - a window resolved during a request must not shift between the count query and the row query - - the tenant's day boundary is the tenant's, not the server's - -DESIGN NOTES THAT ARE EASY TO GET WRONG ------------------------------------------------------------------------------------------------ - - Bounds are INCLUSIVE at both ends and are DATES, not timestamps. `store_query` casts both - sides to TIMESTAMP and appends 23:59:59 to the upper bound, so a bare date is correct there; - the client compares the 10-char ISO prefix, so a bare date is correct there too. - - "Last N days" INCLUDES today. Airtable's "the past week" means the last 7 days up to and - including now, not the 7 days before yesterday. Off-by-one here silently drops today's - orders from every "recent" filter — invisible until someone asks why a sale they just - entered is missing. - - Week starts MONDAY (ISO 8601). Stated rather than defaulted: Python's weekday() is - Monday=0 and JavaScript's getDay() is Sunday=0, so the two engines disagree unless one of - them is explicitly corrected. That correction is the single most likely divergence in this - file, and the gate probes a Sunday and a Monday for exactly that reason. - - A `custom` window with only one bound is legal and means open-ended on the other side. - - An UNRESOLVABLE window returns None rather than a default. A window that quietly becomes - "all time" would widen a filter while still reporting an authoritative count. +Keep this module so existing API and harness consumers retain their stable import path. New +lower-layer consumers import ``core.windows`` directly; the dependency direction stays downward. """ -import datetime as _dt - -#: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts; -#: verify_windows.py asserts the two lists are identical, because a kind the client can emit and -#: the server cannot resolve is a filter that silently stops narrowing. -WINDOW_KINDS = ( - "all_time", - "today", - "yesterday", - "this_week", - "last_week", - "this_month", - "last_month", - "this_quarter", - "last_quarter", - "this_year", - "last_year", - "ytd", - "ytd_last_year", - "ltm", - "past_week", - "past_month", - "past_year", - "last_n_days", - "next_n_days", - "custom", -) - -#: Kinds that carry an integer `n`. -N_KINDS = frozenset({"last_n_days", "next_n_days"}) - -#: How each kind reads in a sentence. The UI renders these; they are here so the label and the -#: arithmetic cannot drift apart in the one place a user would never think to check. -WINDOW_LABELS = { - "all_time": "all time", - "today": "today", - "yesterday": "yesterday", - "this_week": "this week", - "last_week": "last week", - "this_month": "this month", - "last_month": "last month", - "this_quarter": "this quarter", - "last_quarter": "last quarter", - "this_year": "this year", - "last_year": "last year", - "ytd": "year to date", - # "last year to date" (LYTD), not "year to date, last year": the longer phrasing CLIPPED to - # "year to date, last ye" in the 146px window select — and this vocabulary is CLOSED, so - # the rule for it is that it stays readable rather than that it fits. Seen in a live - # screenshot of the owner's own comparison; every assertion in that run was green. - "ytd_last_year": "last year to date", - "ltm": "the last 12 months", - "past_week": "the past week", - "past_month": "the past month", - "past_year": "the past year", - "last_n_days": "the last {n} days", - "next_n_days": "the next {n} days", - "custom": "a custom range", -} - -MAX_N = 3650 # ten years; a bound, not a business rule - -# --- the DATE-VALUE anchors (owner item 3, 2026-07-26) ---------------------------------------- -# The second half of a date CONDITION, as distinct from a measure's window: -# -# Where [Last order] [is before] [one month ago] -# ^ op ^ THIS -# -# A window answers "over what period do I sum"; an anchor answers "which single date am I -# comparing against". Both live in this module for one reason: they are the same two-engine -# contract, resolved from the same `today`, and a gate that holds one in lock-step and not the -# other would leave half the sentence free to drift. - -#: Anchor modes, in the order the picker offers them. `exact` is the historical behaviour — a -#: rule with NO mode is an `exact` rule whose value is an ISO date, which is what every view -#: saved before this change carries. -ANCHOR_MODES = ( - "today", - "yesterday", - "one_week_ago", - "one_month_ago", - "n_days_ago", - "exact", -) - -#: Modes that carry NO value. ⚠ This set is load-bearing far outside this module: a rule whose -#: value is blank normally reads as INACTIVE, and an inactive rule is IGNORED — which WIDENS the -#: result under a count nobody would doubt. Every activeness check (`isRuleActive` in TS, -#: `filter_sql.is_rule_active`, and the validator's value handling) has to know these four are -#: active with an empty value. -ANCHOR_VALUE_FREE = frozenset({"today", "yesterday", "one_week_ago", "one_month_ago"}) - -ANCHOR_LABELS = { - "today": "today", - "yesterday": "yesterday", - "one_week_ago": "one week ago", - "one_month_ago": "one month ago", - "n_days_ago": "{n} days ago", - "exact": "an exact date", -} - - -def _iso(d): - return d.isoformat() - - -def _month_start(d): - return d.replace(day=1) - - -def _month_end(d): - return _next_month(d.replace(day=1)) - _dt.timedelta(days=1) - - -def _next_month(d): - return (d.replace(day=28) + _dt.timedelta(days=4)).replace(day=1) - - -def _quarter_start(d): - return _dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1) - - -def _days_in_month(y, m): - return (_dt.date(y + (m == 12), 1 if m == 12 else m + 1, 1) - _dt.timedelta(days=1)).day - - -def _shift_months(d, n): - """`d` moved `n` months, CLAMPING the day to the target month's length. - - Jan 31 back one month is Dec 31, but Mar 31 back one month is Feb 28 (or 29) — there is no - Feb 31 to land on. Both engines must clamp identically or "the past month" differs by up to - three days for a third of the calendar; the gate probes a 31st and a leap day for exactly it. - """ - total = (d.year * 12 + (d.month - 1)) + n - y, m = divmod(total, 12) - m += 1 - return _dt.date(y, m, min(d.day, _days_in_month(y, m))) - - -def _parse_date(v): - """Accept 'YYYY-MM-DD' (and tolerate a longer ISO timestamp by taking its date part).""" - if isinstance(v, _dt.date): - return v - s = str(v or "").strip()[:10] - if not s: - return None - try: - return _dt.date.fromisoformat(s) - except ValueError: - return None - - -def normalize(spec): - """Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None. - - Fail-closed on the KIND (an unknown kind is not a window), tolerant on the rest — the same - split `clean_filter_tree` uses, so one malformed window cannot cost a user their saved view. - """ - if not isinstance(spec, dict): - return None - kind = spec.get("kind") - if kind not in WINDOW_KINDS: - return None - out = {"kind": kind} - if kind in N_KINDS: - try: - n = int(spec.get("n")) - except (TypeError, ValueError): - return None - if n < 1 or n > MAX_N: - return None - out["n"] = n - if kind == "custom": - f, t = _parse_date(spec.get("from")), _parse_date(spec.get("to")) - if f is None and t is None: - return None # a custom range with no bounds is not a window - if f is not None and t is not None and f > t: - f, t = t, f # the builder cannot enforce order; the engine can - if f is not None: - out["from"] = _iso(f) - if t is not None: - out["to"] = _iso(t) - return out - - -def resolve(spec, today): - """Window spec + the tenant's today -> `(date_from, date_to)`, both inclusive ISO dates. - - Either side may be None, meaning open-ended. Returns None when the spec is not a window at - all — callers MUST treat that as "this condition cannot be evaluated" and refuse, never as - "no window", which would silently widen the result to all time. - """ - spec = normalize(spec) - if spec is None: - return None - kind = spec["kind"] - d = _parse_date(today) - if d is None: - raise ValueError("resolve() needs an explicit `today`; see this module's docstring") - - if kind == "all_time": - return (None, None) - if kind == "today": - return (_iso(d), _iso(d)) - if kind == "yesterday": - y = d - _dt.timedelta(days=1) - return (_iso(y), _iso(y)) - - # ISO 8601: the week starts MONDAY. Python's weekday() is already Monday=0; the TS mirror - # has to correct getDay(), which is Sunday=0. The gate probes both a Sunday and a Monday. - if kind == "this_week": - start = d - _dt.timedelta(days=d.weekday()) - return (_iso(start), _iso(start + _dt.timedelta(days=6))) - if kind == "last_week": - start = d - _dt.timedelta(days=d.weekday() + 7) - return (_iso(start), _iso(start + _dt.timedelta(days=6))) - - if kind == "this_month": - return (_iso(_month_start(d)), _iso(_month_end(d))) - if kind == "last_month": - prev = _month_start(d) - _dt.timedelta(days=1) - return (_iso(_month_start(prev)), _iso(prev)) - - if kind == "this_quarter": - qs = _quarter_start(d) - qe = _month_end(_dt.date(qs.year, qs.month + 2, 1)) - return (_iso(qs), _iso(qe)) - if kind == "last_quarter": - prev_end = _quarter_start(d) - _dt.timedelta(days=1) - qs = _quarter_start(prev_end) - return (_iso(qs), _iso(prev_end)) - - if kind == "this_year": - return (_iso(_dt.date(d.year, 1, 1)), _iso(_dt.date(d.year, 12, 31))) - if kind == "last_year": - return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_dt.date(d.year - 1, 12, 31))) - if kind == "ytd": - # year-to-date ENDS TODAY, unlike this_year which runs to Dec 31. The distinction is the - # whole reason both exist: comparing "this year" against last year double-counts the - # months that have not happened yet. - return (_iso(_dt.date(d.year, 1, 1)), _iso(d)) - if kind == "ytd_last_year": - # SAME PERIOD last year — Jan 1 LY through today's month/day LY. This is the honest - # partner of `ytd`, and it is not a nicety: it mirrors `core.periods.ytd_last_year` - # EXACTLY (including the Feb 29 -> Feb 28 clamp), which is what `pool()` computes the - # `revenue_ly` column from. `Sales[ytd] < Sales[ytd_last_year]` therefore reproduces the - # retired `at_risk > 0` condition rather than approximating it. Comparing `ytd` against - # `last_year` instead would pit seven months against twelve. - return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_shift_months(d, -12))) - if kind == "ltm": - # the last twelve months INCLUDING today: 365 days back, both ends inclusive - return (_iso(d - _dt.timedelta(days=364)), _iso(d)) - - # "The past week/month/year" are ROLLING and end TODAY — Airtable's wording, and NOT the - # calendar kinds above: `last_week` is the previous Monday-Sunday, `past_week` is the seven - # days ending today. Both are offered because a user means different things by them, and - # picking one to serve both would silently answer the other question. - # past_week = the same rule as last_n_days(7) — one period back, PLUS ONE DAY, so today - # is included and the span is exactly 7 days rather than 8. - # past_year agrees with `ltm` except across a leap day, where the calendar shift keeps the - # same month/day and the 364-day subtraction cannot. - if kind == "past_week": - return (_iso(d - _dt.timedelta(days=6)), _iso(d)) - if kind == "past_month": - return (_iso(_shift_months(d, -1) + _dt.timedelta(days=1)), _iso(d)) - if kind == "past_year": - return (_iso(_shift_months(d, -12) + _dt.timedelta(days=1)), _iso(d)) - - if kind == "last_n_days": - # INCLUSIVE of today — "the last 7 days" is today and the 6 before it, not the 7 before - # yesterday. Getting this wrong drops today's orders from every recent-activity filter. - return (_iso(d - _dt.timedelta(days=spec["n"] - 1)), _iso(d)) - if kind == "next_n_days": - return (_iso(d), _iso(d + _dt.timedelta(days=spec["n"] - 1))) - - if kind == "custom": - return (spec.get("from"), spec.get("to")) - raise AssertionError(f"unhandled window kind {kind!r}") # unreachable; WINDOW_KINDS is closed - - -def label(spec): - """How a window reads in the condition sentence ('in the last 90 days').""" - spec = normalize(spec) - if spec is None: - return "an invalid range" - return WINDOW_LABELS[spec["kind"]].format(n=spec.get("n")) - - -def resolve_anchor(mode, value, today): - """A date condition's right-hand side -> ONE inclusive ISO date, or None. - - `mode` is an entry of ANCHOR_MODES; None/'' means `exact`, which is what every view saved - before anchors existed carries (op + an ISO date in `value`). `value` supplies the number for - `n_days_ago` and the date for `exact`, and is ignored by the four value-free modes. - - Returns None when the anchor cannot be resolved — an unknown mode, a non-numeric N, a value - that is not a date. The caller must then treat the CONDITION as unanswerable and match - nothing. It must NOT fall through to "no condition": that widens the result while the count - beside it still looks authoritative, which is the whole reason this returns None rather than - a best guess (same contract as `resolve()` above). - - `n_days_ago` accepts n = 0 where a WINDOW requires n >= 1. A zero-day window is empty and - could only be a mistake; "0 days ago" is today, which is a date a person can mean. - """ - m = mode or "exact" - if m not in ANCHOR_MODES: - return None - d = _parse_date(today) - if d is None: - raise ValueError("resolve_anchor() needs an explicit `today`; see this module's " - "docstring on why the clock is never read here") - if m == "today": - return _iso(d) - if m == "yesterday": - return _iso(d - _dt.timedelta(days=1)) - if m == "one_week_ago": - # A week ago is a DATE (today minus 7), not the past-week RANGE. The two read almost - # identically in English and mean different things in a comparison. - return _iso(d - _dt.timedelta(days=7)) - if m == "one_month_ago": - return _iso(_shift_months(d, -1)) - if m == "n_days_ago": - s = str("" if value is None else value).strip() - # `[0-9]` not `\d`: a Python \d matches every Unicode decimal digit, so a fullwidth - # '30' would resolve here and be rejected by the TS mirror's Number() — the exact - # two-engine divergence filter_sql.to_num was rewritten to avoid. - if not s or not all("0" <= ch <= "9" for ch in s): - return None - n = int(s) - # No `n < 0` guard: the digit test above already excludes a sign, so it would be - # unreachable. The negative control proved that — removing it changed nothing, which is - # how dead code hides in a defensive-looking line. The CAP is the live guard. - if n > MAX_N: - return None - return _iso(d - _dt.timedelta(days=n)) - parsed = _parse_date(value) # exact - return None if parsed is None else _iso(parsed) +from core import windows as _windows +from core.windows import * # noqa: F401,F403 -def anchor_label(mode, value): - """How an anchor reads in the condition sentence ('one month ago', '30 days ago').""" - m = mode or "exact" - if m not in ANCHOR_MODES: - return "an invalid date" - if m == "n_days_ago": - return ANCHOR_LABELS[m].format(n=str("" if value is None else value).strip() or "N") - if m == "exact": - return str("" if value is None else value).strip() or ANCHOR_LABELS[m] - return ANCHOR_LABELS[m] +# Retain the historical public helper without making an otherwise-unused function public in core. +anchor_label = _windows._anchor_label diff --git a/platform/model/metrics/gl.yml b/platform/model/metrics/gl.yml index cec44c6d333fbd726579e4bf7c17d48bb3e1a307..7509f86f9a7ccb236ad51e08ee15d7d7686590a1 100644 --- a/platform/model/metrics/gl.yml +++ b/platform/model/metrics/gl.yml @@ -16,7 +16,7 @@ metrics: # `semantic._one_binding_offer` refuses the key outright (refusal 3), which is why the GL # account and vendor registries offered nothing at all. empty: zero - description: "Posted operating expense from the GL (all expense-type accounts; COGS excluded) — the Expenses dashboard's basis." + description: "Posted operating expense from the GL (all expense-type accounts; COGS excluded). This is the Expenses dashboard's basis." ai_context: "Company-level; includes payroll JEs/depreciation/fees that never hit vendor bills. Watch close-lag: the newest month is often still being booked." validate: method: gl_opex diff --git a/platform/model/metrics/invoice_lines.yml b/platform/model/metrics/invoice_lines.yml index 0af67a191e51649fd6122de3940b1ef0aa3e8f69..59bbb45da84463138ad5bea59e3e68963cb1b937 100644 --- a/platform/model/metrics/invoice_lines.yml +++ b/platform/model/metrics/invoice_lines.yml @@ -56,7 +56,7 @@ metrics: store_filter_sql: "l.move_type = 'out_invoice' AND ca.is_true_agent" format: usd description: >- - Invoice-line sales whose line carries a REAL agent (res.partner.agent = True) — excludes + Invoice-line sales whose line carries a REAL agent (res.partner.agent = True), excluding the internal salespeople who also carry commission lines. ai_context: >- Gross of credit notes by design (refund lines rarely carry commission rows); for the netted @@ -71,7 +71,7 @@ metrics: store_filter_sql: "l.move_type = 'out_invoice' AND ca.is_true_agent IS NOT TRUE" format: usd description: >- - Invoice-line sales with no REAL agent on the line — the strict reading, which counts + Invoice-line sales with no REAL agent on the line. This is the strict reading, which counts salesperson-only lines as unallocated. ai_context: >- Two defensible readings and this is the WIDER one (salesperson-covered business counts as diff --git a/platform/model/metrics/returns.yml b/platform/model/metrics/returns.yml index 7cfc24cebc9e204cefb0d1ba581e6c56eefccbcf..2c82c93f37c682135cbe44c92f269018b8ca9f1c 100644 --- a/platform/model/metrics/returns.yml +++ b/platform/model/metrics/returns.yml @@ -33,7 +33,7 @@ metrics: field: amount_untaxed_signed format: usd description: >- - Untaxed value of posted customer invoices — the denominator the business rates returns + Untaxed value of posted customer invoices, the denominator the business rates returns against. ai_context: >- NOT `revenue`. `revenue` is order-LINE subtotals of confirmed sales orders; this is posted @@ -46,7 +46,7 @@ metrics: numerator: returns denominator: invoiced format: pct - description: "Returns as a share of invoiced sales — the rate modules/returns.py reports." + description: "Returns as a share of invoiced sales, the rate modules/returns.py reports." ai_context: >- A FRACTION (0-1), like every other pct metric here — 0.0159 is 1.59%. The condition builder divides a typed percentage by 100 before comparing (`input_scale`), so a user types 5 and diff --git a/platform/model/metrics/sales.yml b/platform/model/metrics/sales.yml index 7736d945a1a24909b79bf2cecd8757d3a7a520cf..ad5df59da35a431a6417ddec08a988df101e7a55 100644 --- a/platform/model/metrics/sales.yml +++ b/platform/model/metrics/sales.yml @@ -92,7 +92,7 @@ metrics: denominator: units format: usd empty: blank - description: "Revenue per unit sold — the blended average selling price." + description: "Revenue per unit sold: the blended average selling price." ai_context: "revenue ÷ units at identical scope. Mixed units of measure are summed as ordered quantity, so this is per ordered unit, not per piece. Blank when nothing sold: an ASP of $0 on zero units is a false statement, not a missing one." - key: orders @@ -100,7 +100,7 @@ metrics: topic: sales_orders agg: count format: int - description: "Confirmed wholesale orders in the window (order-header count — what the scorecards show)." + description: "Confirmed wholesale orders in the window (an order-header count, which is what the scorecards show)." ai_context: "Order-level count; slightly higher than distinct-orders-from-lines because a few confirmed orders carry zero lines (a data-health artifact the reconciliation contract counts exactly)." validate: method: order_count diff --git a/platform/model/metrics/stock.yml b/platform/model/metrics/stock.yml index 51a10c56143a9b1fed87714e3e7b1d84d9de2e5f..f0410fc3ccb22a5aa91f62dc04d9160e5d233610 100644 --- a/platform/model/metrics/stock.yml +++ b/platform/model/metrics/stock.yml @@ -53,5 +53,5 @@ metrics: expr: "stock_in - stock_out" format: int empty: zero - description: "Arrivals minus departures over the window — the period change in units held." + description: "Arrivals minus departures over the window: the period change in units held." ai_context: "Derived from stock_in and stock_out, so it always reconciles to them. It is a MOVEMENT figure, not the on-hand balance: the balance is `product_data.on_hand` from stock.quant." diff --git a/platform/model/topics/commission_lines.yml b/platform/model/topics/commission_lines.yml index 9ae8ba80dad2050578d3a16609ea7d869ec1c2e5..6cb60740a9ce006d29649e8bd7a93627fd1bc8fb 100644 --- a/platform/model/topics/commission_lines.yml +++ b/platform/model/topics/commission_lines.yml @@ -7,6 +7,15 @@ # # Use `invoice_lines` for revenue-by-agent; use this ONLY for commission accrued / settlement. key: commission_lines +semantic: + approved: [] + date_field: "" + date_col_note: "a.invoice_date" + scope: "Every commission row, including rows attached to non-product lines; sum commission amount in company currency by invoice date, and do not treat commission row counts as sales line counts." + measure_not_yet: + - key: commission_amount + cause: "commission_amount resolves on commission_lines but declares no validate contract against an independent Odoo aggregate." + fix: "Declare a validate method for commission_amount in model/metrics/invoice_lines.yml and add a _v_commission_amount reconciliation to harness/semantic.py::_VALIDATORS." label: Commission lines (agent × invoice line) entity: account.invoice.line.agent diff --git a/platform/model/topics/credit_notes.yml b/platform/model/topics/credit_notes.yml index fd2d907beeed018634d81bde91efda95b5346921..924aabf805d9257a0d8c5722b9886b39c591ed9f 100644 --- a/platform/model/topics/credit_notes.yml +++ b/platform/model/topics/credit_notes.yml @@ -7,6 +7,12 @@ # parity-checked — and parity is the admission bar for filtering on it. Two topics with two # DOMAINS give both paths the same scope by construction. key: credit_notes +semantic: + approved: [] + date_field: "" + date_col_note: "m.invoice_date" + scope: "Posted customer credit notes only (out_refund); company consolidated with no partner exclusion or business-unit filter; negate amount_untaxed_signed to report positive returns by invoice date." + measure: {key: returns, validate: returns_sign} label: Customer credit notes (returns) entity: account.move domain_builder: credit_note_domain diff --git a/platform/model/topics/customer_invoices.yml b/platform/model/topics/customer_invoices.yml index 10878e21e73528f5bf88a07de133fcf227a286fe..0d5c647101cf797bebdbbbed27a009824fe9119c 100644 --- a/platform/model/topics/customer_invoices.yml +++ b/platform/model/topics/customer_invoices.yml @@ -9,6 +9,15 @@ # sales_lines); this is posted invoice documents. They differ by unbilled orders, by billing # timing, and by anything invoiced outside the wholesale sales scope. key: customer_invoices +semantic: + approved: [] + date_field: "" + date_col_note: "m.invoice_date" + scope: "Posted customer invoices only (out_invoice); company consolidated with no partner exclusion or business-unit filter; sum positive amount_untaxed_signed by invoice date." + measure_not_yet: + - key: invoiced + cause: "invoiced resolves on customer_invoices but declares no validate contract against an independent Odoo aggregate." + fix: "Declare a validate method for invoiced in model/metrics/returns.yml and add a _v_invoiced reconciliation to harness/semantic.py::_VALIDATORS." label: Customer invoices entity: account.move domain_builder: customer_invoice_domain diff --git a/platform/model/topics/gl_lines.yml b/platform/model/topics/gl_lines.yml index 5362dd9d10e54d45dca2aabd541986c4f74b4707..5c895bca71b08054942470061b649e0ebe073e3d 100644 --- a/platform/model/topics/gl_lines.yml +++ b/platform/model/topics/gl_lines.yml @@ -2,6 +2,12 @@ # filter — GL isn't team-scoped). BASIS RULE (the glaccount-vs-account lesson, now model law): # this is the RAW-GL basis; NEVER mix it with the management-basis P&L in one analysis. key: gl_lines +semantic: + approved: [] + date_field: "" + date_col_note: "l.date" + scope: "Posted journal entries only, by accounting date; company consolidated with no business-unit split; raw debit-positive balances make expenses positive and income negative before metric sign adjustments." + measure: {key: opex, validate: gl_opex} label: General ledger (raw, posted) entity: account.move.line # store-only topic: no live-path domain builder yet (lands with the Expenses retrofit); diff --git a/platform/model/topics/invoice_lines.yml b/platform/model/topics/invoice_lines.yml index 7959f2927fe55f22c0a9dadf4236b78b097988bf..07358d6844865f711d40e9ad7156983f39901f53 100644 --- a/platform/model/topics/invoice_lines.yml +++ b/platform/model/topics/invoice_lines.yml @@ -11,6 +11,12 @@ # CUSTOMER-MASTER book (res.partner.agent_ids); this one is per-line commission attribution. # They name different people. Always state which source a number came from. key: invoice_lines +semantic: + approved: [] + date_field: "" + date_col_note: "l.date" + scope: "Product lines on posted customer invoices and credit notes only, by line date; untaxed subtotals with refunds negated; business unit comes from the originating sales order, direct invoices retain a null unit, and no partner is excluded." + measure: {key: invoiced_line_sales, validate: line_vs_move_grain} label: Customer invoice lines (with commission agent) entity: account.move.line diff --git a/platform/model/topics/odoo_accounts.yml b/platform/model/topics/odoo_accounts.yml index 6e93832236805152c706cc68cdbe5ed61be0d253..89219f29ff09d702cc9f68198a77251fc05b69a1 100644 --- a/platform/model/topics/odoo_accounts.yml +++ b/platform/model/topics/odoo_accounts.yml @@ -6,6 +6,18 @@ # field row: add the column to the grid contract and re-emit, or the agent is being # trained on a schema the product does not have. key: odoo_accounts +semantic: + approved: [account_code, account_name, odoo_id, account_type, is_expense] + date_field: "" + date_col_note: "gl_lines.l.date" + scope: "Every Odoo account; operating expense comes from posted expense and depreciation lines in gl_lines, excluding cost of goods sold, at company level." + measure: {key: opex, source: gl_lines, validate: gl_opex} + purposes: + account_code: "The chart-of-accounts code used to identify an accounting account." + account_type: "The Odoo account classification used to separate expenses, income, assets and liabilities." + synonyms: + account_code: [account number, ledger code] + account_name: [ledger account, accounting account] label: "Odoo GL accounts" entity: account.account # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). diff --git a/platform/model/topics/odoo_agents.yml b/platform/model/topics/odoo_agents.yml index 9dc78aa32a44b1815505a0c52e7b1737ac898304..f0fc8fc0115b2dc182c4aba2ec5e4a7e75ee78a5 100644 --- a/platform/model/topics/odoo_agents.yml +++ b/platform/model/topics/odoo_agents.yml @@ -6,6 +6,17 @@ # field row: add the column to the grid contract and re-emit, or the agent is being # trained on a schema the product does not have. key: odoo_agents +semantic: + approved: [agent, odoo_id, agent_id, flagged, commissioned] + date_field: "" + date_col_note: "sales_lines.o.date_order" + scope: "Union of partners with commission lines and partners flagged as agents; wholesale sales measures use sales_lines scope, not the commission amount basis." + measure: {key: revenue, source: sales_lines, validate: order_level_revenue} + purposes: + agent: "The partner who is flagged as an agent or carries commission lines." + commissioned: "Whether the partner carries commission lines, independently of the agent flag." + synonyms: + agent: [sales agent, commission agent] label: "Odoo agents" entity: res.partner # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). diff --git a/platform/model/topics/odoo_bills.yml b/platform/model/topics/odoo_bills.yml index 76117ddfd7c2f2a43b9fb499d637fe2b9540ed99..2de49e072c0c03492cd4d41d1f534fc3e6920494 100644 --- a/platform/model/topics/odoo_bills.yml +++ b/platform/model/topics/odoo_bills.yml @@ -6,6 +6,23 @@ # field row: add the column to the grid contract and re-emit, or the agent is being # trained on a schema the product does not have. key: odoo_bills +semantic: + approved: [bill_no, odoo_id, vendor, vendor_id, invoice_date, due_date, amount_untaxed, residual, payment_state, move_type, vendor_link] + date_field: invoice_date + scope: "Posted vendor invoices and vendor credit notes (in_invoice, in_refund); payable residuals are negative in Odoo, so a payable balance uses their absolute value." + measure_not_yet: + - key: ap_outstanding + cause: "No payables fact topic or ap_outstanding metric exists. ar_outstanding resolves on receivables but covers customer documents, so it cannot reconcile a vendor bill balance." + fix: "Add a payables topic for posted in_invoice and in_refund documents, declare ap_outstanding with a validate method in its metric yml, and add a _v_ap_outstanding reconciliation to harness/semantic.py::_VALIDATORS." + purposes: + invoice_date: "The vendor bill date used for dated bill questions." + residual: "The outstanding amount on this posted vendor document; no validated payable aggregate is available yet." + synonyms: + bill_no: [vendor bill, supplier invoice] + residual: [unpaid amount, payable balance] + choices: + move_type: [in_invoice, in_refund] + payment_state: [not_paid, in_payment, paid, partial, reversed, invoicing_legacy] label: "Odoo vendor bills" entity: account.move # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). diff --git a/platform/model/topics/odoo_customers.yml b/platform/model/topics/odoo_customers.yml index e5c6d7a4be1740361e5dcc4a507eab55b4cb53fa..768bdca6dc19de62a55357cc718323c32b2c7c93 100644 --- a/platform/model/topics/odoo_customers.yml +++ b/platform/model/topics/odoo_customers.yml @@ -6,6 +6,21 @@ # field row: add the column to the grid contract and re-emit, or the agent is being # trained on a schema the product does not have. key: odoo_customers +semantic: + approved: [customer, agent, dba, salesperson, city, state, country, customer_since, last_order, pricelist, payment_terms] + date_field: "" + date_col_note: "sales_lines.o.date_order" + scope: "One row per customer in the customer book who ordered in the last 24 months; this registry has no date dimension, and dated wholesale revenue uses sales_lines joined by order_partner." + measure: {key: revenue, source: sales_lines, validate: order_level_revenue} + sensitive: [street, street2, zip, lat, lon, notes] + purposes: + customer: "The customer account in the customer book, identified by partner_id." + partner_id: "The Odoo res.partner id used to join this customer to sales orders and invoices." + last_order: "The date of this customer's latest order; dated revenue questions use the sales_lines order date." + synonyms: + customer: [customer account, buyer] + partner_id: [customer id, partner id] + dba: [business unit, brand] label: "Odoo customers" entity: res.partner # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). @@ -69,7 +84,9 @@ fields: type: text kind: data semantic: - state: draft + # No `state:` here on purpose. R27 ships the Odoo map pre-approved and the topic-level + # `semantic.approved` block above is the single declaration of that; a per-field state + # predating approval would silently outrank it and leave a customer question unanswerable. role: entity_name synonyms: [customer name, account name] sensitivity: internal diff --git a/platform/model/topics/odoo_invoices.yml b/platform/model/topics/odoo_invoices.yml index 598ec1b5628feb3bf93cdca355c9da75426cc22f..7a9fc731b9eecb37c43ca863c42e7f87cab31552 100644 --- a/platform/model/topics/odoo_invoices.yml +++ b/platform/model/topics/odoo_invoices.yml @@ -6,6 +6,20 @@ # field row: add the column to the grid contract and re-emit, or the agent is being # trained on a schema the product does not have. key: odoo_invoices +semantic: + approved: [invoice_no, odoo_id, customer, partner_id, invoice_date, due_date, residual, amount_untaxed, payment_state, move_type, wholesale_scope, origin_order, order_link] + date_field: invoice_date + scope: "Posted customer invoices and credit notes (out_invoice, out_refund); signed residuals are positive for invoices and negative for credits, and receivables are company consolidated." + measure: {key: ar_outstanding, source: receivables, validate: ar_outstanding} + purposes: + invoice_date: "The customer invoice date used for dated billing questions." + residual: "Signed outstanding customer balance, with credit notes netted against invoices." + synonyms: + invoice_no: [invoice number, billing document] + residual: [accounts receivable, outstanding balance, unpaid amount] + choices: + move_type: [out_invoice, out_refund] + payment_state: [not_paid, in_payment, paid, partial, reversed, invoicing_legacy] label: "Odoo invoices" entity: account.move # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). diff --git a/platform/model/topics/odoo_orders.yml b/platform/model/topics/odoo_orders.yml index 3c62b470023bee98fd93b3e038f355fd5e6e93bb..4a42650cb2770dbc1f57973570c7957bda89d2ce 100644 --- a/platform/model/topics/odoo_orders.yml +++ b/platform/model/topics/odoo_orders.yml @@ -6,6 +6,22 @@ # field row: add the column to the grid contract and re-emit, or the agent is being # trained on a schema the product does not have. key: odoo_orders +semantic: + approved: [order_no, odoo_id, customer, partner_id, order_date, amount_untaxed, team, state, invoice_status, wholesale_scope, invoices] + date_field: order_date + scope: "Confirmed orders in state sale or done; quotations and cancellations excluded. The revenue measure uses wholesale sales_lines scope for teams 5 and 6, excluding GIFTWARE DEALS." + measure: {key: revenue, source: sales_lines, validate: order_level_revenue} + purposes: + amount_untaxed: "The untaxed order total, the figure wholesale revenue sums for orders within wholesale scope." + order_date: "The sales order date used for questions such as sales last week." + wholesale_scope: "Whether the order belongs to the wholesale population used by the revenue measure." + synonyms: + amount_untaxed: [order value, order total, untaxed amount] + order_no: [sales order, order number] + team: [business unit, brand] + choices: + state: [sale, done] + invoice_status: [upselling, invoiced, to invoice, "no"] label: "Odoo orders" entity: sale.order # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). diff --git a/platform/model/topics/odoo_products.yml b/platform/model/topics/odoo_products.yml index c1a10fe1d68ec5492639a4cf495ed5ad61c05b8e..ae2a7e23a49465420cbe62e39b0baf094d4794bf 100644 --- a/platform/model/topics/odoo_products.yml +++ b/platform/model/topics/odoo_products.yml @@ -14,9 +14,25 @@ # price_changes, closeouts, notes), and no key here is absent from the contract. So a # field row IS hand-edited, and the rule is the honest one: the key, type and label must # match `aios_grid_fields.json`, because the agent is otherwise trained on a schema the -# product does not have. Building the missing gate is booked; until it exists, this -# paragraph is the only thing keeping the two in step. +# product does not have. ⭐ D-400 GATE, 2026-09-16: `platform/verify_product_pool.py:: +# section_product_field_contract` now holds this file, the json and `pd_fields` in step. +# Its `YML_ABSENT_KEYS` also lists 11 keys absent here WITHOUT a documented reason (fba and +# the ten category/attribute columns); whether they belong here is a decision still owed. key: odoo_products +semantic: + approved: [code, product_id, product, category, supplier, origin_country, lead_days, first_cost, price_1, unit_1, price_2, unit_2, price_3, unit_3] + date_field: "" + date_col_note: "sales_lines.o.date_order" + scope: "Every active product, including unsold products; SKU code is the business key with a product-id fallback for missing codes. The catalogue has no date dimension; product movement uses sales_lines across all channels." + measure: {key: revenue, source: sales_lines, validate: order_level_revenue} + purposes: + code: "The product SKU code, with a product-id fallback when Odoo has no code." + product_id: "The Odoo product.product id used by order, invoice and stock-move relations." + first_cost: "The product's first cost per unit, not a validated cost-of-goods-sold aggregate." + synonyms: + code: [sku, product code, item code] + product: [item, product name] + supplier: [vendor] label: "Odoo products" entity: product.product # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). diff --git a/platform/model/topics/odoo_vendors.yml b/platform/model/topics/odoo_vendors.yml index 076a039f4a2abd29a2b2b9bf881ca8f5673c964e..70f103dba9b398f6606e30d9be4c21c8cdbce570 100644 --- a/platform/model/topics/odoo_vendors.yml +++ b/platform/model/topics/odoo_vendors.yml @@ -6,6 +6,19 @@ # field row: add the column to the grid contract and re-emit, or the agent is being # trained on a schema the product does not have. key: odoo_vendors +semantic: + approved: [vendor, odoo_id, vendor_id, country, email, phone, mobile, vat, vendor_ref, website, street, street2, city, zip, bills] + date_field: "" + date_col_note: "gl_lines.l.date" + scope: "Vendors with posted vendor bills form the population; operating expense uses posted gl_lines grouped by partner at company level, and is not a payable balance." + measure: {key: opex, source: gl_lines, validate: gl_opex} + sensitive: [email, phone, mobile, vat, street, street2, zip] + purposes: + vendor: "The supplier partner with posted vendor bill history." + vendor_id: "The Odoo res.partner id that joins this vendor to bills and ledger lines." + synonyms: + vendor: [supplier, vendor account] + vendor_id: [supplier id, partner id] label: "Odoo vendors" entity: res.partner # The database this topic DESCRIBES — the same store key the nav opens (W33-T46). diff --git a/platform/model/topics/receivables.yml b/platform/model/topics/receivables.yml index 6903e158c2d14f0cf6b98a3825e413cc6644c460..6df764bedcb5172626f96e92c72089d236225e23 100644 --- a/platform/model/topics/receivables.yml +++ b/platform/model/topics/receivables.yml @@ -2,6 +2,12 @@ # (AR is consolidated — the Collections page convention). Store-only topic (live path = the AR # module's own domains; the validator compares against a direct live aggregate). key: receivables +semantic: + approved: [] + date_field: "" + date_col_note: "m.invoice_date" + scope: "Posted customer invoices and credit notes only; signed residuals net credits against invoices at company level with no business-unit split; invoice date selects documents and due date determines overdue status." + measure: {key: ar_outstanding, validate: ar_outstanding} label: Receivables (customer invoices) entity: account.move diff --git a/platform/model/topics/sales_lines.yml b/platform/model/topics/sales_lines.yml index bb5143460f2e3843ec8e17fb180ae9d3b8e0a70c..f2adbd94af7e385b17a234db908d2552cd0b8f8f 100644 --- a/platform/model/topics/sales_lines.yml +++ b/platform/model/topics/sales_lines.yml @@ -3,6 +3,12 @@ # metrics, saved views and the AI Analyst all resolve against — ONE definition of what the # data MEANS, consumed by every surface. key: sales_lines +semantic: + approved: [] + date_field: "" + date_col_note: "o.date_order" + scope: "Confirmed order lines only (sale, done), for wholesale teams Fisch 5 and Royal 6 with optional business-unit narrowing; exclude GIFTWARE DEALS and sum untaxed price_subtotal by parent order date." + measure: {key: revenue, validate: order_level_revenue} label: Sales lines (wholesale) entity: sale.order.line diff --git a/platform/model/topics/sales_orders.yml b/platform/model/topics/sales_orders.yml index 743e386ea621c566a5e36ee1a8ad6724cc873374..ef542a1c82ac2330a3302497150ab8210ea8bead 100644 --- a/platform/model/topics/sales_orders.yml +++ b/platform/model/topics/sales_orders.yml @@ -2,6 +2,12 @@ # same business scope, order grain — for counts and order-level amounts (amount_untaxed). The two # grains reconcile by contract (line Σ == order Σ; order count == line-parents + line-less orders). key: sales_orders +semantic: + approved: [] + date_field: "" + date_col_note: "o.date_order" + scope: "Confirmed order headers only (sale, done), for wholesale teams Fisch 5 and Royal 6 with optional business-unit narrowing; exclude house and GIFTWARE accounts, use untaxed amount_untaxed, and filter by order date." + measure: {key: orders, validate: order_count} label: Sales orders (wholesale) entity: sale.order domain_builder: sale_order_domain diff --git a/platform/model/topics/stock_moves.yml b/platform/model/topics/stock_moves.yml index 14f27d9d82220791db7dc95fb1595e0a6952e745..519d47fa29c37ea25813ab7a9b2150c0193af5ef 100644 --- a/platform/model/topics/stock_moves.yml +++ b/platform/model/topics/stock_moves.yml @@ -9,6 +9,15 @@ # Do not "simplify" one without re-reading it — three of them are traps that produce a # plausible wrong number rather than an error. key: stock_moves +semantic: + approved: [] + date_field: "" + date_col_note: "sm.date" + scope: "Done stock moves only, by movement date and company consolidated; inbound crosses from non-internal to internal locations, outbound crosses the reverse way, and internal transfers count as neither; picking code does not define direction." + measure_not_yet: + - key: stock_in + cause: "stock_in resolves on stock_moves but declares no validate contract against an independent Odoo aggregate; stock_out and stock_net also lack validate contracts." + fix: "Declare validate methods in model/metrics/stock.yml and add _v_stock_in, _v_stock_out and _v_stock_net reconciliations to harness/semantic.py::_VALIDATORS using both location usages." label: Stock movement entity: stock.move domain_builder: ~ diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 2b42358d5df506566e3a4ed305cb277827fc7e06..673446dc32cf17f825723941a5d171b27f396074 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -1431,9 +1431,72 @@ def validate(team_id=None, t=None): checks.extend(validate_price_and_unit_cells(rows)) checks.extend(validate_stock_measures(t=t)) checks.extend(validate_fba(rows)) + checks.extend(validate_linked_metric(team_id=team_id)) return checks +def validate_linked_metric(customer_id=None, product_id=None, team_id=None, + date_from=None, date_to=None): + """Reconcile one customer/product revenue cell to an independent live Odoo aggregate. + + The semantic result uses the shipping pin predicate and entity measure resolver. The + oracle uses Odoo's real partner/product ids and read_group, never the mirror's totals. + A live/mirror gap is reported exactly and fails; sync and rerun to distinguish lag. + Explicit ids/window make the leg reproducible without rebuilding the product pool. + """ + from core import relations + from harness import semantic as sem, datastore as ds + + name = 'linked customer/product revenue reconciles to independent Odoo' + try: + con = ds.ro_cursor() + try: + parts, params = ["o.state IN ('sale','done')"], [] + for expression, value in [('l.order_partner_id', customer_id), + ('l.product_id', product_id), ('o.team_id', team_id)]: + if value is not None: + parts.append(f'{expression} = ?') + params.append(int(value)) + sample = con.execute( + "SELECT l.order_partner_id, l.product_id, " + "COALESCE(NULLIF(p.default_code, ''), 'pid:' || CAST(p.id AS VARCHAR)), " + "CAST(o.date_order AS DATE) FROM sale_order_line l " + "JOIN sale_order o ON o.id=l.order_id JOIN product_product p ON p.id=l.product_id " + f"WHERE {' AND '.join(parts)} ORDER BY o.date_order DESC, l.id LIMIT 1", params).fetchone() + finally: + con.close() + if not sample: + return [dict(check=name, ours=None, theirs=None, ok=False, + detail='No confirmed customer/product sample is available. Sync the sales source and rerun.')] + customer_id, product_id, code, day = sample + date_from, date_to = date_from or str(day), date_to or str(day) + pin = relations.MeasurePin(['validation'], + {'customer_data': [{'pid': customer_id}], + 'product_data': [{'product_id': product_id}]}, + {'sale_order_line': [ + dict(table='customer_data', column='pid', sql='{alias}.order_partner_id'), + dict(table='product_data', column='product_id', sql='{alias}.product_id')]}, + required=('customer_data', 'product_data')) + answer = sem.entity_measure_values('odoo_products', ['revenue'], date_from=date_from, + date_to=date_to, team_id=team_id, related=pin) + if getattr(answer, 'refusals', ()): + return [dict(check=name, ours=None, theirs=None, ok=False, detail=answer.refusals)] + ours = answer.get(code, {}).get('revenue', 0) + domain = O.sale_line_domain(date_from, date_to, team_id=team_id, all_channels=True, + extra=[('order_partner_id', '=', customer_id), ('product_id', '=', product_id)]) + aggregate = O.get_odoo().read_group('sale.order.line', domain, + ['price_subtotal:sum'], [], lazy=False) + theirs = sum(r.get('price_subtotal') or 0 for r in aggregate) + delta = round(ours - theirs, 2) + return [dict(check=name, ours=ours, theirs=theirs, ok=abs(delta) < 0.005, + detail=dict(customer=customer_id, product=product_id, team_id=team_id, + date_from=date_from, date_to=date_to, difference=delta, + recommendation='If different, sync sales and rerun this exact sample; investigate any remaining difference.'))] + except Exception as exc: + return [dict(check=name, ours=None, theirs=None, ok=False, + detail=f'Validation could not read both sources ({type(exc).__name__}). Check source readiness and rerun.')] + + def fba_quant_oracle(): """`(warehouse | None, {product_id: units}, detail)` — the FBA column's INDEPENDENT oracle. diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py index 0d6006c31170a76cec92accdf7844251f89f0b0e..2588c3d579c16a4f82499e0db4e76dc4c4a4d85a 100644 --- a/platform/modules/sheet.py +++ b/platform/modules/sheet.py @@ -33,31 +33,23 @@ from __future__ import annotations import datetime import re +from urllib.parse import urlencode + +from harness.filter_sql import NUMERIC_TYPES as MIRROR_NUMERIC_TYPES +from core.sheet_binding import ( + DRAG_ACROSS, + DRAG_DOWN_NONE, + PERIOD_REF as _PERIOD_REF, + PERIOD_START as _PERIOD_START, + PERIOD_UNITS, +) #: C2's aggregate vocabulary. W59-T14 resolves the `metric_key` arm through a Metric column, #: using only the catalogue's declared fold: see `metric_key_fold` and its additivity guard. AGGREGATE_FNS = ("sum", "count", "avg", "min", "max") -#: ⭐ R9's PERIOD (wave 51, W51-T21). The three units a bound cell can be pinned to, and the month -#: its window starts in. `web/src/sheet/types.ts` carries the same three names. -PERIOD_UNITS = ("month", "quarter", "year") - -#: `YYYY-MM`. A period starts in a MONTH whatever its unit, because a quarter and a year are both -#: named by the month they open on, and one spelling means a copied cell cannot mean two things. -_PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$") - -#: Literal inclusive endpoints of a frozen calendar period, spelled `YYYY-MM-DD`. -_PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$") - -#: A `period.ref` points at a cell that holds the period, so it is a plain A1 address with no `$`, -#: exactly as `core/sheets.py::_ADDR` spells one. -_PERIOD_REF = re.compile(r"^[A-Z]{1,3}[1-9][0-9]{0,6}$") - -#: ⭐ R10's DRAG, and it is TWO settings rather than one because the two axes answer different -#: questions. Across columns a drag walks the PERIOD; down rows it walks the next value of a -#: chosen field. Defaults: across "period", down "none". -DRAG_ACROSS = ("period", "none") -DRAG_DOWN_NONE = "none" +#: Calendar, reference, and drag vocabularies are re-exported from the lower document cleaner so +#: existing callers keep this module's public names without maintaining a second normalizer. #: C2's refusal codes, spelled once. ⚠ Each message reaches a screen, so none carries an em dash or #: an en dash (standing rule 2) and each names both the cause and the subject. @@ -72,7 +64,17 @@ REFUSALS = { "binding_invalid": "That cell's source is not set up correctly.", "aggregate_unknown": "{subject} is not an aggregate this cell can compute.", "field_unknown": "{subject} is not a column on this database.", - "period_invalid": "{subject} is not a period this cell can use.", + "period_invalid": "{subject} is not a period this cell can use. " + "Choose a valid start for the selected period unit.", + "ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.", + "ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year " + "to date, this month or this quarter.", + "ending_cell_unreadable": "Ending from cell {subject} cannot be read. " + "Use a cell containing a date in YYYY-MM-DD format.", + "filter_cell_unreadable": "Filter value from cell {subject} cannot be read. " + "Choose a readable cell containing one nonblank value, or type the filter value.", + "linked_filter_unreadable": "Linked field {subject} cannot be filtered because {detail} " + "Choose a readable field through an available link, or ask an administrator to restore access.", # ── W53-T13's two refusals, and the reason each of them is NOT `field_unknown` ──────────── # # ⛔⛔ `field_unknown`'s sentence is "{subject} is not a column on this database", and for a @@ -140,99 +142,27 @@ class SheetRefusal(Exception): self.detail = str(detail or "") self.message = template.format(subject=self.subject or "this database", detail=self.detail) + if code in ("ending_invalid", "ending_cell_unreadable", "filter_cell_unreadable") and self.detail: + verb = "used" if code == "ending_invalid" else "read" + self.message = self.message.replace(f"cannot be {verb}.", + f"cannot be {verb} because {self.detail}") super().__init__(self.message) def clean_period(raw): - """R9's period, or None. `{unit, start, locked?, ref?}`, with `{ref}` / `{window}` still read. - - A frozen period is `{unit, start, locked?, frozen: true, from, to}`. `frozen` stores only - True so freezing and unfreezing returns to the identical rolling period already stored. - - ⛔ AN UNKNOWN UNIT OR A MALFORMED START IS REFUSED BY NAME, and that is not the same call as - the cosmetic keys next door. A period decides WHICH ROWS ARE COUNTED. Dropping `unit: "week"` - would answer for every row the binding can see and hand back a bigger number with nobody told, - which is standing rule 1's second sentence turned into a defect. - - ⚠ `locked` STORES ONLY TRUE. R9's lock means "a copy of this cell keeps this period"; an - unlocked period is what every cell already is, so storing `false` would make a cell that was - locked and unlocked compare unequal to the cell it started as. - - ⛔⛔ D-660 (W53-T16) - A PERIOD THAT NAMES NO WINDOW IS NOT A PERIOD, AND STORING ONE IS THE - WHOLE DEFECT. `{"unit": "month"}` with no `start`, or `{"locked": true}` on its own, used to be - stored as a period. `period_window` reads either as None, so the CELL still answered all time - - but the binding now carried a `period` key, and the editor reading it back sees a declared unit, - fills the missing start with TODAY'S MONTH and shows it as a chosen period. The next save then - stamps `{unit: "month", start: }` over a cell nobody meant to narrow: an all-time - total silently becomes this month's, which is exactly what D-660 names. So a period must name a - window - a `start`, validated frozen dates, the legacy `window`, or R14's `ref` - or it is - dropped and the cell stays all time. ⚠ THE ORDER MATTERS: an unknown unit or a malformed - start is still REFUSED BY NAME above, as are missing or malformed frozen dates, because a - wrong period and an absent one are different answers. - """ - if not isinstance(raw, dict): - return None - out = {} - - unit = str(raw.get("unit") or "").strip().lower() - if unit: - if unit not in PERIOD_UNITS: - raise SheetRefusal("period_invalid", unit) - out["unit"] = unit - - start = str(raw.get("start") or "").strip() - if start: - if not _PERIOD_START.match(start): - raise SheetRefusal("period_invalid", start) - out["start"] = start - - if raw.get("frozen") is True: - dates = {} - for key in ("from", "to"): - value = str(raw.get(key) or "").strip() - # ⚠ `value or key` because a MISSING date has no value to name, and a refusal with an - # empty subject reads as " is not a period this cell can use" - the same blank-subject - # defect the Metric binding originally exposed. - if not _PERIOD_DAY.fullmatch(value): - raise SheetRefusal("period_invalid", value or key) - dates[key] = value - # ISO days have fixed-width year/month/day parts, so string order is date order. - if dates["to"] < dates["from"]: - raise SheetRefusal("period_invalid", dates["to"]) - out.update({"frozen": True, **dates}) - - if raw.get("locked") is True: - out["locked"] = True - - ref = str(raw.get("ref") or "").strip().upper().replace("$", "") - if ref: - if not _PERIOD_REF.match(ref): - raise SheetRefusal("period_invalid", ref) - if not out.get("frozen"): - out["ref"] = ref - - #: ⚠ The wave-49 spelling, kept so a binding written before R9 still round trips. It is not a - #: second dialect for the same thing: `window` is a raw relative window, `unit` + `start` is a - #: named calendar period, and a document carrying the old one is not rewritten on read. - if (raw.get("window") is not None and "unit" not in out and "start" not in out - and not out.get("frozen")): - out["window"] = raw["window"] - - #: ⛔ D-660: see the docstring. A unit on its own, a lock on its own, or the empty dict names no - #: window, so it is not a period and the cell keeps its all-time answer. A frozen period - #: names its window through the two literal dates already validated above. - if not ({"start", "window", "ref", "frozen"} & set(out)): - return None - return out + """Normalize period metadata through the core document contract.""" + from core.sheet_binding import BindingMetadataRefusal, clean_period as _clean + try: + return _clean(raw) + except BindingMetadataRefusal as exc: + raise SheetRefusal(exc.code, exc.subject, exc.detail) from exc -#: ⭐ R7's PERIOD ARITHMETIC, spelled once. How many months each unit spans, which is the whole -#: difference between the three: a period is always named by the month it OPENS on (`clean_period` -#: enforces the `YYYY-MM` spelling for that reason), so the unit only decides where it closes. +#: Month-based periods retain their existing arithmetic. Day/Week advance in calendar days. PERIOD_MONTHS = {"month": 1, "quarter": 3, "year": 12} -def period_window(period): +def period_window(period, today=None, *, cell_date=None): """⭐⭐ R7 — a bound cell's PERIOD as a `harness.windows` CUSTOM window, or None. OWNER, R7: *"must also be bounded by date time lookup"*. A Metric column carries its OWN @@ -270,14 +200,37 @@ def period_window(period): """ if not isinstance(period, dict): return None + if "ending" in period: + from harness import windows as W + period = clean_period(period) + kind = period["window"]["kind"] + mode = period["ending"]["kind"] + value = cell_date if mode == "cell" else period["ending"].get("date") + if mode == "cell" and W.ending_date(value) is None: + raise SheetRefusal("ending_cell_unreadable", period.get("ref"), + "it has no readable date.") + bounds = W.ending_bounds(kind, mode, value, today or datetime.date.today(), + period["endingStep"], period.get("endingOffset", 0), + period["ending"].get("n", 0), period["ending"].get("unit", "month")) + if bounds is None: + raise SheetRefusal("ending_invalid", W.ENDING_LABELS[mode], + "it falls before the lookback starts or outside the supported calendar.") + return {"kind": "custom", "from": bounds[0], "to": bounds[1]} if period.get("frozen") is True and period.get("from") and period.get("to"): return {"kind": "custom", "from": period["from"], "to": period["to"]} if period.get("window") is not None: return period["window"] start = str(period.get("start") or "").strip() + unit = str(period.get("unit") or "month").strip().lower() + if unit in ("day", "week"): + cleaned = clean_period(period) + if not cleaned or not cleaned.get("start"): + return None + first = datetime.date.fromisoformat(cleaned["start"]) + last = first + datetime.timedelta(days=6 if unit == "week" else 0) + return {"kind": "custom", "from": first.isoformat(), "to": last.isoformat()} if not _PERIOD_START.match(start): return None - unit = str(period.get("unit") or "month").strip().lower() months = PERIOD_MONTHS.get(unit) if not months: return None @@ -289,10 +242,8 @@ def period_window(period): return {"kind": "custom", "from": f"{start}-01", "to": last.isoformat()} -#: The column types `harness/filter_sql.py::NUMERIC_TYPES` compiles as numbers AND -#: `odoo_relational._REGISTRY_COL_TYPE` materialises as a number. The two lists have to agree or a -#: column is stored as one thing and compared as another. -MIRROR_NUMERIC_TYPES = ("currency", "int", "pct") +#: MIRROR_NUMERIC_TYPES is imported from the SQL compiler above, so admission and comparison +#: use one declaration. verify_sheet also checks the evaluator's documented client-only types. #: ⭐ THE COMPUTED KINDS THE MIRROR HAS TO BE TOLD ABOUT. `_REGISTRY_COL_TYPE` maps the ordinary #: field types and falls back to VARCHAR for everything else, so a `metric` or `rating` column @@ -490,23 +441,9 @@ def metric_window_limit(subject, cause): def clean_on_drag(raw): - """R10's two directional drag settings, or None when the caller sent none. - - ⚠ BOTH KEYS COME OUT WHENEVER ONE WENT IN. The two axes are answered together or a drag has - to guess one of them, and a guess is exactly what R10 replaced. The defaults are R10's own: - across "period", down "none". - """ - if not isinstance(raw, dict): - return None - across = str(raw.get("across") or "").strip().lower() - if across not in DRAG_ACROSS: - across = "period" - down = raw.get("down") - if isinstance(down, dict) and str(down.get("field") or "").strip(): - out_down = {"field": str(down["field"]).strip()} - else: - out_down = DRAG_DOWN_NONE - return {"across": across, "down": out_down} + """Normalize drag metadata through the core document contract.""" + from core.sheet_binding import clean_on_drag as _clean + return _clean(raw) def clean_adopted_from(raw): @@ -527,6 +464,66 @@ def clean_adopted_from(raw): return out or None +def validate(date_from, date_to, observed, *, team_id, partner_ids=None): + """Reconcile T08's four sections to live Odoo, at the exact displayed scope. + + sales.validate owns the broader sales-module reconciliation. Its public API + takes a YTD as-of, not arbitrary weeks or customer sets. Reuse its order and + line domains here for the template's narrower independent checks; never use + customer_data.validate (which validates addresses). + """ + import math + from core import odoo as O + from modules import sales + + first, last = datetime.date.fromisoformat(date_from), datetime.date.fromisoformat(date_to) + if first > last or team_id not in O.active_team_ids(): + raise ValueError("Choose a valid date range and one permitted business unit.") + # The template's basis is fixed even if the caller's dashboard is in Invoice mode. + domain = [clause for clause in sales.order_domain(date_from, date_to, team_id, partner_ids) + if clause[0] != "invoice_status"] + lines = [clause for clause in O.sale_line_domain(date_from, date_to, team_id, + partner_ids=partner_ids) + if clause[0] != "order_id.invoice_status"] + revenue = O.sum_field("sale.order", domain, "amount_untaxed") + invoiced = O.sum_field("sale.order", domain + [("invoice_status", "=", "invoiced")], "amount_untaxed") + count = O.get_odoo().search_count("sale.order", domain) + expected = {"invoiced_sales": invoiced, "confirmed_sales": revenue, "order_count": count, + "average_order_value": revenue / count if count else 0.0} + checks = [] + for key, value in expected.items(): + actual = observed.get(key) + valid = isinstance(actual, (float, int)) and not isinstance(actual, bool) and math.isfinite(actual) + checks.append({"check": key, "a": actual, "b": value, + "gap": actual - value if valid else None, + "ok": valid and abs(actual - value) <= (0 if key == "order_count" else 0.01)}) + for label, order_value, extra in (("Confirmed order and line revenue", revenue, []), + ("Fully invoiced order and line revenue", invoiced, + [("order_id.invoice_status", "=", "invoiced")])): + line_value = O.sum_field("sale.order.line", lines + extra, "price_subtotal") + checks.append({"check": label, "a": order_value, "b": line_value, + "gap": order_value - line_value, "ok": abs(order_value - line_value) <= 0.01}) + return checks + + +def sales_summary_partition(all_values, agent_values, unassigned): + """Partition additive measures; AOV must be recomputed from the summed count.""" + import math + keys = ("invoiced_sales", "confirmed_sales", "order_count", "average_order_value") + groups = [all_values, *agent_values, unassigned] + if len(agent_values) < 2 or any(not all(isinstance(g.get(k), (int, float)) + and not isinstance(g[k], bool) and math.isfinite(g[k]) for k in keys) for g in groups): + return False + if any(g["confirmed_sales"] <= 0 or g["order_count"] <= 0 or g["invoiced_sales"] <= 0 + for g in agent_values): + return False + if any(abs(sum(g[k] for g in [*agent_values, unassigned]) - all_values[k]) > + (0 if k == "order_count" else 0.01) for k in keys[:3]): + return False + return all(abs(g["average_order_value"] - (g["confirmed_sales"] / g["order_count"] + if g["order_count"] else 0)) <= 0.01 for g in groups) + + def clean_binding(raw): """Normalize a C2 `Binding`, or raise `SheetRefusal`. @@ -595,6 +592,106 @@ def clean_binding(raw): return out +def filter_cell_nodes(nodes): + """Cell value slots live on leaves at any depth, never on a fixed object path.""" + for node in nodes or (): + if not isinstance(node, dict): + continue + if isinstance(node.get("children"), list): + yield from filter_cell_nodes(node["children"]) + elif "valueRef" in node: + yield node + + +def resolve_filter_cells(nodes, read_cell): + """Resolve transient operands; retain authored references on the stored binding.""" + import math + from harness.filter_sql import MULTI_VALUE_OPS, VALUE_FREE_OPS + from harness.windows import ANCHOR_VALUE_FREE + out = [] + for node in nodes or (): + if not isinstance(node, dict): + out.append(node) + elif isinstance(node.get("children"), list): + out.append({**node, "children": resolve_filter_cells(node["children"], read_cell)}) + elif "valueRef" in node: + slot = node["valueRef"] + ref = str(slot.get("ref") or "").strip() if isinstance(slot, dict) else "" + if (node.get("op") in VALUE_FREE_OPS or node.get("op") == "within" + or node.get("dateMode") in ANCHOR_VALUE_FREE or node.get("rhs") is not None): + raise SheetRefusal("filter_cell_unreadable", ref or "(blank)", + "this condition uses a window or another operand instead of a value.") + value = read_cell(ref) + if (not isinstance(value, (str, int, float, bool)) or str(value).strip() == "" + or isinstance(value, float) and not math.isfinite(value)): + raise SheetRefusal("filter_cell_unreadable", ref or "(blank)", + "its value is blank or is not a single usable value.") + text = str(value).lower() if isinstance(value, bool) else str(value) + leaf = {**node, "value": text, "_cell_value_resolved": True} + if node.get("op") in MULTI_VALUE_OPS: + leaf["values"] = [text] + out.append(leaf) + else: + out.append(node) + return out + + +# W60-T40: explicit persisted identity, separate from both flat columns and cell value slots. +LINKED_FILTER_PREFIX = "linked:" + + +def linked_field_parts(key): + text = str(key or "") + if not text.startswith(LINKED_FILTER_PREFIX): + return None + parts = text.split(":") + if len(parts) != 3 or not all(parts[1:]): + raise SheetRefusal("linked_filter_unreadable", text, "its saved link identity is invalid.") + return parts[1], parts[2] + + +def linked_filter_ids(nodes): + return {key for key in filter_columns_named(nodes) if key.startswith(LINKED_FILTER_PREFIX)} + + +def rewrite_linked_filters(nodes, resolve): + """Preserve every conjunction; the route resolves only explicit linked leaves.""" + out = [] + for node in nodes or (): + if not isinstance(node, dict): + out.append(node) + elif isinstance(node.get("children"), list): + out.append({**node, "children": rewrite_linked_filters(node["children"], resolve)}) + elif linked_filter_ids([node]): + if not linked_field_parts(node.get("colId")) or node.get("rhs") is not None: + raise SheetRefusal("linked_filter_unreadable", ", ".join(sorted(linked_filter_ids([node]))), + "comparing columns across records is not supported. Type a value or use a cell.") + out.append(resolve(node)) + else: + out.append(node) + return out + + +def linked_id_predicate(column, ids): + """Membership using existing numeric equality, with no ID cap or deep SQL OR chain. + + isAnyOf deliberately refuses numeric columns in filter_sql. Balanced groups of eq leaves + express the same membership without changing that pure compiler or truncating its values. + """ + values = sorted({int(pid) for pid in ids}) + + def tree(lo, hi): + if hi == lo: + return {"conj": "and", "children": [ + {"colId": column, "op": "isEmpty"}, {"colId": column, "op": "isNotEmpty"}]} + if hi - lo == 1: + return {"colId": column, "op": "eq", "value": str(values[lo])} + mid = (lo + hi) // 2 + return {"conj": "or", "children": [tree(lo, mid), tree(mid, hi)]} + + return tree(0, len(values)) + + def filter_columns_named(nodes): """Every `colId` a filter tree names, at any depth, including a right-hand column reference. @@ -682,6 +779,27 @@ def compile_scope(nodes, conj, columns, *, today=None, cohort_sets=None): is what a relative date condition needs, and it is exactly the value R14's `period.ref` feeds. """ from harness.filter_sql import compile_filter_tree + import math + unresolved = linked_filter_ids(nodes) + if unresolved: + raise SheetRefusal("linked_filter_unreadable", ", ".join(sorted(unresolved)), + "the linked records have not been resolved.") + for leaf in filter_cell_nodes(nodes): + slot = leaf.get("valueRef") + ref = slot.get("ref") if isinstance(slot, dict) else "(blank)" + if leaf.get("_cell_value_resolved") is not True: + raise SheetRefusal("filter_cell_unreadable", ref, "the source value has not been read.") + try: + compiled = compile_filter_tree([leaf], conj="and", columns=columns or {}, + today=today, cohort_sets=cohort_sets) + except ValueError: + raise SheetRefusal("filter_cell_unreadable", ref, + "its condition uses a value or operator this database cannot apply.") + if (compiled is None or leaf.get("colId") not in compiled.columns_used + or not compiled.params or any(isinstance(p, float) and not math.isfinite(p) + for p in compiled.params)): + raise SheetRefusal("filter_cell_unreadable", ref, + "its condition cannot use this value on the selected column.") return compile_filter_tree(list(nodes or ()), conj=conj or "and", columns=columns or {}, today=today, cohort_sets=cohort_sets) @@ -714,6 +832,9 @@ def aggregate_sql(spec, aggregate, columns): col = (columns or {}).get(key) if not col or not col.get("sql"): raise SheetRefusal("field_unknown", key or "That column") + # An empty SUM is zero. AVG/MIN/MAX retain NULL because there is no value to report. + if fn == "sum": + return f"COALESCE(SUM({col['sql']}), 0)", key return f"{fn.upper()}({col['sql']})", key @@ -800,6 +921,31 @@ def build_ids_query(spec, columns, *, scope_nodes, scope_conj, wall_nodes=None, # ── W49-T10: the dependents provider (contract C3) ─────────────────────────────────────────── +def _sheet_cell_link(workbook_id, tab, addr): + """C2 address from stored identities, or the reason the reader cannot open this cell.""" + if not isinstance(workbook_id, str) or not workbook_id: + return None, "The workbook ID is missing. Restore the workbook before opening this cell." + tab_id = tab.get("id") + if not isinstance(tab_id, str) or not tab_id: + return None, "The tab ID is missing. Restore the tab before opening this cell." + if not isinstance(addr, str) or not _PERIOD_REF.fullmatch(addr): + return None, "The cell address is invalid. Restore its A1 address in the workbook." + # W60-T12, 2026-09-14: T10's reader falls back when a tab, row or column is hidden. + # Such a URL would open another cell, so report the cause and recovery instead. + if tab.get("visibility", "visible") != "visible": + return None, "The tab is hidden. Unhide it in the workbook before opening this cell." + letters = addr.rstrip("0123456789") + row_number = int(addr[len(letters):]) + col_number = 0 + for letter in letters: + col_number = col_number * 26 + ord(letter) - ord("A") + 1 + if row_number in (tab.get("hiddenRows") or ()): + return None, "The cell's row is hidden. Unhide the row before opening this cell." + if col_number in (tab.get("hiddenCols") or ()): + return None, "The cell's column is hidden. Unhide the column before opening this cell." + return "#/sheet?" + urlencode({"wb": workbook_id, "tab": tab_id, "cell": addr}), "" + + def sheet_dependents(st, target): """Which Sheet cells depend on `target`, as C3 `Dependent` rows. @@ -823,35 +969,37 @@ def sheet_dependents(st, target): if not kind or not key: return [] - hits, unreadable = [], 0 + hits, unreadable = [], [] for row in sheet_store.list_workbooks(st): try: book = sheet_store.get_workbook(st, row["id"]) except sheet_store.SheetRefusal: - unreadable += 1 + name = str(row.get("name") or row["id"]) + unreadable.append({ + "kind": "sheet_unreadable", + "label": f"Workbook {name} that could not be read", + "count": 1, "ids": [], "href": None, + "refusal": f"{name} could not be opened because its workbook document could not be read. " + "Restore the document or remove its stale workbook entry.", + }) continue for tab in book.get("sheets") or (): for addr, cell in (tab.get("cells") or {}).items(): if _cell_depends_on(cell, kind, key, database): - hits.append(f"{book.get('name') or book.get('id')} / " - f"{tab.get('name') or tab.get('id')}!{addr}") - - out = [] - if hits: - # ⛔ NO `label` KEY, DELIBERATELY. `dependents.dependents_report` defaults it to - # `noun("sheet_cell", count)`, which AGREES IN NUMBER; a static "Sheet cells" renders - # the user-facing refusal "Leads is used by 1 Sheet cells". Enforced, not merely - # written down: `verify_shared_views.py`'s `(W49-T11b)` leg reds on a static label. - out.append({"kind": "sheet_cell", "count": len(hits), "ids": hits}) - if unreadable: - # ⚠ REPORTED, never folded into the count above. "3 cells depend on this" and "3 cells - # depend on this and 2 workbooks could not be read" are different answers, and only the - # second one lets a person decide whether to proceed. - out.append({"kind": "sheet_unreadable", - "label": ("Workbook that could not be read" if unreadable == 1 - else "Workbooks that could not be read"), - "count": unreadable, "ids": []}) - return out + display = (f"{book.get('name') or book.get('id')} / " + f"{tab.get('name') or tab.get('id')}!{addr}") + href, reason = _sheet_cell_link(row["id"], tab, addr) + # W60-T12, 2026-09-14: C3 now requires a label and a link per cell. + # Retain the count/ids keys and display IDs; each singular label agrees + # with count=1 so legacy refusal readers still describe real dependents. + item = {"kind": "sheet_cell", "count": 1, "ids": [display], + "label": f"Sheet cell in {display}"} + item["href"] = href + if reason: + item["refusal"] = f"{display}: {reason}" + hits.append(item) + # Unreadable workbooks remain separate from the count of known dependent cells. + return hits + unreadable # ── W49-T11 (combine-owned): register the provider with the dependents registry ──────────── diff --git a/requirements.txt b/requirements.txt index 48425ff9155e75f9f6b5ada5e14160cd401cb1e7..5b4bce13e2426d8877f1fd053babb0e3923dfb4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,4 +53,7 @@ python-multipart>=0.0.31 # the line does is turn the fallback into dead weight instead of the live path. # ⚠ BOTH MANIFESTS, NOT ONE. This file's own streamlit story at the top is exactly what pinning # one of them looks like a year later. -anthropic>=0.96 +# D-384: an EXACT pin, equal to `api/requirements.txt`. This is the manifest the Dockerfile +# installs, so a floor here meant the container ran whatever PyPI resolved at build time. +# `api/verify_no_streamlit.py` asserts the two pins agree. +anthropic==0.96.0 diff --git a/web/package-lock.json b/web/package-lock.json index c5d6675a9017fa7953dd2a44a3cae2bcc3a1702e..4bc2d28f73eb6adc7b9f8a85106cdb930b3b091f 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -11,6 +11,8 @@ "@glideapps/glide-data-grid": "^6.0.3", "lodash": "^4.18.1", "marked": "^4.3.0", + "overlayscrollbars": "^2.16.0", + "overlayscrollbars-react": "^0.5.6", "react": "^18.3.1", "react-dom": "^18.3.1", "vega": "^6.3.1", @@ -2358,6 +2360,22 @@ "node": ">=0.10.0" } }, + "node_modules/overlayscrollbars": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/overlayscrollbars/-/overlayscrollbars-2.16.0.tgz", + "integrity": "sha512-N03oje/q7j93D0aLZtoCdsDSYLmhheSsv8H7oSLE7HhdV9P/bmCURtLV/KbPye7P/bpfyt/obSfDpGUYoJ0OWg==", + "license": "MIT" + }, + "node_modules/overlayscrollbars-react": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/overlayscrollbars-react/-/overlayscrollbars-react-0.5.6.tgz", + "integrity": "sha512-E5To04bL5brn9GVCZ36SnfGanxa2I2MDkWoa4Cjo5wol7l+diAgi4DBc983V7l2nOk/OLJ6Feg4kySspQEGDBw==", + "license": "MIT", + "peerDependencies": { + "overlayscrollbars": "^2.0.0", + "react": ">=16.8.0" + } + }, "node_modules/oxlint": { "version": "1.75.0", "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.75.0.tgz", diff --git a/web/package-lock.linux.sha256 b/web/package-lock.linux.sha256 index 64d84248f51f9fed45e676358e7ec8e49ec7d42f..36d50c8749e3210ba58746dd4b0bd447334df181 100644 --- a/web/package-lock.linux.sha256 +++ b/web/package-lock.linux.sha256 @@ -1,2 +1,2 @@ -3875e749397966dc170a4f83ae04cddc47a2989828cd5c46970cad69dedb255a package-lock.json -# resolved on linux/x64 by HF job 6aa6bba021047bf1b0384360 on 2026-09-13, npm ci passed there before upload. Regenerate ONLY with lockfile_linux.py (W-170). +e3e18054f50ef4119f7d9f38fdb8c712c1277df2dfa2ba5418bb683f56afb7af package-lock.json +# resolved on linux/x64 natively in the SDLC 62 wave container (lane H, W62-T33) on 2026-09-16; `npm ci` passed there before this receipt was written. `lockfile_linux.py` submits an HF job to reach a Linux box; this container IS one, so it ran `npm install --save` and `npm ci` here instead. Regenerate ONLY on linux/x64 (W-170). diff --git a/web/package.json b/web/package.json index 29d65db9918a22c1afee6ca71598b6567eaefaf2..016ca63d7dac1d3b76fbd30603574024f2d351f0 100644 --- a/web/package.json +++ b/web/package.json @@ -18,6 +18,8 @@ "@glideapps/glide-data-grid": "^6.0.3", "lodash": "^4.18.1", "marked": "^4.3.0", + "overlayscrollbars": "^2.16.0", + "overlayscrollbars-react": "^0.5.6", "react": "^18.3.1", "react-dom": "^18.3.1", "vega": "^6.3.1", diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT new file mode 100644 index 0000000000000000000000000000000000000000..cb45e778be1e10f5d6fc5799d317d53799914c03 --- /dev/null +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -0,0 +1 @@ +9c7bd0c7d54ff05e2c3907769d9a34c9ffb0d78eb7672498b1e6ca8b806cf1cd diff --git a/web/scripts/test-esm-loader.mjs b/web/scripts/test-esm-loader.mjs new file mode 100644 index 0000000000000000000000000000000000000000..e15d0e830d47a0b8e8f264281233805176bd32d2 --- /dev/null +++ b/web/scripts/test-esm-loader.mjs @@ -0,0 +1,13 @@ +// Resolve TypeScript's extensionless ESM output for source-level Node gates. +// Production is bundled by Vite; this loader gives the gate the same ESM semantics without +// pretending the valid `import.meta.url` worker code is CommonJS. +export async function resolve(specifier, context, nextResolve) { + try { + return await nextResolve(specifier, context); + } catch (error) { + if (specifier.startsWith(".") && !/\.[cm]?js(?:[?#]|$)/.test(specifier)) { + return nextResolve(`${specifier}.js`, context); + } + throw error; + } +} diff --git a/web/src/assistant/AssistantPage.tsx b/web/src/assistant/AssistantPage.tsx index ee0b529288bcb7d8d113761572ae87843f1b8f85..aaceb74e563852320cdb5f19d96f258accbd8a3f 100644 --- a/web/src/assistant/AssistantPage.tsx +++ b/web/src/assistant/AssistantPage.tsx @@ -1,6 +1,6 @@ import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { QUERY_OPEN_EVENT, signal } from "../apiContract"; +import { API_V1, CREDENTIALS, checkTenant, QUERY_OPEN_EVENT, signal, UNAUTHORIZED_EVENT } from "../apiContract"; import { queryCitationLabel } from "../customer-grid/queryPreview"; import { retryEmit } from "../inbox/inboxModel"; import { queryGroups, QueryRail } from "../query/queryParts"; @@ -27,6 +27,81 @@ const QueryWorkspace = lazy(() => import("../query/QueryPage")); export interface AssistantPageProps { granted: NavEntry[]; } +/** Routing rides on /query/tool. Its shared result union belongs to another lane, so keep + * the disclosure and refusal extension local and read it through the single cast in ask(). */ +type RoutedTo = { database: string; label: string; reason: string; auto: boolean; + alternatives: Array<{ database: string; label: string }> }; +type RoutingDisclosure = { + database: string; + label: string; + alternatives: Array<{ database: string; label: string }>; + answered: boolean; +}; +// W61-T30: the drafting extension stays local while the shared API union belongs to another lane. +type SemanticDraft = { + database: string; label: string; canApprove: boolean; sentToAdmins: boolean; + state: "pending" | "approved"; requestedBy: string; + approvedBy?: string | null; approvedAt?: string | null; approvalLine?: string; sentMessage: string; + fields: Array<{ field: string; label: string; why: string; role: string; dataType: string; + sensitivity: string; sample: { redacted: boolean; values: string[]; sample: string[] } }>; +}; +type RoutedQueryResult = (Awaited> | { + status: "refused"; wording: string; speakable: string; artifacts: null; +}) & { routedTo?: RoutedTo; draft?: SemanticDraft }; + +async function draftRequest(database: string, fields?: string[], question?: string): Promise<{ + state?: "pending" | "approved"; draft?: SemanticDraft; approvalLine?: string; sentMessage?: string; +}> { + const response = await fetch(`${API_V1}/query/semantics/${encodeURIComponent(database)}/${fields ? "approve" : "draft"}`, { + credentials: CREDENTIALS, method: fields ? "POST" : "GET", + ...(fields ? { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ fields, question }) } : {}), + }); + if (checkTenant(response)) throw new Error("The workspace changed. Open this chat again."); + if (response.status === 401) signal(UNAUTHORIZED_EVENT); + if (!response.ok) throw new Error("The field request could not be completed, so please try again."); + return response.json(); +} + +/** + * R47 / R6 — the answer names its own source, and the override lives here rather than in front of + * the question. One line: "Answered from · Change". + * + * ⛔ CHANGE IS A SELECT, NOT A ROW OF BUTTONS, AND THAT WAS MEASURED ON A RUNNING TENANT. This + * first shipped as one quiet button per alternative; tenant #0 can read 28 databases, so opening + * Change pushed the answer six rows down the screen behind a wall of names, eleven of which read + * "Formula contract". A select collapses the whole override into the one word the ruling names, + * hands the browser the scrolling, the keyboard and the screen reader, and is the same control + * the model picker one line below already is. Its value is held at "" so the word Change survives + * every switch, and `onChange` ignores that empty value. + * + * ⚠ A refusal renders the control ALONE: the refusal sentence under it already names the database + * it could not answer from, so "Answered from" would be the one thing on the surface that is not + * true. The override still has to be reachable there, which is the state that needs it most. + */ +function RoutingLine({ disclosure, onSwitch, busy }: { + disclosure?: RoutingDisclosure | null; onSwitch?: (database: string) => void; busy?: boolean; +}) { + if (!disclosure?.database || !disclosure.alternatives.length) return null; + return
+ {disclosure.answered ? <> + Answered from {disclosure.label} + {onSwitch ? : null} + : null} + {onSwitch ? ( + + ) : null} +
; +} + /** * ⭐⭐ R14 / CONTRACT C2 — ONE MODULE, ONE ROUTE, TWO LISTS. Owner, 2026-08-16: *"Combine the AI * assistant module AND the Query module. In the secondary navigation at the top, just above @@ -175,12 +250,18 @@ export function AssistantMessage({ onRate, identity, governedArtifacts, + disclosure, + onSwitch, + busy, }: { message: QueryMessage; view?: SavedQuery; citations: QueryCitation[]; onPreview: (id: string) => void; /** Optional so the message can be rendered outside a live conversation without a rating door. */ onRate?: (id: string, rating: "up" | "down" | null, reason?: string) => void; identity?: QueryModelIdentity; governedArtifacts?: QueryToolArtifacts; + disclosure?: RoutingDisclosure | null; + onSwitch?: (database: string) => void; + busy?: boolean; }) { const [openSources, setOpenSources] = useState(false); const [askWhy, setAskWhy] = useState(false); @@ -202,11 +283,17 @@ export function AssistantMessage({ ); } + if (message.reason === "query_no_source_matched") { + return
+

{message.content}

+
; + } const provenance = message.provenance; const sourceFooter = answerSourceFooter(provenance, matching); const numeric = message.numeric; return (
+

{message.content}

Provider {identity?.provider || message.model || "Not reported"} @@ -369,6 +456,7 @@ export default function AssistantPage({ granted }: AssistantPageProps) { const [index, setIndex] = useState(initialIndex); const [activeThread, setActiveThread] = useState(""); const [target, setTarget] = useState(""); + const [targetTouched, setTargetTouched] = useState(false); const [model, setModel] = useState("auto"); const [question, setQuestion] = useState(""); const [busy, setBusy] = useState(false); @@ -384,8 +472,23 @@ export default function AssistantPage({ granted }: AssistantPageProps) { const [alignment, setAlignment] = useState; maxSelections: number; + routed?: RoutedTo; + draft?: SemanticDraft; sent?: boolean; }>(null); const [alignmentFields, setAlignmentFields] = useState([]); + const [draftBusy, setDraftBusy] = useState(false); + const [draftProblem, setDraftProblem] = useState(""); + const [approvalLine, setApprovalLine] = useState(""); + const composerRef = useRef(null); + const alignmentRef = useRef(null); + const requestInFlight = useRef(false); + const alignmentGeneration = useRef(0); + const lastDraftCheck = useRef(0); + useEscapeClose(Boolean(alignment), () => { + alignmentGeneration.current += 1; + setAlignment(null); setAlignmentFields([]); setDraftProblem(""); + composerRef.current?.focus(); + }, { container: () => alignmentRef.current }); const openCancel = useRef<(() => void) | null>(null); const foot = useRef(null); /** A counter, not a random id: the optimistic message only has to be unique within this page. */ @@ -393,6 +496,12 @@ export default function AssistantPage({ granted }: AssistantPageProps) { const databases = useMemo(() => databaseEntries(granted).filter((entry) => entry.kind !== "group"), [granted]); const labels = useMemo(() => new Map(databases.map((entry) => [entry.key, entry.label])), [databases]); + const disclosureFor = useCallback((database: string, answered: boolean): RoutingDisclosure | null => { + if (!labels.has(database)) return null; + return { database, label: labels.get(database)!, answered, + alternatives: databases.filter((entry) => entry.key !== database) + .map((entry) => ({ database: entry.key, label: entry.label })) }; + }, [databases, labels]); /** * ⭐⭐ INSTRUCTION 12 / R11 — NO SOURCE PICKER AT ALL. Owner: *"the Assistant should always have * access to all database that the user have access to. So it doesn't need the database clicks @@ -432,8 +541,8 @@ export default function AssistantPage({ granted }: AssistantPageProps) { return () => controller.abort(); }, []); useEffect(() => () => openCancel.current?.(), []); - // There is no picker any more, so the target DEFAULTS to the first database the caller can - // reach: without this the composer would open with no target and a Send button nothing enables. + // Keep a permitted display target for the placeholder and Send button. Only a touched picker + // chooses the request's database; an untouched picker lets the server route the question. useEffect(() => { const permitted = databases.map((entry) => entry.key); setTarget((current) => permitted.includes(current) ? current : databases[0]?.key ?? ""); @@ -442,8 +551,10 @@ export default function AssistantPage({ granted }: AssistantPageProps) { // clarification instead of replaying the token against a different database. useEffect(() => { if (alignment && alignment.database !== target) { + alignmentGeneration.current += 1; setAlignment(null); setAlignmentFields([]); + setDraftProblem(""); setApprovalLine(""); } }, [alignment, target]); // The conversation reads bottom-up, like every chat the owner named. @@ -520,12 +631,18 @@ export default function AssistantPage({ granted }: AssistantPageProps) { }, [mode, activeQuery, queryList, selectQuery]); const newChat = useCallback(() => { + alignmentGeneration.current += 1; setActiveThread(""); setQuestion(""); setProblem(""); + setDraftProblem(""); setApprovalLine(""); + setTargetTouched(false); setAlignment(null); setAlignmentFields([]); }, []); /** Opening a past chat restores the database it ran against and the model it ran with (R6). */ const openThread = useCallback((id: string) => { + alignmentGeneration.current += 1; setActiveThread(id); setProblem(""); + setDraftProblem(""); setApprovalLine(""); + setTargetTouched(false); setAlignment(null); setAlignmentFields([]); const row = index.threads.find((thread) => thread.id === id); if (!row) return; const permitted = databases.map((entry) => entry.key); @@ -548,30 +665,66 @@ export default function AssistantPage({ granted }: AssistantPageProps) { * the model as if it had [[undo-capture-before-the-write]]. The typed text goes back in the * composer too: losing somebody's sentence to a failed request is the second injury. */ - const ask = useCallback(async (approvedFields?: string[]) => { - const text = approvedFields && alignment ? alignment.prompt : question.trim(); - const database = approvedFields && alignment ? alignment.database : target; - if (!text || !database || busy) return; - setBusy(true); setProblem(""); - let governed: Awaited>; + const ask = useCallback(async (approvedFields?: string[], override?: { database: string; prompt: string }) => { + const replayDraft = !override && alignment?.draft && (approvedFields || alignment.sent); + const replay = (approvedFields || replayDraft) && alignment; + const text = override?.prompt ?? (replay ? replay.prompt : question.trim()); + const database = override?.database ?? (replay ? replay.database : targetTouched ? target : ""); + if (!text || !target || busy || requestInFlight.current) return; + requestInFlight.current = true; + const generation = alignmentGeneration.current; + setBusy(true); setProblem(""); setDraftProblem(""); + if (!replay) setApprovalLine(""); + let governed: RoutedQueryResult; try { - governed = await runQueryTool({ database, prompt: text, - ...(approvedFields ? { fieldKeys: approvedFields, resumeToken: alignment?.resumeToken } : {}) }); - } catch (error) { - setBusy(false); setProblem(error instanceof Error ? error.message : "Query capability check failed."); return; + // One narrow cast for routing and drafts; the shared result union stays in its owning lane. + const runGoverned = async (input: Parameters[0]) => + await runQueryTool(input) as RoutedQueryResult; + if (replayDraft) { + const current = await draftRequest(database); + if (generation !== alignmentGeneration.current) { + requestInFlight.current = false; setBusy(false); return; + } + if (current.draft?.approvalLine) setApprovalLine(current.draft.approvalLine); + } + governed = await runGoverned({ database, prompt: text, + ...(approvedFields && !replayDraft ? { fieldKeys: approvedFields, resumeToken: alignment?.resumeToken } : {}) }); + if (replayDraft && governed.status === "clarification" && !governed.draft) { + // Approval changes the choice set. Only this fresh clarification can authenticate resume. + const choices = new Set(governed.alignmentQuestion.choices.map((choice) => choice.field)); + const fields = (approvedFields || alignmentFields).filter((field) => choices.has(field)); + if (fields.length) governed = await runGoverned({ database, prompt: text, + fieldKeys: fields, resumeToken: governed.resumeToken }); + } + } catch { + requestInFlight.current = false; setBusy(false); + if (generation !== alignmentGeneration.current) return; + if (alignment?.draft) setDraftProblem("The field request could not be completed, so please try again."); + else setProblem("This database could not answer right now. Please try again."); + return; } + if (generation !== alignmentGeneration.current) { + requestInFlight.current = false; setBusy(false); return; + } + const routed = replay && alignment?.routed ? alignment.routed : governed.routedTo; + const chosen = governed.routedTo?.database ?? database; + if (chosen) setTarget(chosen); if (governed.status === "clarification") { - setBusy(false); setAlignmentFields([]); setAlignment({ prompt: text, database, + requestInFlight.current = false; setBusy(false); + const sent = Boolean(replayDraft && alignment?.sent && governed.draft?.state === "pending"); + if (!sent) setAlignmentFields([]); + setAlignment({ prompt: text, database: chosen, routed, draft: governed.draft, sent, resumeToken: governed.resumeToken, question: governed.alignmentQuestion.question, choices: governed.alignmentQuestion.choices, maxSelections: governed.alignmentQuestion.maxSelections }); return; } setAlignment(null); + composerRef.current?.focus(); const pendingId = `pending_${(pending.current += 1)}`; const optimistic: QueryMessage = { id: pendingId, threadId: activeThread, role: "user", content: text, - createdAt: new Date().toISOString(), targetDatabase: database, - sources: [database], requestedModel: model, + createdAt: new Date().toISOString(), targetDatabase: chosen, + sources: chosen ? [chosen] : [], requestedModel: model, }; const withoutPending = (rows: QueryMessage[]) => rows.filter((row) => row.id !== pendingId); setQuestion(""); @@ -579,17 +732,22 @@ export default function AssistantPage({ granted }: AssistantPageProps) { // ⛔ THE TARGET, ALONE. `routes_query.py::_sources` runs EVERY key in this array through the // read scope with no try/except, so one refusal kills the whole request; and the model is only // ever handed the target's snapshot anyway. A wider array buys nothing and breaks the turn. - const result = await submitChat({ question: text, database, sources: [database], + // Automatic refusals go through chat's same routing branch, which saves a plain reply + // without trying to read an unavailable database. Successful replies use the chosen target. + const chatDatabase = governed.status === "refused" && routed?.auto ? "" : chosen; + const result = await submitChat({ question: text, database: chatDatabase, sources: chosen ? [chosen] : [], ...(activeThread ? { threadId: activeThread } : {}), model }); - setBusy(false); + requestInFlight.current = false; setBusy(false); if (!result.ok) { setIndex((current) => ({ ...current, messages: withoutPending(current.messages) })); setQuestion(text); - setProblem(result.message); + setProblem("This question could not be answered right now. Please try again."); return; } setActiveThread(result.value.thread.id); - if (result.value.view) setArtifactByView((current) => ({ ...current, [result.value.view!.id]: governed.artifacts })); + if (result.value.view && governed.status === "answered") { + setArtifactByView((current) => ({ ...current, [result.value.view!.id]: governed.artifacts })); + } setIndex((current) => ({ ...current, threads: [result.value.thread, ...current.threads.filter((row) => row.id !== result.value.thread.id)], @@ -599,7 +757,54 @@ export default function AssistantPage({ granted }: AssistantPageProps) { views: result.value.view ? [result.value.view, ...current.views.filter((row) => row.id !== result.value.view?.id)] : current.views, citations: [...current.citations, ...result.value.citations], })); - }, [activeThread, alignment, busy, model, question, target]); + }, [activeThread, alignment, alignmentFields, busy, model, question, target, targetTouched]); + + const submitDraft = async () => { + if (!alignment?.draft || !alignmentFields.length || busy || draftBusy || requestInFlight.current) return; + const chosenFields = [...alignmentFields]; + const generation = alignmentGeneration.current; + requestInFlight.current = true; + setDraftBusy(true); setDraftProblem(""); + try { + const result = await draftRequest(alignment.database, chosenFields, alignment.prompt); + if (generation !== alignmentGeneration.current) return; + if (result.state === "approved") { + setApprovalLine(result.approvalLine || ""); + requestInFlight.current = false; + await ask(chosenFields); + } else if (result.state === "pending") { + lastDraftCheck.current = Date.now(); + setAlignment((current) => current ? { ...current, sent: true } : null); + } else { + setDraftProblem("The field request could not be completed, so please try again."); + } + } catch { + if (generation === alignmentGeneration.current) + setDraftProblem("The field request could not be completed, so please try again."); + } finally { + requestInFlight.current = false; + setDraftBusy(false); + } + }; + + useEffect(() => { + if (!alignment?.draft || !alignment.sent) return; + const recheck = () => { + if (busy || draftBusy || requestInFlight.current || Date.now() - lastDraftCheck.current < 10000) return; + lastDraftCheck.current = Date.now(); + void ask(); + }; + window.addEventListener("focus", recheck); + return () => window.removeEventListener("focus", recheck); + }, [alignment, ask, busy, draftBusy]); + + const switchDatabase = (database: string, prompt: string) => { + alignmentGeneration.current += 1; + setDraftProblem(""); setApprovalLine(""); + setTarget(database); setTargetTouched(true); setAlignment(null); setAlignmentFields([]); + // Pass the clicked values directly: React state changes are applied after this handler. + void ask(undefined, { database, prompt }); + }; /** ⛔ The chat goes; the views it built STAY (they live in Query). Two-click, in the row. */ const removeThread = useCallback(async (id: string) => { @@ -634,27 +839,70 @@ export default function AssistantPage({ granted }: AssistantPageProps) { selectQuery(qid); }, [pickMode, selectQuery]); + /** + * The question Change re-asks for a past answer. + * + * ⛔ NOT "the nearest message above it". `_submit` stamps the question and the answer with ONE + * `createdAt`, and the index sorts on that key, so a turn's two rows arrive in whichever order + * the stored map held them: a thread whose answer sits ABOVE its question left Change with no + * question to re-ask and therefore unrendered. The shared timestamp IS the pairing, so it is + * what this matches on; the scan above is the fallback for a row that predates that shape. + */ + const questionFor = useCallback((position: number) => { + const row = messages[position]; + if (!row) return ""; + const turn = messages.find((item) => item.role === "user" && item.createdAt === row.createdAt); + if (turn) return turn.content; + for (let index = position - 1; index >= 0; index -= 1) { + if (messages[index].role === "user") return messages[index].content; + } + return ""; + }, [messages]); + const empty = messages.length === 0; const composer = (

- {alignment ?
-

Clarify

{alignment.question}

-
{alignment.choices.map((choice) => { + {approvalLine ?

{approvalLine}

: null} + {alignment ?
+ switchDatabase(database, alignment.prompt)} /> +

{alignment.draft ? "Database fields" : "Clarify"}

+

+ {alignment.sent ? alignment.draft?.sentMessage : alignment.question} +

+
{alignment.choices.filter((choice) => !alignment.sent || alignmentFields.includes(choice.field)).map((choice) => { const on = alignmentFields.includes(choice.field); - return ; + : current.length < alignment.maxSelections ? [...current, choice.field] : current)}> + {choice.label} + {drafted ? <> + {drafted.role} · {drafted.dataType} + {redacted ? "Redacted" + : [...drafted.sample.values, ...drafted.sample.sample].slice(0, 3).join(", ") || "No populated values"} + : null} + ; })}
- + {draftProblem ?

{draftProblem}

: null} + {alignment.sent ? + : }
: null}