diff --git a/RELEASES.json b/RELEASES.json index 44b8ae3ea3e3c65fea2e0fbb9fa997f8e364a40d..719e78679b54f68d4ee1ca48cd3021d4c69a4366 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,437 +1,4 @@ { - "current": "40dceba", - "releases": [ - { - "version": "v73", - "sha": "bb94de5", - "date": "2026-09-13", - "subject": "release v73" - }, - { - "version": "v71", - "sha": "96143dc", - "date": "2026-09-04", - "subject": "release v71 - wave 51, QA'd 24 PASS / 6 FAIL / 5 UNPROVEN" - }, - { - "version": "v70", - "sha": "eedfb6a", - "date": "2026-09-03", - "subject": "release v70 - wave 50 (grid analytics, the Sheet as a spreadsheet, view trash and audit)" - }, - { - "version": "v69", - "sha": "8598f56", - "date": "2026-09-02", - "subject": "release v69: wave48-continuation validated + hotfixes v67/v68/d4e3e31 + go-live QA repairs (Supplier option limit, optio" - }, - { - "version": "v68", - "sha": "62b065a", - "date": "2026-09-01", - "subject": "release v68: unified holiday date offsets" - }, - { - "version": "v67", - "sha": "3b22419", - "date": "2026-09-01", - "subject": "release v67" - }, - { - "version": "v66", - "sha": "73c94f0", - "date": "2026-08-31", - "subject": "release v66" - }, - { - "version": "v65", - "sha": "9f6cfed", - "date": "2026-08-31", - "subject": "release v65" - }, - { - "version": "v64", - "sha": "bb8985c", - "date": "2026-08-31", - "subject": "release v64" - }, - { - "version": "v63", - "sha": "ec75cb8", - "date": "2026-08-31", - "subject": "release v63" - }, - { - "version": "v62", - "sha": "2c8f702", - "date": "2026-08-31", - "subject": "release v62" - }, - { - "version": "v61", - "sha": "bbc4c07", - "date": "2026-08-30", - "subject": "v61 - wave 46 hotfix: D-590 schema overlay reaches the grid read (all three assemblies), D-599 route to zone association" - }, - { - "version": "v60", - "sha": "dd1b062", - "date": "2026-08-30", - "subject": "v60 - wave 46" - }, - { - "version": "v59", - "sha": "3cb73ce", - "date": "2026-08-29", - "subject": "release v59" - }, - { - "version": "v58", - "sha": "c526ced", - "date": "2026-08-29", - "subject": "release v58" - }, - { - "version": "v57", - "sha": "4859c03", - "date": "2026-08-29", - "subject": "release v57" - }, - { - "version": "v56", - "sha": "2970b72", - "date": "2026-08-28", - "subject": "release v56" - }, - { - "version": "v55", - "sha": "b766b90", - "date": "2026-08-28", - "subject": "release v55" - }, - { - "version": "v54", - "sha": "6a8e99a", - "date": "2026-08-27", - "subject": "release v54" - }, - { - "version": "v53", - "sha": "89ed2cb", - "date": "2026-08-24", - "subject": "release v53" - }, - { - "version": "v52", - "sha": "8263493", - "date": "2026-08-23", - "subject": "release v52" - }, - { - "version": "v51", - "sha": "5740d20", - "date": "2026-08-23", - "subject": "release v51" - }, - { - "version": "v50", - "sha": "8027c83", - "date": "2026-08-23", - "subject": "release v50" - }, - { - "version": "v49", - "sha": "9a14c01", - "date": "2026-08-23", - "subject": "release v49" - }, - { - "version": "v48", - "sha": "3c69fd9", - "date": "2026-08-23", - "subject": "release v48" - }, - { - "version": "v47", - "sha": "917f9a0", - "date": "2026-08-23", - "subject": "release v47" - }, - { - "version": "v46", - "sha": "a8dc531", - "date": "2026-08-23", - "subject": "release v46" - }, - { - "version": "v45", - "sha": "0b2b501", - "date": "2026-08-23", - "subject": "release v45" - }, - { - "version": "v44", - "sha": "a7877aa", - "date": "2026-08-23", - "subject": "Hotfix: allow renaming the active Map view" - }, - { - "version": "v43", - "sha": "e2ba461", - "date": "2026-08-23", - "subject": "release v43" - }, - { - "version": "v42", - "sha": "25b885e", - "date": "2026-08-23", - "subject": "release v42" - }, - { - "version": "v41", - "sha": "1d6adcc", - "date": "2026-08-22", - "subject": "v41 vertical route navigation" - }, - { - "version": "v40", - "sha": "1451837", - "date": "2026-08-22", - "subject": "release v40 route UI correction" - }, - { - "version": "v39", - "sha": "a78bdf0", - "date": "2026-08-21", - "subject": "W39 clean stale migration targets" - }, - { - "version": "v38", - "sha": "a556631", - "date": "2026-08-21", - "subject": "W39 reversible Live store cutover" - }, - { - "version": "v37", - "sha": "1ddd8cf", - "date": "2026-08-21", - "subject": "W39: coalesce derived workspace reads" - }, - { - "version": "v36", - "sha": "51db3b7", - "date": "2026-08-21", - "subject": "W39 share-field oid repair" - }, - { - "version": "v35", - "sha": "0f64212", - "date": "2026-08-21", - "subject": "W39 visual QA fixture" - }, - { - "version": "v34", - "sha": "cb4caa1", - "date": "2026-08-21", - "subject": "W39 Staging PostgreSQL egress controls" - }, - { - "version": "v33", - "sha": "2b115e5", - "date": "2026-08-21", - "subject": "W39: stop nonproduction Postgres background egress" - }, - { - "version": "v32", - "sha": "b47b27c", - "date": "2026-08-21", - "subject": "W39: repair Staging Postgres projection" - }, - { - "version": "v31", - "sha": "8880a49", - "date": "2026-08-21", - "subject": "v31: Wave 39 staging candidate" - }, - { - "version": "v30", - "sha": "71776ca", - "date": "2026-08-20", - "subject": "v30: W39-T29 Member-owned View first-share hotfix" - }, - { - "version": "v29", - "sha": "55ced50", - "date": "2026-08-18", - "subject": "v29: the build staging has been running since 2026-08-18 (55ced50)." - }, - { - "version": "v28", - "sha": "2793e4b", - "date": "2026-08-18", - "subject": "release v28" - }, - { - "version": "v27", - "sha": "c05a794", - "date": "2026-08-18", - "subject": "release v27" - }, - { - "version": "v25", - "sha": "bc24829", - "date": "2026-08-14", - "subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)" - }, - { - "version": "v26", - "sha": "6a642e1", - "date": "2026-08-14", - "subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)" - }, - { - "version": "v24", - "sha": "514d829", - "date": "2026-08-09", - "subject": "release v24" - }, - { - "version": "v23", - "sha": "44cc283", - "date": "2026-08-09", - "subject": "release v23" - }, - { - "version": "v22", - "sha": "f14e199", - "date": "2026-08-09", - "subject": "release v22" - }, - { - "version": "v21", - "sha": "5fdffce", - "date": "2026-08-09", - "subject": "release v21" - }, - { - "version": "v20", - "sha": "442e4a1", - "date": "2026-08-09", - "subject": "release v20" - }, - { - "version": "v19", - "sha": "94cb4a4", - "date": "2026-08-09", - "subject": "release v19" - }, - { - "version": "v18", - "sha": "0106ef9", - "date": "2026-08-09", - "subject": "release v18" - }, - { - "version": "v17", - "sha": "308240c", - "date": "2026-08-09", - "subject": "release v17" - }, - { - "version": "v16", - "sha": "a804df5", - "date": "2026-08-09", - "subject": "release v16" - }, - { - "version": "v15", - "sha": "6b84d72", - "date": "2026-08-09", - "subject": "release v15" - }, - { - "version": "v14", - "sha": "bee5eaf", - "date": "2026-08-07", - "subject": "release v14" - }, - { - "version": "v13", - "sha": "314c0e3", - "date": "2026-08-06", - "subject": "release v13" - }, - { - "version": "v12", - "sha": "4908f39", - "date": "2026-08-06", - "subject": "release v12" - }, - { - "version": "v11", - "sha": "3126617", - "date": "2026-08-05", - "subject": "release v11" - }, - { - "version": "v10", - "sha": "43f8042", - "date": "2026-08-05", - "subject": "release v10" - }, - { - "version": "v9", - "sha": "2e26f57", - "date": "2026-08-05", - "subject": "release v9" - }, - { - "version": "v8", - "sha": "8073b3e", - "date": "2026-08-05", - "subject": "release v8" - }, - { - "version": "v7", - "sha": "b1c5a7e", - "date": "2026-08-05", - "subject": "release v7" - }, - { - "version": "v6", - "sha": "a344020", - "date": "2026-08-05", - "subject": "release v6" - }, - { - "version": "v5", - "sha": "c0b0fbb", - "date": "2026-08-05", - "subject": "release v5" - }, - { - "version": "v4", - "sha": "de5037f", - "date": "2026-08-05", - "subject": "release v4" - }, - { - "version": "v3", - "sha": "6b9fa62", - "date": "2026-08-05", - "subject": "release v3" - }, - { - "version": "v2", - "sha": "fd05861", - "date": "2026-08-04", - "subject": "release v2" - }, - { - "version": "v1", - "sha": "5b4e2c4", - "date": "2026-08-04", - "subject": "release v1" - } - ] + "current": "0eda914", + "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index 078d508af88db42c3d1262ab459194361176377a..1e98260efaee6a48bbd314d3cd5eac911a4c1011 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -40dceba +0eda914 diff --git a/api/automation_engine.py b/api/automation_engine.py index 12e563752eefb872c876b18ee09e70ef3ab7040d..7d7e986f3580900519a7523825e5a3e77d946fd8 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -14553,6 +14553,74 @@ def relation_cells(rt, table_key, tables=None): return changes, limits +def related_metric_values(rt, table_key, rows, fields, offer, related, *, team_id=None, + today=None): + """Read-only linked cells at customer, product or declared entity grain. + + The host supplies the same resolved membership as its rows door and its walled catalogue. + A request-local memo never survives a parent, account, permission or tenant change. + """ + from core import measure_resolve, relations + from harness import semantic as sem, windows + import time + + rt.assert_datastore_matches() + today = today or time.strftime('%Y-%m-%d') + by_key = {m['key']: m for m in offer} + out = relations.MeasureAnswer({r['pid']: {f['key']: None for f in fields} for r in rows}) + memo = {} + for field in fields: + spec = field.get('measure') or {} + key = spec.get('key') + if key not in by_key: + continue # The host removes unavailable catalogue entries before serialization. + try: + bounds = windows.resolve(spec.get('window'), today) + if bounds is None: + raise ValueError('The metric date window cannot be resolved. Choose a valid period.') + if table_key == 'customer_data': + errors = [] + values = measure_resolve.column_values( + [field], team_id, [r['pid'] for r in rows], today, today, memo, + on_error=lambda _where, exc: errors.append(exc), related=related) + if errors: + raise ValueError('This metric could not be read for the linked record. Try again after the database is ready.') + out.refusals.extend(getattr(values, 'refusals', ())) + for pid, cells in values.items(): + out[pid].update(cells) + continue + topic = sem.topic_for_grid(table_key) + cache_key = (topic, key, bounds, team_id) + if cache_key not in memo: + memo[cache_key] = sem.entity_measure_values( + topic, [key], date_from=bounds[0], date_to=bounds[1], team_id=team_id, + offer=offer, related=related) + values = memo[cache_key] + markers = getattr(values, 'refusals', ()) + if markers: + out.refusals.extend(dict(marker, field=field['key']) for marker in markers) + continue # Never zero-fill a refused answer or expose its unscoped values. + for row in rows: + identity = row.get('code') if table_key == 'product_data' else row['pid'] + if identity in (None, ''): + raise ValueError('The metric identity cannot be read for this record.') + cell = sem.entity_zero_fill(values.get(identity), [key], offer) + out[row['pid']][field['key']] = cell.get(key) + except Exception as exc: + message = (str(exc) if isinstance(exc, (relations.PinRefusal, sem._EntityMeasureError)) else + 'This metric cannot be read for the linked record. Check its period and source binding, then try again.') + out.refusals.append(dict(code='metric_scope_unavailable', message=message, + path=related.path, field=field['key'], scoped=False)) + # Customer resolvers deliberately retain diagnostics even when their low-level pin refuses. + for marker in out.refusals: + marker['message'] = marker['message'].replace('Values are unscoped.', 'Values are unavailable.').replace( + 'Metric values are unscoped.', 'Metric values are unavailable.') + for cells in out.values(): + if marker.get('field') in cells: + cells[marker['field']] = None + return out + + def related_product_metric_values(rt, customer_id, products, fields, offer, *, team_id=None, agent=None, today=None): """Read-only customer × real product metrics; the caller supplies walled server rows. diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index 7d340dd7df920ddac75041612c607103275f5ac3..c6303734a0a2887e154bbebabe30991ef1df36fd 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -94,6 +94,137 @@ def assert_safe_status(payload): return payload +def first_load_status(tenant_key, rt=None): + """Writer-side progress helper; request status uses sync_state's read-only cursor.""" + from harness import amazon_store + return assert_safe_status(amazon_store.first_load_status(tenant_key, rt=rt)) + + +def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): + """Each Amazon table's sync state, or None when this mirror cannot be read. + + Read only: never _connect_tenant_mirror or first_load_status, which issue DDL. + Missing bookkeeping is unknown, not a measured zero. First-load numbers describe the + whole connection, because that is the scope of the producer's durable progress marker. + """ + import json + from datetime import datetime, timezone + import duckdb + from harness import amazon_store as az + from routes_odoo_tables import _sync_entities, stuck_verdict + + def stamp(value): + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + return (parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)).astimezone(timezone.utc) + except (ValueError, TypeError, OverflowError): + return None + + def iso(value): + return value.isoformat().replace("+00:00", "Z") if value else None + + cadence = cadence_seconds if cadence_seconds is not None else 1800 + try: + path = az._tenant_mirror_path(tenant_key, rt=rt) + if not path.is_file(): + return assert_safe_status(None) + # ⛔⛔ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES + # `ConnectionException: Can't open a connection to same database file with a different + # configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global + # analytical mirror (`datastore.DB_PATH`), so the swallowed refusal below turned this whole + # screen into a permanent "sync status is unknown" on exactly the deployment the ticket is + # for. Measured on staging 2026-09-14, W60-T35. `datastore.ro_con()` is the shipped + # per-thread READ cursor on that same instance, and it takes no lock against the writer. + from harness import datastore as _ds + if path.resolve() == _ds.DB_PATH.resolve(): + # `ro_cursor()` is a cursor on the instance this process ALREADY holds: no second + # connection, no lock contention with the writer, and no DDL. Deliberately NOT + # `ro_con()`, which refuses until `ready()` and would report the whole screen as + # unknown whenever the Odoo backfill is mid-flight, and NOT `datastore.connect()`, + # which issues `CREATE TABLE IF NOT EXISTS`. + entities = _sync_entities(_ds.ro_cursor(), cadence) + else: + with duckdb.connect(str(path), read_only=True) as cur: + entities = _sync_entities(cur, cadence) + except Exception: # noqa: BLE001 + # No exception text: paths, provider payloads and credentials never become status copy. + return assert_safe_status(None) + if entities is None: + return assert_safe_status(None) + + clock = stamp(now) if now is not None else datetime.now(timezone.utc) + states, bad_ticks = {}, set() + for table in az.TABLES: + tick = entities.get(az.entity_key("sync_tick_" + table)) + if tick: + updated = stamp(tick.get("updatedAt")) + if updated is None or tick.get("phase") not in ("attempted", "done", "running"): + bad_ticks.add(table) + else: + states[table] = {"state": tick["phase"], "updated_at": iso(updated)} + # An unreadable input cannot support a cadence promise for any table in the rotation. + plan = az.sync_plan(states, now=clock) if not bad_ticks and clock is not None else None + first_load, load_unknown = None, "" + load = entities.get(az.entity_key(az.FIRST_LOAD_STATUS_KEY)) + if load: + try: + progress = json.loads(load.get("cursor") or "") + remaining, eta = progress["remaining"], progress["etaSeconds"] + if (type(remaining) is not int or not 0 <= remaining <= 2**53 - 1 + or type(eta) is not int or not 0 <= eta <= 2**53 - 1 + or (load.get("phase") == "done" and remaining > 0)): + raise ValueError("Invalid first-load progress") + if remaining > 0: + first_load = {"remaining": remaining, "etaSeconds": eta, + "cause": az.first_load_eta_cause(remaining), + "needsAttention": load.get("phase") == "failed"} + except (ValueError, TypeError, KeyError, OverflowError, RecursionError): + load_unknown = "First-load progress is unknown because its saved marker could not be read. Retry sync to refresh it." + elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"): + load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass." + + answer = {} + for table in az.TABLES: + landed = entities.get(az.entity_key(table)) or {} + tick = entities.get(az.entity_key("sync_tick_" + table)) or {} + watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {} + rows = landed.get("rows") + if type(rows) is not int or rows < 0: + rows = None + successes = [when for marker in (landed, tick, watermark) + if marker.get("phase") == "done" + and (when := stamp(marker.get("updatedAt"))) is not None] + stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or "" + if tick.get("phase") == "attempted": + # The shared stall helper calls an active phase 'running'; Amazon calls it 'attempted'. + updated = stamp(tick.get("updatedAt")) + stuck, stuck_cause = stuck_verdict( + "running", iso(updated), cadence, + now=clock.replace(tzinfo=None) if clock else None) + reason = plan["deferred"].get(table) if plan else None + # Only the planner's explicit due stamp is a due date. Minimum stagger spacing is not + # a promise about when a later resident tick will select this table. + next_due = stamp(reason.partition("next due ")[2]) if reason else None + unknown = [load_unknown] if load_unknown else [] + if rows is None: + unknown.append("This container has no readable row count for this table.") + if plan is None: + unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.") + answer[table] = { + "rowsLoaded": rows, + "lastSuccess": iso(max(successes)) if successes else None, + "inProgress": tick.get("phase") == "attempted" and stuck is False, + "deferred": {"reason": reason} if reason else None, + "nextDue": iso(next_due), + "firstLoad": first_load, + "stuck": stuck, "stuckCause": stuck_cause, + "unknownCause": " ".join(unknown), + } + return assert_safe_status(answer) + + #: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's #: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and #: its seven databases are enabled at the provider boundary rather than masquerading as Odoo @@ -340,7 +471,7 @@ def main(argv): return 1 print("\nreport types this connector will pull (owner ruling R4):") for table, spec in REPORT_TYPES.items(): - rep = spec.get("report") or "NO CONFIRMED REPORT FLAVOR (see O5)" + rep = spec.get("report") or spec.get("report_discover") or "NO CONFIRMED REPORT FLAVOR (see O5)" print(f" {table:<16} {rep}") if "--probe-report" in argv: diff --git a/api/main.py b/api/main.py index 12d7d9584cda8402443c941fa63690a4996c1fd5..d4fece8c84045216a08edaf26f42eba118bd4d74 100644 --- a/api/main.py +++ b/api/main.py @@ -766,6 +766,158 @@ def _pull_amazon(why): return False +class _SyncNowRefused(Exception): + """An application-authored, credential-free sentence for a Sync now receipt.""" + + +def _sync_now_failure(cause): + """Classify actual provider failures without returning vendor text or exception reprs.""" + import re + text = str(cause).lower() + causes = ( + (("reduce the amount of data", "response too large"), + "The provider refused the pull because the response contains too much data."), + (("no ad accounts",), "The Meta token cannot access any ad accounts."), + (("max_pages", "page cap"), "The pull stopped at the provider page limit."), + (("sandbox",), "Amazon sync requires a production credential; this credential is for the sandbox."), + (("deletion", "retention"), "Amazon sync is blocked by an outstanding data deletion requirement."), + (("no stored", "no odoo credentials", "access_token is not set"), + "This tenant has no usable credential for the connector."), + (("enablement document", "capability unavailable"), + "The connector database settings could not be read."), + (("throttl", "rate limit", "too many requests"), + "The provider rate limit stopped the sync. Retry after the provider delay."), + (("timed out", "timeout"), "The sync failed because the provider request timed out."), + (("permission", "access denied", "unauthorized", "invalid oauth", "invalid token", + "expired", "authentication"), "The provider refused the connector credential or its permissions."), + (("duplicate key", "primary key", "constraint"), + "The mirror rejected rows because they violate a database constraint."), + (("connection", "network", "name resolution"), + "The sync could not connect to the provider or tenant database."), + # A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises + # DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error": + # "the loader is broken" and "this server has never held these tables" are different + # operator actions, and only the second one is true here. + (("does not exist", "no such table", "catalog error"), + "This server's mirror does not hold this connector's tables yet, so there was nothing " + "to update. Run the connector's first load on the server that owns this tenant."), + ) + for needles, sentence in causes: + if any(needle in text for needle in needles): + return sentence + status = re.search(r"\bhttp\s+(\d{3})\b", text) + if status: + return f"The provider refused the sync request with HTTP status {status.group(1)}." + if isinstance(cause, TypeError): + return "The connector failed because its provider call used incompatible arguments." + if isinstance(cause, (KeyError, ValueError)): + return "The connector could not read an expected value in the sync response." + return "The connector encountered an unexpected error while processing the sync response." + + +def _run_sync_requests(): + """Drain Sync now intents on the resident resync thread, the mirror's single writer. + + D-29 / intake F6: HTTP handlers only queue intents. Provider calls stay here, before the + resident projection pass. A failed request cannot discard another tenant's queued work. + """ + from core import connector_sync + odoo_requested = False + for receipt in connector_sync.take_sync_requests(): + provider, tenant = receipt["provider"], receipt["tenant"] + ok, detail = False, "" + first_load_progress, continuing = None, False + try: + from harness import runtime, datastore + rt = runtime.get_runtime(tenant) + if rt.key != tenant: + raise _SyncNowRefused("The sync runtime does not match the requesting tenant.") + # These two loaders use a process-global mirror. Never repoint it under readers. + # Amazon's writer opens an explicit tenant file and does not need this restriction. + if provider in ("odoo", "meta_ads") and ( + rt.datastore_path.resolve() != datastore.DB_PATH.resolve() or + datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()): + raise _SyncNowRefused( + "This tenant's mirror is not resident on this server. Sync requires its tenant worker.") + if provider == "odoo": + import core.odoo as odoo + import odoo_relational as rel + if rt.odoo_paused() or rel.frozen(rt): + raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.") + source = rt.odoo_source() + if source is None: + raise _SyncNowRefused("This tenant has no Odoo credential. Connect Odoo before syncing.") + odoo_requested = True + previous_client = getattr(odoo._tlocal, "client", None) + previous_context = getattr(odoo._tlocal, "tenant_ctx", None) + try: + # Raw mirror pulls use the resolved tenant credential and a dedicated + # transport. Do not borrow a request's cached XML-RPC client. + odoo._tlocal.client = odoo.OdooClient(creds=source._creds) + odoo._tlocal.tenant_ctx = None + report = datastore.sync_all(log=lambda *a, **k: None) + finally: + odoo._tlocal.client = previous_client + odoo._tlocal.tenant_ctx = previous_context + if not report: + raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.") + if any(row.get("phase") != "live" for row in report.values()): + raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.") + problems = [] + elif provider == "amazon_sp": + from harness import amazon_store as amazon + if not amazon.credential(tenant, rt=rt): + raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.") + first_load = amazon.needs_first_load(tenant, rt=rt) + options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {} + report = amazon.sync(tenant, rt=rt, scheduled=False, **options, log=lambda *a, **k: None) + from connectors_amazon import assert_safe_status + first_load_progress = assert_safe_status(report.get("firstLoad")) + continuing = report.get("state") == "partial" + problems = list(report.get("problems") or []) + list(report.get("audit_problems") or []) + tables = report.get("tables") or {} + if not problems and not tables and not continuing: + raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.") + if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()): + raise _SyncNowRefused("All Amazon databases are disabled. Enable a database before syncing.") + elif provider == "meta_ads": + from harness import meta_store as meta + if not meta.token(rt): + raise _SyncNowRefused("Meta Ads has no access token. Connect Meta Ads before syncing.") + report = meta.sync(tenant, log=lambda *a, **k: None, insights_days=7) + problems = list(report.get("problems") or []) + if any(row.get("capped") for row in (report.get("tables") or {}).values()): + problems.append("The pull hit the page cap.") + if not problems and not report.get("tables"): + raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.") + else: + raise _SyncNowRefused("This connector has no Sync now runner.") + if continuing: + ok = True + detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; " + f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.") + if problems: + detail += " " + _sync_now_failure(problems[0]) + elif problems: + detail = _sync_now_failure(problems[0]) + else: + ok, detail = True, "Sync finished." + except _SyncNowRefused as exc: + detail = str(exc) + except Exception as exc: # noqa: BLE001 + detail = _sync_now_failure(exc) + if first_load_progress is not None: + connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, + first_load=first_load_progress) + else: + connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail) + if continuing and ok: + # Finish first: the desk refuses a second in-flight intent. Return to the resident + # loop so its Odoo mirror pass runs before this continuation is claimed. + connector_sync.request_sync("amazon_sp", tenant, requested_by="first load") + return odoo_requested + + def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False): """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. @@ -1320,6 +1472,7 @@ def _store_resync_loop(): Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a refreshed pool re-reads the freshly synced store.""" import time as _t + from core import connector_sync passes = 0 last_relational_success_at = None last_meta_fingerprint = None @@ -1388,10 +1541,13 @@ def _store_resync_loop(): # stored fields such as customer_rank can change without partner write_date advancing, # so staging served a provably stale customer population for its first hour. Subsequent # passes still sleep at the freshly resolved cadence, and manual still skips only Odoo. + # The wait is now interruptible so an administrator's Sync now wakes this writer. if passes: - _t.sleep(_every) + connector_sync.sync_wake_event().wait(_every) + connector_sync.sync_wake_event().clear() passes += 1 RESYNC_STATE["passes"] = passes + _odoo_requested = _run_sync_requests() sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes) RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()) RESYNC_STATE["lastPassOk"] = _err is None @@ -1444,6 +1600,10 @@ def _store_resync_loop(): _amzn_previous = last_amzn_fingerprint if _meta_reason is None else None _reason = _resync_rebuild_reason(sync_report, delete_report, last_relational_success_at, _meta_now) + # The scheduled pass may now be quiet because Sync now just advanced this mirror. + # Its zero-change report cannot suppress projection of the explicit Odoo pull. + if _odoo_requested: + _reason = "sync now" _meta_state = {} if _reason is not None: print(f"[aios-api] odoo relational rebuild required ({_reason})") diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 6a8733ddb1fb4b886a5b322f89396e26510b4649..58050f23ef749eff374e259b41ead84f5e81ad41 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -1575,6 +1575,30 @@ def connector_sync_status(connector_key: str, connection_id: str = None, return _sync_payload(connector_key, connection_id, row) +@router.post("/connectors/{connector_key}/sync/now") +def connector_sync_now(connector_key: str, session: Session = Depends(require_session)): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") + try: + return connector_sync.request_sync(connector_key, session.runtime.key, + requested_by=session.uname) + except ValueError: + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None + + +@router.get("/connectors/{connector_key}/sync/now") +def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") + return connector_sync.sync_request(connector_key, session.runtime.key) or { + "provider": connector_key, "tenant": session.runtime.key, "state": "idle", + "requestedAt": None, "startedAt": None, "finishedAt": None, + "requestedBy": "", "detail": "", "accepted": False, + } + + @router.post("/connectors/{connector_key}/sync/retry") def connector_sync_retry(connector_key: str, connection_id: str = None, session: Session = Depends(require_session), database_key: str = None): @@ -2578,11 +2602,25 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None) return _named_databases(rt, is_admin, uname, out) +def _amazon_report_name(table): + """The report type this Amazon table reads, whichever key declares it. + + ⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on + its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter + and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an + empty Source name for settlements on the connector screen, which reads as "this table has no + report" when the truth is "this table is not requested, it is listed". + """ + import connectors_amazon as cx # noqa: PLC0415 - provider owns state + spec = cx.REPORT_TYPES.get(table) or {} + return str(spec.get("report") or spec.get("report_discover") or "") + + def _amazon_databases(rt, is_admin, spec=None): """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM. - Fills the SAME `ConnectorDatabase` shape Odoo does, so the Databases tab needs no client - change: `key`, `label`, `enabled`, `fields`, `excluded`, `sharesFieldsWith`. + Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync` + block. Members retain the shared shape without the field catalogue or sync bookkeeping. ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five of the seven tables have NO published schema, because Amazon publishes artifacts only for its @@ -2611,12 +2649,13 @@ def _amazon_databases(rt, is_admin, spec=None): enabled = enablement.get("databases") or {} cat = amazon_schema.catalogue() tables = cat.get("tables") or {} + states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None # Which OTHER declared tables are filled from the SAME report document. Derived from the # report map so a table added to `REPORT_TYPES` gets this disclosure for free. by_report = {} for name in amazon_schema.PROVENANCE: - rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "") + rep = _amazon_report_name(name) if rep: by_report.setdefault(rep, []).append("ut_amzn_" + name) @@ -2625,7 +2664,7 @@ def _amazon_databases(rt, is_admin, spec=None): key = "ut_amzn_" + name spec = tables.get(name) or {} cols = spec.get("columns") or [] - rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "") + rep = _amazon_report_name(name) fields = [{"key": str(c.get("name") or ""), "label": str(c.get("name") or "").replace("_", " ").capitalize(), "type": "text" if str(c.get("type")) == "VARCHAR" else "number", @@ -2656,6 +2695,7 @@ def _amazon_databases(rt, is_admin, spec=None): # read from Odoo's grid config, and disabling it leaves every stored row and # definition untouched: only this provider config document changes. "enabled": enabled.get(key) is True, + **({"sync": (states or {}).get(name)} if is_admin else {}), "fields": fields, "excluded": excluded, "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]}) @@ -2876,9 +2916,11 @@ def connector_detail(rt, user, uname, is_admin, connector_key): return _safe_connector_status(key, {"overview": overview, "keychain": keychain, "databases": databases, + "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"} + if is_admin and key in connector_sync.providers() else None), "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs", "retryPath": f"/connectors/{key}/sync/retry"} - if is_admin and key in connector_sync.providers() else None), + if is_admin and key == "odoo" else None), # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints. "enablement": enablement, diff --git a/api/routes_customers.py b/api/routes_customers.py index 357010727f17ff95ff21c3912c1caf17efe0a92c..88f6bcdeedb123d74fcff7e162214b2e0df0f8f0 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -603,7 +603,7 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "team_id": team_id, "routes": normalized_routes, "zones": normalized_zones} -def _payload(session: Session): +def _payload(session: Session, related=None): """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py` referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by construction); see `grid_assembly` for what that fixed. @@ -617,11 +617,18 @@ def _payload(session: Session): """ import aios_grid - g = grid_assembly(session) + g = grid_assembly(session, consume_corrections=not bool(related)) rows = aios_grid.rows_from_pool( g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) from core import route_system rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"]) + from routes_odoo_tables import narrow_related_rows + rows = narrow_related_rows(session, MODULE, rows, related) + from routes_tables import scope_linked_metrics, linked_metric_limits + rows, g['fields'], metric_refusals = scope_linked_metrics( + session, MODULE, rows, g['fields'], g['measures'], related) + if related: + g["pids"] = frozenset(row["pid"] for row in rows) # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key. @@ -636,6 +643,8 @@ def _payload(session: Session): # "every document in the tenant" mode to reach for, deliberately. from core import grid_events as _ge return {"fields": g["fields"], "rows": rows, + "refusals": metric_refusals, + "limits": linked_metric_limits(metric_refusals), # `today` rides the payload because every relative date condition must resolve against # the TENANT's day, never the browser's — a client that falls back to its own clock # disagrees with the server for everyone west of it. @@ -718,8 +727,15 @@ def allowed_pids(session: Session): @router.get("/customers") -def customers(session: Session = Depends(module_gate(MODULE))): - return _payload(session) +def customers(session: Session = Depends(module_gate(MODULE)), related: str = None): + return _payload(session, related=related) + + +@router.post("/customers/aggregates") +def customer_aggregates(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from routes_odoo_tables import odoo_table_aggregates + return odoo_table_aggregates(MODULE, body=body, session=session) @router.get("/customers/assigned/{view_id}/rows") diff --git a/api/routes_dependents.py b/api/routes_dependents.py index fcbe5a98b89db0c2f1576017b6de36b780e7ba3f..dec7248ae75082666949344cdb58e205da7ba49f 100644 --- a/api/routes_dependents.py +++ b/api/routes_dependents.py @@ -42,6 +42,8 @@ docstring names the cost of publishing it. This door returns names and a count, "Shared with Anna and 2 others" consumes. """ +from urllib.parse import parse_qs + from fastapi import APIRouter, Depends, Query import core.dependents as dependents_registry @@ -148,6 +150,23 @@ def read_dependents(kind: str = Query(...), key: str = Query(...), target = {"kind": target_kind, "key": target_key, "database": resolved} report = dependents_registry.dependents_report(session.runtime, target) + seen = {resolved: True} + for dependent in report['dependents']: + # An address is not an authorisation. Inbound fields can live on a different + # database, so reuse the same wall before offering either C3 address shape. + for item in [dependent, *dependent.get('items', [])]: + href = item.get('href') + if not isinstance(href, str): + continue + if not href.startswith(('#/data?', '#/sheet?')): + item['href'] = None + item['reason'] = 'This dependent has no supported address available.' + continue + destinations = parse_qs(href.partition('?')[2]).get('database', []) + if (href.startswith('#/data?') and not destinations) or any( + not _may_open(session, destination, seen) for destination in destinations): + item['href'] = None + item['reason'] = 'You do not have access to open this dependent\'s database.' label = _label_for(session, resolved, target_kind, target_key) body = { "target": target, diff --git a/api/routes_grid.py b/api/routes_grid.py index 85e47ea32129d69c6a470da4714b2349b553ec14..a80984f40ec24149142fbe06fe76f6c4b07a8eae 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -2491,9 +2491,11 @@ def _topic_rt(st, module): return st -def _customer_rows(table_key, user, st): +def _customer_rows(table_key, user, st, want_rows=True): """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses.""" import aios_grid + if not want_rows: + return list(aios_grid.FIELDS), [] import core.perm_scope as perm_scope from routes_customers import _pool_for @@ -2502,15 +2504,16 @@ def _customer_rows(table_key, user, st): return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent) -def _product_rows(table_key, user, st): +def _product_rows(table_key, user, st, want_rows=True): """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a BU-pinned reader gets that BU's field contract rather than the consolidated one.""" import core.perm_scope as perm_scope from routes_products import _pool_for, pd_fields - rt = _topic_rt(st, table_key) + rt = _topic_rt(st, table_key) if want_rows else None team_id, _agent = perm_scope.derive_pool_scope(user, table_key) - return pd_fields(consolidated=team_id is None, st=st), _pool_for(rt, team_id) + return pd_fields(consolidated=team_id is None, st=st), ( + _pool_for(rt, team_id) if want_rows else []) def _register_topic_rows(): @@ -2747,7 +2750,8 @@ def _may_open(session: Session, key: str, st=None, memo: dict = None): return _walled_fields(session, key, st=st, memo=memo) is not None -def _source_fields(session: Session, table: str, st=None, memo: dict = None): +def _source_fields(session: Session, table: str, st=None, memo: dict = None, + read_status: dict = None): """The link-bearing field list for `table`, under this session's field wall. ⛔⛔ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS @@ -2769,7 +2773,8 @@ def _source_fields(session: Session, table: str, st=None, memo: dict = None): import odoo_relational fields = fields + list(odoo_relational.module_link_fields(table) or ()) except Exception: # noqa: BLE001 - pass + if read_status is not None: + read_status["cacheable"] = False return fields @@ -2826,7 +2831,7 @@ def _ensure_odoo_spec(): return False -def _unwalled_fields(defs: dict, key: str): +def _unwalled_fields(defs: dict, key: str, read_status: dict = None): """`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only. ⛔⛔ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of @@ -2849,7 +2854,8 @@ def _unwalled_fields(defs: dict, key: str): import odoo_relational fields = fields + list(odoo_relational.module_link_fields(key) or ()) except Exception: # noqa: BLE001 - pass + if read_status is not None: + read_status["cacheable"] = False return fields @@ -2884,7 +2890,7 @@ def _relation_context(session: Session, table: str, memo: dict = None): than a silently empty grid — and `scoped_fields` is a `want_rows=False` door by construction. Nothing here is handed to a mutation; `grid_related` is a GET. ⛔ IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves - only `user_tables` and (lazily, once) `object_shares`, every other store key falls through to + `user_tables`, (lazily, once) `object_shares` and read-only `__shared` overlays; other keys fall through to the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list EQUAL, key for key and value for value, to the one it returned under the bare runtime. @@ -2895,16 +2901,19 @@ def _relation_context(session: Session, table: str, memo: dict = None): """ import core.user_tables as user_tables + read_status = {"cacheable": True} try: defs = dict(user_tables.all_defs(st=session.runtime) or {}) except Exception: # noqa: BLE001 defs = {} + read_status["cacheable"] = False try: lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs}) except Exception: # noqa: BLE001 # A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback # is: the pass simply goes back to paying a round trip per wall resolution. lent = session.runtime + read_status["cacheable"] = False # ⭐ ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo` # below still memoise their own composition (the grant gate, the pre-set link append); this is @@ -2919,24 +2928,30 @@ def _relation_context(session: Session, table: str, memo: dict = None): def may_open(key): if key not in open_memo: open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo) + if not open_memo[key]: + read_status["cacheable"] = False return open_memo[key] def fields_for(key): if key not in field_memo: - field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo) + field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo, + read_status=read_status) + if wall_memo.get(key) is None: + read_status["cacheable"] = False return field_memo[key] def open_fields_for(key): if key not in wide_memo: walled = list(fields_for(key) or ()) seen = {str(f.get("key")) for f in walled if isinstance(f, dict)} - extra = [f for f in _unwalled_fields(defs, key) + extra = [f for f in _unwalled_fields(defs, key, read_status=read_status) if isinstance(f, dict) and str(f.get("key")) not in seen] wide_memo[key] = walled + extra return wide_memo[key] - return {"defs": defs, "may_open": may_open, "fields_for": fields_for, - "open_fields_for": open_fields_for} + read_status.update(defs=defs, may_open=may_open, fields_for=fields_for, + open_fields_for=open_fields_for) + return read_status def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None): @@ -2952,14 +2967,120 @@ def related_path_entries(session: Session, table: str, depth: int, ctx: dict = N """ import core.relations as relations - _ensure_odoo_spec() + spec_ready = _ensure_odoo_spec() table = str(table or "") ctx = ctx or _relation_context(session, table) + if not spec_ready: + ctx["cacheable"] = False return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"], depth=depth, may_open=ctx["may_open"], fields_for=ctx["fields_for"]) +def related_measure_pin(session, target, related, requested): + """Compile metric membership with the rows door's exact path and leg resolvers. + + SQL bindings are host declarations. No SQL, callbacks or row identities are trusted from + the client beyond the origin id, named edge and requested subset of resolved target ids. + Retain through rows as well as explicit hops so a metric cannot expand behind a row wall. + """ + from core import perm_scope, relations + import routes_odoo_tables as door + + if isinstance(related, str): + related = door._json_arg(related, 'related') + allowed = door._relation_target_ids(session, target, related) + if set(requested) - allowed: + raise err(403, 'metric_record_refused', 'A requested record is outside this linked grid.') + source, via = str(related['table']), related.get('via') + path = door._via_path(via) + entries = related_path_entries(session, source, len(path) if path else 1) + entry = relations.pick_path(entries, path) if path else relations.pick(entries, via) + legs = [(source, entry)] + if len(entry.get('path') or ()) > 1: + first, middle, last = door._related_path_legs(session, source, entry, entries) + legs = [(source, first), (middle, last)] + + bindings = { + 'sale_order_line': [ + dict(table='customer_data', column='pid', sql='{alias}.order_partner_id'), + dict(table='product_data', column='product_id', sql='{alias}.product_id'), + dict(table='ut_odoo_orders', column='pid', sql='{alias}.order_id'), + dict(table='ut_odoo_order_lines', column='pid', sql='{alias}.id'), + dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'), + ], + 'sale_order': [ + dict(table='customer_data', column='pid', sql='{alias}.partner_id'), + dict(table='ut_odoo_orders', column='pid', sql='{alias}.id'), + dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'), + dict(table='ut_odoo_order_lines', column='order_id', sql='{alias}.id'), + ], + } + members = {} + def read(table, column, values, where): + columns = {'pid', column} + if table == 'product_data': + columns.add('product_id') + if table == 'ut_odoo_order_lines': + columns.add('order_id') + return door._relation_rows(session, table, columns, match=(column, values), where=where) + + def retain(table, ids): + rows = read(table, 'pid', ids, None) + if table in members: + existing = {r['pid'] for r in members[table]} + if {r['pid'] for r in rows} != existing: + raise err(409, 'metric_scope_unsupported', + 'This metric cannot distinguish repeated databases in the record path. ' + 'Use a direct relation or add a metric binding for these separate roles.') + rows = [r for r in rows if r['pid'] in existing] + members[table] = rows + + with relations.measure_pin_context(read=read, bindings=bindings, + may_open=lambda key: _may_open(session, key)): + ids = {int(related['record'])} + retain(source, ids) + if not members[source]: + raise err(403, 'metric_record_refused', 'The origin record is unavailable to this account.') + for table, leg in legs: + link, via = leg.get('link') or {}, leg.get('via') or {} + through = link.get('through') + if through and link.get('selectionMode') != 'manual': + reverse = via.get('direction') == 'reverse' + left = str(link['throughTo'] if reverse else link['throughOn']) + frm = str(via.get('from') or 'pid') + origins = door._relation_rows(session, table, [frm], ids=ids) + bridge = read(str(through), left, + [r[frm] for r in origins if r.get(frm) is not None], None) + retain(str(through), {r['pid'] for r in bridge}) + ids = door._relation_leg_ids(session, table, ids, leg) + retain(str(leg['target']), ids) + required = tuple(members) + # A manual link need not name the sales bridge. Its metrics still obey that + # database's row wall, intersected with the already resolved record membership. + if 'ut_odoo_order_lines' not in members: + for table, column, join in [('customer_data', 'pid', 'partner_id'), + ('product_data', 'product_id', 'product_id'), + ('ut_odoo_orders', 'pid', 'order_id')]: + if table in members: + bridge = read('ut_odoo_order_lines', join, + [r[column] for r in members[table]], None) + # This extra wall belongs to line metrics. Header metrics use their own + # traversed order membership, including legitimate orders with no lines. + retain('ut_odoo_order_lines', {r['pid'] for r in bridge}) + bindings['sale_order'] = [binding for binding in bindings['sale_order'] + if binding['table'] != 'ut_odoo_order_lines'] + break + # Required path tables must all bind. Binding only the destination would return globals. + pin = relations.MeasurePin(entry['path'], members, bindings, + required=required) + scopes = [perm_scope.derive_pool_scope(session.user, table) for table in required] + teams = {team for team, _agent in scopes if team is not None} + if len(teams) > 1: + raise err(403, 'metric_scope_conflict', 'The linked databases have incompatible business unit scopes.') + return pin, next(iter(teams), None) + + def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None): """W53-T18 / R13 / C3 — WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`. @@ -3031,6 +3152,33 @@ def _refusal_error(diag: dict): return exc +_RELATED_CACHE_LIMIT = 128 + + +def _related_cache_key(session: Session, table: str, depth: int, via): + """Names-only answers follow this user's schema, grant, sync and registrar revisions. + + Tokens are read-through dict lookups, not document downloads. A runtime without revision + support can still serve an uncached answer. Never reuse an answer with unknown revisions. + """ + import core.store as store + import core.shares as shares + import core.user_tables as user_tables + import odoo_relational + + try: + if not session.uname: + return None + return (session.uname, table, depth, via, + session.runtime.revision(user_tables.STORE_KEY)["token"], + session.runtime.revision(shares.SHARES_KEY)["token"], + store.revision("users")["token"], + session.runtime.revision(odoo_relational.CONFIG_KEY)["token"], + user_tables.register_read_through(), user_tables.register_connected_prefix()) + except Exception: # noqa: BLE001 + return None + + @router.get("/grid/{table}/related") def grid_related(table: str, depth: int = Query(default=2), @@ -3102,12 +3250,27 @@ def grid_related(table: str, f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and " f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}") + wanted = tuple(str(v).strip() for v in via if str(v or "").strip()) if via else None + if wanted is not None and len(wanted) > relations.MAX_DEPTH: + raise err(400, "over_depth", relations.over_depth_refusal(table, list(wanted))) + + # This registration must run even on a hit; an import order repair cannot be skipped by + # memoisation. Failed registration or reads are served as before, but never remembered. + spec_ready = _ensure_odoo_spec() + cache = getattr(session.runtime, "related_cache", None) + cache_key = (_related_cache_key(session, table, depth, wanted) + if spec_ready and cache is not None else None) + if spec_ready and cache_key is not None: + from copy import deepcopy + with session.runtime._lock: + cached = cache.get(cache_key) + if cached is not None: + return deepcopy(cached) + ctx = _relation_context(session, table, memo=wall_memo) entries = related_path_entries(session, table, depth, ctx) - if via: - wanted = [str(v) for v in via if str(v or "").strip()] - if len(wanted) > relations.MAX_DEPTH: - raise err(400, "over_depth", relations.over_depth_refusal(table, wanted)) + if wanted is not None: + wanted = list(wanted) hit = relations.pick_path(entries, wanted) if hit is None: # ⭐⭐ W53-T18 / R13 / C3 — THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED @@ -3127,7 +3290,19 @@ def grid_related(table: str, # ⚠ A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other # tabs the wall removed are the LISTING's answer, not this one, and repeating them here # would attach an unasked-for permission report to every successful read. - return {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []} - return {"ok": True, "table": table, "depth": depth, "related": entries, - "refusals": related_refusals(session, table, depth, entries, ctx)} - + answer = {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []} + else: + answer = {"ok": True, "table": table, "depth": depth, "related": entries, + "refusals": related_refusals(session, table, depth, entries, ctx)} + if (spec_ready and cache_key is not None and ctx.get("cacheable", False) and ctx["defs"] + and not answer["refusals"] + and cache_key == _related_cache_key(session, table, depth, cache_key[3])): + from copy import deepcopy + # The Amazon purge's pool-cache tuple does not include this memo. That is safe because + # every lookup checks the current tokens; old entries cannot match changed metadata. + # Limit memory, not the number of related databases: eviction just rebuilds next open. + with session.runtime._lock: + if len(cache) >= _RELATED_CACHE_LIMIT: + cache.clear() + cache[cache_key] = deepcopy(answer) + return answer diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 1e186678eee9165932bc0c465255ff1d02aefb15..c557a66edd40d8f0c54876d788498bf0ed548c13 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -920,6 +920,14 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_ _rel_reconnect(session.runtime) except Exception: # noqa: BLE001 pass + elif etype == "amazon_sp": + # D-29 / intake F6: the HTTP thread only records intent. The resident resync thread + # remains the mirror's single writer. A queue failure must not undo a stored credential. + try: + from core import connector_sync + connector_sync.request_sync("amazon_sp", session.runtime.key, requested_by=session.uname) + except Exception: # noqa: BLE001 + pass if scope != DEFAULT_SCOPE: try: _write_scope(session.runtime, row["id"], scope, session.uname) diff --git a/api/routes_nav.py b/api/routes_nav.py index 13162190d3882c910b5a4e657df93f7f218306d6..2174e41e922015d715db0c9d7da9edda514d4828 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -81,6 +81,36 @@ def _now() -> int: return int(time.time()) +def _clean_linked_layouts(raw): + """R8 personal presentation only. Never accept a SavedView or a field definition.""" + if not isinstance(raw, dict): + return {} + layouts = {} + for key, value in raw.items(): + if not isinstance(key, str) or not key or not isinstance(value, dict): + continue + layout = {} + for name in ("order", "visible"): + if isinstance(value.get(name), list): + layout[name] = list(dict.fromkeys(k for k in value[name] if isinstance(k, str))) + if isinstance(value.get("widths"), dict): + layout["widths"] = {k: w for k, w in value["widths"].items() + if isinstance(k, str) and type(w) in (int, float) and w > 0} + if isinstance(value.get("sorts"), list): + layout["sorts"] = [{"colId": s["colId"], "dir": s["dir"]} + for s in value["sorts"] if isinstance(s, dict) + and isinstance(s.get("colId"), str) and s.get("dir") in ("asc", "desc")] + if isinstance(value.get("filters"), list): + layout["filters"] = [n for n in value["filters"] if isinstance(n, dict)] + if value.get("filterConj") in ("and", "or"): + layout["filterConj"] = value["filterConj"] + if "groupBy" in value and (value["groupBy"] is None or isinstance(value["groupBy"], str)): + layout["groupBy"] = value["groupBy"] + if layout: + layouts[key] = layout + return layouts + + def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. @@ -118,7 +148,35 @@ def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): placement[k] = v elif keep_unknown_ut and k.startswith("ut_"): placement[k] = v - return {"folders": folders, "placement": placement} + clean = {"folders": folders, "placement": placement} + layouts = _clean_linked_layouts(raw.get("linkedLayouts")) + if layouts: + clean["linkedLayouts"] = layouts + return clean + + +def _merge_nav_prefs(stored, body, page_keys, *, keep_unknown_ut=False): + """Replace navigation only when supplied; patch each personal link independently. + + Called inside runtime.update so concurrent link and folder writes see the latest entry. + Folder clients may echo a stale linkedLayouts from GET; only a layout-only request owns it. + """ + stored = stored if isinstance(stored, dict) else {} + body = body if isinstance(body, dict) else {} + navigation = "folders" in body or "placement" in body + merged = {**stored} + if navigation: + merged.update(folders=body.get("folders", []), placement=body.get("placement", {})) + else: + layouts = _clean_linked_layouts(stored.get("linkedLayouts")) + layouts.update(_clean_linked_layouts(body.get("linkedLayouts"))) + merged["linkedLayouts"] = layouts + # A layout write does not own navigation pruning either. + clean = _clean_nav_prefs(merged, page_keys, keep_unknown_ut=keep_unknown_ut) + if not navigation: + for name, default in (("folders", []), ("placement", {})): + clean[name] = stored.get(name, default) + return clean def _placeable_top_keys(session): @@ -959,7 +1017,7 @@ def nav_prefs(session: Session = Depends(require_session)): try: stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} except Exception: - stored = {} + raise err(503, "store_unavailable", "Your saved preferences could not be loaded.") keys, enumerated = _placeable_top_keys(session) return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} @@ -967,17 +1025,19 @@ def nav_prefs(session: Session = Depends(require_session)): @router.post("/nav/prefs") def save_nav_prefs(body: dict = Body(default=None), session: Session = Depends(require_session)): - """Wholesale replace, like the table folder stratum — the client sent its complete - picture, validated here; a partial merge would resurrect deleted folders forever.""" + """Replace the supplied navigation picture or patch personal linked-grid layouts.""" keys, enumerated = _placeable_top_keys(session) - clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) + clean = {} if not session.runtime.available(): raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.") def _up(data): + nonlocal clean data = data if isinstance(data, dict) else {} - if clean["folders"] or clean["placement"]: + clean = _merge_nav_prefs(data.get(session.uname), body, keys, + keep_unknown_ut=not enumerated) + if clean["folders"] or clean["placement"] or clean.get("linkedLayouts"): data[session.uname] = clean else: data.pop(session.uname, None) @@ -987,7 +1047,7 @@ def save_nav_prefs(body: dict = Body(default=None), session.runtime.update(_NAV_PREFS_KEY, _up) except Exception: raise err(503, "store_unavailable", - "the folder change was not saved: the store refused the write.") + "the preference change was not saved: the store refused the write.") return {"prefs": clean} diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index 025d65118a21cdce083f070823f0f2dd54ac028c..c5fd5694b9a8dc2b70970d33ab5a02a55423a6ab 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -23,6 +23,7 @@ import hashlib import hmac import json import time +from contextlib import contextmanager from fastapi import Body, Depends, HTTPException from fastapi import APIRouter, Query @@ -1516,6 +1517,148 @@ def _related_source_value(session, entry, record): return None +def _materialised_table(table_key): + """Storage dispatch, independent of the browser's fallible status roster.""" + return module_cache_source(table_key) is not None or table_key not in _sources() + + +def _relation_rows(session, table, columns, *, ids=None, match=None, where=None): + """Internal join values, after the ordinary row wall, with no row-count ceiling. + + Join columns may be hidden on the wire. Only identities leave the resolver; the native + payload builders still apply their complete hidden-field closure before returning cells. + """ + import core.perm_scope as perm_scope + import core.relations as relations + + columns = set(columns) | {relations.IDENTITY_COL} + if not _materialised_table(table): + read = _compiled_read(table, session) + spec = read["spec"] + bindings = dict(spec["cols"], pid=(spec["id"], int)) + if columns - set(bindings): + raise err(409, "relation_target_unavailable", "A required relation column is unavailable.") + parts = [p for p in (read["where"], where) if p] + params = list(read["params"]) + for key, values in (("pid", ids), match or ("pid", None)): + if values is None: + continue + expr, cast = bindings[key] + values = list(values) + parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0") + params.extend(cast(value) for value in values) + names = sorted(columns) + projection = ", ".join(bindings[key][0] for key in names) + source = spec.get("from_sql") or spec["table"] + predicate = " AND ".join(f"({part})" for part in parts) or "1=1" + return [dict(zip(names, row)) for row in read["cur"].execute( + f"SELECT {projection} FROM {source} WHERE {predicate}", params).fetchall()] + + # This is an internal join, never a cell response. Apply the same permanent wall on the + # complete contract before taking any join value, including a join on a hidden column. + if not perm_scope.may_read(session.user, table, st=session.runtime): + raise err(403, "forbidden_table", "This account cannot open the linked database.") + if module_cache_source(table) is not None: + session.require(table) + fields, rows = perm_scope._read(table, session.user, session.runtime) + rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime) + if where: + raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.") + if ids is not None: + wanted = {str(value) for value in ids} + rows = [row for row in rows if str(row.get("pid")) in wanted] + if match is not None: + key, values = match + wanted = {str(value) for value in values} + rows = [row for row in rows if str(row.get(key)) in wanted] + return [{key: row.get(key) for key in columns} for row in rows] + + +def _relation_cell_ids(value): + """Stored link cells contain comma-separated identities, not a scalar join value.""" + parts = value if isinstance(value, (list, tuple, set)) else str(value or "").split(",") + return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0} + + +def _relation_leg_ids(session, source, source_ids, entry): + """Follow one declared leg, including manual, inverse and through links.""" + target = str(entry["target"]) + via = entry.get("via") or {} + link = entry.get("link") or {} + reverse = via.get("direction") == "reverse" + frm, on = str(via.get("from") or "pid"), str(via.get("on") or "pid") + field = str(via.get("field") or "") + manual = bool(field and link and not link.get("inverse") + and (link.get("selectionMode") == "manual" or not link.get("on"))) + inverse = str(link.get("inverse") or "") + source_col, target_col = frm, on + if manual: + source_col, target_col = ("pid", field) if reverse else (field, "pid") + elif inverse: + source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse) + source_rows = _relation_rows(session, source, [source_col], ids=source_ids) + values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")} + if (manual and not reverse) or (inverse and reverse): + values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows)) + target_col = "pid" + through = link.get("through") if not manual else None + if through: + left = str(link["throughTo"] if reverse else link["throughOn"]) + right = str(link["throughOn"] if reverse else link["throughTo"]) + middle = _relation_rows(session, str(through), [left, right], match=(left, values)) + values = {str(row[right]) for row in middle if row.get(right) not in (None, "")} + if (manual and reverse) or (inverse and not reverse): + rows = _relation_rows(session, target, [target_col]) + rows = [row for row in rows if _relation_cell_ids(row.get(target_col)) & values] + else: + rows = _relation_rows(session, target, [target_col], match=(target_col, values), + where=entry.get("where")) + return {int(row["pid"]) for row in rows} + + +def _relation_target_ids(session, target, related): + """One pin resolver for SQL, pool and tenant rows, selected from the scoped path list.""" + import core.relations as relations + from routes_grid import _may_open, related_path_entries + + if isinstance(related, str): + related = _json_arg(related, "related") + if not isinstance(related, dict) or not related.get("table") or related.get("record") in (None, ""): + raise err(400, "bad_relation", "A pinned read needs {table, record, via}.") + source = str(related["table"]) + via = related.get("via") + path = _via_path(via) + if path and len(path) > relations.MAX_DEPTH: + raise err(400, "over_depth", relations.over_depth_refusal(source, path)) + if not _may_open(session, source): + raise err(403, "forbidden_table", "This account cannot open the source database.") + entries = related_path_entries(session, source, len(path) if path else 1) + entry = relations.pick_path(entries, path) if path else relations.pick(entries, via) + if entry is None or str(entry.get("target")) != str(target): + raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via)) + try: + ids = {int(related["record"])} + except (TypeError, ValueError): + raise err(400, "bad_relation", "The source record must have an integer identity.") + legs = [(source, entry)] + if len(entry.get("path") or ()) > 1: + first, middle, last = _related_path_legs(session, source, entry, entries) + legs = [(source, first), (middle, last)] + for table, leg in legs: + if not _may_open(session, leg["target"]): + raise err(403, "forbidden_table", "This account cannot open the linked database.") + ids = _relation_leg_ids(session, table, ids, leg) + return ids + + +def narrow_related_rows(session, table_key, rows, related): + """Intersect an already walled and stripped payload with one resolved identity set.""" + if related in (None, "", {}): + return rows + ids = _relation_target_ids(session, table_key, related) + return [row for row in rows if int(row["pid"]) in ids] + + def _cache_relation_plan(session, source, via, target): """The offered module link and its explicit bridge, or a cause-specific refusal. @@ -1601,7 +1744,10 @@ def _cache_relation_source_value(session, entry, record): def _cache_relation_target_ids(session, source, record, via, target): - """Resolve one explicit source -> mirror bridge -> module-cache path to target row pids.""" + """Resolve any offered path; retain the legacy Products ordered string transport.""" + if not (source == "customer_data" and target == "product_data" and via == "products_ordered"): + ids = _relation_target_ids(session, target, {"table": source, "record": record, "via": via}) + return {"ok": True, "target": str(target), "ids": sorted(ids)} entry, _link, through, through_on, through_to, on = _cache_relation_plan( session, source, via, target) source_value = _cache_relation_source_value(session, entry, record) @@ -1658,7 +1804,8 @@ def odoo_relation_targets(source: str = Query(...), record: int = Query(..., ge= via: str = Query(...), target: str = Query(...), session: Session = Depends(require_session)): """Target pids for a declared relation whose destination is a runtime-cache database.""" - return _cache_relation_target_ids(session, source, record, via, target) + parsed_via = _json_arg(via, "via") if via.startswith(("[", "{")) else via + return _cache_relation_target_ids(session, source, record, parsed_via, target) def _via_path(via): @@ -1721,308 +1868,24 @@ def _related_path_legs(session, source, entry, entries): return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key) -def _related_hop_values(session, source, record, first, mid_key, last, notes): - """The values hop 2 joins on, resolved from the INTERMEDIATE database's own rows. - - ⭐⭐ W54-T40 — THIS IS THE HALF THAT MAKES A TWO-HOP TAB ANSWER ROWS INSTEAD OF A SENTENCE. - Hop 1 turns the source record into a set of intermediate rows; this returns the value each of - those rows carries in the column hop 2 is keyed on, and the caller pins THIS database by them. - - ⛔⛔ THE INTERMEDIATE IS READ THROUGH `_compiled_read`, WHICH IS THE SAME COMPILER THIS DOOR - USES FOR AN ORDINARY WINDOW, AND THAT IS THE WHOLE PERMISSION ARGUMENT. Its grant gate, its - administrator row wall and its hidden-field closure all run before a single id is produced, so - a traversal cannot be used to learn about rows of the middle database a reader may not see. A - second, lighter read here would be a second permission wall, and this file already carries the - rule that a second copy of that block does not drift towards refusing. - ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ REFUSES BY NAME. `product_data` is a compiled - registry module with no read-through binding, so the one enumerated path that passes through it - answers a sentence carrying the cause and a way round it, rather than an empty grid. - - ⚠ NO CAP, AND THE COST IS REPORTED INSTEAD (standing rule 1). Every distinct value comes back, - however many there are, and the reader is told through `limits` that this tab resolves a second - database on every page. - """ - import core.relations as relations - from fastapi import HTTPException # noqa: PLC0415 - - value = _related_source_value(session, first, record) - if value in (None, ""): - raise err(400, "unknown_relation", - relations.unresolvable_source_refusal(source, first)) - try: - mid = _compiled_read(mid_key, session) - except HTTPException as exc: - why = "" - if isinstance(getattr(exc, "detail", None), dict): - why = str((exc.detail.get("error") or {}).get("message") or "") - raise err(409, "relation_not_served", - f"this relation reaches '{last.get('target')}' through '{mid_key}', and that " - f"database could not be read here to follow it" - + (f" ({why})" if why else "") - + f". Open '{mid_key}' on its own, find the records this one is linked to, and " - f"follow the next relation from there") - mid_spec = mid["spec"] - on1 = str((first.get("via") or {}).get("on") or "") - step = (mid_spec.get("cols") or {}).get(on1) - if not step: - raise err(409, "relation_not_served", - f"'{mid_key}' has no live expression for '{on1}', which is the column the first " - f"hop of this relation joins on, so the traversal cannot be resolved. Open " - f"'{mid_key}' on its own, or ask for '{on1}' to be added to its live connection") - # ⭐ THE COLUMN HOP 2 IS KEYED ON, READ OFF THE INTERMEDIATE. `pid` is the row IDENTITY and is - # not a column of the mirror spec, so it resolves to the spec's own identity expression, which - # is the same one this route trusts everywhere else and is per table rather than a bare `id`. - frm2 = str((last.get("via") or {}).get("from") or relations.IDENTITY_COL) - take = (mid_spec["id"] if frm2 == relations.IDENTITY_COL - else ((mid_spec.get("cols") or {}).get(frm2) or (None,))[0]) - if not take: - raise err(409, "relation_not_served", - f"'{mid_key}' has no live expression for '{frm2}', which is the value the second " - f"hop of this relation is keyed on, so the traversal cannot be resolved. Open " - f"'{mid_key}' on its own, or ask for '{frm2}' to be added to its live connection") - try: - value = step[1](value) - except Exception: # noqa: BLE001 - value = str(value) - mid_from = mid_spec["from_sql"] if mid_spec.get("from_sql") else mid_spec["table"] - parts = [p for p in (mid["base_where"], first.get("where")) if p] - parts.append(f"{step[0]} = ?") - pred = " AND ".join(f"({p})" for p in parts) - rows = mid["cur"].execute( - f"SELECT DISTINCT {take} FROM {mid_from} WHERE {pred}", - list(mid["wall_params"]) + [value]).fetchall() - values = [r[0] for r in rows if r and r[0] is not None] - # ⭐⭐ STANDING RULE 1'S SECOND SENTENCE, AND `effect` IS CHOSEN SO THE READER GETS **THIS** - # SENTENCE RATHER THAN A NEIGHBOUR'S. `counts.ts::plainLimit` maps a known `effect` to one - # fixed client phrase and falls through to the SERVER's own first sentence for an effect it has - # not learned; `slow` maps to *"Rows this far down load slowly, each page is reached by walking - # every row above it"*, which is a true statement about a deep page and a FALSE one about a - # seven-row traversal. `traversal` is deliberately not in that table, so the strip prints the - # cause below, which is what actually happened. - # ⚠ THE FIRST CLAUSE IS THE ONE THAT REACHES THE STRIP: `firstSentence` cuts at the first full - # stop and truncates past 120 characters, so the mechanism is stated before the count. - notes.append({ - "subject": "related", "effect": "traversal", - "cause": f"these rows are reached through '{mid_key}', so every page resolves that " - f"database first. This record matched {len(values):,} of its records, and this " - f"database is then narrowed to the rows those reach", - "recommendation": f"open '{mid_key}' on its own if you are paging through a large set, or " - f"filter this database beside the relation to ask a smaller question"}) - return values def _related_pin(table_key, session, spec, base_where, wall_params, cur, related): - """`(sql, params, notes)` for C6's pinned read, or `(None, [], [])` when nothing is pinned. - - ⛔⛔ WHY THIS DOES NOT CALL `compile_filter_tree(member_ids=)`, WHICH IS WHAT C6'S SENTENCE - SAYS. Measured against `platform/harness/filter_sql.py` on 2026-09-03, that call cannot express - a pinned read and the failure is silent in the widening direction, twice over: - - 1. `:583` returns None the moment the tree is INACTIVE, and the `member_ids` branch is BELOW - that return. A pin with no client filter is exactly the inactive case, so the ids would be - dropped and the whole table served. - 2. `:588` emits `(tree OR id_sql IN (...))`. Its own docstring: *"`member_ids` reproduces - `members.has(r.pid) || matchFilterTree(...)`"*. That is a UNION. ORing a pin onto a - filter WIDENS; a pinned read needs an INTERSECTION. - - So the tail is compiled by `core.relations.pin_sql`, in `filter_sql`'s OWN spelling (proven - byte for byte by `verify_pivot.py`, not by this comment), and ANDed in. `filter_sql.py` is lane - B's file this wave; a narrowing member mode there is an AMENDMENT and is booked in the handoff, - not edited in from here. - - ⭐ `id_sql` IS PASSED EXPLICITLY, AS `spec["id"]`, AND IS NEVER DEFAULTED. `harness/semantic.py` - defaults it to `f"{alias}.id"` inside its OWN compile calls; a call from the route layer never - reaches that line, and `filter_sql:586` RAISES without it. It matters that it is per table - rather than a literal `id`: `ut_odoo_order_lines` is join-shaped and its identity is `sol.id`, - so a bare `id` would be ambiguous across `sale_order_line` and `sale_order`. It is the same - expression this route already trusts as `tiebreak_sql` and as the fallback ORDER BY. - - ⭐⭐ W54-T40 / INSTRUCTION 22 / R12 — **A `via` MAY NOW BE A PATH, AND THE PATH IS SERVED.** - The owner's second quoted error was this function's own `bad_relation` sentence, and it fired - on 18 of the 25 rail tabs of `customer_data`: R12 says the client pins a traversal by naming - each edge, so a two-hop tab sends `via` as the ARRAY `["invoices","order_link"]`, and the - guard below accepted only the one-hop bag. MEASURED 2026-09-06, 6 of 25 tabs answered 200 and - 18 answered 400 on the first click. - - ⛔⛔ ACCEPTING THE ARRAY IS HALF A FIX AND WOULD HAVE MOVED THE ERROR RATHER THAN CLOSED IT. - `relations.pick` matches `via.direction` against `DIRECTIONS`, and a two-hop entry's direction - is the literal `path`, deliberately outside that tuple so nothing serves half a traversal. So - a path resolves through `relations.pick_path` against the DEPTH-AWARE listing, and the read is - then answered in TWO steps: hop 1 resolves the intermediate database's own rows to the values - hop 2 joins on, and hop 2 pins this database by them. Sending the object form alone answered - `400 unknown_relation` and changed nothing the reader could see. - - ⭐ EVERY HOP IS STILL PERMISSION CHECKED, AND AT PIN TIME, NOT ONLY AT LISTING TIME (R13). The - intermediate is resolved through `_compiled_read`, i.e. THE SAME compiler this door uses for - an ordinary read, so its grant, its row wall and its hidden-field closure all apply before a - single id crosses into this table's predicate. A second, laxer read of the intermediate would - be exactly the permission wall with a hole in it that `related_entries` exists to prevent. - ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ IS A **REPORTED LIMIT**, never a silent empty: - `product_data` is a registry module with no read-through binding, so the one enumerated path - that passes through it refuses by name with a cause and a recommendation. - """ + """Compile the same resolved identity set on both mirror transports, as an intersection.""" import core.relations as relations if isinstance(related, str): related = _json_arg(related, "related") if related in (None, "", {}): return None, [], [] - if not isinstance(related, dict): - raise err(400, "bad_relation", - "`related` must be an object naming the record to pin these rows to: " - "{table, record, via}") - - source = str(related.get("table") or "") - record = related.get("record") - via = related.get("via") - # ⭐⭐ W54-T40 — `via` IS EITHER ONE EDGE'S BAG OR A PATH OF EDGE KEYS, and `_via_path` is the - # one place that decides which. It answers None for the bag, which is what keeps the guard - # below spelling the SAME refusal for a genuinely incomplete pin. - path = _via_path(via) - if not source or record in (None, "") or (path is None and not isinstance(via, dict)): - raise err(400, "bad_relation", - "a pinned read needs the record's database, the record id and the relation to " - "follow, as {table, record, via}, where `via` is one relation or the array of " - "edge keys naming each hop. Ask GET /api/v1/grid/" + (source or "{table}") - + "/related for the relations this account may follow") - - from routes_grid import _may_open, related_entries, related_path_entries - - if not _may_open(session, source): - # ⚠ THE CAUSE IS NAMED AS A PAIR, ON PURPOSE. `_may_open` is fail-closed and answers False - # both for a genuine permission refusal and for a database it could not read at all, so a - # sentence claiming only the first would tell a reader their access was revoked when the - # store had merely hiccupped. Naming both is honest and still refuses. - raise err(403, "not_permitted", - f"'{source}' could not be opened for your account, either because it is not " - f"granted to you or because it could not be read just now, so rows here cannot " - f"be pinned to one of its records. Ask an administrator for access to that " - f"database, or try again") - - # ⛔ RESOLVED AGAINST THE **SCOPED** LISTING, which is what makes "an edge you cannot be - # offered is an edge you cannot pin" structural rather than remembered. `related_entries` is - # the same function `GET /grid/{table}/related` answers with; there is no second resolution. - # ⭐ W54-T40 — AND A PATH RESOLVES AGAINST THE DEPTH-AWARE LISTING, WHICH IS THE SAME RULE ONE - # LEVEL DEEPER: `related_path_entries` is what `GET /grid/{table}/related?depth=2` answers - # with, so a traversal this account is not offered is a traversal it cannot pin either. - # ⚠ THE DEPTH ASKED FOR IS THE LENGTH OF THE PATH, never a flat 2. A one-element path costs - # the depth-1 enumeration it needs and not the depth-2 one, which on `customer_data` is the - # difference between roughly half a second and roughly two. - if path is None: - entries = related_entries(session, source) - entry = relations.pick(entries, via) - else: - if len(path) > relations.MAX_DEPTH: - raise err(400, "over_depth", relations.over_depth_refusal(source, path)) - entries = related_path_entries(session, source, len(path)) - entry = relations.pick_path(entries, path) - if entry is None: - raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via)) - entry = dict(entry, __source__=source) - - if str(entry.get("target") or "") != str(table_key): - raise err(400, "unknown_relation", - f"the relation '{entry.get('key')}' on '{source}' points at " - f"'{entry.get('target')}', not at '{table_key}', so it cannot narrow this " - f"database. Read the database the relation names, or ask for the relation that " - f"reaches this one") - if not entry.get("served", True): - raise err(409, "relation_not_served", relations.unserved_target_refusal(entry)) - - # ⭐⭐ W54-T40 — THE **LAST** HOP IS WHAT JOINS THIS DATABASE, and on a two-hop entry that is - # not `entry`. A composite carries `via = {direction: 'path', key, path}` and NO `on`, no - # `from` and no `where`: `relations._two_hop_entry` refuses to flatten the two hops' predicates - # onto the pair, because one of them may carry a predicate the other does not (the open-invoice - # bridge is exactly that edge). So the legs are resolved, each from its OWN table's scoped - # listing, and the last one is what this block below reads. + ids = _relation_target_ids(session, table_key, related) + sql, params = relations.pin_sql(spec["id"], sorted(ids)) notes = [] - first, mid_key, last = entry, "", entry - if len(entry.get("path") or ()) > 1: - first, mid_key, last = _related_path_legs(session, source, entry, entries) - - on = str((last.get("via") or {}).get("on") or "") - # ⚠ `spec["cols"]`, NOT the caller's narrowed `cols`. The join column is a JOIN, not a value - # this response returns: a relation must keep working when an administrator has hidden the id - # column it runs over, and no cell of it is ever projected. The hidden-field closure still - # governs everything that reaches the wire, one layer up. - # ⭐⭐ W54-T40 — AND THE ROW IDENTITY IS NOT ONE OF THOSE COLUMNS. `relations` defaults an edge - # with no declared `on` to `pid`, which is the wire's name for the row identity and is never a - # key of a mirror spec's `cols` (measured across all eight bound grains on 2026-09-06: not one - # carries it). Every such edge therefore refused with "has no live expression for 'pid'" while - # the expression was in hand the whole time, as `spec["id"]` - the same per-table identity this - # route already uses for the pin itself, for the tiebreak and for the fallback ORDER BY, and - # deliberately not a bare `id`, which is ambiguous on a join-shaped grain. - if on == relations.IDENTITY_COL: - binding = (spec["id"], int) - else: - binding = (spec.get("cols") or {}).get(on) - if not binding: - raise err(409, "relation_not_served", - f"'{table_key}' has no live expression for '{on}', which is the column this " - f"relation joins on, so the rows cannot be narrowed to one record here. Open " - f"this database directly, or ask for '{on}' to be added to its live connection") - on_expr, on_cast = binding[0], binding[1] - - if mid_key: - # ⭐ HOP 1, ANSWERED BY THE INTERMEDIATE DATABASE ITSELF, UNDER ITS OWN WALL. See - # `_related_hop_values`. An empty answer is an ANSWER (this record reaches nothing through - # that database) and compiles to `1=0` two blocks down, never to "no pin". - values = _related_hop_values(session, source, record, first, mid_key, last, notes) - else: - value = _related_source_value(session, entry, record) - if value in (None, ""): - raise err(400, "unknown_relation", - relations.unresolvable_source_refusal(source, entry)) - values = [value] - cast = [] - for one in values: - try: - cast.append(on_cast(one)) - except Exception: # noqa: BLE001 - cast.append(str(one)) - - # ── the member ids, in ONE SQL over the bridge column (the ticket's `how:`) ──────────────── - # ⛔ NOT THROUGH `datastore.window`: that clamps at `WINDOW_MAX` (5,000) rows, so a customer - # with more order lines than that would be pinned to the first 5,000 of them and the tab would - # show a smaller, entirely plausible number with nothing red anywhere. Standing rule 1: no cap - # on data from a connected source. - # ⚠ THE WALL RIDES ALONG. Resolving ids under `base_where` means the id SET a scoped reader can - # produce is already narrowed to rows they may see, so the pin cannot be used to learn how many - # rows exist beyond their boundary. - # ⭐ W54-T40 — `= ?` FOR ONE VALUE AND `IN (...)` FOR MANY, and the one-value spelling is kept - # BYTE FOR BYTE so a one-hop pin compiles exactly the SQL it compiled before this ticket. The - # many-value arm is only ever reached by the second hop of a traversal. - frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"] - ids = [] - if cast: - parts = [p for p in (base_where, last.get("where")) if p] - parts.append(f"{on_expr} = ?" if len(cast) == 1 - else f"{on_expr} IN ({','.join('?' for _ in cast)})") - pred = " AND ".join(f"({p})" for p in parts) - ids = [int(r[0]) for r in cur.execute( - f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {pred}', - list(wall_params) + list(cast)).fetchall() if r and r[0] is not None] - - # ⭐ STANDING RULE 1'S SECOND SENTENCE, ON A COST RATHER THAN ON A CAP. Nothing is limited - # here; the reader is told that this particular relation is answered by reading the source - # database once per page, because it is keyed on a column rather than on the record id. - if str((first.get("via") or {}).get("from") or "") != relations.IDENTITY_COL: - notes.append({ - "subject": "related", "effect": "slow", - "cause": f"this relation is keyed on the '{first['via']['from']}' column of " - f"'{source}' rather than on its record id, so every page of these rows is " - f"resolved by reading that database first", - "recommendation": "read this database on its own and filter it on " - f"'{on}' if you are paging through a large set"}) - if len(ids) >= _PIN_REPORT_AT: + if len(ids) > _PIN_REPORT_AT: notes.append({ "subject": "related", "effect": "slow", - "cause": f"this record is linked to {len(ids):,} rows, and each page names every one " - f"of them in the query that narrows this database to them", - "recommendation": "add a filter beside the relation to ask a smaller question, or " - "read this database on its own"}) - - sql, params = relations.pin_sql(spec["id"], ids) + "cause": f"This relation matched {len(ids):,} records, all of which are included.", + "recommendation": "Add a filter to reduce the result while continuing to page through every match."}) return sql, params, notes @@ -2615,6 +2478,87 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= "pred": pred} +@contextmanager +def _materialised_read(table_key, session, query, related): + """One compiler for native rows and their aggregate sibling, over the walled payload. + + The temporary connection contains only this request's already stripped, pinned rows and is + closed before returning. It never registers data on the tenant's shared mirror connection. + """ + import duckdb + + if table_key == "customer_data": + from routes_customers import _payload + session.require(table_key) + payload = _payload(session, related=related) + elif table_key == "product_data": + from routes_products import products + session.require(table_key) + payload = products(session=session, related=related) + else: + from routes_tables import table_rows + payload = table_rows(table_key, session=session, related=related) + fields = payload["fields"] + with duckdb.connect(":memory:") as cur: + cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)", + [json.dumps(payload["rows"], default=str)]) + bindings = {} + for field in fields: + key = str(field["key"]) + pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''") + expr = f"json_extract_string(payload, '{pointer}')" + numeric = field.get("type") in ("int", "currency", "pct", "float", "number") + bindings[key] = (f"TRY_CAST({expr} AS DOUBLE)" if numeric else expr, + float if numeric else str) + spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)", + "cols": bindings, "where": None} + cols = _filter_columns(spec, fields) + limits = list(payload.get("limits") or []) + tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols, + None, [], cur, limits) + missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols) + if missing: + raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.") + try: + pred = _fs().compile_filter_tree(tree, conj=query["filterConj"], columns=cols, + id_sql=spec["id"], today=payload.get("today") or _today(), + cohort_sets=cohorts) + except ValueError as exc: + raise err(400, "filter_unsupported", str(exc)) + if pred is not None and pred.uses_aggregate: + raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.") + where, params = (pred.sql, list(pred.params)) if pred is not None else (None, []) + search = _fs().compile_search(query["search"], cols) if query["search"] else None + if search is not None: + where = f"({where}) AND {search.sql}" if where else search.sql + params.extend(search.params) + yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params, + "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload} + + +def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related): + from harness import datastore + + query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj, + "sorts": _json_arg(sorts, "sorts"), "search": search}) + with _materialised_read(table_key, session, query, related) as read: + spec = read["spec"] + order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"]) + win = datastore.window(select=f'{spec["id"]} AS pid', where=read["where"], + params=tuple(read["params"]), order_by=order or spec["id"], + offset=offset, limit=limit or None, cur=read["cur"], **read["frm"]) + payload = read["payload"] + by_id = {int(row["pid"]): row for row in payload["rows"]} + rows = [by_id[int(row[0])] for row in win["rows"]] + if win["clamped"]: + read["limits"].append({"subject": "limit", "effect": "window_clamped", + "cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.", + "recommendation": "Continue paging with offset. Every matching row remains available."}) + return {**payload, "ok": True, "rows": rows, "total": win["total"], + "totalUnfiltered": len(by_id), "offset": win["offset"], "limit": win["limit"], + "limits": read["limits"], "scope": {"table": table_key, "readThrough": True}} + + @router.get("/odoo-tables/{table_key}/rows") def odoo_table_rows(table_key: str, offset: int = Query(default=0, ge=0), @@ -2652,6 +2596,10 @@ def odoo_table_rows(table_key: str, import core.perm_scope as perm_scope from harness import datastore + if _materialised_table(table_key): + return _materialised_window(table_key, session, offset, limit, filters, filterConj, + sorts, search, related) + # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the # hidden-field closure, the user predicate and the search all live in `_compiled_read` above, # because C1 promises that `/aggregates` answers over *the same compiled predicate this window @@ -3041,6 +2989,13 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None), if group_by is not None and not isinstance(group_by, str): raise err(400, "bad_selection", "groupBy must be a column id") + if _materialised_table(table_key): + with _materialised_read(table_key, session, query, payload.get("related")) as read: + out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"], + read["params"], read["frm"], aggs, group_by=group_by, + limits=read["limits"]) + return {"ok": True, **out, "scope": {"table": table_key, "readThrough": True}} + # ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3). # `/rows` takes `related` as a JSON query STRING and this door takes it as a BODY KEY; a worker # who assumed both were bodies would have wired the rows and left the summaries answering over diff --git a/api/routes_products.py b/api/routes_products.py index 9c0537a9c7905d102c332a189180243de1f38fca..c281b92b46ecb39dd3264751af280dc6cb837813 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -561,7 +561,7 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str @router.get("/products") -def products(session: Session = Depends(module_gate(MODULE))): +def products(session: Session = Depends(module_gate(MODULE)), related: str = None): """The product table for this session's scope — the /customers envelope byte-for-byte (`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`). @@ -573,13 +573,21 @@ def products(session: Session = Depends(module_gate(MODULE))): import aios_grid import modules.product_data as pd - g = product_assembly(session) + g = product_assembly(session, consume_corrections=not bool(related)) rows = aios_grid.rows_from_pool( g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) rows = _seed_image(_seed_shared(rows, g["rows_src"], g["fields"]), g["fields"]) from core import user_tables as _zones rows = _zones.zone_cells("product_data", rows, allowed_row_ids=g["pids"], st=session.runtime) + from routes_odoo_tables import narrow_related_rows + rows = narrow_related_rows(session, MODULE, rows, related) + from routes_tables import scope_linked_metrics, linked_metric_limits + rows, g['fields'], metric_refusals = scope_linked_metrics( + session, MODULE, rows, g['fields'], g['measures'], related) + if related: + pinned = {row["pid"] for row in rows} + g["rows_src"] = [row for row in g["rows_src"] if row["pid"] in pinned] # ⭐⭐ W41-T17 (owner instruction 25) — THE IDENTITY REPORT RIDES THE ENVELOPE, because a # report nothing reads is not a report. `product_data.identity_report` is standing rule 1's # second sentence as data for this grid's two identity columns: `product_id` (the Odoo @@ -599,12 +607,21 @@ def products(session: Session = Depends(module_gate(MODULE))): # The envelope check beside it filters to the four `/customers` keys, so an additive key is # the shape this route already established with `identity` and `scope`. return {"fields": g["fields"], "rows": rows, + "refusals": metric_refusals, + "limits": linked_metric_limits(metric_refusals), "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"), "identity": {"pid": "pid", "businessKey": "code"}, "identity_report": pd.identity_report(g["rows_src"]), "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}} +@router.post("/products/aggregates") +def product_aggregates(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from routes_odoo_tables import odoo_table_aggregates + return odoo_table_aggregates(MODULE, body=body, session=session) + + @router.patch("/products/{pid}") def patch_product(pid: int, body: dict = Body(default=None), session: Session = Depends(module_gate(MODULE))): diff --git a/api/routes_sheet.py b/api/routes_sheet.py index fb6e7d8c9fb3477d656367473a775f45dfe05e29..a4b242f2c5bd9e9021f1814606bd02a9b00e674e 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -248,6 +248,159 @@ def _registry_assembly(session: Session, table_key: str): "shown rather than shown from an older set of columns. Try again in a moment.") +def _linked_customer_bridges(database): + """Only declared reverse customer edges with an unconditional, served relation.""" + from core.relations import BRIDGES + return [bridge for bridge in BRIDGES if bridge.get("target") == database + and bridge.get("source") == "customer_data" and bridge.get("served") + and bridge.get("cardinality") == "one_to_many" and not bridge.get("where")] + + +def _linked_customer_fields(session, bridge): + from core import perm_scope + _require_database(session, bridge["source"]) + g = _registry_assembly(session, bridge["source"]) + fields = list(g.get("fields") or []) + hidden = perm_scope.hidden_keys(session.user, bridge["source"], fields, st=session.runtime) + # Only values the server can read per record. A browser formula or linked rollup needs + # another evaluator; offering it here would make a missing value look like a blank value. + fields = [f for f in fields if f.get("key") not in hidden + and f.get("type") not in {"formula", "link", "lookup", "rollup", "metric"} + and not f.get("measure")] + return g, fields + + +def _linked_relation_column(session, bridge, cur, spec, columns): + from core import perm_scope + from modules import sheet as sheet_mod + key = str(bridge.get("target_col") or "") + subject = bridge["key"] + ":" + (key or "(missing column)") + hidden = perm_scope.hidden_keys(session.user, bridge["target"], spec.get("fields") or [], + st=session.runtime) + if key in hidden: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the relation column is hidden from this account.") + # The catalogue owns expressions such as sol.order_partner_id and aml.partner_id. + # The declaration supplies the key, and is a fallback only when the catalogue has none. + if (columns.get(key) or {}).get("sql"): + return key + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the link does not declare a readable relation column.") + source = spec.get("from_sql") or spec.get("table") + try: + cur.execute(f'SELECT {key} FROM {source} LIMIT 0') + except Exception: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the declared relation column is unavailable on this database.") + columns[key] = {"sql": key, "type": "int", "aggregate": False} + return key + + +@router.get("/sheets/filter-fields/{database}") +def linked_filter_fields(database: str, session: Session = Depends(require_session)): + """The linked Filter offer, through the same field wall as the related grid.""" + from modules import sheet as sheet_mod + database = _require_database(session, database) + bridges = _linked_customer_bridges(database) + if not bridges: + return {"fields": []} + cur, spec, columns = _mirror(session)(database) + columns = dict(columns) + offered = [] + for bridge in bridges: + try: + _linked_relation_column(session, bridge, cur, spec, columns) + _g, fields = _linked_customer_fields(session, bridge) + except HTTPException as exc: + if exc.status_code == 403: + continue + raise + except sheet_mod.SheetRefusal: + continue + link_label = f'{bridge.get("source_label") or bridge["source"]} via {bridge.get("label") or bridge["key"]}' + for field in fields: + # Copy the existing field vocabulary, including options. No customer values or + # hidden definitions travel on the picker wire. + offered.append({**field, + "key": f'{sheet_mod.LINKED_FILTER_PREFIX}{bridge["key"]}:{field["key"]}', + "label": f'{link_label}: {field.get("label") or field["key"]}'}) + return {"fields": offered} + + +def _linked_filter_scope(session, database, nodes, cur, spec, columns, *, today=None): + """Read linked values in Python, then give the relation-blind compiler flat predicates.""" + from modules import sheet as sheet_mod + if not sheet_mod.linked_filter_ids(nodes): + return nodes + import aios_grid + from core import perm_scope, shared_overlay, view_templates + from harness import filter_eval, filter_sql + + bridges = {b["key"]: b for b in _linked_customer_bridges(database)} + related = {} + + def resolve(leaf): + subject = leaf["colId"] + bridge_key, field_key = sheet_mod.linked_field_parts(subject) + bridge = bridges.get(bridge_key) + if bridge is None: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "this database no longer has that available customer link.") + try: + relation_column = _linked_relation_column(session, bridge, cur, spec, columns) + if bridge_key not in related: + g, fields = _linked_customer_fields(session, bridge) + # Use the rows door's wall, including its shared-field enrichment. The pool's + # identity is pid; it must never be confused with the orders' partner_id. + rows = perm_scope.apply_row_scope(g["rows_src"], session.user, bridge["source"], + aios_grid.FIELDS, st=session.runtime) + related[bridge_key] = (g, fields, rows) + g, fields, source_rows = related[bridge_key] + except HTTPException as exc: + if exc.status_code != 403: + raise + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "this account cannot read the linked customer database.") + field = next((f for f in fields if f.get("key") == field_key), None) + if field is None: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the field is hidden, removed, or has no readable per-record value.") + local = {**leaf, "colId": field_key} + # Validate before evaluating even an empty pool, so malformed/stale leaves refuse + # rather than looking like a genuine zero. Cell slots were resolved by W60-T06 above. + probe_columns = {field_key: {"sql": "linked_value", "type": sheet_mod.mirror_field_type(field)}} + try: + compiled = sheet_mod.compile_scope([local], "and", probe_columns, today=today) + if (local.get("op") not in filter_sql.FILTER_OPS or compiled is None + or field_key not in compiled.columns_used): + raise ValueError("unusable linked condition") + except (ValueError, sheet_mod.SheetRefusal): + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "its value or operator cannot be applied to this field.") + storage_key = view_templates.workspace_key(bridge["source"]) + try: + definitions = shared_overlay.fields(storage_key, st=session.runtime) + rows = aios_grid.rows_from_pool(source_rows, fields, (g.get("ws") or {}).get("overlays"), + derived=g.get("derived")) + if field_key in definitions: + cells = shared_overlay.cells(storage_key, [r["pid"] for r in source_rows], st=session.runtime) + rows = [{**row, field_key: (cells.get(str(int(row["pid"]))) or {}).get(field_key, "")} + for row in rows] + ctx = filter_eval.EvalCtx(today=today) + ids = [row[bridge["source_col"]] for row in rows + if filter_eval.permits({"conj": "and", "nodes": [local]}, row, fields, ctx)] + return sheet_mod.linked_id_predicate(relation_column, ids) + except Exception: + raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject, + "the linked customer values could not be read. Retry after the data is available.") + + try: + return sheet_mod.rewrite_linked_filters(nodes, resolve) + except sheet_mod.SheetRefusal as exc: + raise err(409, exc.code, exc.message) + + def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None): """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period. @@ -642,10 +795,8 @@ def _refuse(exc): nothing else, so the 409 and the 413 are raised here directly rather than by bending `err`, which would change the error shape of every route in the app. - ⚠ `workbook_revision_stale` (the PUT door's code) ALSO ENDS `_stale` and still lands here. It - carries no `rev_current`, so its body is unchanged and the client's existing PUT branch keeps - working. ⭐ D-639's `index_revision_stale` lands here too and DOES carry one, so the rail's - new guard needed no branch of its own: matching the family's spelling was the whole cost. + Workbook, retained cell history and folder index refusals all carry the current revision + through the same existing envelope. """ if exc.code in _CAP_CODES: return HTTPException(status_code=413, @@ -739,13 +890,22 @@ def _role_on_workbook(session: Session, book): round because a workbook that has never been shared has no registry record at all. Asking `shares.role_for` alone would answer `None` for the person who created it. """ - from core import shares - if session.admin: - return "owner" - me = str(session.uname or "").strip().lower() - if me and _workbook_owner(book) == me: - return "owner" - return shares.role_for("workbook", book.get("id"), me, st=_store(session)) + from core import sheets + return sheets.workbook_role(_store(session), book, session.uname, is_admin=session.admin) + + +def _require_reader(session: Session, book): + """R28: use the same predicate as the index; hidden and missing ids answer identically.""" + from core import sheets + if not sheets.may_read_workbook(book, _role_on_workbook(session, book)): + raise _refuse(sheets.SheetRefusal("workbook_missing", "That workbook no longer exists.")) + + +def _index_for(session: Session): + """C2: every list response uses the index's sole visibility filter and role helper.""" + from core import sheets + return sheets.read_index(_store(session), viewer=session.uname, + role_on_workbook=lambda book: _role_on_workbook(session, book)) def _require_editor(session: Session, book): @@ -778,6 +938,8 @@ def _wire_shares(rows): out = [] for row in rows or (): row = dict(row) + if "role" in row: + row["role"] = "owner" if row["role"] == "owner" else wire_role(row["role"]) row["shares"] = [{"who": s.get("who"), "role": wire_role(s.get("role"))} for s in (row.get("shares") or []) if wire_role(s.get("role"))] @@ -809,7 +971,24 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str: return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}) -def _reresolved_for_reader(session: Session, book): +def _has_ending(binding): + return (isinstance(binding, dict) and isinstance(binding.get("period"), dict) + and "ending" in binding["period"]) + + +def _has_cell_inputs(binding): + from modules import sheet as sheet_mod + if _has_ending(binding): + return True + scope = binding.get("scope") if isinstance(binding, dict) else None + nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None) + # Linked values and related-table grants can change independently of the orders' cache key. + if sheet_mod.linked_filter_ids(nodes): + return True + return any(sheet_mod.filter_cell_nodes(nodes)) + + +def _reresolved_for_reader(session: Session, book, *, endings_only=False): """A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5). ⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with @@ -850,6 +1029,8 @@ def _reresolved_for_reader(session: Session, book): for addr, cell in (tab.get("cells") or {}).items(): if not isinstance(cell, dict) or cell.get("kind") != "bound": continue + if endings_only and not _has_cell_inputs(cell.get("binding")): + continue touched += 1 binding = cell.get("binding") or {} db = str(binding.get("database") or "") @@ -872,7 +1053,7 @@ def _reresolved_for_reader(session: Session, book): cell_key = tab_ref + "!" + addr scope_key = _binding_scope_key(session, db, binding) hit = None - if wid and scope_key: + if wid and scope_key and not _has_cell_inputs(binding): try: hit = sheets.cached_scope_value(store, wid, cell_key, scope_key) except Exception: # noqa: BLE001 @@ -890,7 +1071,7 @@ def _reresolved_for_reader(session: Session, book): continue try: - out = _resolve_binding(session, binding, view_memo=view_memo) + out = _resolve_cell_binding(session, binding, fresh, tab_ref, view_memo=view_memo) except HTTPException as exc: detail = getattr(exc, "detail", None) or {} info = detail.get("error") if isinstance(detail, dict) else None @@ -901,6 +1082,7 @@ def _reresolved_for_reader(session: Session, book): "code": (info or {}).get("code", "sheet_unresolved"), "message": (info or {}).get("message", "This cell could not be refreshed."), } + cell.pop("limits", None) continue cell["state"] = "fresh" cell["value"] = out["value"] @@ -932,15 +1114,24 @@ def _reresolved_for_reader(session: Session, book): return fresh +def _response_workbook(book): + """Omit private write history from response tabs without changing the stored document.""" + return {**book, "sheets": [{key: value for key, value in tab.items() + if key not in ("writes", "writesFrom")} + for tab in book.get("sheets") or []]} + + def _read_for(session: Session, book): """The workbook this session is entitled to see: stored for the owner, re-resolved otherwise. - ⚠ A STRANGER IS A NON-OWNER AND TAKES THE SAME PATH, deliberately. Nothing in this file - refuses a stranger's READ today (see the module note the measurement is booked under), so the - fail-closed answer to "what should they be shown" is: whatever THEIR OWN scope computes, - never the owner's cached numbers. + W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers + and grantees still receive their own scoped numbers, never the owner's cached numbers. """ + book = _response_workbook(book) if _role_on_workbook(session, book) == "owner": + if any(_has_cell_inputs(cell.get("binding")) + for tab in book.get("sheets") or [] for cell in (tab.get("cells") or {}).values()): + return _reresolved_for_reader(session, book, endings_only=True) return book return _reresolved_for_reader(session, book) @@ -951,8 +1142,7 @@ def list_sheets(session: Session = Depends(require_session)): ⚠ C5's `shares` rides here as well as on `/sheets/index`, because the client reads both. """ - from core import sheets - rows = sheets.project_shares(_store(session), sheets.list_workbooks(_store(session))) + rows = _index_for(session)["workbooks"] return {"workbooks": _wire_shares(rows)} @@ -967,7 +1157,7 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req created_by=session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) - return {"workbook": book} + return {"workbook": _response_workbook(book)} # ── the rail's own document (wave 50, W50-T17) ─────────────────────────────────────────────── @@ -982,17 +1172,10 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req # ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the # Sheet is granted to everyone, and the rail's folders name no database. # -# ⛔⛔ MEASURED IN W51-T25 AND BOOKED, NOT FIXED: THAT SENTENCE MEANS EVERY SIGNED IN ACCOUNT CAN -# ALREADY READ EVERY WORKBOOK. Nothing in `core.sheets.list_workbooks` or `::read_index` filters by -# owner, and `GET /sheets/{id}` walls on the session alone, so an account holding NO share reads a -# stranger's workbook with **200**. Driven in `verify_sheet.py` section 5c with a third identity. -# ⇒ `sharedWithMe` is therefore a PARTITION of a list everyone can already see, and C5's -# viewer/editor wall guards the WRITE doors rather than closing a read that was never shut. -# Refusing a stranger's read is not in W51-T25's `done-when` and widening it here would change the -# behaviour of a door six other tickets are built on, so it is an amendment finding for the root. -# ⚠ WHAT THIS TICKET DOES GUARANTEE MEANWHILE is the half that leaks data rather than existence: -# a non-owner never receives the owner's stored numbers, because every cell-bearing read -# re-resolves under the caller's own scope first. +# W60-T09, 2026-09-14: R28 supersedes the W51-T25 amendment finding. Both lists and all three +# workbook read shapes use core.sheets.may_read_workbook with _role_on_workbook. New workbooks +# are private to their creator and grantees; administrators see all. Unmarked or creatorless +# documents retain legacy visibility. Admitted non-owners still re-resolve under their own scope. @router.get("/sheets/index") @@ -1007,8 +1190,7 @@ def read_sheet_index(session: Session = Depends(require_session)): can reach: a client cannot send a revision it was never given, and a server that never checks one is a client stamping a number nobody reads. """ - from core import sheets - return {"index": _wire_index(sheets.read_index(_store(session), viewer=session.uname))} + return {"index": _wire_index(_index_for(session))} @router.put("/sheets/index") @@ -1035,7 +1217,7 @@ def write_sheet_index(body: dict = Body(default=None), expect_rev=raw.get("rev")) except sheets.SheetRefusal as exc: raise _refuse(exc) - return {"index": _wire_index(out)} + return {"index": _wire_index({**_index_for(session), "repairs": out.get("repairs", [])})} # ── C4: the .xlsx IMPORT, which is a JOB and not a request (wave 51, W51-T24) ──────────────── @@ -1133,7 +1315,7 @@ def _run_import(job_id, path, filename, runtime, uname): stamp = _now_iso() made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname) saved = sheets.put_workbook(runtime, made["id"], - dict(book, refreshed_at=stamp, rev=made["rev"])) + dict(book, refreshed_at=stamp, rev=made["rev"]), actor=uname) _job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"], sentence=_import_sentence(report)) except sheet_import.ImportRefusal as exc: @@ -1258,6 +1440,7 @@ def read_sheet(workbook_id: str, tabs: str = "", from core import sheets try: book = sheets.get_workbook(_store(session), workbook_id) + _require_reader(session, book) except sheets.SheetRefusal as exc: raise _refuse(exc) if str(tabs or "").strip().lower() == "meta": @@ -1294,6 +1477,7 @@ def read_sheet_tab(workbook_id: str, tab: str, from core import sheets try: book = sheets.get_workbook(_store(session), workbook_id) + _require_reader(session, book) served = _read_for(session, book) found = sheets.find_tab(served, tab) except sheets.SheetRefusal as exc: @@ -1319,9 +1503,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None same patch has landed. A CAP breach is the opposite: it is a whole request 413, because the document as a whole cannot be stored and there is no partial version of that. - ⚠ `rev` IN THE BODY MAKES THE WRITE CONDITIONAL, exactly as it does on the PUT door. Omit it - and the last writer wins; send the `rev` the tab read and a concurrent change answers 409 - `sheet_revision_stale` carrying `rev_current`, which is the revision to refetch. + With retained history, a stale revision merges by address and the later value wins; + `conflicts` names other people's overlapping writes, and only missing history refuses. ⛔⛔ C5's READ-ONLY CHECK RUNS BEFORE THAT COMPARISON AND THE ORDER IS THE CONTRACT. A viewer is refused 403 `workbook_read_only`; a viewer holding a stale rev is ALSO refused @@ -1347,7 +1530,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None expect = raw.get("rev") try: out = sheets.patch_cells(_store(session), workbook_id, tab, raw.get("cells"), - expect_rev=expect) + expect_rev=expect, actor=session.uname, + actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) accepted = out.pop("accepted", []) @@ -1396,11 +1580,31 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c (found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes} _recompute_formulas(book, fr, sheet_mod, cache_key=(session.tenant, session.uname, workbook_id), changes=edits) + # Date and filter-cell edits also change database totals. Recompute all readers, then formulas. + ending_cells = set() + for source in book.get("sheets") or []: + for addr, cell in (source.get("cells") or {}).items(): + binding = cell.get("binding") or {} + if cell.get("kind") != "bound" or not _has_cell_inputs(binding): + continue + try: + result = _resolve_cell_binding(session, binding, book, source.get("id")) + cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh") + cell.pop("refusal", None) + except HTTPException as exc: + info = ((exc.detail or {}).get("error") or {}) + cell.update(value=None, display="", state="refused", refusal={ + "code": info.get("code", "ending_cell_unreadable"), + "message": info.get("message", "The Ending date could not be read. Choose a valid date.")}) + ending_cells.add((source.get("id"), addr)) + if ending_cells: + _recompute_formulas(book, fr, sheet_mod) found = sheets.find_tab(book, tab_ref) out = {} for addr, cell in (found.get("cells") or {}).items(): if not isinstance(cell, dict) or (cell.get("kind") != "formula" - and "spill" not in cell and "spillParent" not in cell): + and "spill" not in cell and "spillParent" not in cell + and (found.get("id"), addr) not in ending_cells): continue row = {"value": cell.get("value"), "display": "" if cell.get("value") is None else str(cell.get("display") or ""), @@ -1435,10 +1639,12 @@ def write_sheet(workbook_id: str, body: dict = Body(default=None), # `rev` inside the body IS the expectation; `core.sheets.put_workbook` reads it. This door # does not translate it into a second argument, because two ways to say the same thing is # how one of them ends up ignored. - book = sheets.put_workbook(_store(session), workbook_id, raw) + book = sheets.put_workbook(_store(session), workbook_id, raw, + actor=session.uname, + actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) - return {"workbook": book} + return {"workbook": _response_workbook(book)} @router.delete("/sheets/{workbook_id}") @@ -1509,6 +1715,14 @@ def share_sheet(workbook_id: str, body: dict = Body(default=None), rec = shares.set_grants("workbook", book["id"], entries, owner=(None if held.get("owner") else _workbook_owner(book)), granter=session.uname, st=_store(session), expect=prior) + # A successful explicit share narrows legacy visibility; ordinary saves never do. + if "access" not in book: + # W60-T09, 2026-09-14: the first share decides VISIBILITY, making the owner gate load bearing. + # A non-owner must not reach this narrowing on an ownerless legacy workbook + # and lock themselves out by sharing it. + sheets.narrow_workbook(_store(session), book["id"]) + except sheets.SheetRefusal as exc: + raise _refuse(exc) except shares.GrantsChanged as exc: # ⚠ `grants_changed`, THE SAME CODE `routes_shares.py` ALREADY ANSWERS, and a plain 409 # rather than the `_stale` shape: there is no revision to hand back here, and a @@ -1613,20 +1827,24 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) if failure is not None: cell["state"] = "refused" cell["value"] = None + cell["display"] = "" cell["refusal"] = {"code": "database_unreadable", "message": f"You cannot read {db}."} + cell.pop("limits", None) continue try: - out = _resolve_binding(session, binding, view_memo=view_memo) + out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo) except HTTPException as exc: detail = getattr(exc, "detail", None) or {} info = detail.get("error") if isinstance(detail, dict) else None cell["state"] = "refused" cell["value"] = None + cell["display"] = "" cell["refusal"] = { "code": (info or {}).get("code", "sheet_unresolved"), "message": (info or {}).get("message", "This cell could not be refreshed."), } + cell.pop("limits", None) continue cell["state"] = "fresh" cell["value"] = out["value"] @@ -1654,14 +1872,16 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) try: saved = sheets.put_workbook(_store(session), workbook_id, {"name": book.get("name"), "sheets": book.get("sheets"), - "refreshed_at": stamp, "rev": book.get("rev")}) + "refreshed_at": stamp, "rev": book.get("rev")}, + actor=session.uname, + actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) # ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE # VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET # recomputes once, under their own scope, and fills its own slot again from there. sheets.clear_scope_cache(_store(session), workbook_id) - return {"workbook": saved, "recalc_ms": recalc_ms} + return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms} def _now_iso(): @@ -1916,7 +2136,120 @@ def _metric_column_aggregate(session, database, aggregate, window): return column(raw) -def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None): +def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()): + """Resolve a period from authored cells. Submitted ending.date is never trusted for a ref.""" + from modules import sheet as sheet_mod + from harness import windows as W + period = binding.get("period") + if not isinstance(period, dict) or (period.get("ending") or {}).get("kind") != "cell": + return sheet_mod.period_window(period, today) + ref = period.get("ref") or "(blank)" + value = _read_binding_cell(session, ref, book=book, tab=tab, today=today, seen=seen) + date = W.ending_cell_date(value, book.get("dateSystem", "1900")) + if date is None: + raise sheet_mod.SheetRefusal("ending_cell_unreadable", ref, "its value is not a real date.") + return sheet_mod.period_window(period, today, cell_date=date) + + +def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(), + code="ending_cell_unreadable"): + """Shared cell read for Ending and Filter, including scoped bound and formula inputs.""" + from modules import sheet as sheet_mod + import formula_runtime as fr + + def refuse(reason): + raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason) + + if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref): + refuse("its reference is invalid.") + if book is None: + refuse("the source workbook is unavailable.") + tabs = book.get("sheets") or [] + names = _tab_scope_names(tabs) + home = _resolve_home_tab(book, tabs, names, tab) + key = _asked_key(ref, home, fr) + if key in seen: + refuse("the reference is circular.") + cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names) + for addr, cell in (source.get("cells") or {}).items()} + cell = cells.get(key) + if not isinstance(cell, dict) or cell.get("kind") == "empty": + refuse("the cell or its sheet is missing or blank.") + if cell.get("kind") == "bound": + try: + value = _resolve_binding(session, cell.get("binding"), today=today, + book=book, tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"] + except HTTPException as exc: + reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.") + refuse(reason) + elif cell.get("kind") == "formula": + # The existing formula graph discovers actual inputs, including dynamic references. + # A database input starts refused, so a cached value can never bypass this reader's wall. + bound = {k: c for k, c in cells.items() if c.get("kind") == "bound"} + specs = {k: ({"refused": "ending_input_pending"} if k in bound + else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()} + engine = fr.SheetCalculation(specs, sheets=names, today=today, + named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900")) + attempted, failures = set(), {} + while True: + dependencies, pending = set(), [key] + while pending: + dependency = pending.pop() + if dependency in dependencies: + continue + dependencies.add(dependency) + pending.extend(engine.consumed.get(dependency, ())) + needed = (dependencies & set(bound)) - attempted + if not needed: + break + changes = {} + for dependency in needed: + attempted.add(dependency) + try: + result = _resolve_binding(session, bound[dependency].get("binding"), today=today, + book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency)) + changes[dependency] = {"value": result["value"]} + except HTTPException as exc: + info = ((exc.detail or {}).get("error") or {}) + failures[dependency] = info.get("message", "a database input is refused.") + changes[dependency] = {"refused": info.get("code", "ending_cell_unreadable")} + # Each pass resolves previously unread database cells, so the workbook bounds the loop. + engine.update(changes, revision=engine.revision + 1, today=today) + if engine.errors.get(key): + error_code = engine.errors[key] + reason = next((failures[k] for k in dependencies if k in failures), None) + if not reason: + reason = (sheet_mod.SheetRefusal(error_code, ref, "a formula input is refused.").message + if error_code in sheet_mod.REFUSALS else fr._sheet_error_message(error_code)) + refuse(reason) + value = engine.values.get(key) + else: + if cell.get("state") == "refused": + refuse((cell.get("refusal") or {}).get("message") or "the cell is refused.") + if cell.get("state") == "stale": + refuse("the cell needs to be refreshed.") + value = cell.get("value", cell.get("text")) + return value + + +def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen=()): + from modules import sheet as sheet_mod + try: + return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell( + session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable")) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) + + +def _resolve_cell_binding(session, binding, book, tab, **kwargs): + """Keep the legacy resolver calling convention for bindings without workbook inputs.""" + if _has_cell_inputs(binding): + return _resolve_binding(session, binding, book=book, tab=tab, **kwargs) + return _resolve_binding(session, binding, **kwargs) + + +def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None, + book=None, tab=None, ending_seen=()): """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`. The order below is the contract, not a preference: @@ -1944,7 +2277,10 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non raise err(400, exc.code, exc.message) database = _require_database(session, binding["database"]) - window = sheet_mod.period_window(binding.get("period")) + try: + window = _ending_window(session, binding, book=book, tab=tab, today=today, seen=ending_seen) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) # W59-T14 / R10: the per-database wall is contract step 1. A metric key must not be looked # up or acted on before the caller passes it; the ordinary column guards below still apply. if binding["aggregate"].get("metric_key"): @@ -1966,12 +2302,16 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non scope.get("view_name", ""), memo=view_memo) else: scope_nodes, scope_conj = _fe_parts(scope.get("filter")) + scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab, + today=today, seen=ending_seen) needed = (sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes)) if binding["aggregate"].get("field"): needed.add(str(binding["aggregate"]["field"])) cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed) + cols = dict(cols) + scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today) # ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a # real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder @@ -2069,7 +2409,15 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None), field through the database's ordinary field writer; reusing a column needs no field write. """ raw = dict(body or {}) - out = _resolve_binding(session, raw.get("binding")) + binding = raw.get("binding") or {} + book = None + if _has_cell_inputs(binding): + from core import sheets + try: + book = sheets.get_workbook(_store(session), workbook_id) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + out = _resolve_cell_binding(session, binding, book, raw.get("tab")) return {"value": out["value"], "display": "" if out["value"] is None else str(out["value"]), "limits": out["limits"]} @@ -2203,7 +2551,7 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None), saved = dict(cell) limits, resolved = [], None try: - resolved = _resolve_binding(session, binding) + resolved = _resolve_cell_binding(session, binding, book, tab_ref) except HTTPException as exc: #: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door #: and a cell refused by that one read identically. It is PERSISTED rather than raised: @@ -2226,7 +2574,9 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None), saved.pop("refusal", None) try: - landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved}) + landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved}, + actor=session.uname, + actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: raise _refuse(exc) @@ -2317,14 +2667,24 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s # `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the # window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from # the column set instead, where the time actually is. + try: + scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab.get("id")) + window = _ending_window(session, binding, book=book, tab=tab.get("id")) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) cur, spec, cols = _mirror(session)( - database, metric_window=sheet_mod.period_window(binding.get("period")), + database, metric_window=window, needed=(sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes))) + cols = dict(cols) + scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols) - count_sql, count_params, limits = sheet_mod.build_aggregate_query( - spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes, - scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj) + try: + count_sql, count_params, limits = sheet_mod.build_aggregate_query( + spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes, + scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) total = (cur.execute(count_sql, count_params).fetchone() or [0])[0] ids_sql, ids_params, _ = sheet_mod.build_ids_query( @@ -2411,8 +2771,8 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None), ⛔⛔ AND HERE IS WHAT THE SHAPE CANNOT CARRY, STATED RATHER THAN HIDDEN: a bound cell whose binding was REFUSED just now (a database wall, a row wall, a scope condition this database - cannot compile) comes back as `null`, which is the same thing an aggregate over zero matching - rows returns. The cause is real and this door knows it; `{addr: value}` has nowhere to put it. + cannot compile) comes back as `null`, which is also an empty AVG/MIN/MAX result. An empty + SUM returns zero. The cause is real and this door knows it; `{addr: value}` has nowhere to put it. `POST /sheets/{id}/refresh` and `GET /sheets/{id}` both still carry the full `{code, message}` refusal on the cell, so nothing is lost from the document - what is lost is the client's ability to paint THIS cell's refusal at commit time. Raised as a dated PRD @@ -2479,11 +2839,12 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None), if cell is None: continue binding = cell.get("binding") or {} - slot = json.dumps(binding, sort_keys=True, default=str) + own_tab = _asked_key(label, home, fr).rpartition("!")[0] + slot = json.dumps([own_tab, binding], sort_keys=True, default=str) if slot not in by_binding: try: - by_binding[slot] = _resolve_binding(session, binding, - view_memo=view_memo)["value"] + by_binding[slot] = _resolve_cell_binding(session, binding, book, own_tab, + view_memo=view_memo)["value"] except HTTPException: # ⛔ SWALLOWED HERE AND NOWHERE ELSE. See the docstring: the flat body has no # room for a cause, and failing the whole batch on one walled cell would take diff --git a/api/routes_starred.py b/api/routes_starred.py index e362f508289c23604ab1fa5ea252acfda7b47247..e2a1e9df918c9c1d10829566af403e979cb6e51e 100644 --- a/api/routes_starred.py +++ b/api/routes_starred.py @@ -39,17 +39,17 @@ from deps import Session, err, require_session router = APIRouter(prefix="/api/v1") -#: The per-user bucket holding the three ID-LIST kinds. `{username: {databases, agents, queries}}` +#: The per-user bucket holding ID-LIST kinds. `{username: {databases, agents, queries, sheets}}` #: #: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about #: the object, so a tenant-wide star would put one person's shortlist on everybody's Home. _STARRED_KEY = "starred" -#: The four kinds C2 declares. `view` is the odd one out — see the header. -KINDS = ("database", "agent", "query", "view", "application") +#: Accepted star kinds. Views and applications keep their own record-backed write paths. +KINDS = ("database", "agent", "query", "view", "application", "sheet") -#: The three kinds this file's own bucket holds, and the payload key each answers under. -_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries"} +#: The kinds this file's own bucket holds, and the payload key each answers under. +_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries", "sheet": "sheets"} #: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with #: the cause and the action (unstar something), rather than being accepted and truncated. A star @@ -90,7 +90,7 @@ def _clean_list(raw): def _read_lists(session): - """This caller's three id lists. A store blip answers empty rather than taking the page down.""" + """This caller's id lists. A store blip answers empty rather than taking the page down.""" try: mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {} except Exception: @@ -289,7 +289,7 @@ def _query_names(session, qids): def _payload(session, keys=None, cache=None): - """The C2 answer: the three id lists, the starred views, and what could not be read. + """The star answer: the caller's id lists, starred records, and what could not be read. ⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s `omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a @@ -307,6 +307,8 @@ def _payload(session, keys=None, cache=None): # validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can # disclose nothing they did not already know, and B/C render only objects they can already see # (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap. + # Sheets are also the CALLER'S OWN writes. R28 filters the sheet index; C2 forbids a second + # access check here, so sheet ids pass through unchanged. lists["databases"] = [k for k in lists["databases"] if k in keys] # ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers # "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a @@ -327,7 +329,7 @@ def _payload(session, keys=None, cache=None): except Exception: application_rows = [] out = {"databases": lists["databases"], "agents": lists["agents"], - "queries": lists["queries"], "queryNames": qnames, + "queries": lists["queries"], "sheets": lists["sheets"], "queryNames": qnames, "views": views, "applications": application_rows, "degraded": unread} if unread: # R6's second sentence, in the payload: the CAUSE and what it means, not a bare list. @@ -503,7 +505,7 @@ def set_starred(body: dict = Body(default=None), session: Session = Depends(require_session)): """C2 — star or unstar ONE object, and answer with the caller's whole new list. - Body: `{"kind": "database"|"agent"|"query"|"view", "id": str, "on": bool}`, plus an OPTIONAL + Body: `{"kind": "database"|"agent"|"query"|"view"|"application"|"sheet", "id": str, "on": bool}`, plus an OPTIONAL `"database"` when `kind` is `view` (the fast path — see `_find_view`). ⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled diff --git a/api/routes_tables.py b/api/routes_tables.py index c6b1d7cd39880750563dcff17fc96d5158bae101..4d78c402e51690adad61022431d82fd2ac1dada7 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -68,33 +68,45 @@ class _MetricReadRuntime: raise AttributeError(name) -def _related_product_fields(session, rows): +def _related_product_fields(session, rows, table_key='product_data'): """Resolve authored Metric definitions through the existing workspace and field wall.""" import aios_grid from core import grid_events, perm_scope, shared_overlay from modules import product_data as pd + import routes_customers + + scope = {'product_data': 'product', 'customer_data': 'customer'}.get(table_key, table_key) + if table_key == 'product_data': + table, base = pd.table_ops(session.runtime), aios_grid.product_fields() + shared_key = pd.TABLE_KEY + elif table_key == 'customer_data': + table, base = routes_customers._customer_table(session), aios_grid.FIELDS + shared_key = routes_customers._shared_key() + else: + table = _ops(session, table_key) + base = perm_scope.scoped_fields(session.user, table_key, st=session.runtime) + shared_key = table_key pids = frozenset(r["pid"] for r in rows) ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=set(), admin=session.admin, fallback_ws=None, - seen_ids={}, st=session.runtime, scope_key="product", - table=pd.table_ops(session.runtime)) + seen_ids={}, st=session.runtime, scope_key=scope, table=table) ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) _, fields, _, _ = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key="product", with_cohorts=False, - fields_base=aios_grid.product_fields(), st=session.runtime) + ws, session.uname, set(pids), defs={}, scope_key=scope, with_cohorts=False, + fields_base=base, st=session.runtime) have = {f["key"] for f in fields} fields = list(fields) + [dict(f, source="overlay", shared=True) - for key, f in shared_overlay.fields(pd.TABLE_KEY, st=session.runtime).items() + for key, f in shared_overlay.fields(shared_key, st=session.runtime).items() if key not in have] - hidden = perm_scope.hidden_keys(session.user, "product_data", fields, st=session.runtime) + hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime) return [f for f in fields if f.get("measure") and f["key"] not in hidden] @router.post("/tables/{table_key}/related-metrics") def related_metrics(table_key: str, body: dict = Body(default=None), session: Session = Depends(require_session)): - """C6: a visible Products ordered edge, origin customer and real destination ids. + """R5: any offered relation, with the rows door's exact walled membership. The request supplies only grid identities and a named relation. Fields, periods, source identities and values are read from this tenant; nothing is authored by this endpoint. @@ -106,66 +118,79 @@ def related_metrics(table_key: str, body: dict = Body(default=None), import automation_engine body = body if isinstance(body, dict) else {} - pin = body.get("related") or {} + pin = body.get("related") pids = body.get("pids") - if (table_key != "product_data" or not isinstance(pin, dict) - or pin.get("table") != "customer_data" - or type(pin.get("record")) is not int or pin["record"] <= 0 - or not isinstance(pids, list) or not pids + if (not isinstance(pin, dict) or not isinstance(pids, list) or any(type(pid) is not int or pid <= 0 for pid in pids)): - raise err(400, "bad_metric_scope", "Name the origin customer and related product records.") + raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.") viewer = copy.copy(session) viewer.runtime = _MetricReadRuntime(session.runtime) try: - # Both endpoints and the actual bridge are permission-bearing databases. - for key in ("customer_data", "product_data", "ut_odoo_order_lines"): - viewer.require(key) - entries = routes_grid.related_path_entries(viewer, "customer_data", 1) - via = pin.get("via") - edge = relations.pick_path(entries, via) if isinstance(via, list) else relations.pick(entries, via) - if (edge is None or edge.get("target") != "product_data" - or edge.get("via", {}).get("field") != "products_ordered"): - raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.") - source_fields = routes_grid._source_fields(viewer, "customer_data") - hidden = perm_scope.hidden_keys(viewer.user, "customer_data", source_fields, st=viewer.runtime) - if "products_ordered" in hidden: - raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.") - customers = perm_scope.scoped_table(viewer.user, "customer_data", st=viewer.runtime) - if not any(r.get("pid") == pin["record"] for r in customers): - raise err(403, "metric_record_refused", "The origin record is not available to this account.") - products = perm_scope.scoped_table(viewer.user, "product_data", st=viewer.runtime) + compiled, team_id = routes_grid.related_measure_pin(viewer, table_key, pin, pids) + # Read actual rows, not the fields-only door introduced by T24. + products = perm_scope.scoped_table(viewer.user, table_key, st=viewer.runtime) requested = set(pids) selected = [r for r in products if r.get("pid") in requested] - if (len(selected) != len(requested) - or any(type(r.get("product_id")) is not int or r["product_id"] <= 0 for r in selected)): + if len(selected) != len(requested): raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.") - # A bridge restriction that cannot be translated must never be silently dropped. - bridge_fields = perm_scope.scoped_fields(viewer.user, "ut_odoo_order_lines", st=viewer.runtime) - declared = perm_scope.entry(viewer.user, "ut_odoo_order_lines") or {} - if (perm_scope.row_scope_applies(viewer.user, "ut_odoo_order_lines") - or (not viewer.admin and declared.get("hiddenFields")) - or not {"partner_id", "product_id"}.issubset({f["key"] for f in bridge_fields})): - raise err(409, "metric_scope_unsupported", "Order-line permission restrictions cannot be applied to these Metrics yet.") - scopes = [perm_scope.derive_pool_scope(viewer.user, key) - for key in ("customer_data", "product_data", "ut_odoo_order_lines")] - teams = {team for team, _ in scopes if team is not None} - agents = {agent for _, agent in scopes if agent is not None} - if len(teams) > 1 or len(agents) > 1: - raise err(403, "metric_scope_conflict", "The related databases have incompatible permission scopes.") - fields = _related_product_fields(viewer, products) - values, reasons = automation_engine.related_product_metric_values( - viewer.runtime, pin["record"], selected, fields, routes_products.product_measures(viewer), - team_id=next(iter(teams), None), agent=next(iter(agents), None)) + fields = _related_product_fields(viewer, products, table_key) + from core import measure_resolve + if not perm_scope.may_metrics(viewer.user, table_key): + offer = [] + elif table_key == 'customer_data': + offer = measure_resolve.offer(team_id) + elif table_key == 'product_data': + offer = routes_products.product_measures(viewer) + else: + offer = ut_measures(table_key, viewer) + admitted = {m['key'] for m in offer} + fields = [f for f in fields if f['measure'].get('key') in admitted] + values = automation_engine.related_metric_values( + viewer.runtime, table_key, selected, fields, offer, compiled, team_id=team_id) return {"ok": True, "values": values, "fields": [f["key"] for f in fields], - "scope": {"customer": pin["record"], - "products": {r["pid"]: r["product_id"] for r in selected}}, - "reason": " ".join(reasons)} + "refusals": values.refusals, + "scope": {"table": table_key, "related": pin, "pids": sorted(requested), + "customer": pin.get('record'), + "products": {r['pid']: r.get('product_id') for r in selected}}, + "reason": ' '.join(dict.fromkeys(r['message'] for r in values.refusals))} except (perm_scope.Denied, perm_scope.UnknownTable): raise err(403, "metric_record_refused", "The related records are not available to this account.") except (perm_scope.Unresolvable, RuntimeError, ValueError): raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.") +def scope_linked_metrics(session, table_key, rows, fields, offer, related): + """Replace metric cells before a pinned payload can be filtered or aggregated.""" + if not related or not any(f.get('measure') for f in fields): + return rows, fields, [] + from core import perm_scope + import routes_grid + import automation_engine + + admitted = {m['key'] for m in offer} if perm_scope.may_metrics(session.user, table_key) else set() + hidden = {f['key'] for f in fields if f.get('measure') + and f['measure'].get('key') not in admitted} + fields = [f for f in fields if f['key'] not in hidden] + rows = [{k: v for k, v in row.items() if k not in hidden} for row in rows] + metrics = [f for f in fields if f.get('measure')] + if not metrics: + return rows, fields, [] + pin, team_id = routes_grid.related_measure_pin(session, table_key, related, + [row['pid'] for row in rows]) + answer = automation_engine.related_metric_values(session.runtime, table_key, rows, metrics, + offer, pin, team_id=team_id) + return [{**row, **answer[row['pid']]} for row in rows], fields, answer.refusals + + +def linked_metric_limits(refusals): + """The same refusal accompanies rows, filters and aggregate siblings.""" + if not refusals: + return [] + return [dict(subject='Linked metrics', + cause=' '.join(dict.fromkeys(marker['message'] for marker in refusals)), + recommendation='Check the metric source binding and period, then reopen this tab.')] + + def _ut(): import core.user_tables as user_tables return user_tables @@ -2277,7 +2302,8 @@ def _thin_json(fields, merged, table_key=""): @router.get("/tables/{table_key}/rows") -def table_rows(table_key: str, session: Session = Depends(require_session)): +def table_rows(table_key: str, session: Session = Depends(require_session), + related: str = None): """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, identity}` — so the client's generic topic fetch consumes it with zero new parsing. @@ -2299,7 +2325,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the # value it replaced (contract C5). _lent = _ut().lend(session.runtime) - g = ut_assembly(session, table_key, st=_lent) + g = ut_assembly(session, table_key, st=_lent, consume_corrections=not bool(related)) # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge # this line does not perform since the route was written (owner item 3, 2026-08-09: # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). @@ -2361,6 +2387,12 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. _stamp_ai_states(table_key, g["fields"], rows, st=_lent) + from routes_odoo_tables import narrow_related_rows + rows = narrow_related_rows(session, table_key, rows, related) + rows, g['fields'], metric_refusals = scope_linked_metrics( + session, table_key, rows, g['fields'], g['measures'], related) + if related: + g["pids"] = frozenset(row["pid"] for row in rows) # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this # database says so here, with its cause and the recommendation, rather than waiting to be @@ -2382,16 +2414,24 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): if str(pid) in _zone_pids]) for zone in _ut().zones(table_key, st=_lent)] return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"], + "refusals": metric_refusals, "zones": _zones, "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, admin=session.admin, st=session.runtime), "pulled_at": time.strftime("%Y-%m-%d %H:%M"), "identity": {"pid": "pid"}, "scope": {"table": table_key, "rowCount": len(rows)}, - "limits": [_report] if _report else [], + "limits": ([_report] if _report else []) + linked_metric_limits(metric_refusals), "recordsMutable": _ut().records_mutable(table_key, st=_lent)} +@router.post("/tables/{table_key}/aggregates") +def table_aggregates(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + from routes_odoo_tables import odoo_table_aggregates + return odoo_table_aggregates(table_key, body=body, session=session) + + #: The per-cell provenance a row carries on the wire, one key per enrichment column. #: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. #: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new diff --git a/platform/core/connector_sync.py b/platform/core/connector_sync.py index 457a3ae1539d6ab509e32769934148a2fa7d4db6..24f32ae4ddec014bb13b30736e9cecbe1da3e8e0 100644 --- a/platform/core/connector_sync.py +++ b/platform/core/connector_sync.py @@ -8,6 +8,7 @@ from __future__ import annotations import datetime as _dt import hashlib import json +import threading import time import uuid @@ -20,6 +21,77 @@ _ERROR_LIMIT = 50 _DEFAULT_CONNECTION = "default" _PROVIDERS = {} +# One in-flight Sync now per (provider, tenant). Only the resident resync thread drains this +# intent desk. Nothing here imports a provider, calls a runner, or writes to the mirror/store. +_SYNC_REQUESTS = {} +_SYNC_REQUEST_LOCK = threading.Lock() +_SYNC_WAKE = threading.Event() + + +def request_sync(provider, tenant, *, requested_by="", now=None): + """Queue an intent. A duplicate queued/running request returns accepted=False.""" + if provider not in _PROVIDERS: + raise ValueError("This connector is not registered for sync.") + tenant = str(tenant or "").strip() + if not tenant: + raise ValueError("A tenant is required for sync.") + with _SYNC_REQUEST_LOCK: + previous = _SYNC_REQUESTS.get((provider, tenant)) + if previous and previous["state"] in ("queued", "running"): + return {**previous, "accepted": False} + receipt = {"provider": provider, "tenant": tenant, "state": "queued", + "requestedAt": time.time() if now is None else now, + "startedAt": None, "finishedAt": None, "detail": "", + "requestedBy": str(requested_by or ""), "accepted": True} + if requested_by == "first load" and previous and previous.get("firstLoad"): + # A continuation keeps the safe numeric progress visible while waiting for the writer. + receipt.update(detail=previous["detail"], firstLoad=dict(previous["firstLoad"])) + _SYNC_REQUESTS[(provider, tenant)] = receipt + _SYNC_WAKE.set() + return dict(receipt) + + +def sync_request(provider, tenant): + """Return a detached current receipt, or None.""" + with _SYNC_REQUEST_LOCK: + receipt = _SYNC_REQUESTS.get((provider, tenant)) + if receipt is None: + return None + result = {**receipt, "accepted": False} + if "firstLoad" in result: + result["firstLoad"] = dict(result["firstLoad"]) + return result + + +def take_sync_requests(): + """Claim every queued intent atomically, on the resident resync thread.""" + with _SYNC_REQUEST_LOCK: + _SYNC_WAKE.clear() + claimed = [] + for receipt in _SYNC_REQUESTS.values(): + if receipt["state"] == "queued": + receipt.update(state="running", startedAt=time.time(), accepted=False) + claimed.append(dict(receipt)) + return claimed + + +def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load=None): + """Finish a claimed intent. The API runner supplies a safe cause sentence, never raw errors.""" + with _SYNC_REQUEST_LOCK: + receipt = _SYNC_REQUESTS.get((provider, tenant)) + if receipt is None or receipt["state"] != "running": + return None + receipt.update(state="done" if ok else "failed", detail=str(detail or "")[:300], + finishedAt=time.time() if now is None else now, accepted=False) + if first_load is not None: + receipt["firstLoad"] = dict(first_load) + return dict(receipt) + + +def sync_wake_event(): + """The event the resident thread waits on between passes.""" + return _SYNC_WAKE + # One receipt per immutable database identity. No credentials, session, or source values. JOBS_KEY = "connector_materialization_jobs" MAX_ATTEMPTS = 3 @@ -363,3 +435,5 @@ register_provider( label="Amazon", retry_hint="No sync runner is wired for this connector yet.", ) +register_provider("meta_ads", label="Meta Ads", + retry_hint="Use Sync now to retry the Meta Ads pull.") diff --git a/platform/core/dependents.py b/platform/core/dependents.py index fac771e19254ff547fd5d9f3561d2627651da995..5086758b133a06605dda03b02a0dd0a9cd19b246 100644 --- a/platform/core/dependents.py +++ b/platform/core/dependents.py @@ -47,7 +47,13 @@ walked directly instead, exactly as the shape allows. FIELD KEY (a string) of the link the rollup traverses. A rollup may also carry no `link` at all (the `{'source': {...}}` shape, which joins through a topic dim). Every walker below branches on `field['type']` first and on which key is present second. + +W60-T31 / C3: grouped kind/label/count/ids retain their meanings. Additive `items` names each +id in order and gives its address or a reason it cannot be opened. Data addresses use only +the existing #/data? prefix; workbook addresses belong to the Sheet provider (C2). """ +from urllib.parse import parse_qs, urlencode + import core.chart_system as chart_system import core.route_system as route_system import core.shares as shares @@ -60,6 +66,13 @@ from core.table_store import ASSIGNED_KEY, SHARED_KEY #: happened to register first (W49-T11 registers `sheet_cell` after the merge). KIND_ORDER = ('view', 'route', 'link', 'rollup', 'grant', 'chart', 'sheet_cell') +#: Built-in Default views are not dependents, including their saved owner copies. Their ids are +#: shared by EVERY database, so counting them charges one database's Default view to another. +#: `aios_grid.py` mints `all-customers` on every topic; its `IG_OVERVIEW_ID` is `tpl_overview`. +#: Hardcoded here because `core/` may not reach UP into `aios_grid`; `verify_shared_views.py` +#: pins this pair against that module so the declarations cannot drift. +PINNED_VIEW_IDS = frozenset({'all-customers', 'tpl_overview'}) + #: Singular / plural noun phrase per kind. ⛔ THE CASING IS R23's OWN — lowercase `views`, capital #: `Route`, capital `Sheet cells` — because the ruling quotes the sentence the user reads. _NOUNS = { @@ -75,6 +88,36 @@ _NOUNS = { _PROVIDERS = {} +DATA_HASH_PREFIX = '#/data?' + + +def data_hash(database, **params): + """Build a #/data?database=... address. The only hash builder in this module.""" + database = str(database or '').strip() + if not database: + return None + query = {'database': database} + query.update({key: str(value) for key, value in params.items() + if key != 'database' and value is not None and str(value) != ''}) + return DATA_HASH_PREFIX + urlencode(query) + + +def _dependent_item(item_id, label, href=None, reason=''): + """A named thing with an existing hash address or an explicit explanation.""" + item_id = str(item_id or '') + label = str(label or item_id) + href = href if isinstance(href, str) and href.startswith((DATA_HASH_PREFIX, '#/sheet?')) else None + return {'id': item_id, 'label': label, 'href': href, + 'reason': str(reason or '') if href else + str(reason or f'{label} has no direct address available.')} + + +def _field_item(table_key, field): + key = str(field.get('key') or '') + return _dependent_item(f'{table_key}:{key}', field.get('label') or key, + data_hash(table_key, field=key) if key else None) + + def noun(kind, count): """The noun phrase for `count` of `kind`, agreeing in number.""" pair = _NOUNS.get(str(kind or '')) @@ -120,7 +163,7 @@ def clean_target(target): return {'kind': kind, 'key': key, 'database': database} -def _dependent(kind, ids): +def _dependent(kind, ids, items=None): """A Dependent, or None when nothing depends. ⛔ NONE IS THE ZERO CASE — never `count: 0`.""" seen, out = set(), [] for raw in ids or (): @@ -130,7 +173,13 @@ def _dependent(kind, ids): out.append(value) if not out: return None - return {'kind': kind, 'label': noun(kind, len(out)), 'count': len(out), 'ids': out} + result = {'kind': kind, 'label': noun(kind, len(out)), 'count': len(out), 'ids': out} + if items is not None: + by_id = {} + for item in items: + by_id.setdefault(str(item.get('id') or ''), item) + result['items'] = [by_id.get(value) or _dependent_item(value, value) for value in out] + return result def dependents(st, target): @@ -162,10 +211,36 @@ def dependents_report(st, target): count = int(item.get('count') or len(ids) or 0) if count <= 0: continue - found.append({'kind': str(item.get('kind') or kind), + normalized = {'kind': str(item.get('kind') or kind), 'label': str(item.get('label') or noun(kind, count)), 'count': count, - 'ids': [str(i) for i in ids]}) + 'ids': [str(i) for i in ids]} + # Lane A may supply a single Sheet address on the dependent itself. Preserve + # both that shape and per-id items; neither changes the legacy count or label. + if 'href' in item: + normalized['href'] = item['href'] if isinstance(item['href'], str) else None + if 'reason' in item: + normalized['reason'] = str(item.get('reason') or '') + # W60-T12 (combine-owned), 2026-09-14: `refusal` is the OTHER half of the same fact + # and was being dropped at this seam. A provider says "no address" with `href: None` + # AND says WHY in `refusal`; carrying the null while dropping the cause turns an + # explained dead end into an unexplained one, which is the defect R23's sentence + # exists to prevent. Both keys are optional and neither changes count or label. + if 'refusal' in item: + normalized['refusal'] = str(item.get('refusal') or '') + if isinstance(item.get('items'), list): + normalized['items'] = [] + for entry in item['items']: + entry = entry if isinstance(entry, dict) else {'id': entry} + href = entry.get('href') if isinstance(entry.get('href'), str) else None + normalized['items'].append({ + 'id': str(entry.get('id') or ''), + 'label': str(entry.get('label') or entry.get('id') or ''), + 'href': href, + 'reason': str(entry.get('reason') or ('' if href else + 'This dependent has no direct address available.')), + }) + found.append(normalized) order = {kind: index for index, kind in enumerate(KIND_ORDER)} found.sort(key=lambda item: (order.get(item['kind'], len(order)), item['kind'])) return {'dependents': found, 'errors': errors} @@ -334,16 +409,27 @@ def view_dependents(st, target): """ kind, key, database = target['kind'], target['key'], target['database'] if kind == 'table': - return _dependent('view', [vid for _owner, vid, _v in walk_views(st, key)]) + items = [_dependent_item(vid, view.get('name') or vid, data_hash(key, view=vid)) + for _owner, vid, view in walk_views(st, key) if vid not in PINNED_VIEW_IDS] + return _dependent('view', [item['id'] for item in items], items=items) if kind == 'view': if not database: return None - return _dependent('view', _assigned_ids(st, database, key)) - hits = [] + ids = _assigned_ids(st, database, key) + assignments = _table_doc(st, database).get(ASSIGNED_KEY) or {} + items = [] + for item_id in ids: + recipient, _, view_id = item_id.partition(':') + view = assignments[recipient]['views'][view_id] + label = f'{recipient}: {view.get("name") or view_id}' + items.append(_dependent_item(item_id, label, data_hash(database, view=view_id))) + return _dependent('view', ids, items=items) + items = [] for _owner, view_id, view in walk_views(st, database): - if _view_names_field(view, key): - hits.append(view_id) - return _dependent('view', hits) + if view_id not in PINNED_VIEW_IDS and _view_names_field(view, key): + items.append(_dependent_item(view_id, view.get('name') or view_id, + data_hash(database, view=view_id))) + return _dependent('view', [item['id'] for item in items], items=items) def _view_names_field(view, field_key): @@ -383,21 +469,31 @@ def route_dependents(st, target): a `geographicField`, which is what makes them a FIELD dependent too. """ kind, key, database = target['kind'], target['key'], target['database'] - routes, hits = _routes(st), [] + routes, matched_routes, matched_zones = _routes(st), [], [] if kind == 'view': - hits += [str(r.get('id')) for r in routes if str(r.get('sourceViewId') or '') == key] - hits += [str(z.get('id')) for z in _zones(st, database or None) - if str(z.get('sourceViewId') or '') == key] - return _dependent('route', hits) - if kind == 'table': - view_ids = {vid for _owner, vid, _v in walk_views(st, key)} - hits += [str(r.get('id')) for r in routes - if str(r.get('sourceViewId') or '') in view_ids] - hits += [str(z.get('id')) for z in _zones(st, key)] - return _dependent('route', hits) - hits += [str(z.get('id')) for z in _zones(st, database) - if str(z.get('geographicField') or '') == key] - return _dependent('route', hits) + matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') == key] + matched_zones = [z for z in _zones(st, database or None) + if str(z.get('sourceViewId') or '') == key] + elif kind == 'table': + view_ids = {vid for _owner, vid, _v in walk_views(st, key) + if vid not in PINNED_VIEW_IDS} + # A-QI7: Routes store no source database and are created only from customer_data. + # The database test must therefore use the target table key, not the tenant-wide Route + # list or a shared view id. Zones already carry their own table scope. + if str(key) == route_system.DEFAULT_ZONE_TABLE: + matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') in view_ids] + matched_zones = _zones(st, key) + else: + matched_zones = [z for z in _zones(st, database) + if str(z.get('geographicField') or '') == key] + items = [_dependent_item(str(r.get('id')), r.get('name') or r.get('id'), + data_hash(route_system.DEFAULT_ZONE_TABLE, route=r.get('id'))) + for r in matched_routes] + for zone in matched_zones: + name = zone.get('name') or zone.get('id') + items.append(_dependent_item(str(zone.get('id')), name, reason= + f'{name} has no direct address. Edit this zone in the Map view Zones menu.')) + return _dependent('route', [item['id'] for item in items], items=items) def link_dependents(st, target): @@ -410,18 +506,18 @@ def link_dependents(st, target): kind, key = target['kind'], target['key'] if kind == 'view': return None - hits = [] + items = [] for table_key, definition in _defs(st).items(): for field in _fields(definition): bag = _link_bag(field) if bag is None: continue if kind == 'table' and str(bag.get('table') or '') == key and str(table_key) != key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) elif kind == 'field' and str(bag.get('inverse') or '') == key \ and str(bag.get('table') or '') == str(target['database']): - hits.append(f'{table_key}:{field.get("key")}') - return _dependent('link', hits) + items.append(_field_item(table_key, field)) + return _dependent('link', [item['id'] for item in items], items=items) def rollup_dependents(st, target): @@ -435,7 +531,7 @@ def rollup_dependents(st, target): kind, key = target['kind'], target['key'] if kind == 'view': return None - hits = [] + items = [] for table_key, definition in _defs(st).items(): fields = _fields(definition) by_key = {str(f.get('key') or ''): f for f in fields} @@ -449,18 +545,18 @@ def rollup_dependents(st, target): hop_bag = _link_bag(hop_field) if hop_field else None if kind == 'table': if source and str(source.get('topic') or '') == key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) elif hop_bag is not None and str(hop_bag.get('table') or '') == key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) continue if str(target['database']) and str(table_key) != str(target['database']): continue if hop == key or str(bag.get('field') or '') == key: - hits.append(f'{table_key}:{field.get("key")}') + items.append(_field_item(table_key, field)) elif source and key in (str(source.get('on') or ''), str(source.get('groupBy') or ''), str(source.get('measure') or '')): - hits.append(f'{table_key}:{field.get("key")}') - return _dependent('rollup', hits) + items.append(_field_item(table_key, field)) + return _dependent('rollup', [item['id'] for item in items], items=items) def grant_dependents(st, target): @@ -471,13 +567,21 @@ def grant_dependents(st, target): * field -> the `field` grant on `:` """ kind, key, database = target['kind'], target['key'], target['database'] + + def named_grants(names): + subject = {'view': 'view', 'table': 'database', 'field': 'field'}[kind] + items = [_dependent_item(name, name, reason= + f'{name} has no direct access address. Change their access from this ' + f'{subject}\'s share dialog.') for name in names] + return _dependent('grant', names, items=items) + if kind == 'view': - return _dependent('grant', share_audience(st, key)['names']) + return named_grants(share_audience(st, key)['names']) if kind == 'field': if not database: return None record = shares.grants('field', shares.field_oid(database, key), st=st) - return _dependent('grant', [str(e.get('user') or '') for e in (record.get('entries') or ())]) + return named_grants([str(e.get('user') or '') for e in (record.get('entries') or ())]) holders = [str(e.get('user') or '') for e in (shares.grants('database', key, st=st).get('entries') or ())] try: @@ -490,21 +594,110 @@ def grant_dependents(st, target): continue for entry in (shares.grants('field', oid, st=st).get('entries') or ()): holders.append(str(entry.get('user') or '')) - return _dependent('grant', holders) + return named_grants(holders) + + +def _sheet_chart_items(st, target): + """Sheet charts drawn over a cell that depends on `target` (W60-T12, contract C2). + + ⛔⛔ THE DEPENDENCY DECISION IS NOT REMADE HERE, AND THAT IS THE WHOLE DESIGN. Whether a CELL + depends on the target is `modules/sheet.py`'s answer, and `core/` may not reach UP into + `modules/` to ask it directly -- the same layering rule `PINNED_VIEW_IDS` is hardcoded for. So + this reads the ALREADY-REGISTERED `sheet_cell` provider's own rows and takes the cells it + named. One source of truth for the predicate, so a chart can never disagree with the cell it + is drawn over; and if nothing registered `sheet_cell` there are no sheet charts to report and + this is silent, exactly as an absent provider is everywhere else in this file. + + ⚠ A CELL WITH NO ADDRESS CONTRIBUTES NO CHART ADDRESS. `sheet_cell` answers `href: None` plus a + cause when the destination is hidden or unreadable, and a link built anyway would open some + OTHER cell -- which is the defect W60-T12 exists to close, not one to reintroduce one layer up. + """ + provider = _PROVIDERS.get('sheet_cell') + if provider is None: + return [] + try: + rows = provider(st, dict(target)) + except Exception: + return [] # a provider that raises omits its kind; so does this + wanted = {} + for row in rows if isinstance(rows, (list, tuple)) else (): + if not isinstance(row, dict) or row.get('kind') != 'sheet_cell': + continue + href = row.get('href') + if not isinstance(href, str) or not href.startswith('#/sheet?'): + continue + params = parse_qs(href.split('?', 1)[1]) + workbook_id = (params.get('wb') or [''])[0] + tab_id = (params.get('tab') or [''])[0] + addr = (params.get('cell') or [''])[0] + if workbook_id and tab_id and addr: + wanted.setdefault((workbook_id, tab_id), []).append(addr) + if not wanted: + return [] + + from core import sheets as sheet_store + items, seen = [], set() + for workbook_id in dict.fromkeys(book for book, _ in wanted): + try: + book = sheet_store.get_workbook(st, workbook_id) + except Exception: + continue # an unreadable workbook is `sheet_cell`'s to report + for tab in book.get('sheets') or (): + tab_id = str(tab.get('id') or '') + addrs = wanted.get((workbook_id, tab_id)) or [] + if not addrs: + continue + # ⚠ THE ADDRESS MATHS IS `core/sheets.py`'s, not a second copy. `_clean_range` accepts + # both `"B4"` and the chart's `{col0,row0,col1,row1}` and answers the same zero-based + # rectangle for each, so a cell and the chart over it cannot drift apart. + cells = [rect for rect in (sheet_store._clean_range(a) for a in addrs) if rect] + for chart in tab.get('charts') or (): + if not isinstance(chart, dict): + continue + chart_id = str(chart.get('id') or '') + rect = sheet_store._clean_range(chart.get('range')) + if not chart_id or chart_id in seen or not rect: + continue + if not any(rect['col0'] <= c['col0'] <= rect['col1'] + and rect['row0'] <= c['row0'] <= rect['row1'] for c in cells): + continue + seen.add(chart_id) + label = str(chart.get('name') or chart.get('title') or '').strip() or ( + f"Chart in {book.get('name') or workbook_id} / {tab.get('name') or tab_id}") + items.append(_dependent_item(chart_id, label, '#/sheet?' + urlencode( + {'wb': workbook_id, 'tab': tab_id, 'chart': chart_id}))) + return items def chart_dependents(st, target): - """Saved charts filed against the target DATABASE. + """Saved charts filed against the target DATABASE, and Sheet charts drawn over its cells. ⛔ A CHART NEVER DEPENDS ON A VIEW, and that is `chart_system`'s stated contract rather than an omission here: a chart's `config` is OPAQUE and this file must not start reading keys out of it to guess at a reference. Deleting a view cannot delete a chart, so it must not disclose one. + + ⭐ W60-T12 (combine-owned) ADDED THE SECOND SOURCE. A Sheet chart is not reachable from + `sheet_dependents` at all -- it is a sibling of the cells, not one of them -- so before this a + database could be switched off while a chart still drew from it, and R23's refusal would not + have said so. Charts are not read from the sheet document here; see `_sheet_chart_items`. """ kind, key = target['kind'], target['key'] if kind != 'table': return None - return _dependent('chart', [str(c.get('id')) for c in _charts(st) - if str(c.get('database') or '') == key]) + items = [_dependent_item(str(c.get('id')), c.get('name') or c.get('id'), c.get('href'), + '' if c.get('href') else + 'This chart has no direct address. Open it from Charts.') + for c in _charts(st) if str(c.get('database') or '') == key] + items += _sheet_chart_items(st, target) + row = _dependent('chart', [item['id'] for item in items], items=items) + # ⭐ WHEN EXACTLY ONE CHART DEPENDS ON THE TARGET, THE ROW ADDRESSES ONE THING, so it carries + # that address itself -- the single-address-on-the-dependent shape `dependents_report` already + # preserves. ⛔ With several it carries none: a row-level link would have to pick one chart and + # would silently mean the others, which is the naming-by-display defect W60-T12 closed. + if row and len(items) == 1: + row['href'] = items[0]['href'] + row['reason'] = items[0]['reason'] + return row register('view', view_dependents) diff --git a/platform/core/measure_resolve.py b/platform/core/measure_resolve.py index 53633d1226c3177a498bfec750894b5cd1a6a3b2..bb65aa676899b4f42a373e866b4a8c28f4ab3597 100644 --- a/platform/core/measure_resolve.py +++ b/platform/core/measure_resolve.py @@ -152,8 +152,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No `related` is the row-window {table, record, via} pin. The host binds its scoped readers with relations.measure_pin_context and publishes the returned mapping's `refusals` beside - its cells. A refused pin retains unscoped values plus a per-field named marker, on memo - hits too. Every measure and its conditions use the same query scope and caller's pool. + its cells. A refused pin publishes a per-field marker and no values, on memo hits too. + Every measure and its conditions use the same query scope and caller's pool. """ mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)] if not mfields: @@ -197,6 +197,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No if related is not None: out.refusals.extend(dict(marker, field=fkey) for marker in getattr(vals, 'refusals', ())) + if getattr(vals, 'refusals', ()): + continue for pid, v in vals.items(): out.setdefault(pid, {})[fkey] = v if len(memo) > 100: diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index de0a35a7606287201446a71f6e90e21c10b35c06..f7494bdcc99bff23462183cbbd6818f34fab7ef5 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -40,6 +40,7 @@ twice in this codebase (`modules: []` and `bus: []` both read as UNR perms block; without this clause an explicit-marker scheme locks the owner out of their own product at exactly the moment the store is broken. """ +import inspect import re import core.perms as perms @@ -1400,7 +1401,7 @@ class Unresolvable(RuntimeError): #: Row readers DECLARED by the app layer, keyed by EXACT database key. -#: `reader(table_key, user, st) -> (fields, rows)`. +#: `reader(table_key, user, st[, want_rows=True]) -> (fields, rows)`. #: #: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and #: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache @@ -1409,6 +1410,7 @@ class Unresolvable(RuntimeError): #: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being #: interrogated by it."* _ROW_SOURCES = {} +_ROW_SOURCE_WANT_ROWS = {} #: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror. #: `reader(table_key, field_keys, st) -> rows`. @@ -1423,10 +1425,18 @@ def register_rows(reader, *table_keys): says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write door's return keeps one construction site of the set instead of two. """ + # Decide the calling convention once. Never retry a reader's own TypeError as an old + # signature: doing so can silently replace a failed read with a different answer. + parameters = inspect.signature(reader).parameters + accepts_want_rows = any(p.kind == inspect.Parameter.VAR_KEYWORD + for p in parameters.values()) or ( + 'want_rows' in parameters and parameters['want_rows'].kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)) for key in table_keys: k = str(key or '').strip() if k: _ROW_SOURCES[k] = reader + _ROW_SOURCE_WANT_ROWS[k] = accepts_want_rows return frozenset(_ROW_SOURCES) @@ -1568,8 +1578,8 @@ def scoped_fields(user, table_key, st=None): second path to the store (W1). So both come from here, off one wall. ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a - registry topic it goes through the registered reader, which builds that topic's pool; the - pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`. + registry topic it asks the registered reader for fields only when supported. Legacy + three-argument readers retain their existing call contract. """ fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False) return fields @@ -1613,7 +1623,10 @@ def _read(table_key, user, st, want_rows=True): """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`.""" reader = _ROW_SOURCES.get(table_key) if reader is not None: - fields, rows = reader(table_key, user, st) + if _ROW_SOURCE_WANT_ROWS.get(table_key, False): + fields, rows = reader(table_key, user, st, want_rows=want_rows) + else: + fields, rows = reader(table_key, user, st) return list(fields or ()), list(rows or ()) ut = _ut() if not table_key.startswith(ut.KEY_PREFIX): diff --git a/platform/core/relations.py b/platform/core/relations.py index 301c18461cfe04f98d3f886d6870b5a31b9a925c..f07cc091b4b258506c338ebb321c6697b4a1d41a 100644 --- a/platform/core/relations.py +++ b/platform/core/relations.py @@ -745,6 +745,10 @@ def measure_pin_key(related): if related is None: return None ctx = _MEASURE_PIN_CONTEXT.get() or {} + if isinstance(related, MeasurePin): + # Compiled pins contain the server's walled membership, not just a wire path. + related = dict(path=related.path, rows=related.rows, bindings=related.bindings, + refusal=related.refusal, required=related.required) return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key') @@ -754,11 +758,17 @@ class MeasurePin: rows: dict bindings: dict refusal: dict = None + required: tuple = () def predicate(self, store): """AND every declared, traversed member set into this metric's raw rows.""" if self.refusal is not None: return None, [] + bound = {b['table'] for b in self.bindings.get(store['table'], ())} + if set(self.required) - bound: + raise PinRefusal('relation_binding_unavailable', + 'This metric cannot be restricted to every database in this record path. ' + 'Ask an administrator to add the missing metric binding.', self.path) parts, params = [], [] for binding in self.bindings.get(store['table'], ()): table, column = binding['table'], binding['column'] @@ -1126,6 +1136,15 @@ def one_hop(table, fields=None, defs=None, include_retired=False): out = [] for entry in base: entry = dict(entry) + # Keep related()'s legacy shape. Depth-aware pins also need the declared manual, + # inverse or through configuration, which remains schema-only cache input. + via = entry.get('via') or {} + owner_fields = (((defs or {}).get(entry['target']) or {}).get('fields') or [] + if via.get('direction') == 'reverse' else fields or []) + declared = next((field for field in owner_fields + if field.get('key') == via.get('field')), None) + if declared and _link_bag(declared): + entry['link'] = dict(_link_bag(declared)) got = cards.get(entry['key']) or bridge_cards.get(entry['key']) entry['cardinality'] = str(got or 'many_to_many') out.append(entry) diff --git a/platform/core/sheet_templates.py b/platform/core/sheet_templates.py index b393ff560a24262bb0f339bf3260995bfdaaed7f..87e86aa0664caccfb3e7190fa844adf6128c24e9 100644 --- a/platform/core/sheet_templates.py +++ b/platform/core/sheet_templates.py @@ -43,6 +43,118 @@ def _scope(*conditions): for field, op, value in conditions]} +def _sales_summary(): + """W60-T08: Field aggregates need date predicates, not a Metric-only period. + + The existing T06 valueRef reader resolves these formula dates and dropdowns; + T40 resolves the shared customer field. No raw records enter the workbook. + """ + tabs, bindings, formats = [], [], [] + for index, name in enumerate(("Summary (All)", "Summary (SR)")): + cells = { + "A1": _literal("Daily sales performance"), + "A2": _literal("Ending date"), "B2": _literal(date.today().isoformat()), + "A3": _literal("Business unit"), + "B3": {**_literal("Royal"), "validation": {"list": ["Fisch", "Royal"]}}, + "A4": _literal("Assigned agent" if index else "Scope"), + "B4": ({**_literal("Choose an assigned agent"), + "validation": {"list": ["Choose an assigned agent"]}} if index else + _literal("Confirmed wholesale orders, untaxed, by order date")), + "A6": _literal("Week starting"), "A7": _literal("Before"), + "A8": _literal("Sales, fully invoiced"), "A9": _literal("Confirmed sales orders"), + "A10": _literal("Order count"), "A11": _literal("Average order value"), + "A14": _literal("Year to date"), "B14": _literal("Current year"), + "C14": _literal("Previous year"), + "A15": _literal("From"), "A16": _literal("Before"), + "B15": _formula("=DATE(YEAR(B2),1,1)"), + "C15": _formula("=DATE(YEAR(B2)-1,1,1)"), + "B16": _formula('=DATEADD(B2,1,"days")'), + "C16": _formula('=DATEADD(EDATE(B2,-12),1,"days")'), + "A17": _literal("Sales, fully invoiced"), "A18": _literal("Confirmed sales orders"), + "A19": _literal("Order count"), "A20": _literal("Average order value"), + "A23": _literal("Sales basis"), + "B23": _literal("Fully invoiced means invoice_status = invoiced, by order date. " + "Partial invoices and invoice-date totals are not included."), + "A24": _literal("Workbook difference"), + "B24": _literal("Uses confirmed orders and order date. The Excel workbook uses " + "creation dates and all states, so its totals differ by design."), + "A25": _literal("Assigned agent setup"), + "B25": _literal("Create Assigned agent on Customers using the column menu, add " + "agent choices, then share the field. It starts empty; no Odoo " + "salesperson or commission-agent values are copied."), + "A27": _literal("Workbook exclusions"), + "B27": _literal("HubSpot activities and companies need a connector. Delivery time " + "needs a validated delivery-completion measure. Raw tabs, Backend, " + "Mastersheet and the other customer worklists are outside this summary template."), + "A28": _literal("Import limits"), + "B28": _literal("The source is a 39 MB .xlsm; import accepts .xlsx up to 25 MB. " + "Use this Cell database summary or export smaller .xlsx reports. " + "Excel macros and print automation are not imported."), + "A29": _literal("Formula limits"), + "B29": _literal("Ranges above 5,000 cells and tabs above 100,000 cells are refused. " + "Eight source tabs exceed the tab cap. Use Cell database aggregates " + "instead of raw dumps and whole-column formulas."), + "A30": _literal("Long formulas"), + "B30": _literal("26,934 source Mastersheet formulas exceed 500 characters. " + "Move those calculations to database fields or split them into smaller formulas."), + "A31": _literal("Workbook size"), + "B31": _literal("Saved workbooks are limited to 64 MiB. The expanded raw workbook " + "has not been measured; keep data in databases and aggregate it into cells."), + } + if index: + cells["B2"] = _formula("='Summary (All)'!B2") + cells["B3"] = _formula("='Summary (All)'!B3") + for offset, col in enumerate("BCDEFGHIJKLMN"): + cells[f"{col}6"] = _formula( + f'=DATEADD($B$2,1-WEEKDAY($B$2,2)-{(12-offset)*7},"days")') + cells[f"{col}7"] = _formula(f'=DATEADD({col}6,7,"days")') + + for col, start, end, first in [*( (c, f"{c}6", f"{c}7", 8) for c in "BCDEFGHIJKLMN"), + ("B", "B15", "B16", 17), ("C", "C15", "C16", 17)]: + filters = [ + {"colId": "order_date", "op": "gte", "valueRef": {"ref": start}}, + {"colId": "order_date", "op": "lt", "valueRef": {"ref": end}}, + {"colId": "business_unit", "op": "eq", "valueRef": {"ref": "'Summary (All)'!$B$3"}}, + {"colId": "wholesale", "op": "eq", "value": "1"}, + {"colId": "state", "op": "isAnyOf", "values": ["sale", "done"]}, + ] + if index: + filters.append({"colId": "assigned_agent", "op": "eq", + "valueRef": {"ref": "$B$4"}}) + for row, fn, field in ((first, "sum", "revenue"), + (first+1, "sum", "revenue"), (first+2, "count", "order")): + nodes = deepcopy(filters) + if row == first: + nodes.append({"colId": "invoice_status", "op": "eq", "value": "invoiced"}) + bindings.append({"sheet": index, "addr": f"{col}{row}", "field": field, + "fn": fn, "scope": {"filter": nodes}}) + cells[f"{col}{first+3}"] = _formula( + f'=IF({col}{first+2}=0,0,{col}{first+1}/{col}{first+2})') + formats.extend({"sheet": index, "addr": f"{col}{row}", "kind": kind} + for row, kind in ((first, "currency"), (first+1, "currency"), + (first+2, "number"), (first+3, "currency"))) + formats.extend({"sheet": index, "addr": addr, "kind": "date"} + for addr in ["B2", "B15", "C15", "B16", "C16", + *(f"{col}{row}" for col in "BCDEFGHIJKLMN" for row in (6, 7))]) + for addr, cell in cells.items(): + if addr.startswith("A") or addr in ("B14", "C14"): + cell["style"] = {"bold": True} + tabs.append({"name": name, "cells": cells}) + return { + "key": "odoo_daily_sales_performance", "title": "Daily sales performance", + "source": "odoo", "database_hint": "ut_odoo_orders", + "answers": "How do thirteen weeks and year-to-date sales compare by business unit " + "and the customer's shared Assigned agent? Uses confirmed, untaxed orders " + "by order date, so totals differ from the Excel workbook.", + "semantic_fields": {"order_date": ("order_date",), "revenue": ("amount_untaxed",), + "order": ("odoo_id",), "business_unit": ("team",), + "wholesale": ("wholesale_scope",), "state": ("state",), + "invoice_status": ("invoice_status",), + "assigned_agent": ("linked:bridge_customer_orders:custom_assigned_agent",)}, + "sheets": tabs, "bindings": bindings, "formats": formats, + } + + # C4 has exactly two additions: answers supplies the gallery's short question # summary; semantic_fields lets bindings name concepts such as "revenue" rather # than tenant column ids. Tenants can map that concept to different source keys. @@ -50,6 +162,7 @@ def _scope(*conditions): # an applied workbook never silently moves a labelled period to another month. _AS_OF = date.today() TEMPLATES = [ + _sales_summary(), { "key": "odoo_monthly_sales", "title": "Monthly sales", "source": "odoo", "database_hint": "ut_odoo_orders", @@ -205,8 +318,15 @@ def build_cells(template, resolved, database_key): for entry in template["bindings"]: cells, addr = target(entry) + if set(entry) - {"sheet", "addr", "fn", "field", "metric_key", "scope", "period"}: + raise TemplateRefusal("template_invalid", detail="The template binding has an unknown setting.") + aggregate = ({"metric_key": entry["metric_key"]} if "metric_key" in entry else + {"fn": entry["fn"], "field": resolved[entry["field"]]}) + if "metric_key" in entry and (not entry["metric_key"] or "field" in entry or "fn" in entry + or not entry.get("period")): + raise TemplateRefusal("template_invalid", detail="A template metric needs a period and no field aggregate.") binding = {"database": database_key, - "aggregate": {"fn": entry["fn"], "field": resolved[entry["field"]]}, + "aggregate": aggregate, "scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)} if "period" in entry: binding["period"] = deepcopy(entry["period"]) @@ -224,6 +344,40 @@ def build_cells(template, resolved, database_key): return tabs +def _sales_summary_inputs(st, descriptor, created_by): + """Read existing shared field metadata only; creation and promotion stay in the column menu.""" + from core import perm_scope, shared_overlay, users, view_templates + user = {**(st.get("users").get(created_by) or {}), "username": created_by} + if not perm_scope.may_access(user, "customer_data"): + raise TemplateRefusal("template_fields_missing", missing=("a readable shared Assigned agent field",)) + shared = shared_overlay.fields(view_templates.workspace_key("customer_data"), st=st) + visible = perm_scope.visible_fields(list(shared.values()), user, "customer_data", st=st) + candidates = [f for f in visible if f.get("label", "").casefold() in + ("assigned agent", "w60-t08 assigned agent") and f.get("type") in ("text", "select", "user")] + if len(candidates) != 1: + raise TemplateRefusal("template_fields_missing", missing=( + "one shared Assigned agent field on Customers (create it with the column menu and share it)",)) + field = candidates[0] + descriptor = deepcopy(descriptor) + key = "linked:bridge_customer_orders:" + field["key"] + descriptor["semantic_fields"]["assigned_agent"] = (key,) + # BU choice metadata follows the same permission helper as the rest of the product. + bus = [label for label in users.allowed_bus_labels(user) if label != "All"] + all_cells, sr_cells = (tab["cells"] for tab in descriptor["sheets"]) + all_cells["B2"] = _literal(date.today().isoformat()) + all_cells["B3"] = {**_literal(bus[0]), "validation": {"list": bus}} + choices = [str(option.get("value") or option.get("label") or "") if isinstance(option, dict) + else str(option) for option in field.get("options", [])] + choices = list(dict.fromkeys(choice for choice in choices if choice.strip())) + if choices: + sr_cells["B4"] = {**_literal(choices[0]), "validation": {"list": choices[:200]}} + if len(choices) > 200: + sr_cells["A26"] = _literal("Agent choices") + sr_cells["B26"] = _literal(f"Showing 200 of {len(choices)} choices. The dropdown holds at most " + "200; type another agent value or reduce the field's choices.") + return descriptor, key + + def apply_template(st, key, database_key, *, name="", created_by=""): descriptor = template(key) if descriptor is None: @@ -233,11 +387,19 @@ def apply_template(st, key, database_key, *, name="", created_by=""): raise TemplateRefusal("unknown_database", detail="That database does not exist.") if source_of_database(database_key) != descriptor["source"]: raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.") - resolved = resolve_fields(descriptor, (field["key"] for field in definition.get("fields", []))) + field_keys = [field["key"] for field in definition.get("fields", [])] + if key == "odoo_daily_sales_performance": + descriptor, linked_key = _sales_summary_inputs(st, descriptor, created_by) + field_keys.append(linked_key) + resolved = resolve_fields(descriptor, field_keys) prepared = build_cells(descriptor, resolved, database_key) # Validate tab names, extents and every cell before the first store write. checked = sheets.migrate_workbook({"sheets": [ {"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]}) + if key == "odoo_daily_sales_performance": + for tab in checked["sheets"]: + tab["colWidths"] = {"A": 230, **{col: 125 for col in "BCDEFGHIJKLMN"}} + tab["freeze"] = {"rows": 7, "cols": 1} book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by) try: tabs = [{**tab, "cells": {}} for tab in checked["sheets"]] diff --git a/platform/core/sheets.py b/platform/core/sheets.py index c80931acb68d3fdd2763f09116f14a64006dcdba..a8331cffce91bb45488051242f522e48c2e4d768 100644 --- a/platform/core/sheets.py +++ b/platform/core/sheets.py @@ -81,6 +81,7 @@ AMAZON_DATA_ORIGIN_NAME = "Amazon Services API" SHEET_SCHEMA_VERSION = 1 DOCUMENT_MAX_COLS = 16384 DOCUMENT_MAX_ROWS = 1048576 +WRITE_LOG_REVS = 200 #: ⛔ THE FROZEN VOCABULARY (contract C1, amended by C2). `verify_api.py` and `verify_sheet.py` #: assert `web/src/sheet/types.ts` carries exactly these, name for name. @@ -230,7 +231,7 @@ SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", #: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata. SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze", "rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights", - "visibility", "print", "excel") + "visibility", "view", "print", "excel") #: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of #: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab @@ -1197,6 +1198,38 @@ def _clean_print(raw): return out +#: Wave 60 base kit: `SheetTabView` in `sheet/types.ts`. Display settings, so a bad value is +#: DROPPED rather than refused: a sheet must never fail to save over a zoom level. +_VIEW_ZOOM_MIN, _VIEW_ZOOM_MAX = 10, 400 +_VIEW_PAGE_VIEWS = ("normal", "pageBreak", "pageLayout") + + +def _clean_view(raw): + """A tab's display settings: gridlines, headings, zoom, pageView and split, nothing else. + + Booleans stay booleans, zoom clamps to 10..400 (a non-number is dropped), pageView keeps only + the three words, split keeps non-negative integer rows and cols or is dropped. An empty + result is None, so the tab stores no empty stratum. + """ + if not isinstance(raw, dict): + return None + out = {} + for key in ("gridlines", "headings"): + if isinstance(raw.get(key), bool): + out[key] = raw[key] + zoom = raw.get("zoom") + if not isinstance(zoom, bool) and isinstance(zoom, (int, float)) and math.isfinite(zoom): + out["zoom"] = max(_VIEW_ZOOM_MIN, min(_VIEW_ZOOM_MAX, zoom)) + if raw.get("pageView") in _VIEW_PAGE_VIEWS: + out["pageView"] = raw["pageView"] + split = raw.get("split") + if isinstance(split, dict): + parts = [split.get("rows"), split.get("cols")] + if all(isinstance(n, int) and not isinstance(n, bool) and n >= 0 for n in parts): + out["split"] = {"rows": parts[0], "cols": parts[1]} + return out or None + + def _clean_imported(raw): """C4's import record: when, from what, and WHAT DID NOT SURVIVE. @@ -1269,6 +1302,7 @@ def _clean_sheet(raw, index=0, *, lossless=False): "rowHeight": _clean_base_row_height(raw.get("rowHeight")), "rowHeights": _clean_row_heights(raw.get("rowHeights")), "visibility": visibility, + "view": _clean_view(raw.get("view")), "print": _clean_print(raw.get("print")), "excel": excel, } @@ -1287,7 +1321,7 @@ def _clean_sheet(raw, index=0, *, lossless=False): if value: sheet[key] = value extensions = {key: copy.deepcopy(value) for key, value in raw.items() - if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount"}} + if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount", "writes", "writesFrom"}} sheet.update(_json_safe(extensions, "sheet extensions")) if lossless: sheet["cols"] = max(cols, (sheet.get("freeze") or {}).get("cols", 0)) @@ -1346,10 +1380,11 @@ def new_workbook(name="", created_by="", workbook_id=""): "name": _text(name, _MAX_NAME) or "Untitled workbook", "sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1", "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {}, - "usedExtent": None}], + "usedExtent": None, "writesFrom": 0}], "rev": 1, "refreshed_at": None, "created_by": _text(created_by, _MAX_ID), + "access": "private", "created_at": stamp, "updated_at": stamp, } @@ -1429,6 +1464,30 @@ def create_workbook(st, name="", created_by=""): return book +def _carry_write_logs(book, prior): + """Only stored tabs may supply write history; client metadata never owns it.""" + held = {tab.get("id"): tab for tab in prior.get("sheets") or []} + for tab in book.get("sheets") or []: + source = held.get(tab.get("id")) or {} + tab.pop("writes", None) + tab["writesFrom"] = int(source.get("writesFrom") or 0) + if "writes" in source: + tab["writes"] = copy.deepcopy(source["writes"]) + return book + + +def _append_write_log(tab, rev, addrs, *, actor="", actor_name=""): + """Record accepted addresses and the oldest revision with complete retained history.""" + if not addrs: + return + writes = [*(tab.get("writes") or []), {"rev": rev, "by": actor, + "byName": actor_name or actor, "addrs": list(addrs)}] + dropped = writes[:-WRITE_LOG_REVS] + if dropped: + tab["writesFrom"] = max(int(tab.get("writesFrom") or 0), int(dropped[-1]["rev"])) + tab["writes"] = writes[-WRITE_LOG_REVS:] + + def get_workbook(st, workbook_id): """One workbook, or a named refusal when the document is not there.""" wid = _slug_id(workbook_id) @@ -1436,13 +1495,13 @@ def get_workbook(st, workbook_id): if not doc or not doc.get("id"): raise SheetRefusal("workbook_missing", "That workbook no longer exists.") - return migrate_workbook(doc) + return _carry_write_logs(migrate_workbook(doc), doc) -def put_workbook(st, workbook_id, raw, expect_rev=None): +def put_workbook(st, workbook_id, raw, expect_rev=None, *, actor="", actor_name=""): """Replace a workbook's name and sheets, bumping `rev`. - ⛔ `id`, `created_by` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by + ⛔ `id`, `created_by`, `access` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by sending a different id, and a client that never read the document cannot blank its provenance. ⚠ `expect_rev` makes the write CONDITIONAL, and when it is not passed explicitly the `rev` @@ -1472,9 +1531,10 @@ def put_workbook(st, workbook_id, raw, expect_rev=None): if prior is None: raise SheetRefusal("workbook_missing", "That workbook no longer exists.") if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev): - raise SheetRefusal( + raise SheetStaleRefusal( "workbook_revision_stale", - "That workbook changed somewhere else, so this save was not applied.") + "That workbook changed somewhere else, so this save was not applied.", + rev_current=int(prior.get("rev") or 0)) book = migrate_workbook(prior) # Immutable ownership and server read projections cannot be reassigned by a PUT. # ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may @@ -1483,16 +1543,24 @@ def put_workbook(st, workbook_id, raw, expect_rev=None): # ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an # unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned # cells by ``_merge_data_origins`` immediately below. - protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at", + protected = {"id", "created_by", "access", "created_at", "updated_at", "rev", "refreshed_at", "sheets", "name", "refreshedFor", "shares", "role", "permissions", "dataOrigins"} book.update({key: value for key, value in incoming.items() if key not in protected}) book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook" - book["sheets"] = sheets + book["sheets"] = copy.deepcopy(sheets) + _carry_write_logs(book, prior) # DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is # ignored, so a client cannot erase or replace the server-maintained attribution list. _merge_data_origins(book, sheets=sheets) book["rev"] = int(prior.get("rev") or 0) + 1 + prior_tabs = {tab.get("id"): tab for tab in prior.get("sheets") or []} + for tab in book["sheets"]: + before = (prior_tabs.get(tab.get("id")) or {}).get("cells") or {} + after = tab.get("cells") or {} + changed = sorted(addr for addr in before.keys() | after.keys() + if before.get(addr) != after.get(addr)) + _append_write_log(tab, book["rev"], changed, actor=actor, actor_name=actor_name) if refreshed_at is not None: book["refreshed_at"] = _text(refreshed_at, 120) or None book["updated_at"] = _now() @@ -1574,7 +1642,7 @@ def workbook_meta(book): return meta -def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): +def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None, *, actor="", actor_name=""): """Merge a RANGE of cells into ONE tab. Returns `{rev, tab, refused}` (contract C3). ⭐⭐ ONE BAD CELL DOES NOT THROW THE COMMIT AWAY. Every address is cleaned on its own and a @@ -1640,13 +1708,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): if prior is None: raise SheetRefusal("workbook_missing", "That workbook no longer exists.") current = int(prior.get("rev") or 0) - if expect_rev is not None and current != int(expect_rev): - raise SheetStaleRefusal( - "sheet_revision_stale", - "This sheet changed somewhere else while you had it open, so this save was not " - "applied. Reload the tab and try again.", - rev_current=current) - book = migrate_workbook(prior) + book = _carry_write_logs(migrate_workbook(prior), prior) tabs = book.get("sheets") or [] at = _tab_index(tabs, tab_ref) if at is None: @@ -1654,6 +1716,21 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): "tab_missing", "Tab " + (str(tab_ref) or "(unnamed)") + " is not in this workbook.") tab = tabs[at] + writes = tab.get("writes") or [] + touched = list(dict.fromkeys([*merged, *deleted])) + conflicts = {} + if touched and expect_rev is not None and int(expect_rev) < current: + revision = int(expect_rev) + if revision < int(tab.get("writesFrom") or 0): + raise SheetStaleRefusal( + "sheet_revision_stale", + "This tab's retained change history was truncated after your revision. Reload the tab and try again.", + rev_current=current) + for write in writes: + if int(write["rev"]) > revision and write["by"] != actor: + for addr in sorted(set(touched).intersection(write["addrs"])): + conflicts[addr] = {"addr": addr, "by": write["by"], + "byName": write.get("byName") or write["by"]} cells = dict(tab.get("cells") or {}) for key in deleted: cells.pop(key, None) @@ -1670,6 +1747,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): book["sheets"] = tabs _merge_data_origins(book, sheets=tabs) book["rev"] = current + 1 + _append_write_log(tab, book["rev"], touched, actor=actor, actor_name=actor_name) book["updated_at"] = _now() _json_safe(book, "workbook") out["rev"] = book["rev"] @@ -1677,6 +1755,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): out["cells"] = len(cells) out["schemaVersion"] = SHEET_SCHEMA_VERSION out["usedExtent"] = tab["usedExtent"] + out["conflicts"] = list(conflicts.values()) # Server-cleaned keys only. The route consumes this delta for calculation context. out["accepted"] = list(dict.fromkeys([*merged, *deleted])) saved.update(book) @@ -1883,6 +1962,19 @@ def _index_upsert(st, book): #: registry's words, so a second spelling cannot be introduced here by accident. +def narrow_workbook(st, workbook_id): + """Persist explicit sharing's one way visibility decision on the latest document.""" + wid = _slug_id(workbook_id) + + def _apply(state): + if not state.get("id"): + raise SheetRefusal("workbook_missing", "That workbook no longer exists.") + state.setdefault("access", "private") + return state, None + + _persist(st, workbook_key(wid), _apply) + + def workbook_shares(st, workbook_id): """C5's `shares` list for ONE workbook, computed from `core.shares` and never stored. @@ -1904,11 +1996,31 @@ def workbook_shares(st, workbook_id): return [{"who": e["user"], "role": e["role"]} for e in (rec.get("entries") or [])] -def project_shares(st, rows): +def workbook_role(st, book, viewer, *, is_admin=False): + """Document ownership plus the existing grant reader, shared with the route's role helper.""" + from core import shares + me = str(viewer or "").strip().lower() + if me and str(book.get("created_by") or "").strip().lower() == me: + return "owner" + return shares.role_for("workbook", book.get("id"), me, is_admin=is_admin, st=st) + + +def may_read_workbook(book, role): + """R28: a role admits a reader; unmarked or creatorless documents remain legacy-visible. + + Creation and explicit sharing stamp access. Read the document every time: + _index_upsert rebuilds its row after every save, and PUT must preserve marker absence too. + Legacy visibility grants no ownership or editing role. + """ + return bool(role) or "access" not in book or not book.get("created_by") + + +def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=None): """Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched. - ⚠ Called by `read_index`, `save_folders` and `routes_sheet.list_sheets` — the three doors a - client reads the rail through. `list_workbooks` itself stays a bare pointer list on purpose: + A named viewer filters rows against each live workbook document and receives their role. + The route supplies its existing role helper; core callers use workbook_role directly. + `list_workbooks` itself stays a bare pointer list on purpose: `iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry read per workbook there would put a permission lookup inside a scan that asks no permission question at all. @@ -1918,6 +2030,15 @@ def project_shares(st, rows): if not isinstance(row, dict) or not row.get("id"): continue joined = dict(row) + if viewer: + book = _read(st, workbook_key(row["id"])) + if not book or not book.get("id"): + continue + role = (role_on_workbook(book) if role_on_workbook is not None + else workbook_role(st, book, viewer, is_admin=is_admin)) + if not may_read_workbook(book, role): + continue + joined["role"] = role or "view" joined["shares"] = workbook_shares(st, row["id"]) out.append(joined) return out @@ -2374,7 +2495,31 @@ def _index_rev(doc): return 0 -def read_index(st, viewer=None): +def _visible_rail(folders, placed, starred, ids): + """Prune hidden placements, stars and folders occupied only by hidden workbooks. + + Keep visible folders' ancestors and intentionally empty folders. Otherwise creating an + empty folder would immediately hide it from the person who needs to put a workbook in it. + """ + parents = {f["id"]: f.get("parent") for f in folders} + + def ancestors(folder_ids): + found = set() + for fid in folder_ids: + while fid in parents and fid not in found: + found.add(fid) + fid = parents[fid] + return found + + visible = {wid: fid for wid, fid in placed.items() if wid in ids} + occupied = ancestors(placed.values()) + needed = ancestors(set(visible.values()) | (parents.keys() - occupied)) + hidden_only = occupied - needed + return ([f for f in folders if f["id"] not in hidden_only], visible, + _clean_starred(starred, ids, [])) + + +def read_index(st, viewer=None, *, is_admin=False, role_on_workbook=None): """The whole rail document: `{workbooks, folders, itemFolders, starred, rev}`, plus `sharedWithMe` when a `viewer` is named. @@ -2399,6 +2544,7 @@ def read_index(st, viewer=None): doc = _read(st, INDEX_KEY) rows = [dict(r) for r in (doc.get("workbooks") or []) if isinstance(r, dict) and r.get("id")] + rows = project_shares(st, rows, viewer, is_admin=is_admin, role_on_workbook=role_on_workbook) ids = {r["id"] for r in rows} placements = doc.get("itemFolders") try: @@ -2407,11 +2553,14 @@ def read_index(st, viewer=None): flat = [dict(f, parent=None) for f in (doc.get("folders") or []) if isinstance(f, dict) and f.get("id")] folders, placed, _repairs = validate_folder_tree(flat, placements) + starred = _clean_starred(doc.get("starred"), ids, []) + if viewer: + folders, placed, starred = _visible_rail(folders, placed, doc.get("starred"), ids) out = { - "workbooks": project_shares(st, rows), + "workbooks": rows, "folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids}, - "starred": _clean_starred(doc.get("starred"), ids, []), + "starred": starred, "rev": _index_rev(doc), } if viewer: diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 897cc75b80971f01362bbaefb841b26b3b1626e7..02c9f7296ae4f4c8c2b74f186e196d3bf20d5bd0 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -4473,10 +4473,13 @@ def may_edit_database(table_key, viewer, is_admin=False, st=None): #: #: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through #: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the -#: runtime through `__getattr__` — so this can serve a stale value for exactly two keys, for +#: runtime through `__getattr__` — so this can serve a stale value for the two named keys and +#: dynamically named `__shared` overlay buckets, for #: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level #: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS -#: `createdBy`). +#: `createdBy`). Shared overlays are also read-only on this path: `_shared_value_keys` only +#: reads their field keys, never mutates the document. Their first read is lent for one pass; +#: `fresh=True` and write verbs still reach the runtime, never the snapshot. _LENDABLE = (STORE_KEY, 'object_shares') @@ -4487,7 +4490,7 @@ class _Lent: self._runtime, self._docs = runtime, dict(docs or {}) def get(self, name, fresh=False): - if fresh or name not in _LENDABLE: + if fresh or (name not in _LENDABLE and not str(name).endswith('__shared')): return self._runtime.get(name, fresh) if fresh else self._runtime.get(name) if name not in self._docs: self._docs[name] = self._runtime.get(name) # lazily, once, for this pass @@ -4507,7 +4510,7 @@ class _Lent: A re-projection would cost a second copy to convert the first case and change nothing about the second. """ - if name not in _LENDABLE: + if name not in _LENDABLE and not str(name).endswith('__shared'): return self._runtime.get_projection(name, drop=drop) if name not in self._docs: self._docs[name] = self._runtime.get_projection(name, drop=drop) @@ -4518,7 +4521,7 @@ class _Lent: def lend(st=None, **docs): - """A handle that serves `docs` for `_LENDABLE` keys and defers everything else to `st`. + """Lend `_LENDABLE` keys and `__shared` suffix buckets; defer other keys to `st`. `lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and hand the result to every predicate in that pass; see the note above for what makes it safe. diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index 895ecb0265731657da7d212c8384c18481b30b15..ca274b336b7f9e8a162d0a32df236aff23c29506 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -245,8 +245,10 @@ REPORT_TYPES = { # reads whatever V2 actually sends and nothing here needs to know the column list -- which is # exactly the property that makes swapping the report type a one-line change. # · https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement + # Amazon creates settlements on its own schedule and refuses createReport for them. + # This type is a getReports discovery filter, never a requestable report flavor. "settlements": { - "report": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2", + "report_discover": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2", "validates_against": "settlement summary vs settlement lines", "role": "Finance and Accounting", }, @@ -324,6 +326,13 @@ OPERATIONS = { "name": "createReport", "roles": _BULK_REPORT_ROLES, "method": "POST", "path": "reports/2021-06-30/reports", "report_types": _APPROVED_REPORT_TYPES, }, + # Discovery is approved only for Amazon-issued settlements and their finance role. + # Sharing the Reports path grants neither the other bulk types nor their roles. + "getReports": { + "name": "getReports", "roles": ("Finance and Accounting",), "method": "GET", + "path": "reports/2021-06-30/reports", + "report_types": (REPORT_TYPES["settlements"]["report_discover"],), + }, "getReport": { "name": "getReport", "roles": _BULK_REPORT_ROLES, "method": "GET", "path": "reports/2021-06-30/reports/{reportId}", @@ -967,6 +976,69 @@ def report_document(fields, report_type, start=None, end=None, options=None, "compression": meta.get("compressionAlgorithm")} +SETTLEMENT_REPORT_PAGE_CAP = 1000 + + +def settlement_reports(fields, *, since=None, timeout=_TIMEOUT, log=None): + """List settlements Amazon has already created, oldest first. + + Amazon issues these on its own fortnightly cycle, so this lists rather than requests. + Only safe, named metadata leaves this function; vendor extras and tokens stay here. + """ + require_operation("GET", _REPORTS_CREATE_PATH, + report_type=REPORT_TYPES["settlements"]["report_discover"]) + import re + + rtype = REPORT_TYPES["settlements"]["report_discover"] + token = access_token(fields, timeout=timeout) + region = str((fields or {}).get("region") or DEFAULT_REGION) + params = {"reportTypes": rtype, "processingStatuses": "DONE", "pageSize": 100} + if since is not None: + params["createdSince"] = str(since) + reports = [] + for _page in range(SETTLEMENT_REPORT_PAGE_CAP): + # _request owns the same _retry policy as the existing report/document reads. + got = _request(_REPORTS_CREATE_PATH, token, region=region, timeout=timeout, + sandbox=is_sandbox(fields), **params) + for raw in got.get("reports") or (): + row = {key: str(raw[key]) if raw.get(key) is not None else "" for key in ( + "reportId", "reportDocumentId", "reportType", "dataStartTime", + "dataEndTime", "createdTime")} + # Match the API status door without importing upward into the API layer. + # Refuse unsafe values with a fixed sentence that cannot echo the credential. + if any(len(value) > 512 or "Atzr|" in value + or re.search(r"\bbearer\s+\S+", value, flags=re.IGNORECASE) + or any(ch in value for ch in ("\u2014", "\u2013")) + for value in row.values()): + raise AmazonError("Settlement discovery returned unsafe metadata; no status was emitted.") + if row["reportType"] != rtype or not row["reportDocumentId"]: + raise AmazonError("Settlement discovery returned an unexpected type or no document ID.") + reports.append(row) + next_token = got.get("nextToken") + if not next_token: + if log: + log(f"getReports discovered {len(reports)} settlement reports") + return sorted(reports, key=lambda row: (row["createdTime"], row["reportId"])) + # Amazon continuation tokens replace the original filters, which the token carries. + params = {"nextToken": str(next_token)} + raise AmazonError( + f"Settlement discovery hit the {SETTLEMENT_REPORT_PAGE_CAP}-page cap because Amazon kept " + "returning nextToken; retry with a narrower createdSince window.") + + +def settlement_document(fields, document_id, *, timeout=_TIMEOUT): + """One settlement document's text through getReportDocument and the presigned URL.""" + require_operation("GET", f"reports/2021-06-30/documents/{document_id}") + token = access_token(fields, timeout=timeout) + region = str((fields or {}).get("region") or DEFAULT_REGION) + meta = _request(f"reports/2021-06-30/documents/{document_id}", token, region=region, + timeout=timeout, sandbox=is_sandbox(fields)) + url = str(meta.get("url") or "") + if not url: + raise AmazonError("The settlement document carried no download URL.") + return _download_document(url, str(meta.get("compressionAlgorithm") or "")) + + def _download_document(url, compression="", timeout=120): """Fetch one presigned report document and decode it. -> text. diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index 775cc8e841a70fe41fcba74d981ac9d13fba7353..d20c9c2d4f4f5f49b07a0126c9ca00467eeb3cad 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -70,6 +70,9 @@ PREFIX = "amzn_" TABLES = ("orders", "order_items", "listings", "settlements", "inventory", "pricing", "brand_analytics") +#: Head -> lines: both tables read the same document; a second report buys no new rows. +SHARED_REPORT_SIBLINGS = {"orders": "order_items"} + # C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity. ENABLEMENT_KEY = "amazon_connector_enablement" # Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing. @@ -149,8 +152,20 @@ def entity_key(table): #: How far back a first sync reaches when the caller names no window. Amazon's flat-file order #: reports accept at most ~31 days per request, so a backfill is sliced (`_windows`). DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730) +#: Per-table marker whose detail is the ISO UTC end of the last successfully landed window. +WATERMARK_PREFIX = "sync_mark_" #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one. MAX_WINDOW_DAYS = 30 +#: R18: the most recent month lands before any history, so a newly connected account has usable +#: rows in minutes instead of hours. One window, because Amazon's own ceiling is 30 days. +FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS +#: F4: Amazon paces createReport (burst 15, one per 60 s, 5 s gap), so 7 tables x 25 windows is +#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident +#: thread for hours and the Odoo resync behind it would never run. +FIRST_LOAD_REPORTS_PER_PASS = 15 +BACKFILL_FLOOR_PREFIX = "sync_floor_" +FIRST_LOAD_END_KEY = "sync_first_load_end" +FIRST_LOAD_STATUS_KEY = "sync_first_load_status" # Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores; # JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata. @@ -447,7 +462,7 @@ def _cell(value, kind): return None -def _ensure(con, table, cols, rows=None, declared=None): +def _ensure(con, table, cols, rows=None, declared=None, *, preserve_rows=False): """Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched. ⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data @@ -479,7 +494,15 @@ def _ensure(con, table, cols, rows=None, declared=None): clash = [c for c, ty in want.items() if c in have and have[c] != ty and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")] - if clash: + if clash and preserve_rows: + # Settlement document markers preclude replay: widening must retain prior lines. + # The measured types are VARCHAR, BIGINT and DOUBLE; never narrow historical data. + for c in clash: + widened = "VARCHAR" if "VARCHAR" in (have[c], want[c]) else "DOUBLE" + if have[c] != widened: + con.execute(f'ALTER TABLE "{table}" ALTER COLUMN "{c}" TYPE {widened}') + have[c] = want[c] = widened + elif clash: con.execute(f"DROP TABLE {table}") have = {} if not have: @@ -508,6 +531,47 @@ def _replace(con, table, cols, types, rows): return len(rows) +def replace_keys_for(table): + """Replace on declared identity excluding lineage, which changes on every fetch.""" + return tuple(c for c in (amazon_schema.TABLE_SPECS.get(table) or {}).get("row_identity", ()) + if c not in _ORIGIN_TYPES) + + +def replace_window(con, table, cols, types, rows, keys): + """Replace incoming key groups atomically, or report why only insertion is possible.""" + missing = [c for c in keys if c not in cols] + if missing: + note = (f"{table}: window replacement unavailable; missing identity columns: " + f"{', '.join(missing)}; rows inserted; use a report carrying these columns") + return _replace(con, table, cols, types, rows), note + if not keys or not rows: + return _replace(con, table, cols, types, rows), None + quote = lambda name: '"' + str(name).replace('"', '""') + '"' + target = quote(table) + temporary = quote("amazon_window_keys_" + uuid.uuid4().hex) + columns = ", ".join(quote(c) for c in keys) + # Use the mirror's types and the insert's cell conversion so NULL and numeric keys agree. + incoming = list(dict.fromkeys(tuple(_cell(row.get(c), types.get(c, "VARCHAR")) + for c in keys) for row in rows)) + con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE") + try: + con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})", + incoming) + predicate = " AND ".join( + f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys) + con.execute("BEGIN TRANSACTION") + try: + con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})') + count = _replace(con, table, cols, types, rows) + con.execute("COMMIT") + except Exception: + con.execute("ROLLBACK") + raise + return count, None + finally: + con.execute(f"DROP TABLE {temporary}") + + def _windows(start, end, size=MAX_WINDOW_DAYS): """Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden.""" out, cur = [], start @@ -518,6 +582,49 @@ def _windows(start, end, size=MAX_WINDOW_DAYS): return out +def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS): + """R18's request order: every recent month, then each table's history, oldest last. + + Settlements have one discovery slot (None, None), never a createReport window. + This planner is pure; durable floors filter its fixed windows when a pass resumes. + """ + start, end = _utc(start), _utc(end) + if start >= end: + return [] + size = min(float(size), MAX_WINDOW_DAYS) + recent_days = FIRST_LOAD_RECENT_DAYS if recent_days is None else recent_days + recent_days = min(float(recent_days), size) + if size <= 0 or recent_days <= 0: + raise ValueError("Amazon report windows must have a positive size.") + recent_start = max(start, end - timedelta(days=recent_days)) + tables = list(dict.fromkeys(tables)) + recent = [(table, None, None) if table == "settlements" else (table, recent_start, end) + for table in tables] + history = [(table, w0, w1) for table in tables if table != "settlements" + for w0, w1 in reversed(_windows(start, recent_start, size))] + return recent + history + + +def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None): + """Amazon createReport pacing floor in seconds; polling and other work can add time.""" + from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS + if report_count <= 0: + return 0 + stagger = REPORT_STAGGER_SECONDS if stagger_seconds is None else stagger_seconds + tokens = REPORT_BURST if burst is None else burst + rate = 60 / REPORT_SUSTAINED_SECONDS if per_minute is None else per_minute + seconds = max(0, (report_count - 1) * stagger, (report_count - tokens) * 60 / rate) + return int(seconds) + (seconds > int(seconds)) + + +def first_load_eta_cause(report_count): + """Screen-safe explanation of the vendor ceiling and the automatic continuation.""" + from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS + return (f"Amazon's report rate allows a burst of {REPORT_BURST}, then one report every " + f"{REPORT_SUSTAINED_SECONDS:g} seconds. Keep the connection enabled while history " + "loads automatically. Report processing and other syncs can add time.") + + def _iso(dt): return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1071,6 +1178,94 @@ def _read_state(con, key): return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]} +def settlement_is_new(con, document_id): + """A completed document is durable even when it contained no mirror lines.""" + state = _read_state(con, entity_key("settlement_doc_" + str(document_id))) + return not state or state["state"] != "done" + + +def read_watermark(con, table): + """Missing or malformed detail means no successful window, never the current time.""" + state = _read_state(con, entity_key(WATERMARK_PREFIX + table)) + detail = (state or {}).get("detail") + if not detail: + return None + try: + return _utc(detail) + except (ValueError, TypeError, AttributeError, OverflowError): + return None + + +def record_watermark(con, table, when, rows=0): + """Persist only after this table's pass succeeds.""" + _record_state(con, entity_key(WATERMARK_PREFIX + table), "done", rows, _iso(_utc(when))) + + +def read_backfill_floor(con, table): + """Oldest contiguous landed window; missing or corrupt markers restart safely.""" + state = _read_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table)) + try: + return _utc(state["detail"]) if state and state.get("detail") else None + except (ValueError, TypeError, AttributeError, OverflowError): + return None + + +def record_backfill_floor(con, table, when): + _record_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table), "done", detail=_iso(_utc(when))) + + +def needs_first_load(tenant_key, rt=None): + """A fresh account or an unfinished durable load uses bounded passes, even after one table finishes.""" + con, _path = _connect_tenant_mirror(tenant_key, rt) + try: + load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) + return bool(load and load["state"] != "done") or not any( + read_watermark(con, table) for table in TABLES) + finally: + con.close() + + +def first_load_status(tenant_key, rt=None): + """Durable numeric progress for the API's safe-status projection; no vendor payloads.""" + con, _path = _connect_tenant_mirror(tenant_key, rt) + try: + state = _read_state(con, entity_key(FIRST_LOAD_STATUS_KEY)) + if not state: + return None + values = json.loads(state["detail"]) + return {"state": state["state"], "remaining": int(values["remaining"]), + "etaSeconds": int(values["etaSeconds"]), "backfillDays": DEFAULT_BACKFILL_DAYS, + "cause": first_load_eta_cause(values["remaining"])} + finally: + con.close() + + +def sync_window(table, mark, *, now, since=None, backfill_days=None): + """Pure request decision for one table: start, end, report, incremental and cause.""" + end = _utc(now) + days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days + try: + mark = _utc(mark) if mark else None + except (ValueError, TypeError, AttributeError, OverflowError): + mark = None + if since is not None: + start, incremental = _utc(since), False + cause = f"caller named the window starting {_iso(start)}" + elif mark is not None: + start, incremental = mark, True + cause = f"incremental window since watermark {_iso(mark)}" + else: + start, incremental = end - timedelta(days=days), False + cause = f"first full load: {days} days of backfill" + spec = _client().REPORT_TYPES.get(table) or {} + rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report") + if table == "settlements": + # The watermark filters issued reports by creation time; it never requests one. + rtype = spec.get("report_discover") + return {"start": start, "end": end, "report": rtype, + "incremental": incremental, "cause": cause} + + def _sync_event(kind, tenant, table, detail, rt, report, log): """Sync is nondestructive: preserve C2's loud failure, but continue the data pull.""" try: @@ -1108,7 +1303,7 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log): def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, - max_wait=900, *, scheduled=False): + max_wait=900, *, scheduled=False, first_load=False, report_budget=None): """Pull every declared table into the tenant's mirror. -> a report dict. Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the @@ -1142,7 +1337,6 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None want = [t for t in (TABLES if tables is None else tables) if t in TABLES] end = datetime.now(timezone.utc) - start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS)) report = {"tenant": tenant_key, "tables": {}, "started": _iso(end), "problems": [], "audit_problems": []} enablement = read_database_enablement(rt) @@ -1171,6 +1365,16 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None if obligation["earliestTriggerAt"]: con.close() raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data") + load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) + if scheduled and load and load["state"] != "done": + # The pinned inline producer still calls sync(scheduled=True). Do not let it start an + # unbounded history pull between the resident writer's bounded first-load passes. + for table in want: + report["tables"][table] = {"state": "deferred", "reason": + "The first load is continuing in bounded passes on the resident sync thread."} + report["finished"] = _iso(datetime.now(timezone.utc)) + con.close() + return report if scheduled: plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}, want, now=end) @@ -1179,9 +1383,60 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None report["tables"][table] = {"state": "deferred", "reason": reason} log(f"{table}: DEFERRED: {reason}") - for table in want: - spec = cx.REPORT_TYPES.get(table) or {} - rtype = spec.get("report") + # Heads go first even when the caller listed lines first. Scheduled plans still select one. + pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items() + if head in want and sibling in want] + want)) + handled, noted = set(), set() + + def write_window(table, cols, types, rows): + count, note = replace_window(con, table_name(table), cols, types, rows, + replace_keys_for(table)) + if note and table not in noted: + noted.add(table) + report["problems"].append(note) + log(note) + return count + + work, plans, paired, contexts = [], {}, {}, {} + documents, settlement_remaining = 0, 0 + failed, landed = set(), set() + if first_load: + # Freeze the cohort end across passes. Recomputing it would move every window boundary. + end = _utc(load["detail"]) if load else _utc(_iso(end)) + _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end)) + plans = {table: sync_window(table, None, now=end) for table in pending} + floors = {table: read_backfill_floor(con, table) for table in pending} + wanted = [table for table in pending if not read_watermark(con, table) + or read_watermark(con, table) < end] + for task in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end): + table, w0, w1 = task + if w1 is None or floors[table] is None or w1 <= floors[table]: + work.append(task) + # A failed enrichment after the final report retries completion without fetching again. + for table in wanted: + if not any(task[0] == table for task in work): + work.append((table, None, None)) + # Reuse a document for head and lines only when BOTH durable cursors agree. + for head, sibling in SHARED_REPORT_SIBLINGS.items(): + head_tasks = [(w0, w1) for table, w0, w1 in work if table == head] + line_tasks = [(w0, w1) for table, w0, w1 in work if table == sibling] + if head_tasks and head_tasks == line_tasks: + paired[head] = sibling + work = [task for task in work if task[0] != sibling] + else: + work = [(table, None, None) for table in pending] + + for index, (table, task_start, task_end) in enumerate(work): + if first_load and table in failed: + continue + if (first_load and isinstance(report_budget, int) and documents >= max(0, report_budget) + and (task_end is not None or table == "settlements")): + break + if table in handled: + continue + plan = plans[table] if first_load else sync_window( + table, read_watermark(con, table), now=end, since=since) + rtype = plan["report"] key, mirror = entity_key(table), table_name(table) if not rtype: report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"} @@ -1189,15 +1444,89 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None continue prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync") root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1] + log(f"{table}: {plan['cause']}") + sibling = SHARED_REPORT_SIBLINGS.get(table) + sibling_plan = None + if first_load: + sibling = paired.get(table) + sibling_plan = plans.get(sibling) + elif sibling in want and sibling not in handled: + sibling_plan = sync_window(sibling, read_watermark(con, sibling), now=end, since=since) + differs = [part for part in ("start", "end", "report") + if plan[part] != sibling_plan[part]] + if differs: + log(f"{table} and {sibling}: cannot share this pass; differing {', '.join(differs)}") + sibling = None + else: + handled.add(sibling) + log(f"{sibling}: {sibling_plan['cause']}") + else: + sibling = None + operation = "report_document" try: # Attempt first, so a refusal or process interruption cannot starve the other tables. - _record_state(con, entity_key("sync_tick_" + table), "attempted") + if not first_load: + _record_state(con, entity_key("sync_tick_" + table), "attempted") + if sibling and not first_load: + _record_state(con, entity_key("sync_tick_" + sibling), "attempted") + sibling_written, sibling_cleared = 0, False written, cols, cleared = 0, [], False pricing_asins = [] - for w0, w1 in _windows(start, end): + operations = {} + if first_load: + cleared = floors[table] is not None + sibling_cleared = bool(sibling and floors[sibling] is not None) + pricing_asins = contexts.setdefault(table, []) + if table == "settlements": + operation = "getReports" + _sync_event("amazon.sync", tenant_key, table, + {"state": "access", "operation": operation}, rt, report, log) + discovered = cx.settlement_reports( + fields, since=_iso(plan["start"]) if plan["incremental"] else None, + log=lambda message: log(f"settlements: {message}")) + downloaded, skipped = 0, 0 + for issued_index, issued in enumerate(discovered): + document_id = str(issued.get("reportDocumentId") or "") + if not document_id: + raise cx.AmazonError("Settlement discovery returned no document ID.") + if not settlement_is_new(con, document_id): + skipped += 1 + continue + if first_load and isinstance(report_budget, int) and documents >= max(0, report_budget): + settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or "")) + for item in discovered[issued_index:]) + break + operation = "getReportDocument" + documents += 1 + _sync_event("amazon.sync", tenant_key, table, + {"state": "access", "operation": operation}, rt, report, log) + text = cx.settlement_document(fields, document_id) + cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype) + count = 0 + if cols and rows: + types = _ensure(con, mirror, cols, rows, preserve_rows=True) + # Never clear settlement history: durable markers prevent its replay. + # A newly issued document still replaces its own settlement's lines. + count = write_window(table, cols, types, rows) + written += count + # Record completion after the write, including header-only and blank files. + _record_state(con, entity_key("settlement_doc_" + document_id), "done", + count, issued.get("dataEndTime")) + downloaded += 1 + operations["getReports"] = { + "state": "done", "reports": len(discovered), + "downloaded": downloaded, "skipped": skipped} + if first_load and settlement_remaining: + break + windows = () + if table != "settlements": + windows = (([(task_start, task_end)] if task_end is not None else []) if first_load + else _windows(plan["start"], plan["end"])) + for w0, w1 in (() if table == "settlements" else windows): # W59-T43 / C2 has no amazon.access kind. Record each credential-backed # vendor attempt as a sync ACCESS phase, distinct from the completion event. # Lane F/combine own the vocabulary decision; no credential or row enters C2. + documents += 1 _sync_event("amazon.sync", tenant_key, table, {"state": "access", "operation": "report_document"}, rt, report, log) text, meta = cx.report_document( @@ -1237,7 +1566,25 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None text, document_id=meta.get("reportDocumentId"), report_type=rtype) declared = {} if not cols: - continue + if first_load: + # Even an empty recent window clears old rows exactly once. + for empty_table in (table, sibling): + if empty_table and floors[empty_table] is None: + if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?", + [table_name(empty_table)]).fetchone(): + con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1") + cleared = True + else: + continue + if sibling: + sibling_mirror = table_name(sibling) + sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared, + preserve_rows=first_load) if cols else {} + if cols and not sibling_cleared and not sibling_plan["incremental"]: + con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1") + sibling_cleared = True + if cols: + sibling_written += write_window(sibling, cols, sibling_types, rows) if table == "orders": rows = _dedupe_orders(rows) if table == "inventory": @@ -1246,16 +1593,32 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None cols.append("inventory_source") if table == "pricing": pricing_asins.extend(_pricing_asins(rows)) - types = _ensure(con, mirror, cols, rows, declared=declared) - # Clear once, after the table exists. The old multi-window branch deleted nothing, - # so every backfill appended a second copy of every historical row. - if not cleared: + types = _ensure(con, mirror, cols, rows, declared=declared, + preserve_rows=first_load) if cols else {} + # Explicit windows and first loads replace the mirror; incremental passes keep history. + if not cleared and not plan["incremental"]: con.execute(f"DELETE FROM {mirror} WHERE 1=1") cleared = True - written += _replace(con, mirror, cols, types, rows) + if cols: + written += write_window(table, cols, types, rows) + if first_load: + for reached in (table, sibling): + if reached: + record_backfill_floor(con, reached, w0) + floors[reached] = w0 + + if first_load: + landed.add((table, task_start, task_end)) + if any(next_table == table for next_table, _w0, _w1 in work[index + 1:]): + continue - operations = {} if table == "pricing": + if first_load: + # Completion can be in another process/pass than the recent ASIN report. + available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()} + pricing_asins = list(dict.fromkeys(pricing_asins + [str(row[0]) + for col in ("child_asin", "parent_asin") if col in available + for row in con.execute(f'SELECT DISTINCT "{col}" FROM {mirror} WHERE "{col}" IS NOT NULL').fetchall()])) state_key = ENTITY_STATE_KEYS[table] if not pricing_asins: _record_state(con, state_key, "unproven", 0, @@ -1291,11 +1654,11 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None awd_rows = _awd_rows(inventory) if awd_rows: awd_cols = sorted({c for row in awd_rows for c in row}) - awd_types = _ensure(con, mirror, awd_cols, awd_rows) - if not cleared: + awd_types = _ensure(con, mirror, awd_cols, awd_rows, preserve_rows=first_load) + if not cleared and not plan["incremental"]: con.execute(f"DELETE FROM {mirror} WHERE 1=1") cleared = True - written += _replace(con, mirror, awd_cols, awd_types, awd_rows) + written += write_window(table, awd_cols, awd_types, awd_rows) _record_state(con, state_key, "done", len(awd_rows)) operations["listInventory"] = { "state": "done", "rows": len(awd_rows)} @@ -1312,21 +1675,67 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None column_count, total_rows = len(cols), written report["tables"][table] = { "rows": total_rows, "columns": column_count, "provenance": prov, - "report": rtype, "operations": operations} + "report": rtype, "window": plan["cause"], "incremental": plan["incremental"], + "operations": operations} _record_state(con, key, "done", total_rows) _record_state(con, entity_key("sync_tick_" + table), "done", total_rows) + if not any(v["state"] in ("refused", "throttled") for v in operations.values()): + record_watermark(con, table, plan["end"], total_rows) _sync_event("amazon.sync", tenant_key, table, {"state": "done", "rows": total_rows, "operations_ok": all(v["state"] == "done" for v in operations.values())}, rt, report, log) log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}") + if sibling: + sibling_mirror = table_name(sibling) + try: + sibling_columns = len(con.execute(f"PRAGMA table_info('{sibling_mirror}')").fetchall()) + sibling_total = int(con.execute(f"SELECT COUNT(*) FROM {sibling_mirror}").fetchone()[0]) + except Exception: + sibling_columns, sibling_total = len(cols), sibling_written + sibling_prov = amazon_schema.PROVENANCE.get(sibling, "measured_at_sync") + report["tables"][sibling] = { + "rows": sibling_total, "columns": sibling_columns, "provenance": sibling_prov, + "report": rtype, "window": sibling_plan["cause"], + "incremental": sibling_plan["incremental"], "operations": {}} + _record_state(con, entity_key(sibling), "done", sibling_total) + _record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total) + record_watermark(con, sibling, sibling_plan["end"], sibling_total) + _sync_event("amazon.sync", tenant_key, sibling, + {"state": "done", "rows": sibling_total, "operations_ok": True}, + rt, report, log) + log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}") except Exception as e: # noqa: BLE001 + # A failed pass retains its previous watermark so its window will be retried. # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN # ticket answering itself, and it must not cost the six tables that did work. - problem = _sync_problem(e, table, "report_document", tenant_key, rt, report, log) + problem = _sync_problem(e, table, operation, tenant_key, rt, report, log) + failed.add(table) report["tables"][table] = {"rows": 0, **problem} _sync_event("amazon.sync", tenant_key, table, {"state": problem["state"], "rows": 0}, rt, report, log) + if sibling: + problem = _sync_problem(e, sibling, "report_document", tenant_key, rt, report, log) + report["tables"][sibling] = {"rows": 0, **problem} + _sync_event("amazon.sync", tenant_key, sibling, + {"state": problem["state"], "rows": 0}, rt, report, log) + if first_load: + remaining = sum(task not in landed for task in work) + if settlement_remaining: + remaining += settlement_remaining - 1 # Replace the discovery slot with known documents. + # Keep healthy tables moving after an isolated refusal. Once only failed work remains, + # stop automatic continuation instead of endlessly retrying the same vendor refusal. + continuable = any(task not in landed and task[0] not in failed for task in work) + state = ("partial" if remaining and continuable else + "failed" if remaining or report["problems"] or report["audit_problems"] else "done") + progress = {"state": state, "remaining": remaining, + "etaSeconds": first_load_eta(remaining, burst=0), + "cause": first_load_eta_cause(remaining), "backfillDays": DEFAULT_BACKFILL_DAYS} + report.update(progress) + report["firstLoad"] = dict(progress) + _record_state(con, entity_key(FIRST_LOAD_END_KEY), state, detail=_iso(end)) + _record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), state, + detail=json.dumps({key: progress[key] for key in ("remaining", "etaSeconds")})) report["finished"] = _iso(datetime.now(timezone.utc)) con.close() return report diff --git a/platform/harness/measure_filter.py b/platform/harness/measure_filter.py index 06389b14bb79590da1d076eccccbd7b4062ad1d9..0941b4e474057e9527cc84c92c64a56949078f17 100644 --- a/platform/harness/measure_filter.py +++ b/platform/harness/measure_filter.py @@ -302,7 +302,7 @@ def _base_topic(key): def _related_scope(related, keys): """Prepare all bases before querying: a refused composite must be wholly unscoped.""" - if related is None or isinstance(related, _relations.MeasurePin): + if related is None or (isinstance(related, _relations.MeasurePin) and related.refusal): return related try: pin = _relations.measure_pin(related) @@ -673,6 +673,10 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related= con = _sem._store_con() try: totals = _totals(con, key, window, today, team_id, related) + guard = None + metric = _sem.metrics().get(key, {}) + if related is not None and not related.refusal and metric.get('agg') == 'ratio': + guard = _totals(con, metric['denominator'], window, today, team_id, related) finally: con.close() scale = ADMITTED.get(key, {}).get('input_scale') or 1.0 @@ -682,7 +686,8 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related= # round(x, None) returns an int — exactly the pool's `_round` convention for money/counts. # The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction. return _related_answer( - {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}, related) + {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe + if guard is None or guard.get(p)}, related) #: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to diff --git a/platform/harness/runtime.py b/platform/harness/runtime.py index 9e66f9a23c1793fc387e358dca9567931bb482cf..85ee5016e7c02fd819cd6c9319382fa7ad6eb724 100644 --- a/platform/harness/runtime.py +++ b/platform/harness/runtime.py @@ -126,6 +126,8 @@ class TenantRuntime: tenant: object # harness.base.Tenant store_namespace: str = "" pool_cache: dict = field(default_factory=dict) + # Names/edges only, bounded and revision-keyed by routes_grid; no row pools live here. + related_cache: dict = field(default_factory=dict) #: EXIT-5 — the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps #: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on #: (stamp, scope, pool identity, question) so nothing user-shaped lives in them. diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py index 1de3e2bcfb725c3525cef7d5a6beb8b84031063e..f2fa899ca82487e9e08ad84e5c3b873e9e2af258 100644 --- a/platform/harness/semantic.py +++ b/platform/harness/semantic.py @@ -827,13 +827,24 @@ def _scope_for_binding(store, scope_variant=None): def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None, team_id=None, filters=None, sort=None, limit=1000, exclude_services=False, - filter_tree=None, filter_conj="and", today=None, scope_variant=None): + filter_tree=None, filter_conj="and", today=None, scope_variant=None, related=None): """The semantic query over the tenant store — the engine behind the Analyst's run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model.""" t = _model()["topics"].get(topic) if not t or "store" not in t: raise ModelError(f"topic {topic!r} has no store binding") s = t["store"] + pin_predicate, pin_params = None, [] + if related is not None: + from core import relations + try: + pin = relations.measure_pin(related) + if pin.refusal: + raise relations.PinRefusal(pin.refusal['code'], pin.refusal['message'], pin.path) + pin_predicate, pin_params = pin.predicate(s) + except relations.PinRefusal as exc: + return {"topic": topic, "rows": [], "row_count": 0, "truncated": False, + "refusals": [exc.marker]} alias = s["alias"] dims = s.get("dims") or {} # ⭐ A GROUPED QUERY IS BOUNDED BY GROUPS, NOT BY ROWS — and the 5,000 row ceiling applied to @@ -869,7 +880,9 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date foreign_vals = {} for k in foreign: sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from, - date_to=date_to, team_id=team_id) + date_to=date_to, team_id=team_id, related=related) + if sub.get("refusals"): + return sub foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0 if not base_keys: # purely foreign scalar request return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)", @@ -895,6 +908,9 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}") where = [_scope_for_binding(s, scope_variant)] + if pin_predicate is not None: + where.append(f'({pin_predicate})') + params.extend(pin_params) # TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes — # datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date). # Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' < @@ -911,7 +927,7 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date # intersecting either with a `team_id` restates the same dimension a second time and the two # statements can disagree. Concretely, `channel_amazon` AND `team_id = 5` returns an empty # cell that reads as "this SKU sold nothing on Amazon" — a FALSE number, not a narrower one. - if team_id and scope_variant is None: + if team_id and (scope_variant is None or related is not None): if not s.get("team_col"): raise ModelError(f"topic {topic!r} is company-level: it has no business-unit filter") where.append(f"{s['team_col']} = ?"); params.append(int(team_id)) @@ -1505,7 +1521,7 @@ def _one_binding_offer(topic, b, strict=False, ready=None): def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None, - exclude_services=False, offer=None): + exclude_services=False, offer=None, related=None): """`{group key: {measure key: value}}` for an entity topic's lookback columns. ONE grouped query for all `keys` together — they share a topic, a scope and a window, so @@ -1548,11 +1564,15 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non groups.setdefault((by_key[k]["topic"], by_key[k]["dim"], by_key[k].get("scope_variant")), []).append(k) if len(groups) > 1: - merged = {} + from core import relations + merged = relations.MeasureAnswer() if related is not None else {} for (_src, _dim, _variant), ks in groups.items(): - for gk, cell in entity_measure_values( + answer = entity_measure_values( topic, ks, date_from=date_from, date_to=date_to, team_id=team_id, - exclude_services=exclude_services, offer=offer).items(): + exclude_services=exclude_services, offer=offer, related=related) + if related is not None: + merged.refusals.extend(getattr(answer, 'refusals', ())) + for gk, cell in answer.items(): merged.setdefault(gk, {}).update(cell) return merged (src, dim, scope_variant), want = next(iter(groups.items())) @@ -1568,7 +1588,11 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non base_of = {k: (by_key[k].get("base_key") or k) for k in want} res = store_query(src, [base_of[k] for k in want], group_by=[dim], date_from=date_from, date_to=date_to, team_id=team_id, exclude_services=exclude_services, - limit=MAX_GROUPS, scope_variant=scope_variant) + limit=MAX_GROUPS, scope_variant=scope_variant, related=related) + if res.get("refusals"): + from core import relations + return relations.MeasureAnswer(refusals=[dict(marker, measure=k) + for marker in res['refusals'] for k in want]) # ⛔ A TRUNCATED GROUP SET IS A WRONG ANSWER PER ROW, not a short list — the same refusal # `rollup_sql.group_values` makes, for the same reason. Never write cells from one. if res.get("truncated"): diff --git a/platform/harness/windows.py b/platform/harness/windows.py index e0e615021db71e86d57d87284ac66e72468eca05..075b50c78e912f89dcb030f9eb63b0db6cd9c8ab 100644 --- a/platform/harness/windows.py +++ b/platform/harness/windows.py @@ -40,6 +40,8 @@ DESIGN NOTES THAT ARE EASY TO GET WRONG "all time" would widen a filter while still reporting an authoritative count. """ import datetime as _dt +import re +import math #: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts; #: verify_windows.py asserts the two lists are identical, because a kind the client can emit and @@ -191,6 +193,91 @@ def _parse_date(v): return None +# Sheet Ending metadata belongs to the binding period, never to a window spec. +ENDING_KINDS = ("today", "yesterday", "last_week", "last_month", "last_quarter", + "last_year", "fixed", "cell") +ENDING_LABELS = dict(zip(ENDING_KINDS, ("Today (live)", "Yesterday", "End of last week", + "End of last month", "End of last quarter", "End of last year", "A fixed date", "From a cell"))) +TO_DATE_KINDS = ("ytd", "ytd_last_year", "this_month", "this_quarter") +ENDING_STEPS = ("day", "week", "month", "quarter", "year") + + +def ending_step_default(kind): + return "quarter" if kind == "this_quarter" else "month" + + +def ending_date(value): + """A real ISO day only. No truncation, calendar rollover, or blank fallback.""" + if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value.strip()): + return None + d = _parse_date(value) + return _iso(d) if d else None + + +def resolve_ending(mode, value, today, step="month", offset=0): + """Inclusive ending date; cell values are supplied by the caller's trusted read.""" + d = _parse_date(today) + if d is None: + raise ValueError("Ending needs a valid today date") + if mode not in ENDING_KINDS or step not in ENDING_STEPS: + return None + if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0: + return None + if mode in ("fixed", "cell"): + parsed = ending_date(value) + if parsed is None: + return None + end = _parse_date(parsed) + elif mode == "today": + end = d + elif mode == "yesterday": + end = d - _dt.timedelta(days=1) + elif mode == "last_week": + end = d - _dt.timedelta(days=d.weekday() + 1) + elif mode == "last_month": + end = _month_start(d) - _dt.timedelta(days=1) + elif mode == "last_quarter": + end = _quarter_start(d) - _dt.timedelta(days=1) + else: + end = _dt.date(d.year - 1, 12, 31) + try: + if step in ("day", "week"): + end += _dt.timedelta(days=offset * (7 if step == "week" else 1)) + else: + end = _shift_months(end, offset * {"month": 1, "quarter": 3, "year": 12}[step]) + return _iso(end) + except (ValueError, OverflowError): + return None + + +def ending_cell_date(value, date_system="1900"): + """Sheet dates may be ISO days or Excel serials in the workbook's declared epoch.""" + if not isinstance(value, (int, float)) or isinstance(value, bool): + return ending_date(value) + if not math.isfinite(value) or value < (0 if date_system == "1904" else 1): + return None + days = math.floor(value) + if date_system != "1904" and days == 60: + return None # Excel's fictional February 29, 1900 is not a calendar date. + epoch = (_dt.date(1904, 1, 1) if date_system == "1904" else + _dt.date(1899, 12, 31) if days < 60 else _dt.date(1899, 12, 30)) + try: + return _iso(epoch + _dt.timedelta(days=days)) + except (ValueError, OverflowError): + return None + + +def ending_bounds(kind, mode, value, today, step="month", offset=0): + """Keep the kind's own start. An ending before it refuses instead of reversing bounds.""" + if kind not in TO_DATE_KINDS: + return None + end = resolve_ending(mode, value, today, step, offset) + if end is None: + return None + start = resolve({"kind": kind}, today)[0] + return (start, end) if start <= end else None + + def normalize(spec): """Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None. diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 2b42358d5df506566e3a4ed305cb277827fc7e06..673446dc32cf17f825723941a5d171b27f396074 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -1431,9 +1431,72 @@ def validate(team_id=None, t=None): checks.extend(validate_price_and_unit_cells(rows)) checks.extend(validate_stock_measures(t=t)) checks.extend(validate_fba(rows)) + checks.extend(validate_linked_metric(team_id=team_id)) return checks +def validate_linked_metric(customer_id=None, product_id=None, team_id=None, + date_from=None, date_to=None): + """Reconcile one customer/product revenue cell to an independent live Odoo aggregate. + + The semantic result uses the shipping pin predicate and entity measure resolver. The + oracle uses Odoo's real partner/product ids and read_group, never the mirror's totals. + A live/mirror gap is reported exactly and fails; sync and rerun to distinguish lag. + Explicit ids/window make the leg reproducible without rebuilding the product pool. + """ + from core import relations + from harness import semantic as sem, datastore as ds + + name = 'linked customer/product revenue reconciles to independent Odoo' + try: + con = ds.ro_cursor() + try: + parts, params = ["o.state IN ('sale','done')"], [] + for expression, value in [('l.order_partner_id', customer_id), + ('l.product_id', product_id), ('o.team_id', team_id)]: + if value is not None: + parts.append(f'{expression} = ?') + params.append(int(value)) + sample = con.execute( + "SELECT l.order_partner_id, l.product_id, " + "COALESCE(NULLIF(p.default_code, ''), 'pid:' || CAST(p.id AS VARCHAR)), " + "CAST(o.date_order AS DATE) FROM sale_order_line l " + "JOIN sale_order o ON o.id=l.order_id JOIN product_product p ON p.id=l.product_id " + f"WHERE {' AND '.join(parts)} ORDER BY o.date_order DESC, l.id LIMIT 1", params).fetchone() + finally: + con.close() + if not sample: + return [dict(check=name, ours=None, theirs=None, ok=False, + detail='No confirmed customer/product sample is available. Sync the sales source and rerun.')] + customer_id, product_id, code, day = sample + date_from, date_to = date_from or str(day), date_to or str(day) + pin = relations.MeasurePin(['validation'], + {'customer_data': [{'pid': customer_id}], + 'product_data': [{'product_id': product_id}]}, + {'sale_order_line': [ + dict(table='customer_data', column='pid', sql='{alias}.order_partner_id'), + dict(table='product_data', column='product_id', sql='{alias}.product_id')]}, + required=('customer_data', 'product_data')) + answer = sem.entity_measure_values('odoo_products', ['revenue'], date_from=date_from, + date_to=date_to, team_id=team_id, related=pin) + if getattr(answer, 'refusals', ()): + return [dict(check=name, ours=None, theirs=None, ok=False, detail=answer.refusals)] + ours = answer.get(code, {}).get('revenue', 0) + domain = O.sale_line_domain(date_from, date_to, team_id=team_id, all_channels=True, + extra=[('order_partner_id', '=', customer_id), ('product_id', '=', product_id)]) + aggregate = O.get_odoo().read_group('sale.order.line', domain, + ['price_subtotal:sum'], [], lazy=False) + theirs = sum(r.get('price_subtotal') or 0 for r in aggregate) + delta = round(ours - theirs, 2) + return [dict(check=name, ours=ours, theirs=theirs, ok=abs(delta) < 0.005, + detail=dict(customer=customer_id, product=product_id, team_id=team_id, + date_from=date_from, date_to=date_to, difference=delta, + recommendation='If different, sync sales and rerun this exact sample; investigate any remaining difference.'))] + except Exception as exc: + return [dict(check=name, ours=None, theirs=None, ok=False, + detail=f'Validation could not read both sources ({type(exc).__name__}). Check source readiness and rerun.')] + + def fba_quant_oracle(): """`(warehouse | None, {product_id: units}, detail)` — the FBA column's INDEPENDENT oracle. diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py index 0d6006c31170a76cec92accdf7844251f89f0b0e..8cfb79fff4a1b8f1adcead60c637886d5dde9454 100644 --- a/platform/modules/sheet.py +++ b/platform/modules/sheet.py @@ -33,20 +33,21 @@ from __future__ import annotations import datetime import re +from urllib.parse import urlencode + +from harness.filter_sql import NUMERIC_TYPES as MIRROR_NUMERIC_TYPES #: C2's aggregate vocabulary. W59-T14 resolves the `metric_key` arm through a Metric column, #: using only the catalogue's declared fold: see `metric_key_fold` and its additivity guard. AGGREGATE_FNS = ("sum", "count", "avg", "min", "max") -#: ⭐ R9's PERIOD (wave 51, W51-T21). The three units a bound cell can be pinned to, and the month -#: its window starts in. `web/src/sheet/types.ts` carries the same three names. -PERIOD_UNITS = ("month", "quarter", "year") +#: Calendar units shared with `web/src/sheet/types.ts`; W60-T06 adds daily and weekly summaries. +PERIOD_UNITS = ("day", "week", "month", "quarter", "year") -#: `YYYY-MM`. A period starts in a MONTH whatever its unit, because a quarter and a year are both -#: named by the month they open on, and one spelling means a copied cell cannot mean two things. +#: Month, Quarter and Year keep their opening-month encoding unchanged. _PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$") -#: Literal inclusive endpoints of a frozen calendar period, spelled `YYYY-MM-DD`. +#: Day/Week starts and inclusive endpoints of a frozen period, spelled `YYYY-MM-DD`. _PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$") #: A `period.ref` points at a cell that holds the period, so it is a plain A1 address with no `$`, @@ -72,7 +73,17 @@ REFUSALS = { "binding_invalid": "That cell's source is not set up correctly.", "aggregate_unknown": "{subject} is not an aggregate this cell can compute.", "field_unknown": "{subject} is not a column on this database.", - "period_invalid": "{subject} is not a period this cell can use.", + "period_invalid": "{subject} is not a period this cell can use. " + "Choose a valid start for the selected period unit.", + "ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.", + "ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year " + "to date, this month or this quarter.", + "ending_cell_unreadable": "Ending from cell {subject} cannot be read. " + "Use a cell containing a date in YYYY-MM-DD format.", + "filter_cell_unreadable": "Filter value from cell {subject} cannot be read. " + "Choose a readable cell containing one nonblank value, or type the filter value.", + "linked_filter_unreadable": "Linked field {subject} cannot be filtered because {detail} " + "Choose a readable field through an available link, or ask an administrator to restore access.", # ── W53-T13's two refusals, and the reason each of them is NOT `field_unknown` ──────────── # # ⛔⛔ `field_unknown`'s sentence is "{subject} is not a column on this database", and for a @@ -140,6 +151,10 @@ class SheetRefusal(Exception): self.detail = str(detail or "") self.message = template.format(subject=self.subject or "this database", detail=self.detail) + if code in ("ending_invalid", "ending_cell_unreadable", "filter_cell_unreadable") and self.detail: + verb = "used" if code == "ending_invalid" else "read" + self.message = self.message.replace(f"cannot be {verb}.", + f"cannot be {verb} because {self.detail}") super().__init__(self.message) @@ -150,7 +165,7 @@ def clean_period(raw): True so freezing and unfreezing returns to the identical rolling period already stored. ⛔ AN UNKNOWN UNIT OR A MALFORMED START IS REFUSED BY NAME, and that is not the same call as - the cosmetic keys next door. A period decides WHICH ROWS ARE COUNTED. Dropping `unit: "week"` + the cosmetic keys next door. A period decides WHICH ROWS ARE COUNTED. Dropping `unit: "fortnight"` would answer for every row the binding can see and hand back a bigger number with nobody told, which is standing rule 1's second sentence turned into a defect. @@ -174,6 +189,39 @@ def clean_period(raw): return None out = {} + # An Ending is a period modifier. Validate before the legacy dialect can discard it. + if "ending" in raw: + from harness import windows as W + window = W.normalize(raw.get("window")) + kind = (window or {}).get("kind") + if kind not in W.TO_DATE_KINDS: + raise SheetRefusal("ending_not_supported", kind or "This period") + ending = raw["ending"] + mode = ending.get("kind") if isinstance(ending, dict) else None + if mode not in W.ENDING_KINDS: + raise SheetRefusal("ending_invalid", str(mode or "selection"), "the choice is unknown.") + step = raw.get("endingStep", W.ending_step_default(kind)) + offset = raw.get("endingOffset", 0) + if step not in W.ENDING_STEPS or isinstance(offset, bool) or not isinstance(offset, int) or offset < 0: + raise SheetRefusal("ending_invalid", "step", "its size or copy offset is invalid.") + out = {"window": window, "ending": {"kind": mode}, "endingStep": step} + if offset: + out["endingOffset"] = offset + if raw.get("locked") is True: + out["locked"] = True + if mode == "fixed": + day = W.ending_date(ending.get("date")) + if day is None: + raise SheetRefusal("ending_invalid", "fixed date", "it is not a real ISO date.") + out["ending"]["date"] = day + if mode == "cell": + ref = str(raw.get("ref") or "").strip() + # Preserve sheet names and dollar anchors. The read layer validates the target. + if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref): + raise SheetRefusal("ending_cell_unreadable", ref or "(blank)", "its reference is invalid.") + out["ref"] = ref + return out + unit = str(raw.get("unit") or "").strip().lower() if unit: if unit not in PERIOD_UNITS: @@ -182,8 +230,16 @@ def clean_period(raw): start = str(raw.get("start") or "").strip() if start: - if not _PERIOD_START.match(start): + pattern = _PERIOD_DAY if unit in ("day", "week") else _PERIOD_START + if not pattern.fullmatch(start): raise SheetRefusal("period_invalid", start) + if unit in ("day", "week"): + try: + day = datetime.date.fromisoformat(start) + except ValueError: + raise SheetRefusal("period_invalid", start) + if unit == "week": + start = (day - datetime.timedelta(days=day.weekday())).isoformat() out["start"] = start if raw.get("frozen") is True: @@ -226,13 +282,11 @@ def clean_period(raw): return out -#: ⭐ R7's PERIOD ARITHMETIC, spelled once. How many months each unit spans, which is the whole -#: difference between the three: a period is always named by the month it OPENS on (`clean_period` -#: enforces the `YYYY-MM` spelling for that reason), so the unit only decides where it closes. +#: Month-based periods retain their existing arithmetic. Day/Week advance in calendar days. PERIOD_MONTHS = {"month": 1, "quarter": 3, "year": 12} -def period_window(period): +def period_window(period, today=None, *, cell_date=None): """⭐⭐ R7 — a bound cell's PERIOD as a `harness.windows` CUSTOM window, or None. OWNER, R7: *"must also be bounded by date time lookup"*. A Metric column carries its OWN @@ -270,14 +324,36 @@ def period_window(period): """ if not isinstance(period, dict): return None + if "ending" in period: + from harness import windows as W + period = clean_period(period) + kind = period["window"]["kind"] + mode = period["ending"]["kind"] + value = cell_date if mode == "cell" else period["ending"].get("date") + if mode == "cell" and W.ending_date(value) is None: + raise SheetRefusal("ending_cell_unreadable", period.get("ref"), + "it has no readable date.") + bounds = W.ending_bounds(kind, mode, value, today or datetime.date.today(), + period["endingStep"], period.get("endingOffset", 0)) + if bounds is None: + raise SheetRefusal("ending_invalid", W.ENDING_LABELS[mode], + "it falls before the lookback starts or outside the supported calendar.") + return {"kind": "custom", "from": bounds[0], "to": bounds[1]} if period.get("frozen") is True and period.get("from") and period.get("to"): return {"kind": "custom", "from": period["from"], "to": period["to"]} if period.get("window") is not None: return period["window"] start = str(period.get("start") or "").strip() + unit = str(period.get("unit") or "month").strip().lower() + if unit in ("day", "week"): + cleaned = clean_period(period) + if not cleaned or not cleaned.get("start"): + return None + first = datetime.date.fromisoformat(cleaned["start"]) + last = first + datetime.timedelta(days=6 if unit == "week" else 0) + return {"kind": "custom", "from": first.isoformat(), "to": last.isoformat()} if not _PERIOD_START.match(start): return None - unit = str(period.get("unit") or "month").strip().lower() months = PERIOD_MONTHS.get(unit) if not months: return None @@ -289,10 +365,8 @@ def period_window(period): return {"kind": "custom", "from": f"{start}-01", "to": last.isoformat()} -#: The column types `harness/filter_sql.py::NUMERIC_TYPES` compiles as numbers AND -#: `odoo_relational._REGISTRY_COL_TYPE` materialises as a number. The two lists have to agree or a -#: column is stored as one thing and compared as another. -MIRROR_NUMERIC_TYPES = ("currency", "int", "pct") +#: MIRROR_NUMERIC_TYPES is imported from the SQL compiler above, so admission and comparison +#: use one declaration. verify_sheet also checks the evaluator's documented client-only types. #: ⭐ THE COMPUTED KINDS THE MIRROR HAS TO BE TOLD ABOUT. `_REGISTRY_COL_TYPE` maps the ordinary #: field types and falls back to VARCHAR for everything else, so a `metric` or `rating` column @@ -504,8 +578,34 @@ def clean_on_drag(raw): down = raw.get("down") if isinstance(down, dict) and str(down.get("field") or "").strip(): out_down = {"field": str(down["field"]).strip()} + if down.get("mode") in ("value", "presence", "member"): + out_down["mode"] = down["mode"] else: out_down = DRAG_DOWN_NONE + if isinstance(raw.get("steps"), list): + steps = [] + for step in raw["steps"]: + if not isinstance(step, dict): + continue + field = step.get("field") + if not isinstance(field, str) or not field.strip(): + continue + if step.get("axis") not in ("across", "down"): + continue + if "size" in step and step["size"] not in ("day", "week", "month", "quarter", "year"): + continue + cleaned = {"field": field.strip(), "axis": step["axis"]} + if step.get("mode") in ("value", "presence", "member"): + cleaned["mode"] = step["mode"] + if "size" in step: + cleaned["size"] = step["size"] + steps.append(cleaned) + first_down = next((step for step in steps if step["axis"] == "down"), None) + out_down = ({key: first_down[key] for key in ("field", "mode") if key in first_down} + if first_down else DRAG_DOWN_NONE) + across = "period" if any(step["axis"] == "across" and step["field"] in + ("__period", "__ending") for step in steps) else "none" + return {"across": across, "down": out_down, "steps": steps} return {"across": across, "down": out_down} @@ -527,6 +627,66 @@ def clean_adopted_from(raw): return out or None +def validate(date_from, date_to, observed, *, team_id, partner_ids=None): + """Reconcile T08's four sections to live Odoo, at the exact displayed scope. + + sales.validate owns the broader sales-module reconciliation. Its public API + takes a YTD as-of, not arbitrary weeks or customer sets. Reuse its order and + line domains here for the template's narrower independent checks; never use + customer_data.validate (which validates addresses). + """ + import math + from core import odoo as O + from modules import sales + + first, last = datetime.date.fromisoformat(date_from), datetime.date.fromisoformat(date_to) + if first > last or team_id not in O.active_team_ids(): + raise ValueError("Choose a valid date range and one permitted business unit.") + # The template's basis is fixed even if the caller's dashboard is in Invoice mode. + domain = [clause for clause in sales.order_domain(date_from, date_to, team_id, partner_ids) + if clause[0] != "invoice_status"] + lines = [clause for clause in O.sale_line_domain(date_from, date_to, team_id, + partner_ids=partner_ids) + if clause[0] != "order_id.invoice_status"] + revenue = O.sum_field("sale.order", domain, "amount_untaxed") + invoiced = O.sum_field("sale.order", domain + [("invoice_status", "=", "invoiced")], "amount_untaxed") + count = O.get_odoo().search_count("sale.order", domain) + expected = {"invoiced_sales": invoiced, "confirmed_sales": revenue, "order_count": count, + "average_order_value": revenue / count if count else 0.0} + checks = [] + for key, value in expected.items(): + actual = observed.get(key) + valid = isinstance(actual, (float, int)) and not isinstance(actual, bool) and math.isfinite(actual) + checks.append({"check": key, "a": actual, "b": value, + "gap": actual - value if valid else None, + "ok": valid and abs(actual - value) <= (0 if key == "order_count" else 0.01)}) + for label, order_value, extra in (("Confirmed order and line revenue", revenue, []), + ("Fully invoiced order and line revenue", invoiced, + [("order_id.invoice_status", "=", "invoiced")])): + line_value = O.sum_field("sale.order.line", lines + extra, "price_subtotal") + checks.append({"check": label, "a": order_value, "b": line_value, + "gap": order_value - line_value, "ok": abs(order_value - line_value) <= 0.01}) + return checks + + +def sales_summary_partition(all_values, agent_values, unassigned): + """Partition additive measures; AOV must be recomputed from the summed count.""" + import math + keys = ("invoiced_sales", "confirmed_sales", "order_count", "average_order_value") + groups = [all_values, *agent_values, unassigned] + if len(agent_values) < 2 or any(not all(isinstance(g.get(k), (int, float)) + and not isinstance(g[k], bool) and math.isfinite(g[k]) for k in keys) for g in groups): + return False + if any(g["confirmed_sales"] <= 0 or g["order_count"] <= 0 or g["invoiced_sales"] <= 0 + for g in agent_values): + return False + if any(abs(sum(g[k] for g in [*agent_values, unassigned]) - all_values[k]) > + (0 if k == "order_count" else 0.01) for k in keys[:3]): + return False + return all(abs(g["average_order_value"] - (g["confirmed_sales"] / g["order_count"] + if g["order_count"] else 0)) <= 0.01 for g in groups) + + def clean_binding(raw): """Normalize a C2 `Binding`, or raise `SheetRefusal`. @@ -595,6 +755,106 @@ def clean_binding(raw): return out +def filter_cell_nodes(nodes): + """Cell value slots live on leaves at any depth, never on a fixed object path.""" + for node in nodes or (): + if not isinstance(node, dict): + continue + if isinstance(node.get("children"), list): + yield from filter_cell_nodes(node["children"]) + elif "valueRef" in node: + yield node + + +def resolve_filter_cells(nodes, read_cell): + """Resolve transient operands; retain authored references on the stored binding.""" + import math + from harness.filter_sql import MULTI_VALUE_OPS, VALUE_FREE_OPS + from harness.windows import ANCHOR_VALUE_FREE + out = [] + for node in nodes or (): + if not isinstance(node, dict): + out.append(node) + elif isinstance(node.get("children"), list): + out.append({**node, "children": resolve_filter_cells(node["children"], read_cell)}) + elif "valueRef" in node: + slot = node["valueRef"] + ref = str(slot.get("ref") or "").strip() if isinstance(slot, dict) else "" + if (node.get("op") in VALUE_FREE_OPS or node.get("op") == "within" + or node.get("dateMode") in ANCHOR_VALUE_FREE or node.get("rhs") is not None): + raise SheetRefusal("filter_cell_unreadable", ref or "(blank)", + "this condition uses a window or another operand instead of a value.") + value = read_cell(ref) + if (not isinstance(value, (str, int, float, bool)) or str(value).strip() == "" + or isinstance(value, float) and not math.isfinite(value)): + raise SheetRefusal("filter_cell_unreadable", ref or "(blank)", + "its value is blank or is not a single usable value.") + text = str(value).lower() if isinstance(value, bool) else str(value) + leaf = {**node, "value": text, "_cell_value_resolved": True} + if node.get("op") in MULTI_VALUE_OPS: + leaf["values"] = [text] + out.append(leaf) + else: + out.append(node) + return out + + +# W60-T40: explicit persisted identity, separate from both flat columns and cell value slots. +LINKED_FILTER_PREFIX = "linked:" + + +def linked_field_parts(key): + text = str(key or "") + if not text.startswith(LINKED_FILTER_PREFIX): + return None + parts = text.split(":") + if len(parts) != 3 or not all(parts[1:]): + raise SheetRefusal("linked_filter_unreadable", text, "its saved link identity is invalid.") + return parts[1], parts[2] + + +def linked_filter_ids(nodes): + return {key for key in filter_columns_named(nodes) if key.startswith(LINKED_FILTER_PREFIX)} + + +def rewrite_linked_filters(nodes, resolve): + """Preserve every conjunction; the route resolves only explicit linked leaves.""" + out = [] + for node in nodes or (): + if not isinstance(node, dict): + out.append(node) + elif isinstance(node.get("children"), list): + out.append({**node, "children": rewrite_linked_filters(node["children"], resolve)}) + elif linked_filter_ids([node]): + if not linked_field_parts(node.get("colId")) or node.get("rhs") is not None: + raise SheetRefusal("linked_filter_unreadable", ", ".join(sorted(linked_filter_ids([node]))), + "comparing columns across records is not supported. Type a value or use a cell.") + out.append(resolve(node)) + else: + out.append(node) + return out + + +def linked_id_predicate(column, ids): + """Membership using existing numeric equality, with no ID cap or deep SQL OR chain. + + isAnyOf deliberately refuses numeric columns in filter_sql. Balanced groups of eq leaves + express the same membership without changing that pure compiler or truncating its values. + """ + values = sorted({int(pid) for pid in ids}) + + def tree(lo, hi): + if hi == lo: + return {"conj": "and", "children": [ + {"colId": column, "op": "isEmpty"}, {"colId": column, "op": "isNotEmpty"}]} + if hi - lo == 1: + return {"colId": column, "op": "eq", "value": str(values[lo])} + mid = (lo + hi) // 2 + return {"conj": "or", "children": [tree(lo, mid), tree(mid, hi)]} + + return tree(0, len(values)) + + def filter_columns_named(nodes): """Every `colId` a filter tree names, at any depth, including a right-hand column reference. @@ -682,6 +942,27 @@ def compile_scope(nodes, conj, columns, *, today=None, cohort_sets=None): is what a relative date condition needs, and it is exactly the value R14's `period.ref` feeds. """ from harness.filter_sql import compile_filter_tree + import math + unresolved = linked_filter_ids(nodes) + if unresolved: + raise SheetRefusal("linked_filter_unreadable", ", ".join(sorted(unresolved)), + "the linked records have not been resolved.") + for leaf in filter_cell_nodes(nodes): + slot = leaf.get("valueRef") + ref = slot.get("ref") if isinstance(slot, dict) else "(blank)" + if leaf.get("_cell_value_resolved") is not True: + raise SheetRefusal("filter_cell_unreadable", ref, "the source value has not been read.") + try: + compiled = compile_filter_tree([leaf], conj="and", columns=columns or {}, + today=today, cohort_sets=cohort_sets) + except ValueError: + raise SheetRefusal("filter_cell_unreadable", ref, + "its condition uses a value or operator this database cannot apply.") + if (compiled is None or leaf.get("colId") not in compiled.columns_used + or not compiled.params or any(isinstance(p, float) and not math.isfinite(p) + for p in compiled.params)): + raise SheetRefusal("filter_cell_unreadable", ref, + "its condition cannot use this value on the selected column.") return compile_filter_tree(list(nodes or ()), conj=conj or "and", columns=columns or {}, today=today, cohort_sets=cohort_sets) @@ -714,6 +995,9 @@ def aggregate_sql(spec, aggregate, columns): col = (columns or {}).get(key) if not col or not col.get("sql"): raise SheetRefusal("field_unknown", key or "That column") + # An empty SUM is zero. AVG/MIN/MAX retain NULL because there is no value to report. + if fn == "sum": + return f"COALESCE(SUM({col['sql']}), 0)", key return f"{fn.upper()}({col['sql']})", key @@ -800,6 +1084,31 @@ def build_ids_query(spec, columns, *, scope_nodes, scope_conj, wall_nodes=None, # ── W49-T10: the dependents provider (contract C3) ─────────────────────────────────────────── +def _sheet_cell_link(workbook_id, tab, addr): + """C2 address from stored identities, or the reason the reader cannot open this cell.""" + if not isinstance(workbook_id, str) or not workbook_id: + return None, "The workbook ID is missing. Restore the workbook before opening this cell." + tab_id = tab.get("id") + if not isinstance(tab_id, str) or not tab_id: + return None, "The tab ID is missing. Restore the tab before opening this cell." + if not isinstance(addr, str) or not _PERIOD_REF.fullmatch(addr): + return None, "The cell address is invalid. Restore its A1 address in the workbook." + # W60-T12, 2026-09-14: T10's reader falls back when a tab, row or column is hidden. + # Such a URL would open another cell, so report the cause and recovery instead. + if tab.get("visibility", "visible") != "visible": + return None, "The tab is hidden. Unhide it in the workbook before opening this cell." + letters = addr.rstrip("0123456789") + row_number = int(addr[len(letters):]) + col_number = 0 + for letter in letters: + col_number = col_number * 26 + ord(letter) - ord("A") + 1 + if row_number in (tab.get("hiddenRows") or ()): + return None, "The cell's row is hidden. Unhide the row before opening this cell." + if col_number in (tab.get("hiddenCols") or ()): + return None, "The cell's column is hidden. Unhide the column before opening this cell." + return "#/sheet?" + urlencode({"wb": workbook_id, "tab": tab_id, "cell": addr}), "" + + def sheet_dependents(st, target): """Which Sheet cells depend on `target`, as C3 `Dependent` rows. @@ -823,35 +1132,37 @@ def sheet_dependents(st, target): if not kind or not key: return [] - hits, unreadable = [], 0 + hits, unreadable = [], [] for row in sheet_store.list_workbooks(st): try: book = sheet_store.get_workbook(st, row["id"]) except sheet_store.SheetRefusal: - unreadable += 1 + name = str(row.get("name") or row["id"]) + unreadable.append({ + "kind": "sheet_unreadable", + "label": f"Workbook {name} that could not be read", + "count": 1, "ids": [], "href": None, + "refusal": f"{name} could not be opened because its workbook document could not be read. " + "Restore the document or remove its stale workbook entry.", + }) continue for tab in book.get("sheets") or (): for addr, cell in (tab.get("cells") or {}).items(): if _cell_depends_on(cell, kind, key, database): - hits.append(f"{book.get('name') or book.get('id')} / " - f"{tab.get('name') or tab.get('id')}!{addr}") - - out = [] - if hits: - # ⛔ NO `label` KEY, DELIBERATELY. `dependents.dependents_report` defaults it to - # `noun("sheet_cell", count)`, which AGREES IN NUMBER; a static "Sheet cells" renders - # the user-facing refusal "Leads is used by 1 Sheet cells". Enforced, not merely - # written down: `verify_shared_views.py`'s `(W49-T11b)` leg reds on a static label. - out.append({"kind": "sheet_cell", "count": len(hits), "ids": hits}) - if unreadable: - # ⚠ REPORTED, never folded into the count above. "3 cells depend on this" and "3 cells - # depend on this and 2 workbooks could not be read" are different answers, and only the - # second one lets a person decide whether to proceed. - out.append({"kind": "sheet_unreadable", - "label": ("Workbook that could not be read" if unreadable == 1 - else "Workbooks that could not be read"), - "count": unreadable, "ids": []}) - return out + display = (f"{book.get('name') or book.get('id')} / " + f"{tab.get('name') or tab.get('id')}!{addr}") + href, reason = _sheet_cell_link(row["id"], tab, addr) + # W60-T12, 2026-09-14: C3 now requires a label and a link per cell. + # Retain the count/ids keys and display IDs; each singular label agrees + # with count=1 so legacy refusal readers still describe real dependents. + item = {"kind": "sheet_cell", "count": 1, "ids": [display], + "label": f"Sheet cell in {display}"} + item["href"] = href + if reason: + item["refusal"] = f"{display}: {reason}" + hits.append(item) + # Unreadable workbooks remain separate from the count of known dependent cells. + return hits + unreadable # ── W49-T11 (combine-owned): register the provider with the dependents registry ──────────── diff --git a/web/src/automation/AutomationSurface.tsx b/web/src/automation/AutomationSurface.tsx index fb259b5fbf47d66bc868b3a2d64254f8dc9377a2..eeab8f63138486bb1cd857af12f35b0980c0cb46 100644 --- a/web/src/automation/AutomationSurface.tsx +++ b/web/src/automation/AutomationSurface.tsx @@ -26,6 +26,7 @@ import { AUTOMATION_OPEN_EVENT } from "../apiContract"; // WAVE'S BRAND while a comment asserted parity. So this imports the component that POINTS at the // generated artifact. [[loopable-nav-logo-toggle]] — one element paints the mark. import { Mark } from "../shell/Brand"; +import { agentAddressHash, readAgentAddress } from "../shell/nav"; import type { Automation, AutomationList } from "./automationApi"; import { AutomationError, @@ -34,6 +35,7 @@ import { listVoiceAgents, listAutomations, liveStepOf, + nextAutomationName, patchAutomation, rememberAutomation, replaceVoiceAgent, @@ -212,29 +214,6 @@ if (typeof window !== "undefined") { * that a database can be created BY asking for an automation. */ -/** - * A default name for a brand-new automation, and it has to be one the server ACCEPTS. - * - * MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create - * with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name - * cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that - * sorts by name with nothing to tell the rows apart. Numbering from the existing list is a - * client-side convenience over a server that does not care: if two tabs race, the loser gets a - * duplicate name, which is legal, visible and renameable — never an error the user has to read. - */ -function nextAutomationName(existing: Automation[]): string { - const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase())); - for (let n = existing.length + 1; ; n += 1) { - // ⭐ WAVE 34 · R12 — "Agent N", not "Automation N". ⚠ THIS ONE IS STORED, so it is the only - // string in the rename that leaves a visible seam: automations created before today keep - // "Automation 3" as their NAME until somebody renames them. That is correct and deliberate — - // a name is a person's own text, and rewriting stored names to match a caption change would - // edit user data to tidy a word. The seam is in the rail for a while and then gone. - const candidate = `Agent ${n}`; - if (!taken.has(candidate.toLowerCase())) return candidate; - } -} - export default function AutomationSurface() { /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes through `data === null` and therefore never paints the skeleton. The lazy initialiser runs @@ -282,6 +261,8 @@ export default function AutomationSurface() { wantedId.current = id; setActiveId(id); setActiveVoiceId(""); + const address = agentAddressHash("agent", id); + if (window.location.hash !== address) window.location.hash = address; }, []); const selectVoice = useCallback((id: string) => { @@ -289,8 +270,28 @@ export default function AutomationSurface() { wantedId.current = ""; setActiveId(""); setActiveVoiceId(id); + const address = agentAddressHash("voice", id); + if (window.location.hash !== address) window.location.hash = address; }, []); + useEffect(() => { + const readAddress = () => { + const address = readAgentAddress(); + if (address.agent) select(address.agent); + else if (address.voice) selectVoice(address.voice); + else { + wantedId.current = ""; + setActiveId(""); + setActiveVoiceId(""); + setSlackOpen(false); + } + }; + // Select before the list arrives; a missing id simply has no matching rail row. + readAddress(); + window.addEventListener("hashchange", readAddress); + return () => window.removeEventListener("hashchange", readAddress); + }, [select, selectVoice]); + /** * ⭐⭐ W33 (C-18, from lane C) — WHICH PANE AN INBOX ITEM ASKED FOR, and it was DROPPED here. * diff --git a/web/src/automation/automationApi.ts b/web/src/automation/automationApi.ts index 23397cfa8f9bd3fa6e0a5685b93ba152675dd5dc..39db9996451f97f690409097f5dfb48541d9bac6 100644 --- a/web/src/automation/automationApi.ts +++ b/web/src/automation/automationApi.ts @@ -1396,6 +1396,29 @@ export function forgetAutomation(id: string): AutomationList | null { return data; } +/** + * A default name for a brand-new automation, and it has to be one the server ACCEPTS. + * + * MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create + * with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name + * cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that + * sorts by name with nothing to tell the rows apart. Numbering from the existing list is a + * client-side convenience over a server that does not care: if two tabs race, the loser gets a + * duplicate name, which is legal, visible and renameable — never an error the user has to read. + */ +export function nextAutomationName(existing: { name: string }[]): string { + const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase())); + for (let n = existing.length + 1; ; n += 1) { + // ⭐ WAVE 34 · R12 — "Agent N", not "Automation N". ⚠ THIS ONE IS STORED, so it is the only + // string in the rename that leaves a visible seam: automations created before today keep + // "Automation 3" as their NAME until somebody renames them. That is correct and deliberate — + // a name is a person's own text, and rewriting stored names to match a caption change would + // edit user data to tidy a word. The seam is in the rail for a while and then gone. + const candidate = `Agent ${n}`; + if (!taken.has(candidate.toLowerCase())) return candidate; + } +} + export function createAutomation(body: unknown): Promise<{ automation: Automation }> { return send<{ automation: Automation }>("/automations", { method: "POST", diff --git a/web/src/connectors/ConnectorModal.tsx b/web/src/connectors/ConnectorModal.tsx index 22eba06675165ec1132b09fd89f739ad51a529c1..636bbb471f1afc0de56f383ce02df5bf5c52289e 100644 --- a/web/src/connectors/ConnectorModal.tsx +++ b/web/src/connectors/ConnectorModal.tsx @@ -275,6 +275,8 @@ export interface ConnectorDetail { databases: ConnectorDatabase[]; enablement: ConnectorEnablement | null; materialization?: MaterializationCapability | null; + /** Present only when the server says this administrator can request a provider sync. */ + syncNow?: { provider: string; path: string } | null; introspection: ConnectorIntrospection; relationships: ConnectorRelationshipGraph; } @@ -503,6 +505,9 @@ export function parseConnectorDetail(body: unknown, selectedKey: string): Connec keychain: parseKeychainHalf(source.keychain), databases: parseDatabases(source.databases), enablement: parseConnectorEnablement(source.enablement, selectedKey), + syncNow: obj(source.syncNow).provider === selectedKey && + str(obj(source.syncNow).path) === `/connectors/${selectedKey}/sync/now` + ? { provider: selectedKey, path: `/connectors/${selectedKey}/sync/now` } : null, materialization: obj(source.materialization).provider === selectedKey && str(obj(source.materialization).statusPath) === `/connectors/${selectedKey}/jobs` && str(obj(source.materialization).retryPath) === `/connectors/${selectedKey}/sync/retry` @@ -815,11 +820,13 @@ export default function ConnectorModal({ */ const [pulling, setPulling] = useState(false); /** - * ⛔⛔ THE PULL DIALOG'S OWN SURFACE, HELD IN STATE RATHER THAN A REF, because + * ⛔⛔ THE PULL AND CONFIRM DIALOGS' OWN SURFACES, HELD IN STATE RATHER THAN REFS, because * `useOverlayLayer` reads `outsideElements` on every RENDER and a ref that fills in after mount * would never reach it. See the dismissal note on the layer below. */ const [pullSurface, setPullSurface] = useState(null); + const [tabConfirmSurface, setTabConfirmSurface] = useState(null); + const [leaveSurface, setLeaveSurface] = useState(null); const canPull = isAmazonConnector(row.key); const unsavedMessage = databasePendingCount === 1 @@ -862,14 +869,15 @@ export default function ConnectorModal({ * * `useOverlayLayer` guards its handlers with `stack.isTop(id)` and that guard is INERT in this * app: `OverlaySurface.tsx` exports `OverlayProvider` and nothing mounts it, so the context is - * null and every layer believes it is the top one. Both layers also listen on `document` in + * null and every layer believes it is the top one. Both layers listen for pointers on `document` in * the CAPTURE phase, where `stopPropagation` does not stop a sibling listener on the same - * node. So Escape inside the pull dialog would close the dialog AND this modal underneath it. - * Answering here keeps the fix in this ticket's own files: while the pull dialog is open, - * this layer's dismissal closes THAT and stops. + * node. So a pointer press can dismiss the dialog AND this modal underneath it. + * Answering here keeps the fix in this ticket's own files: while either confirm is open, + * this layer refuses dismissal; while the pull dialog is open, it closes THAT and stops. + * Escape already belongs to the topmost entry in `useEscapeClose`'s shared stack. */ onDismiss: () => { - if (pendingLeave) return; + if (pendingLeave || tabConfirmSurface) return; if (pulling) { setPulling(false); return; @@ -878,13 +886,16 @@ export default function ConnectorModal({ }, dismissOnOutside: true, /** - * ⛔ AND THE POINTER HALF OF THE SAME PROBLEM. The dialog portals as a SIBLING of this - * panel, so a press inside it is literally outside `panelRef` and would dismiss this modal - * out from under it in the middle of a paid pull. `outsideElements` is the hook's own answer. + * ⛔ AND THE POINTER HALF OF THE SAME PROBLEM. These dialogs portal as SIBLINGS of this + * panel, so a press inside one is literally outside `panelRef` and would dismiss this modal + * out from under a paid pull or a staged change. `outsideElements` is the hook's own answer. + * Each surface includes its scrim, so a stray press there cannot dismiss this modal either. */ - outsideElements: pullSurface ? [pullSurface] : [], + outsideElements: [pullSurface, tabConfirmSurface, leaveSurface].filter( + (element): element is HTMLElement => element !== null, + ), initialFocus: "[data-overlay-autofocus]", - trapFocus: pendingLeave === null, + trapFocus: pendingLeave === null && tabConfirmSurface === null, // The layer captures whatever was focused when it mounted and focuses it again on unmount, // which is the card's own open button. That is the done-when's "focus returns to the card". restoreFocus: true, @@ -1013,6 +1024,7 @@ export default function ConnectorModal({ onChanged={reread} onPendingChange={setDatabasePendingCount} onBusyChange={setDatabaseTabBusy} + onConfirmSurface={setTabConfirmSurface} reveal={databaseReveal} /> @@ -1198,6 +1210,7 @@ export default function ConnectorModal({ tone="destructive" onConfirm={confirmLeave} onCancel={() => setPendingLeave(null)} + surfaceRef={setLeaveSurface} /> ); diff --git a/web/src/connectors/DatabasesTab.tsx b/web/src/connectors/DatabasesTab.tsx index dea355b83a301d3e55a4ae95606f374b4887eb7b..4f57b79812f27e92fe9e5441f33ba879b4bccd2e 100644 --- a/web/src/connectors/DatabasesTab.tsx +++ b/web/src/connectors/DatabasesTab.tsx @@ -57,6 +57,7 @@ // --------------------------------------------------------------------------- import { forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react"; +import type { MouseEvent } from "react"; import { API_V1, CREDENTIALS, WORKSPACE_STALE_EVENT, signal } from "../apiContract"; import { ConfirmDialog } from "../ui/ConfirmDialog"; import { useEscapeClose } from "../ui/useEscapeClose"; @@ -65,9 +66,200 @@ import type { ConnectorRow } from "./ConnectorsPage"; import { watchMaterialization } from "./materializationJobs"; import type { MaterializationJob } from "./materializationJobs"; +interface AmazonTableSync { + rowsLoaded: number | null; + lastSuccess: string | null; + inProgress: boolean; + deferred: { reason: string } | null; + nextDue: string | null; + firstLoad: { remaining: number; etaSeconds: number; cause: string; needsAttention: boolean } | null; + stuck: boolean | null; + stuckCause: string; + unknownCause: string; +} + +function syncObject(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record : {}; +} + +function syncText(value: unknown): string { + return typeof value === "string" && value.length <= 512 ? value : ""; +} + +function syncCount(value: unknown): number | null { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; +} + +function syncStamp(value: unknown): string | null { + const text = syncText(value); + return text && Number.isFinite(Date.parse(text)) ? text : null; +} + +/** ConnectorModal's parser omits provider additions. Parse our own raw detail read here. */ +export function parseAmazonSyncStates(value: unknown): Record | null { + const payload = syncObject(value); + if (syncObject(payload.overview).key !== "amazon_sp" || !Array.isArray(payload.databases)) return null; + const states: Record = {}; + for (const raw of payload.databases) { + const database = syncObject(raw); + if (typeof database.key !== "string" || !database.key.startsWith("ut_amzn_")) continue; + if (!database.sync || typeof database.sync !== "object" || Array.isArray(database.sync)) { + states[database.key] = null; + continue; + } + const state = syncObject(database.sync); + const load = syncObject(state.firstLoad); + const remaining = syncCount(load.remaining); + const etaSeconds = syncCount(load.etaSeconds); + const cause = syncText(load.cause); + const reason = syncText(syncObject(state.deferred).reason); + states[database.key] = { + rowsLoaded: syncCount(state.rowsLoaded), lastSuccess: syncStamp(state.lastSuccess), + inProgress: state.inProgress === true, + deferred: reason ? { reason } : null, nextDue: syncStamp(state.nextDue), + firstLoad: remaining !== null && remaining > 0 && etaSeconds !== null && cause + ? { remaining, etaSeconds, cause, needsAttention: load.needsAttention === true } : null, + stuck: state.stuck === true ? true : state.stuck === false ? false : null, + stuckCause: syncText(state.stuckCause), unknownCause: syncText(state.unknownCause), + }; + } + return states; +} + +function useAmazonSyncStates(enabled: boolean, detail: ConnectorDetail, syncPhase?: string) { + const [states, setStates] = useState>({}); + const [error, setError] = useState(""); + useEffect(() => { + setStates({}); + setError(""); + if (!enabled) return; + const controller = new AbortController(); + let reading = false; + const poll = async () => { + if (reading || controller.signal.aborted) return; + reading = true; + try { + const response = await fetch(`${API_V1}/connectors/amazon_sp/detail`, { + credentials: CREDENTIALS, signal: controller.signal, + }); + const raw: unknown = await response.json(); + if (controller.signal.aborted) return; + const next = response.ok ? parseAmazonSyncStates(raw) : null; + if (!next) throw new Error("Unreadable sync status"); + setStates(next); + setError(""); + } catch { + if (controller.signal.aborted) return; + setStates({}); + setError("Sync status could not be read. Reload to check again."); + } finally { + reading = false; + } + }; + void poll(); + const timer = window.setInterval(() => { + if (document.visibilityState !== "hidden") void poll(); + }, 5000); + return () => { controller.abort(); window.clearInterval(timer); }; + }, [enabled, detail, syncPhase]); + return { states, error }; +} + +function readableSyncStamp(value: string): string { + return new Date(value).toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); +} + +function plural(count: number, unit: string): string { + return `${count.toLocaleString()} ${unit}${count === 1 ? "" : "s"}`; +} + +function firstLoadEstimate(seconds: number): string { + if (seconds < 60) return plural(seconds, "second"); + const minutes = Math.ceil(seconds / 60); + if (minutes < 60) return plural(minutes, "minute"); + const hours = Math.floor(minutes / 60); + const rest = minutes % 60; + return rest ? `${plural(hours, "hour")} ${plural(rest, "minute")}` : plural(hours, "hour"); +} + +export function AmazonSyncChip({ state, error }: { state?: AmazonTableSync | null; error?: string }) { + const phase = state?.inProgress ? "syncing" : state?.stuck ? "failed" + : state?.deferred ? "deferred" : state?.rowsLoaded != null ? "ready" : "unknown"; + return
+ {state?.rowsLoaded == null ? "Not loaded yet" + : `${state.rowsLoaded.toLocaleString()} rows loaded`} + {state?.lastSuccess ? Last success {readableSyncStamp(state.lastSuccess)} : null} + {state?.inProgress ? Syncing : null} + {state?.deferred ? Deferred: {state.deferred.reason} : null} + {state?.nextDue ? Next due {readableSyncStamp(state.nextDue)} : null} + {state?.stuckCause ? {state.stuckCause}. Retry sync to refresh progress. : null} + {state?.firstLoad ? + First load for connection: {state.firstLoad.remaining.toLocaleString()} reports remaining. + {" "}Estimated minimum time: {firstLoadEstimate(state.firstLoad.etaSeconds)}. + {" "}{state.firstLoad.cause} + {state.firstLoad.needsAttention ? " First load needs attention. Retry sync to continue." : ""} + : null} + {state?.unknownCause ? {state.unknownCause} : null} + {!state ? {error || "Sync status is unknown in this container. Reload to check again."} : null} +
; +} + +interface DependentItem { id: string; label: string; href: string | null; reason: string } +interface DependentKind { + kind: string; label: string; count: number; ids: string[]; + items?: DependentItem[]; href?: string | null; reason?: string; +} + +function dependentHref(value: unknown): string | null { + return typeof value === "string" && (value.startsWith("#/data?") || value.startsWith("#/sheet?")) + ? value : null; +} + +function parseDependents(value: unknown): DependentKind[] { + if (!Array.isArray(value)) return []; + return value.flatMap((entry: unknown): DependentKind[] => { + if (!entry || typeof entry !== "object") return []; + const row = entry as Record; + if (typeof row.kind !== "string" || typeof row.label !== "string" || + typeof row.count !== "number" || !Number.isFinite(row.count) || row.count <= 0 || + !Array.isArray(row.ids) || !row.ids.every((id) => typeof id === "string")) return []; + const dependent: DependentKind = { + kind: row.kind, label: row.label, count: row.count, ids: row.ids, + }; + if ("href" in row) dependent.href = dependentHref(row.href); + if (typeof row.reason === "string") dependent.reason = row.reason; + if (Array.isArray(row.items)) { + dependent.items = row.items.flatMap((raw: unknown): DependentItem[] => { + if (!raw || typeof raw !== "object") return []; + const item = raw as Record; + if (typeof item.id !== "string" || typeof item.label !== "string" || + typeof item.reason !== "string" || + !(item.href === null || typeof item.href === "string")) return []; + return [{ id: item.id, label: item.label, href: dependentHref(item.href), reason: item.reason }]; + }); + } + return [dependent]; + }); +} + +function dependentRows(dependent: DependentKind): DependentItem[] { + if (dependent.items?.length) return dependent.items; + // Lane A may supply its workbook address and name on the dependent itself instead + // of an items list. Render that shape too without guessing any Sheet address. + if ("href" in dependent) return [{ + id: dependent.ids.join(":"), label: dependent.label, href: dependent.href ?? null, + reason: dependent.reason || "This dependent has no direct address available.", + }]; + return dependent.ids.map((id) => ({ + id, label: id, href: null, + reason: dependent.reason || "This dependent has no direct address available.", + })); +} + /** What `GET /dependents` said, or why it could not say anything. */ type DependentsAnswer = - | { ok: true; refusal: string } + | { ok: true; refusal: string; dependents: DependentKind[] } | { ok: false; message: string }; /** @@ -84,12 +276,15 @@ async function readDependents(tableKey: string): Promise { { credentials: CREDENTIALS } ); const body = (await res.json().catch(() => null)) as - | { refusal?: unknown; error?: { message?: string } } + | { refusal?: unknown; dependents?: unknown; error?: { message?: string } } | null; if (!res.ok) { return { ok: false, message: body?.error?.message || `The server answered ${res.status}.` }; } - return { ok: true, refusal: typeof body?.refusal === "string" ? body.refusal : "" }; + return { + ok: true, refusal: typeof body?.refusal === "string" ? body.refusal : "", + dependents: parseDependents(body?.dependents), + }; } catch { return { ok: false, message: "Cannot reach the server." }; } @@ -113,6 +308,85 @@ async function write(path: string, method: "PUT" | "PATCH" | "POST", body: unkno } } +type SyncNowReceipt = { + provider: string; + state: "idle" | "queued" | "running" | "done" | "failed"; + detail: string; +}; + +async function readSyncNow(path: string, provider: string): Promise< + { receipt: SyncNowReceipt; message: "" } | { receipt: null; message: string } +> { + try { + const res = await fetch(`${API_V1}${path}`, { credentials: CREDENTIALS }); + const payload = await res.json().catch(() => null) as + | { provider?: unknown; state?: unknown; detail?: unknown; error?: { message?: string } } + | null; + if (!res.ok) return { receipt: null, message: payload?.error?.message || `The server answered ${res.status}.` }; + if (payload?.provider !== provider || typeof payload.state !== "string" || + !["idle", "queued", "running", "done", "failed"].includes(payload.state) || + typeof payload.detail !== "string") { + return { receipt: null, message: "The server returned an unreadable sync status." }; + } + return { receipt: { provider, state: payload.state as SyncNowReceipt["state"], detail: payload.detail }, message: "" }; + } catch { + return { receipt: null, message: "Cannot reach the server to check sync progress." }; + } +} + +function useSyncNow(capability: ConnectorDetail["syncNow"]) { + const path = capability?.path; + const provider = capability?.provider; + const [receipt, setReceipt] = useState(null); + const [message, setMessage] = useState(""); + const [requestError, setRequestError] = useState(""); + const [posting, setPosting] = useState(false); + const [reads, setReads] = useState(0); + const postInFlight = useRef(false); + const readGeneration = useRef(0); + useEffect(() => { + if (!path || !provider) return; + const generation = ++readGeneration.current; + let stopped = false; + let reading = false; + const poll = async () => { + if (reading || stopped || generation !== readGeneration.current) return; + reading = true; + const answer = await readSyncNow(path, provider); + reading = false; + if (stopped || generation !== readGeneration.current) return; + setMessage(answer.message); + setPosting(false); + if (!answer.receipt) return; // A failed read does not prove that the run finished. + setReceipt(answer.receipt); + if (!["queued", "running"].includes(answer.receipt.state)) window.clearInterval(timer); + }; + const timer = window.setInterval(() => void poll(), 1500); + void poll(); + return () => { stopped = true; readGeneration.current += 1; window.clearInterval(timer); }; + }, [path, provider, reads]); + const active = receipt !== null && receipt.provider === provider && + (receipt.state === "queued" || receipt.state === "running"); + const start = async () => { + if (!path || active || postInFlight.current) return; + const generation = ++readGeneration.current; + postInFlight.current = true; + setPosting(true); + setReceipt(null); + setMessage(""); + setRequestError(""); + const error = await write(path, "POST", {}); + postInFlight.current = false; + if (generation !== readGeneration.current) return; + setRequestError(error); + // Read after every response, including an uncertain network failure. The server may have + // accepted the intent already. Only a readback may claim that it is queued or finished. + setReads((value) => value + 1); + }; + return { receipt: receipt?.provider === provider ? receipt : null, message: requestError || message, + busy: Boolean(active || posting), start }; +} + /** * Enabling sets the flag and THEN materialises, so a refusal can still have changed the server. * @@ -652,6 +926,7 @@ const DatabasesTab = forwardRef void; /** Locks the modal actions while the dependents check or the batch is in flight. */ onBusyChange: (busy: boolean) => void; + onConfirmSurface?: (element: HTMLElement | null) => void; /** The enabled row and its result survive the modal's loading remount. */ reveal?: DatabaseReveal | null; }>(function DatabasesTab({ @@ -660,6 +935,7 @@ const DatabasesTab = forwardRef(null); const [refusals, setRefusals] = useState>({}); + const [refusalDependents, setRefusalDependents] = useState>({}); + const [pendingDestination, setPendingDestination] = useState(null); const [autoEnableResults, setAutoEnableResults] = useState>( () => reveal?.key && reveal.status ? { [reveal.key]: reveal.status } : {} ); @@ -684,7 +962,10 @@ const DatabasesTab = forwardRef([]); const [jobError, setJobError] = useState(""); const [retrying, setRetrying] = useState(""); - const capability = detail.materialization; + const capability = row.key === "amazon_sp" ? null : detail.materialization; + const syncNow = useSyncNow(detail.syncNow); + const showsAmazonSync = row.key === "amazon_sp" && introspection.fieldsVisible; + const amazonSync = useAmazonSyncStates(showsAmazonSync, detail, syncNow.receipt?.state); useEffect(() => { if (!capability) return; return watchMaterialization(capability, (next, message) => { @@ -771,6 +1052,7 @@ const DatabasesTab = forwardRef ({ ...current, [database.key]: "" })); + setRefusalDependents((current) => ({ ...current, [database.key]: [] })); setAutoEnableResults((current) => ({ ...current, [database.key]: "" })); if (!next) { const answer = await readDependents(database.key); @@ -787,6 +1069,7 @@ const DatabasesTab = forwardRef ({ ...current, [database.key]: answer.refusal })); + setRefusalDependents((current) => ({ ...current, [database.key]: answer.dependents })); return; } if (!usesProviderEnablement) { @@ -847,6 +1130,19 @@ const DatabasesTab = forwardRef, href: string) => { + if (busy) { + event.preventDefault(); + return; + } + // Hash navigation never fires beforeunload. Reuse the modal's discard wording + // and ConfirmDialog surface registration without extending its prop contract. + if (pendingCount || renaming) { + event.preventDefault(); + setPendingDestination(href); + } + }; + const savePending = useCallback(async () => { const snapshot = pending; if (!pendingChangeCount(snapshot)) return; @@ -933,7 +1229,7 @@ const DatabasesTab = forwardRef; - } - - if (!databases.length) { - // One line plus the action that fills it (DESIGN.md section 4), and it names the cause - // rather than implying the connector creates nothing. - return ( -

- This connection has not declared any databases in this workspace. Connect it from the - Keychain tab to read its schema. -

- ); - } - - return ( -
- {showsPopulation && introspection.fieldsVisible ? ( + const showCensus = showsPopulation && introspection.fieldsVisible; + const header = showCensus || detail.syncNow ? (
+ {showCensus ? <>

Count rows stops at 101 rows per source table and displays 100+ from 100 onward. Each table still needs a request, so this can take about a minute or longer. @@ -1012,7 +1293,28 @@ const DatabasesTab = forwardRef ) : null} + : null} + {detail.syncNow ? ( +

+ {syncNow.receipt?.state === "queued" ? "Sync queued." + : syncNow.receipt?.state === "running" ? "Sync running." + : syncNow.receipt?.state === "done" ? "Sync finished." + : syncNow.receipt?.state === "failed" ? `Sync failed. ${syncNow.receipt.detail}` + : syncNow.busy ? "Checking sync progress..." : ""} +

+ ) : null} + {detail.syncNow && syncNow.message ? ( +

{syncNow.message}

+ ) : null}
+ {detail.syncNow ? ( + + ) : null} + {showCensus ? <> + : null}
- ) : null} + ) : null; + + if (!introspection.introspectable) { + return
{header}
; + } + + if (!databases.length) { + // Sync now remains reachable before the first schema has arrived. + return ( +
+ {header} +

+ This connection has not declared any databases in this workspace. Connect it from the + Keychain tab to read its schema. +

+
+ ); + } + + return ( +
+ {header} {error ? (

{error} @@ -1138,6 +1462,8 @@ const DatabasesTab = forwardRef{job.error} : null}

) : null} + {showsAmazonSync ? : null}
); diff --git a/web/src/customer-grid/ColumnMenu.tsx b/web/src/customer-grid/ColumnMenu.tsx index 414d695fbd4b1118236b169abb86bf49f94130bb..b20372688f1e5ba0c6c91b67017807998a099e57 100644 --- a/web/src/customer-grid/ColumnMenu.tsx +++ b/web/src/customer-grid/ColumnMenu.tsx @@ -196,6 +196,10 @@ interface ColumnMenuProps { locked: boolean; /** Product-owned preset fields may still drive view actions, but their schema is immutable. */ schemaLocked?: boolean; + /** Linked grids expose personal layout actions only, even when edit callbacks exist. */ + embedded?: boolean; + columnWidth?: number; + onColumnWidth?: (width: number) => void; /** * ⭐⭐ W40-T24 — the grid is a `ut_*` runtime database rather than an Odoo-backed topic. It * feeds `columnDefinitionOffer` (display.ts), which is this menu's ONE statement of the pre-set @@ -472,7 +476,7 @@ type CreatePosition = "left" | "right" | "end"; * choices/stars/formula config), and the Change-field control that swaps the column to show * another field. */ -type MenuPane = "menu" | "edit" | "format" | "summary" | "showAs"; +type MenuPane = "menu" | "edit" | "format" | "summary" | "showAs" | "width"; /** * How each creatable type reads in the menu. DERIVED from CREATABLE_TYPES rather than listed @@ -2511,6 +2515,9 @@ export default function ColumnMenu({ geocodable, locked, schemaLocked = false, + embedded = false, + columnWidth = 160, + onColumnWidth, isUserTable = false, viewer, sortedDir, @@ -2578,7 +2585,7 @@ export default function ColumnMenu({ * refusal in this file could only subtract from a door the host deliberately opened. */ const offer = columnDefinitionOffer(field, { isUserTable, isPrimary: locked }); - const canEditDefinition = !schemaLocked && mayEditFieldDefinition(field, viewer); + const canEditDefinition = !embedded && !schemaLocked && mayEditFieldDefinition(field, viewer); /** * ⭐⭐ W41-T31 (owner instruction 1, ruling R19) — THE DESCRIPTION-ONLY DOOR. * @@ -2602,7 +2609,7 @@ export default function ColumnMenu({ * gate — but it is the handler the whole pane exists to reach, and `typeof` is how you write * that without TS2774 ("this condition will always return true"). */ - const canEditDescription = schemaLocked && typeof onNote === "function"; + const canEditDescription = !embedded && schemaLocked && typeof onNote === "function"; /** * Either door opens the Edit-field pane; WHICH pane is decided at the render, below. Read by * the menu row and by the seed, so the two can never disagree about whether it is on offer. @@ -2615,12 +2622,13 @@ export default function ColumnMenu({ */ const editPaneOffered = canEditDefinition || canEditDescription; const [pane, setPane] = useState( - initialPane === "edit" + embedded ? "menu" : initialPane === "edit" ? (editPaneOffered ? "edit" : "menu") : (schemaLocked ? "menu" : initialPane ?? "menu") ); const [note, setNote] = useState(field.note ?? ""); - const [position, setPosition] = useState(initialPosition ?? null); + const [position, setPosition] = useState(embedded ? null : initialPosition ?? null); + const [widthDraft, setWidthDraft] = useState(String(Math.round(columnWidth))); /** Delete is DESTRUCTIVE (a custom field's stored values go with it) — first click arms, * second click fires. Reopening the menu disarms (state is per-mount, keyed by field). */ const [label, setLabel] = useState(""); @@ -3575,6 +3583,8 @@ export default function ColumnMenu({ * Item 6 — Insert left/right (and Add at end) jump STRAIGHT to the create form: name + type * autofocused, and none of the note/change-field/other-column chrome rendered around it. */ + // R8: a retained pane or a caller-supplied initial pane cannot expose schema controls. + if (!embedded) { if (position) { return ( = 50 && width <= 600; + return ( + + +
{ + event.preventDefault(); + if (!validWidth) return; + onColumnWidth(Math.round(width)); + onClose(); + }}> + +
+ + +
+
+
+ ); + } + // ------------------------------------------------------------------ the MENU pane // Wave-5 items 1/2/3: a flat action list. Editors open their OWN windows (the panes above); // nothing edits inline here. Conditional entries render IFF they apply to the current view. @@ -4852,10 +4908,11 @@ export default function ColumnMenu({ onDismiss={onClose} role="dialog" ariaLabel={`Column settings for ${field.label}`} - initialFocus="[data-overlay-autofocus]" + initialFocus={embedded ? "first" : "[data-overlay-autofocus]"} + dismissOnOutside={!embedded} dataKind="column-menu" > - + {/* Item 8c — any measure-carrying column exposes its PERIOD, the property that defines it. Item 21 (2026-08-02) moved the EDITOR into the Edit-field pane; what stays here is one line that STATES the period and opens that pane, because the window is the @@ -4879,6 +4936,7 @@ export default function ColumnMenu({ an icon on EVERY action. Presentation only — every handler and event is the one it was before; entries not in Airtable's list slot into the nearest sensible group. */}
+ {!embedded && <> {/* Owner item 8 (2026-07-31) — EDIT FIELD sits at the very top, above Duplicate: the one window for the field's name, type (with per-type choices/stars/formula) and the Change-field control. Offered on EVERY field — what a read-only field @@ -5057,6 +5115,8 @@ export default function ColumnMenu({
+ } + {/* Group 3 — sort. Direction wording stays the shipped per-type directionLabel. */} {onSort && fieldOfferable && <> )} - {onUnpin && ( + {!embedded && onUnpin && ( @@ -5122,6 +5182,15 @@ export default function ColumnMenu({
+ {embedded && onColumnWidth && ( + + )} + {/* Group 5 — hide · delete. */} + {!embedded && <> {/* ⭐⭐ WAVE-34 (owner ruling R13) — THE BULK RUN, and it is TWO rows rather than one because "run this column" is two different questions with two different bills. ⛔ NEITHER LABEL SAYS "everything". `ai_enrich_may_write` refuses a human-edited cell @@ -5300,6 +5370,7 @@ export default function ColumnMenu({ Only {deleteOwner} or an administrator can delete this field.

)} + }
); diff --git a/web/src/customer-grid/CustomerGrid.tsx b/web/src/customer-grid/CustomerGrid.tsx index 3b16d0bd2f56730cc5c51a5fe5aa697f1be6bc76..a53eb5761e1bd84f1f2a14920326bbe4988bdb18 100644 --- a/web/src/customer-grid/CustomerGrid.tsx +++ b/web/src/customer-grid/CustomerGrid.tsx @@ -26,7 +26,7 @@ import type { } from "@glideapps/glide-data-grid"; import "@glideapps/glide-data-grid/dist/index.css"; -import { useCustomerData } from "./useCustomerData"; +import { useCustomerData, usePersonalGridLayout } from "./useCustomerData"; import type { SurfaceScope } from "./apiBridge"; import { mutateQueryWorkspace, QUERY_BINDING_EVENT, refuseQueryMutation } from "../query/queryApi"; import type { QueryVirtualBinding } from "../query/queryApi"; @@ -34,7 +34,7 @@ import { acceptsQueryPreview, routeQueryViewMutation } from "./queryPreview"; // ⭐ WAVE 30 · W30-T42 (contract C2) — the WINDOWED grid's arithmetic, pure and node-run in // `verify_grid_ux.py`, because every one of these decisions is taken inside a callback where // only its own source text could otherwise be checked. -import { WINDOW_ROWS, fetchAggregates } from "./apiBridge"; +import { WINDOW_ROWS, fetchAggregates, personalGridLayout } from "./apiBridge"; // ⭐⭐ W50-T03 / CONTRACT C1 — the windowed aggregate's envelope. Lane F serves the route // (W50-T31); this side is written to C1's shape and degrades to the loaded window until it // lands, which is the done-when's own observable rather than a workaround. @@ -124,7 +124,7 @@ import ViewSidebar, { viewMoveRefusal } from "./ViewSidebar"; import ColumnMenu from "./ColumnMenu"; import type { ColumnMenuState } from "./ColumnMenu"; import { HEADER_ICONS } from "./iconShapes"; -import { emitHostEvent, eventId } from "./hostBridge"; +import { bindHostEvent, eventId } from "./hostBridge"; // ⭐ WAVE 27 item 21 (R14) — which of this table's views carry an alert. The SERVER's // list is the only answer; the rail's alert door is a one-way create. import { fetchAlerts } from "../alerts/alertsApi"; @@ -857,7 +857,7 @@ export interface FieldBulkDoorDeps { lockedKey: string; fieldByKey: Map; isUserTable: boolean; - scope: string; + scope: SurfaceScope; rowsPath: string; setFields: (update: (current: Field[]) => Field[]) => void; stampFieldDelete: (key: string) => void; @@ -894,6 +894,7 @@ function reportFieldDeleteRefusals( export function useFieldBulkDoors(deps: FieldBulkDoorDeps): FieldBulkDoors { const { config, updateConfig, order, visible, lockedKey, fieldByKey, isUserTable, scope, rowsPath, setFields, stampFieldDelete } = deps; + const emitHostEvent = useMemo(() => bindHostEvent(scope), [scope]); const onSetFieldsHidden = useCallback((keys: string[], hidden: boolean) => { const next = new Set(visible); for (const key of keys) { @@ -948,7 +949,7 @@ export function useFieldBulkDoors(deps: FieldBulkDoorDeps): FieldBulkDoors { } })(); }, [config, fieldByKey, isUserTable, lockedKey, order, rowsPath, scope, setFields, - stampFieldDelete, updateConfig, visible]); + stampFieldDelete, updateConfig, visible, emitHostEvent]); const onShareFields = useCallback((keys: string[]) => { if (keys.length > 0) signal(TOAST_EVENT, @@ -1640,6 +1641,12 @@ export function serverGroupHeaders( }); } +/** Cell-owned presentation reported to the caller without a dependency on Sheet bindings. */ +export interface CellDisplayChoice { + mode: "grid" | "list" | "map"; + visible: string[]; +} + interface CustomerGridProps { scope?: SurfaceScope; /** Shell-owned Data workspace location; grid resolves its view/field, never the global rail. */ @@ -1658,6 +1665,8 @@ interface CustomerGridProps { /** Read-only Grid view embedded in a linked-record modal. It keeps the standard * filter/sort/search toolbar while withdrawing schema and row mutations. */ embedded?: boolean; + /** R8 per-user layout identity: source table + link; Sheet supplies its cell identity. */ + personalLayoutKey?: string; /** Complete caller-scoped projection for an embedded grid. Null withholds rows while * loading/refused; undefined retains the ordinary target reader. */ embeddedPayload?: CustomersPayload | null; @@ -1680,10 +1689,9 @@ interface CustomerGridProps { * | `embedded` | HIDDEN (`hostViews = []`) | the WHOLE table | filtered CLIENT-side | * | **`related`** | **SHOWN, the target's own** | **server, PRE-FILTERED** | **only linked** | * - * ⛔⛔ `related` IS NOT ROUTED THROUGH `embedded`, AND THAT IS THE WHOLE TICKET. `embedded` - * keeps hiding the rail and keeps fetching whole tables; R5 asks for the OTHER DATABASE'S OWN - * SAVED VIEWS, its rail included, with the record pinned. So this mode leaves every - * `embedded` guard alone and narrows the READ instead. + * W60 R8: `related` still selects the server-pinned READ and the target's view rail. + * Its row/schema write guards now match `embedded`; view selection is read-only and + * personalLayoutKey persists the user's layout separately from those target views. * * `label` is the pinned record's NAME and is display-only: the chip reads "Linked to *
@@ -801,6 +960,75 @@ export function HomeSurface({ */ export default function HomePage(props: HomeProps) { const { load: starLoad, starred, busy, counts, toggle } = useStarred(); + const [sheets, setSheets] = useState([]); + const [sheetsKnown, setSheetsKnown] = useState(false); + const [sheetsError, setSheetsError] = useState(""); + const [sheetBusy, setSheetBusy] = useState(false); + const sheetCreating = useRef(false); + useEffect(() => { + if (props.mode === "app" || props.mode === "data" || props.mode === "agents") return; + let active = true; + setSheetsKnown(false); + setSheetsError(""); + void readIndex().then((result) => { + if (!active) return; + setSheetsKnown(true); + if (result.ok) { + // C2/R28: render exactly the server's workbook directory, in its order. + setSheets(result.value.workbooks); + } else { + setSheetsError("Sheets could not be loaded. Reload to retry."); + } + }).catch(() => { + if (!active) return; + setSheetsKnown(true); + setSheetsError("Sheets could not be loaded. Reload to retry."); + }); + return () => { active = false; }; + }, [props.mode]); + + const onNewSheet = async () => { + if (sheetCreating.current) return; + sheetCreating.current = true; + setSheetBusy(true); + try { + const result = await createWorkbook(); + if (result.ok) { + window.location.hash = "#/sheet?wb=" + encodeURIComponent(result.value.id); + } else { + setSheetsError("The sheet could not be created. Try again."); + } + } catch { + setSheetsError("The sheet could not be created. Try again."); + } finally { + sheetCreating.current = false; + setSheetBusy(false); + } + }; + const [agentBusy, setAgentBusy] = useState(false); + const [agentsError, setAgentsError] = useState(""); + const agentCreating = useRef(false); + const onNewAgent = async () => { + if (agentCreating.current) return; + agentCreating.current = true; + setAgentBusy(true); + setAgentsError(""); + try { + const result = await createAutomation({ name: nextAutomationName(props.automations) }); + if (result.automation?.id) { + window.location.hash = agentAddressHash("agent", result.automation.id); + } else { + setAgentsError("The agent could not be created. Try again."); + } + } catch (failure) { + setAgentsError(failure instanceof AutomationError && failure.status === 403 + ? "You do not have permission to create an agent." + : "The agent could not be created. Try again."); + } finally { + agentCreating.current = false; + setAgentBusy(false); + } + }; const [applications, setApplications] = useState([]); const [applicationsKnown, setApplicationsKnown] = useState(false); const [applicationsError, setApplicationsError] = useState(""); @@ -833,7 +1061,7 @@ export default function HomePage(props: HomeProps) { }; }, []); useEffect(() => { - if (props.mode === "data") return; + if (props.mode === "data" || props.mode === "sheet" || props.mode === "agents") return; const controller = new AbortController(); const requestEpoch = ++applicationRequestEpoch.current; setApplicationsError(""); @@ -926,6 +1154,14 @@ export default function HomePage(props: HomeProps) { busy={busy} counts={counts} toggle={toggle} + sheets={sheets} + sheetsKnown={sheetsKnown} + sheetsError={sheetsError} + onNewSheet={onNewSheet} + sheetBusy={sheetBusy} + onNewAgent={onNewAgent} + agentBusy={agentBusy} + agentsError={agentsError || props.agentsError || ""} applications={applications} applicationsKnown={applicationsKnown} applicationsError={applicationsError} @@ -942,3 +1178,14 @@ export default function HomePage(props: HomeProps) { ); } + +// verify_shell_rail.py pins Shell's FIRST HomePage tag to its Home/Data/App mode expression. +// Keep this distinct tag for the Sheet gallery so that mount contract remains intact. +export function SheetGalleryPage(props: HomeProps) { + return ; +} + +// A distinct Agents tag preserves the same FIRST HomePage mount contract as SheetGalleryPage. +export function AgentsGalleryPage(props: HomeProps) { + return ; +} diff --git a/web/src/sheet/BindingEditor.tsx b/web/src/sheet/BindingEditor.tsx index bd5cd7b5daeb9b71ddfd999da42420576024f456..7a9edd595dc31e6fe50240aa23959f2581f0c4a3 100644 --- a/web/src/sheet/BindingEditor.tsx +++ b/web/src/sheet/BindingEditor.tsx @@ -37,16 +37,19 @@ // ease-out on opacity and transform, no emoji, and no em dash or en dash in any string on screen. import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; +import { API_V1, CREDENTIALS } from "../apiContract"; import { fetchTopicRows, fetchWorkspace } from "../customer-grid/apiBridge"; import type { SurfaceScope } from "../customer-grid/apiBridge"; import { dayText } from "../customer-grid/display"; import { FieldSelectButton } from "../customer-grid/FieldSelect"; -import { FieldTypeIcon } from "../customer-grid/icons"; +import { FieldTypeIcon, MenuIcon } from "../customer-grid/icons"; import { BodyPortal, OverlayProvider, useOverlayLayer } from "../customer-grid/OverlaySurface"; import type { Field, FilterTree, GridWorkspace, Measure, Row, SavedView } from "../customer-grid/types"; -import { choiceVocabulary, topicForScope } from "../customer-grid/types"; +import { choiceVocabulary, topicForScope, MULTI_VALUE_OPS } from "../customer-grid/types"; import type { WindowSpec } from "../customer-grid/windows"; -import { normalizeWindow } from "../customer-grid/windows"; +import { normalizeWindow, ENDING_KINDS, ENDING_LABELS, ENDING_STEPS, TO_DATE_KINDS, + endingBounds, endingStepDefault, ANCHOR_VALUE_FREE } from "../customer-grid/windows"; +import type { EndingKind, EndingStep } from "../customer-grid/windows"; // ⭐⭐ W53-T15 (ruling R8) — THE FILTER BUILDER IS NO LONGER MOUNTED IN THIS COLUMN. The SHARED // `FilterBuilderPanel` the Data module also uses is untouched; `SheetFilterPopup` wraps it in the // base kit's centred dialog so its condition rows have room, and only the Sheet mounts that @@ -55,19 +58,21 @@ import { countConditions, isNumericType } from "../filter-kit/ops"; import { SheetFilterPopup } from "../filter-kit/SheetFilterPopup"; import { WindowPicker } from "../filter-kit/WindowPicker"; import { useEscapeClose } from "../ui/useEscapeClose"; -import { previewCell } from "./sheetApi"; -import { bindingAggregate, bindingDisplay, bindingFilterClauses } from "./types"; -import type { BindingAggregate } from "./types"; -import { bindingStepMode, bindingStepValues, seedBindingStep } from "./bindingRefs"; +import { previewCell, readWorkbook } from "./sheetApi"; +import { formatRef, parseRef } from "./a1"; +import { bindingAggregate, bindingDisplay, bindingFilterClauses, bindingOnDrag, BINDING_ENDING_STEP, BINDING_PERIOD_STEP } from "./types"; +import type { BindingAggregate, BindingPeriod, BindingDragStep, BindingOnDrag } from "./types"; +import { bindingStepMode, bindingStepValues, resolveEndingCell, seedBindingStep, + bindingRefSlots, filterValueLeaves, resolveFilterCell, updateFilterValueLeaf } from "./bindingRefs"; import "../styles/wave57/binding-actions.css"; import "../styles/wave58/sheet.css"; import "../styles/wave59/binding.css"; const BINDING_SECTIONS = [ - { key: "data", label: "Data" }, - { key: "rows", label: "Filter" }, - { key: "period", label: "Period" }, - { key: "copied", label: "When copied" }, + { key: "data", label: "Data", icon: }, + { key: "rows", label: "Filter", icon: }, + { key: "period", label: "Period", icon: }, + { key: "copied", label: "When copied", icon: }, ] as const; const AGGREGATES = [ @@ -133,9 +138,8 @@ const scopeOf = (database: string): SurfaceScope | null => { // ══ THE PERIOD (R9) ═══════════════════════════════════════════════════════════════════════════ // // ⛔⛔ CONTRACT C2 SPELLS THIS ON THE WIRE AND THE UI DOES NOT GET A VOTE: -// `period: {unit: "month"|"quarter"|"year", start: "YYYY-MM", locked: boolean, ref?: A1}`. The -// unit values are LOWERCASE on the wire even though the picker reads Month / Quarter / Year, and -// `start` is a MONTH string for every unit. +// Unit values are lowercase on the wire. Day/Week starts use `YYYY-MM-DD`, with Week starting +// on Monday; Month/Quarter/Year starts retain `YYYY-MM`. // A frozen period keeps that pair for returning to the calendar, and adds only // `frozen: true, from: "YYYY-MM-DD", to: "YYYY-MM-DD"` in place of a cell reference. // @@ -145,7 +149,7 @@ const scopeOf = (database: string): SurfaceScope | null => { // server compiler reads the pair, and two readings of `2026-08` under `quarter` (August, or Q3) // are a silent disagreement that no screen shows. -export type PeriodUnit = "month" | "quarter" | "year"; +export type PeriodUnit = "day" | "week" | "month" | "quarter" | "year"; export interface PeriodBinding { unit: PeriodUnit; @@ -156,9 +160,14 @@ export interface PeriodBinding { from?: string; to?: string; window?: WindowSpec; + ending?: BindingPeriod["ending"]; + endingStep?: EndingStep; + endingOffset?: number; } export const PERIOD_UNITS: { unit: PeriodUnit; label: string }[] = [ + { unit: "day", label: "Day" }, + { unit: "week", label: "Week" }, { unit: "month", label: "Month" }, { unit: "quarter", label: "Quarter" }, { unit: "year", label: "Year" }, @@ -170,11 +179,11 @@ const MONTH_NAMES = [ ]; /** How many months one step of a unit covers. The only place that arithmetic is stated. */ -const MONTHS_PER_UNIT: Record = { month: 1, quarter: 3, year: 12 }; +const MONTHS_PER_UNIT: Partial> = { month: 1, quarter: 3, year: 12 }; /** The noun a sentence uses for one step. Sentence case, and no dash anywhere near it. */ const UNIT_NOUN: Record = { - month: "month", quarter: "quarter", year: "year", + day: "day", week: "week", month: "month", quarter: "quarter", year: "year", }; const pad2 = (n: number) => String(n).padStart(2, "0"); @@ -194,7 +203,14 @@ const joinYm = (y: number, m: number) => `${y}-${pad2(m)}`; /** THE PERIOD'S FIRST MONTH, which is what `start` means. See the block above. */ export function normalizePeriodStart(unit: PeriodUnit, ym: string): string { - const { y, m } = splitYm(ym); + if (unit === "day" || unit === "week") { + const text = /^\d{4}-\d{2}$/.test(ym) ? `${ym}-01` : ym; + const date = new Date(`${text}T00:00:00Z`); + if (!Number.isFinite(date.getTime()) || date.toISOString().slice(0, 10) !== text) return text; + if (unit === "week") date.setUTCDate(date.getUTCDate() - (date.getUTCDay() + 6) % 7); + return date.toISOString().slice(0, 10); + } + const { y, m } = splitYm(ym.slice(0, 7)); if (unit === "year") return joinYm(y, 1); if (unit === "quarter") return joinYm(y, 1 + 3 * Math.floor((m - 1) / 3)); return joinYm(y, m); @@ -202,8 +218,14 @@ export function normalizePeriodStart(unit: PeriodUnit, ym: string): string { /** Move a start by whole periods. Negative steps go back; this is also the drag rule's maths. */ export function shiftPeriodStart(unit: PeriodUnit, ym: string, steps: number): string { + if (unit === "day" || unit === "week") { + const date = new Date(`${normalizePeriodStart(unit, ym)}T00:00:00Z`); + if (!Number.isFinite(date.getTime())) return ym; + date.setUTCDate(date.getUTCDate() + steps * (unit === "week" ? 7 : 1)); + return date.toISOString().slice(0, 10); + } const { y, m } = splitYm(normalizePeriodStart(unit, ym)); - const total = y * 12 + (m - 1) + steps * MONTHS_PER_UNIT[unit]; + const total = y * 12 + (m - 1) + steps * MONTHS_PER_UNIT[unit]!; return joinYm(Math.floor(total / 12), (total % 12) + 1); } @@ -211,6 +233,12 @@ export function shiftPeriodStart(unit: PeriodUnit, ym: string, steps: number): s * `platform/modules/sheet.py::period_window`, whose `to` is the LAST DAY of the period. */ export function periodRange(unit: PeriodUnit, ym: string): { from: string; to: string } { const start = normalizePeriodStart(unit, ym); + if (unit === "day" || unit === "week") { + const last = new Date(`${start}T00:00:00Z`); + if (!Number.isFinite(last.getTime())) return { from: "", to: "" }; + last.setUTCDate(last.getUTCDate() + (unit === "week" ? 6 : 0)); + return { from: start, to: last.toISOString().slice(0, 10) }; + } const last = new Date(`${shiftPeriodStart(unit, start, 1)}-01T00:00:00Z`); last.setUTCDate(last.getUTCDate() - 1); return { from: `${start}-01`, to: last.toISOString().slice(0, 10) }; @@ -259,6 +287,12 @@ export function migratePeriod(raw: unknown): PeriodBinding | null { start: normalizePeriodStart("month", todayYm()), locked: true, window, + ...(p.ending && typeof p.ending === "object" ? { + ending: p.ending as BindingPeriod["ending"], + endingStep: (p.endingStep ?? endingStepDefault(window.kind)) as EndingStep, + endingOffset: Number(p.endingOffset ?? 0), locked: p.locked === true, + ...(ref ? { ref } : {}), + } : {}), }; } const declared = PERIOD_UNITS.some((u) => u.unit === p.unit); @@ -270,7 +304,8 @@ export function migratePeriod(raw: unknown): PeriodBinding | null { const unit = p.unit as PeriodUnit; const stored = typeof p.start === "string" ? p.start.trim() : ""; // D-660: no window named, and no cell to take one from, so there is no period to read. - if (!/^\d{4}-\d{2}$/.test(stored) && !ref) return null; + const startPattern = unit === "day" || unit === "week" ? /^\d{4}-\d{2}-\d{2}$/ : /^\d{4}-\d{2}$/; + if (!startPattern.test(stored) && !ref && p.frozen !== true) return null; const start = normalizePeriodStart(unit, stored || todayYm()); const locked = p.locked === true; if (p.frozen === true) { @@ -293,6 +328,7 @@ export type DragAcross = "period" | "none"; export interface OnDragBinding { across: DragAcross; down: { field: string } | "none"; + steps?: readonly BindingDragStep[]; } /** A copy step walks a column's stored values. Measure-backed and Metric columns are computed @@ -304,14 +340,8 @@ export function isDragDownField(f: Field): boolean { /** The saved `onDrag`, defaulted. `down` is carried as a field key, empty meaning nothing. */ export function readOnDrag(raw: unknown): { across: DragAcross; down: string } { - const d = (raw && typeof raw === "object" ? raw : {}) as Record; - const across: DragAcross = d.across === "none" ? "none" : "period"; - const down = d.down; - if (down && typeof down === "object") { - const key = (down as Record).field; - if (typeof key === "string" && key) return { across, down: key }; - } - return { across, down: "" }; + const rules = bindingOnDrag({ onDrag: raw }); + return { across: rules.across, down: rules.down === "none" ? "" : rules.down.field }; } /** `scope.adoptedFrom` (R11): the view was purged and its filter was copied into this cell. */ @@ -667,6 +697,10 @@ export default function BindingEditor({ // built out of groups is not undercounted into reading as empty. const conditionCount = useMemo(() => countConditions(tree.nodes ?? []), [tree]); const [fields, setFields] = useState([]); + const [linkedOffer, setLinkedOffer] = useState<{ database: string; fields: Field[] }>({ database: "", fields: [] }); + const [linkedNotice, setLinkedNotice] = useState(""); + const filterFields = useMemo(() => [...fields, + ...(linkedOffer.database === database ? linkedOffer.fields : [])], [fields, linkedOffer, database]); const catalogueDatabase = useRef(""); const [measures, setMeasures] = useState([]); const [views, setViews] = useState([]); @@ -738,7 +772,7 @@ export default function BindingEditor({ // render would re-read `initial` after the person had already changed a control, which is how an // editor forgets an edit the moment its parent re-renders for an unrelated reason. const migrated = useState(() => migratePeriod(initial?.period))[0]; - const savedDrag = useState(() => readOnDrag(initial?.onDrag))[0]; + const savedDrag = useState(() => bindingOnDrag(initial))[0]; const [unit, setUnit] = useState(migrated?.unit ?? "month"); const [start, setStart] = useState( @@ -787,6 +821,17 @@ export default function BindingEditor({ migrated?.window ? "window" : "calendar", ); const [windowSpec, setWindowSpec] = useState(migrated?.window ?? { kind: "ytd" }); + const [ending, setEnding] = useState(migrated?.ending?.kind ?? "today"); + const [endingFixed, setEndingFixed] = useState(migrated?.ending?.date ?? ""); + const [endingStep, setEndingStep] = useState( + migrated?.endingStep ?? endingStepDefault(windowSpec.kind)); + const [endingOffset, setEndingOffset] = useState(migrated?.endingOffset ?? 0); + const [endingSheets, setEndingSheets] = useState<{ id: string; name: string }[]>([]); + const [endingSheet, setEndingSheet] = useState(() => parseRef(migrated?.ref ?? "")?.sheet ?? ""); + const parsedEndingRef = parseRef(periodRef); + const endingRef = parsedEndingRef && !parsedEndingRef.sheet && endingSheet + ? formatRef({ ...parsedEndingRef, sheet: endingSheet }) : periodRef.trim(); + const hasEnding = (TO_DATE_KINDS as readonly string[]).includes(windowSpec.kind); // "Use a cell" is a LINK that REVEALS the A1 input (R9). The input was the whole period control // before this ticket and it is not deleted, only demoted to the thing behind that link. const [refOpen, setRefOpen] = useState(Boolean(migrated?.ref)); @@ -805,8 +850,33 @@ export default function BindingEditor({ // at all, which is standing rule 1's second sentence - a limit nobody was told about. The switch // below states which of the two a cell is, and either answer is one click from the other. const [periodOn, setPeriodOn] = useState(Boolean(migrated)); - const [across, setAcross] = useState(savedDrag.across); - const [downField, setDownField] = useState(savedDrag.down); + useEffect(() => { + if (!periodOn || periodMode !== "window" || ending !== "cell") return; + let cancelled = false; + void readWorkbook(workbookId).then((got) => { + if (cancelled) return; + if (!got.ok) { setNotice(got.message); return; } + setEndingSheets(got.value.sheets.map(({ id, name }) => ({ id, name }))); + setEndingSheet((held) => held || got.value.sheets[0]?.name || ""); + }); + return () => { cancelled = true; }; + }, [workbookId, periodOn, periodMode, ending]); + const [copySteps, setCopySteps] = useState(() => + (savedDrag.steps ?? []).filter((step) => + (step.field !== BINDING_PERIOD_STEP && step.field !== BINDING_ENDING_STEP) || !migrated?.locked)); + const [copyStepsEdited, setCopyStepsEdited] = useState(false); + const updateCopySteps = (steps: readonly BindingDragStep[]) => { + setCopySteps(steps); setCopyStepsEdited(true); + }; + const across: DragAcross = copySteps.some((step) => + step.field === BINDING_PERIOD_STEP || step.field === BINDING_ENDING_STEP) ? "period" : "none"; + const setAcross = (value: DragAcross) => { + const field = hasEnding ? BINDING_ENDING_STEP : BINDING_PERIOD_STEP; + const kept = copySteps.filter((step) => step.field !== field); + updateCopySteps(value === "none" ? kept : [...kept, { field, axis: "across", + ...(hasEnding ? { size: endingStep } : {}), + }]); + }; const adoptedFrom = adoptedFromOf(initial); @@ -943,6 +1013,30 @@ export default function BindingEditor({ return () => { live = false; }; }, [database, rowsWanted]); + // The related grid's field wall is checked on the server. Keep its labelled fields in the + // existing Filter picker only; a customer's amount is not an order aggregate column. + useEffect(() => { + if (!rowsWanted || !database) return undefined; + const controller = new AbortController(); + setLinkedOffer({ database, fields: [] }); + setLinkedNotice(""); + (async () => { + try { + const response = await fetch(`${API_V1}/sheets/filter-fields/${encodeURIComponent(database)}`, { + credentials: CREDENTIALS, signal: controller.signal, + }); + const body = await response.json().catch(() => null); + if (!response.ok || !Array.isArray(body?.fields)) + throw new Error(body?.error?.message || "Linked fields could not be loaded. Close and reopen this panel to retry."); + if (!controller.signal.aborted) setLinkedOffer({ database, fields: body.fields as Field[] }); + } catch (error) { + if (!controller.signal.aborted) setLinkedNotice(error instanceof Error ? error.message + : "Linked fields could not be loaded. Close and reopen this panel to retry."); + } + })(); + return () => controller.abort(); + }, [database, rowsWanted]); + /** The binding names a view the fresh list does not have. Said in the panel, not after a save. * * ⛔ IT STOPS BEING SAID THE MOMENT THE PERSON ANSWERS IT. The sentence asks for one of two acts, @@ -962,7 +1056,7 @@ export default function BindingEditor({ * already gated, so this is not a second definition of the rule, only a second caller of it. */ const statusValues = useMemo(() => { const map: Record = {}; - for (const f of fields) { + for (const f of filterFields) { if (f.type === "user") { map[f.key] = userOptions; continue; @@ -971,7 +1065,7 @@ export default function BindingEditor({ map[f.key] = choiceVocabulary(f, rawRows); } return map; - }, [fields, rawRows, userOptions]); + }, [filterFields, rawRows, userOptions]); // Offer every numeric column this database can total, including Metrics created in the app. // `isNumericType` checks numeric behaviour and `canTotal` checks whether the database can @@ -1012,24 +1106,27 @@ export default function BindingEditor({ field: candidate, values: bindingStepValues(candidate, rawRows, userOptions), })); - const savedDragDriverMissing = Boolean(downField) && rowsResolved - && !stepFields.some((candidate) => candidate.key === downField); - const chooseStepField = (key: string) => { - if (!key) { setDownField(""); return; } + const savedDragDriverMissing = rowsResolved && copySteps.some((step) => step.field + && step.field !== BINDING_ENDING_STEP && step.field !== BINDING_PERIOD_STEP + && !stepFields.some((candidate) => candidate.key === step.field)); + const chooseStepField = (index: number, key: string) => { + const replace = (step: BindingDragStep) => updateCopySteps(copySteps.map((held, i) => i === index ? step : held)); + if (key === BINDING_ENDING_STEP || key === BINDING_PERIOD_STEP) { + replace({ field: key, axis: copySteps[index].axis, + ...(key === BINDING_ENDING_STEP ? { size: endingStepDefault(windowSpec.kind) } : {}), + }); + setLocked(false); + return; + } + if (!key) { replace({ field: "", axis: copySteps[index].axis }); return; } const offer = filterClauseOffers.find((candidate) => candidate.field.key === key); if (!offer?.values.length || !rowsResolved) return; - setTree(seedBindingStep(driverTree, offer.field, offer.values[0])); + setTree(bindingFilterClauses(driverTree).some((clause) => clause.colId === key) + ? driverTree : seedBindingStep(driverTree, offer.field, offer.values[0])); setScopeKind("filter"); setViewId(""); - setDownField(key); + replace({ field: key, axis: copySteps[index].axis, mode: bindingStepMode(offer.field) }); }; - const localDriver = bindingFilterClauses(tree).find((clause) => clause.colId === downField); - const driverField = fields.find((candidate) => candidate.key === downField); - const driverValue = localDriver?.op === "isEmpty" - ? (driverField?.type === "checkbox" ? "Unchecked" : "Empty") - : localDriver?.op === "isNotEmpty" - ? (driverField?.type === "checkbox" ? "Checked" : "Filled") - : localDriver?.value || localDriver?.values?.[0] || ""; /** * ⭐⭐ W50-T21 (owner instruction 24: *"Sheet column picker (sum/count) shows the Field icons."*) @@ -1069,7 +1166,7 @@ export default function BindingEditor({ setMetricKey(""); setField(""); setFn("sum"); - setPeriodOn(false); + setPeriodOn(kind === "number"); setPreview(""); setColPickerOpen(false); }; @@ -1079,8 +1176,8 @@ export default function BindingEditor({ setField(""); setViewId(""); setTree({ conj: "and", nodes: [] }); - setDownField(""); - setPeriodOn(false); + updateCopySteps([]); + setPeriodOn(chosenIsMetric); setPreview(""); }; @@ -1117,7 +1214,11 @@ export default function BindingEditor({ // ⛔ C5 - EMIT EXACTLY ONE DIALECT. `clean_period` prefers unit/start when both are present, // so combining these branches would silently discard the rolling window. if (periodMode === "window") { - out.period = { window: windowSpec }; + out.period = { window: windowSpec, ...(hasEnding ? { + ending: { kind: ending, ...(ending === "fixed" ? { date: endingFixed } : {}) }, + endingStep, ...(endingOffset ? { endingOffset } : {}), + locked: across === "none", ...(ending === "cell" ? { ref: endingRef } : {}), + } : {}) }; } else { const period: Record = { unit, start, locked: chosenIsMetric ? locked : true, @@ -1133,14 +1234,55 @@ export default function BindingEditor({ // ⛔⛔ R8 REVOKED W51-T12'S ONE-AXIS-AT-A-TIME RULE. The wire keys keep C2's spelling // because `clean_on_drag` rebuilds anything else; emitting both slots independently lets one // Metric cell step its period and one filter clause without either choice erasing the other. - out.onDrag = { - across: chosenIsMetric && periodOn && periodMode === "calendar" && !locked ? across : "none", - down: downField ? { field: downField, ...(driverField ? { mode: bindingStepMode(driverField) } : {}) } : "none", - }; + const steps = copySteps.filter((step) => step.field && (step.field === BINDING_ENDING_STEP + ? periodOn && hasEnding && periodMode === "window" : step.field === BINDING_PERIOD_STEP + ? periodOn && chosenIsMetric && !locked && periodMode === "calendar" : true)); + // An unchanged legacy editor keeps the old copy semantics on save as well as on read. + out.onDrag = !copyStepsEdited && !Array.isArray((initial?.onDrag as BindingOnDrag | undefined)?.steps) + ? { across: savedDrag.across, down: savedDrag.down } + : bindingOnDrag({ onDrag: { steps } }); return out; }, [database, sourceKind, metricKey, fn, field, scopeKind, viewId, views, tree, initial, unit, start, locked, frozen, frozenFrom, frozenTo, refOpen, periodRef, periodOn, periodMode, windowSpec, - across, downField, driverField, chosenIsMetric]); + across, copySteps, copyStepsEdited, savedDrag, chosenIsMetric, hasEnding, ending, endingFixed, endingStep, endingOffset, endingRef]); + + // Read before either send; the server reads the stored source again independently. + const prepareBinding = async (): Promise | null> => { + let value = endingFixed; + let prepared = binding; + try { + const filterRefs = bindingRefSlots(binding).filter((slot) => slot.kind === "filter"); + if (filterRefs.length) { + const got = await readWorkbook(workbookId); + if (!got.ok) throw new Error(got.message); + const scope = binding.scope as { filter: FilterTree }; + let previewTree = scope.filter; + for (const slot of filterRefs) { + const operand = resolveFilterCell(got.value, slot.text); + previewTree = updateFilterValueLeaf(previewTree, slot.indices!, (node) => ({ ...node, + value: operand, ...((MULTI_VALUE_OPS as ReadonlySet).has(node.op) ? { values: [operand] } : {}), + })); + } + prepared = { ...binding, scope: { ...scope, filter: previewTree } }; + } + if (!periodOn || periodMode !== "window" || !hasEnding) return prepared; + if (ending === "cell") { + const got = await readWorkbook(workbookId); + if (!got.ok) throw new Error(got.message); + value = resolveEndingCell(got.value, endingRef); + } + const today = new Date(); + const day = `${today.getFullYear()}-${pad2(today.getMonth() + 1)}-${pad2(today.getDate())}`; + if (!endingBounds(windowSpec.kind, ending, value, day, endingStep, endingOffset)) + throw new Error("Ending cannot be used because its date is invalid or falls before the lookback starts. Choose a later ending date."); + return ending === "cell" ? { ...prepared, period: { + ...(binding.period as Record), ending: { kind: "cell", date: value }, + } } : prepared; + } catch (error) { + setNotice(error instanceof Error ? error.message : "The Ending date could not be read. Choose a valid date."); + return null; + } + }; // ⭐ THE NUMBER APPEARS BEFORE SAVING. `preview` evaluates the binding without persisting it, // which is what lets someone see whether they picked the right thing while they can still change @@ -1148,7 +1290,9 @@ export default function BindingEditor({ const runPreview = async () => { setNotice(""); setPreview(""); - const got = await previewCell(workbookId, addr, binding); + const prepared = await prepareBinding(); + if (!prepared) return; + const got = await previewCell(workbookId, addr, prepared); if (!got.ok) { setNotice(got.message); return; @@ -1253,7 +1397,7 @@ export default function BindingEditor({ <>
@@ -1454,6 +1601,49 @@ export default function BindingEditor({ : `Edit filter (${conditionCount === 1 ? "1 condition" : `${conditionCount} conditions`})`} + {filterValueLeaves(tree).map(({ node, indices }, index) => { + const title = filterFields.find((candidate) => candidate.key === node.colId)?.label || node.colId; + const supported = !["isEmpty", "isNotEmpty", "aboveAvg", "belowAvg", "within"].includes(node.op) + && !node.colId.startsWith("__") && !node.rhs && !ANCHOR_VALUE_FREE.has(node.dateMode ?? ""); + if (!supported && !node.valueRef) return null; + return
+ + {node.valueRef && <> + + +

Include the sheet name when this workbook has several sheets.

+ } +
; + })} {/* ⛔⛔ MOUNTED ONLY WHILE OPEN, and that is what orders Escape correctly rather than a flag inside the dialog. `useOverlayLayer` claims Escape as soon as it runs and `ui/useEscapeClose` gives the key to the LAST entry pushed, so a @@ -1461,10 +1651,11 @@ export default function BindingEditor({ editor opened. Mounting it here, later in time, puts it ABOVE this panel's own claim (pushed at mount, a few hundred lines up) and unmounting it hands Escape straight back, so one press closes the filter and the next closes the panel. */} + {linkedNotice &&

{linkedNotice}

} {filterPopupOpen && ( setPeriodOn((v) => !v)} + disabled={chosenIsMetric} + title={chosenIsMetric ? "A Metric cell always covers a period." : undefined} + onClick={() => { if (!chosenIsMetric) setPeriodOn((v) => !v); }} > Limit this cell to a period @@ -1525,10 +1718,48 @@ export default function BindingEditor({ {periodMode === "window" ? ( + <> + {hasEnding && <> + + {ending === "fixed" && } + {ending === "cell" && <> + {!parsedEndingRef?.sheet && } + + } + {endingOffset > 0 &&

+ {`Ending moved later by ${endingOffset} ${endingStep}${endingOffset === 1 ? "" : "s"} when copied.`} +

} + } + ) : ( <> -
+ {unit === "day" || unit === "week" ? ( + + ) :
-

- Right and down move forward. Left and up move backward. -

- - )} - - {/* ⛔ C5 - A ROLLING WINDOW HAS NO START TO SHIFT. Offering the calendar step here - would promise an action `dragBoundCell` cannot perform. */} - {periodOn && periodMode === "window" && ( -

- A rolling window is measured from today, so every copy of this cell answers the - same window. Use a calendar period when the period should step as the cell is copied. -

- )} - - {/* ⚠ AN ALL-TIME METRIC SAYS WHY NO PERIOD PICKER IS PRESENT. Without this note the - absent control looks like a loading or configuration failure. */} - {chosenIsMetric && !periodOn && ( -

- Choose a Period or Lookback before saving this Metric. -

- )} - -
- Step field - ({ - key: candidate.key, - label: `Step ${candidate.label}`, - type: candidate.type, - disabled: !values.length, - hint: values.length ? undefined : "No values", - })), - ...(savedDragDriverMissing ? [{ - key: downField, - label: `${downField} (unavailable)`, - disabled: true, - }] : []), - ]} - /> -
+ {isEnding && } + {localDriver && driverField && {`${driverField.label}: ${driverValue}`}} + +
; + })} + {!rowsResolved &&

{rowsNotice || "Loading all field values…"}

} {savedDragDriverMissing &&

The saved Step field is unavailable. Choose another field.

} - {localDriver && driverField && ( -
- {`${driverField.label}: ${driverValue}`} -
- )} -

- Right and down choose later values. Left and up choose earlier values. - Copies stop at the first or last value. -

)} @@ -1816,7 +2050,9 @@ export default function BindingEditor({ - diff --git a/web/src/sheet/ChartDialog.tsx b/web/src/sheet/ChartDialog.tsx index 403a2a10c2f196bc0f84ebfb5264efcaf147eed9..e11553a31275195f0955e579f800e53d1b3d0d96 100644 --- a/web/src/sheet/ChartDialog.tsx +++ b/web/src/sheet/ChartDialog.tsx @@ -43,19 +43,26 @@ // rather than a spinner while the database list is in flight, press feedback that does not change // layout, reduced motion honoured, and no em dash or en dash in any string that reaches a screen. -import { createContext, useContext, useEffect, useId, useMemo, useRef, useState } from "react"; +import { createContext, Fragment, useContext, useEffect, useId, useMemo, useRef, useState } from "react"; +import type { ReactNode } from "react"; import { BodyPortal, useOverlayLayer } from "../customer-grid/OverlaySurface"; // The document's own addresser, in SHEET coordinates. A second spelling of A1 in this file would // be a second answer to a question the document already answers. import { addressOf } from "./sheetState"; import { parseRange } from "./rules"; -import type { Cell, SheetChartRange, SheetChartSpec, SheetTab, SheetChartPresentation, SheetChartColor } from "./types"; +import type { Cell, SheetChartRange, SheetChartSpec, SheetTab, SheetChartPresentation, SheetChartColor, ChartAxisFormat } from "./types"; import { defaultChartPresentation } from "./types"; -import { chartDataForRange, DrawnChart } from "./SheetChart"; +import { chartDataForRange, chartLegendEntries, DrawnChart } from "./SheetChart"; import type { SheetChartData } from "./SheetChart"; import "../styles/wave58/sheet.css"; import "../styles/wave59/chart-dialog.css"; +/** R-instruction 11: a new chart is born named, so every label is something the user can click and + * rename. A chart saved before this release keeps whatever titles it had, which is why this is + * applied at CREATION and never in `defaultChartPresentation` or at render time. */ +const UNTITLED = "Untitled"; +const BOUND_KEYS = ["yMin", "yMax", "yMajor", "yMinor", "xMin", "xMax"] as const; + /** The page owns the atomic workbook write; isolated legacy mounts retain their old callback. */ export const ChartWorkbookContext = createContext<{ sheets: SheetTab[]; activeSheetId: string; @@ -368,6 +375,27 @@ export function planDatabaseBlock(opts: { }; } +// Database plans keep real period labels but stale values, which the chart treats as empty. +// Preview-only samples preserve the labels and chart shape without pretending to be real figures. +const PREVIEW_SAMPLE = [42, 68, 51, 84, 63, 97, 75, 89, 58, 72, 46, 91]; + +export function previewDatabaseCells(plan: DatabaseBlockPlan): Record { + const cells = { ...plan.cells }; + let i = 0; + for (let col = plan.range.col0; col <= plan.range.col1; col += 1) { + for (let row = plan.range.row0; row <= plan.range.row1; row += 1) { + const address = addressOf(col, row); + if (cells[address]?.state === "stale") { + cells[address] = { + kind: "literal", state: "typed", text: String(PREVIEW_SAMPLE[i % PREVIEW_SAMPLE.length]), + }; + i += 1; + } + } + } + return cells; +} + type SourceTab = "range" | "database"; @@ -438,9 +466,11 @@ export default function ChartDialog({ const [chartHeight, setChartHeight] = useState(String(chart?.geometry?.height ?? 360)); const [section, setSection] = useState(opening.section); const [galleryTab, setGalleryTab] = useState<"recommended" | "all">(editing || initialKind ? "all" : "recommended"); - const [bounds, setBounds] = useState>(() => Object.fromEntries(["yMin","yMax","yMajor","yMinor"].map(k=>[k,String(chart?.presentation?.[k as "yMin"]??"")]))); + const [bounds, setBounds] = useState>(() => Object.fromEntries(BOUND_KEYS.map(k=>[k,String(chart?.presentation?.[k]??"")]))); const setOption = (key: K, value: SheetChartPresentation[K]) => setPresentation(current => ({...current,[key]:value})); + const setAxis = (axis: "valueAxis" | "categoryAxis", patch: Partial) => + setOption(axis, { ...(presentation[axis] ?? {}), ...patch }); const [tab, setTab] = useState("range"); const [kind, setKind] = useState(opening.kind); @@ -506,6 +536,22 @@ export default function ChartDialog({ const model = candidates.find(candidate => candidate.kind === kind)!.model; const recommended = useMemo(() => range ? recommendChartKinds(candidates) : [], [candidates,range]); const previewError = range ? chartPreviewError(kind, model) : "Enter a valid data range."; + const databasePlan = useMemo(() => tab === "database" ? planDatabaseBlock({ + anchor: landingOf(selection), database, fn, field: field.trim(), unit, periods: Number(periods), + }) : null, [tab, selection, database, fn, field, unit, periods]); + const databasePresentation = useMemo(() => ({ + ...presentation, orientation: "rows", headers: "none", categoryLabels: "firstColumn", + }), [presentation]); + const databaseModel = useMemo(() => databasePlan ? chartDataForRange( + previewDatabaseCells(databasePlan), databasePlan.range, databasePresentation, [databasePlan.range], kind, + ) : null, [databasePlan, databasePresentation, kind]); + const previewModel = tab === "database" && databaseModel ? databaseModel : model; + const previewPresentation = tab === "database" ? databasePresentation : presentation; + const previewMessage = tab === "range" ? previewError + : !database || !databaseModel ? "Choose a database to see a preview." : chartPreviewError(kind, databaseModel); + const previewNote = previewMessage || (tab === "database" + ? "Sample values. The chart reads the real figures once the periods land in the sheet." + : CHART_DESCRIPTIONS[kind]); const galleryKinds = galleryTab === "recommended" ? recommended : CHART_KINDS.map(entry => entry.kind); const thumbnailPresentation: SheetChartPresentation = { ...presentation, showLegend: false, showDataLabels: false, showAxisTitles: false, layout: "compact", @@ -522,13 +568,15 @@ export default function ChartDialog({ railRef.current?.querySelector('[aria-selected="true"]') ?.scrollIntoView({ block: "nearest" }); }, [kind, galleryTab, section, tab]); - const offeredSeries = useMemo(() => chartDataForRange(sourceCells, range ?? seedRange, {...presentation,series:[]}, ranges ?? [seedRange], kind).series, [sourceCells,range,presentation,ranges,kind]); + const offeredModel = useMemo(() => chartDataForRange(sourceCells, range ?? seedRange, {...presentation,series:[]}, ranges ?? [seedRange], kind), [sourceCells,range,presentation,ranges,kind]); + const offeredSeries = offeredModel.series; const configuredSeries = (presentation.series ?? []).flatMap(setting => { const offered = offeredSeries.find(s => s.sourceKey === setting.key || s.y === setting.key); return offered ? [{...setting,key:offered.sourceKey || offered.y}] : []; }); const settings: NonNullable = [...configuredSeries, ...offeredSeries.filter(s => !configuredSeries.some(c => c.key === (s.sourceKey||s.y))).map(s => ({key:s.sourceKey||s.y}))]; + const legendEntries = chartLegendEntries(kind, settings.filter(s => !s.hidden).length, offeredModel.rows.length); const seriesName = (s: NonNullable[number]) => s.label || offeredSeries.find(offer => (offer.sourceKey||offer.y) === s.key)?.label || s.key; const updateSeries = (key: string, patch: Partial[number]>) => @@ -538,9 +586,10 @@ export default function ChartDialog({ [reordered[index],reordered[other]]=[reordered[other],reordered[index]]; setOption("series",reordered); }; - const boundsError = Object.entries(bounds).some(([key,value])=>value!==""&&(!Number.isFinite(Number(value))||((key==="yMajor"||key==="yMinor")&&Number(value)<=0))) + const boundsError = BOUND_KEYS.some(key=>bounds[key]!==""&&(!Number.isFinite(Number(bounds[key]))||((key==="yMajor"||key==="yMinor")&&Number(bounds[key])<=0))) ? "Enter finite bounds and positive units." - : bounds.yMin!==""&&bounds.yMax!==""&&Number(bounds.yMin)>=Number(bounds.yMax) ? "The Y maximum must exceed the minimum." : ""; + : bounds.yMin!==""&&bounds.yMax!==""&&Number(bounds.yMin)>=Number(bounds.yMax) ? "The Y maximum must exceed the minimum." + : bounds.xMin!==""&&bounds.xMax!==""&&Number(bounds.xMin)>=Number(bounds.xMax) ? "The X maximum must exceed the minimum." : ""; const validGeometry = location && location.col0===location.col1 && location.row0===location.row1 && Number(chartWidth)>=320 && Number(chartWidth)<=2400 && Number(chartHeight)>=240 && Number(chartHeight)<=1600; const newNameError = destination==="__new__" && (!newName.trim() || [...newName].some(c => "[]:*?/\\\\'".includes(c)) || workbook?.sheets.some(s=>s.name.toLowerCase()===newName.trim().toLowerCase())); @@ -558,9 +607,11 @@ export default function ChartDialog({ }) : null; const dataRanges = plan ? [plan.range] : ranges!; const display = {...presentation, series: configuredSeries}; - for (const key of ["yMin","yMax","yMajor","yMinor"] as const) { + for (const key of BOUND_KEYS) { if (bounds[key] === "") delete display[key]; else display[key] = Number(bounds[key]); } + if (!editing) for (const key of ["title", "xTitle", "yTitle"] as const) + if (!display[key]) display[key] = UNTITLED; if (plan) { display.orientation = "rows"; display.headers = "none"; display.categoryLabels = "firstColumn"; } const next: SheetChartSpec = { ...chart, id: chart?.id ?? `chart-${crypto.randomUUID()}`, kind, range:dataRanges[0], @@ -573,8 +624,15 @@ export default function ChartDialog({ else onCreate?.(kind,next.range,plan?.cells); }; - const rangeField = ( -
+ const group = (name: string, body: ReactNode, busy?: boolean) => ( +
+

{name}

+
{body}
+
+ ); + + const rangeFieldRows = ( + <> ) : null} -
+ ); const kindPicker = ; @@ -611,7 +669,9 @@ export default function ChartDialog({
-
+ {/* [W60-T16] The sentence used to live under the gallery's own preview, which moved out to + the dialog's right column. An empty tablist is not an answer, so it is said here. */} + {galleryKinds.length ?
{galleryKinds.map(candidateKind => { const candidateModel = candidates.find(candidate => candidate.kind === candidateKind)!.model; const error = range ? chartPreviewError(candidateKind, candidateModel) : "Enter a valid data range."; @@ -626,17 +686,7 @@ export default function ChartDialog({ } ; })} -
-
- {galleryKinds.length ? <> -

{CHART_KINDS.find(entry => entry.kind === kind)!.label}

-

{previewError || CHART_DESCRIPTIONS[kind]}

- {!previewError ?
- -
: null} - :

Choose a range with numeric values to see recommendations.

} -
+
:

Choose a range with numeric values to see recommendations.

}
; const select = (label: string,key:K,choices: readonly (readonly [string,string])[]) => @@ -645,11 +695,49 @@ export default function ChartDialog({ ; const toggle = (label:string,key:"showLegend"|"showAxisTitles"|"showDataLabels") => ; + const axisFields = (axis: "valueAxis" | "categoryAxis") => { + const format = presentation[axis] ?? {}; + const fieldId = (name: string) => `${titleId}-${axis}-${name}`; + const value = axis === "valueAxis"; + return <> + {BOUND_KEYS.filter(key => value + ? key.startsWith("y") && (!["gauge", "radar"].includes(kind) || key === "yMin" || key === "yMax") + : key.startsWith("x") && ["scatter", "bubble"].includes(kind)).map(key => + + { + const next = event.target.value; + setBounds(current => ({...current, [key]: next})); + setOption(key, next === "" ? undefined : Number(next)); + }}/> + )} + + setAxis(axis, {decimals: event.target.value === "" ? null : Number(event.target.value)})}/> + + + + + + + {value ? <> + + + : null} + ; + }; const resetFormatting = () => { const defaults=defaultChartPresentation(); setPresentation({...defaults,headers:presentation.headers,categoryLabels:presentation.categoryLabels,orientation:presentation.orientation, series:configuredSeries.map(s=>({key:s.key,label:s.label,hidden:s.hidden})),ganttMode:presentation.ganttMode,title:presentation.title,xTitle:presentation.xTitle,yTitle:presentation.yTitle,categoryTitle:presentation.categoryTitle}); - setBounds({yMin:"",yMax:"",yMajor:"",yMinor:""}); + setBounds(Object.fromEntries(BOUND_KEYS.map(key => [key, ""]))); }; const settingsPanel =
{section==="Chart"?<>{tab==="range" ? chartGallery : kindPicker} @@ -666,14 +754,15 @@ export default function ChartDialog({ :null} {section==="Text"?<>{text("Chart title","title")}{text("X axis title","xTitle")}{text("Y axis title","yTitle")}{text("Category name","categoryTitle")} - {toggle("Show legend","showLegend")}{toggle("Show axis titles","showAxisTitles")}{toggle("Show data labels","showDataLabels")} - {select("Legend position","legendPosition",[["bottom","Bottom"],["top","Top"],["right","Right"]])} - {select("Legend font","legendFont",[["small","Small"],["medium","Medium"],["large","Large"]])} + {legendEntries>=2?toggle("Show legend","showLegend"):null}{toggle("Show axis titles","showAxisTitles")}{toggle("Show data labels","showDataLabels")} + {legendEntries>=2?select("Legend position","legendPosition",[["bottom","Bottom"],["top","Top"],["right","Right"]]):null} + {legendEntries>=2?select("Legend font","legendFont",[["small","Small"],["medium","Medium"],["large","Large"]]):null} {select("Axis font","axisFont",[["small","Small"],["medium","Medium"],["large","Large"]])} -

Charts with two or more series always keep their legend.

+ {legendEntries>=2?

Charts with two or more series always keep their legend.

:null} :null} - {section==="Axes"?<>{["pie","donut","funnel","treemap","polar"].includes(kind)?

This chart type has no value axis.

:<>{(["yMin","yMax","yMajor","yMinor"] as const).filter(key=>!["gauge","radar"].includes(kind)||key==="yMin"||key==="yMax").map((key,i)=>)} - {select("Display units","displayUnits",[["none","None"],["thousands","Thousands"],["millions","Millions"],["percent","Percent"]])} + {section==="Axes"?<>{["pie","donut","funnel","treemap","polar"].includes(kind)?

This chart type has no value axis.

:<> + {group("Value axis", axisFields("valueAxis"))} + {group("Category axis", axisFields("categoryAxis"))} }:null} {section==="Series"?<>{settings.length?settings.map((s,i)=>
@@ -690,18 +779,40 @@ export default function ChartDialog({ + :null}
; + const previewPane = ( +
+

{CHART_KINDS.find(entry => entry.kind === kind)!.label}

+

{previewNote}

+ {previewMessage ? null : ( +
+ +
+ )} +
+ ); + return (
+ {/* [W60-T16, R23] The one split: settings scroll on the left, the preview never hides on + the right. The two strips and the panel below are UNCHANGED and simply sit one level + deeper; the children keep their old indentation so the move stays readable as a diff. */} +
+
{section==="Data" && !editing ? (
{SOURCE_TABS.map((entry) => ( @@ -761,16 +877,47 @@ export default function ChartDialog({ > {section!=="Data" ? settingsPanel : tab === "range" ? ( <> - {workbook?:null} - {rangeField} - {select("Headings","headers",[["auto","Automatic"],["firstRow","First row"],["none","None"]])} - {select("Category labels","categoryLabels",[["auto","Automatic"],["firstColumn","First column"],["none","None"]])} - {select("Series in","orientation",[["columns","Columns"],["rows","Rows"]])} - + {workbook ? group("Source", <> + + + ) : null} + {group("Range", rangeFieldRows)} + {group("Headers and categories", <> + + + + + )} + {group("Series orientation", <> + + +