diff --git a/Dockerfile b/Dockerfile index 0558431db11754088ad3954f9d7d3f0df67f99b9..56fc23ab2036fd1bd3331714aba19be683f2764b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,19 +23,21 @@ WORKDIR /web # The lockfile is the audited supply-chain boundary. It now resolves entirely from the public # registry; discarding it here made every image an unreviewed dependency upgrade. # -# ⛔⛔ `npm ci` WAS TRIED HERE AND CANNOT WORK FROM A WINDOWS-RESOLVED LOCKFILE. Measured -# 2026-09-13: the build died with `Missing: @emnapi/runtime@1.11.3 from lock file`. npm on Windows -# prunes the linux-x64 rolldown binding and its transitive `@emnapi/runtime`, and `npm ci` treats -# any absent entry as a fatal lockfile mismatch rather than a platform gap. Regenerating with -# `--os=linux --cpu=x64` only inverts the failure: the lockfile then builds on Linux and breaks -# `npm ci` on every developer's Windows box. -# -# `npm install` WITH the lockfile present keeps the audited versions as the boundary — it honours -# every pin it can resolve — while repairing a platform-only gap instead of aborting. This is NOT -# the pre-83bfed7 `--no-package-lock`, which discarded the audit entirely. -# The real fix is a lockfile generated on Linux; booked as W-170. +# ⛔⛔ THE LOCKFILE IS GENERATED ON LINUX, NEVER ON WINDOWS (W-170, 2026-09-13). A Windows-resolved +# lockfile fails `npm ci` in this stage (`Missing: @emnapi/runtime@1.11.3 from lock file`): npm +# records the platform closure of the box that ran it, and `npm ci` treats an absent entry as a +# fatal mismatch. A lockfile resolved natively on Linux (no `--os`/`--cpu`) passes `npm ci` on +# BOTH Linux and Windows -- measured the same day, 198 packages in 52s on the laptop. Regenerate +# ONLY with `python .claude/skills/deploy/scripts/lockfile_linux.py` (one small HF job that runs +# `npm ci` there first and writes `web/package-lock.linux.sha256`); `verify_lockfile.py` reddens +# the battery when the lockfile on disk is not the one that receipt attests. +# W59-T35 reached the same Linux resolve independently in the wave-59 container (Node 22.23.2 / +# npm 10.9.8) and added `--engine-strict`, which makes the `engines` pin in package.json a build +# failure on a mismatched toolchain rather than a warning. COPY web/package.json web/package-lock.json ./ -RUN npm install --no-audit --no-fund --ignore-scripts +RUN npm ci --no-audit --no-fund --ignore-scripts --engine-strict +# Keep the resolved build tree: the final image has the bundle but no Node installation. +RUN mkdir -p /deps && npm ls --all --json > /deps/web-deps.json COPY web/ ./ RUN npm run build # -> /web/dist @@ -43,7 +45,12 @@ RUN npm run build # -> /web/dist FROM python:3.12-slim WORKDIR /app COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +# W59-T35, 2026-09-13: pin the locally installed resolver instead of upgrading pip implicitly. +# The Space uploader still sends aios-web/requirements.txt with open bounds and excludes the +# pinned api/requirements.txt. Wiring those pins into the upload is a separate deployment fix; +# changing COPY to api/requirements.txt alone would refer to a file absent from the Space. +RUN python -m pip install --no-cache-dir --disable-pip-version-check --upgrade "pip==25.0.1" \ + && python -m pip install --no-cache-dir --disable-pip-version-check -r requirements.txt COPY api/ ./api/ # the reused data layer (modules/core/harness) # ⛔ COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction: @@ -60,6 +67,7 @@ COPY platform/ ./platform/ # up: a file the product opens BY PATH is invisible to every import-following guard we own. COPY jobs/ ./jobs/ COPY --from=web /web/dist ./web/dist +COPY --from=web /deps/web-deps.json ./deps/web-deps.json # Run unprivileged — the C2 security baseline, and matched to ops/Dockerfile.api so the two images # do not differ on a property that would only surface after a migration. uid 1000 is what HF @@ -69,6 +77,7 @@ USER user ENV RI_DIR=/app/platform \ PORT=7860 \ + FORWARDED_ALLOW_IPS=* \ PYTHONUNBUFFERED=1 \ AIOS_PREWARM=1 # ⛔ AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound @@ -85,4 +94,10 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ # a launcher argument — which is what makes moving hosts a config change instead of a code change. # Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time # and are never baked; the repo-root .dockerignore drops **/.env from the build context. -CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"] +# W59-T35: the Space terminates TLS at its ingress proxy, so honour its forwarded scheme/client. +# '*' assumes the container is unreachable except through that trusted proxy, which must replace +# client-supplied forwarded headers. For any direct ingress, set FORWARDED_ALLOW_IPS to the actual +# proxy IPs/CIDRs before exposing the port; do not carry wildcard trust onto a public listener. +# Suppress the server banner and release idle keep-alive sockets after five seconds. W59-T36 +# owns the API posture read-back and application host/header policy; these flags are its launcher seam. +CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 5"] diff --git a/RELEASES.json b/RELEASES.json index e731c9ab24b0a24ec174d258239663d0f0431dca..81ab60306e081379e2d3c62152a259bd9b3eb769 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,12 @@ { - "current": "v73 (bb94de5)", + "current": "v74 (40dceba)", "releases": [ + { + "version": "v74", + "sha": "40dceba", + "date": "2026-09-14", + "subject": "release v74: waves 58+59 incl. the Amazon credential door (W59-T21/22/23) and sync producer (W59-T42); hotfix promote so" + }, { "version": "v73", "sha": "bb94de5", diff --git a/VERSION b/VERSION index 1e15e29d9ad5fcc8920c4f4747bfa4fd135d93f9..86c64d99f4dec766401a6d295a87d91df6d320cd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v73 (bb94de5) +v74 (40dceba) diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index 9dfa5d00a5b354f31aba524728bf4c4a119e393c..7d340dd7df920ddac75041612c607103275f5ac3 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -44,6 +44,56 @@ from harness.amazon_client import ( # noqa: F #: comments in `keychain.py` warn about it in as many words. ENTRY_TYPE = "amazon_sp" + +class AmazonStatusSafetyError(ValueError): + """A status response contains a forbidden credential shape.""" + + +def status_payload_has_secret(serialized_payload): + """Scan serialized JSON for C2's credential keys, token shapes and oversized values. + + Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content. + The credential key roster is the same one the connector accepts at connect time. + """ + import json + import re + + try: + pending = [json.loads(serialized_payload, object_pairs_hook=lambda pairs: pairs)] + except (TypeError, ValueError, UnicodeError, RecursionError): + return True + credential_keys = {key.casefold() for key in CREDENTIAL_FIELDS} + while pending: + value = pending.pop() + if isinstance(value, tuple): + key, item = value + if key.casefold() in credential_keys: + return True + pending.extend((key, item)) + elif isinstance(value, list): + pending.extend(value) + elif isinstance(value, str): + if (len(value) > 512 or 'Atzr|' in value + or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)): + return True + elif value is not None and len(str(value)) > 512: + return True + return False + + +def assert_safe_status(payload): + """Return a status only after scanning its serialized form, or raise without its content.""" + import json + + try: + serialized = json.dumps(payload, ensure_ascii=False, allow_nan=False) + except (TypeError, ValueError, RecursionError): + raise AmazonStatusSafetyError('Amazon status could not be safely serialized.') from None + if status_payload_has_secret(serialized): + raise AmazonStatusSafetyError('Amazon status contains a forbidden credential shape.') + return payload + + #: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's #: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and #: its seven databases are enabled at the provider boundary rather than masquerading as Odoo @@ -139,7 +189,13 @@ def enablement_capability(rt, is_admin): "available": state["available"]} -def probe_credential(fields, timeout=8): +def probe_credential(fields, timeout=8, *, fetch_participations=None): + """Return the guarded Amazon status; a disclosure violation must raise at this boundary.""" + return assert_safe_status(_probe_credential_result( + fields, timeout, fetch_participations=fetch_participations)) + + +def _probe_credential_result(fields, timeout=8, *, fetch_participations=None): """`{ok, message}` for a stored `amazon_sp` credential: the keychain's liveness probe. ⛔ IT LIVES HERE, IN THE API LAYER, AND IS REGISTERED INTO `core.keychain` RATHER THAN IMPORTED @@ -151,26 +207,27 @@ def probe_credential(fields, timeout=8): reports zero ad accounts that way: a credential that authenticates but reaches nothing is an authorization answer, and `ok: true` would hide the one thing the admin has to fix. """ - f = fields or {} - missing = [k for k in ("client_id", "client_secret", "refresh_token") if not f.get(k)] + f = fields if isinstance(fields, dict) else {} + missing = [key for key in CREDENTIAL_FIELDS + if not isinstance(f.get(key), str) or not f[key].strip()] if missing: return {"ok": False, - "message": "store " + ", ".join(f"`{m}`" for m in missing) - + " alongside `seller_id` and `marketplace_id`"} + "message": "Missing " + ", ".join(missing) + ". Enter every required field."} try: - parts = marketplace_participations(f, timeout=timeout) - except AmazonError as e: - return {"ok": False, "message": str(e)[:200]} - live = [p for p in parts - if ((p or {}).get("participation") or {}).get("isParticipating") is not False] - n = len(live) - if not n: + # Bind only when explicitly tested. Shape gates can inject a local response function. + fetch = fetch_participations if fetch_participations is not None else marketplace_participations + parts = fetch(f, timeout=timeout) + live = [part for part in parts if isinstance(part, dict) + and isinstance(part.get("participation"), dict) + and part["participation"].get("isParticipating") is True] + if not live: + return {"ok": False, + "message": "Amazon reported no active marketplace access. Check authorization and test again."} + except Exception: # noqa: BLE001 + # AmazonError includes provider-supplied text and an invalid region can echo user input. return {"ok": False, - "message": "the credential authenticates but participates in NO marketplace: " - "that is an authorization answer, not an empty account"} - names = ", ".join(sorted({str(((p or {}).get("marketplace") or {}).get("countryCode") or "?") - for p in live})) - return {"ok": True, "message": f"Amazon answered. {n} marketplace(s) reachable: {names}"} + "message": "Amazon did not confirm marketplace access. Check the five fields and test again."} + return {"ok": True, "message": "Amazon confirmed marketplace access."} def register(): @@ -294,11 +351,11 @@ def main(argv): print("") try: probe_report(f, report_type=which) - except Exception as e: # noqa: BLE001 + except Exception: # noqa: BLE001 # ⚠ REPORTED, NOT RAISED. The probe is a diagnostic: a refusal is a RESULT the operator # needs to read (a role they lack, a throttle, an unsupported sandbox report type), not # a traceback that buries it. - print(f"REPORT PROBE REFUSED: {type(e).__name__}: {e}") + print("REPORT PROBE REFUSED: Amazon did not confirm the report. Check authorization and try again.") return 1 return 0 diff --git a/api/main.py b/api/main.py index 268093d6d988998f1062e2c5f5c3a190db4510f7..12d7d9584cda8402443c941fa63690a4996c1fd5 100644 --- a/api/main.py +++ b/api/main.py @@ -286,6 +286,28 @@ def health(): return {"ok": True} +# W59-T43: boot stays available while a cross-lane module is absent, but never silently. +# verify_wiring.W59_PENDING_CROSS_LANE_HALVES owns the shrink-only declarations and rejects +# any failure outside them, including a missing dependency INSIDE a declared router. +ROUTER_MOUNT_FAILURES = {} + + +def _mount_router(module_name): + """Mount the module's own router/prefix and retain every failed boot attempt for checks.""" + try: + from importlib import import_module + app.include_router(import_module(module_name).router) + return True + except Exception as exc: # Boot must answer even when a router cannot load. + failure = {"error": type(exc).__name__, + "missing_module": exc.name if isinstance(exc, ModuleNotFoundError) else ""} + ROUTER_MOUNT_FAILURES.setdefault(module_name, []).append(failure) + # Exception messages can contain configuration values; report the class/module only. + print(f"[aios-api] ROUTER MOUNT FAILED: {module_name}: {failure}; " + "routes unavailable; check W59_PENDING_CROSS_LANE_HALVES in verify_wiring.py") + return False + + app.include_router(routes_auth.router) app.include_router(routes_nav.router) app.include_router(routes_customers.router) @@ -463,6 +485,7 @@ app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent h # rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) app.include_router(routes_external.router) +_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount. # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) @@ -632,6 +655,7 @@ def _seed_and_sync_store(): # than in tomorrow's. It is idempotent, so every later boot is one `list_entries` read. _migrate_env_odoo("boot") _pull_meta("boot") + _pull_amazon("boot") _rebuild_odoo_relational("boot") # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. # ⚠ AFTER the two above and not before: those advance the mirror and can take minutes, and @@ -716,6 +740,32 @@ def _pull_meta(why): return False +def _pull_amazon(why): + """Produce Amazon rows in this container before projecting them, on the caller's thread.""" + try: + from harness import amazon_store as _amazon + from harness import runtime as _runtime + # The existing boot/resync projection targets tenant #0. Keep credential, mirror and + # projection on that same tenant; never fall back to an operator's environment credential. + tenant_key = "royal-imports" + rt = _runtime.get_runtime(tenant_key) + if not _amazon.credential(tenant_key, rt=rt): + print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential " + "for royal-imports; the connector is idle, not broken") + return False + rep = _amazon.sync(tenant_key, rt=rt, scheduled=True, + log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")) + for problem in rep.get("problems") or []: + print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}") + print(f"[aios-api] amazon sync done ({why}): " + ", ".join( + f"{table}={row.get('state', str(row.get('rows', 0)) + ' rows')}" + for table, row in rep["tables"].items())) + return not bool(rep.get("problems")) + except Exception as exc: # noqa: BLE001 + print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}") + return False + + def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False): """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. @@ -1386,6 +1436,7 @@ def _store_resync_loop(): # from being permanently absent. # ⚠ Cheap when there is nothing to do: no token => one line and return. _meta_pull_confirmed = _pull_meta("resync") + _pull_amazon("resync") _meta_now = _t.time() _meta_reason = _meta_rebuild_force_reason( _meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now) diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 67f9051819bf97e177b17ce069a8598028f7a628..7e225b28ec364e4d1dce3b04a61056898441670b 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -2002,6 +2002,66 @@ def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=Fa return result +def refresh_model_catalog_entry(rt, model, connection_id=None, client=None): + """Refresh one model's fields with two read-only calls, without enumerating other models.""" + current = readable_model_catalog(rt, connection_id) + + def _read_fields(bound_client): + readable = bound_client.execute_kw( + model, "check_access_rights", ["read"], {"raise_exception": False}) + if not bool(readable): + return None + fields = bound_client.fields_get( + model, attributes=("string", "type", "required", "readonly", "relation")) + clean_fields = {} + for name, meta in (fields if isinstance(fields, dict) else {}).items(): + if not str(name).strip() or not isinstance(meta, dict): + continue + clean_fields[str(name)] = {k: meta.get(k) for k in + ("string", "type", "required", "readonly", "relation") + if k in meta} + return clean_fields + + try: + if client is not None: + fields = _read_fields(client) + else: + source = (rt.odoo_source() if connection_id is None + else rt.odoo_source(connection_id)) + if source is None: + return current + from core.odoo import get_odoo + with source._bound(): # tenant credential + scope + fields = _read_fields(get_odoo()) + except Exception: # noqa: BLE001 + # No answer (including unavailable credentials) is not evidence of denied model access. + return current + if fields is None: + raise Refused(f"This Odoo account cannot read the model {model}. " + "Ask your Odoo administrator to grant read access in Odoo, not here.") + slot = _catalog_slot(connection_id) + + def _cache(current_cache): + current_cache = dict(current_cache) if isinstance(current_cache, dict) else {} + connections = dict(current_cache.get("connections") or {}) + result = dict(connections.get(slot) or current) + rows = list(result.get("models") or ()) + previous = next((row for row in rows if row.get("model") == model), None) + row = {"model": model, + "name": " ".join(str((previous or {}).get("name") or model).split()), + "fields": fields} + result["models"] = [row if item.get("model") == model else item for item in rows] + if previous is None: + result["models"].append(row) + # One model neither captures a whole catalogue nor changes its completeness. Preserve + # complete and capturedAt exactly, along with every unrelated row and connection slot. + connections[slot] = result + return {"version": MODEL_CATALOG_VERSION, "connections": connections} + + rt.update(MODEL_CATALOG_KEY, _cache, flush="sync") + return readable_model_catalog(rt, connection_id) + + # The population fact is deliberately separate from the readable-model catalogue. Measured on # Royal Imports on 2026-09-08: discovery found 499 readable models in 22.62 s, then exact # search_count calls took 67.87 s for 494 models at 137 ms per model; 239 held rows. A sampled @@ -2143,7 +2203,7 @@ def population_census(rt, connection_id=None): """The last census, from the tenant store. Reading it never contacts Odoo.""" empty = {"version": POPULATION_CENSUS_VERSION, "capturedAt": "", "complete": False, "durationMs": 0, "counts": {}, "errors": [], "modelsCounted": 0, - "modelsWithRows": 0} + "modelsWithRows": 0, "mode": "unknown"} try: cached = rt.get(POPULATION_CENSUS_KEY) if rt is not None else None except Exception: # noqa: BLE001 @@ -2172,6 +2232,8 @@ def population_census(rt, connection_id=None): return {"version": POPULATION_CENSUS_VERSION, "capturedAt": str(slot.get("capturedAt") or ""), + # Documents predating bounded counts were always exact. + "mode": (slot.get("mode", "exact") if slot.get("capturedAt") else "unknown"), "complete": slot.get("complete") is True, "durationMs": _nonnegative_int(slot.get("durationMs")), "counts": counts, @@ -2180,10 +2242,15 @@ def population_census(rt, connection_id=None): "modelsWithRows": _nonnegative_int(slot.get("modelsWithRows"))} -def refresh_population_census(rt, connection_id=None, client=None, force=False): - """Count every readable model with read-only ``search_count`` and cache the result.""" +def refresh_population_census(rt, connection_id=None, client=None, force=False, *, exact=False): + """Cache model counts using at most 101 ids, or an explicitly requested exact count. + + This limit bounds census work only; connected-source row readers remain uncapped. + Clients without search retain the exact search_count path and record that fallback. + """ current = population_census(rt, connection_id) - if current["capturedAt"] and not force: + mode = "exact" if exact else "bounded" + if current["capturedAt"] and not force and current["mode"] == mode: return current # Start before possible discovery so durationMs is the wall clock the person waiting sees: @@ -2208,10 +2275,16 @@ def refresh_population_census(rt, connection_id=None, client=None, force=False): errors.append("catalogue: no complete readable model catalogue is available") def _count_all(bound_client): + nonlocal mode + search = getattr(bound_client, "search", None) if not exact else None + if not exact and not callable(search): + mode = "exact_fallback" for model in models: try: - # search_count is the only Odoo operation in the census. It cannot mutate data. - counts[model] = int(bound_client.search_count(model, [])) + # Both operations are reads. search_count has no limit argument in Odoo. + counts[model] = (len(search(model, [], limit=101)) + if mode == "bounded" + else int(bound_client.search_count(model, []))) except Exception as exc: # noqa: BLE001 errors.append(f"{model}: {type(exc).__name__}: {exc}") @@ -2233,6 +2306,7 @@ def refresh_population_census(rt, connection_id=None, client=None, force=False): result = { "version": POPULATION_CENSUS_VERSION, "capturedAt": _dt.datetime.now(_dt.timezone.utc).isoformat(), + "mode": mode, "complete": catalogue.get("complete") is True and not errors and len(counts) == len(models), "durationMs": int((_time.monotonic() - started) * 1000), @@ -3641,8 +3715,9 @@ def _model_field_specs(catalogue_row): def publish_database_schema(rt, table_key, connection_id=None, username="automation", today=None): - """Publish one cached, permission-visible database definition without source calls or row work. + """Publish one permission-visible database definition without row work. + Make at most two source calls on one model, on the miss path only. Enable owns the choice and a durable sync job owns materialization. This seam only merges the schema, so retrying it preserves existing rows, authored fields and the navigation rename. """ @@ -3664,6 +3739,9 @@ def publish_database_schema(rt, table_key, connection_id=None, username="automat raw = (_ut().odoo_schema_snapshot(rt).get("models") or {}).get(model) if isinstance(raw, dict): row = {"model": model, "fields": raw} + if not row or not row.get("fields"): + catalogue = refresh_model_catalog_entry(rt, model, connection_id=namespace) + row = next((row for row in catalogue["models"] if row.get("model") == model), None) if not row or not row.get("fields"): raise Refused("the field catalogue is unavailable for this database. Refresh the catalogue first.") raw_fields = _model_field_specs(row) diff --git a/api/routes_auth.py b/api/routes_auth.py index aa43d6eeb3704bc39acfb142eaf975a7c09d99cc..3dbe9f68d9ac2ea259312ece79176ef596b11255 100644 --- a/api/routes_auth.py +++ b/api/routes_auth.py @@ -7,15 +7,48 @@ replaces, and it is deliberately the same account store so the migration is a sw CREDENTIAL check, not of the user model. """ import base64 as _b64 +import hashlib as _hashlib import re as _re from fastapi import APIRouter, Body, Depends, Request, Response import aios_session +from core import audit from deps import Session, err, require_session, users, perms router = APIRouter(prefix="/api/v1/auth") +# GET /me, /mfa and /password-policy deliberately emit nothing: polling these reads would +# evict useful security history from the bounded document. Candidate MFA secrets are stateless; +# only their confirmation changes the account and earns an enrolled event. +# Each emit catches only AuditRefused at this HTTP boundary so a programming error is loud +# without breaking a production sign-in. audit.record itself owns best-effort store failures. +def _audit_unresolved(identifier): + """Never persist a typed login name: it might be a password pasted into the wrong box.""" + # Deliberately NOT users' lockout namespace: these digests cannot be joined to that budget. + normalised = str(identifier or "").strip().lower() + return "digest:" + _hashlib.sha256( + ("aios-security-event-identity-v1\0" + normalised).encode("utf-8")).hexdigest() + + +#: ⛔⛔ W-171 - FAILING CLOSED IS RIGHT; FAILING CLOSED SILENTLY IS NOT. On 2026-09-13 staging +#: answered 503 `auth_state_unavailable` to every sign-in, *including a correct password*, and the +#: container printed NOTHING: each helper below caught `Exception` and raised the generic 503 with +#: no trace of what it had caught. The cause (`core.data_binding.StoreWriteRefused`, because the +#: deploy's `AIOS_SANDBOX_TENANTS` omitted `royal-imports` and the lockout counter is a WRITE into +#: tenant #0's store on BOTH the failure and the success path) could only be found by reproducing +#: the call locally against the staging database. One line per helper now names the type and the +#: message on the way out. It goes to stdout in `api/main.py`'s `[aios-api] ...` shape because that +#: is what this service uses and what the Space's log collects; the CLIENT message is unchanged, +#: so nothing about the diagnosis reaches the browser. +def _auth_store_down(where, exc): + """Print the swallowed cause, then return the 503 every caller raises.""" + print(f"[aios-api] auth lockout store unavailable in {where}: " + f"{type(exc).__name__}: {exc}") + return err(503, "auth_state_unavailable", + "sign in protection is temporarily unavailable. Try again later") + + #: THE SERVER OWNS BACKOFF: the login page never paces itself. The budget lives in the shared #: tenant-0 auth store, not this process, so worker rotation or a deploy cannot buy ten more tries. #: Username and email aliases resolve to one canonical account key; only a SHA-256 digest of that @@ -23,9 +56,8 @@ router = APIRouter(prefix="/api/v1/auth") def _throttle_key(identifier): try: return users.canonical_login_identifier(identifier) - except Exception: - raise err(503, "auth_state_unavailable", - "sign in protection is temporarily unavailable. Try again later") + except Exception as exc: + raise _auth_store_down("_throttle_key", exc) def _throttled(key, now=None): @@ -33,9 +65,8 @@ def _throttled(key, now=None): return 0 try: return users.login_lockout_remaining(key, now=now) - except Exception: - raise err(503, "auth_state_unavailable", - "sign in protection is temporarily unavailable. Try again later") + except Exception as exc: + raise _auth_store_down("_throttled", exc) def _note_failure(key, now=None): @@ -43,9 +74,8 @@ def _note_failure(key, now=None): return 0 try: return users.note_login_failure(key, now=now) - except Exception: - raise err(503, "auth_state_unavailable", - "sign in protection is temporarily unavailable. Try again later") + except Exception as exc: + raise _auth_store_down("_note_failure", exc) def _clear_failures(key): @@ -53,9 +83,8 @@ def _clear_failures(key): return try: users.clear_login_failures(key) - except Exception: - raise err(503, "auth_state_unavailable", - "sign in protection is temporarily unavailable. Try again later") + except Exception as exc: + raise _auth_store_down("_clear_failures", exc) def _public_user(user): @@ -108,6 +137,11 @@ def set_avatar(body: dict = Body(default=None), users.set_avatar(session.uname, raw) u2 = dict(session.user) u2["avatar"] = raw + try: + audit.record("config.changed", session.tenant, session.user["username"], + {"setting": "avatar"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") return {"user": _public_user(u2)} @@ -119,6 +153,11 @@ def clear_avatar(session: Session = Depends(require_session)): users.set_avatar(session.uname, None) u2 = dict(session.user) u2.pop("avatar", None) + try: + audit.record("config.changed", session.tenant, session.user["username"], + {"setting": "avatar"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") return {"user": _public_user(u2)} @@ -148,6 +187,12 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) throttle_key = None if user else _throttle_key(uname) wait = _throttled(throttle_key) if wait: + try: + # No account is proved at this point. "unknown" avoids inventing tenant attribution. + audit.record("auth.lockout", "unknown", _audit_unresolved(uname), + {"seconds_remaining": wait}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(429, "too_many_attempts", f"too many failed attempts. Try again in {wait} seconds") if not user: @@ -162,6 +207,11 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) except Exception: pass _note_failure(throttle_key) + try: + # A failed credential resolves no account or tenant; the raw identifier stays out. + audit.record("auth.login_failed", "unknown", _audit_unresolved(uname)) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(401, "invalid_credentials", "that username and password do not match") tenant = str(user.get("tenant") or "royal-imports").strip().lower() @@ -170,6 +220,11 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) runtime.get_runtime(tenant) except KeyError: _note_failure(throttle_key) + try: + audit.record("auth.login_failed", tenant, user["username"], + {"reason": "tenant_unavailable"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(401, "invalid_credentials", "that username and password do not match") # W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or @@ -193,6 +248,7 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) # W57-T36 / C12: every STORED account needs its second factor before any cookie is minted. # qa_runner is the sole exception and is double-gated out of live deployments. + recovery_code_used = False if not user.get("qa_runner"): try: rec = (users.registry() or {}).get(user["username"]) @@ -208,9 +264,15 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.") # TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's # ordinary code already proves the factor, and never reset the guess budget on password. - if not (users.totp_valid(rec.get("mfa_secret"), code) - or users.consume_recovery_code(user["username"], code)): + factor_valid = users.totp_valid(rec.get("mfa_secret"), code) + if not factor_valid: + recovery_code_used = users.consume_recovery_code(user["username"], code) + if not (factor_valid or recovery_code_used): _note_failure(throttle_key) + try: + audit.record("auth.mfa_failed", tenant, user["username"]) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(401, "mfa_invalid", _MFA_INVALID) # ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this @@ -236,11 +298,13 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) # that security write is unavailable, authentication fails closed. _clear_failures(throttle_key) session_epoch = int(user.get("epoch") or 0) + epoch_bumped = False if not user.get("qa_runner"): try: bumped = users.bump_epoch(user["username"]) if bumped is not None: session_epoch = int(bumped) + epoch_bumped = True except Exception as exc: print(f"[aios-api] login for {user['username']!r}: could not bump the session epoch " f"({type(exc).__name__}). This sign-in stands, but the account's previous " @@ -261,6 +325,11 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) import deps as _deps if not user.get("qa_runner"): _deps.note_active(user["username"]) + try: + audit.record("auth.login", tenant, user["username"], + {"recovery_code_used": bool(recovery_code_used), "epoch_bumped": epoch_bumped}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") return {"user": _public_user(user)} @@ -279,6 +348,12 @@ def _mfa_credentials(uname, password): throttle_key = _throttle_key(uname) wait = _throttled(throttle_key) if wait: + try: + # Password proof has not resolved an account, so no tenant can be asserted. + audit.record("auth.lockout", "unknown", _audit_unresolved(uname), + {"seconds_remaining": wait}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(429, "too_many_attempts", f"too many failed attempts. Try again in {wait} seconds") user = users.verify(uname, password) if (uname and password) else None @@ -288,6 +363,11 @@ def _mfa_credentials(uname, password): except Exception: pass _note_failure(throttle_key) + try: + audit.record("auth.login_failed", "unknown", _audit_unresolved(uname), + {"door": "mfa"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(401, "invalid_credentials", "that username and password do not match") return user, throttle_key @@ -311,6 +391,11 @@ def _mfa_session_tenant(user, rec, throttle_key): runtime.get_runtime(tenant) except KeyError: _note_failure(throttle_key) + try: + audit.record("auth.login_failed", tenant, user["username"], + {"door": "mfa", "reason": "tenant_unavailable"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(401, "invalid_credentials", "that username and password do not match") # Confirming a phone must not become a second login door that accepts an expired password. # Check before writing the factor, so the person can rotate and resume the same setup. @@ -387,12 +472,20 @@ def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(d secret = str(body.get("secret") or "").strip().upper() if not users.totp_valid(secret, str(body.get("code") or "")): _note_failure(throttle_key) + try: + audit.record("auth.mfa_failed", user["tenant"], user["username"], {"door": "enroll"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(401, "mfa_invalid", _MFA_INVALID) tenant = _mfa_session_tenant(user, rec, throttle_key) codes = users.new_recovery_codes() # The store guard repeats the check atomically: two workers cannot issue two first sets. landed = _mfa_set(user, secret, codes, require_unenrolled=True) _mfa_session(request, response, user, rec, tenant, landed, throttle_key) + try: + audit.record("auth.mfa_enrolled", tenant, user["username"], {"reason": "first"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") return {"user": _public_user(user), "recoveryCodes": codes} @@ -427,6 +520,71 @@ def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(d codes = users.new_recovery_codes() landed = _mfa_set(user, rec["mfa_secret"], codes) _mfa_session(request, response, user, rec, tenant, landed, throttle_key) + try: + # "credential" is itself a forbidden secret key; this label identifies the kind only. + audit.record("credential.rotated", tenant, user["username"], + {"credential_kind": "recovery_codes"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") + return {"recoveryCodes": codes} + + +@router.post("/mfa/device") +def mfa_device(response: Response, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Offer a replacement phone without removing the factor that admitted this session. + + This is NOT the first-enrolment door: R1 still refuses an unenrolled login without a cookie. + A candidate is stateless and changes nothing until the new phone proves its first code. + """ + # Refuse before any credential lookup: the synthetic runner must not read tenant #0. + if session.user.get("qa_runner"): + raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.") + # The session supplies the identity. Body username/tenant fields can never choose a target. + user, _ = _mfa_credentials(session.uname, str((body or {}).get("password") or "")) + rec = _mfa_record(session.uname) + if not users.mfa_enrolled(rec): + raise err(409, "mfa_not_enrolled", "Sign in again to set up your authenticator app.") + secret = users.new_mfa_secret() + response.headers["Cache-Control"] = "no-store" + return {"secret": secret, "otpauthUri": users.otpauth_uri(secret, user["username"]), + "issuer": users.MFA_ISSUER, "digits": users.MFA_DIGITS, "period": users.MFA_PERIOD} + + +@router.post("/mfa/device/confirm") +def mfa_device_confirm(request: Request, response: Response, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Prove the CANDIDATE phone, replace the factor and revoke the account's other sessions. + + Checking the installed secret would bless an unproved replacement. Requiring unenrolled + would refuse every legitimate caller. Re-mint from the landed epoch or this credential + change signs out its own caller; the replacement recovery codes are shown only once. + """ + # Refuse before any credential lookup, even when the runner posts a wrong password. + if session.user.get("qa_runner"): + raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.") + body = body or {} + # The session supplies the identity. Body username/tenant fields can never choose a target. + user, throttle_key = _mfa_credentials(session.uname, str(body.get("password") or "")) + rec = _mfa_record(session.uname) + if not users.mfa_enrolled(rec): + raise err(409, "mfa_not_enrolled", "Sign in again to set up your authenticator app.") + secret = str(body.get("secret") or "").strip().upper() + if not users.totp_valid(secret, str(body.get("code") or "")): + _note_failure(throttle_key) + try: + audit.record("auth.mfa_failed", user["tenant"], user["username"], {"door": "device"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") + raise err(401, "mfa_invalid", _MFA_INVALID) + tenant = _mfa_session_tenant(user, rec, throttle_key) + codes = users.new_recovery_codes() + landed = _mfa_set(user, secret, codes) + _mfa_session(request, response, user, rec, tenant, landed, throttle_key) + try: + audit.record("auth.mfa_enrolled", tenant, user["username"], {"reason": "device_change"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") return {"recoveryCodes": codes} @@ -454,6 +612,12 @@ def change_password(body: dict = Body(default=None)): throttle_key = _throttle_key(uname) wait = _throttled(throttle_key) if wait: + try: + # This unauthenticated door has no proved tenant either. + audit.record("auth.lockout", "unknown", _audit_unresolved(uname), + {"seconds_remaining": wait}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(429, "too_many_attempts", f"too many failed attempts. Try again in {wait} seconds") user = users.verify(uname, pw) if (uname and pw) else None @@ -463,6 +627,11 @@ def change_password(body: dict = Body(default=None)): except Exception: pass _note_failure(throttle_key) + try: + audit.record("auth.login_failed", "unknown", _audit_unresolved(uname), + {"door": "password"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") raise err(401, "invalid_credentials", "that username and password do not match") try: # verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed @@ -471,6 +640,12 @@ def change_password(body: dict = Body(default=None)): except users.PasswordPolicyError as exc: raise err(400, "weak_password", "That password does not meet the policy: " + str(exc)) _clear_failures(throttle_key) + try: + # Use a descriptive label; C2 forbids the key "credential" even for non-secret values. + audit.record("credential.rotated", user["tenant"], user["username"], + {"credential_kind": "password"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") return Response(status_code=204) @@ -494,6 +669,21 @@ def logout(request: Request): """ out = Response(status_code=204) aios_session.clear_cookie(out, request) + # Only verified cookie claims can name an account here. These were minted from the resolved + # username; no new admission or store lookup is needed to clear an already-revoked session. + try: + claims = aios_session.read(request.cookies.get(aios_session.COOKIE_NAME)) + except (TypeError, ValueError, OverflowError): + # A malformed signature (including non-ASCII bytes) must not prevent cookie clearing. + # Its parser exception could contain the cookie, so it never belongs in a log either. + claims = None + # Invalid/expired/absent cookies prove no tenant. Never use their raw bytes as attribution. + tenant = claims["t"] if claims else "unknown" + principal = claims["u"] if claims else _audit_unresolved("logout-without-identity") + try: + audit.record("auth.logout", tenant, principal) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") return out diff --git a/api/routes_connectors.py b/api/routes_connectors.py index c53e701a1dad037105daab1ba959e4468e18b11c..6a8733ddb1fb4b886a5b322f89396e26510b4649 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -23,6 +23,7 @@ from __future__ import annotations import os import time import threading +from datetime import datetime, timezone from fastapi import APIRouter, Body, Depends @@ -36,6 +37,8 @@ from deps import Session, err, require_session #: connector is in. from core import connector_directory from core import connector_sync +from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD +from harness import amazon_client #: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default, #: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already #: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module @@ -76,6 +79,26 @@ from routes_monid import router as monid_router # same URL. `verify_api` now asserts the prefix on every mounted router instead. router = APIRouter(prefix="/api/v1") +CONNECTOR_DETAIL_DEADLINE_SECONDS = 30 + + +# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention. +# A thread pool deadline was tried and produced three consecutive 30 s 504s because +# core/store.py uses reentrant locks whose ownership does not survive a thread +# boundary while the request thread waits on the result. Keep the work on the +# request thread. This budget is therefore cooperative: it bounds a read whose +# phases are each finite and cannot interrupt a single call that is already wedged. +class _ConnectorDetailBudget: + def __init__(self): + self._started = time.monotonic() + + def check(self): + if time.monotonic() - self._started >= CONNECTOR_DETAIL_DEADLINE_SECONDS: + raise err(504, "connector_detail_timeout", + f"The connector store did not answer within {CONNECTOR_DETAIL_DEADLINE_SECONDS} " + "seconds, so the read was stopped.") + + #: The to-do list (R8). Order is the owner's reading order, not alphabetical: the things most #: likely to come next first. `queue` mirrors the standing provider queue in wave22-split C5-A2 #: so the two cannot drift into disagreeing about what is coming. @@ -650,7 +673,12 @@ def _entry_types(rt, uname=""): types, scopes = set(), {} for e in rows: t = str(e.get("type") or "") - types.add(t) + if t == "amazon_sp": + from core import keychain as kc + if kc.entry_verified(rt, e.get("id")): + types.add(t) + else: + types.add(t) if scopes.get(t) != rk.DEFAULT_SCOPE: scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE) return types, scopes, None @@ -658,6 +686,26 @@ def _entry_types(rt, uname=""): return set(), {}, f"{type(e).__name__}: {e}" +def _stored_amazon_rotation_deadline(rt, entry_id): + """Project only the public timestamp from the stored row, without decrypting a credential.""" + from core import keychain as kc + stored = ((rt.get(kc.KEY) or {}).get("entries") or {}).get(str(entry_id)) or {} + if stored.get("type") != "amazon_sp": + return "" + deadline = stored.get(AMAZON_ROTATION_DEADLINE_FIELD) + return deadline if isinstance(deadline, str) else "" + + +def _amazon_rotation_expired(deadline): + if not deadline: + return False + try: + parsed = datetime.fromisoformat(deadline.replace("Z", "+00:00")) + return parsed.tzinfo is not None and parsed <= datetime.now(timezone.utc) + except (TypeError, ValueError): + return False + + def directory(rt, uname, is_admin=False, incomplete=None): """The composed row set. Pure over the session, so the gate drives it without a request. @@ -751,11 +799,21 @@ def directory(rt, uname, is_admin=False, incomplete=None): # --- Token connectors (R8): connectable today, and honest that data flows arrive later. def _tokens(): + amazon_expired = False + if "amazon_sp" in scopes: + import routes_keychain as rk + amazon_expired = any( + _amazon_rotation_expired(_stored_amazon_rotation_deadline(rt, entry.get("id"))) + for entry in rk.visible_entries(rt, uname) or [] + if entry.get("type") == "amazon_sp") for t in TOKEN_CONNECTORS: rows.append({ "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"], "hint": t["hint"], - "state": "connected" if t["entry"] in have else "available", + "state": ("expired" if t["entry"] == "amazon_sp" and amazon_expired + else "connected" if t["entry"] in have else "reconnect" + if t["entry"] == "amazon_sp" and t["entry"] in scopes + else "available"), # ⭐ W32-T11 / C1 / R4 — the scope of the credential BEHIND this card, for THIS # reader. A token connector is the case R4 is really about: a member's own Stripe # key is theirs, and the card must say which kind of connection it is looking at. @@ -893,6 +951,7 @@ def directory(rt, uname, is_admin=False, incomplete=None): # thing the toggle must never do is hide a connection the reader actually holds. for row in rows: row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key")))) + _safe_connector_status(row.get("key"), row) return rows @@ -1067,6 +1126,9 @@ def connectors_directory(session: Session = Depends(require_session)): # number of cards behind it by construction, and never a sum of endpoint counts (the ticket's # trap; `chip_counts` carries the argument). served = connector_directory.to_client_rows(rows) + for row in served: + _safe_connector_status(row.get("key"), row) + _safe_connector_status("amazon_sp", {"incomplete": incomplete}) return {"connectors": served, "incomplete": incomplete, # ⭐⭐ W53-T29 / OWNER RULING R6 — MAY THIS READER SEE THE COMPANY SIDE AT ALL. @@ -1124,16 +1186,16 @@ def _sync_payload(connector_key, connection_id, row=None): meta = connector_sync.providers()[connector_key] slot = str(connection_id or "").strip() or "default" if row is None: - return {"provider": connector_key, "label": meta["label"], + return _safe_connector_status(connector_key, {"provider": connector_key, "label": meta["label"], "connectionId": slot, "recorded": False, "state": None, "at": None, "stateAt": None, "from": None, "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0, - "databases": [], "retryable": False, "retryHint": meta["retryHint"]} + "databases": [], "retryable": False, "retryHint": meta["retryHint"]}) answer = dict(row) answer.update({"provider": connector_key, "connectionId": slot, "label": meta["label"], "recorded": True, "stateAt": row.get("at"), "retryHint": meta["retryHint"]}) - return answer + return _safe_connector_status(connector_key, answer) def _odoo_sync_runner(session, connection_id): @@ -1451,11 +1513,13 @@ def _schema_catalog_rt(rt, database_key=None, connection_id=None): return [] -def _odoo_field_key(rt, database_key, model, raw_name): - import odoo_relational +def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...): + # An explicitly resolved None must not trigger another lookup. + if connection_id is ...: + import odoo_relational + connection_id = odoo_relational.connection_for_table_key(database_key, rt) return connector_sync.field_key( - "odoo", odoo_relational.connection_for_table_key(database_key, rt), - database_key, model, raw_name) + "odoo", connection_id, database_key, model, raw_name) @router.get("/connectors/odoo/status") @@ -1554,8 +1618,8 @@ def connector_database_jobs(connector_key: str, session: Session = Depends(requi _sync_admin(session) if connector_key not in connector_sync.providers(): raise err(404, "sync_provider_unknown", "This connector has no database sync runner.") - return {"provider": connector_key, "jobs": connector_sync.jobs( - session.runtime, connector_key, connection_id)} + return _safe_connector_status(connector_key, {"provider": connector_key, "jobs": connector_sync.jobs( + session.runtime, connector_key, connection_id)}) @router.get("/connectors/odoo/schema") @@ -1582,17 +1646,20 @@ def odoo_schema(session: Session = Depends(require_session)): @router.post("/connectors/odoo/census") -def odoo_population_census(session: Session = Depends(require_session)): - """Explicitly count every readable Odoo model; connector detail only reads this cache.""" +def odoo_population_census(session: Session = Depends(require_session), exact: bool = False): + """Count up to 101 ids per model by default; exact=true requests the full census.""" _odoo_admin(session) import odoo_relational - census = odoo_relational.refresh_population_census(session.runtime, force=True) + census = odoo_relational.refresh_population_census(session.runtime, force=True, exact=exact) return {"capturedAt": str(census.get("capturedAt") or ""), + "mode": census["mode"], + "counts": census["counts"], "complete": census.get("complete") is True, "durationMs": int(census.get("durationMs") or 0), "modelsCounted": int(census.get("modelsCounted") or 0), "modelsWithRows": int(census.get("modelsWithRows") or 0), - "errors": [str(value) for value in (census.get("errors") or ())][:50]} + "errors": [str(value).replace("\u2014", ",").replace("\u2013", "-") + for value in (census.get("errors") or ())][:50]} def _materialisation_failure_reason(result): @@ -1611,6 +1678,9 @@ def _materialisation_failure_reason(result): candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,)) reason = str(next((value for value in candidates if str(value or "").strip()), "")).strip() # Provider errors can carry typographic punctuation. Screen strings use plain punctuation. + # ⛔ Standing rule 2 names the EM DASH and the EN DASH, never the ASCII hyphen. A blanket + # strip of every Unicode dash category also breaks "read-only" and "e-commerce", so the + # two characters the rule names are rewritten and nothing else is touched. reason = reason.replace("\u2014", ",").replace("\u2013", "-") return reason[:1].upper() + reason[1:] if reason else "" @@ -1707,11 +1777,6 @@ def _enable_odoo_database(table_key, session, connection_id=None): return cur failure = None - try: - session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") - except Exception: # noqa: BLE001 - raise err(503, "store_unavailable", "the database setting was not saved. Try again.") - try: result = odoo_relational.publish_database_schema( session.runtime, key, connection_id=namespace, username=session.uname) @@ -1721,8 +1786,15 @@ def _enable_odoo_database(table_key, session, connection_id=None): failure = (409, "odoo_database_refused", str(exc)) except Exception: # noqa: BLE001 failure = (500, "database_schema_failed", - "The setting was enabled, but the database was not created because the " - "field catalogue could not be published.") + "The setting was not changed and the database was not created because the " + "field catalogue could not be published. Refresh the catalogue and try again.") + + # A refused publication must leave both the flag and its relock stamp untouched. + if failure is None: + try: + session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", "the database setting was not saved. Try again.") facts = (_rename_rights(session.runtime, session.uname, bool(session.admin), [key]).get(key) or {"exists": False, "canRename": False}) @@ -1733,9 +1805,8 @@ def _enable_odoo_database(table_key, session, connection_id=None): # `key`, `enabled` and the fresh facts BESIDE that key so the client could read them off a # failure. Two shapes for one thing is what this module's own comments call the defect it has # paid for twice, and a caller that learns to read a sibling off an error will keep doing it. - # The client does not need them: the FLAG is written before materialisation is attempted, so - # every outcome of this door can change what `/detail` answers, and `flipTable` therefore - # re-reads on both paths. + # The client re-reads `/detail` on both paths; publication precedes the flag write and row + # materialisation remains separate work. if failure is not None: raise err(*failure) if not exists: @@ -1985,11 +2056,27 @@ def _entry_type_for(connector_key): #: connector whose credential grows a field would take a client edit to become storable. #: ⚠ `secret` decides the input type, so a field marked secret is typed into a password box and #: never echoed back — the masked preview is the only form the stored value takes afterwards. +#: Sentence case with the initialism capitalised, which is what the rest of this app does. +_AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret", + "refresh_token": "Refresh token", "seller_id": "Seller ID", + "marketplace_id": "Marketplace ID"} _CONNECT_FIELDS = { "odoo": [{"key": "url", "label": "Server URL (https://...)", "secret": False}, {"key": "db", "label": "Database", "secret": False}, {"key": "user", "label": "API user", "secret": False}, {"key": "api_key", "label": "API key", "secret": True}], + # ⭐ THE KEYS AND THEIR ORDER ARE DERIVED, NEVER TYPED. `amazon_client.CREDENTIAL_FIELDS` is + # the one roster, and lane G's `verify_amazon` asserts this sequence against it, so a field + # added there appears here rather than drifting. Labels are the one thing NOT derived: + # `"client_id".replace("_", " ").capitalize()` renders "Client id", and this app writes the + # initialism. The map carries a derived fallback so a new credential field still renders. + "amazon_sp": [ + {"key": key, + "label": _AMAZON_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()), + "secret": key in ("client_secret", "refresh_token"), + **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})} + for key in amazon_client.CREDENTIAL_FIELDS + ], } #: Everything else is one pasted token. The field name is `api_key` because that is the first of #: `core.keychain._PREVIEW_FIELDS`, so the stored entry gets a preview rather than a blank cell — @@ -1997,6 +2084,14 @@ _CONNECT_FIELDS = { _CONNECT_TOKEN_FIELD = [{"key": "api_key", "label": "API key", "secret": True}] +def _safe_connector_status(connector_key, payload): + """Apply the provider's disclosure guard at the status boundary, outside fallback catches.""" + if connector_key == "amazon_sp": + import connectors_amazon as cx + return cx.assert_safe_status(payload) + return payload + + def _detail_connect(entry_type, is_admin): """C4's `keychain.connect` half: what the Connect form must ask for, and what it may store. @@ -2010,7 +2105,7 @@ def _detail_connect(entry_type, is_admin): """ if not entry_type: return None - fields = _CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD + fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD) tenant_wide = entry_type in TENANT_WIDE_TYPES return {"type": entry_type, "fields": [dict(field) for field in fields], @@ -2027,7 +2122,7 @@ def _replacement_field_specs(entry_type): This is deliberately a projection of the Connect descriptor rather than another provider vocabulary. URL, database, account and every future non-secret field stay server-held during - replacement; only fields the existing descriptor marks secret cross this door. + replacement. Optional rotation metadata is accepted separately from this descriptor. """ spec = _detail_connect(entry_type, True) or {} return [dict(field) for field in (spec.get("fields") or []) @@ -2117,6 +2212,9 @@ def _detail_keychain(rt, uname, is_admin, row): # a masked preview happened to be visible. The replacement endpoint changes # this row's encrypted payload in place; no secret is added to this DTO. "replaceable": can_replace, + **({AMAZON_ROTATION_DEADLINE_FIELD: + _stored_amazon_rotation_deadline(rt, entry.get("id"))} + if entry_type == "amazon_sp" else {}), }) except Exception: # noqa: BLE001 entries = [] @@ -2191,6 +2289,9 @@ def _connector_spec_items(connector_key): except Exception: # noqa: BLE001 return () key = str(connector_key or "") + # The directory uses the keychain type; the table registry uses the provider name. + if key == "amazon_sp": + key = "amazon" found = None for name in ("tables_for", "for_connector", "tables", "spec", "spec_for"): fn = getattr(connector_spec, name, None) @@ -2427,14 +2528,34 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None) except Exception: # noqa: BLE001 catalog = {} enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {} + # Connection resolution reaches keychain.list_entries and therefore the store (Postgres). + # It ran 12,824 times for fields and database identities, then 501 with a per-key memo. + # Resolve the shared default lazily once instead. This local belongs only to this detail + # build: request scope keeps the answer per runtime and tenant, never process-wide. + default_connection_id = ... + + def resolve_connection_id(key): + nonlocal default_connection_id + base, cid = odoo_relational.split_table_key(key) + if base is None: + return None + if cid: + return cid + if default_connection_id is ...: + default_connection_id = (odoo_relational.default_connection_id(rt) + or odoo_relational.DEFAULT_CONNECTION) + return default_connection_id + out = [] for choice in choices: key = str(choice.get("key") or "") + resolved_connection_id = resolve_connection_id(key) model = str(choice.get("model") or models.get(key) or "") fields, excluded = catalog.get(model, ([], [])) scoped_fields = [] for field in fields: - field_key = _odoo_field_key(rt, key, model, field["source_field"]) + field_key = _odoo_field_key(rt, key, model, field["source_field"], + connection_id=resolved_connection_id) scoped_fields.append({**field, "key": field_key, "enabled": enabled_fields.get(field_key, field.get("enabled") if namespace is None else False) is True}) @@ -2444,7 +2565,7 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None) source_rows = counts.get(model) if census_at and model in counts else None out.append({"key": key, "identity": connector_sync.database_identity( - "odoo", odoo_relational.connection_for_table_key(key, rt), key, model), + "odoo", resolved_connection_id, key, model), "label": str(declared.get("label") or choice.get("label") or key), "defaultLabel": str(declared.get("label") or choice.get("label") or key), "native": str(declared.get("native") or model), @@ -2728,7 +2849,9 @@ def connector_detail(rt, user, uname, is_admin, connector_key): Pure over the session for the same reason `directory` is: a gate can drive it without a request, so the modal's payload is provable without a browser. """ + budget = _ConnectorDetailBudget() rows = directory(rt, uname, bool(is_admin)) + budget.check() key = str(connector_key or "") row = next((candidate for candidate in rows if str(candidate.get("key") or "") == key), None) @@ -2737,27 +2860,37 @@ def connector_detail(rt, user, uname, is_admin, connector_key): graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal( key, "No actual relationship graph was returned for this connection. A provider template " "is not shown.") + budget.check() # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a # question about a PERSON and a table, not about an administrator flag alone. databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or "")) - return {"overview": _detail_overview(row, databases, graph), - "keychain": _detail_keychain(rt, uname, bool(is_admin), row), + budget.check() + overview = _detail_overview(row, databases, graph) + budget.check() + keychain = _detail_keychain(rt, uname, bool(is_admin), row) + budget.check() + enablement = _detail_enablement(rt, bool(is_admin), row) + budget.check() + introspection = _detail_introspection(rt, bool(is_admin), row) + budget.check() + return _safe_connector_status(key, {"overview": overview, + "keychain": keychain, "databases": databases, "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs", "retryPath": f"/connectors/{key}/sync/retry"} if is_admin and key in connector_sync.providers() else None), # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints. - "enablement": _detail_enablement(rt, bool(is_admin), row), + "enablement": enablement, # ⭐ W49-T14 / R24 — WHETHER THE DATABASES TAB MAY DISCLOSE ANYTHING AT ALL, and what # THIS reader may do there. An addition to C4, declared on `_detail_introspection`: # without it an empty `databases` list is "this connector has no schema" and "this # connector's schema could not be read" at the same time. - "introspection": _detail_introspection(rt, bool(is_admin), row), + "introspection": introspection, # ⛔ THE `relationship_graphs` SHAPE, UNCHANGED. C4 says so in as many words, and the # client already has a reader for it (`parseRelationshipGraph`) that refuses a graph # bound to another connection. A second shape here would need a second reader. - "relationships": graph} + "relationships": graph}) def _set_amazon_database_enablement(database_key, body, session): @@ -2818,8 +2951,9 @@ def connectors_detail(connector_key: str, session: Session = Depends(require_ses workspace is connected to. What a member does NOT get is the tenant-wide credential door (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue. """ - detail = connector_detail(session.runtime, session.user, session.uname, - bool(session.admin), connector_key) + detail = connector_detail( + session.runtime, session.user, session.uname, + bool(session.admin), connector_key) if detail is None: raise err(404, "unknown_connector", "this workspace has no connector with that key") @@ -2856,14 +2990,18 @@ def _encrypted_replacement_payload(kc, label, entry_type, fields, username): row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {}) if not row.get("enc"): raise RuntimeError("the keychain encryptor returned no encrypted payload") - return {"enc": str(row["enc"]), "preview": str(row.get("preview") or "")} + payload = {"enc": str(row["enc"]), "preview": str(row.get("preview") or ""), + "envelope_version": row["envelope_version"]} + if entry_type == "amazon_sp": + payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD] + return payload def _replace_connector_key(connector_key, entry_id, body, session): """Replace only one connector entry's declared secrets while preserving its identity. The existing row remains untouched through validation, decryption, merge and encryption. The - final store update compares the complete old row before swapping only ``enc`` and ``preview``; + final store update compares the complete old row before swapping the prepared envelope; a concurrent rename or credential change therefore conflicts instead of being overwritten. Scope and owner live in a separate bucket and are never written by this operation. """ @@ -2895,15 +3033,16 @@ def _replace_connector_key(connector_key, entry_id, body, session): raise err(404, "no_entry", "no such key for this connector") secret_specs = _replacement_field_specs(entry_type) - secret_keys = [str(field["key"]) for field in secret_specs] + secret_keys = [str(field["key"]) for field in secret_specs if field.get("secret") is True] + optional_keys = {AMAZON_ROTATION_DEADLINE_FIELD} if entry_type == "amazon_sp" else set() request = body if isinstance(body, dict) else {} supplied = request.get("fields") if set(request) != {"fields"} or not isinstance(supplied, dict): raise err(400, "bad_replacement", "send only the connector's new secret fields") - if set(supplied) != set(secret_keys): + if not set(secret_keys) <= set(supplied) <= set(secret_keys) | optional_keys: raise err(400, "bad_replacement", - "send every new secret field and no account or server settings") + "send every new secret field, optional rotation metadata, and no account or server settings") replacements = {} for field in secret_keys: value = supplied.get(field) @@ -2911,8 +3050,18 @@ def _replace_connector_key(connector_key, entry_id, body, session): raise err(400, "bad_replacement", "a replacement secret cannot be blank") replacements[field] = value.strip() + # Snapshot before decryption so a concurrent replacement cannot become our baseline later. + from copy import deepcopy + raw_bucket = session.runtime.get(kc.KEY) or {} + original = deepcopy((raw_bucket.get("entries") or {}).get(str(entry_id))) + if not isinstance(original, dict): + raise err(409, "credential_changed", + "the stored credential changed while replacement was prepared. Reload it.") try: current_fields = kc.read_fields(session.runtime, entry_id) + except kc.KeychainEnvelopeVersionError: + raise err(409, "credential_envelope_unsupported", + "This credential uses an unsupported encryption envelope version. Nothing was changed.") except kc.KeychainLocked as exc: raise err(503, "keychain_locked", f"the keychain is locked: {exc}. Nothing was changed.") @@ -2924,12 +3073,9 @@ def _replace_connector_key(connector_key, entry_id, body, session): "the stored credential is incomplete. Nothing was changed.") merged_fields = {**current_fields, **replacements} - raw_bucket = session.runtime.get(kc.KEY) or {} - original = ((raw_bucket.get("entries") or {}).get(str(entry_id))) - if not isinstance(original, dict): - raise err(409, "credential_changed", - "the stored credential changed while replacement was prepared. Reload it.") - original = dict(original) + if entry_type == "amazon_sp": + # The previous secret's deadline does not describe its replacement. Absence is unknown. + merged_fields[AMAZON_ROTATION_DEADLINE_FIELD] = supplied.get(AMAZON_ROTATION_DEADLINE_FIELD, "") try: encrypted = _encrypted_replacement_payload( kc, entry.get("label"), entry_type, merged_fields, session.uname) @@ -2948,10 +3094,10 @@ def _replace_connector_key(connector_key, entry_id, body, session): if not isinstance(stored, dict) or stored != original: raise _ReplacementConflict() replacement = dict(stored) - replacement["enc"] = encrypted["enc"] - replacement["preview"] = encrypted["preview"] - entries[str(entry_id)] = replacement - return cur + replacement.update(encrypted) + replacement.pop("probe", None) + # Copy on write: a store failure after this callback must not mutate its cached old row. + return {**cur, "entries": {**entries, str(entry_id): replacement}} try: session.runtime.update(kc.KEY, _swap, flush="sync") diff --git a/api/routes_security.py b/api/routes_security.py new file mode 100644 index 0000000000000000000000000000000000000000..36b4cdae5258e65584391fb69c66cbe79b477c38 --- /dev/null +++ b/api/routes_security.py @@ -0,0 +1,501 @@ +"""Deployment posture read-back, not edge provisioning or an independent network probe. + +Ingress observations describe this request and are reported against every mounted public +route. TLS termination hides the negotiated version from this process: HTTPS plus an +explicit minimum declaration is the limited evidence accepted here, only with bounded +proxy trust. No declaration substitutes for the four missing infrastructure producers. +""" +from dataclasses import dataclass +from functools import cached_property +import ast +import hmac +import inspect +import json +import os +from pathlib import Path +import re +import time +from typing import Callable + +from fastapi import APIRouter, Depends, Request, Response +from starlette.routing import Mount +from starlette.staticfiles import StaticFiles + +import aios_session +from deps import Session, err +from core import audit +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1/security") +POSTURE_STATES = ("met", "not_met", "unknown") + + +def _answer(state, evidence): + return {"state": state, "evidence": evidence} + + +@dataclass +class Observation: + request: object + tenant: str + + @cached_property + def security_events(self): + # One read per payload, never one read per route and never a process-wide cache. + if not audit.AUDIT_KINDS: + return _answer("not_met", "The audit module declares no security event kinds.") + try: + audit.read(self.tenant, limit=1) + except Exception: + # A store exception can carry secrets; neither its text nor its values escape. + return _answer("unknown", "The tenant security event document could not be read.") + return _answer("met", "The audit module declares event kinds and the tenant event document was read.") + + +def tls_min_version(observation, route): + """Read scheme plus declared minimum, never claim to observe negotiated TLS version. + + Uvicorn owns forwarded-header validation. Reading the raw forwarded header ourselves + would bypass its peer check. An absent, empty or wildcard allow-list takes precedence + over scheme because even the apparent HTTPS scheme can then be client supplied. + """ + allowed = os.environ.get("FORWARDED_ALLOW_IPS", "").strip() + if not allowed or "*" in allowed: + return _answer("unknown", "The forwarded scheme cannot be trusted under a wildcard allow-list, including an absent or empty declaration.") + scheme = getattr(getattr(observation.request, "url", None), "scheme", None) + if scheme in ("http", "ws"): + return _answer("not_met", "The request scheme reports plain HTTP without TLS.") + if scheme not in ("https", "wss"): + return _answer("unknown", "No HTTPS request scheme is available for the TLS control.") + declared = os.environ.get("AIOS_TLS_MIN_VERSION", "").strip() + if not re.fullmatch(r"[0-9]+\.[0-9]+", declared): + return _answer("unknown", "The request reports HTTPS but the minimum TLS version declaration is absent or unrecognised.") + # Compare integer components, not a float (a minor version need not be one digit). + minimum = tuple(int(part) for part in declared.split(".")) + if minimum < (1, 2): + return _answer("not_met", "The declared minimum TLS version is below the required minimum.") + return _answer("met", "The request reports HTTPS with bounded proxy trust and a declared TLS minimum of at least 1.2; the negotiated version is not observable.") + + +def origin_bypass_prevented(observation, route): + declared = os.environ.get("AIOS_ORIGIN_TOKEN", "") + if not declared.strip(): + return _answer("unknown", "No shared origin token is declared for the origin control.") + headers = getattr(observation.request, "headers", {}) + supplied = headers.get("x-aios-origin-token", "") + if not supplied or not hmac.compare_digest(supplied.encode("utf-8"), declared.encode("utf-8")): + return _answer("not_met", "The request did not carry the declared shared origin token.") + return _answer("met", "The request carried the declared shared origin token; this is request ingress evidence.") + + +def waf_in_path(observation, route): + return _answer("unknown", "A WAF association and request inspection read-back producer is required.") + + +def firewall_acl(observation, route): + return _answer("unknown", "A firewall or ACL rule read-back producer for the serving network is required.") + + +def network_segmentation(observation, route): + return _answer("unknown", "A network topology and isolation rule read-back producer is required.") + + +def intrusion_detection_monitored(observation, route): + return _answer("unknown", "A detection service and active monitoring read-back producer is required.") + + +def security_events_written(observation, route): + return observation.security_events + + +@dataclass(frozen=True) +class Control: + id: str + label: str + producer: Callable + + +# The module-level seam is injectable without a server, live store or network. +CONTROL_ROSTER = ( + Control("tls_min_version", "TLS minimum", tls_min_version), + Control("waf_in_path", "Web application firewall", waf_in_path), + Control("origin_bypass_prevented", "Origin protection", origin_bypass_prevented), + Control("firewall_acl", "Firewall rules", firewall_acl), + Control("network_segmentation", "Network segmentation", network_segmentation), + Control("intrusion_detection_monitored", "Intrusion monitoring", intrusion_detection_monitored), + Control("security_events_written", "Security events", security_events_written), +) +_CONTROL_IDS = tuple(control.id for control in CONTROL_ROSTER) + + +def internet_routes(app): + """Walk original_router recursively, as verify_api._mounted_paths does. + + Also include direct app routes and mounted apps. FastAPI's include_context carries + the actual include prefix; the nested router's own prefix is already in its leaf + paths. Exclude only StaticFiles mounts and the GET/HEAD /api/health liveness route. + Opaque mounts and routes of uncertain exposure remain in the inventory. + """ + paths = [] + + def walk(owner, base): + for route in getattr(owner, "routes", ()): + nested = getattr(route, "original_router", None) + if nested is not None: + context = getattr(route, "include_context", None) + walk(nested, base + getattr(context, "prefix", "")) + continue + path = base + (getattr(route, "path", None) or "") + if isinstance(route, Mount): + if isinstance(route.app, StaticFiles): + continue + if getattr(route.app, "routes", None): + walk(route.app, path) + continue + path += "/{path:path}" + methods = sorted(getattr(route, "methods", None) or ( + ["WEBSOCKET"] if "WebSocket" in type(route).__name__ else ["UNKNOWN"])) + if path == "/api/health" and set(methods) <= {"GET", "HEAD"}: + continue + paths.append({"path": path or "/{path:path}", "methods": methods}) + + walk(app, "") + return paths + + +def posture_allowed(controls): + """Unknown, malformed, missing and duplicate controls all refuse the roll-up.""" + return (len(controls) == len(_CONTROL_IDS) + and {control["id"] for control in controls} == set(_CONTROL_IDS) + and all(control.get("state") == "met" for control in controls)) + + +def build_posture(request, tenant): + observation = Observation(request, tenant) + routes = [] + for route in internet_routes(getattr(request, "app", None)): + controls = [] + for control in CONTROL_ROSTER: + try: + answer = control.producer(observation, route) + if answer["state"] not in POSTURE_STATES: + raise ValueError("Unrecognised posture state") + # Producers return fixed source descriptions, never raw headers or env values. + audit._string("evidence", answer["evidence"]) + except Exception: + answer = _answer("unknown", "The control producer did not return usable read-back evidence.") + controls.append({"id": control.id, "label": control.label, + "state": answer["state"], "evidence": answer["evidence"]}) + routes.append({**route, "controls": controls, "allowed": posture_allowed(controls)}) + failing = [key for key in _CONTROL_IDS if not routes or any( + not any(c["id"] == key and c["state"] == "met" for c in route["controls"]) + for route in routes)] + return {"routes": routes, "allowed": bool(routes) and all(r["allowed"] for r in routes), + "failing_control_ids": failing, + "evidence_scope": "Ingress evidence describes this request, not separate probes of every route.", + "route_evidence": ("The application route tree was read at request time." if routes else + "No internet-facing route inventory is available; enablement is refused.")} + + +def _principal(request, tenant): + if request is None: + return "system:amazon-posture" + try: + claims = aios_session.read(request.cookies.get(aios_session.COOKIE_NAME)) + except (TypeError, ValueError, OverflowError): + claims = None + if claims and claims["t"] == tenant: + return claims["u"] + return "unresolved:amazon-posture" + + +def require_amazon_posture(request, tenant) -> None: + """503 means a deployment precondition is unmet, not bad caller input. + + Enablement callers must authenticate first. Boot sync may pass request=None, which + refuses because there is no request ingress or route evidence; never fabricate either. + Only refused attempts emit an event. The audit writer owns store failures, and only + AuditRefused is caught here, matching routes_auth's boundary convention. + """ + posture = build_posture(request, tenant) + if posture["allowed"]: + return None + failing = posture["failing_control_ids"] or list(_CONTROL_IDS) + try: + audit.record("config.changed", tenant, _principal(request, tenant), + {"setting": "amazon_enablement", "allowed": False, + "failing_controls": ", ".join(failing)}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") + raise err(503, "amazon_posture_unmet", + "Amazon enablement refused: unmet deployment controls: " + ", ".join(failing) + ".") + + +@router.get("/posture") +def security_posture(request: Request, session: Session = Depends(admin_gate)): + return build_posture(request, session.tenant) + + +COMPLIANCE_STATES = {"met": "Met", "not_met": "Not met", "unproven": "Unproven"} +_APPROVED_ROLES = ( + ("role-inventory", "Inventory and Order Tracking"), + ("role-fulfillment", "Amazon Fulfillment"), + ("role-awd", "Amazon Warehousing and Distribution"), + ("role-finance", "Finance and Accounting"), + ("role-pricing", "Pricing"), + ("role-listing", "Product Listing"), + ("role-analytics", "Brand Analytics"), + ("role-insights", "Selling Partner Insights"), +) +# RD10 is deliberately unavailable until production access and QI13 verification exist. +# These are the six CREDENTIAL_RED labels, not imports of a verification executable. +_PRODUCTION_CHECKS = ( + ("production-stored", "[live] a stored amazon_sp credential resolves for tenant #0"), + ("production-authentication", "[live] the credential authenticates against marketplaceParticipations"), + ("production-sync", "[live] every declared table has synced at least once"), + ("production-roles", "[live] Pricing and AWD role operations completed without refusal"), + ("production-reconciliation", "[live] validate reconciles each table to Amazon's own summary"), + ("production-brand", "[live] brand_analytics returns rows for a registered brand (R5's exit condition)"), +) + + +def _compliance_read(producer, missing): + try: + return producer() + except audit.AuditRefused: + # A secret-shaped result refuses the response; it must never become a redacted claim. + raise + except Exception: + # Exception text may contain a connection string or a credential value. + return _answer("unproven", missing) + + +def _authentication_control(): + from core import users + from routes_auth import login + + policy = (users.PASSWORD_MIN_LENGTH, users.PASSWORD_HISTORY, + users.PASSWORD_MIN_AGE_DAYS, users.PASSWORD_MAX_AGE_DAYS) + rules = {rule["rule"] for rule in users.password_rules()} + function = ast.parse(inspect.getsource(login)).body[0] + parents = {child: node for node in ast.walk(function) for child in ast.iter_child_nodes(node)} + # T33's scope predicate: extra role guards, else arms and early cookie mints cannot pass. + expected = {"users.totp_valid": ["not user.get('qa_runner')"], + "users.consume_recovery_code": ["not user.get('qa_runner')", "not factor_valid"]} + factors = [] + covered = True + for name, guards in expected.items(): + calls = [node for node in ast.walk(function) + if isinstance(node, ast.Call) and ast.unparse(node.func) == name] + if len(calls) != 1: + covered = False + continue + factors.extend(calls) + ancestors, node = [], calls[0] + while node in parents: + parent = parents[node] + if isinstance(parent, ast.If): + covered = covered and node in parent.body + ancestors.insert(0, ast.unparse(parent.test)) + node = parent + covered = covered and ancestors == guards + first = min((node.lineno for node in factors), default=0) + covered = covered and not any(getattr(node, "lineno", 0) < first and ( + isinstance(node, ast.Return) or isinstance(node, ast.Call) + and ast.unparse(node.func) in {"aios_session.mint", "aios_session.set_cookie"}) + for node in ast.walk(function)) + refusals = {ast.unparse(node.test): node for node in ast.walk(function) + if isinstance(node, ast.If)} + for guard, code in (("not users.mfa_enrolled(rec)", "mfa_enrolment_required"), + ("not code", "mfa_required"), + ("not (factor_valid or recovery_code_used)", "mfa_invalid")): + block = refusals.get(guard) + covered = covered and block is not None and any( + isinstance(node, ast.Raise) and any(isinstance(value, ast.Constant) and value.value == code + for value in ast.walk(node)) + for node in ast.walk(block)) + met = (policy == (12, 10, 1, 90) and covered and + rules >= {"length", "uppercase", "lowercase", "digit", "symbol", "identifier"}) + return _answer("met" if met else "not_met", ( + "Runtime policy reads 12 characters, ten prior passwords and 1/90 day age limits; the inspected login block requires a factor for stored accounts with a QA identity exception." + if met else "The runtime password policy or inspected login factor block does not match the authentication baseline.")) + + +def _telemetry_control(tenant): + audit.read(tenant, limit=1) + return _answer("met", "The tenant audit document was read through the security event reader; event contents are excluded.") + + +def _role_control(role): + from harness.amazon_client import ROLES_TAKEN + + for name in ROLES_TAKEN: + audit._string("role", name) + exact = (len(ROLES_TAKEN) == len(_APPROVED_ROLES) + and set(ROLES_TAKEN) == {name for _, name in _APPROVED_ROLES}) + if role not in ROLES_TAKEN: + return _answer("not_met", "The live connector role allow-list does not include this approved role.") + if not exact: + return _answer("not_met", "The live connector role allow-list differs from the eight approved roles.") + return _answer("met", "The live connector role allow-list includes this role; production authorization is unavailable.") + + +def _audit_retention_control(tenant): + seconds = audit.RETENTION_SECONDS + if type(seconds) is not int or seconds <= 0: + return _answer("not_met", "The audit reader has no positive retention interval.") + audit.read(tenant, limit=1) + return _answer("met", f"The live audit reader filters events older than {seconds} seconds; stored events are pruned on writes, not by an Amazon data retention job.") + + +def _vulnerability_control(): + # T35 writes an npm dependency inventory, which is not a vulnerability scan result. + with Path("/deps/web-deps.json").open(encoding="utf-8") as handle: + inventory = json.load(handle) + if not isinstance(inventory, dict) or not isinstance(inventory.get("dependencies"), dict): + return _answer("unproven", "The dependency inventory is unreadable as an inventory and no current vulnerability scan producer is available.") + return _answer("unproven", "The dependency inventory was read, but no current vulnerability scan producer is available.") + + +def _posture_compliance_control(posture, control_id): + routes = posture["routes"] + answers = [next(control for control in route["controls"] if control["id"] == control_id) + for route in routes] + if not answers: + return _answer("unproven", "No request-time internet route inventory is available for this control.") + # Keep every child row; an unknown child prevents a green category roll-up. + states = {answer["state"] for answer in answers} + state = "unproven" if "unknown" in states else "not_met" if "not_met" in states else "met" + if not states <= set(POSTURE_STATES): + raise ValueError("Unrecognised posture state") + if state == "met" and control_id in posture["failing_control_ids"]: + return _answer("unproven", "The posture roll-up and route control observations disagree.") + target = "unknown" if state == "unproven" else state + evidence = next(answer["evidence"] for answer in answers if answer["state"] == target) + return _answer(state, evidence) + + +def read_compliance_controls(request, tenant): + """Live observations only, with one posture read and isolated producer failures.""" + observations = { + "q1": _compliance_read(_authentication_control, "The runtime password policy or login factor block could not be read."), + "q5": _compliance_read(lambda: _telemetry_control(tenant), "The tenant audit document could not be read."), + "retention-audit": _compliance_read(lambda: _audit_retention_control(tenant), "The audit retention interval or tenant audit document could not be read."), + "vulnerability": _compliance_read(_vulnerability_control, "The dependency inventory at /deps/web-deps.json could not be read and no current vulnerability scan producer is available."), + } + posture = _compliance_read(lambda: build_posture(request, tenant), "The deployment posture could not be read.") + for control in CONTROL_ROSTER: + observations["posture-" + control.id] = _compliance_read( + lambda key=control.id: _posture_compliance_control(posture, key), + "The request-time posture observation for this control could not be read.") + for key, role in _APPROVED_ROLES: + observations[key] = _compliance_read(lambda name=role: _role_control(name), + "The live connector role allow-list could not be read.") + return observations + + +def _check_compliance_strings(claims): + for claim in claims: + for key, value in claim.items(): + if isinstance(value, str): + audit._string("compliance field", value) + if chr(0x2014) in value or chr(0x2013) in value or "\n" in value or "\r" in value: + raise audit.AuditRefused("compliance field", "unsupported prose or line break") + if claim["state"] not in COMPLIANCE_STATES: + raise ValueError("Unrecognised compliance state") + + +def compliance_roster(observations): + """Pure roster assembly; absent observations never default to affirmative evidence.""" + claims = [] + + def add(key, group, claim, source=None, missing="No producing control is available.", answer=None): + observed = answer if answer is not None else observations.get(key) if source is not None else None + observed = observed if observed is not None else _answer("unproven", missing) + evidence = observed["evidence"] + if group == "Question predicates": + evidence = evidence.removesuffix(".") + f"; verbatim {key.upper()} text is unavailable to this deployment." + claims.append({"id": key, "group": group, "claim": claim, + "state": observed["state"], "evidence": evidence, "source": source}) + + add("q1", "Question predicates", "Q1: Authentication and MFA", "aios-web/api/routes_auth.py:login", + "The runtime password policy or login factor block could not be read.") + add("q2", "Question predicates", "Q2: Role and scope", + missing="No production role authorization and scope read-back producer is available.") + add("q3", "Question predicates", "Q3: Credential handling", + missing="No end-to-end credential handling compliance read-back producer is available.") + add("q4", "Question predicates", "Q4: Information lifecycle", + missing="No Amazon data retention job or restore-residue observation is available.") + add("q5", "Question predicates", "Q5: Telemetry", "platform/core/audit.py:read", + "The tenant audit document could not be read.") + posture_children = [observations.get("posture-" + key, _answer("unproven", "Missing control.")) + for key in _CONTROL_IDS] + children = [*posture_children, observations.get("vulnerability", _answer("unproven", "Missing scan."))] + state = ("unproven" if any(row["state"] not in ("met", "not_met") for row in children) + else "not_met" if any(row["state"] == "not_met" for row in children) else "met") + unproven = sum(row["state"] not in ("met", "not_met") for row in children) + not_met = sum(row["state"] == "not_met" for row in children) + add("q6", "Question predicates", "Q6: Supply chain and posture", "aios-web/api/routes_security.py:build_posture", + answer=_answer(state, f"The posture and vulnerability reads leave {unproven} of {len(children)} controls unproven and {not_met} not met; each control is listed below.")) + add("q7", "Question predicates", "Q7: Compliance evidence", + missing="No authoritative questionnaire or reviewed compliance evidence producer is available.") + for control_id, label in (("tls_min_version", "TLS minimum"), ("waf_in_path", "Web application firewall"), + ("origin_bypass_prevented", "Origin protection"), ("firewall_acl", "Firewall rules"), + ("network_segmentation", "Network segmentation"), + ("intrusion_detection_monitored", "Intrusion monitoring"), + ("security_events_written", "Security events")): + add("posture-" + control_id, "Deployment posture", label, + "aios-web/api/routes_security.py:" + control_id, + "The request-time posture observation for this control is unavailable.") + for key, role in _APPROVED_ROLES: + add(key, "Approved roles", role, "platform/harness/amazon_client.py:ROLES_TAKEN", + "The live connector role allow-list could not be read.") + add("retention-audit", "Information lifecycle", "Audit event retention", "platform/core/audit.py:RETENTION_SECONDS", + "The audit retention interval or tenant audit document could not be read.") + add("retention-amazon", "Information lifecycle", "Amazon data retention", + missing="No Amazon data retention job enforces the declared limit of 30 days after order delivery.") + add("restore", "Information lifecycle", "Restore residue", + missing="No restore-residue read-back control is available; purge helpers alone do not prove a clean restore.") + add("vulnerability", "Operational readiness", "Vulnerability state", "aios-web/api/routes_security.py:_vulnerability_control", + "No current vulnerability scan producer is available.") + add("incident", "Operational readiness", "Incident state", + missing="No incident status read-back producer is available.") + for key, label in _PRODUCTION_CHECKS: + add(key, "Production checks", label, + answer=_answer("unproven", "Unavailable: Amazon production access has not been granted and this live verification is deferred under RD10 and QI13.")) + _check_compliance_strings(claims) + return claims + + +def build_compliance(request, tenant): + return compliance_roster(read_compliance_controls(request, tenant)) + + +def render_compliance_export(claims): + _check_compliance_strings(claims) + lines = ["Amazon compliance", ""] + for claim in claims: + lines.extend([f"{claim['id']} | {claim['group']} | {COMPLIANCE_STATES[claim['state']]}", + claim["claim"], claim["evidence"], "Source: " + (claim["source"] or "No producing control"), ""]) + return "\n".join(lines) + + +@router.get("/compliance") +def security_compliance(request: Request, response: Response, session: Session = Depends(admin_gate)): + response.headers["Cache-Control"] = "no-store" + return {"claims": build_compliance(request, session.tenant), "generated_at": time.time()} + + +@router.get("/compliance/export") +def security_compliance_export(request: Request, session: Session = Depends(admin_gate)): + # Both doors call build_compliance once; the renderer takes that roster, so file and pane cannot drift. + body = render_compliance_export(build_compliance(request, session.tenant)) + try: + audit.record("config.changed", session.tenant, session.uname, + {"setting": "amazon_compliance_export", "action": "download"}) + except audit.AuditRefused as exc: + print(f"[aios-api] {exc}") + return Response(body, media_type="text/plain", headers={ + "Content-Disposition": 'attachment; filename="amazon-compliance.txt"', + "Cache-Control": "no-store", + }) diff --git a/api/routes_sheet.py b/api/routes_sheet.py index a763843d106bada64acd62447e2d3066fda27f65..fb6e7d8c9fb3477d656367473a775f45dfe05e29 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -38,6 +38,7 @@ while its surface is a 404. import hashlib import json import os +import re import tempfile import threading import uuid @@ -815,7 +816,8 @@ def _reresolved_for_reader(session: Session, book): `put_workbook`, and a read path that copied that body would write the READER's numbers into the OWNER's document: the same leak as serving stale owner values, in the opposite direction, caused by a GET, and therefore reachable by anybody the workbook was shared with. So this - works on a `copy.deepcopy` and returns it, and nothing here touches the store. + works on a `copy.deepcopy` and returns it without saving the reader's workbook values. + R10 may create a missing Metric field through the database's ordinary field writer. ⭐ THE WALL RUNS ONCE PER DISTINCT DATABASE, exactly as `refresh_sheet` does (R16): a reader who has lost access to a bound database is refused on every cell of it rather than shown the @@ -861,6 +863,7 @@ def _reresolved_for_reader(session: Session, book): if failure is not None: cell["state"] = "refused" cell["value"] = None + cell["display"] = "" cell["refusal"] = {"code": "database_unreadable", "message": f"You cannot read {db}."} cell.pop("limits", None) @@ -893,6 +896,7 @@ def _reresolved_for_reader(session: Session, book): info = detail.get("error") if isinstance(detail, dict) else None cell["state"] = "refused" cell["value"] = None + cell["display"] = "" cell["refusal"] = { "code": (info or {}).get("code", "sheet_unresolved"), "message": (info or {}).get("message", "This cell could not be refreshed."), @@ -1795,6 +1799,123 @@ def _formula_message(code, subject=""): }.get(code, "This formula could not be calculated.") +_METRIC_COLUMN_CREATE_LOCK = threading.RLock() + + +def _forget_sheet_database(session, database): + """A field write invalidates both request-local schema and materialised-column memos.""" + getattr(session, "_sheet_assemblies", {}).pop(database, None) + mirrors = getattr(session, "_sheet_mirrors", {}) + for slot in list(mirrors): + if slot[0] == database: + mirrors.pop(slot, None) + + +def _metric_column_aggregate(session, database, aggregate, window): + """R10: admit the measure, reuse its column, or create through the ordinary event door. + + The caller has passed the database wall. This writes a field, never a record: a locked + connector database still permits adding fields (DESIGN.md section 4). The existing handler + owns validation, creator stamps, shared persistence and grant bookkeeping. + """ + import aios_grid + from core import grid_events, shared_overlay + from harness import measure_filter, semantic, windows + from modules import sheet as sheet_mod + from routes_grid import _ctx + + key = aggregate["metric_key"] + + def refuse(code, subject=key): + exc = sheet_mod.SheetRefusal(code, subject) + raise err(409, exc.code, exc.message) + + # These are the two measure-backed registry mirrors. Never fall through to the customer + # assembly for another database: that would create a field on the wrong database. + if database not in ("customer_data", "product_data"): + refuse("measure_not_offered") + g = _registry_assembly(session, database) + # Customer assembly gates measure_fields(team_id) on may_metrics. Intersect the catalogue + # with the served offer too: product has a narrower entity catalogue, and revoked Metric + # capability makes the account's offer empty. A browser key can never widen either offer. + visible_measures = {m["key"] for m in g.get("measures") or ()} + offered = {m["key"]: m for m in measure_filter.measure_fields(g.get("team_id")) + if m["key"] in visible_measures} + if key not in offered: + refuse("measure_not_offered") + label = offered[key].get("label") or key + try: + fold = sheet_mod.metric_key_fold(label, (semantic.metrics().get(key) or {}).get("agg")) + except sheet_mod.SheetRefusal as exc: + raise err(409, exc.code, exc.message) + window = windows.normalize(window) + if window is None: + # An absent period or unresolved period.ref must never select the column's default. + refuse("metric_period_required", label) + + def existing(fields): + candidates = [f for f in fields or () if isinstance(f, dict) and f.get("key") + and isinstance(f.get("measure"), dict) and f["measure"].get("key") == key] + wanted = sheet_mod.measure_column_key(key, window) + return next((f for f in candidates if sheet_mod.measure_column_key( + key, windows.normalize(f["measure"].get("window"))) == wanted), + candidates[0] if candidates else None) + + def column(field): + return {"kind": "column", "fn": fold, "field": field["key"]} + + found = existing(g.get("fields")) + if found is not None: + return column(found) + + # One column per measure, not per cell period. LTM is only the column's declared DEFAULT; + # _mirror(metric_window=window) re-resolves it at the period each cell actually names. + # Hash the full key so slug truncation does not alias measures, and check occupied keys + # before writing so even a hash collision cannot silently redefine an existing field. + slug = re.sub(r"[^a-z0-9_]+", "_", key.lower()).strip("_")[:40] or "metric" + suffix = hashlib.sha256(key.encode("utf-8")).hexdigest()[:12] + raw = {"key": aios_grid.MEASURE_FIELD_PREFIX + slug + "_" + suffix, + "label": label, "measure": {"key": key, "window": {"kind": "ltm"}}, + "permissions": {"edit": "collaborative"}} + if aios_grid.clean_measure_field(raw, offered) is None: + refuse("measure_not_offered", label) + + # A second request may have assembled before the first write. Serialise automatic creates + # in this process and re-read occupied keys through the scoped assembly before reusing one. + with _METRIC_COLUMN_CREATE_LOCK: + try: + ctx = _ctx(session, g["fields"], g["pids"], + scope_key="product" if database == "product_data" else "customer", + measure_keys=frozenset(offered), measure_offer=tuple(offered.values())) + storage_key, _topic = grid_events._field_share_keys(ctx) + stored = shared_overlay.fields(storage_key, st=session.runtime) + personal = grid_events.table_workspace( + ctx, allowed_pids=g["pids"], consume_corrections=False).get("fields") or {} + if (raw["key"] in stored or raw["key"] in personal or raw["key"] in ctx.hidden_keys + or raw["key"] in {f.get("key") for f in g["fields"]}): + _forget_sheet_database(session, database) + found = existing(_registry_assembly(session, database).get("fields")) + if found is not None: + return column(found) + refuse("metric_column_conflict", label) + try: + # Event objects use `type`, not the handler's local variable name `kind`. + applied = grid_events.handle_one({"id": "sheet-metric-" + uuid.uuid4().hex, + "type": "field_upsert", "field": raw}, ctx) + finally: + _forget_sheet_database(session, database) + except grid_events.StoreUnavailable: + exc = sheet_mod.SheetRefusal("metric_column_create_failed", label) + raise err(503, exc.code, exc.message) + # A True handler result can also mean "repaint the refusal"; inspect both channels. + if ctx.out.refusals: + failure = ctx.out.refusals[0] + raise err(409, failure["code"], failure["message"]) + if not applied: + refuse("metric_column_create_failed", label) + return column(raw) + + def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None): """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`. @@ -1822,14 +1943,13 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non except sheet_mod.SheetRefusal as exc: raise err(400, exc.code, exc.message) - # ⛔ C2's OTHER AGGREGATE ARM IS ANSWERED HERE, NOT LEFT TO FALL THROUGH TO `aggregate_sql`, - # where it died as `field_unknown` with an empty subject. See `metric_key_refusal`. - metric_refusal = sheet_mod.metric_key_refusal(binding["aggregate"]) - if metric_refusal is not None: - raise err(400, metric_refusal.code, metric_refusal.message) - database = _require_database(session, binding["database"]) window = sheet_mod.period_window(binding.get("period")) + # W59-T14 / R10: the per-database wall is contract step 1. A metric key must not be looked + # up or acted on before the caller passes it; the ordinary column guards below still apply. + if binding["aggregate"].get("metric_key"): + binding["aggregate"] = _metric_column_aggregate( + session, database, binding["aggregate"], window) # ⭐⭐ W54-T36 — THE WALL AND THE SCOPE ARE READ BEFORE THE MIRROR IS OPENED, so the mirror can # be built over the columns this cell can possibly read instead of over the whole database. @@ -1864,9 +1984,8 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non raise err(409, agg_refusal.code, agg_refusal.message) if bound_field and bound_field.get("measure") and not binding.get("period"): label = bound_field.get("label") or bound_field.get("key") - raise err(409, "metric_period_required", - f"{label} is measured over a period, so this cell needs one. " - "Open the cell's Period section and choose a period.") + refusal = sheet_mod.SheetRefusal("metric_period_required", label) + raise err(409, refusal.code, refusal.message) # ── the ROW wall, fail-closed ──────────────────────────────────────────────────────────── if wall_nodes: @@ -1946,8 +2065,8 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None), """Evaluate a C2 binding WITHOUT saving it (contract C1). ⭐ It takes the binding from the BODY rather than from the stored cell, which is what lets the - binding editor show a number before anyone commits to it, and what lets this door be exercised - on a tenant whose store is read-only. + binding editor show a number before anyone commits the cell. R10 may create a missing Metric + field through the database's ordinary field writer; reusing a column needs no field write. """ raw = dict(body or {}) out = _resolve_binding(session, raw.get("binding")) diff --git a/api/routes_templates.py b/api/routes_templates.py index 227355e69e0a549d5659e35c9aa4243c7a5304a7..ca5b6745664c79131a426c452d9c595caaca3361 100644 --- a/api/routes_templates.py +++ b/api/routes_templates.py @@ -166,3 +166,93 @@ def apply_template(key: str, body: dict = Body(default=None), alert_id = None return {"key": key, "table": body.get("table"), "views": applied, "alert": alert_id, "alerted": bool(alert_id)} + + +def _store(session: Session): + return session.runtime + + +@router.get("/sheet-templates") +def list_sheet_templates(session: Session = Depends(require_session)): + """The workbook gallery, including sources this tenant has not connected.""" + from core import sheet_templates, user_tables + from fastapi import HTTPException + from routes_sheet import _require_database + + definitions = user_tables.all_defs(_store(session)) + databases = [] + for database_key, definition in definitions.items(): + if sheet_templates.source_of_database(database_key) is None: + continue + try: + _require_database(session, database_key) + except HTTPException as exc: + if exc.status_code in (403, 404): + continue + raise + databases.append({"key": database_key, "label": definition.get("label") or database_key}) + return {"templates": [ + {"key": descriptor["key"], "title": descriptor["title"], + "answers": descriptor["answers"], "source": descriptor["source"], + "databaseHint": descriptor["database_hint"], + "databases": [database for database in databases + if sheet_templates.source_of_database(database["key"]) == descriptor["source"]]} + for descriptor in sheet_templates.TEMPLATES + ]} + + +@router.post("/sheet-templates/{key}/apply") +def apply_sheet_template(key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Materialise and calculate a workbook through the existing Sheet engine.""" + from core import sheet_templates, sheets, user_tables + from modules.sheet import SheetRefusal as BindingRefusal + from routes_sheet import _require_database, refresh_sheet + + descriptor = sheet_templates.template(key) + if descriptor is None: + raise err(404, "unknown_sheet_template", "That sheet template does not exist.") + body = body if isinstance(body, dict) else {} + database_key = body.get("database") + name = body.get("name", "") + if not isinstance(database_key, str) or not database_key.strip(): + raise err(404, "unknown_database", "That database does not exist.") + if not isinstance(name, str): + raise err(400, "bad_request", "The workbook name must be text.") + database_key = database_key.strip() + st = _store(session) + if database_key not in user_tables.all_defs(st): + raise err(404, "unknown_database", "That database does not exist.") + _require_database(session, database_key) + if sheet_templates.source_of_database(database_key) != descriptor["source"]: + raise err(400, "template_source_mismatch", + "That database uses a different connector from this template.") + + book = None + try: + book = sheet_templates.apply_template(st, key, database_key, name=name, + created_by=session.uname) + # Owners read their persisted values. Use the same permission-scoped + # refresh as Sheet so the first open already contains computed cells. + refreshed = refresh_sheet(book["id"], session=session)["workbook"] + failures = [] + for tab in refreshed["sheets"]: + for addr, cell in tab["cells"].items(): + if cell.get("state") == "refused" or cell.get("error") or cell.get("refusal"): + sentence = (cell.get("refusal") or {}).get("message") + failures.append(tab["name"] + "!" + addr + (": " + sentence if sentence else "")) + if failures: + raise sheet_templates.TemplateRefusal( + "template_cells_refused", + detail="The template could not calculate cells: " + "; ".join(failures) + ".") + return {"workbook": refreshed} + except Exception as exc: + if book is not None: + sheets.delete_workbook(st, book["id"]) + if isinstance(exc, sheet_templates.TemplateRefusal): + status = {"unknown_sheet_template": 404, "unknown_database": 404, + "template_source_mismatch": 400}.get(exc.code, 409) + raise err(status, exc.code, exc.sentence()) from exc + if isinstance(exc, (sheets.SheetRefusal, BindingRefusal)): + raise err(409, exc.code, exc.message) from exc + raise diff --git a/api/test_odoo_rank_refresh.py b/api/test_odoo_rank_refresh.py index 9b304ffb9971690ee33dc31e39fec9165a67ceea..4215d9296dd890ccf90a3cba957f99a14dda04a3 100644 --- a/api/test_odoo_rank_refresh.py +++ b/api/test_odoo_rank_refresh.py @@ -139,7 +139,15 @@ class RankRefreshTests(unittest.TestCase): _resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False, tenant='synthetic', tenantSource='test', tenantSeconds=60, envSeconds=60, disagreement=None), + # ⛔ THIS NAMESPACE IS HAND-MAINTAINED, so a NEW collaborator inside + # `_store_resync_loop` arrives here as a bare `NameError` from a synthetic + # module rather than as a readable failure. W59-T42 added `_pull_amazon` + # one line after `_pull_meta` at `main.py:1416`, and all four resident-loop + # tests died on it -- in the MERGED tree only: lane G does not run this + # gate and lane D's run predates lane G. The real `main.py` defines the + # function at module scope and the deployed Space boots fine. _resync_mirror_pass=sync, _pull_meta=lambda *_: True, + _pull_amazon=lambda *_: True, _rebuild_odoo_relational=rebuild, _rebuild_meta_relational=lambda *_: {}, _rebuild_amazon_relational=lambda *_: {}) diff --git a/platform/core/audit.py b/platform/core/audit.py new file mode 100644 index 0000000000000000000000000000000000000000..989948b65624e6ae60020819462a1675701544e1 --- /dev/null +++ b/platform/core/audit.py @@ -0,0 +1,176 @@ +"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O.""" +from collections.abc import Mapping +import math +import re +import time + +from harness import runtime + + +AUDIT_KINDS = frozenset({ + "auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled", + "auth.mfa_failed", "auth.logout", "admin.grant", "credential.added", + "credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled", + "amazon.authorization_lost", "data.purge", "data.restore", "config.changed", +}) +BUCKET = "security_events" +MAX_EVENTS = 2000 +RETENTION_SECONDS = 400 * 24 * 60 * 60 +_ATTRIBUTION_MAX = 256 +_KEY_MAX = 128 + + +def _key_name(key): + return "".join(c for c in key.casefold() if c.isalnum()) + + +# Restated from core.keychain._PREVIEW_FIELDS to avoid coupling a log to the keychain. +# Keep the first six names and that tuple in step; client_secret is also an Amazon secret. +SECRET_KEYS = frozenset(_key_name(key) for key in ( + "api_key", "access_token", "value", "password", "token", "refresh_token", + "client_secret", "secret", "private_key", "authorization", "cookie", "session", + "mfa_secret", "recovery_code", "otpauth", "credential", +)) +_BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE) +_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.") + + +class AuditRefused(ValueError): + """A call-site error carrying only the field name and refusal reason, never its value.""" + + def __init__(self, key, reason): + self.key = key + self.reason = reason + # Explaining a refusal with the value would put the secret into its traceback. + super().__init__(f"audit refused key {key!r}: {reason}") + + +def _string(key, value): + if len(value) > 512: + raise AuditRefused(key, "string exceeds 512 characters") + if value.startswith("Atzr|"): + raise AuditRefused(key, "Atzr refresh-token prefix") + if _BEARER.search(value): + raise AuditRefused(key, "bearer token shape") + if _JWT.search(value): + raise AuditRefused(key, "JWT token shape") + + +def _attribution(key, value): + if not isinstance(value, str) or not value.strip(): + raise AuditRefused(key, "attribution requires a non-empty string") + _string(key, value) + value = value.strip() + _string(key, value) + if len(value) > _ATTRIBUTION_MAX: + raise AuditRefused(key, "attribution exceeds 256 characters") + return value.lower() if key == "tenant" else value + + +def event(kind, tenant, principal, detail=None, *, now=None) -> dict: + """Validate without I/O; an explicit epoch makes this pure and deterministic for gates.""" + if not isinstance(kind, str) or kind not in AUDIT_KINDS: + raise AuditRefused("kind", "unknown audit kind") + tenant = _attribution("tenant", tenant) + principal = _attribution("principal", principal) + if detail is None: + detail = {} + if not isinstance(detail, Mapping): + raise AuditRefused("detail", "detail must be a flat mapping") + clean = {} + for key, value in detail.items(): + if not isinstance(key, str) or not key.strip(): + raise AuditRefused("detail", "detail keys must be non-empty strings") + # A malformed key can itself contain a token. Never echo that key in diagnostics. + _string("detail key", key) + if len(key) > _KEY_MAX or any(ord(c) < 32 for c in key): + raise AuditRefused("detail key", "key exceeds 128 characters or contains controls") + if _key_name(key) in SECRET_KEYS: + raise AuditRefused(key, "secret key name") + if isinstance(value, (Mapping, list, tuple, set, frozenset)): + raise AuditRefused(key, "nested value violates flat detail") + if type(value) not in (str, int, bool, float): + raise AuditRefused(key, "unsupported detail value type") + if isinstance(value, str): + _string(key, value) + if isinstance(value, float) and not math.isfinite(value): + raise AuditRefused(key, "detail number must be finite") + clean[key] = value + try: + ts = float(time.time() if now is None else now) + except (TypeError, ValueError, OverflowError): + raise AuditRefused("ts", "timestamp must be finite epoch seconds") from None + if not math.isfinite(ts): + raise AuditRefused("ts", "timestamp must be finite epoch seconds") + return {"ts": ts, "kind": kind, "tenant": tenant, "principal": principal, "detail": clean} + + +def ordered(doc, tenant, limit=200) -> list[dict]: + """Return ascending timestamps; a limit retains the most recent rows, still ascending. + + The stored tenant must match exactly even inside an already scoped runtime. This second + wall protects readers if a future migration ever puts multiple tenants in one document. + """ + tenant = _attribution("tenant", tenant) + limit = max(0, int(limit)) + if not limit or not isinstance(doc, Mapping): + return [] + rows = doc.get("events") + if not isinstance(rows, list): + return [] + retained = [] + for row in rows: + if not isinstance(row, dict) or row.get("tenant") != tenant: + continue + if type(row.get("ts")) not in (int, float): + # In particular, None must not ask event() for today's clock on a pure read. + continue + try: + # Revalidate persisted rows too, so legacy/corrupt documents cannot leak a secret. + retained.append(event(row.get("kind"), tenant, row.get("principal"), + row.get("detail"), now=row["ts"])) + except (AuditRefused, KeyError): + continue + return sorted(retained, key=lambda row: row["ts"])[-limit:] + + +def record(kind, tenant, principal, detail=None) -> dict | None: + """Refusals raise before I/O. Store failures lose the event, never the user's sign-in.""" + item = event(kind, tenant, principal, detail) + tenant = item["tenant"] + + def _append(doc): + # Prune and cap INSIDE the atomic update, as users.note_login_failure does. + rows = ordered(doc, tenant, MAX_EVENTS) + rows.append(item) + rows = [row for row in rows if row["ts"] >= item["ts"] - RETENTION_SECONDS] + return {"events": sorted(rows, key=lambda row: row["ts"])[-MAX_EVENTS:]} + + try: + # Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door. + # ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default + # flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits + # on the sign-in path: every login would then wait on a network round trip, and on the hf + # backend every login would spend one of that repo's 256 commits per hour. The async + # branch applies the mutation to the in-process cache and a coalescing worker commits it, + # which is the same posture the table-workspace hot path has used since wave 7. + # ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes + # loses the events still in its cache. That is the same window `record` already accepts by + # swallowing a store failure, and it is the right side of the trade: an audit log must + # never be the reason a person cannot sign in. + runtime.get_runtime(str(tenant)).update(BUCKET, _append, flush="async") + except Exception as exc: + # Exception text can contain credentials from a provider or connection string. + print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: " + f"store unavailable ({type(exc).__name__}).") + return None + return item + + +def read(tenant, limit=200, now=None) -> list[dict]: + """Read one tenant's retained events. Read failures propagate instead of claiming no history.""" + tenant = _attribution("tenant", tenant) + now = time.time() if now is None else float(now) + doc = runtime.get_runtime(str(tenant)).get(BUCKET) + rows = ordered(doc, tenant, limit) + return [row for row in rows if row["ts"] >= now - RETENTION_SECONDS] diff --git a/platform/core/keychain.py b/platform/core/keychain.py index 3e595010eaf85907eceee6cabe19fe9350b423c8..2cbe3c3a760bea3a86dfddee40b2e64fb1dcdefe 100644 --- a/platform/core/keychain.py +++ b/platform/core/keychain.py @@ -14,19 +14,26 @@ decrypts and writes are refused — fail closed, never a plaintext fallback. SHAPE (bucket `keychain`, per-tenant via the runtime handle): - {entries: {id: {id, connection_id, slot, label, type, enc, preview, created, createdBy}}} + {entries: {id: {id, connection_id, slot, label, type, enc, envelope_version, + preview, created, createdBy}}} -`enc` = Fernet token over the JSON fields dict. `preview` = a masked hint ("····1234") +Amazon rows also carry `client_secret_rotation_deadline`, empty when unknown, and may carry +the ciphertext-bound `probe` proof. `enc` = Fernet token over the JSON fields dict. +`preview` = a masked hint ("····1234") computed ONCE at write from the entry's most secret-looking field — the ONLY part of a secret that ever leaves this module through a route. `read_fields` exists for the CONNECTOR LAYER; no route returns its output. """ import datetime as _dt +import hashlib import json import os import secrets as _secrets KEY = 'keychain' +ORIGINAL_ENVELOPE_VERSION = 1 +ENVELOPE_VERSION = ORIGINAL_ENVELOPE_VERSION +AMAZON_ROTATION_DEADLINE_FIELD = 'client_secret_rotation_deadline' #: ⭐ WAVE 23 (C11/R8) — `stripe` and `shopify` joined. They are TOKEN connectors, not OAuth #: ones, and that is the whole reason they could ship in this wave: both issue a long-lived #: restricted key / custom-app token an admin pastes in, so they need no OAuth client, no @@ -69,6 +76,50 @@ class KeychainLocked(Exception): """Raised when AIOS_KEYCHAIN_KEY is absent/wrong — callers answer 503, never {}.""" +class KeychainEnvelopeError(ValueError): + """A stored credential envelope is malformed or unsupported.""" + + +class KeychainEnvelopeVersionError(KeychainEnvelopeError): + """An unsupported envelope is distinct from an unavailable encryption key.""" + + +def _check_envelope_version(row): + # Untagged rows use the original Fernet envelope without rewriting storage. + version = row.get('envelope_version', ORIGINAL_ENVELOPE_VERSION) + if type(version) is not int or version != ORIGINAL_ENVELOPE_VERSION: + raise KeychainEnvelopeVersionError('This credential uses an unsupported encryption envelope version.') + + +def _decoded_fields(raw): + try: + data = json.loads(raw.decode('utf-8')) + except Exception: + raise KeychainEnvelopeError('The stored credential payload is invalid.') from None + if not isinstance(data, dict) or not data: + raise KeychainEnvelopeError('The stored credential payload is invalid.') + return data + + +def _amazon_rotation_deadline(value): + """Validate an Amazon-issued timestamp supplied by the admin, never infer an expiry.""" + if value is None or value == '': + return '' + message = 'Enter the Amazon-issued secret rotation deadline as an ISO 8601 timestamp with a timezone.' + if not isinstance(value, str) or len(value) > 64: + raise ValueError(message) + value = value.strip() + if not value: + return '' + try: + stamp = _dt.datetime.fromisoformat(value.replace('Z', '+00:00')) + if stamp.tzinfo is None or stamp.utcoffset() is None: + raise ValueError() + return stamp.astimezone(_dt.timezone.utc).isoformat().replace('+00:00', 'Z') + except (ValueError, OverflowError): + raise ValueError(message) from None + + def _fernet(): key = (os.environ.get('AIOS_KEYCHAIN_KEY') or '').strip() if not key: @@ -217,13 +268,25 @@ def add_entry(rt, label, etype, fields, username): etype = str(etype or '').strip().lower() if etype not in ENTRY_TYPES: raise ValueError(f"type must be one of {', '.join(sorted(ENTRY_TYPES))}") - if not isinstance(fields, dict) or not fields: + if etype == 'amazon_sp' and not isinstance(fields, dict): + fields = {} + if not isinstance(fields, dict) or (not fields and etype != 'amazon_sp'): raise ValueError('fields must be a non-empty object') + deadline = (_amazon_rotation_deadline(fields.get(AMAZON_ROTATION_DEADLINE_FIELD)) + if etype == 'amazon_sp' else '') clean = {} for k, v in list(fields.items())[:12]: k = str(k).strip()[:40] + if etype == 'amazon_sp' and k == AMAZON_ROTATION_DEADLINE_FIELD: + continue if k and isinstance(v, (str, int, float, bool)): clean[k] = str(v)[:MAX_FIELD_LEN] + if etype == 'amazon_sp': + from harness import amazon_client + missing = [key for key in amazon_client.CREDENTIAL_FIELDS + if not isinstance(fields.get(key), str) or not clean.get(key, '').strip()] + if missing: + raise ValueError('Missing ' + ', '.join(missing) + '. Enter every required field.') if not clean: raise ValueError('no usable fields') token = _fernet().encrypt(json.dumps(clean, ensure_ascii=False).encode('utf-8')) @@ -235,9 +298,12 @@ def add_entry(rt, label, etype, fields, username): label = f'{kind_label(etype)} {slot}' row = {'id': eid, 'connection_id': eid, 'slot': slot, 'label': label, 'type': etype, 'enc': token.decode('ascii'), + 'envelope_version': ENVELOPE_VERSION, 'preview': preview, 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), 'createdBy': str(username or '')} + if etype == 'amazon_sp': + row[AMAZON_ROTATION_DEADLINE_FIELD] = deadline def _up(cur): entries = cur.setdefault('entries', {}) @@ -296,19 +362,20 @@ def read_fields(rt, entry_id): None when the entry does not exist; KeychainLocked when the key is absent/wrong (a wrong key must be loud — silently empty creds would read as 'not configured').""" e = (_bucket(rt).get('entries') or {}).get(str(entry_id)) - if not isinstance(e, dict) or not e.get('enc'): + if e is None: return None + if not isinstance(e, dict): + raise KeychainEnvelopeError('The stored credential envelope is invalid.') + _check_envelope_version(e) + if not e.get('enc'): + raise KeychainEnvelopeError('The stored credential ciphertext is missing.') try: raw = _fernet().decrypt(str(e['enc']).encode('ascii')) except KeychainLocked: raise except Exception: - raise KeychainLocked('this entry does not decrypt under the configured key') - try: - data = json.loads(raw.decode('utf-8')) - return data if isinstance(data, dict) else None - except Exception: - return None + raise KeychainLocked('this entry does not decrypt under the configured key') from None + return _decoded_fields(raw) def first_entry_of_type(rt, etype): @@ -674,11 +741,13 @@ def put_user_secret(rt, username, provider, fields): raise ValueError('no usable fields') token = _fernet().encrypt(json.dumps(clean, ensure_ascii=False).encode('utf-8')) row = {'enc': token.decode('ascii'), + 'envelope_version': ENVELOPE_VERSION, 'updated': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')} def _up(cur): - cur.setdefault('user_secrets', {}).setdefault(username, {})[provider] = row - return cur + slots = dict(cur.get('user_secrets') or {}) + slots[username] = {**(slots.get(username) or {}), provider: row} + return {**cur, 'user_secrets': slots} rt.update(KEY, _up, flush='sync') return True @@ -689,19 +758,20 @@ def read_user_secret(rt, username, provider): never read as 'not connected').""" slot = ((_bucket(rt).get('user_secrets') or {}).get(str(username)) or {}).get( str(provider or '').lower()) - if not isinstance(slot, dict) or not slot.get('enc'): + if slot is None: return None + if not isinstance(slot, dict): + raise KeychainEnvelopeError('The stored credential envelope is invalid.') + _check_envelope_version(slot) + if not slot.get('enc'): + raise KeychainEnvelopeError('The stored credential ciphertext is missing.') try: raw = _fernet().decrypt(str(slot['enc']).encode('ascii')) except KeychainLocked: raise except Exception: - raise KeychainLocked('this slot does not decrypt under the configured key') - try: - data = json.loads(raw.decode('utf-8')) - return data if isinstance(data, dict) else None - except Exception: - return None + raise KeychainLocked('this slot does not decrypt under the configured key') from None + return _decoded_fields(raw) def drop_user_secret(rt, username, provider): @@ -747,6 +817,51 @@ def register_prober(etype, fn): return etype in _PROBES +def entry_verified(rt, entry_id): + """Read proof for the current ciphertext without decrypting or making a network call.""" + entry = (_bucket(rt).get('entries') or {}).get(str(entry_id)) + if not isinstance(entry, dict) or not entry.get('enc'): + return False + proof = entry.get('probe') + return (isinstance(proof, dict) and proof.get('ok') is True + and proof.get('encrypted_digest') == hashlib.sha256( + str(entry['enc']).encode('utf-8')).hexdigest()) + + +def _test_amazon_entry(rt, entry_id, entry, timeout): + """Only an explicit test calls the registered probe; storage and listing stay offline. + + Bind the result to ciphertext so a replacement cannot inherit a previous credential's proof. + Neither provider messages nor exception text cross this boundary: both can echo input. + """ + result = {'ok': False, + 'message': 'Amazon did not confirm marketplace access. Check the five fields and test again.'} + try: + fields = read_fields(rt, entry_id) or {} + prober = _PROBES.get('amazon_sp') + got = prober(dict(fields), timeout) if prober is not None else None + if isinstance(got, dict) and got.get('ok') is True: + result = {'ok': True, 'message': 'Amazon confirmed marketplace access.'} + except Exception: # noqa: BLE001 + pass + + def _record(cur): + current = (cur.get('entries') or {}).get(str(entry_id)) + if not isinstance(current, dict) or current.get('enc') != entry.get('enc'): + raise ValueError('The credential changed during the test. Reload and test again.') + current['probe'] = { + 'ok': result['ok'], + 'encrypted_digest': hashlib.sha256(str(entry.get('enc') or '').encode('utf-8')).hexdigest(), + } + return cur + + try: + rt.update(KEY, _record, flush='sync') + except Exception: # noqa: BLE001 + return {'ok': False, 'message': 'The test result could not be saved. Reload and test again.'} + return result + + def test_entry(rt, entry_id, timeout=8): """A cheap, READ-ONLY liveness probe. odoo: xmlrpc `common.version()` (no auth, no data); generic: shape-only. Always answers {ok, message} — an exception here is a result, not a @@ -755,6 +870,8 @@ def test_entry(rt, entry_id, timeout=8): e = (_bucket(rt).get('entries') or {}).get(str(entry_id)) if not isinstance(e, dict): return {'ok': False, 'message': 'no such entry'} + if e.get('type') == 'amazon_sp': + return _test_amazon_entry(rt, entry_id, dict(e), timeout) fields = read_fields(rt, entry_id) or {} # ⭐ W31 — A REGISTERED PROBER ANSWERS FOR ITS OWN TYPE, if the connector layer filled the # slot. A shape check says "you pasted something", which is the least useful thing an diff --git a/platform/core/sheet_templates.py b/platform/core/sheet_templates.py new file mode 100644 index 0000000000000000000000000000000000000000..b393ff560a24262bb0f339bf3260995bfdaaed7f --- /dev/null +++ b/platform/core/sheet_templates.py @@ -0,0 +1,253 @@ +"""Loopable's prebuilt connector workbooks (W59-T29, contract C4). + +Descriptors are data; applying them uses the existing workbook persistence engine. +The API owns permission checks and calculation under the calling user's row scope. +""" +from __future__ import annotations + +from copy import deepcopy +from datetime import date, timedelta +import re + +from core import sheets, user_tables + + +# Do not use connector_spec.connector_of: it only covers SPEC'd tables and calls +# Amazon "amazon", not "amazon_sp" (D-718, owned by lane D). Definition.source is +# "Automation" and field.source is "overlay"; the spawn prefix carries the source. +SOURCE_PREFIXES = (("ut_odoo_", "odoo"), ("ut_amzn_", "amazon_sp")) +_A1 = re.compile(r"[A-Z]+[1-9][0-9]*\Z") + + +def source_of_database(database_key): + return next((source for prefix, source in SOURCE_PREFIXES + if str(database_key or "").startswith(prefix)), None) + + +def _literal(value): + return {"kind": "literal", "text": str(value), "value": value} + + +def _formula(value): + return {"kind": "formula", "formula": value} + + +def _month(offset): + today = date.today() + year, month = divmod(today.year * 12 + today.month - 1 + offset, 12) + return date(year, month + 1, 1).isoformat() + + +def _scope(*conditions): + return {"filter": [{"colId": field, "op": op, "value": value} + for field, op, value in conditions]} + + +# C4 has exactly two additions: answers supplies the gallery's short question +# summary; semantic_fields lets bindings name concepts such as "revenue" rather +# than tenant column ids. Tenants can map that concept to different source keys. +# Calendar labels and filters are pinned together in the descriptor, so refreshing +# an applied workbook never silently moves a labelled period to another month. +_AS_OF = date.today() +TEMPLATES = [ + { + "key": "odoo_monthly_sales", "title": "Monthly sales", "source": "odoo", + "database_hint": "ut_odoo_orders", + "answers": "How did sales change across the last twelve calendar months?", + "semantic_fields": {"order_date": ("order_date",), "revenue": ("amount_untaxed",)}, + "sheets": [{"name": "Monthly sales", "cells": { + "A1": _literal("Month"), "B1": _literal("Sales"), + **{f"A{row}": _literal(_month(row - 13)[:7]) for row in range(2, 14)}, + "A14": _literal("Total"), "B14": _formula("=SUM(B2:B13)"), + }}], + "bindings": [{"sheet": 0, "addr": f"B{row}", "field": "revenue", "fn": "sum", + "scope": _scope(("order_date", "gte", _month(row - 13)), + ("order_date", "lt", _month(row - 12)))} + for row in range(2, 14)], + "formats": [{"sheet": 0, "addr": f"B{row}", "kind": "currency"} + for row in range(2, 15)], + }, + { + "key": "odoo_receivables_aging", "title": "Receivables aging", "source": "odoo", + "database_hint": "ut_odoo_invoices", + "answers": "How much remains unpaid in each overdue age band?", + "semantic_fields": {"due_date": ("due_date",), "open_amount": ("residual",), + "customer": ("customer",)}, + "sheets": [{"name": "Receivables aging", "cells": { + "A1": _literal("Age band"), "B1": _literal("Open amount"), + "A2": _literal("Not yet due"), "A3": _literal("1 to 30 days overdue"), + "A4": _literal("31 to 60 days overdue"), "A5": _literal("61 to 90 days overdue"), + "A6": _literal("Over 90 days overdue"), "A7": _literal("No due date"), + "A8": _literal("Total open"), "B8": _formula("=SUM(B2:B7)"), + "A10": _literal("Invoices with an open balance"), + "A12": _literal("Aging date"), "B12": _literal(_AS_OF.isoformat()), + }}], + "bindings": [ + {"sheet": 0, "addr": "B2", "field": "open_amount", "fn": "sum", + "scope": _scope(("due_date", "gte", _AS_OF.isoformat()), + ("open_amount", "gt", "0"))}, + *[{"sheet": 0, "addr": f"B{row}", "field": "open_amount", "fn": "sum", + "scope": _scope(("due_date", "gte", (_AS_OF - timedelta(days=end)).isoformat()), + ("due_date", "lt", (_AS_OF - timedelta(days=start)).isoformat()), + ("open_amount", "gt", "0"))} + for row, start, end in ((3, 0, 30), (4, 30, 60), (5, 60, 90))], + {"sheet": 0, "addr": "B6", "field": "open_amount", "fn": "sum", + "scope": _scope(("due_date", "lt", (_AS_OF - timedelta(days=90)).isoformat()), + ("due_date", "isNotEmpty", ""), ("open_amount", "gt", "0"))}, + {"sheet": 0, "addr": "B7", "field": "open_amount", "fn": "sum", + "scope": _scope(("due_date", "isEmpty", ""), ("open_amount", "gt", "0"))}, + {"sheet": 0, "addr": "B10", "field": "customer", "fn": "count", + "scope": _scope(("open_amount", "gt", "0"))}, + ], + "formats": [{"sheet": 0, "addr": f"B{row}", "kind": "currency"} + for row in range(2, 9)] + [ + {"sheet": 0, "addr": "B10", "kind": "number"}, + {"sheet": 0, "addr": "B12", "kind": "date"}], + }, + { + "key": "odoo_margin_by_product", "title": "Margin by product", "source": "odoo", + "database_hint": "ut_odoo_order_lines", + "answers": "What revenue and margin do product lines generate?", + "semantic_fields": {"product": ("product",), "margin": ("margin",), + "revenue": ("price_subtotal",)}, + "sheets": [{"name": "Product margins", "cells": { + "A1": _literal("Products"), "B1": _literal("Revenue"), + "C1": _literal("Margin"), "D1": _literal("Margin rate"), + "A2": _literal("All products"), "D2": _formula("=IF(B2=0,0,C2/B2)"), + "A4": _literal("Product lines"), + }}], + "bindings": [ + {"sheet": 0, "addr": "B2", "field": "revenue", "fn": "sum"}, + {"sheet": 0, "addr": "C2", "field": "margin", "fn": "sum"}, + {"sheet": 0, "addr": "B4", "field": "product", "fn": "count"}, + ], + "formats": [{"sheet": 0, "addr": addr, "kind": kind} + for addr, kind in (("B2", "currency"), ("C2", "currency"), + ("D2", "percent"), ("B4", "number"))], + }, + { + "key": "amazon_settlement_summary", "title": "Settlement summary", "source": "amazon_sp", + "database_hint": "ut_amzn_settlements", + "answers": "What net amount and transaction count do settlement lines contain?", + # V2 flat-file columns are normalized by amazon_store._snake. Sum the + # line amount, never a repeated settlement-level total_amount header. + "semantic_fields": {"settlement": ("settlement_id",), "net_amount": ("amount",)}, + "sheets": [{"name": "Settlement summary", "cells": { + "A1": _literal("Measure"), "B1": _literal("Value"), + "A2": _literal("Net settlement amount"), "A3": _literal("Settlement lines"), + "A4": _literal("Average amount per line"), "B4": _formula("=IF(B3=0,0,B2/B3)"), + }}], + "bindings": [{"sheet": 0, "addr": "B2", "field": "net_amount", "fn": "sum"}, + {"sheet": 0, "addr": "B3", "field": "settlement", "fn": "count"}], + "formats": [{"sheet": 0, "addr": addr, "kind": kind} + for addr, kind in (("B2", "number"), ("B3", "number"), ("B4", "number"))], + }, +] + + +def template(key): + return next((row for row in TEMPLATES if row["key"] == key), None) + + +def templates_for_sources(sources): + allowed = set(sources) + return [row for row in TEMPLATES if row["source"] in allowed] + + +class TemplateRefusal(Exception): + def __init__(self, code, *, missing=(), detail=""): + self.code = code + self.missing = tuple(missing) + self.detail = detail + super().__init__(self.sentence()) + + def sentence(self): + if self.code == "template_fields_missing": + return "This database is missing fields required by the template: " + ", ".join(self.missing) + "." + return self.detail or "This sheet template could not be applied." + + +def resolve_fields(template, field_keys): + available = set(field_keys) + resolved, missing = {}, [] + for semantic, candidates in template["semantic_fields"].items(): + chosen = next((key for key in candidates if key in available), None) + if chosen is None: + missing.append(semantic) + else: + resolved[semantic] = chosen + if missing: + raise TemplateRefusal("template_fields_missing", missing=missing) + return resolved + + +def _resolved_scope(value, resolved): + if isinstance(value, list): + return [_resolved_scope(item, resolved) for item in value] + if isinstance(value, dict): + return {key: resolved[item] if key == "colId" else _resolved_scope(item, resolved) + for key, item in value.items()} + return deepcopy(value) + + +def build_cells(template, resolved, database_key): + # The ticket explicitly uses modules.sheet's existing binding normalizer; + # no API or calculation runtime is imported into this data layer. + from modules.sheet import clean_binding + + tabs = [(i, tab["name"], deepcopy(tab["cells"])) for i, tab in enumerate(template["sheets"])] + + def target(entry): + index, addr = entry["sheet"], entry["addr"] + if type(index) is not int or not 0 <= index < len(tabs) or not _A1.fullmatch(addr): + raise TemplateRefusal("template_invalid", detail="The template names an invalid sheet or cell.") + return tabs[index][2], addr + + for entry in template["bindings"]: + cells, addr = target(entry) + binding = {"database": database_key, + "aggregate": {"fn": entry["fn"], "field": resolved[entry["field"]]}, + "scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)} + if "period" in entry: + binding["period"] = deepcopy(entry["period"]) + cells[addr] = {**cells.get(addr, {}), "kind": "bound", "binding": clean_binding(binding)} + for entry in template["formats"]: + cells, addr = target(entry) + cell = cells.setdefault(addr, {"kind": "empty"}) + style = cell.get("style", {}) + cell["style"] = {**style, "format": {**style.get("format", {}), "kind": entry["kind"]}} + for _, _, cells in tabs: + for addr, cell in cells.items(): + if not _A1.fullmatch(addr): + raise TemplateRefusal("template_invalid", detail="The template names an invalid cell: " + addr + ".") + cells[addr] = sheets.clean_cell(cell, addr) + return tabs + + +def apply_template(st, key, database_key, *, name="", created_by=""): + descriptor = template(key) + if descriptor is None: + raise TemplateRefusal("unknown_sheet_template", detail="That sheet template does not exist.") + definition = user_tables.all_defs(st).get(database_key) + if definition is None: + raise TemplateRefusal("unknown_database", detail="That database does not exist.") + if source_of_database(database_key) != descriptor["source"]: + raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.") + resolved = resolve_fields(descriptor, (field["key"] for field in definition.get("fields", []))) + prepared = build_cells(descriptor, resolved, database_key) + # Validate tab names, extents and every cell before the first store write. + checked = sheets.migrate_workbook({"sheets": [ + {"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]}) + book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by) + try: + tabs = [{**tab, "cells": {}} for tab in checked["sheets"]] + book = sheets.put_workbook(st, book["id"], {"name": book["name"], "sheets": tabs, "rev": book["rev"]}) + for index, _, cells in prepared: + result = sheets.patch_cells(st, book["id"], book["sheets"][index]["id"], cells) + if result.get("refused"): + addresses = [str(item["addr"]) for item in result["refused"]] + raise TemplateRefusal("template_cells_refused", detail="The template could not apply cells: " + ", ".join(addresses) + ".") + return sheets.get_workbook(st, book["id"]) + except Exception: + sheets.delete_workbook(st, book["id"]) + raise diff --git a/platform/core/users.py b/platform/core/users.py index 8eead67bd78b84d763c2a337246dabaac06b7cdc..85cdd5a4241bb01a5799ff7ca5d11538f246365a 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -488,8 +488,8 @@ def note_login_failure(identifier, now=None): def _note(doc): doc = dict(doc) if isinstance(doc, dict) else {} - # Keep the security document bounded under random-identifier abuse. An active lock is - # never pruned, even if a malformed/old ``last_failed_at`` accompanies it. + # Prune stale budgets under random-identifier abuse. An active lock is never pruned, + # even if a malformed/old ``last_failed_at`` accompanies it. retained = {} for stored_key, value in doc.items(): if not isinstance(value, dict): @@ -520,12 +520,24 @@ def note_login_failure(identifier, now=None): doc[key] = {'count': count, 'last_failed_at': now, 'locked_until': until} if len(doc) > _LOGIN_LOCKOUT_MAX_TRACKED: - # Discard expired/incomplete budgets before active locks. A pathological set of more - # than 2,048 simultaneous active identifiers still evicts oldest-first: the document - # stays bounded rather than becoming an attacker-controlled memory allocation. - oldest = sorted(doc, key=lambda stored_key: ( - float((doc.get(stored_key) or {}).get('locked_until') or 0) > now, - float((doc.get(stored_key) or {}).get('last_failed_at') or 0))) + # W59-T33 / instruction 24: the size target cannot shorten a thirty-minute lock. + # Only expired/incomplete budgets may be evicted. If active locks fill the target, + # retain them plus this attempt's budget until expiry permits pruning again. + # ⭐⭐ THIS REVERSES A DELIBERATE EARLIER CHOICE, SO KEEP BOTH REASONS. The previous + # rule sorted active locks last but still evicted them once the document passed the + # target, to stop a random-identifier flood becoming an attacker-controlled + # allocation. The cost of that rule is a BYPASS of the control itself: an attacker + # who wants to keep guessing one account sprays enough other identifiers to evict + # that account's lock, and a lock that can be evicted is not the thirty-minute lock + # Amazon's baseline was told exists. The accepted cost of the new rule is the one the + # old comment named: while more than 2,048 locks are simultaneously ACTIVE the + # document grows past its target. That is bounded in practice, because every active + # lock costs ten failed proofs and each burns a PBKDF2 hash, and it self-corrects + # within thirty minutes as locks expire and become prunable again. + oldest = sorted( + (stored_key for stored_key in doc + if float((doc.get(stored_key) or {}).get('locked_until') or 0) <= now), + key=lambda stored_key: float((doc.get(stored_key) or {}).get('last_failed_at') or 0)) excess = len(doc) - _LOGIN_LOCKOUT_MAX_TRACKED for stored_key in [candidate for candidate in oldest if candidate != key][:excess]: doc.pop(stored_key, None) diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index 8e3c6175d661e6be3bba645ac9363a16b380b374..895ecb0265731657da7d212c8384c18481b30b15 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -298,6 +298,135 @@ ROLES_TAKEN = ( ) ROLES_REFUSED = ("Buyer Communication", "Buyer Solicitation") +#: W59-T39: the connector's approved operation scope, not Amazon's entire role catalogue. +#: Join the report roster rather than maintaining another copy of its role labels. A compound +#: label describes TWO data legs: inventory uses Fulfillment for its report and AWD for its +#: entity read; pricing uses Brand Analytics for its report and Pricing for its entity read. +#: Keep those source labels intact and expose both roles, without granting either entity role +#: the other role's report access. +REPORT_ROLES = { + table: tuple(spec["role"].replace(" / ", " + ").split(" + ")) + for table, spec in REPORT_TYPES.items() +} +_REPORT_ROLE_INDEX = {"inventory": 0, "pricing": 1} +_BULK_REPORT_ROLES = tuple(dict.fromkeys( + roles[_REPORT_ROLE_INDEX.get(table, 0)] for table, roles in REPORT_ROLES.items())) +_APPROVED_REPORT_TYPES = tuple(dict.fromkeys( + spec[key] for spec in REPORT_TYPES.values() for key in ("report", "report_incremental") + if spec.get(key))) + +#: Exact method/path pairs. Braced ids match ONE opaque path segment, never a suffix, query, +#: escaped slash or traversal. The two POSTs only request a report or wrap fixed read-only GETs. +#: LWA, presigned documents and the schema module's public GitHub refresh are not SP-API +#: operations. The gate enumerates those transports separately; they confer no operation scope. +OPERATIONS = { + "createReport": { + "name": "createReport", "roles": _BULK_REPORT_ROLES, "method": "POST", + "path": "reports/2021-06-30/reports", "report_types": _APPROVED_REPORT_TYPES, + }, + "getReport": { + "name": "getReport", "roles": _BULK_REPORT_ROLES, "method": "GET", + "path": "reports/2021-06-30/reports/{reportId}", + }, + "getReportDocument": { + "name": "getReportDocument", "roles": _BULK_REPORT_ROLES, "method": "GET", + "path": "reports/2021-06-30/documents/{documentId}", + }, + "getMarketplaceParticipations": { + "name": "getMarketplaceParticipations", "roles": ("Selling Partner Insights",), + "method": "GET", "path": "sellers/v1/marketplaceParticipations", + }, + "getCompetitiveSummary": { + "name": "getCompetitiveSummary", "roles": ("Pricing",), "method": "POST", + "path": "batches/products/pricing/2022-05-01/items/competitiveSummary", + }, + "listInventory": { + "name": "listInventory", "roles": ("Amazon Warehousing and Distribution",), + "method": "GET", "path": "awd/2024-05-09/inventory", + }, +} + +#: Answer "what does this approved role let this connector read?" directly from data. +#: The report-side roles above are derived from REPORT_TYPES, including its compound labels. +ROLE_OPERATIONS = { + role: tuple(name for name, operation in OPERATIONS.items() if role in operation["roles"]) + for role in ROLES_TAKEN +} + +# Containment record for the Amazon approval-control series, measured 2026-09-13. +# Git ancestry is not the record here: a packed wave container carries no source +# history. The laptop ancestry check lives in intake finding F23 in +# AutoWiki/research/ideation/wave59-intake.md. +# Each entry claims what exists, never that the control is correct. `contained` +# records full containment; False preserves a documented partial contribution. +# `checkable` means the module can be imported from platform/ without reaching up. +# False is an honest ownership boundary, not a gap to paper over: platform must +# not import the web API, and a Dockerfile is not an importable Python module. +# This is data only; resolving symbols belongs to the checker (W59-T39). +APPROVAL_CONTROLS = { + "f1d803d": { + "carries": "shared persistent login-attempt lockout", + "module": "platform/core/users.py", + "symbol": "LOGIN_LOCKOUT_BUCKET", + "contained": True, + "checkable": True, + }, + "be68048": { + "carries": "password minimum age in the password policy", + "module": "platform/core/users.py", + "symbol": "PASSWORD_MIN_AGE_DAYS", + "contained": True, + "checkable": True, + }, + "83bfed7": { + "carries": "the audited lockfile is the dependency boundary", + "module": "aios-web/Dockerfile", + "symbol": None, + "contained": False, + "checkable": False, + "note": ( + "The lockfile COPY at aios-web/Dockerfile:37 stands; the npm ci half " + "does not: :38 is npm install --no-audit --no-fund --ignore-scripts. " + "The cause is a Windows-resolved lockfile, booked as HARNESS W-170. " + "Dockerfile and the lockfile belong to lane F in Wave 59. " + "This web build control cannot be resolved here because Dockerfile " + "is not an importable Python module." + ), + }, + "849067e": { + "carries": "Pricing and AWD read operations, and the eight approved roles", + "module": "platform/harness/amazon_client.py", + "symbol": "ROLES_TAKEN", + "contained": True, + "checkable": True, + }, + "7e556f8": { + "carries": "exact-tenant raw-mirror purge", + "module": "platform/harness/amazon_store.py", + "symbol": "purge", + "contained": True, + "checkable": True, + }, + "9c9454b": { + "carries": "origin attribution on materialised Amazon tables", + "module": "aios-web/api/amazon_relational.py", + "symbol": "source_attribution", + "contained": True, + "checkable": False, + "note": ( + "Owned by the web API layer, which platform may not import; " + "source_attribution cannot be resolved from this layer." + ), + }, + "df1252b": { + "carries": "origin attribution on Sheet workbooks and the scope cache", + "module": "platform/core/sheets.py", + "symbol": "AMAZON_DATA_ORIGIN_NAME", + "contained": True, + "checkable": True, + }, +} + _UA = "AIOS-Loopable/1.0 (+read-only)" _TIMEOUT = 20 @@ -306,6 +435,14 @@ class AmazonError(RuntimeError): """Any refusal from LWA or SP-API, already stripped of anything secret.""" +class AmazonOperationRefused(AmazonError): + """A local scope refusal, never a throttle and never eligible for throttling retries.""" + + def __init__(self, operation): + super().__init__( + f"Refused {_safe(operation)}: this connector is limited to the approved read-only set") + + class AmazonThrottled(AmazonError): """An SP-API 429, with an optional Retry-After delay in seconds.""" @@ -331,6 +468,36 @@ def _safe(url): return "" +def operation_for(method, path): + """Resolve an exact approved operation, or None. Query parameters belong in **params.""" + parts = str(path).lstrip("/").split("/") + for operation in OPERATIONS.values(): + template = operation["path"].split("/") + if str(method).upper() != operation["method"] or len(parts) != len(template): + continue + if all( + (bool(part) and part not in (".", "..") + and all(c in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." + for c in part)) + if segment.startswith("{") and segment.endswith("}") else part == segment + for part, segment in zip(parts, template) + ): + return operation + return None + + +def require_operation(method, path, *, report_type=None): + """Fail closed before constructing any URL or touching a transport.""" + operation = operation_for(method, path) + if operation is None: + raise AmazonOperationRefused(f"{str(method).upper()} {_safe(path)}") + # A Reports URL alone is insufficient: restricted report families use this same POST. + # Admit only the bulk flavors joined from the seven declared tables, including incrementals. + if report_type is not None and str(report_type) not in operation.get("report_types", ()): + raise AmazonOperationRefused(f"{operation['name']} report {report_type}") + return operation + + def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random, exhausted_fix=None): """Retry only SP-API throttling; return successes and propagate other refusals unchanged.""" for n in range(1, RETRY_MAX_ATTEMPTS + 1): @@ -389,6 +556,8 @@ def _pace_entity(name, interval, *, sleep=time.sleep, now=time.monotonic): def _post_form(url, data, timeout=_TIMEOUT): + # LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes + # LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS. body = urllib.parse.urlencode(data).encode("ascii") req = urllib.request.Request(url, data=body, method="POST") req.add_header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8") @@ -442,6 +611,7 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT, """ if str(method).upper() != "GET": raise AmazonError("this connector is read-only: only GET is permitted") + require_operation(method, path) def attempt(): """One raw read-only SP-API attempt, with no retry or pacing policy.""" @@ -500,6 +670,7 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT, def _competitive_summary_batch(token, asins, marketplace_id, region=DEFAULT_REGION, timeout=_TIMEOUT, sandbox=False): """Read one fixed `getCompetitiveSummary` batch. -> its response list.""" + require_operation("POST", _COMPETITIVE_SUMMARY_BATCH_PATH) items = [str(asin or "").strip() for asin in (asins or ()) if str(asin or "").strip()] if not 1 <= len(items) <= COMPETITIVE_SUMMARY_BATCH_SIZE: raise AmazonError( @@ -669,6 +840,8 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION, ⛔ NO `path` PARAMETER, ON PURPOSE. See `_REPORTS_CREATE_PATH` above: the whole safety argument is that this function can reach exactly one operation and a caller cannot redirect it. """ + require_operation("POST", _REPORTS_CREATE_PATH, report_type=str(report_type)) + def attempt(): """One raw attempt at the declared report-creation POST.""" base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( @@ -745,6 +918,8 @@ def report_document(fields, report_type, start=None, end=None, options=None, `IN_PROGRESS` after `max_wait` raises with the reportId in the message, so the next call can resume from it rather than paying for the whole build again. """ + # Refuse an undeclared report even before its LWA exchange; _create_report guards its own door. + require_operation("POST", _REPORTS_CREATE_PATH, report_type=str(report_type)) say = log or (lambda *_a, **_k: None) token = access_token(fields, timeout=timeout) region = str((fields or {}).get("region") or DEFAULT_REGION) @@ -801,6 +976,8 @@ def _download_document(url, compression="", timeout=120): one smart quote. Decoded with replacement, and that is a deliberate lossy choice on a presentation field rather than on a number. """ + # This is the presigned URL returned by getReportDocument, not an SP-API operation. Do not + # apply OPERATIONS here: its host and signature belong to Amazon's document download service. req = urllib.request.Request(url, method="GET") req.add_header("User-Agent", _UA) try: diff --git a/platform/harness/amazon_schema.py b/platform/harness/amazon_schema.py index 00f9e149b4e115b011296ab17c2c0cf22642d7da..fc14c2db3ee589ca1405403a7f9c82b1b2fc0117 100644 --- a/platform/harness/amazon_schema.py +++ b/platform/harness/amazon_schema.py @@ -265,6 +265,120 @@ def columns_for(table): return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])] +# --------------------------------------------------------------------------- contract C1 + +#: The one translation from C1's short report keys to the authored database namespace. +#: A-C1-KEYS preserves stored enablement keys and route paths while lane D adopts this map. +DATABASE_PREFIX = "ut_amzn_" + + +def _table_spec(table, identity, relationships=(), *, grain="entity", line_discriminator=None): + """Build the common descriptor fields from the LOCAL pin, never a vendor call.""" + return { + "columns": [{"name": name, "type": kind} for name, kind in columns_for(table)], + "columns_source": { + "path": "platform/harness/proto/amazon-report-schemas.json", + "reader": "amazon_schema.columns_for", + "provenance": PROVENANCE[table], + "assumption": ( + "A-QI14: This wave declares from REPORT_TYPES and the pinned artifacts. " + "The five measured_at_sync tables' columns are re-measured against the first " + "production sync during the deferred QI13 hotfix (intake RDA6)."), + }, + "row_identity": identity, + "row_identity_mode": "composite", + "row_identity_missing": "require_all", + "row_identity_alternatives": (), + "row_identity_status": "declared_pending_first_production_sync", + "grain": grain, + "line_discriminator": line_discriminator, + "line_discriminator_source": ({ + "kind": "declared_mirror_metadata", + "document_column": "_report_document_id", + "note": ( + "Sync must retain the report document ID and the one-based parsed row position " + "before filtering or deduplication, including identical repeated lines. These " + "are proposed mirror metadata, not measured Amazon header columns. Replaying " + "one document keeps its positions; reconciling overlapping documents is a " + "separate sync responsibility. Never fall back to the order or settlement ID."), + } if line_discriminator else None), + "relationships": relationships, + "provenance": PROVENANCE[table], + "read_through": True, + "row_cap": None, + "read_through_status": "declared_pending_consumer_wiring", + "read_through_reason": ( + "Standing rule 1: these are connector records with records_mutable=False. " + "Serve history read-through from the mirror without a record cap."), + } + + +#: ⭐⭐ TABLE_SPECS IS CONTRACT C1'S SINGLE SOURCE OF TRUTH. One entry per REPORT_TYPES key, +#: checked strictly by verify_amazon. Columns and types come ONLY through columns_for's pinned +#: catalogue; the five measured_at_sync lists remain EMPTY. columns_source carries A-QI14 in the +#: descriptor itself, so a consumer cannot mistake this wave's declaration for production evidence. +#: +#: ⛔ IDENTITIES ARE COMPOSITES, NEVER FALLING PREFERENCES. Every component participates and no +#: proper subset is an alternative identity FOR THAT TABLE. orders is the deduplicated header +#: (_dedupe_orders), while order_items and settlements retain source lines. An order item ID is +#: not guaranteed in the flat-file header, and even identical settlement charges must survive: +#: both line tables therefore declare document + position lineage beside their parent ID. The +#: lineage columns are proposed MIRROR METADATA, not invented vendor columns in the catalogue. +#: +#: ⚠ Identity uniqueness, line metadata and relationship column spellings await the first real +#: sync. All identities say so, including the two tables with published COLUMN artifacts. Nine +#: relationships retain BOTH join-column names and short target keys; the gate compares them to +#: connector_spec.tables_for("amazon"), which amazon_relational still renders. Read-through is a +#: REQUIRED POLICY, not a claim that the existing materialisation consumer already implements it. +TABLE_SPECS = { + "orders": _table_spec("orders", ("amazon_order_id",), ( + ("sku", "listings", "seller_sku", "many_to_one"), + ), grain="order"), + "order_items": _table_spec("order_items", ( + "amazon_order_id", "_report_document_id", "_report_row_number", + ), ( + ("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), + ("sku", "listings", "seller_sku", "many_to_one"), + ), grain="line", line_discriminator="_report_row_number"), + "listings": _table_spec("listings", ("seller_sku",)), + "settlements": _table_spec("settlements", ( + "settlement_id", "_report_document_id", "_report_row_number", + ), ( + ("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), + ("sku", "listings", "seller_sku", "many_to_one"), + ), grain="line", line_discriminator="_report_row_number"), + "inventory": _table_spec("inventory", ("asin", "fnsku", "msku"), ( + ("msku", "listings", "seller_sku", "many_to_one"), + ("asin", "listings", "asin1", "many_to_one"), + )), + "pricing": _table_spec("pricing", ("parent_asin", "child_asin", "sku"), ( + ("child_asin", "listings", "asin1", "many_to_one"), + ("sku", "listings", "seller_sku", "many_to_one"), + )), + "brand_analytics": _table_spec("brand_analytics", ( + "department_name", "search_term", "clicked_asin", "click_share_rank", + )), +} + + +def row_identity_for(table, row): + """Evaluate the declared full composite. Missing lineage is an error, never a parent ID. + + This is a descriptor evaluator, not a production uniqueness measurement or a sync adapter. + Return a tuple so delimiters inside values cannot merge two different component sequences. + """ + spec = TABLE_SPECS[table] + if spec["row_identity_mode"] != "composite" or spec["row_identity_missing"] != "require_all": + raise ValueError(f"{table}: unsupported row identity policy") + identity = spec["row_identity"] + if not identity or spec["row_identity_alternatives"]: + raise ValueError(f"{table}: exactly one complete row identity is required") + missing = [name for name in identity if row.get(name) in (None, "")] + if missing: + raise ValueError(f"{table}: missing declared row identity components: {', '.join(missing)}") + return tuple(row[name] for name in identity) + + def disclosure(): """What the onboarding screen shows: one row per table, with its provenance and sync date. diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index 24f4ba7a0306a75a2b4e53fb13e300c1ea73a791..775cc8e841a70fe41fcba74d981ac9d13fba7353 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -45,8 +45,12 @@ import csv import io import json import os +import shutil import sys +import tempfile import time +import uuid +from contextlib import closing from datetime import datetime, timedelta, timezone from pathlib import Path @@ -66,6 +70,63 @@ PREFIX = "amzn_" TABLES = ("orders", "order_items", "listings", "settlements", "inventory", "pricing", "brand_analytics") +# C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity. +ENABLEMENT_KEY = "amazon_connector_enablement" +# Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing. +SETTLEMENT_SYNC_SECONDS = 14 * 24 * 60 * 60 + + +def read_database_enablement(rt): + """Read C1 without importing the web layer; malformed/unreadable state fails closed.""" + keys = tuple(amazon_schema.DATABASE_PREFIX + table for table in TABLES) + unavailable = {"available": False, "databases": dict.fromkeys(keys, False)} + try: + current = rt.get(ENABLEMENT_KEY) if rt is not None else None + except Exception: + return unavailable + if current is None or current == {}: + return {"available": True, "databases": dict.fromkeys(keys, True)} + if not isinstance(current, dict) or not isinstance(current.get("databases"), dict): + return unavailable + stored = current["databases"] + if set(stored) - set(keys) or any(type(value) is not bool for value in stored.values()): + return unavailable + return {"available": True, "databases": {key: stored.get(key, True) for key in keys}} + + +def sync_plan(states, tables=None, *, now=None): + """Pure cadence decision: one oldest due table per tick, with no stagger sleep. + + Attempt markers live in this tenant's mirror, so a replacement mirror starts fresh. Failed + tables rotate behind their peers and retry on a later tick, including failed settlements. + The client still owns pacing/retries within a table; this spreads tables across caller ticks. + """ + from harness.amazon_client import REPORT_STAGGER_SECONDS + now = _utc(now) + want = list(TABLES if tables is None else tables) + stamps = {table: _utc(state["updated_at"]) for table, state in states.items() if state} + deferred = {} + due = [] + for table in want: + stamp = stamps.get(table) + if (table == "settlements" and stamp is not None + and states[table]["state"] == "done" + and (now - stamp).total_seconds() < SETTLEMENT_SYNC_SECONDS): + deferred[table] = "settlements are on a 14-day cycle; next due " + _iso( + stamp + timedelta(seconds=SETTLEMENT_SYNC_SECONDS)) + else: + due.append(table) + # Boot and the resident first pass can be adjacent. Report the delay instead of sleeping. + recent = max(stamps.values()) if stamps else None + ready = recent is None or (now - recent).total_seconds() >= REPORT_STAGGER_SECONDS + selected = min(due, key=lambda table: stamps.get( + table, datetime.min.replace(tzinfo=timezone.utc))) if due and ready else None + for table in due: + if table != selected: + deferred[table] = ("table stagger: waiting for a later cadence tick; " + f"minimum spacing {REPORT_STAGGER_SECONDS:g}s") + return {"tables": [selected] if selected else [], "deferred": deferred} + #: Durable proof that the two non-report role operations actually ran. A report-backed table can #: contain rows while one of these role grants is still missing, so table presence alone is not #: evidence that Pricing or AWD was authorized. @@ -91,6 +152,21 @@ DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730) #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one. MAX_WINDOW_DAYS = 30 +# Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores; +# JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata. +_ORIGIN_TYPES = {"_report_document_id": "VARCHAR", "_report_row_number": "BIGINT", + "_data_origin": "VARCHAR", "_report_type": "VARCHAR"} +DELETION_DAYS = 30 +DELETION_TRIGGERS = ("revocation", "authorization_loss", "termination", "notice") + + +def _origin(row, document_id, row_number, report_type): + if not document_id or not report_type: + raise ValueError("Amazon row origin requires a document ID and report type") + row.update({"_report_document_id": str(document_id), "_report_row_number": row_number, + "_data_origin": "Amazon Services API", "_report_type": str(report_type)}) + return row + #: Columns that must be text whatever they look like. An Amazon order id is `111-1234567-1234567` #: and an ASIN is `B0…`; a settlement id is a long decimal that exceeds 2^53. Same ruling as Meta's #: ids and `ig_id` before it, for the same reason: a float path corrupts them silently. @@ -140,7 +216,7 @@ def _dedupe(cols): return out -def parse_flat_file(text): +def parse_flat_file(text, *, document_id=None, report_type=None): """One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header. ⭐ THIS FUNCTION IS WHY NO ORDER, LISTING OR SETTLEMENT COLUMN IS TRANSCRIBED ANYWHERE. Line 1 @@ -163,16 +239,20 @@ def parse_flat_file(text): cols = _dedupe([_snake(h) for h in header]) width = len(cols) rows = [] - for raw in reader: + # The read-only connectors_amazon report probe measures headers without capture metadata. + # Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies + # BOTH keywords, and a missing document ID with a report type is an error, never a fallback. + attributed = document_id is not None or report_type is not None + for row_number, raw in enumerate(reader, 1): if not raw or (len(raw) == 1 and not str(raw[0]).strip()): continue vals = list(raw[:width]) + [None] * max(0, width - len(raw)) - rows.append({c: (v if (v is None or str(v) != "") else None) - for c, v in zip(cols, vals)}) - return cols, rows + row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)} + rows.append(_origin(row, document_id, row_number, report_type) if attributed else row) + return cols + (list(_ORIGIN_TYPES) if attributed else []), rows -def parse_json_report(text, root): +def parse_json_report(text, root, *, document_id=None, report_type=None): """One JSON report document -> `(columns, rows)` flattened onto the pinned artifact's names. ⚠ THE COLUMN LIST IS THE PINNED SCHEMA'S, NOT THE PAYLOAD'S. Amazon omits null fields from a @@ -188,8 +268,10 @@ def parse_json_report(text, root): if isinstance(body, dict): body = [body] rows = [] - for item in body: - rows.append(_flatten_row(item or {})) + attributed = document_id is not None or report_type is not None + for row_number, item in enumerate(body, 1): + row = _flatten_row(item or {}) + rows.append(_origin(row, document_id, row_number, report_type) if attributed else row) return sorted({k for r in rows for k in r}), rows @@ -205,11 +287,12 @@ def _flatten_row(obj, prefix=""): path = f"{prefix}.{k}" if prefix else k if isinstance(v, dict): out.update(_flatten_row(v, path)) - elif isinstance(v, list): - out[amazon_schema._snake(path)] = json.dumps(v, separators=(",", ":"), - ensure_ascii=False) else: - out[amazon_schema._snake(path)] = v + name = amazon_schema._snake(path) + if name in _ORIGIN_TYPES: + raise ValueError("Amazon JSON field collides with reserved origin metadata") + out[name] = json.dumps(v, separators=(",", ":"), ensure_ascii=False) if isinstance( + v, list) else v return out @@ -227,13 +310,16 @@ def _competitive_rows(responses): return out -def _awd_rows(inventory): +def _awd_rows(inventory, operation_id=None): """AWD inventory summaries -> flattened rows that remain distinguishable from FBA ledger.""" rows = [] - for item in inventory or (): + # AWD is a paginated operation, not a report. Retain its full response-list position and + # an explicitly operation-prefixed capture ID instead of inventing an Amazon document ID. + operation_id = operation_id or "operation:" + uuid.uuid4().hex + for row_number, item in enumerate(inventory or (), 1): row = _flatten_row(item or {}) row["inventory_source"] = "AWD" - rows.append(row) + rows.append(_origin(row, operation_id, row_number, "listInventory")) return rows @@ -309,6 +395,8 @@ def _coltype(name, rows=None): column that is numeric for 500 rows and then carries `N/A` must not make the insert fail after five tables are already written — the exact failure `meta_store._coltype` was rewritten for. """ + if name in _ORIGIN_TYPES: + return _ORIGIN_TYPES[name] if name in _FORCE_TEXT: return "VARCHAR" if not rows: @@ -405,15 +493,14 @@ def _ensure(con, table, cols, rows=None, declared=None): def _replace(con, table, cols, types, rows): - """Delete-then-insert for one report window. -> rows written. - - ⚠ A REPORT DOCUMENT IS THE TRUTH FOR ITS WINDOW, so the window is replaced wholesale rather - than merged row by row. That is what makes a re-sync idempotent when Amazon RESTATES a row - (refunds, cancellations and settlement adjustments all arrive as changes to old rows), and it - is why `orders` syncs incrementally by LAST UPDATE rather than by order date. - """ + """Insert one window. The sync caller clears the table ONCE before its first window.""" if not rows: return 0 + if not set(_ORIGIN_TYPES).issubset(cols) or any( + any(r.get(c) in (None, "") for c in _ORIGIN_TYPES) + or type(r.get("_report_row_number")) is not int + or r["_report_row_number"] < 1 for r in rows): + raise ValueError("refusing Amazon rows without complete source attribution") ph = ", ".join("?" for _ in cols) con.executemany( f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES ({ph})', @@ -453,9 +540,18 @@ def credential(tenant_key="royal-imports", rt=None): before pasting it. Serving it here would hand one tenant another's Amazon seller account, which is precisely the elevation `keychain.TENANT_WIDE_TYPES` was written to stop. """ + key = _tenant_key(tenant_key) + runtime = rt if rt is not None else _runtime_for(key) + # Keep identity failures OUTSIDE the optional-keychain try: fail closed with the diagnosis, + # not a silent "no credential". A resolved runtime must match too, not only an explicit one. + if runtime is not None: + runtime_key = str(getattr(runtime, "key", "") or "").strip().lower() + if runtime_key != key: + raise RuntimeError( + f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}") try: import core.keychain as kc - return kc.amazon_creds(rt if rt is not None else _runtime_for(tenant_key)) + return kc.amazon_creds(runtime) except Exception: # noqa: BLE001 return None @@ -524,6 +620,151 @@ def _quoted_identifier(value): return '"' + str(value).replace('"', '""') + '"' +class AmazonRetentionOverdue(RuntimeError): + """A dated deletion obligation is overdue, not a throttle or operation refusal.""" + + +class AmazonAuditUnavailable(RuntimeError): + """The mandatory C2 event failed; its failure is durable in the lifecycle ledger.""" + + +def _utc(value=None): + value = value or datetime.now(timezone.utc) + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("a timezone-aware deletion timestamp is required") + return value.astimezone(timezone.utc) + + +def _control_stamp(value=None): + # Preserve subsecond ordering: a notice arriving AFTER a purge in the same second is new + # work. Report-request windows keep their existing second-precision _iso contract. + return _utc(value).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +LIFECYCLE_KEY = "amazon_lifecycle" + + +def _ledger_runtime(tenant_key, rt=None): + """Resolve the ONE tenant store; an unavailable obligation ledger must fail loudly.""" + key = _tenant_key(tenant_key) + rt = rt if rt is not None else _runtime_for(key) + if rt is None: + raise RuntimeError("Amazon lifecycle ledger requires a resolved tenant runtime") + runtime_key = str(getattr(rt, "key", "") or "").strip().lower() + if runtime_key != key: + raise RuntimeError( + f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}") + return rt + + +def _ledger(tenant_key, rt=None): + # The tenant-store document SURVIVES purge and mirror restore: neither touches this key. + # Only tenant slug, event kind, UTC dates and booleans. No Amazon rows, document IDs, + # report IDs or credentials. It is not an amzn_ object or a _sync_state row, so the mirror's + # collateral snapshot stays exact. Fresh reads cannot hide another process's obligation. + rt = _ledger_runtime(tenant_key, rt) + return rt.get(LIFECYCLE_KEY, fresh=True) or { + "triggers": [], "checks": [], "audit_failures": []} + + +def _append_ledger(tenant_key, rt, section, row): + rt = _ledger_runtime(tenant_key, rt) + + def append(doc): + for part in ("triggers", "checks", "audit_failures"): + doc.setdefault(part, []) + # Preserve INSERT OR IGNORE's exact-event idempotency for triggers only. + if section != "triggers" or row not in doc[section]: + doc[section].append(row) + + # Commit through the tenant's bound backend before returning; never buffer an obligation. + rt.update(LIFECYCLE_KEY, append, flush="sync") + + +def _c2_event(kind, tenant, principal, detail, rt=None): + try: + # C2 is declared by lane F. Missing or failing audit is NEVER a successful emission. + from importlib import import_module + audit = import_module("core.audit") + audit.record(kind, tenant, principal, detail) + except Exception as exc: + _append_ledger(tenant, rt, "audit_failures", + {"tenant": tenant, "kind": kind, "checked_at": _control_stamp()}) + raise AmazonAuditUnavailable( + f"mandatory C2 event {kind} failed; recorded in the lifecycle ledger") from exc + + +def record_deletion_trigger(tenant_key, trigger, *, occurred_at=None, rt=None, + principal="system"): + """Record the real event time; calling this schedules an obligation, never deletes data. + + Credential-revocation, termination and notice owners must call this at their event boundary. + Authorization-loss owners use the same door, which also emits C2. Current keychain absence + alone is NOT evidence of when revocation occurred. + """ + key = _tenant_key(tenant_key) + if trigger not in DELETION_TRIGGERS: + raise ValueError("unknown Amazon deletion trigger") + when = _utc(occurred_at) + if when > _utc(): + raise ValueError("a deletion trigger cannot be dated in the future") + rt = _ledger_runtime(key, rt) + _append_ledger(key, rt, "triggers", + {"tenant": key, "kind": trigger, "occurred_at": _control_stamp(when)}) + if trigger == "authorization_loss": + _c2_event("amazon.authorization_lost", key, principal, + {"trigger": trigger, "occurred_at": _control_stamp(when)}, rt) + return retention_check(key, rt=rt) + + +def retention_check(tenant_key, *, rt=None, checked_at=None, scope="all", raise_overdue=False): + """Dated obligation check. Only a real readback can complete the covered mirror scope. + + Full-service completion remains unproven until workbook/export/backup owners wire deletion. + A local purge must not move their deadline or turn this default all-scope check green. + """ + key, now = _tenant_key(tenant_key), _utc(checked_at) + if scope not in ("all", "mirror"): + raise ValueError("deletion scope must be all or mirror") + ledger = _ledger(key, rt) + triggers = sorted(r["occurred_at"] for r in ledger["triggers"] if r["tenant"] == key) + checks = sorted(r["checked_at"] for r in ledger["checks"] + if r["tenant"] == key and r["kind"] == "purge" and r["success"]) + # A check in the future cannot prove deletion at the requested historical observation time. + checks = [v for v in checks if _utc(v) <= now] + triggers = [v for v in triggers if _utc(v) <= now] + outstanding = [v for v in triggers if not ( + scope == "mirror" and any(c >= v for c in checks))] + earliest = outstanding[0] if outstanding else None + deadline = _utc(earliest) + timedelta(days=DELETION_DAYS) if earliest else None + state = ("OVERDUE" if now > deadline else "DUE") if deadline else "NO_TRIGGER" + if not outstanding and triggers and scope == "mirror" and checks: + # Historical late deletion stays late, even after a later successful retry. + state = "COMPLETED_LATE" if any( + _utc(next(c for c in checks if c >= t)) > _utc(t) + timedelta(days=DELETION_DAYS) + for t in triggers if _utc(t) <= now) else "COMPLETED_ON_TIME" + result = {"tenant": key, "scope": scope, "state": state, "checkedAt": _control_stamp(now), + "earliestTriggerAt": earliest, + "deadlineAt": _control_stamp(deadline) if deadline else None, + "mirrorDeletedAt": checks[-1] if checks else None} + if state == "OVERDUE" and raise_overdue: + raise AmazonRetentionOverdue("Amazon deletion deadline has passed; see the dated check") + return result + + +def _record_deletion_check(key, rt, kind, success): + stamp = _control_stamp() + _append_ledger(key, rt, "checks", + {"tenant": key, "kind": kind, "checked_at": stamp, "success": bool(success)}) + return stamp + + +def _is_amazon_object(name): + return str(name).startswith((PREFIX, "ut_amzn_")) + + def _amazon_objects(con): """Every live mirror object reserved to Amazon by the `amzn_` naming contract.""" rows = con.execute( @@ -532,7 +773,7 @@ def _amazon_objects(con): out = [] for name, kind in rows: name = str(name) - if not name.startswith(PREFIX): + if not name.startswith(PREFIX) and not name.startswith("ut_amzn_"): continue out.append((name, str(kind or "BASE TABLE").upper())) return sorted(out, key=lambda row: ("VIEW" not in row[1], row[0])) @@ -545,18 +786,30 @@ def _sync_state_rows(con): "FROM _sync_state ORDER BY entity").fetchall()] -def _purge_mirror(con): +def _collateral_rows(con): + """Compare complete non-Amazon row multisets, including duplicate counts, not just names.""" + from collections import Counter + names = [str(r[0]) for r in con.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() + if not _is_amazon_object(r[0]) and r[0] != "_sync_state"] + return {name: Counter(repr(tuple(row)) for row in con.execute( + f"SELECT * FROM {_quoted_identifier(name)}").fetchall()) for name in names} + + +def _purge_mirror(con, tenant_key, confirmation): """Transactionally delete every `amzn_` mirror object and state marker, then read it back. The prefix is the connector's reserved namespace, so this also catches a future Amazon table that has not yet been added to `TABLES`. Non-Amazon tables and sync markers are snapshotted and compared inside the transaction and again after commit. No caller supplies a table name. """ + _require_purge_confirmation(tenant_key, confirmation) objects = _amazon_objects(con) before_tables = {str(row[0]) for row in con.execute( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='main'").fetchall()} - before_other_tables = sorted(name for name in before_tables if not name.startswith(PREFIX)) + before_other_tables = sorted(name for name in before_tables if not _is_amazon_object(name)) + before_other_rows = _collateral_rows(con) before_state = _sync_state_rows(con) amazon_state = [str(row[0]) for row in before_state if str(row[0]).startswith(PREFIX)] other_state = [row for row in before_state if not str(row[0]).startswith(PREFIX)] @@ -577,12 +830,14 @@ def _purge_mirror(con): raise RuntimeError("Amazon mirror objects remain inside the purge transaction") inside_other_tables = sorted(str(row[0]) for row in con.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() - if not str(row[0]).startswith(PREFIX)) + if not _is_amazon_object(row[0])) if inside_other_tables != before_other_tables: raise RuntimeError("a non-Amazon mirror object changed during the purge") inside_state = _sync_state_rows(con) if inside_state != other_state: raise RuntimeError("a non-Amazon sync marker changed during the purge") + if _collateral_rows(con) != before_other_rows: + raise RuntimeError("non-Amazon rows changed during the purge") con.execute("COMMIT") except Exception: try: @@ -596,13 +851,14 @@ def _purge_mirror(con): if str(row[0]).startswith(PREFIX)] after_other_tables = sorted(str(row[0]) for row in con.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() - if not str(row[0]).startswith(PREFIX)) + if not _is_amazon_object(row[0])) after_other_state = [row for row in _sync_state_rows(con) if not str(row[0]).startswith(PREFIX)] if residue_objects or residue_state: raise RuntimeError( "Amazon mirror purge committed but independent readback found residue") - if after_other_tables != before_other_tables or after_other_state != other_state: + if (after_other_tables != before_other_tables or after_other_state != other_state + or _collateral_rows(con) != before_other_rows): raise RuntimeError( "Amazon mirror purge committed but independent readback found collateral change") return { @@ -614,7 +870,56 @@ def _purge_mirror(con): } -def purge(tenant_key, confirmation, rt=None, log=print): +def _purge_materialized(rt, tenant_key, confirmation): + """Extend the operator purge through the existing tenant-bound authorship store seam.""" + _require_purge_confirmation(tenant_key, confirmation) + _tenant_mirror_path(tenant_key, rt) + if not callable(getattr(rt, "update", None)): + return {"materializedVerified": False, "materializedTables": 0, "materializedRows": 0} + from copy import deepcopy + removed, kept = {}, {} + + def remove(doc): + removed.clear() + kept.clear() + for name, value in doc.items(): + if str(name).startswith("ut_amzn_"): + removed[name] = len((value or {}).get("rows") or []) + else: + kept[name] = deepcopy(value) + for name in removed: + del doc[name] + + rt.update("user_tables", remove, flush="sync") + readback = rt.get("user_tables", fresh=True) or {} + if any(str(name).startswith("ut_amzn_") for name in readback) or readback != kept: + raise RuntimeError("materialized Amazon purge readback found residue or collateral change") + return {"materializedVerified": True, "materializedTables": len(removed), + "materializedRows": sum(removed.values())} + + +def _amazon_cache_key(value): + if isinstance(value, (tuple, list)): + return any(_amazon_cache_key(item) for item in value) + return isinstance(value, str) and _is_amazon_object(value) + + +def _purge_runtime_caches(rt, tenant_key, confirmation): + _require_purge_confirmation(tenant_key, confirmation) + _tenant_mirror_path(tenant_key, rt) + count = 0 + for attr in ("pool_cache", "measure_memo", "mset_memo", "series_memo"): + cache = getattr(rt, attr, {}) + for key in list(cache): + if _amazon_cache_key(key): + del cache[key] + count += 1 + if any(_amazon_cache_key(key) for key in cache): + raise RuntimeError("Amazon runtime cache residue remains") + return count + + +def purge(tenant_key, confirmation, rt=None, log=print, *, principal="operator"): """Permanently remove one tenant's live Amazon mirror after credential revocation. This is an operator-only lifecycle control. It never revokes a credential itself and refuses @@ -631,17 +936,102 @@ def purge(tenant_key, confirmation, rt=None, log=print): raise RuntimeError( f"tenant {key!r} has no existing mirror file; refusing to create one as purge proof") _assert_amazon_credential_revoked(rt) - con, bound_path = _connect_tenant_mirror(key, rt) + success, result = False, {} try: - result = _purge_mirror(con) + con, bound_path = _connect_tenant_mirror(key, rt) + try: + result = _purge_mirror(con, key, confirmation) + finally: + con.close() + result.update(_purge_materialized(rt, key, confirmation)) + result["runtimeCacheEntries"] = _purge_runtime_caches(rt, key, confirmation) + # Keep the existing post-commit query and add a fresh, independently opened cursor. + import duckdb + with closing(duckdb.connect(str(bound_path))) as readback: + if _amazon_objects(readback) or any( + str(r[0]).startswith(PREFIX) for r in _sync_state_rows(readback)): + raise RuntimeError("independent mirror file readback found Amazon residue") + success = True finally: - con.close() - report = {"tenant": key, "database": bound_path.name, **result} + stamp = _record_deletion_check(key, rt, "purge", success) + # Counts only. Even database-discovered table names could contain customer data, so + # never send them (or exception strings) to C2's flat, secret-refusing detail object. + _c2_event("data.purge", key, principal, + {"success": success, "checked_at": stamp, "scope": "local", + "rows": result.get("rows", 0), "tables": len(result.get("tables", [])), + "known_tables": ",".join(table_name(t) for t in TABLES + if table_name(t) in result.get("tables", [])), + "materialized_tables": result.get("materializedTables", 0), + "materialized_rows": result.get("materializedRows", 0)}, rt) + report = {"tenant": key, "database": bound_path.name, **result, "checkedAt": stamp, + "complete": False, + "unverifiedScopes": ["workbooks", "service_retained_exports", "backups", + "other_process_and_unattributed_caches"], + "retention": retention_check(key, rt=rt)} + if not result.get("materializedVerified"): + report["unverifiedScopes"].append("materialized_store") log(f" tenant {key}: permanently deleted {result['rows']:,} Amazon mirror rows from " f"{len(result['tables']):,} tables and {result['syncMarkers']:,} sync markers") return report +def _restore_readback(path): + import duckdb + with closing(duckdb.connect(str(path), read_only=True)) as con: + objects = _amazon_objects(con) + state = [r for r in _sync_state_rows(con) if str(r[0]).startswith(PREFIX)] + if objects or state: + raise RuntimeError("restored Amazon mirror contains purged residue") + return _collateral_rows(con) + + +def restore(tenant_key, confirmation, backup_path, destination, *, rt=None, + principal="operator"): + """Restore a real backup to a NEW local copy, read it back before publishing that copy. + + This never activates a service store or rewrites the live mirror. A dirty pre-purge backup + is refused, not silently sanitized by a second purge implementation. The authoritative + lifecycle ledger must stay outside the backup being restored. + """ + key = _require_purge_confirmation(tenant_key, confirmation) + rt = rt if rt is not None else _runtime_for(key) + if rt is None: + raise RuntimeError("restore requires a resolved tenant runtime") + live_path = _tenant_mirror_path(key, rt).resolve() + _assert_amazon_credential_revoked(rt) + source, target = Path(backup_path).resolve(), Path(destination).resolve() + if target.parent != live_path.parent or target == live_path or target.exists(): + raise RuntimeError("restore requires a new copy beside this tenant's mirror") + if not source.is_file(): + raise RuntimeError("restore requires an existing backup file") + proof = any(r["tenant"] == key and r["kind"] == "purge" and r["success"] + for r in _ledger(key, rt)["checks"]) + if not proof: + raise RuntimeError("restore requires a recorded real purge readback") + success, temporary = False, None + try: + fd, name = tempfile.mkstemp(prefix=".amazon-restore-", suffix=".duckdb", dir=target.parent) + os.close(fd) + temporary = Path(name) + shutil.copyfile(source, temporary) + before = _restore_readback(source) + if _restore_readback(temporary) != before: + raise RuntimeError("restored non-Amazon rows differ from the backup") + # Hard-link publication refuses an existing target, including a concurrent creator. + os.link(temporary, target) + if _restore_readback(target) != before: + raise RuntimeError("published restore readback differs from the backup") + success = True + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + stamp = _record_deletion_check(key, rt, "restore", success) + _c2_event("data.restore", key, principal, + {"success": success, "checked_at": stamp, "scope": "mirror_copy"}, rt) + return {"tenant": key, "checkedAt": stamp, "residueTables": [], + "residueSyncMarkers": [], "scope": "mirror_copy", "success": success} + + def _client(): """The SP-API transport, from the DATA LAYER. @@ -681,8 +1071,44 @@ def _read_state(con, key): return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]} +def _sync_event(kind, tenant, table, detail, rt, report, log): + """Sync is nondestructive: preserve C2's loud failure, but continue the data pull.""" + try: + _c2_event(kind, tenant, "system", {"table": table, **detail}, rt=rt) + except Exception as exc: + # The emitter records the failure in the lifecycle ledger before raising. If that store + # also fails, the report/log remains the fallback; never claim an event was delivered. + message = (f"{table}: {kind} AUDIT UNAVAILABLE ({type(exc).__name__}); " + "restore core.audit and the tenant lifecycle store; sync continues") + report["audit_problems"].append(message) + log(message) + + +def _sync_problem(exc, table, operation, tenant, rt, report, log): + from harness.amazon_client import AmazonThrottled, AmazonOperationRefused + cause = f"{type(exc).__name__}: {exc}" + if isinstance(exc, AmazonThrottled): + retry = exc.retry_after + state = "throttled" + fix = (f"retry_after={retry if retry is not None else 'unspecified'}; " + "retry after the vendor delay, widen report windows or reduce sync frequency") + _sync_event("amazon.throttled", tenant, table, + {"operation": operation, "retry_after": str(retry), "cause": "HTTP 429"}, + rt, report, log) + elif isinstance(exc, AmazonOperationRefused): + state = "refused" + fix = "use an operation in the approved read-only scope; check the connector operation mapping" + else: + state = "refused" + fix = "check the report and operation's required roles, credential and vendor response; retry" + message = f"{table}: {operation} {state.upper()}: {cause}; fix: {fix}" + report["problems"].append(message) + log(message) + return {"state": state, "reason": message} + + def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, - max_wait=900): + max_wait=900, *, scheduled=False): """Pull every declared table into the tenant's mirror. -> a report dict. Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the @@ -714,11 +1140,44 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None "reach a grid (standing rule 8, and R3's reconciliation). Use it to prove the client " "(`connectors_amazon.py --check`) and sync only a PRODUCTION credential.") - want = [t for t in (tables or TABLES) if t in TABLES] + want = [t for t in (TABLES if tables is None else tables) if t in TABLES] end = datetime.now(timezone.utc) start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS)) + report = {"tenant": tenant_key, "tables": {}, "started": _iso(end), + "problems": [], "audit_problems": []} + enablement = read_database_enablement(rt) + report["available"] = enablement["available"] + for table in want: + if not enablement["databases"][amazon_schema.DATABASE_PREFIX + table]: + reason = ("database disabled; re-enable it in Settings, Connectors to resume; existing data preserved" + if enablement["available"] else + "enablement document malformed or unreadable; capability unavailable; " + "repair amazon_connector_enablement with known database keys and boolean values " + "and restore tenant-store access; existing data preserved") + report["tables"][table] = {"state": "disabled", "reason": reason} + log(f"{table}: SKIPPED: {reason}") + if not enablement["available"]: + report["problems"].append(f"{table}: {reason}") + want = [table for table in want if table not in report["tables"]] + if not want: + report["finished"] = _iso(datetime.now(timezone.utc)) + return report con, _path = _connect_tenant_mirror(tenant_key, rt) - report = {"tenant": tenant_key, "tables": {}, "started": _iso(end)} + try: + obligation = retention_check(tenant_key, rt=rt, raise_overdue=True) + except Exception: + con.close() + raise + if obligation["earliestTriggerAt"]: + con.close() + raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data") + if scheduled: + plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table)) + for table in TABLES}, want, now=end) + want = plan["tables"] + for table, reason in plan["deferred"].items(): + report["tables"][table] = {"state": "deferred", "reason": reason} + log(f"{table}: DEFERRED: {reason}") for table in want: spec = cx.REPORT_TYPES.get(table) or {} @@ -731,13 +1190,22 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync") root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1] try: + # Attempt first, so a refusal or process interruption cannot starve the other tables. + _record_state(con, entity_key("sync_tick_" + table), "attempted") written, cols, cleared = 0, [], False pricing_asins = [] for w0, w1 in _windows(start, end): + # W59-T43 / C2 has no amazon.access kind. Record each credential-backed + # vendor attempt as a sync ACCESS phase, distinct from the completion event. + # Lane F/combine own the vocabulary decision; no credential or row enters C2. + _sync_event("amazon.sync", tenant_key, table, + {"state": "access", "operation": "report_document"}, rt, report, log) text, meta = cx.report_document( - fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, log=log) + fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, + log=lambda message: log(f"{table}: {message}")) if prov == "schema_artifact" and root: - parsed_cols, rows = parse_json_report(text, root) + parsed_cols, rows = parse_json_report( + text, root, document_id=meta.get("reportDocumentId"), report_type=rtype) pinned = [c for c, _t in amazon_schema.columns_for(table)] # ⛔⛔ THE UNION, NOT THE PIN, AND AMAZON'S OWN ARTIFACT IS WHY. # @@ -765,7 +1233,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None # ⚠ NO DECLARED TYPES HERE ON PURPOSE: a flat-file report publishes no schema, # so sniffing the data is the ONLY evidence there is. That is the whole # provenance split, arriving one layer down. - cols, rows = parse_flat_file(text) + cols, rows = parse_flat_file( + text, document_id=meta.get("reportDocumentId"), report_type=rtype) declared = {} if not cols: continue @@ -796,6 +1265,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None log(" pricing: getCompetitiveSummary UNPROVEN, no ASIN was returned") else: try: + _sync_event("amazon.sync", tenant_key, table, + {"state": "access", "operation": "getCompetitiveSummary"}, + rt, report, log) responses = cx.competitive_summaries(fields, pricing_asins) response_rows, updated = _apply_competitive(con, mirror, responses) _record_state(con, state_key, "done", response_rows) @@ -805,15 +1277,16 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, " f"{updated:,} mirror rows enriched") except Exception as e: # noqa: BLE001 - detail = f"{type(e).__name__}: {str(e)[:160]}" - _record_state(con, state_key, "refused", 0, detail) - operations["getCompetitiveSummary"] = { - "state": "refused", "reason": detail} - log(f" pricing: getCompetitiveSummary REFUSED: {str(e)[:160]}") + problem = _sync_problem(e, table, "getCompetitiveSummary", tenant_key, + rt, report, log) + _record_state(con, state_key, problem["state"], 0, problem["reason"]) + operations["getCompetitiveSummary"] = problem if table == "inventory": state_key = ENTITY_STATE_KEYS[table] try: + _sync_event("amazon.sync", tenant_key, table, + {"state": "access", "operation": "listInventory"}, rt, report, log) inventory = cx.awd_inventory(fields) awd_rows = _awd_rows(inventory) if awd_rows: @@ -828,10 +1301,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None "state": "done", "rows": len(awd_rows)} log(f" inventory: AWD listInventory {len(awd_rows):,} rows") except Exception as e: # noqa: BLE001 - detail = f"{type(e).__name__}: {str(e)[:160]}" - _record_state(con, state_key, "refused", 0, detail) - operations["listInventory"] = {"state": "refused", "reason": detail} - log(f" inventory: AWD listInventory REFUSED: {str(e)[:160]}") + problem = _sync_problem(e, table, "listInventory", tenant_key, rt, report, log) + _record_state(con, state_key, problem["state"], 0, problem["reason"]) + operations["listInventory"] = problem try: column_count = len(con.execute(f"PRAGMA table_info('{mirror}')").fetchall()) @@ -842,13 +1314,21 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None "rows": total_rows, "columns": column_count, "provenance": prov, "report": rtype, "operations": operations} _record_state(con, key, "done", total_rows) + _record_state(con, entity_key("sync_tick_" + table), "done", total_rows) + _sync_event("amazon.sync", tenant_key, table, + {"state": "done", "rows": total_rows, + "operations_ok": all(v["state"] == "done" for v in operations.values())}, + rt, report, log) log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}") except Exception as e: # noqa: BLE001 # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN # ticket answering itself, and it must not cost the six tables that did work. - report["tables"][table] = {"rows": 0, "state": f"{type(e).__name__}: {str(e)[:160]}"} - log(f" {table:<16} REFUSED: {str(e)[:160]}") + problem = _sync_problem(e, table, "report_document", tenant_key, rt, report, log) + report["tables"][table] = {"rows": 0, **problem} + _sync_event("amazon.sync", tenant_key, table, + {"state": problem["state"], "rows": 0}, rt, report, log) report["finished"] = _iso(datetime.now(timezone.utc)) + con.close() return report @@ -991,9 +1471,11 @@ def selftest(log=print): tsv = ("amazon-order-id\tpurchase-date\tItem Price\t2nd Ref\tsku\n" "111-2223334-4445556\t2026-08-01T00:00:00Z\t27.30\tx\tRI-001\n" "111-2223334-4445557\t2026-08-02T00:00:00Z\t14.00\ty\tRI-002\n") - cols, rows = parse_flat_file(tsv) + origin = {"document_id": "selftest-document", "report_type": "selftest-report"} + cols, rows = parse_flat_file(tsv, **origin) ck("NC-AZ1 flat-file columns are measured from the header", - cols == ["amazon_order_id", "purchase_date", "item_price", "c_2nd_ref", "sku"], cols) + cols == ["amazon_order_id", "purchase_date", "item_price", "c_2nd_ref", "sku"] + + list(_ORIGIN_TYPES), cols) # ⚠ ASSERTED AGAINST THE SANITIZER, NOT AGAINST A DEAD `.replace`. The first version of this # check asked "is there a BOM left in the output", which is true no matter what the code does # once every non-alphanumeric becomes `_`. It survived its own perturbation. It now asserts @@ -1012,13 +1494,14 @@ def selftest(log=print): # quoting then treats the rest of the document as one quoted string and silently returns a # single merged row. That is the real Amazon case, a title like `"Special" Vase`. q = 'sku\ttitle\nRI-9\t"12 inch Vase\nRI-10\tPlain\n' - qc, qr = parse_flat_file(q) + qc, qr = parse_flat_file(q, **origin) ck("NC-AZ4 a field opening with a quote does not swallow the rows after it", len(qr) == 2 and qr[0]["title"] == '"12 inch Vase' and qr[1]["sku"] == "RI-10", qr) # NC-AZ5 — a duplicate header cell must not collide. - dc, _ = parse_flat_file("sku\tsku\tqty\nA\tB\t1\n") - ck("NC-AZ5 duplicate headers are disambiguated", dc == ["sku", "sku_2", "qty"], dc) + dc, _ = parse_flat_file("sku\tsku\tqty\nA\tB\t1\n", **origin) + ck("NC-AZ5 duplicate headers are disambiguated", + dc == ["sku", "sku_2", "qty"] + list(_ORIGIN_TYPES), dc) # NC-AZ6..7 — typing sniffs the DATA, and ids stay text however numeric they look. numeric = [{"qty": "3", "price": "27.30", "settlement_id": "9007199254740993"}] @@ -1038,7 +1521,7 @@ def selftest(log=print): {"parentAsin": "B01", "salesByAsin": {"unitsOrdered": 3, "orderedProductSales": {"amount": 27.3}}, "trafficByAsin": {"buyBoxPercentage": 91.2}}]}) - jc, jr = parse_json_report(doc, "salesAndTrafficByAsin") + jc, jr = parse_json_report(doc, "salesAndTrafficByAsin", **origin) ck("NC-AZ9 nested JSON flattens to the pinned column spelling", jr and jr[0].get("sales_by_asin_ordered_product_sales_amount") == 27.3 and jr[0].get("traffic_by_asin_buy_box_percentage") == 91.2, jr[:1]) @@ -1074,7 +1557,8 @@ def selftest(log=print): # NC-AZ15 — an empty parse is empty, never a phantom row. ck("NC-AZ15 an empty document yields no columns and no rows", - parse_flat_file("") == ([], []) and parse_json_report("", "x") == ([], [])) + parse_flat_file("", **origin) == ([], []) + and parse_json_report("", "x", **origin) == ([], [])) # NC-AZ16..18 - both role-specific payloads keep their complete source shape and source label. price_input = [ @@ -1131,7 +1615,7 @@ def selftest(log=print): mem.execute("INSERT INTO keep_unrelated VALUES (99)") _record_state(mem, "amzn_orders", "done", 2) _record_state(mem, "odoo_orders", "live", 11) - purged = _purge_mirror(mem) + purged = _purge_mirror(mem, "fixture", "fixture") ck("NC-AZ21 purge removes every amzn_ table, including a future one", purged["tables"] == ["amzn_future_table", "amzn_orders", "amzn_pricing"] and purged["rows"] == 5, purged) @@ -1144,7 +1628,7 @@ def selftest(log=print): "SELECT phase, rows FROM _sync_state WHERE entity='odoo_orders'").fetchone() ck("NC-AZ23 purge preserves unrelated tables and sync markers byte for byte", kept == [(99,)] and kept_state == ("live", 11), (kept, kept_state)) - second = _purge_mirror(mem) + second = _purge_mirror(mem, "fixture", "fixture") ck("NC-AZ24 purge readback is residue-free and a retry is idempotent", second == {"tables": [], "rows": 0, "syncMarkers": 0, "residueTables": [], "residueSyncMarkers": []}, second) @@ -1199,12 +1683,34 @@ def selftest(log=print): key = "qa-purge" datastore_path = purge_path + def __init__(self, entries): + super().__init__(entries) + self.docs = {} + + def get(self, name, fresh=False): + from copy import deepcopy + return deepcopy(self.docs[name]) if name in self.docs else super().get(name) + + def update(self, name, fn, flush="sync"): + from copy import deepcopy + assert flush == "sync" + doc = deepcopy(self.docs.get(name) or {}) + fn(doc) + self.docs[name] = doc + def assert_datastore_matches(self): if Path(datastore.DB_PATH).resolve() != purge_path.resolve(): raise RuntimeError("wrong test mirror") - public_detail = purge( - "qa-purge", "qa-purge", rt=_PurgeRuntime({}), log=lambda *_a: None) + # A C2 shim is confined to this offline fixture; product paths never substitute it. + from types import ModuleType + from unittest.mock import patch + audit_fixture = ModuleType("core.audit") + events = [] + audit_fixture.record = lambda *args: events.append(args) + with patch.dict(sys.modules, {"core.audit": audit_fixture}): + public_detail = purge( + "qa-purge", "qa-purge", rt=_PurgeRuntime({}), log=lambda *_a: None) datastore.use_path(prior_path) readback = duckdb.connect(str(purge_path), read_only=True) tables_after = sorted(str(row[0]) for row in readback.execute("SHOW TABLES").fetchall()) @@ -1213,7 +1719,8 @@ def selftest(log=print): public_ok = (public_detail["tenant"] == "qa-purge" and public_detail["database"] == "qa-purge.duckdb" and public_detail["rows"] == 1 - and tables_after == ["_sync_state"] and state_after == []) + and tables_after == ["_sync_state"] and state_after == [] + and len(events) == 1 and events[0][0] == "data.purge") finally: datastore.use_path(prior_path) ck("NC-AZ27 the public purge wrapper deletes and reads back an isolated tenant file", @@ -1230,12 +1737,35 @@ def main(argv=None): ap.add_argument("--validate", action="store_true") ap.add_argument("--purge", action="store_true", help="permanently delete every Amazon mirror table after credential revocation") + ap.add_argument("--deletion-trigger", choices=DELETION_TRIGGERS) + ap.add_argument("--trigger-at", help="original event timestamp with UTC offset") + ap.add_argument("--retention-check", action="store_true") + ap.add_argument("--restore-from", help="existing backup file to restore as a checked copy") + ap.add_argument("--restore-to", help="new copy beside the tenant mirror, never the live file") ap.add_argument("--selftest", action="store_true") ap.add_argument("--tenant", default=None) ap.add_argument("--confirm-tenant", default=None, - help="for --purge, repeat the exact tenant slug") + help="for --purge or --restore-from, repeat the exact tenant slug") ap.add_argument("--table", action="append", help="limit to one table (repeatable)") a = ap.parse_args(argv) + if sum(bool(v) for v in (a.sync, a.status, a.validate, a.purge, a.selftest, + a.deletion_trigger, a.retention_check, a.restore_from)) > 1: + ap.error("choose exactly one action") + if a.trigger_at and not a.deletion_trigger: + ap.error("--trigger-at requires --deletion-trigger") + if bool(a.restore_from) != bool(a.restore_to): + ap.error("--restore-from and --restore-to are required together") + if a.deletion_trigger or a.retention_check or a.restore_from: + if not a.tenant or a.table: + ap.error("lifecycle actions require an explicit --tenant and no --table") + if a.deletion_trigger: + rep = record_deletion_trigger(a.tenant, a.deletion_trigger, occurred_at=a.trigger_at) + elif a.restore_from: + rep = restore(a.tenant, a.confirm_tenant, a.restore_from, a.restore_to) + else: + rep = retention_check(a.tenant) + print(json.dumps(rep, indent=2)) + return 1 if rep.get("state") == "OVERDUE" else 0 tenant = a.tenant or "royal-imports" if a.purge: if not a.tenant: diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py index f40b5ce5c62828b64305a531a666ceb6bb665fcd..0d6006c31170a76cec92accdf7844251f89f0b0e 100644 --- a/platform/modules/sheet.py +++ b/platform/modules/sheet.py @@ -34,8 +34,8 @@ from __future__ import annotations import datetime import re -#: C2's aggregate vocabulary. `metric_key` is the other arm of that union, and W53-T13 answers it -#: by NAME rather than leaving it dead: see `metric_key_refusal` and its refusal's own note. +#: C2's aggregate vocabulary. W59-T14 resolves the `metric_key` arm through a Metric column, +#: using only the catalogue's declared fold: see `metric_key_fold` and its additivity guard. AGGREGATE_FNS = ("sum", "count", "avg", "min", "max") #: ⭐ R9's PERIOD (wave 51, W51-T21). The three units a bound cell can be pinned to, and the month @@ -46,6 +46,9 @@ PERIOD_UNITS = ("month", "quarter", "year") #: named by the month they open on, and one spelling means a copied cell cannot mean two things. _PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$") +#: Literal inclusive endpoints of a frozen calendar period, spelled `YYYY-MM-DD`. +_PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$") + #: A `period.ref` points at a cell that holds the period, so it is a plain A1 address with no `$`, #: exactly as `core/sheets.py::_ADDR` spells one. _PERIOD_REF = re.compile(r"^[A-Z]{1,3}[1-9][0-9]{0,6}$") @@ -83,15 +86,26 @@ REFUSALS = { # column somebody asked to sum), and computing it in one place is what stops three doors # wording the same fact three ways ([[one-question-two-normalizers]]). "field_not_aggregable": "{subject} cannot be totalled on this database, because {detail}", - # ⭐ C2's OTHER aggregate arm. `{metric_key}` names a number measured over a whole SET, and a - # set measure is not additive: summing a per-record average or a percentage over the rows this - # cell scopes produces a confident wrong number. The Metric COLUMN path answers the same - # question honestly, per database and per period, which is exactly ruling R7, so the sentence - # sends the reader there rather than stopping at "no". + # W59-T14 / R10: now unreachable. The route creates or reuses the Metric column after + # admission and the declared-aggregate check, resolving the missing-column condition. + # Keep the code declared for the refusal vocabulary contract. "metric_key_unsupported": "{subject} is measured over a whole set of records rather than " "stored on each one, so this cell cannot total it directly. Add a " "Metric column for it on this database, then point this cell at " "that column.", + "metric_not_additive": "{subject} cannot be totalled from per-record values. Choose a sum " + "or count measure in the cell's Metric section instead.", + "metric_column_conflict": "A column already uses the key for {subject}. Open this " + "database's Fields and bind the cell to a visible Metric column " + "for that measure.", + "metric_period_required": "{subject} is measured over a period, so this cell needs one. " + "Open the cell's Period section and choose a period.", + "measure_not_offered": "{subject} is not an available Metric on this database. Choose a " + "Metric offered here or select a database that offers it.", + "metric_column_unavailable": "{subject} cannot be materialised as a Metric column with a " + "supported aggregate. Choose another Metric or use a numeric field.", + "metric_column_create_failed": "The Metric column for {subject} could not be saved. Try " + "binding the cell again or add it from the database's Fields panel.", # ── C5's two sharing refusals (wave 51, W51-T25) ────────────────────────────────────────── # # ⭐⭐ THEY LIVE HERE, WITH C2's, BECAUSE THIS DICT IS THE ONE PLACE A SHEET REFUSAL IS @@ -130,7 +144,10 @@ class SheetRefusal(Exception): def clean_period(raw): - """R9's period, or None. `{unit, start, locked, ref?}`, with `{ref}` / `{window}` still read. + """R9's period, or None. `{unit, start, locked?, ref?}`, with `{ref}` / `{window}` still read. + + A frozen period is `{unit, start, locked?, frozen: true, from, to}`. `frozen` stores only + True so freezing and unfreezing returns to the identical rolling period already stored. ⛔ AN UNKNOWN UNIT OR A MALFORMED START IS REFUSED BY NAME, and that is not the same call as the cosmetic keys next door. A period decides WHICH ROWS ARE COUNTED. Dropping `unit: "week"` @@ -148,9 +165,10 @@ def clean_period(raw): fills the missing start with TODAY'S MONTH and shows it as a chosen period. The next save then stamps `{unit: "month", start: }` over a cell nobody meant to narrow: an all-time total silently becomes this month's, which is exactly what D-660 names. So a period must name a - window - a `start`, the legacy `window`, or R14's `ref` - or it is dropped and the cell stays - all time. ⚠ THE ORDER MATTERS: an unknown unit or a malformed start is still REFUSED BY NAME - above, because a wrong period and an absent one are different answers. + window - a `start`, validated frozen dates, the legacy `window`, or R14's `ref` - or it is + dropped and the cell stays all time. ⚠ THE ORDER MATTERS: an unknown unit or a malformed + start is still REFUSED BY NAME above, as are missing or malformed frozen dates, because a + wrong period and an absent one are different answers. """ if not isinstance(raw, dict): return None @@ -168,6 +186,21 @@ def clean_period(raw): raise SheetRefusal("period_invalid", start) out["start"] = start + if raw.get("frozen") is True: + dates = {} + for key in ("from", "to"): + value = str(raw.get(key) or "").strip() + # ⚠ `value or key` because a MISSING date has no value to name, and a refusal with an + # empty subject reads as " is not a period this cell can use" - the same blank-subject + # defect the Metric binding originally exposed. + if not _PERIOD_DAY.fullmatch(value): + raise SheetRefusal("period_invalid", value or key) + dates[key] = value + # ISO days have fixed-width year/month/day parts, so string order is date order. + if dates["to"] < dates["from"]: + raise SheetRefusal("period_invalid", dates["to"]) + out.update({"frozen": True, **dates}) + if raw.get("locked") is True: out["locked"] = True @@ -175,17 +208,20 @@ def clean_period(raw): if ref: if not _PERIOD_REF.match(ref): raise SheetRefusal("period_invalid", ref) - out["ref"] = ref + if not out.get("frozen"): + out["ref"] = ref #: ⚠ The wave-49 spelling, kept so a binding written before R9 still round trips. It is not a #: second dialect for the same thing: `window` is a raw relative window, `unit` + `start` is a #: named calendar period, and a document carrying the old one is not rewritten on read. - if raw.get("window") is not None and "unit" not in out and "start" not in out: + if (raw.get("window") is not None and "unit" not in out and "start" not in out + and not out.get("frozen")): out["window"] = raw["window"] #: ⛔ D-660: see the docstring. A unit on its own, a lock on its own, or the empty dict names no - #: window, so it is not a period and the cell keeps its all-time answer. - if not ({"start", "window", "ref"} & set(out)): + #: window, so it is not a period and the cell keeps its all-time answer. A frozen period + #: names its window through the two literal dates already validated above. + if not ({"start", "window", "ref", "frozen"} & set(out)): return None return out @@ -218,6 +254,9 @@ def period_window(period): the following month into a monthly cell. It is computed as "the day before the month after the period", never as a 28/30/31 table, so February and a leap year need no special case. + A frozen period returns the person's own two dates and ignores `unit` / `start`, which is + why unfreezing restores their derived range. + ⚠ A `{window}` PERIOD (the wave-49 spelling `clean_period` still round trips) IS RETURNED AS IT STANDS. It already IS a window spec; normalising it here would be this module's second opinion about a vocabulary `harness/windows.py` owns, and `normalize` refuses anything it does @@ -231,6 +270,8 @@ def period_window(period): """ if not isinstance(period, dict): return None + if period.get("frozen") is True and period.get("from") and period.get("to"): + return {"kind": "custom", "from": period["from"], "to": period["to"]} if period.get("window") is not None: return period["window"] start = str(period.get("start") or "").strip() @@ -353,29 +394,38 @@ def aggregate_refusal(aggregate, field): return None -def metric_key_refusal(aggregate): - """The named refusal for C2's `{metric_key}` arm, or None when the binding is not that arm. - - ⛔⛔ IT IS ANSWERED RATHER THAN LEFT TO FALL THROUGH, and that is the whole of W53-T13's (d). - `clean_binding` accepts `{metric_key}` and nothing downstream ever resolved it, so it reached - `aggregate_sql` with no `fn` and no `field` and died as `field_unknown` with an EMPTY subject: - "That column is not a column on this database." Measured on the running app for - `{"metric_key": "revenue"}`. That sentence names nothing and is not even true. - - ⭐ WHY IT REFUSES RATHER THAN RESOLVES, and this is standing rule 9 read forwards. A metric is - measured over a SET: `harness/measure_filter.py::resolve_series`' own docstring records the - additivity law it obeys, that a non-additive measure (an average, a percentage) is computed at - POOL grain and never folded from per-record answers. A bare metric key carries no aggregate - and no window, so resolving it here would mean choosing both, and choosing `sum` would print a - wrong number for every ratio metric in the catalogue with nothing to notice. The Metric COLUMN - path answers the same question with the aggregate and the window both DECLARED, per database, - which is precisely what R7 asked for, so the sentence sends the reader one step sideways - rather than stopping them. +def metric_key_fold(measure_key, declared_agg): + """The column fold for a declared additive measure; otherwise a named refusal. + + W53-T13's original refusal answered a bare key rather than letting it die as field_unknown + with an empty subject. Its hazard remains: a metric is measured over a SET, and + `harness/measure_filter.py::resolve_series` computes non-additive measures at POOL grain, + never by folding per-record answers. A bare metric key carries no aggregate and no window, + so resolving it here would mean choosing both, and choosing `sum` would print a wrong + number for every ratio metric in the catalogue with nothing to notice. + + R10 resolves only `sum` and `count`: both add across entities, so their per-record answers + fold with SUM, never COUNT of the column. Ratio, derived and distinct-count declarations + still refuse. The route reads `semantic.metrics()[key]['agg']` and requires the cell's + declared period before creating a column; this pure module guesses neither one. """ - key = str((aggregate or {}).get("metric_key") or "").strip() - if not key: - return None - return SheetRefusal("metric_key_unsupported", key) + if declared_agg in ("sum", "count"): + return "sum" + raise SheetRefusal("metric_not_additive", measure_key) + + +def measure_column_key(measure_key, window): + """Identity of a measure at an already normalised window, without a second window cleaner. + + Mirrors `customer-grid/types.ts::windowKey` (`kind|n|from|to`) and `measureColumnIndex`. + Callers normalise through `harness.windows.normalize`, just as the client calls + `normalizeWindow`. The route prefers this exact identity but reuses any column of the same + measure when no window matches, because the cell re-resolves it at its own period. + """ + w = window if isinstance(window, dict) else {} + parts = [str(w.get(k)) if w.get(k) is not None else "" + for k in ("kind", "n", "from", "to")] + return str(measure_key) + "|" + ("|".join(parts) if w else "") def period_limits(period, aggregate, field, bounded_keys=(), depends=()): @@ -482,7 +532,8 @@ def clean_binding(raw): `Binding = {database, aggregate:{fn, field?} | {metric_key}, scope:{view_id} | {filter} (+ adoptedFrom?), - period?: {unit, start, locked, ref?} | {window}, onDrag?: {across, down}}` + period?: {unit, start, locked?, ref?} | {unit, start, locked?, frozen: true, from, to} + | {window}, onDrag?: {across, down}}` """ if not isinstance(raw, dict): raise SheetRefusal("binding_invalid") diff --git a/web/package-lock.json b/web/package-lock.json index 19eba657181a54f4eef9d7b52c03413cddeedbdc..c5d6675a9017fa7953dd2a44a3cae2bcc3a1702e 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -26,6 +26,10 @@ "typescript": "~6.0.2", "vite": "^8.1.1", "vite-plugin-singlefile": "^2.3.3" + }, + "engines": { + "node": "22.x", + "npm": "10.x" } }, "node_modules/@babel/code-frame": { @@ -56,7 +60,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -326,6 +329,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "@emnapi/wasi-threads": "1.2.3", "tslib": "^2.4.0" @@ -338,6 +342,19 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -1184,7 +1201,6 @@ "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -1202,7 +1218,6 @@ "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -1341,7 +1356,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.44", "caniuse-lite": "^1.0.30001806", @@ -1386,7 +1400,8 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/cliui": { "version": "9.0.1", @@ -2210,8 +2225,7 @@ "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/loose-envify": { "version": "1.4.0", @@ -2239,7 +2253,6 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -2328,6 +2341,7 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -2444,7 +2458,6 @@ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -2486,6 +2499,7 @@ "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", @@ -2497,7 +2511,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -2510,7 +2523,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -2524,6 +2536,7 @@ "resolved": "https://registry.npmjs.org/react-easy-swipe/-/react-easy-swipe-0.0.21.tgz", "integrity": "sha512-OeR2jAxdoqUMHIn/nS9fgreI5hSpgGoL5ezdal4+oO7YSSgJR8ga+PkYGJrSrJ9MKlPcQjMQXnketrD7WNmNsg==", "license": "MIT", + "peer": true, "dependencies": { "prop-types": "^15.5.8" }, @@ -2544,7 +2557,8 @@ "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/react-number-format": { "version": "5.4.5", @@ -2804,7 +2818,6 @@ "resolved": "https://registry.npmjs.org/vega/-/vega-6.3.1.tgz", "integrity": "sha512-mX5tvY3ISCSiPPmunuZyQfccq0XlUvJd2t5oyc6SNS0N0TwnPNoklx0mVod5n+qbzuUoiuxl7Ve/SvoRqH4RzA==", "license": "BSD-3-Clause", - "peer": true, "dependencies": { "vega-crossfilter": "~5.1.2", "vega-dataflow": "~6.1.2", @@ -3026,7 +3039,6 @@ "resolved": "https://registry.npmjs.org/vega-lite/-/vega-lite-6.4.3.tgz", "integrity": "sha512-d/7hPjfz560UERaQuTmGgIVfXAe3g2hJWeC+igDeaGohUdEoNrHLXgR/yTOBT8vV/lIuuKnw+0/xWWblkDwkMQ==", "license": "BSD-3-Clause", - "peer": true, "dependencies": { "json-stringify-pretty-compact": "~4.0.0", "tslib": "~2.8.1", @@ -3297,7 +3309,6 @@ "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", diff --git a/web/package-lock.linux.sha256 b/web/package-lock.linux.sha256 new file mode 100644 index 0000000000000000000000000000000000000000..64d84248f51f9fed45e676358e7ec8e49ec7d42f --- /dev/null +++ b/web/package-lock.linux.sha256 @@ -0,0 +1,2 @@ +3875e749397966dc170a4f83ae04cddc47a2989828cd5c46970cad69dedb255a package-lock.json +# resolved on linux/x64 by HF job 6aa6bba021047bf1b0384360 on 2026-09-13, npm ci passed there before upload. Regenerate ONLY with lockfile_linux.py (W-170). diff --git a/web/package.json b/web/package.json index f19240d9f05e302af0b877b3adfd2c2cc77050b2..29d65db9918a22c1afee6ca71598b6567eaefaf2 100644 --- a/web/package.json +++ b/web/package.json @@ -3,6 +3,10 @@ "private": true, "version": "0.0.0", "type": "module", + "engines": { + "node": "22.x", + "npm": "10.x" + }, "scripts": { "dev": "vite", "build": "tsc -b && vite build", diff --git a/web/src/connectors/ConnectorModal.tsx b/web/src/connectors/ConnectorModal.tsx index 8af7a654d793533725ed25679f1b2603367269f2..22eba06675165ec1132b09fd89f739ad51a529c1 100644 --- a/web/src/connectors/ConnectorModal.tsx +++ b/web/src/connectors/ConnectorModal.tsx @@ -45,7 +45,7 @@ // --------------------------------------------------------------------------- import { createContext, useCallback, useContext, useEffect, useRef, useState } from "react"; -import { API_V1, CREDENTIALS } from "../apiContract"; +import { API_V1, CREDENTIALS, UNAUTHORIZED_EVENT } from "../apiContract"; import { BodyPortal, useOverlayLayer } from "../customer-grid/OverlaySurface"; import { ConfirmDialog } from "../ui/ConfirmDialog"; import { brandForConnector } from "./brandMarks"; @@ -53,7 +53,7 @@ import DatabasesTab from "./DatabasesTab"; import type { MaterializationCapability } from "./materializationJobs"; import "../styles/wave58/connectors.css"; import type { DatabaseReveal, DatabasesTabHandle } from "./DatabasesTab"; -import KeychainTab from "./KeychainTab"; +import KeychainTab, { amazonRotationNotice, RotationNotice, useRotationClock } from "./KeychainTab"; import MonidRunDialog from "./MonidRunDialog"; import { SyncCadenceTab } from "./OdooConfig"; import RelationshipDiagram from "./RelationshipDiagram"; @@ -68,6 +68,7 @@ import { parseRelationshipGraph } from "./ConnectorsPage"; import "../styles/wave49/connectors.css"; import "../styles/wave52/connectors.css"; import "../styles/wave55/connectors.css"; +import "../styles/wave59/connectors.css"; /** R26's original four, plus "cadence" (W52-T04, the done-when's "sync cadence" section). */ export type ConnectorTab = "overview" | "keychain" | "databases" | "relationships" | "cadence"; @@ -133,6 +134,8 @@ export interface ConnectorKeychainEntry { testable: boolean; /** Server-owned admin permission for the write-only replacement flow. */ replaceable: boolean; + /** Public Amazon-issued timestamp, empty or absent when unknown. Never a credential field. */ + client_secret_rotation_deadline?: string; } /** @@ -148,6 +151,7 @@ export interface ConnectorConnectField { label: string; /** Renders as a password box, and is never echoed back. */ secret: boolean; + default?: string; } export interface ConnectorConnectSpec { @@ -300,6 +304,7 @@ function parseKeychainHalf(raw: unknown): ConnectorKeychainHalf { scope: str(entry.scope).trim(), testable: entry.testable === true, replaceable: entry.replaceable === true, + client_secret_rotation_deadline: str(entry.client_secret_rotation_deadline).trim(), }); } return { @@ -327,7 +332,9 @@ export function parseConnectSpec(raw: unknown): ConnectorConnectSpec | null { const field = obj(item); const key = str(field.key).trim(); if (!key) continue; - fields.push({ key, label: str(field.label).trim() || key, secret: field.secret === true }); + const parsed: ConnectorConnectField = { key, label: str(field.label).trim() || key, secret: field.secret === true }; + if (typeof field.default === "string" && field.default.trim()) parsed.default = field.default; + fields.push(parsed); } if (!fields.length) return null; const scopes = parseStrings(source.scopes); @@ -729,6 +736,22 @@ type Load = | { phase: "ready"; detail: ConnectorDetail } | { phase: "error"; message: string }; +const CONNECTOR_DETAIL_FRESHNESS_MS = 60_000; +// Measured reads take 1.34 s cold and 0.01 s warm, with a boot contended read above 8 s. Twenty seconds caps the wait a person sees. +// This is deliberately shorter than the server's thirty seconds so this component shows its own readable sentence before a bare 504. +const CONNECTOR_DETAIL_DEADLINE_MS = 20_000; +// Closing the modal unmounts it; keep the last successful read for the next open. +const connectorDetailCache = new Map(); + +// Shell.tsx::signOut should call this beside clearCustomersCache when its lane can wire it. +export function clearConnectorDetailCache(): void { + connectorDetailCache.clear(); +} + +if (typeof window !== "undefined") { + window.addEventListener(UNAUTHORIZED_EVENT, clearConnectorDetailCache); +} + type PendingLeave = { kind: "close" } | { kind: "tab"; tab: ConnectorTab } | null; /** @@ -759,9 +782,13 @@ export default function ConnectorModal({ */ onOpenDatabases?: () => void; }) { + const connectorKey = row.key; const panelRef = useRef(null); const [tab, setTab] = useState(initialTab); - const [load, setLoad] = useState({ phase: "loading" }); + const [load, setLoad] = useState(() => { + const cached = connectorDetailCache.get(connectorKey); + return cached ? { phase: "ready", detail: cached.detail } : { phase: "loading" }; + }); const databasesRef = useRef(null); const [databasePendingCount, setDatabasePendingCount] = useState(0); const [databaseTabBusy, setDatabaseTabBusy] = useState(false); @@ -776,9 +803,11 @@ export default function ConnectorModal({ const [reads, setReads] = useState(0); const [databaseReveal, setDatabaseReveal] = useState(null); const reread = useCallback((reveal?: DatabaseReveal) => { + // Evict immediately so closing just after a write cannot reuse the previous read. + connectorDetailCache.delete(connectorKey); if (reveal) setDatabaseReveal(reveal); setReads((n) => n + 1); - }, []); + }, [connectorKey]); /** * ⭐⭐ W49-T19 · R31 — THE AMAZON REVIEWS PULL, OPENED FROM THE OVERVIEW TAB. * A second overlay above this one, which is why it takes the PORTAL step and this modal keeps @@ -861,46 +890,83 @@ export default function ConnectorModal({ restoreFocus: true, }); - const connectorKey = row.key; useEffect(() => { let dead = false; - setLoad((current) => reads > 0 && current.phase === "ready" ? current : { phase: "loading" }); + const cached = reads === 0 ? connectorDetailCache.get(connectorKey) : undefined; + if (reads > 0) { + connectorDetailCache.delete(connectorKey); + setLoad((current) => current.phase === "ready" ? current : { phase: "loading" }); + } else if (cached) { + setLoad({ phase: "ready", detail: cached.detail }); + if (Date.now() - cached.at < CONNECTOR_DETAIL_FRESHNESS_MS) return; + } else { + setLoad({ phase: "loading" }); + } + + const controller = new AbortController(); + let timedOut = false; + const timeoutMessage = `Reading ${row.label} stopped after ${CONNECTOR_DETAIL_DEADLINE_MS / 1000} seconds because the server did not answer.`; + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + if (!dead && !cached) setLoad({ phase: "error", message: timeoutMessage }); + }, CONNECTOR_DETAIL_DEADLINE_MS); void (async () => { try { const res = await fetch( - `${API_V1}/connectors/${encodeURIComponent(connectorKey)}/detail`, - { credentials: CREDENTIALS } + `${API_V1}/connectors/${encodeURIComponent(connectorKey)}/detail${cached ? "?revalidate=1" : ""}`, + { credentials: CREDENTIALS, signal: controller.signal } ); + if (res.status === 401 || res.status === 403) clearConnectorDetailCache(); const body = (await res.json().catch(() => null)) as - | { error?: { message?: string } } + | { error?: { message?: string }; detail?: { error?: { message?: string } } } | null; if (dead) return; + if (timedOut) throw new Error(timeoutMessage); if (!res.ok) { - setLoad({ - phase: "error", - message: body?.error?.message || `The server answered ${res.status}.`, - }); - return; + throw new Error( + body?.error?.message || body?.detail?.error?.message || `The server answered ${res.status}.` + ); } const detail = parseConnectorDetail(body, connectorKey); if (!detail) { + throw new Error( + "This connector's details were refused because the answer described a different connection." + ); + } + connectorDetailCache.set(connectorKey, { detail, at: Date.now() }); + setLoad({ phase: "ready", detail }); + } catch (error) { + // A failed background read must leave the already painted detail intact. + if (!dead && !cached) { setLoad({ phase: "error", - message: - "This connector's details were refused because the answer described a different connection.", + message: timedOut ? timeoutMessage + : error instanceof Error ? error.message : "Cannot reach the server.", }); - return; } - setLoad({ phase: "ready", detail }); - } catch { - if (!dead) setLoad({ phase: "error", message: "Cannot reach the server." }); + } finally { + clearTimeout(timeout); } })(); return () => { dead = true; + clearTimeout(timeout); + controller.abort(); }; + // eslint-disable-next-line react-hooks/exhaustive-deps -- Omit row.label (timeout wording only); connectorKey identifies the row; verify_ui.py keychain-tab asserts this pair. }, [connectorKey, reads]); + const rotationNow = useRotationClock(); + const rotationEntry = row.key === "amazon_sp" && load.phase === "ready" + ? load.detail.keychain.entries + .filter((entry) => amazonRotationNotice(entry.client_secret_rotation_deadline, rotationNow)) + .sort((a, b) => Date.parse(a.client_secret_rotation_deadline || "") - + Date.parse(b.client_secret_rotation_deadline || ""))[0] + : undefined; + const rotationNotice = rotationEntry + ? amazonRotationNotice(rotationEntry.client_secret_rotation_deadline, rotationNow) : null; + const renderTab = useCallback(() => { // ⛔ THE KEYCHAIN TAB IS BELOW THE LOAD GUARD, and the placeholder it replaces was above it. // C4 serves the credential rows on the SAME payload as the overview, so a keychain tab that @@ -915,9 +981,12 @@ export default function ConnectorModal({ } if (load.phase === "error") { return ( -

- {load.message} -

+
+

{load.message}

+ +
); } if (tab === "keychain") { @@ -982,12 +1051,21 @@ export default function ConnectorModal({ ); } return ( + <> + {rotationNotice ? ( + setTab("keychain") : undefined} + /> + ) : null} setPulling(true) : undefined} /> + ); - }, [canPull, databaseReveal, load, onOpenDatabases, reread, row, tab]); + }, [canPull, databaseReveal, load, onOpenDatabases, reread, rotationEntry, rotationNotice, row, tab]); const saveDatabaseChanges = useCallback(async () => { if (!databasePendingCount || !databasesRef.current) return; @@ -1013,7 +1091,8 @@ export default function ConnectorModal({ else setTab(destination.tab); }, [discardDatabaseChanges, onClose, pendingLeave]); - const status = load.phase === "ready" ? load.detail.overview.status : row.state; + const expired = rotationNotice?.tone === "expired"; + const status = expired ? "expired" : load.phase === "ready" ? load.detail.overview.status : row.state; const title = load.phase === "ready" ? load.detail.overview.label || row.label : row.label; return ( @@ -1036,7 +1115,10 @@ export default function ConnectorModal({ card to read a state off, so until `/detail` lands there is nothing to say and an empty pill would be a coloured hole in the header. */} {status ? ( - {status} + {status} ) : null} {tab === "databases" ? (
diff --git a/web/src/connectors/DatabasesTab.tsx b/web/src/connectors/DatabasesTab.tsx index 19f4ebe8f6747cc68f200031bf29ae86f967cd55..dea355b83a301d3e55a4ae95606f374b4887eb7b 100644 --- a/web/src/connectors/DatabasesTab.tsx +++ b/web/src/connectors/DatabasesTab.tsx @@ -56,9 +56,10 @@ // `ConnectorOverviewPanel` and `KeychainTab` are exported panels. // --------------------------------------------------------------------------- -import { forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useState } from "react"; +import { forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react"; import { API_V1, CREDENTIALS, WORKSPACE_STALE_EVENT, signal } from "../apiContract"; import { ConfirmDialog } from "../ui/ConfirmDialog"; +import { useEscapeClose } from "../ui/useEscapeClose"; import type { ConnectorDatabase, ConnectorDetail } from "./ConnectorModal"; import type { ConnectorRow } from "./ConnectorsPage"; import { watchMaterialization } from "./materializationJobs"; @@ -260,14 +261,16 @@ type PopulationCountAnswer = durationMs: number; modelsCounted: number; modelsWithRows: number; + mode: string; + exactCounts: Record; errors: string[]; } | { ok: false; message: string }; -/** The explicit long-running door. Connector detail never starts this work. */ -async function countSourceRows(): Promise { +/** Both passes start on a click. Connector detail never starts this work. */ +async function countSourceRows(exact = false): Promise { try { - const res = await fetch(`${API_V1}/connectors/odoo/census`, { + const res = await fetch(`${API_V1}/connectors/odoo/census${exact ? "?exact=true" : ""}`, { method: "POST", credentials: CREDENTIALS, }); @@ -286,8 +289,20 @@ async function countSourceRows(): Promise { ? Math.max(0, Math.trunc(value)) : 0; const rawErrors = payload?.errors; + const mode = typeof payload?.mode === "string" ? payload.mode : "unknown"; + const exactCounts: Record = {}; + if ((mode === "exact" || mode === "exact_fallback") && payload?.counts && + typeof payload.counts === "object" && !Array.isArray(payload.counts)) { + for (const [model, value] of Object.entries(payload.counts)) { + if (typeof value === "number" && Number.isInteger(value) && value >= 0) { + exactCounts[model] = value; + } + } + } return { ok: true, + mode, + exactCounts, complete: payload?.complete === true, capturedAt: typeof payload?.capturedAt === "string" ? payload.capturedAt : "", durationMs: number(payload?.durationMs), @@ -333,6 +348,13 @@ function populationResult( (complete ? "" : " The count was incomplete because some source tables could not be read."); } +function sourceRowsTitle(exactCount: number | undefined): string { + const explanation = "The display threshold is 100 for speed and does not limit data pulled from the source"; + return typeof exactCount === "number" + ? `Exact total: ${exactCount.toLocaleString("en-US")} rows. ${explanation}.` + : `${explanation}; use Exact totals beside Count rows for the full total.`; +} + /** The on/off control, one shape for both rails (DESIGN.md section 4: one control shape). */ function Switch({ on, @@ -383,6 +405,7 @@ export function DatabasesUnavailable({ row, reason }: { row: ConnectorRow; reaso /** One database's disclosure: its fields, and what the catalogue left out. */ function DatabasePanel({ database, + sourceLabel, siblingLabels, fieldsVisible, canToggleFields, @@ -392,6 +415,7 @@ function DatabasePanel({ onField, }: { database: ConnectorDatabase; + sourceLabel: string; /** The OTHER databases reading the same Odoo model, by label. See the shared note below. */ siblingLabels: string[]; fieldsVisible: boolean; @@ -412,10 +436,10 @@ function DatabasePanel({

The field list for this database is administrator only.

); } - if (!catalogueComplete) { + if (!catalogueComplete && !database.fields.length) { return (

- No schema sync has finished, so this workspace holds no field list from Odoo yet. + No field list is available for this database yet.

); } @@ -428,8 +452,8 @@ function DatabasePanel({ database change. */} {siblingLabels.length ? (

- These fields come from the same Odoo source as {siblingLabels.join(", ")}, so switching - one on or off changes it there too. + These fields come from the same {sourceLabel} source as {siblingLabels.join(", ")} + {canToggleFields ? ", so switching one on or off changes it there too." : "."}

) : null} {database.fields.length ? ( @@ -469,7 +493,7 @@ function DatabasePanel({ ) : ( -

Odoo reported no fields for this database.

+

{sourceLabel} reported no fields for this database.

)}

Excluded

@@ -508,6 +532,15 @@ function DatabasePanel({ * `routes_connectors.py::_enable_odoo_database`), so there is no waiting state left to describe. * A reader who may not rename gets no control at all, rather than a sentence about a wait. */ +function PenIcon() { + return ( + + + + + ); +} + function NameCell({ database, editing, @@ -528,25 +561,29 @@ function NameCell({ /** An empty string CLEARS, which is what falling back to our default means. */ onSave: (next: string) => void; }) { + const penRef = useRef(null); + const wasEditing = useRef(editing); + useEffect(() => { + if (wasEditing.current && !editing) penRef.current?.focus(); + wasEditing.current = editing; + }, [editing]); + // Escape is claimed through useEscapeClose: the arbiter listens at document capture, + // so a local handler cannot see the key. Enter stays local because nothing arbitrates it. + useEscapeClose(editing, onCancel); + if (!editing) { - // ⚠ `conn-db-head` IS THE ROW SHAPE THIS TAB ALREADY USES for a name beside its control, and - // it is reused rather than re-declared: two adjacent inline controls with no rule between - // them set flush against each other, which is the run-on this surface reads as one word. return ( -
- {database.override ? ( - {database.override} - ) : ( - Inherits the default name - )} +
+ {database.override || database.defaultLabel} {database.canRename ? ( ) : null}
@@ -563,8 +600,16 @@ function NameCell({ id={`conn-db-rename-${database.key}`} value={draft} disabled={busy} + autoFocus autoComplete="off" onChange={(event) => onDraft(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + event.stopPropagation(); + if (!(busy || !draft.trim())) onSave(draft); + } + }} />
@@ -659,6 +704,7 @@ const DatabasesTab = forwardRef>({}); /** The one row being renamed, and the text in its box. Empty means nobody is renaming. */ const [renaming, setRenaming] = useState(""); const [draft, setDraft] = useState(""); @@ -698,6 +744,7 @@ const DatabasesTab = forwardRef database), [databases, pending.tables] ); + const hasRenamableDatabase = orderedDatabases.some((database) => database.canRename); const stageTable = useCallback( (database: ConnectorDatabase, next: boolean) => { @@ -830,7 +877,8 @@ const DatabasesTab = forwardRef pending.tables[database.key] ?? database.enabled, }), [discardPending, flipTable, pending.tables, savePending]); - const runPopulationCount = useCallback(async () => { + const runPopulationCount = useCallback(async (exact = false) => { setCensusBusy(true); setCensusResult(""); + setExactCounts({}); setError(""); - const answer = await countSourceRows(); + const answer = await countSourceRows(exact); setCensusBusy(false); if (!answer.ok) { setError(answer.message); return; } - setCensusResult(populationResult( + setExactCounts(answer.exactCounts); + const pass = answer.mode === "bounded" ? "Counted up to 101 rows per source table. " + : answer.mode === "exact" ? "Counted exact totals. " + : answer.mode === "exact_fallback" + ? "Counted exact totals because this connection does not support bounded counts. " + : ""; + const exactHint = Object.keys(answer.exactCounts).length + ? " Exact totals are available in each Source rows tooltip." : ""; + setCensusResult(pass + populationResult( answer.modelsCounted, answer.modelsWithRows, answer.durationMs, answer.capturedAt, answer.complete - )); + ) + exactHint); onChanged(); }, [onChanged]); @@ -938,8 +995,9 @@ const DatabasesTab = forwardRef

- Counting every source table takes about a minute and a half on a workspace this - size. You can keep this tab open while it runs. + Count rows stops at 101 rows per source table and displays 100+ from 100 onward. + Each table still needs a request, so this can take about a minute or longer. + Counting exact totals takes considerably longer than the default pass.

{censusResult ? (

{censusResult}

@@ -964,6 +1022,15 @@ const DatabasesTab = forwardRef {censusBusy ? "Counting rows..." : "Count rows"} +
) : null} {error ? ( @@ -977,12 +1044,13 @@ const DatabasesTab = forwardRef {/* ⛔ THE IDENTITY BLOCK IS EXPLAINED, NOT INFERRED FROM A GREYED CONTROL. Which of the - three names is ours, which is theirs and which one a rename touches is the question this + two names is ours, which is theirs and which one the pen edits is the question this screen exists to answer, so it is answered in words above the table. */}

- Source name is what {row.label} calls the table, so it cannot be changed here. Default - name is ours. Your name replaces it everywhere in this workspace, including the - navigation rail, and clearing it brings the default back. + Source name belongs to {row.label} and cannot be changed here. Default name is ours + {hasRenamableDatabase + ? "; the pen changes the workspace name, including in the navigation rail." + : "."}

{showsPopulation ? (

@@ -995,7 +1063,6 @@ const DatabasesTab = forwardRef Source name Default name - Your name Fields {showsPopulation ? Source rows : null} Enabled @@ -1037,7 +1104,6 @@ const DatabasesTab = forwardRef {database.native || Not published} - {database.defaultLabel} {database.sourceRows === 0 ? ( Empty + ) : typeof database.sourceRows === "number" && database.sourceRows >= 100 ? ( + 100+ ) : typeof database.sourceRows === "number" ? ( - database.sourceRows.toLocaleString("en-US") + + {database.sourceRows.toLocaleString("en-US")} + ) : ( Not counted yet )} @@ -1119,9 +1189,9 @@ const DatabasesTab = forwardRef - {!effectiveEnabled && siblingLabels.length ? ( + {!effectiveEnabled && introspection.canToggleFields && siblingLabels.length ? ( - +

{renderedDatabase.label} shares its Odoo source table with {sentenceList(siblingLabels)}, so populated fields will be switched on there too. @@ -1131,7 +1201,7 @@ const DatabasesTab = forwardRef - +

{autoEnableStatus}

@@ -1143,7 +1213,7 @@ const DatabasesTab = forwardRef - +

{refusal}

@@ -1152,9 +1222,10 @@ const DatabasesTab = forwardRef - + 30 * DAY_MS) return null; + if (remaining <= 0) return { tone: "expired", message: "Client secret expired." }; + const days = Math.ceil(remaining / DAY_MS); + const time = days === 1 ? "within a day" : `in ${days} days`; + return remaining <= 7 * DAY_MS + ? { tone: "urgent", message: `Secret expires ${time}.` } + : { tone: "attention", message: `Client secret expires ${time}.` }; +} + +export function RotationNotice({ notice, onRotate, disabled = false }: { + notice: NonNullable>; + onRotate?: () => void; + disabled?: boolean; +}) { + return ( +
+ {notice.message} + {onRotate ? ( + + ) : null} +
+ ); +} + +/** Recalculate even when the tab stays open across a deadline or resumes from sleep. */ +export function useRotationClock() { + const [now, setNow] = useState(Date.now); + useEffect(() => { + const refresh = () => setNow(Date.now()); + const timer = window.setInterval(refresh, 60_000); + window.addEventListener("focus", refresh); + document.addEventListener("visibilitychange", refresh); + return () => { + window.clearInterval(timer); + window.removeEventListener("focus", refresh); + document.removeEventListener("visibilitychange", refresh); + }; + }, []); + return now; +} + +function RotationDeadlineField({ + id, value, onChange, disabled, replacement = false, +}: { + id: string; + value: string; + onChange: (value: string) => void; + disabled: boolean; + replacement?: boolean; +}) { + return ( +
+ + onChange(event.target.value)} + /> +

+ Amazon issues this with the client secret. Include its timezone; blank means no expiry warnings. +

+
+ ); +} + /** * ⭐ WAVE 32 · R4 / contract C1 — WHO A CONNECTION IS FOR, in the two words the server declares. * @@ -67,7 +156,7 @@ export function hasOwnDisconnect(row: ConnectorRow): boolean { return row.key === "odoo" || row.kind === "oauth"; } -async function post(path: string, body?: unknown): Promise { +async function post(path: string, body?: unknown): Promise<{ message: string; entryId: string }> { try { const res = await fetch(`${API_V1}${path}`, { method: "POST", @@ -75,14 +164,33 @@ async function post(path: string, body?: unknown): Promise { headers: { "Content-Type": "application/json" }, body: JSON.stringify(body ?? {}), }); - const payload = (await res.json().catch(() => null)) as { error?: { message?: string } } | null; - if (!res.ok) return payload?.error?.message || `The server answered ${res.status}.`; - return ""; + const payload = (await res.json().catch(() => null)) as + | { entry?: { id?: string }; error?: { message?: string } } + | null; + if (!res.ok) { + return { message: payload?.error?.message || `The server answered ${res.status}.`, entryId: "" }; + } + return { message: "", entryId: typeof payload?.entry?.id === "string" ? payload.entry.id : "" }; } catch { - return "Cannot reach the server."; + return { message: "Cannot reach the server.", entryId: "" }; } } +async function testAmazonEntry(entryId: string): Promise { + if (!entryId) return "The stored key could not be identified. Close and reload this connection."; + try { + const res = await fetch(`${API_V1}/admin/keychain/${encodeURIComponent(entryId)}/test`, { + method: "POST", + credentials: CREDENTIALS, + }); + const body = (await res.json().catch(() => null)) as { ok?: boolean } | null; + if (res.ok && body?.ok === true) return ""; + } catch { + // Storage already succeeded. Retrying must test the same entry, never create another one. + } + return "The key is stored, but Amazon access is unconfirmed. Test again or close and use Replace."; +} + async function del(path: string): Promise { try { const res = await fetch(`${API_V1}${path}`, { method: "DELETE", credentials: CREDENTIALS }); @@ -115,7 +223,11 @@ function ConnectForm({ onCancel: () => void; }) { const [entryLabel, setEntryLabel] = useState(label); - const [values, setValues] = useState>({}); + const [values, setValues] = useState>(() => + Object.fromEntries(spec.fields.map((field) => [field.key, field.default ?? ""])) + ); + const [storedId, setStoredId] = useState(null); + const [rotationDeadline, setRotationDeadline] = useState(""); const [scope, setScope] = useState(spec.scopes[0] || "business"); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); @@ -124,33 +236,50 @@ function ConnectForm({ if (busy) return; setBusy(true); setError(""); - const fields: Record = {}; - for (const field of spec.fields) { - const value = (values[field.key] || "").trim(); - if (value) fields[field.key] = value; + let entryId = storedId; + if (entryId === null) { + const fields: Record = {}; + for (const field of spec.fields) { + const value = (values[field.key] || "").trim(); + if (value) fields[field.key] = value; + } + if (spec.type === "amazon_sp") { + fields[AMAZON_ROTATION_DEADLINE_FIELD] = rotationDeadline.trim(); + } + const result = await post("/admin/keychain", { + label: entryLabel.trim(), + type: spec.type, + scope, + fields, + }); + if (result.message) { + setError(result.message); + setBusy(false); + return; + } + entryId = result.entryId; + setStoredId(entryId); + setValues({}); + setRotationDeadline(""); } - const message = await post("/admin/keychain", { - label: entryLabel.trim(), - type: spec.type, - scope, - fields, - }); + const message = spec.type === "amazon_sp" ? await testAmazonEntry(entryId) : ""; if (message) { setError(message); setBusy(false); return; } - // The form is dropped rather than cleared: a secret that has been stored must not sit in a - // React state object waiting for the next render. + // Values were cleared at storage; close only after the required connection test answers. onDone(); }; return (
+ {storedId === null ? <>
setEntryLabel(event.target.value)} @@ -161,6 +290,7 @@ function ConnectForm({ {spec.scopes.length > 1 ? (
))} + {spec.type === "amazon_sp" ? ( + + ) : null} + : null} {error ? (

{error}

) : null}
-
@@ -238,6 +378,8 @@ function ReplaceForm({ onCancel: () => void; }) { const [values, setValues] = useState>({}); + const [rotationDeadline, setRotationDeadline] = useState(""); + const [stored, setStored] = useState(false); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); const fields = spec.fields.filter((field) => field.secret); @@ -245,16 +387,30 @@ function ReplaceForm({ fields.length > 0 && fields.every((field) => Boolean((values[field.key] || "").trim())); const save = async () => { - if (busy || !complete) return; + if (busy || (!stored && !complete)) return; setBusy(true); setError(""); - const replacement = Object.fromEntries( - fields.map((field) => [field.key, (values[field.key] || "").trim()]) - ); - const message = await post( - `/connectors/${encodeURIComponent(connectorKey)}/keychain/${encodeURIComponent(entry.id)}/replace`, - { fields: replacement } - ); + if (!stored) { + const replacement = Object.fromEntries( + fields.map((field) => [field.key, (values[field.key] || "").trim()]) + ); + if (spec.type === "amazon_sp") { + replacement[AMAZON_ROTATION_DEADLINE_FIELD] = rotationDeadline.trim(); + } + const result = await post( + `/connectors/${encodeURIComponent(connectorKey)}/keychain/${encodeURIComponent(entry.id)}/replace`, + { fields: replacement } + ); + if (result.message) { + setError(result.message); + setBusy(false); + return; + } + setStored(true); + setValues({}); + setRotationDeadline(""); + } + const message = spec.type === "amazon_sp" ? await testAmazonEntry(entry.id) : ""; if (message) { setError(message); setBusy(false); @@ -267,10 +423,18 @@ function ReplaceForm({ return (
-

Replace key

+

{stored ? "Test connection" : spec.type === "amazon_sp" ? "Rotate secret" : "Replace key"}

+ {!stored ? <>

The current key stays unchanged until the complete replacement is stored.

+ {fields.length > 1 ? ( +

+ {spec.type === "amazon_sp" + ? "Re-enter both the client secret and refresh token together. The connection stays in place." + : "To rotate one secret, re-enter all secrets together."} +

+ ) : null} {fields.map((field) => (
))} + {spec.type === "amazon_sp" ? ( + + ) : null} + : null}
{error ? (

@@ -297,15 +471,15 @@ function ReplaceForm({

@@ -333,6 +507,7 @@ export default function KeychainTab({ const [error, setError] = useState(""); const [adding, setAdding] = useState(false); const [replacing, setReplacing] = useState(null); + const now = useRotationClock(); const runTest = useCallback(async (entry: ConnectorKeychainEntry) => { setTested((current) => ({ ...current, [entry.id]: "Testing" })); @@ -351,7 +526,8 @@ export default function KeychainTab({ } catch { setTested((current) => ({ ...current, [entry.id]: "Cannot reach the server." })); } - }, []); + if (row.key === "amazon_sp") onChanged(); + }, [onChanged, row.key]); const remove = useCallback( async (entry: ConnectorKeychainEntry) => { @@ -385,9 +561,9 @@ export default function KeychainTab({ setError(""); let message = ""; if (row.key === "odoo") { - message = await post("/admin/connectors/odoo/disconnect"); + message = (await post("/admin/connectors/odoo/disconnect")).message; } else if (row.kind === "oauth") { - message = await post(`/oauth/${encodeURIComponent(row.key)}/disconnect`); + message = (await post(`/oauth/${encodeURIComponent(row.key)}/disconnect`)).message; } else { for (const entry of keychain.entries) { message = await del(`/admin/keychain/${encodeURIComponent(entry.id)}`); @@ -406,7 +582,10 @@ export default function KeychainTab({ setAdding(false)} + onCancel={() => { + setAdding(false); + onChanged(); + }} onDone={() => { setAdding(false); onChanged(); @@ -421,7 +600,10 @@ export default function KeychainTab({ connectorKey={row.key} spec={keychain.connect} entry={replacing} - onCancel={() => setReplacing(null)} + onCancel={() => { + setReplacing(null); + onChanged(); + }} onDone={() => { setReplacing(null); onChanged(); @@ -439,7 +621,10 @@ export default function KeychainTab({ ) : null} {keychain.entries.length ? (
    - {keychain.entries.map((entry) => ( + {keychain.entries.map((entry) => { + const notice = row.key === "amazon_sp" + ? amazonRotationNotice(entry.client_secret_rotation_deadline, now) : null; + return (
  • {entry.label} @@ -451,21 +636,29 @@ export default function KeychainTab({ {" · "} {scopeLabel(entry.scope)} - {tested[entry.id] ? ( + {notice ? ( + setReplacing(entry) : undefined} + /> + ) : null} + {tested[entry.id] && notice?.tone !== "expired" ? ( {tested[entry.id]} ) : null}
    - {entry.replaceable && keychain.connect ? ( + {entry.replaceable && keychain.connect && !notice ? ( ) : null} {entry.testable ? ( @@ -491,7 +684,8 @@ export default function KeychainTab({ )}
  • - ))} + ); + })}
) : ( // One line, plus the action that fills it. Never a paragraph (DESIGN.md). @@ -506,7 +700,11 @@ export default function KeychainTab({ diff --git a/web/src/settings/CompliancePane.tsx b/web/src/settings/CompliancePane.tsx new file mode 100644 index 0000000000000000000000000000000000000000..a0453179c46e10d77ce2a76d55e4d2ad0dcac399 --- /dev/null +++ b/web/src/settings/CompliancePane.tsx @@ -0,0 +1,151 @@ +import { useEffect, useState } from "react"; +import { API_V1, CREDENTIALS, checkTenant, UNAUTHORIZED_EVENT } from "../apiContract"; +import "../styles/wave59/security.css"; + +type ClaimState = "met" | "not_met" | "unproven"; +type Claim = { + id: string; + group: string; + claim: string; + state: ClaimState; + evidence: string; + source: string | null; +}; +type Compliance = { claims: Claim[]; generated_at: number }; + +const STATE_WORDS: Record = { + met: "Met", not_met: "Not met", unproven: "Unproven", +}; +const UNAVAILABLE = "Amazon compliance could not be loaded. Try reopening Settings."; + +function requireResponse(response: Response): void { + if (checkTenant(response)) throw new Error(UNAVAILABLE); + if (response.status === 401) { + window.dispatchEvent(new CustomEvent(UNAUTHORIZED_EVENT)); + throw new Error("Sign in again to view Amazon compliance."); + } + if (response.status === 403) throw new Error("Amazon compliance is available to administrators only."); + if (!response.ok) throw new Error(UNAVAILABLE); +} + +function isClaim(value: unknown): value is Claim { + if (!value || typeof value !== "object") return false; + const row = value as Record; + return ["id", "group", "claim", "evidence"].every((key) => + typeof row[key] === "string" && row[key].trim().length > 0) + && (row.state === "met" || row.state === "not_met" || row.state === "unproven") + && (row.source === null || typeof row.source === "string"); +} + +function readCompliance(value: unknown): Compliance { + if (!value || typeof value !== "object") throw new Error(UNAVAILABLE); + const payload = value as Record; + if (!Array.isArray(payload.claims) || !payload.claims.length || !payload.claims.every(isClaim) + || new Set(payload.claims.map((row) => row.id)).size !== payload.claims.length + || typeof payload.generated_at !== "number" || !Number.isFinite(payload.generated_at)) { + throw new Error(UNAVAILABLE); + } + return { claims: payload.claims, generated_at: payload.generated_at }; +} + +export default function CompliancePane() { + const [payload, setPayload] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(""); + const [downloading, setDownloading] = useState(false); + const [notice, setNotice] = useState(""); + + useEffect(() => { + const controller = new AbortController(); + async function load() { + try { + const response = await fetch(`${API_V1}/security/compliance`, { + credentials: CREDENTIALS, cache: "no-store", signal: controller.signal, + }); + requireResponse(response); + const result = readCompliance(await response.json()); + if (!controller.signal.aborted) setPayload(result); + } catch (failure) { + if (!controller.signal.aborted) { + setError(failure instanceof Error && [UNAVAILABLE, + "Sign in again to view Amazon compliance.", + "Amazon compliance is available to administrators only."].includes(failure.message) + ? failure.message : UNAVAILABLE); + } + } finally { + if (!controller.signal.aborted) setLoading(false); + } + } + void load(); + return () => controller.abort(); + }, []); + + async function downloadEvidence() { + if (downloading) return; + setDownloading(true); + setError(""); + setNotice(""); + try { + const response = await fetch(`${API_V1}/security/compliance/export`, { + credentials: CREDENTIALS, cache: "no-store", + }); + requireResponse(response); + if (!response.headers.get("Content-Type")?.startsWith("text/plain")) throw new Error(UNAVAILABLE); + const body = await response.text(); + const url = URL.createObjectURL(new Blob([body], { type: "text/plain;charset=utf-8" })); + const anchor = window.document.createElement("a"); + anchor.href = url; + anchor.download = "amazon-compliance.txt"; + window.document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + window.setTimeout(() => URL.revokeObjectURL(url), 0); + setNotice("amazon-compliance.txt downloaded. It contains no credential or other secret."); + } catch { + setError("The evidence file could not be downloaded. Try reopening Settings."); + } finally { + setDownloading(false); + } + } + + const groups = new Map(); + for (const claim of payload?.claims ?? []) { + const rows = groups.get(claim.group) ?? []; + rows.push(claim); + groups.set(claim.group, rows); + } + + return ( +
+

Amazon compliance

+ {loading ?

Reading the current controls…

: null} + {error ?

{error}

: null} + {payload ? <> +

Observed {new Date(payload.generated_at * 1000).toLocaleString()}.

+ + {notice ?

{notice}

: null} + {Array.from(groups, ([group, claims]) => ( +
+

{group}

+
    + {claims.map((claim) => ( +
  • +
    + {claim.claim} + + {STATE_WORDS[claim.state]} + +
    +

    {claim.evidence}

    +
  • + ))} +
+
+ ))} + : null} +
+ ); +} diff --git a/web/src/settings/SecurityPane.tsx b/web/src/settings/SecurityPane.tsx index 74cb94894443bcd3cc7aee412954cd9df385ed19..4b34b25010e22a5a548b412df0cc90813ae3cf0b 100644 --- a/web/src/settings/SecurityPane.tsx +++ b/web/src/settings/SecurityPane.tsx @@ -1,20 +1,164 @@ // --------------------------------------------------------------------------- // settings / SecurityPane.tsx - W57-T36, R15 / C12. // -// Every stored account owns its second factor, so this is a session-scoped -// room for every user. It reads status, never the installed secret. A new phone -// goes through the admin reset and the login enrolment door, not a second editor. +// Every stored account owns its second factor. W59-T31 also renders this pane +// before a session exists: R1 moves the enrollment screen, never the MFA wall. +// Settings reads status, never the installed secret, and can replace a phone +// only after password re-authentication and proof of the candidate factor. // --------------------------------------------------------------------------- -import { useEffect, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import type { FormEvent } from "react"; -import { mfaStatus, regenerateRecoveryCodes } from "../shell/session"; -import type { MFAStatus } from "../shell/session"; -import { RecoveryCodes } from "../shell/LoginPage"; +import { errorMessage, mfaEnrollConfirm, mfaEnrollStart, mfaStatus, regenerateRecoveryCodes, SERVER_UNAVAILABLE } from "../shell/session"; +import type { MFASetup, MFAStatus, SessionUser } from "../shell/session"; +import { API_V1, CREDENTIALS, checkTenant } from "../apiContract"; import { useEscapeClose } from "../ui/useEscapeClose"; +import { encodeQr } from "../shell/qrcode"; import "../styles/wave57/login-mfa.css"; +import "../styles/wave59/security.css"; -export default function SecurityPane() { +function SetupQr({ uri }: { uri: string }) { + const drawing = useMemo(() => { + try { + const qr = encodeQr(uri); + const quiet = 4; + // One path keeps the modal's DOM small; the four-module light border + // belongs in the viewBox so responsive sizing cannot crop it away. + const path = qr.modules.flatMap((row, y) => row.flatMap((dark, x) => + dark ? [`M${x + quiet} ${y + quiet}h1v1h-1z`] : [])).join(""); + return { extent: qr.size + 2 * quiet, path }; + } catch { + // Keep enrollment usable, but never draw a plausible square on failure. + return null; + } + }, [uri]); + if (!drawing) return

The QR code could not be drawn.

; + return ( + <> +

Or scan this code with your authenticator app.

+ + + + + + ); +} + +/** W57-T36 / C12: one show-once treatment in both doors. Copy is a quiet, + * explicit export to the person's clipboard, never persistence on their behalf. + * Only the acknowledgement can dismiss this surface; Escape names what remains. + * Browser reload/close can only be warned about, never made impossible by a page. */ +export function RecoveryCodes({ codes, onSaved, onError }: { + codes: string[]; + onSaved: () => void; + onError: (message: string) => void; +}) { + const [copied, setCopied] = useState(false); + const [copying, setCopying] = useState(false); + useEscapeClose(true, () => onError("Save your recovery codes before continuing.")); + useEffect(() => { + const warn = (event: BeforeUnloadEvent) => { + event.preventDefault(); + event.returnValue = ""; + }; + window.addEventListener("beforeunload", warn); + return () => window.removeEventListener("beforeunload", warn); + }, []); + + async function copy() { + if (copying) return; + setCopying(true); + try { + await navigator.clipboard.writeText(codes.join("\n")); + setCopied(true); + onError(""); + } catch { + setCopied(false); + onError("Copy did not work. Select the codes and copy them manually."); + } finally { + setCopying(false); + } + } + + return ( + <> +

This screen is the only place these recovery codes will ever be shown.

+

Save them somewhere safe; each code works once if you lose your authenticator.

+
    + {codes.map((code) =>
  • {code}
  • )} +
+
+ + {copied ? "Copied" : ""} +
+ {/* A type=button acknowledgement cannot be submitted by Enter left over + from the previous code field. There is no Back or secondary exit. */} + + + ); +} + +type AuthFailure = { ok: false; message: string }; + +// Keep the literal fetch doors visible to D-48. These session-scoped helpers +// mirror authPost's tenant guard and error policy without creating a second session module. +async function startDevice(password: string): Promise<({ ok: true } & MFASetup) | AuthFailure> { + let res: Response; + try { + res = await fetch(`${API_V1}/auth/mfa/device`, { + method: "POST", credentials: CREDENTIALS, cache: "no-store", + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ password }), + }); + } catch { + return { ok: false, message: errorMessage(0, null) }; + } + if (checkTenant(res)) return { ok: false, message: SERVER_UNAVAILABLE }; + const body = await res.json().catch(() => null); + if (!res.ok) return { ok: false, message: errorMessage(res.status, body) }; + if (!body || typeof body.secret !== "string" || !body.secret.trim() + || typeof body.otpauthUri !== "string" || !body.otpauthUri.trim() + || typeof body.issuer !== "string" || !body.issuer.trim()) + return { ok: false, message: SERVER_UNAVAILABLE }; + return { ok: true, secret: body.secret, otpauthUri: body.otpauthUri, issuer: body.issuer }; +} + +async function confirmDevice(password: string, secret: string, code: string): Promise< + { ok: true; recoveryCodes: string[] } | AuthFailure +> { + let res: Response; + try { + res = await fetch(`${API_V1}/auth/mfa/device/confirm`, { + method: "POST", credentials: CREDENTIALS, cache: "no-store", + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ password, secret, code }), + }); + } catch { + return { ok: false, message: errorMessage(0, null) }; + } + if (checkTenant(res)) return { ok: false, message: SERVER_UNAVAILABLE }; + const body = await res.json().catch(() => null); + if (!res.ok) return { ok: false, message: errorMessage(res.status, body) }; + const codes: unknown = body?.recoveryCodes; + // Accept the whole fresh set or refuse success, just like the existing issuers. + if (!Array.isArray(codes) || codes.length === 0 + || !codes.every((value): value is string => typeof value === "string" && value.trim().length > 0) + || new Set(codes).size !== codes.length) + return { ok: false, message: SERVER_UNAVAILABLE }; + return { ok: true, recoveryCodes: codes }; +} + +export type EnrolmentGate = { + username: string; + password: string; + onEnrolled: (user: SessionUser, recoveryCodes: string[]) => void; + onBack: () => void; +}; + +export default function SecurityPane({ gate }: { gate?: EnrolmentGate } = {}) { const [status, setStatus] = useState(null); const [loading, setLoading] = useState(true); const [reload, setReload] = useState(0); @@ -22,12 +166,22 @@ export default function SecurityPane() { const [busy, setBusy] = useState(false); const [codes, setCodes] = useState(null); const [error, setError] = useState(""); + const [device, setDevice] = useState(false); + const [setup, setSetup] = useState(null); + const [code, setCode] = useState(""); + const [preparing, setPreparing] = useState(false); + const [setupRetry, setSetupRetry] = useState(0); + const [enrolledUser, setEnrolledUser] = useState(null); const flight = useRef(false); const dialog = useRef(null); const passwordField = useRef(null); + const heading = useRef(null); const locked = busy || codes !== null; + const gateUsername = gate?.username; + const gatePassword = gate?.password; useEffect(() => { + if (gate) return; let current = true; setLoading(true); setError(""); @@ -38,7 +192,26 @@ export default function SecurityPane() { if (!next) setError("Security status could not be loaded. Try again."); }); return () => { current = false; }; - }, [reload]); + }, [gate, reload]); + + // Scalar dependencies keep a parent re-render from replacing the candidate. + // Starting setup is read-only; ignore a late response after Back or a retry. + useEffect(() => { + if (gateUsername === undefined || gatePassword === undefined) return; + let current = true; + heading.current?.focus(); + setPreparing(true); + setSetup(null); + setCode(""); + setError(""); + void mfaEnrollStart(gateUsername, gatePassword).then((result) => { + if (!current) return; + setPreparing(false); + if (result.ok === false) setError(result.message); + else setSetup(result); + }); + return () => { current = false; }; + }, [gateUsername, gatePassword, setupRetry]); // Lock BEFORE issuing codes, not only after receiving them: closing Settings // during the POST would throw away its only clear response. A native modal @@ -77,10 +250,60 @@ export default function SecurityPane() { setBusy(false); } + async function enrol(event: FormEvent) { + event.preventDefault(); + if (flight.current || locked || preparing) return; + if (!setup) { + if (gate) { + setSetupRetry((value) => value + 1); + return; + } + if (!status?.enrolled || !password) return; + flight.current = true; + setPreparing(true); + setError(""); + try { + const result = await startDevice(password); + if (result.ok === false) { + setError(result.message); + setPassword(""); + } else setSetup(result); + } finally { + flight.current = false; + setPreparing(false); + } + return; + } + if (!code || (!gate && (!status?.enrolled || !password))) return; + flight.current = true; + setBusy(true); + setError(""); + try { + const result: { ok: true; user?: SessionUser; recoveryCodes: string[] } | AuthFailure = gate + ? await mfaEnrollConfirm(gate.username, gate.password, setup.secret, code) + : await confirmDevice(password, setup.secret, code); + if (result.ok === false) setError(result.message); + else { + // A cookie exists only now. The frame still waits for the show-once acknowledgement. + if (result.user) setEnrolledUser(result.user); + setCodes(result.recoveryCodes); + setSetup(null); + setDevice(false); + } + } finally { + setPassword(""); + setCode(""); + flight.current = false; + setBusy(false); + } + } + return (
-

Security

- {loading ?

Loading security status…

: status ? ( +

Security

+ {gate ? ( +

Two step sign in is required before the rest of the app opens.

+ ) : loading ?

Loading security status…

: status ? ( <>
Two step sign in
{status.enrolled ? "On" : "Off"}
@@ -91,21 +314,29 @@ export default function SecurityPane() { ) : null}
Recovery codes left
{status.recoveryCodesRemaining}
-

To enrol a new phone, ask an administrator to reset two step sign in, then sign in again.

{status.enrolled ? ( <> -

Generating new recovery codes replaces the old codes and signs this account's other sessions out.

- {!locked ? ( -
- - { setPassword(event.target.value); setError(""); }} /> - -
+ {!device && !locked ? ( + + ) : null} + {!device ? ( + <> +

Generating new recovery codes replaces the old codes and signs this account's other sessions out.

+ {!locked ? ( +
+ + { setPassword(event.target.value); setError(""); }} /> + +
+ ) : null} + ) : null} ) :

Sign in again to set up your authenticator app.

} @@ -115,6 +346,47 @@ export default function SecurityPane() { Try again )} + {(gate || device) && !locked ? ( +
+ {!gate ? ( + <> +

Confirming a new device replaces your old recovery codes and signs this account's other sessions out.

+ + { setPassword(event.target.value); setError(""); }} /> + + ) : null} + {setup ? ( + <> + {/* The in-app encoder adds no dependency or external QR service. + Keep manual setup available even when the QR cannot be drawn. */} +

Type this setup code into any authenticator app.

+ {setup.issuer} setup code + {setup.secret.match(/.{1,4}/g)?.join(" ")} +
+ + Authenticator setup link + {setup.otpauthUri} +
+ + { setCode(event.target.value); setError(""); }} + aria-invalid={error ? true : undefined} aria-describedby="security-message" /> + + ) : preparing ?

Preparing your setup code…

: null} + + +
+ ) : null}

{locked ? "" : error}

{/* R15/C12's explicit acknowledgement is the exception to ordinary @@ -127,6 +399,13 @@ export default function SecurityPane() {
{codes ? ( { + if (gate) { + if (!enrolledUser) return; + gate.onEnrolled(enrolledUser, codes); + setEnrolledUser(null); + setCodes(null); + return; + } setCodes(null); setError(""); // Count only, independently re-read after acknowledgement. diff --git a/web/src/sheet/BindingEditor.tsx b/web/src/sheet/BindingEditor.tsx index 553db297b4976267ae966d28504a64256d429c98..bd5cd7b5daeb9b71ddfd999da42420576024f456 100644 --- a/web/src/sheet/BindingEditor.tsx +++ b/web/src/sheet/BindingEditor.tsx @@ -39,6 +39,7 @@ import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; import { fetchTopicRows, fetchWorkspace } from "../customer-grid/apiBridge"; import type { SurfaceScope } from "../customer-grid/apiBridge"; +import { dayText } from "../customer-grid/display"; import { FieldSelectButton } from "../customer-grid/FieldSelect"; import { FieldTypeIcon } from "../customer-grid/icons"; import { BodyPortal, OverlayProvider, useOverlayLayer } from "../customer-grid/OverlaySurface"; @@ -60,10 +61,11 @@ import type { BindingAggregate } from "./types"; import { bindingStepMode, bindingStepValues, seedBindingStep } from "./bindingRefs"; import "../styles/wave57/binding-actions.css"; import "../styles/wave58/sheet.css"; +import "../styles/wave59/binding.css"; const BINDING_SECTIONS = [ { key: "data", label: "Data" }, - { key: "rows", label: "Rows" }, + { key: "rows", label: "Filter" }, { key: "period", label: "Period" }, { key: "copied", label: "When copied" }, ] as const; @@ -134,6 +136,8 @@ const scopeOf = (database: string): SurfaceScope | null => { // `period: {unit: "month"|"quarter"|"year", start: "YYYY-MM", locked: boolean, ref?: A1}`. The // unit values are LOWERCASE on the wire even though the picker reads Month / Quarter / Year, and // `start` is a MONTH string for every unit. +// A frozen period keeps that pair for returning to the calendar, and adds only +// `frozen: true, from: "YYYY-MM-DD", to: "YYYY-MM-DD"` in place of a cell reference. // // ⭐ `start` IS THE PERIOD'S FIRST MONTH, and that is the decision the ticket leaves implicit. // `unit` and `start` have to jointly name exactly one period, so Q3 2026 is `2026-07` and the year @@ -148,6 +152,9 @@ export interface PeriodBinding { start: string; locked: boolean; ref?: string; + frozen?: true; + from?: string; + to?: string; window?: WindowSpec; } @@ -200,6 +207,15 @@ export function shiftPeriodStart(unit: PeriodUnit, ym: string, steps: number): s return joinYm(Math.floor(total / 12), (total % 12) + 1); } +/** The inclusive day range a calendar period covers. Mirrors + * `platform/modules/sheet.py::period_window`, whose `to` is the LAST DAY of the period. */ +export function periodRange(unit: PeriodUnit, ym: string): { from: string; to: string } { + const start = normalizePeriodStart(unit, ym); + const last = new Date(`${shiftPeriodStart(unit, start, 1)}-01T00:00:00Z`); + last.setUTCDate(last.getUTCDate() - 1); + return { from: `${start}-01`, to: last.toISOString().slice(0, 10) }; +} + // ⚠ W53-T16 (instruction 13) - `periodLabel` LIVED HERE and is gone with the summary block that // was its only caller. The period a person reads is now the Year / Month / Quarter pickers // themselves, which is where they set it, so a second rendering of the same value had nothing left @@ -257,6 +273,13 @@ export function migratePeriod(raw: unknown): PeriodBinding | null { if (!/^\d{4}-\d{2}$/.test(stored) && !ref) return null; const start = normalizePeriodStart(unit, stored || todayYm()); const locked = p.locked === true; + if (p.frozen === true) { + return { + unit, start, locked, frozen: true, + from: typeof p.from === "string" ? p.from : "", + to: typeof p.to === "string" ? p.to : "", + }; + } return ref ? { unit, start, locked, ref } : { unit, start, locked }; } @@ -272,9 +295,11 @@ export interface OnDragBinding { down: { field: string } | "none"; } -/** Every permission-visible tenant field participates in the type-aware Step catalogue. */ +/** A copy step walks a column's stored values. Measure-backed and Metric columns are computed + * numbers with no stored values to walk, so offering one promises a step `dragBoundCell` + * cannot perform. */ export function isDragDownField(f: Field): boolean { - return Boolean(f.key) && !f.key.startsWith("__"); + return Boolean(f.key) && !f.key.startsWith("__") && !f.measure && f.type !== "metric"; } /** The saved `onDrag`, defaulted. `down` is carried as a field key, empty meaning nothing. */ @@ -720,6 +745,26 @@ export default function BindingEditor({ migrated?.start ?? normalizePeriodStart("month", todayYm()), ); const [locked, setLocked] = useState(migrated?.locked ?? false); + // An absent flag is rolling, including every binding saved before frozen dates existed. + const [frozen, setFrozen] = useState(Boolean(migrated?.frozen)); + const [frozenFrom, setFrozenFrom] = useState( + () => migrated?.from ?? periodRange(unit, start).from, + ); + const [frozenTo, setFrozenTo] = useState( + () => migrated?.to ?? periodRange(unit, start).to, + ); + const range = frozen ? { from: frozenFrom, to: frozenTo } : periodRange(unit, start); + const resumeRollingPeriod = () => { + setFrozen(false); + setFrozenFrom(""); + setFrozenTo(""); + }; + const editPeriodDate = (key: "from" | "to", value: string) => { + // Capture the other displayed endpoint too: the dropdowns may have moved since mount. + setFrozenFrom(key === "from" ? value : range.from); + setFrozenTo(key === "to" ? value : range.to); + setFrozen(true); + }; // ⛔⛔ W57-T17 - THE DATABASE IS SEEDED AT MOUNT OUT OF A LIST THAT ARRIVES LATER, AND AN EMPTY // SEED NEVER HEALED. `databases` is fetched by `SheetPage` from `/api/v1/nav`; a panel opened // before that answer lands seeds `database` to `""`, and nothing re-seeds it. A ` changeSource("number")} /> - Number + Metric
@@ -1218,8 +1308,11 @@ export default function BindingEditor({ key: m.key, label: m.label, type: "measure" as const, }))} /> - {measures.length === 0 &&

No Metrics are available on this database.

} -
@@ -1445,6 +1538,7 @@ export default function BindingEditor({ aria-label="Period unit" onChange={(e) => { const next = e.target.value as PeriodUnit; + resumeRollingPeriod(); setUnit(next); // Re-seat the start on the new unit's first month, or the pair stops naming one // period: `2026-08` under `quarter` is not a quarter anybody can compile. @@ -1494,6 +1588,44 @@ export default function BindingEditor({ )}
+
+ + +
+

+ {dayText(range.from)} to {dayText(range.to)} +

+ {frozen && ( + <> +

Frozen to these dates until you choose a period.

+ + + )} + {/* W56-T21 (owner instruction 13) - THE QUICK PICKS ARE GONE. Unit, year and month-or-quarter already reach every period the This and Last buttons could, so a second control for the same value was a second way to say one @@ -1539,7 +1671,14 @@ export default function BindingEditor({ @@ -1604,31 +1743,43 @@ export default function BindingEditor({ absent control looks like a loading or configuration failure. */} {chosenIsMetric && !periodOn && (

- Choose a Period or Lookback before saving this Number. + Choose a Period or Lookback before saving this Metric.

)} -
diff --git a/web/src/sheet/Ribbon.tsx b/web/src/sheet/Ribbon.tsx index 54ecafa1af461723f7757eaba68569294b61ad47..795a445811b7db0d123e7f73fd6278e0d80f2394 100644 --- a/web/src/sheet/Ribbon.tsx +++ b/web/src/sheet/Ribbon.tsx @@ -46,7 +46,7 @@ // labels with no ALL CAPS chrome; motion is 150 ms ease-out on opacity and transform only; z-order // rides a `--cg-z-*` token. No em dash or en dash in any string that reaches a screen. -import { createContext, useContext, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { Children, cloneElement, createContext, isValidElement, useContext, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import type { KeyboardEvent, MouseEvent as ReactMouseEvent, ReactNode } from "react"; import type { Cell, CellStyle, CellStylePatch, SheetChartSpec, Workbook, SheetPrintSettings } from "./types"; import { AnchoredOverlay } from "../customer-grid/OverlaySurface"; @@ -246,6 +246,7 @@ const ACTIONS = [ type TabId = (typeof TABS)[number]["id"]; type ControlKey = + | "overflow" | (typeof TABS)[number]["key"] | (typeof ACTIONS)[number]["key"] | (typeof TABS)[number]["keys"][number]; @@ -302,6 +303,7 @@ export interface RibbonTabContext { kt?: string, ktOpens?: "gallery", controlDisabled?: boolean, + popupRole?: "menu" | "dialog", ) => ReactNode; group: (name: string, body: ReactNode, extra?: string) => ReactNode; word: (text: string) => ReactNode; @@ -401,6 +403,18 @@ function IconChevron({ direction }: { direction: "up" | "down" | "left" | "right ); } +/** Keep each tab's wrappers, including Insert's native disabled fieldset, on both sides. */ +function partitionRibbonGroups(node: ReactNode, visibleCount: number, folded: boolean): ReactNode { + return Children.map(node, (child) => { + if (!isValidElement<{ children?: ReactNode; "data-ribbon-group"?: number }>(child)) return child; + const index = child.props["data-ribbon-group"]; + if (index !== undefined) return (index >= visibleCount) === folded ? child : null; + if (!child.props.children) return child; + const children = partitionRibbonGroups(child.props.children, visibleCount, folded); + return Children.count(children) ? cloneElement(child, undefined, children) : null; + }); +} + export default function Ribbon({ style, disabled = false, frozen = false, canImport = false, onApply, onAction, insert, }: RibbonProps) { @@ -429,8 +443,10 @@ export default function Ribbon({ const [formulaBarVisible, setFormulaBarVisible] = useState(readFormulaBar); const rootRef = useRef(null); const scrollRef = useRef(null); - const [canScrollLeft, setCanScrollLeft] = useState(false); - const [canScrollRight, setCanScrollRight] = useState(false); + const overflowRef = useRef(null); + const overflowTriggerRef = useRef(null); + const [overflowOpen, setOverflowOpen] = useState(false); + const [groupFit, setGroupFit] = useState<{ tab: TabId; count: number } | null>(null); useEffect(() => { const wrap = rootRef.current?.closest(".sheet-grid-wrap"); @@ -449,48 +465,8 @@ export default function Ribbon({ /** Which registered tab is showing. */ const [active, setActive] = useState(TABS[0].id); const activeTab = TABS.find((tab) => tab.id === active) ?? TABS[0]; - - useEffect(() => { - const scroller = scrollRef.current; - if (!scroller) { - setCanScrollLeft(false); - setCanScrollRight(false); - return; - } - const updateScrollAvailability = () => { - const toolbar = scroller.parentElement; - const toolbarStyle = toolbar ? window.getComputedStyle(toolbar) : null; - // Measure the full row so the chevrons cannot keep a fitting tab overflowing by themselves. - const fullWidth = toolbar && toolbarStyle - ? toolbar.clientWidth - parseFloat(toolbarStyle.paddingLeft) - - parseFloat(toolbarStyle.paddingRight) - : scroller.clientWidth; - const overflows = scroller.scrollWidth > fullWidth + 1; - setCanScrollLeft(overflows && scroller.scrollLeft > 1); - setCanScrollRight(overflows - && scroller.scrollLeft + scroller.clientWidth < scroller.scrollWidth - 1); - }; - updateScrollAvailability(); - scroller.addEventListener("scroll", updateScrollAvailability, { passive: true }); - window.addEventListener("resize", updateScrollAvailability); - const observer = typeof ResizeObserver === "undefined" - ? null : new ResizeObserver(updateScrollAvailability); - observer?.observe(scroller); - return () => { - scroller.removeEventListener("scroll", updateScrollAvailability); - window.removeEventListener("resize", updateScrollAvailability); - observer?.disconnect(); - }; - }, [active, collapsed]); - - const scrollGroups = (direction: -1 | 1) => { - const scroller = scrollRef.current; - if (!scroller) return; - scroller.scrollBy({ - left: direction * scroller.clientWidth * 0.8, - behavior: window.matchMedia("(prefers-reduced-motion: reduce)").matches ? "auto" : "smooth", - }); - }; + const visibleGroupCount = groupFit?.tab === active ? groupFit.count : Infinity; + const hasOverflow = Number.isFinite(visibleGroupCount); /** * The keys the arrows may reach right now. Collapsed, that is the strip and nothing else — and @@ -500,10 +476,10 @@ export default function Ribbon({ * actually has an import door, in both branches. */ const reachable = useMemo( - () => (collapsed ? STRIP : [...STRIP, ...activeTab.keys]) + () => (collapsed ? STRIP : [...STRIP, ...activeTab.keys, ...(hasOverflow ? ["overflow" as const] : [])]) .filter((key) => (key !== "import" || canImport) && (key !== "export" || !exportUnavailable)), - [activeTab, canImport, collapsed, exportUnavailable], + [activeTab, canImport, collapsed, exportUnavailable, hasOverflow], ); // ⛔ FOCUS MUST LAND ON SOMETHING THAT EXISTS. Collapsing while the focus sits on a control that @@ -514,11 +490,29 @@ export default function Ribbon({ if (!reachable.includes(focus)) setFocus(reachable[0]); }, [focus, reachable]); + useEffect(() => { + if (hasOverflow && !overflowOpen && overflowRef.current?.querySelector(`[data-ribbon="${focus}"]`)) { + setFocus("overflow"); + } + }, [focus, hasOverflow, overflowOpen, visibleGroupCount]); + const focusControl = useCallback((key: ControlKey) => { setFocus(key); - rootRef.current - ?.querySelector(`[data-ribbon="${key}"]`) - ?.focus(); + const selector = `[data-ribbon="${key}"]`; + const foldedControl = overflowRef.current?.querySelector(selector); + if (foldedControl) { + // Closed folded groups remain mounted for measurement; reveal them before focusing. + setOverflowOpen(true); + requestAnimationFrame(() => overflowRef.current?.querySelector(selector)?.focus()); + } else { + rootRef.current?.querySelector(selector)?.focus(); + } + }, []); + + const closeOverflow = useCallback(() => { + setOverflowOpen(false); + setFocus("overflow"); + overflowTriggerRef.current?.querySelector("button")?.focus(); }, []); const toggleCollapsed = useCallback(() => { @@ -528,6 +522,7 @@ export default function Ribbon({ return next; }); setOpen(""); + setOverflowOpen(false); }, []); /** @@ -602,31 +597,41 @@ export default function Ribbon({ // would be a second claimant on the same key. // The toolbar's arrows belong to the toolbar only while nothing is open above it. if (open) return; - const at = Math.max(0, reachable.indexOf(focus)); + if (event.target instanceof HTMLElement && event.target.matches("input, textarea, select, [contenteditable=true]")) return; + const enabledKeys = reachable.filter(key => { + const selector = `[data-ribbon="${key}"]`; + const button = rootRef.current?.querySelector(selector) + ?? overflowRef.current?.querySelector(selector); + return button && !button.matches(":disabled"); + }); + if (!enabledKeys.length) return; + const at = Math.max(0, enabledKeys.indexOf(focus)); if (event.key === "ArrowRight") { event.preventDefault(); - focusControl(reachable[(at + 1) % reachable.length]); + focusControl(enabledKeys[(at + 1) % enabledKeys.length]); } else if (event.key === "ArrowLeft") { event.preventDefault(); - focusControl(reachable[(at - 1 + reachable.length) % reachable.length]); + focusControl(enabledKeys[(at - 1 + enabledKeys.length) % enabledKeys.length]); } else if (event.key === "Home") { event.preventDefault(); - focusControl(reachable[0]); + focusControl(enabledKeys[0]); } else if (event.key === "End") { event.preventDefault(); - focusControl(reachable[reachable.length - 1]); + focusControl(enabledKeys[enabledKeys.length - 1]); } }, [focus, focusControl, open, reachable]); const apply = useCallback((patch: CellStylePatch) => { setOpen(""); + if (overflowOpen) closeOverflow(); onApply(patch); - }, [onApply]); + }, [closeOverflow, onApply, overflowOpen]); const act = useCallback((kind: RibbonActionKind, arg?: string | number) => { setOpen(""); + if (overflowOpen) closeOverflow(); onAction({ kind, arg }); - }, [onAction]); + }, [closeOverflow, onAction, overflowOpen]); /** * ⭐⭐ W53-T07 — CLOSING A MENU RETURNS FOCUS TO THE CONTROL THAT OWNS IT, and that is the @@ -693,6 +698,7 @@ export default function Ribbon({ kt?: string, ktOpens?: "gallery", controlDisabled?: boolean, + popupRole: "menu" | "dialog" = "menu", ) => ( @@ -904,44 +985,38 @@ export default function Ribbon({ role="toolbar" aria-label={activeTab.toolbarLabel} > - {/* Scroll buttons stay outside the roving order because keyboard focus already scrolls controls into view. */} - {canScrollLeft && ( - - )} - {/* ⛔ `min-width: 0` ON THE SCROLLER IS LOAD BEARING. A flex child will not scroll - without it. Import, Refresh, Export and Full screen do not live in this row - (W52-T15 moved them to the tab strip above), so the only thing that can walk off - the right edge at a narrow window now is a group itself, which scrolling permits. */}
- {/* ⭐⭐ THE ACTIVE TAB'S OWN GROUPS. `ribbonHome.tsx` holds every one of them, in R8's - order, and this is the whole of the shell's knowledge of what they are. */} - {activeTab.groups(tabContext)} -
- {canScrollRight && ( - - )} + {control("overflow", "More groups", <>{word("More groups")}, + toggleOverflow, false, overflowOpen, hasOverflow ? "Y" : undefined, "gallery", undefined, "dialog")} + +
)} + {!collapsed && (overflowOpen && hasOverflow ? ( + + {overflowContent} + + ) : ( + + ))} {/* ⭐⭐ W52-T17 (instruction 20, ruling R7) — mounted ONCE, absolutely positioned via its own `position: fixed` overlay (styles/wave52/ribbon.css), so it adds no layout of its own (the design contract's "appear without layout shift"). It reads this DOM tree through diff --git a/web/src/sheet/SheetChart.tsx b/web/src/sheet/SheetChart.tsx index e093851d221093c63c2828e764372f8ebbaa3bdc..5fe02634e2493912487abe5990c8d7ee268c9051 100644 --- a/web/src/sheet/SheetChart.tsx +++ b/web/src/sheet/SheetChart.tsx @@ -129,10 +129,22 @@ export function chartDataForRange( if (!matrix.length || !matrix[0].length) return empty(); // A lone row is a numeric run, retaining the legacy one-row selection behavior. if (matrix.length === 1) matrix = matrix[0].map(v => [v]); + // [W59-T06] Labels above one numeric run become categories; blanks retain missing values. + const headerRowCategories = !["stock", "gantt", "scatter", "bubble"].includes(kind) + && (presentation.headers ?? "auto") !== "none" + && (presentation.categoryLabels ?? "auto") === "auto" + && (presentation.orientation ?? "columns") === "columns" + && matrix.length === 2 && matrix[0].length > 1 + && matrix[0].every(v => v.text.trim() && v.number === null) + && matrix[1].some(v => v.number !== null) + && matrix[1].every(v => v.number !== null || !v.text.trim()); + if (headerRowCategories) matrix = matrix[0].map((_, c) => matrix.map(r => r[c])); const first = matrix[0]; - const header = presentation.headers === "firstRow" || (presentation.headers !== "none" + const numericHeaderCols = first.map((_, c) => c).filter(c => matrix.slice(1).some(r => r[c].number !== null)); + // [W59-T06] Auto headers have no numbers in value columns; a rotated header is already categories. + const header = !headerRowCategories && (presentation.headers === "firstRow" || (presentation.headers !== "none" && first.some(v => v.text.trim() && v.number === null) - && matrix.slice(1).some(r => r.some(v => v.number !== null))); + && numericHeaderCols.length > 0 && numericHeaderCols.every(c => first[c].number === null))); const body = matrix.slice(header ? 1 : 0); if (!body.length) return empty(); const numericCols = first.map((_, c) => c).filter(c => body.some(r => r[c].number !== null)); @@ -146,7 +158,7 @@ export function chartDataForRange( if (!valueCols.length) return empty(["stock", "gantt"].includes(kind) ? "Choose numeric chart values." : undefined); let series = valueCols.map(c => ({ y: `value${c}`, - sourceKey: presentation.orientation === "rows" ? `row:${first[c].row+1}` : `column:${addressOf(first[c].col,0).replace(/[0-9]+$/, "")}`, + sourceKey: presentation.orientation === "rows" || headerRowCategories ? `row:${first[c].row+1}` : `column:${addressOf(first[c].col,0).replace(/[0-9]+$/, "")}`, label: header ? first[c].text || addressOf(first[c].col,first[c].row) : addressOf(first[c].col,first[c].row).replace(/[0-9]+$/, ""), color: undefined as SheetChartColor | undefined, })); diff --git a/web/src/sheet/SheetGrid.tsx b/web/src/sheet/SheetGrid.tsx index 8afeb83d3c5ba0b1d412248486072f383a8d5bef..cf1397f63a1f4eafc52657817bb7758d232b0365 100644 --- a/web/src/sheet/SheetGrid.tsx +++ b/web/src/sheet/SheetGrid.tsx @@ -197,8 +197,8 @@ import type { SavedViewShadow } from "./bindingRefs"; // `ShortcutsPanel.tsx` renders it, so the two cannot advertise different keys. import { arrowStep, columnBand, commitDirection, commitTarget, currentRegion, edgeIndex, edgeStart, extendEdgeRect, isArrowKey, KEY_TIP_GRID_ARROW_EVENT, keyTipGridArrow, matchShortcut, - rowBand, shortcutOwner, structureAxis, tabRunOrigin } from "./shortcuts"; -import type { SelectionAxis, SelectionRect, ShortcutId } from "./shortcuts"; + rowBand, selectAllTarget, shortcutOwner, structureAxis, tabRunOrigin } from "./shortcuts"; +import type { CommitDirection, SelectionAxis, SelectionRect, ShortcutId } from "./shortcuts"; /** * ⭐⭐ W50-T20 — THE EXTENT IS THE SHEET'S NOW, AND THESE ARE ONLY ITS DEFAULTS. @@ -695,8 +695,8 @@ export default function SheetGrid({ * * `tabRunAt` holds the column a run of Tabs STARTED at, so the Enter that ends the run can * return to it; `shortcuts.ts::tabRunOrigin` owns the rule for what it becomes on each press. - * `commitNav` is raised for exactly one selection change, by the commit-navigation arm in - * `onKeyDown`, and read back in `onSelectionChange` below. + * `commitNav` is raised for exactly one selection change by the canvas commit arm and consumed + * in `onSelectionChange` below. Editor commits restore their run after that door resets it. * * ⛔ THE FLAG IS WHY THE RESET CAN LIVE IN `onSelectionChange` AT ALL. The done-when asks the * run to reset when the selection moves by anything other than Tab, and that handler is the ONE @@ -710,6 +710,14 @@ export default function SheetGrid({ */ const tabRunAt = useRef(null); const commitNav = useRef(false); + const overlayCommit = useRef<{ dir: CommitDirection; at: number; from: Item } | null>(null); + /** + * W59-T02 / R4: the Ctrl+A run remembers its sheet anchor and consecutive press count. + * Its own selection must preserve the run, so `selectAllNav` is consumed once at the same + * selection door as `commitNav`. A separate flag matters: a Tab commit must end this run. + */ + const selectAllRunAt = useRef<{ col: number; row: number; count: number } | null>(null); + const selectAllNav = useRef(false); /** * ⭐ W51-T17 — THE TWO SURFACES THE KEYBOARD OPENS, and both live INSIDE this component's own * box rather than in a portal: portalling drops every descendant-scoped rule with no error at @@ -1188,8 +1196,8 @@ export default function SheetGrid({ * identity also gives it a new callback identity. Keeping one callback lets a same-tab refresh * remain a repaint rather than look like navigation. */ - const selectionContextRef = useRef({ addrAt, onSelect, tab, viewRowOf, visCols }); - selectionContextRef.current = { addrAt, onSelect, tab, viewRowOf, visCols }; + const selectionContextRef = useRef({ addrAt, bandRows, onSelect, tab, viewRowOf, visCols, visRows }); + selectionContextRef.current = { addrAt, bandRows, onSelect, tab, viewRowOf, visCols, visRows }; /** * What the NEXT call to `onSelectionChange` is: a reader gesture or an app-owned move. @@ -1223,8 +1231,43 @@ export default function SheetGrid({ const replacementCell = replacementSelection?.current?.cell; const offeredCell = next.current?.cell; let offered = next; + let overlayRun: number | null | undefined; + // W59-T03 / D-697: a captured chord alone is not a grid commit. Require both a fresh stamp + // from the prior cell and glide's matching, unmodified move. The canvas commit arm already + // aged the run and raised commitNav, so consuming its stamp must not age it a second time. + const stamp = overlayCommit.current; + overlayCommit.current = null; + if (!commitNav.current && fromGlide && stamp && prior && offeredCell + && Date.now() - stamp.at < 300 + && prior[0] === stamp.from[0] && prior[1] === stamp.from[1]) { + const bounds = { + minCol: 0, maxCol: Math.max(0, context.visCols.length - 1), + minRow: -context.bandRows, maxRow: Math.max(0, context.visRows.length - 1), + }; + const from = { col: prior[0], row: prior[1] }; + const expected = commitTarget(from, stamp.dir, null, bounds); + if (offeredCell[0] === expected.col && offeredCell[1] === expected.row) { + const origin = tabRunAt.current; + if ((stamp.dir === "down" || stamp.dir === "up") && origin !== null) { + const land = commitTarget(from, stamp.dir, origin, bounds); + offered = { + ...offered, + current: { + ...offered.current!, + cell: [land.col, land.row] as Item, + range: { x: land.col, y: land.row, width: 1, height: 1 }, + }, + }; + } + overlayRun = tabRunOrigin(origin, stamp.dir, prior[0]); + } + } + const overlayCommitted = overlayRun !== undefined; + + // An overlay commit can replace the workbook before glide offers its next cell. Its + // validated move wins over that replacement's reconciliation handoff, just as a click does. if (replacementSelection && replacementCell && offeredCell - && fromGlide && !readerCaused + && fromGlide && !readerCaused && !overlayCommitted && (replacementCell[0] !== offeredCell[0] || replacementCell[1] !== offeredCell[1])) { offered = { ...replacementSelection, @@ -1252,6 +1295,13 @@ export default function SheetGrid({ // own move and this consumes it, so a Tab keeps the run and everything else ends it. if (commitNav.current) commitNav.current = false; else tabRunAt.current = null; + // W59-T03 / D-697: a validated glide overlay commit is a commit too. The reset above cannot + // know that, because glide never reaches `onKeyDown` and so nothing claimed the move; this + // puts back the run the commit should leave behind, computed from the origin read BEFORE it. + if (overlayRun !== undefined) tabRunAt.current = overlayRun; + // W59-T02 / R4: only Ctrl+A's own selection keeps its run, including at the same anchor. + if (selectAllNav.current) selectAllNav.current = false; + else selectAllRunAt.current = null; // R11: the bar is anchored to the selection and unmounts when the selection changes. Clearing // the anchor here is what makes "disappears when the selection changes" true rather than // approximately true. @@ -1478,7 +1528,7 @@ export default function SheetGrid({ }, [formulaEdit]); /** Enter/Tab outside point mode, and the second half of PM-12 inside it. */ - const commitFormulaEditor = useCallback(() => { + const commitFormulaEditor = useCallback((move: CommitDirection = "down") => { if (!formulaEdit) return; const { addr, draft, inBand, viewCell } = formulaEdit; // ⭐⭐ W53-T02 — READY IS RESTORED FIRST, THE WRITE SECOND, and the order is the whole point. @@ -1497,22 +1547,32 @@ export default function SheetGrid({ // cancelled rather than tidied up and saved. const closed = autoCloseFormula(draft); commitWithRecalc({ [addr]: commitTyped(cellAt(tab, addr), closed, cellValueGetter) }); - // Mirrors glide's own default Enter-commit: the selection moves down one row, so the custom - // editor does not feel like a different control from editing any other cell. + // W59-T01 / instruction 3: choose the move inside the commit so Tab lands horizontally once. + // The default stays down; commitTarget now clamps at the last visible row, matching the + // canvas Enter path instead of letting the editor select beyond the sheet. // ⚠ NOT `applySelection` -- that helper is declared FURTHER DOWN this component (it is the // one thing a controlled selection needs `gridRef` for, beside the structural ops below), and // this callback is defined ahead of it. `onSelectionChange` is the same door one layer down. if (!inBand) { - const next: Item = [viewCell[0], viewCell[1] + 1]; + const origin = tabRunAt.current; + const land = commitTarget( + { col: viewCell[0], row: viewCell[1] }, move, origin, + { minCol: 0, maxCol: Math.max(0, visCols.length - 1), + minRow: -bandRows, maxRow: Math.max(0, visRows.length - 1) }, + ); + const next: Item = [land.col, land.row]; selectionIntentRef.current = "reader"; onSelectionChange({ columns: CompactSelection.empty(), rows: CompactSelection.empty(), current: { cell: next, range: { x: next[0], y: next[1], width: 1, height: 1 }, rangeStack: [] }, }); + // W59-T03 / D-697's second half: that door resets the run for any move it was not told + // about. This is a commit, so the run continues from the origin read above. + tabRunAt.current = tabRunOrigin(origin, move, viewCell[0]); gridRef.current?.scrollTo(next[0], next[1]); } - }, [cellValueGetter, commitWithRecalc, formulaEdit, onSelectionChange, tab]); + }, [bandRows, cellValueGetter, commitWithRecalc, formulaEdit, onSelectionChange, tab, visCols, visRows]); /** * ⭐⭐ W54-T21 — THE BLUR DOOR, AND THE ONE REASON IT IS NO LONGER THE COMMIT ITSELF. @@ -1850,8 +1910,14 @@ export default function SheetGrid({ return; } if (e.key === "Enter") { e.preventDefault(); commitFormulaEditor(); return; } - // D-1: Tab is captured ONLY while the dropdown is open (handled above); otherwise it traverses - // focus like any other control, so nothing here intercepts it. + // W59-T01 / instruction 3: plain-edit Tab commits and moves right; Shift+Tab moves left. + // The dropdown and point-mode branches above keep ownership of their Tab gestures. + if (e.key === "Tab") { + if (e.ctrlKey || e.metaKey || e.altKey) return; + e.preventDefault(); + commitFormulaEditor(e.shiftKey ? "left" : "right"); + return; + } // ⛔ W54-T21: and NOTHING in this handler reads Escape, in any branch. The arbiter's listener // is capture phase on the document and stops propagation, so a branch here could never fire // anyway - it would be dead code that a later reader would trust. @@ -1875,14 +1941,14 @@ export default function SheetGrid({ /** * ⭐⭐ CR-1/CR-5/CR-10/PM-6 — EVERY CELL A LIVE REFERENCE TOUCHES, KEYED BY ITS REAL 1-BASED - * `"col,row"` ADDRESS, TO THE COLOUR IT OWNS. `refsIn` already scans the draft on every - * keystroke (CR-10: "recolour on every keystroke"), and since a point-mode proposal is spliced + * `"col,row"` ADDRESS, TO ITS COLOUR AND REFERENCE ORDINAL. `refsIn` already scans the draft + * on every keystroke (CR-10: "recolour on every keystroke"), and since a point-mode proposal is spliced * into the SAME draft text as any other reference (`proposeCell` above), the proposal gets its * border for free through this one map -- PM-6 needs no separate mechanism. * - * ⚠ EACH CELL OF A RANGE GETS ITS OWN BORDER, not one rectangle traced around the range's outer - * edge. Simpler, and still says "these cells" unambiguously; tracing only the outer edge is the - * tidy follow up. + * W59-T05 / INSTRUCTION 17 / R7: ONE OUTER PERIMETER PER REFERENCE. Each cell paints only + * edges whose neighbour belongs to a different reference or none. Reference ordinals keep + * touching ranges separate even when their slot colours repeat; no interior fill is added. * * ⭐⭐ W53-T02 / INSTRUCTION 18 — RANGE FINDER ALSO FIRES ON A BARE SELECTION. Microsoft's own * page documents only the edit-mode trigger ("double-click the cell ... Excel highlights each @@ -1893,12 +1959,14 @@ export default function SheetGrid({ * (`paintRefBorder` further down) - there is no second outline mechanism to keep in step. */ const formulaRefCells = useMemo(() => { - const out = new Map(); + const out = new Map(); const source = formulaEdit ? formulaEdit.draft : (selected?.cell?.kind === "formula" ? (selected.cell.formula ?? "") : ""); if (!source) return out; + let refOrdinal = 0; for (const ref of refsIn(source)) { + const ordinal = refOrdinal++; // Count every reference, including ranges skipped by the cap. let c0 = colToIndex(ref.start.col); let c1 = colToIndex(ref.end.col); let r0 = ref.start.row; @@ -1915,7 +1983,7 @@ export default function SheetGrid({ const color = colorForSlot(ref.colorIndex, "light"); for (let c = Math.min(c0, c1); c <= Math.max(c0, c1); c += 1) { for (let r = Math.min(r0, r1); r <= Math.max(r0, r1); r += 1) { - out.set(`${c},${r}`, color); + out.set(`${c},${r}`, { color, ref: ordinal }); } } } @@ -2009,8 +2077,8 @@ export default function SheetGrid({ */ const refPaintKey = useMemo(() => { const parts: string[] = []; - for (const [key, color] of formulaRefCells) { - parts.push(`${key}:${color}:${formulaPointCells.has(key) ? "dash" : "solid"}`); + for (const [key, entry] of formulaRefCells) { + parts.push(`${key}:${entry.color}:${formulaPointCells.has(key) ? "dash" : "solid"}:${entry.ref}`); } for (const key of formulaPointCells) { if (!formulaRefCells.has(key)) parts.push(`${key}:none:dash`); @@ -3745,8 +3813,9 @@ export default function SheetGrid({ const cur = selection?.current; switch (hit) { - // -- the five the grid canvas owns. Unreachable while the table says `canvas`. -------- - case "extend-cell": case "go-home": case "page-up": case "page-down": case "select-all": + // -- the four the canvas owns: extend-cell, go-home, page-up, page-down. --------------- + // Unreachable while the table says `canvas`. + case "extend-cell": case "go-home": case "page-up": case "page-down": return; // -- moving and selecting ------------------------------------------------------------- @@ -3810,6 +3879,21 @@ export default function SheetGrid({ if (!cur) return; applySelection(rowBand(cur.cell[1], visCols.length), cur.cell); return; + case "select-all": { + if (!cur) return; + const anchor = { col: realCol(cur.cell[0]), row: realRow(cur.cell[1]) }; + const run = selectAllRunAt.current; + const count = run && run.col === anchor.col && run.row === anchor.row + ? run.count + 1 : 1; + selectAllRunAt.current = { ...anchor, count }; + const region = selectAllTarget(anchor, extent, (c, r) => isFilled(tab, c, r), count); + selectAllNav.current = true; + applySelection({ + col0: viewIndexOf(visCols, region.col0), row0: viewRowOf(region.row0), + col1: viewIndexOf(visCols, region.col1), row1: viewRowOf(region.row1), + }, cur.cell); + return; + } case "select-region": { if (!cur) return; const region = currentRegion({ col: realCol(cur.cell[0]), row: realRow(cur.cell[1]) }, @@ -4150,15 +4234,34 @@ export default function SheetGrid({ // same reference's slot colour, which is the state instruction 18 describes. The colour is // what tells it apart from the copy marquee's own dash further down. const refKey = `${realCol(col) + 1},${realRow(row) + 1}`; - const refColor = formulaRefCells.get(refKey); + const refEntry = formulaRefCells.get(refKey); const refProposed = formulaPointCells.has(refKey); const paintRefBorder = () => { - if (!refColor) return; + if (!refEntry) return; + const sameRef = (dc: number, dr: number) => + formulaRefCells.get(`${realCol(col) + 1 + dc},${realRow(row) + 1 + dr}`)?.ref === refEntry.ref; ctx.save(); - ctx.strokeStyle = refColor; + ctx.strokeStyle = refEntry.color; ctx.lineWidth = 2; if (refProposed) ctx.setLineDash([4, 3]); - ctx.strokeRect(rect.x + 1, rect.y + 1, rect.width - 2, rect.height - 2); + ctx.beginPath(); + if (!sameRef(-1, 0)) { + ctx.moveTo(rect.x + 1, rect.y); + ctx.lineTo(rect.x + 1, rect.y + rect.height); + } + if (!sameRef(1, 0)) { + ctx.moveTo(rect.x + rect.width - 1, rect.y); + ctx.lineTo(rect.x + rect.width - 1, rect.y + rect.height); + } + if (!sameRef(0, -1)) { + ctx.moveTo(rect.x, rect.y + 1); + ctx.lineTo(rect.x + rect.width, rect.y + 1); + } + if (!sameRef(0, 1)) { + ctx.moveTo(rect.x, rect.y + rect.height - 1); + ctx.lineTo(rect.x + rect.width, rect.y + rect.height - 1); + } + ctx.stroke(); ctx.setLineDash([]); ctx.restore(); }; @@ -4581,6 +4684,12 @@ export default function SheetGrid({
{ + const dir = commitDirection(event); + const from = selectionRef.current?.current?.cell; + if (dir === null || formulaEdit || !from) return; + overlayCommit.current = { dir, at: Date.now(), from: [from[0], from[1]] }; + }} onPointerDownCapture={() => { pointerSelectingRef.current = true; }} onPointerUpCapture={() => { queueMicrotask(() => { pointerSelectingRef.current = false; }); diff --git a/web/src/sheet/SheetRail.tsx b/web/src/sheet/SheetRail.tsx index 16142222c70dde3baea4b26007ddda54fe204e73..3aeb8560bf499df313b13d948ab59eb45e7e8beb 100644 --- a/web/src/sheet/SheetRail.tsx +++ b/web/src/sheet/SheetRail.tsx @@ -42,12 +42,14 @@ import "../styles/wave52/parity.css"; // path row's seating is decided by weight rather than by emission order. See the file's header // for the two measured row heights that make the extra pixel necessary. import "../styles/wave53/sheet-rail.css"; +import "../styles/wave59/templates.css"; import { AnchoredOverlay } from "../customer-grid/OverlaySurface"; import { FolderMark, MenuLabel } from "../customer-grid/icons"; import { SHARE_SHAPE, SHEET_SHAPE } from "../customer-grid/iconShapes"; import { StarIcon } from "../ui/icons"; import { ConfirmDialog } from "../ui/ConfirmDialog"; import ImportDialog from "./ImportDialog"; +import TemplateGallery from "./templates/TemplateGallery"; import { ROOT_FOLDER_ID } from "./types"; import type { SheetFolder, WorkbookRow } from "./types"; @@ -341,6 +343,7 @@ export default function SheetRail({ * be ticking behind it. Unmounting is one of the poll's five terminal conditions. */ const [importing, setImporting] = useState(false); + const [pickingTemplate, setPickingTemplate] = useState(false); /** * ⭐⭐ W53-T09 (owner instruction 20) — THE RIBBON'S IMPORT BUTTON IS THE SECOND DOOR ONTO THIS @@ -922,6 +925,13 @@ export default function SheetRail({ > + )} @@ -1140,6 +1150,13 @@ export default function SheetRail({ onOpened={(workbookId) => { setImporting(false); onOpen(workbookId); }} /> ) : null} + {pickingTemplate ? ( + setPickingTemplate(false)} + onOpened={(workbookId) => { setPickingTemplate(false); onOpen(workbookId); }} + /> + ) : null} ); } diff --git a/web/src/sheet/ribbonIcons.tsx b/web/src/sheet/ribbonIcons.tsx index ce83a3da1fae1a1b842427121d8e14e921ce1119..4c5ef5458db3a9c5d71bbba4b693f5f12fea8c25 100644 --- a/web/src/sheet/ribbonIcons.tsx +++ b/web/src/sheet/ribbonIcons.tsx @@ -22,6 +22,74 @@ export function IconPaste() { ); } +/** Chart shortcuts share the ribbon's 16 by 16 drawing vocabulary. */ +export function IconChartColumn() { + return ( + + ); +} + +export function IconChartBar() { + return ( + + ); +} + +export function IconChartLine() { + return ( + + ); +} + +export function IconChartArea() { + return ( + + ); +} + +export function IconChartPie() { + return ( + + ); +} + +export function IconChartDonut() { + return ( + + ); +} + +export function IconChartScatter() { + return ( + + ); +} + +export function IconChartRecommended() { + return ( + + ); +} + /** Two handle loops and the crossing blades. */ export function IconCut() { return ( diff --git a/web/src/sheet/ribbonInsert.tsx b/web/src/sheet/ribbonInsert.tsx index 46d742e92b7efeba15ffcfc32b86adabb54728ee..6f3feecfe389296f7e28b68a0236bcefc84af46c 100644 --- a/web/src/sheet/ribbonInsert.tsx +++ b/web/src/sheet/ribbonInsert.tsx @@ -54,6 +54,11 @@ import { useState } from "react"; import type { FormEvent } from "react"; import type { RibbonTabContext } from "./Ribbon"; +import { + IconChartColumn, IconChartBar, IconChartLine, IconChartArea, + IconChartPie, IconChartDonut, IconChartScatter, IconChartRecommended, +} from "./ribbonIcons"; +// W59-T08 adds shortcuts into the same dialog; chart creation still uses its single commit door. // ⭐⭐ W54-T11 (ruling R5) — THE CHART GALLERY IS GONE AND A DIALOG STANDS WHERE IT STOOD. R5 reads // "A chart reads BOTH. A cell range, exactly like Excel, is the primary path; a database source is // a second tab in the same dialog. Ship both." A gallery of eight kinds cannot carry a second @@ -90,7 +95,18 @@ import { AnchoredOverlay } from "../customer-grid/OverlaySurface"; // imports its own. See the stylesheet's header for what it restates and why. import "../styles/wave54/ribbon-insert.css"; -const CHARTS = ["chart"] as const; +const CHART_SHORTCUTS = [ + { key: "chart-column", moreKey: "chart-column-choices", kind: "column", label: "Column", Icon: IconChartColumn, kt: "V" }, + { key: "chart-bar", moreKey: "chart-bar-choices", kind: "bar", label: "Bar", Icon: IconChartBar, kt: "B" }, + { key: "chart-line", moreKey: "chart-line-choices", kind: "line", label: "Line", Icon: IconChartLine, kt: "L" }, + { key: "chart-area", moreKey: "chart-area-choices", kind: "area", label: "Area", Icon: IconChartArea, kt: "A" }, + { key: "chart-pie", moreKey: "chart-pie-choices", kind: "pie", label: "Pie", Icon: IconChartPie, kt: "P" }, + { key: "chart-donut", moreKey: "chart-donut-choices", kind: "donut", label: "Donut", Icon: IconChartDonut, kt: "O" }, + { key: "chart-scatter", moreKey: "chart-scatter-choices", kind: "scatter", label: "Scatter", Icon: IconChartScatter, kt: "X" }, +] as const; +const CHARTS = [ + "chart", ...CHART_SHORTCUTS.flatMap(({ key, moreKey }) => [key, moreKey]), "chart-recommended", +] as const; const SPARKLINES = ["sparkline-line", "sparkline-column", "sparkline-winloss"] as const; const LINKS = ["link"] as const; const COMMENTS = ["comment"] as const; @@ -169,9 +185,8 @@ function IconComment() { } /** - * ⚠ `IconCaret` IS GONE WITH THE GALLERY (W54-T11). A caret says "a menu opens under this button", - * and the Charts control now opens a centred modal instead; wearing the caret would have promised - * the wrong surface. The two form popovers below never wore one. + * The large Chart opener stays a single command. Chart-kind split controls use a separate + * choices half to open the same dialog at that kind, without inserting before confirmation. */ // ═════════════════════════════════════════════════════════════════════════════════════════════ @@ -386,6 +401,13 @@ export const insertGroups = (ctx: RibbonTabContext) => { // oversight: the only menu this tab ever painted was the chart gallery, and the eight kinds are // buttons inside `ChartDialog` now. The shell still offers the helper to every tab. const { open, anchorEl, openPop, closePop, control, group, word, insert, act, openCellData } = ctx; + const chartShortcut = CHART_SHORTCUTS.find(({ key, moreKey }) => open === key || open === moreKey); + const chartOpen = open === "chart" || open === "chart-recommended" || !!chartShortcut; + const chartCaret = ( + + ); /** * ⭐ THE ANCHOR CELL, IN SHEET COORDINATES. The door hands the selection over already translated @@ -504,21 +526,41 @@ export const insertGroups = (ctx: RibbonTabContext) => { `styles/wave54/ribbon-insert.css`. */
{group("Charts", ( - + <> + {control("chart", "Chart", <>{word("Chart")}, - openPop("chart"), false, open === "chart", "C", "gallery")} + openPop("chart"), false, open === "chart", "C", "gallery", undefined, "dialog")} + + + {CHART_SHORTCUTS.map(({ key, moreKey, label, Icon, kt }) => ( + + {control(key, `${label} chart`, , + openPop(key), false, open === key, kt, "gallery", undefined, "dialog")} + + {control(moreKey, `${label} chart choices`, chartCaret, + openPop(moreKey), false, open === moreKey, undefined, "gallery", undefined, "dialog")} + + + ))} + + {control("chart-recommended", "Recommended charts", , + openPop("chart-recommended"), false, open === "chart-recommended", "R", "gallery", undefined, "dialog")} + + {/* ⚠ MOUNTED CONDITIONALLY, never rendered behind an `open` prop: `useOverlayLayer` claims Escape the moment it runs, so a hidden dialog would hold the key. The `insert` guard is belt and braces, and it is not the fieldset's job here: the dialog portals to the body and is therefore OUTSIDE the disabled fence. */} - {open === "chart" && insert && ( + {chartOpen && insert && ( )} - + ))} {/* ⭐ THREE CONTROLS RATHER THAN ONE WITH A GALLERY. Excel's Sparklines group is exactly @@ -528,7 +570,7 @@ export const insertGroups = (ctx: RibbonTabContext) => { badge with no such attribute as a plain command and never descends; these three now open a popover, so they have to say so or the key tip would flash and land nowhere. */} {group("Sparklines", ( - + {control("sparkline-line", "Line sparkline", , openPop("sparkline-line"), false, open === "sparkline-line", "SL", "gallery")} {control("sparkline-column", "Column sparkline", , @@ -559,7 +601,7 @@ export const insertGroups = (ctx: RibbonTabContext) => { ))} {group("Links", ( - + {control("link", "Link", <>{word("Link")}, openPop("link"), false, open === "link", "I", "gallery")} {/* ⚠ MOUNTED CONDITIONALLY AND OUTSIDE THE FIELDSET, for the two reasons the Charts @@ -584,7 +626,7 @@ export const insertGroups = (ctx: RibbonTabContext) => { ))} {group("Comments", ( - + {control("comment", "Comment", <>{word("Comment")}, openPop("comment"), false, open === "comment", "M", "gallery")} {open === "comment" && ( @@ -628,13 +670,13 @@ export const insertGroups = (ctx: RibbonTabContext) => { )} ))} - {group("Cell data", control("cell-data", "Cell database", <> + {group("Cell data", {control("cell-data", "Cell database", <> {word("Cell database")} - , openCellData, undefined, undefined, "D"))} + , openCellData, undefined, undefined, "D")})}
); }; diff --git a/web/src/sheet/shortcuts.ts b/web/src/sheet/shortcuts.ts index 5c436ab4eb8a7e2087ce153f96bc545776f65229..e24a0b9260cffb9bd8039ee586a01afe1856f754 100644 --- a/web/src/sheet/shortcuts.ts +++ b/web/src/sheet/shortcuts.ts @@ -154,9 +154,9 @@ export const SHORTCUTS: readonly Shortcut[] = [ { id: "select-all", keys: "Ctrl+A", - action: "Select every cell on the sheet", - when: "The grid has the keyboard", - owner: "canvas", + action: "Select the current region, then the whole sheet on a second press", + when: "The grid has the keyboard. With no filled neighbours, select the whole sheet immediately", + owner: "sheet", }, { id: "select-column", @@ -829,6 +829,27 @@ export const currentRegion = ( }; }; +/** + * W59-T02 / R4: Ctrl+A selects the current region first, then the whole sheet. + * A one-cell region goes straight to the sheet, even when the anchor itself is filled: + * it is the absence of filled neighbours that matters. Counts are one based; zero and + * negative counts use the first-press rule too. The region walk stays in {@link currentRegion}. + */ +export const selectAllTarget = ( + anchor: { col: number; row: number }, + extent: { cols: number; rows: number }, + filled: (col: number, row: number) => boolean, + pressCount: number, +): SelectionRect => { + const sheet = { + col0: 0, row0: 0, + col1: Math.max(0, extent.cols - 1), row1: Math.max(0, extent.rows - 1), + }; + if (pressCount >= 2) return sheet; + const region = currentRegion(anchor, extent, filled); + return region.col0 === region.col1 && region.row0 === region.row1 ? sheet : region; +}; + /** The rectangle Ctrl+Space means: one whole column, in whatever coordinates it was handed. */ export const columnBand = (col: number, rows: number) => ({ col0: col, col1: col, row0: 0, row1: Math.max(0, rows - 1), diff --git a/web/src/sheet/templates/TemplateGallery.tsx b/web/src/sheet/templates/TemplateGallery.tsx new file mode 100644 index 0000000000000000000000000000000000000000..fd05cdec8ef5120edd4c20c45c488b3e13b2dd07 --- /dev/null +++ b/web/src/sheet/templates/TemplateGallery.tsx @@ -0,0 +1,167 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { ConfirmDialog } from "../../ui/ConfirmDialog"; +import { applySheetTemplate, fetchSheetTemplates } from "./templatesApi"; +import { CONNECTORS_ROUTE } from "../../shell/nav"; +import type { SheetTemplate } from "./types"; + +export default function TemplateGallery({ open, onCancel, onOpened }: { + open: boolean; + onCancel: () => void; + onOpened: (workbookId: string) => void; +}) { + const [templates, setTemplates] = useState([]); + const [phase, setPhase] = useState<"loading" | "loaded" | "failed">("loading"); + const [fetchError, setFetchError] = useState(""); + const [selectedKey, setSelectedKey] = useState(""); + const [database, setDatabase] = useState(""); + const [applyError, setApplyError] = useState(""); + const [busy, setBusy] = useState(false); + const requestRef = useRef(null); + const applyingRef = useRef(false); + const selected = templates.find((template) => template.key === selectedKey); + const canCreate = phase === "loaded" && Boolean( + selected?.databases.some((option) => option.key === database), + ); + + const load = useCallback(() => { + const controller = new AbortController(); + requestRef.current = controller; + applyingRef.current = false; + setTemplates([]); + setPhase("loading"); + setFetchError(""); + setSelectedKey(""); + setDatabase(""); + setApplyError(""); + setBusy(false); + void fetchSheetTemplates(controller.signal).then((result) => { + if (controller.signal.aborted) return; + if (!result.ok) { + setFetchError(result.message); + setPhase("failed"); + return; + } + // Preselect, so the primary button is never a control that refuses. R8: a + // fake affordance is worse than no affordance. + const offered = result.value.filter((template) => template.databases.length > 0); + const first = offered[0]; + setTemplates(offered); + if (first) { + setSelectedKey(first.key); + setDatabase(first.databases.find((option) => option.key === first.databaseHint)?.key + ?? first.databases[0].key); + } + setPhase("loaded"); + }); + return () => controller.abort(); + }, []); + + // ⭐ THE RETRY GOES THROUGH STATE, NOT THROUGH A SECOND CALL TO `load`. The effect owns the + // AbortController's cleanup, so a dialog closed mid retry aborts the request it started rather + // than leaking it and setting state after unmount. + const [reloadKey, setReloadKey] = useState(0); + + useEffect(() => { + if (!open) return undefined; + return load(); + }, [open, load, reloadKey]); + + const createWorkbook = async () => { + const controller = requestRef.current; + if (!canCreate || !selected || applyingRef.current || !controller || controller.signal.aborted) return; + applyingRef.current = true; + setBusy(true); + setApplyError(""); + try { + const result = await applySheetTemplate(selected.key, database); + if (controller.signal.aborted) return; + if (result.ok) onOpened(result.value.id); + else setApplyError(result.message); + } finally { + if (!controller.signal.aborted) { + applyingRef.current = false; + setBusy(false); + } + } + }; + + // ⛔ THE SHARED KIT HAS NO `confirmDisabled`, and reaching into `.lp-dialog` to disable its + // button imperatively was the first draft of this file. That fights React and pins this + // component to another component's DOM. Instead the PRIMARY ACTION CHANGES WITH THE PHASE, so + // the button always does the one thing that makes sense where the person is standing. + const primary = phase === "failed" + ? { label: "Try again", run: () => { setReloadKey((n) => n + 1); } } + : templates.length === 0 && phase === "loaded" + ? { label: "Connect a source", + run: () => { onCancel(); window.location.hash = "#/" + CONNECTORS_ROUTE; } } + : { label: "Create workbook", run: () => { void createWorkbook(); } }; + + return ( + + {phase === "loading" ? ( +
+ {[0, 1, 2].map((row) => ( + + ))} +
+ ) : phase === "failed" ? ( +

{fetchError}

+ ) : templates.length === 0 ? ( +

No connector has published a database yet.

+ ) : ( + <> +
+ {templates.map((template) => ( + + ))} +
+ {selected ? ( + + ) : null} +

{applyError}

+ + )} +
+ } + /> + ); +} diff --git a/web/src/sheet/templates/templatesApi.ts b/web/src/sheet/templates/templatesApi.ts new file mode 100644 index 0000000000000000000000000000000000000000..b819abc5d6b37671a919a506419b60f9c42f384d --- /dev/null +++ b/web/src/sheet/templates/templatesApi.ts @@ -0,0 +1,87 @@ +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { normalizeCellBinding } from "../types"; +import type { Result } from "../sheetApi"; +import type { Workbook } from "../types"; +import type { SheetTemplate, SheetTemplateDatabase } from "./types"; + +const object = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const text = (value: unknown): value is string => + typeof value === "string" && value.trim().length > 0; + +const database = (value: unknown): value is SheetTemplateDatabase => + object(value) && text(value.key) && text(value.label); + +const template = (value: unknown): value is SheetTemplate => + object(value) && text(value.key) && text(value.title) && text(value.answers) && + text(value.source) && text(value.databaseHint) && Array.isArray(value.databases) && + value.databases.every(database); + +const workbook = (value: unknown): value is Workbook => + object(value) && text(value.id) && text(value.name) && + typeof value.rev === "number" && Number.isInteger(value.rev) && value.rev > 0 && + (value.refreshed_at === null || typeof value.refreshed_at === "string") && + Array.isArray(value.sheets) && value.sheets.length > 0 && value.sheets.every((tab: unknown) => + object(tab) && text(tab.id) && text(tab.name) && object(tab.cells) && + Object.entries(tab.cells).every(([addr, cell]) => + /^[A-Z]+[1-9][0-9]*$/.test(addr) && object(cell) && + typeof cell.kind === "string" && ["literal", "formula", "bound", "empty"].includes(cell.kind) && + typeof cell.state === "string" && ["fresh", "stale", "typed", "refused"].includes(cell.state) && + (cell.kind !== "formula" || text(cell.formula)) && + (cell.kind !== "bound" || (object(cell.binding) && text(cell.binding.database) && + object(cell.binding.scope) && object(cell.binding.aggregate) && + typeof cell.binding.aggregate.fn === "string" && + ["sum", "count", "avg", "min", "max"].includes(cell.binding.aggregate.fn) && + (cell.binding.aggregate.fn === "count" || text(cell.binding.aggregate.field)))))); + +const normalizedWorkbook = (book: Workbook): Workbook => ({ + ...book, + sheets: book.sheets.map((tab) => ({ + ...tab, + cells: Object.fromEntries(Object.entries(tab.cells).map(([addr, cell]) => [addr, + cell.binding ? { ...cell, binding: normalizeCellBinding(cell.binding) } : cell, + ])), + })), +}); + +async function call( + path: string, + init: RequestInit, + parse: (body: Record) => T | null, +): Promise> { + let response: Response; + try { + response = await fetch(`${API_V1}${path}`, { credentials: CREDENTIALS, ...init }); + } catch { + return { ok: false, status: 0, code: "unreachable", message: "Cannot reach the server." }; + } + const body: unknown = await response.json().catch(() => null); + if (!response.ok) { + const detail = object(body) && object(body.detail) ? body.detail : body; + const error = object(detail) && object(detail.error) ? detail.error : null; + return { + ok: false, status: response.status, + code: error && text(error.code) ? error.code : "unknown", + // Keep the server's sentence intact, including every missing field name. + message: error && text(error.message) ? error.message : "The sheet template request failed.", + }; + } + const value = object(body) ? parse(body) : null; + return value === null + ? { ok: false, status: response.status, code: "invalid_response", + message: "The server returned an invalid sheet template response." } + : { ok: true, value }; +} + +export const fetchSheetTemplates = (signal?: AbortSignal): Promise> => + call("/sheet-templates", { method: "GET", signal }, (body) => + Array.isArray(body.templates) ? body.templates.filter(template) : null); + +export const applySheetTemplate = ( + key: string, database: string, name?: string, +): Promise> => + call(`/sheet-templates/${encodeURIComponent(key)}/apply`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ database, ...(name === undefined ? {} : { name }) }), + }, (body) => workbook(body.workbook) ? normalizedWorkbook(body.workbook) : null); diff --git a/web/src/sheet/templates/types.ts b/web/src/sheet/templates/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..a5286a321319de41ed5a97fd7199f2042ccbb6d7 --- /dev/null +++ b/web/src/sheet/templates/types.ts @@ -0,0 +1,13 @@ +export interface SheetTemplateDatabase { + key: string; + label: string; +} + +export interface SheetTemplate { + key: string; + title: string; + answers: string; + source: string; + databaseHint: string; + databases: SheetTemplateDatabase[]; +} diff --git a/web/src/shell/LoginPage.tsx b/web/src/shell/LoginPage.tsx index 6db87d041ad1956c068c827c898fd3c3c553ec51..998e9cf367f83bf64cc05d75133c6028ab435cc6 100644 --- a/web/src/shell/LoginPage.tsx +++ b/web/src/shell/LoginPage.tsx @@ -22,71 +22,12 @@ import { useEffect, useRef, useState } from "react"; import type { FormEvent } from "react"; import { Brand } from "./Brand"; -import { changePassword, login, mfaEnrollConfirm, mfaEnrollStart, passwordPolicy } from "./session"; -import type { LoginOutcome, MFASetup, PasswordPolicy, SessionUser } from "./session"; -import { useEscapeClose } from "../ui/useEscapeClose"; +import { changePassword, login, passwordPolicy } from "./session"; +import type { LoginOutcome, PasswordPolicy, SessionUser } from "./session"; +import SecurityPane from "../settings/SecurityPane"; import "../styles/wave57/login-mfa.css"; -type Step = "credentials" | "expired" | "mfa" | "enrol" | "codes"; -type IssuedCodes = { user: SessionUser; recoveryCodes: string[] }; - -/** W57-T36 / C12: one show-once treatment in both doors. Copy is a quiet, - * explicit export to the person's clipboard, never persistence on their behalf. - * Only the acknowledgement can dismiss this surface; Escape names what remains. - * Browser reload/close can only be warned about, never made impossible by a page. */ -export function RecoveryCodes({ codes, onSaved, onError }: { - codes: string[]; - onSaved: () => void; - onError: (message: string) => void; -}) { - const [copied, setCopied] = useState(false); - const [copying, setCopying] = useState(false); - useEscapeClose(true, () => onError("Save your recovery codes before continuing.")); - useEffect(() => { - const warn = (event: BeforeUnloadEvent) => { - event.preventDefault(); - event.returnValue = ""; - }; - window.addEventListener("beforeunload", warn); - return () => window.removeEventListener("beforeunload", warn); - }, []); - - async function copy() { - if (copying) return; - setCopying(true); - try { - await navigator.clipboard.writeText(codes.join("\n")); - setCopied(true); - onError(""); - } catch { - setCopied(false); - onError("Copy did not work. Select the codes and copy them manually."); - } finally { - setCopying(false); - } - } - - return ( - <> -

This screen is the only place these recovery codes will ever be shown.

-

Save them somewhere safe; each code works once if you lose your authenticator.

-
    - {codes.map((code) =>
  • {code}
  • )} -
-
- - {copied ? "Copied" : ""} -
- {/* A type=button acknowledgement cannot be submitted by Enter left over - from the previous code field. There is no Back or secondary exit. */} - - - ); -} +type Step = "credentials" | "expired" | "mfa" | "enrol"; /** The reveal toggle the host's password field ships natively (BaseWeb's eye). * Two doors, one affordance. */ @@ -110,8 +51,6 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs const [newPassword, setNewPassword] = useState(""); const [confirmPassword, setConfirmPassword] = useState(""); const [code, setCode] = useState(""); - const [setup, setSetup] = useState(null); - const [issued, setIssued] = useState(null); const [policy, setPolicy] = useState(null); const [policyLoading, setPolicyLoading] = useState(false); const flight = useRef(false); @@ -136,21 +75,12 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs // the new task to assistive tech, and focus the heading rather than a submit. useEffect(() => { stepHeading.current?.focus(); }, [step]); - async function startEnrolment(currentPassword: string) { - setSetup(null); - setCode(""); - const result = await mfaEnrollStart(username, currentPassword); - if (result.ok === false) setError(result.message); - else setSetup({ secret: result.secret, otpauthUri: result.otpauthUri, issuer: result.issuer }); - } - /** Correct credentials must survive a challenge. Clearing the password on * every 401 would make the second-factor POST an invalid-credentials loop. * It stays only in this mounted tree, and leaves with success or Back. */ - async function accept(result: LoginOutcome, currentPassword: string) { + async function accept(result: LoginOutcome) { if (result.ok === true) { setPassword(""); - setSetup(null); setCode(""); onSignedIn(result.user); return; @@ -160,18 +90,15 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs case "password_expired": setPolicyLoading(true); setStep("expired"); - setSetup(null); setCode(""); break; case "mfa_required": case "mfa_invalid": setStep("mfa"); - setSetup(null); setCode(""); break; case "mfa_enrolment_required": setStep("enrol"); - await startEnrolment(currentPassword); break; default: // Keep the current challenge on network/backoff errors so retry uses @@ -182,7 +109,7 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs async function submit(e: FormEvent) { e.preventDefault(); - if (flight.current || step === "codes") return; + if (flight.current) return; flight.current = true; setBusy(true); setError(""); @@ -203,26 +130,9 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs setNewPassword(""); setConfirmPassword(""); setStep("credentials"); - await accept(await login(username, newPassword), newPassword); - } else if (step === "enrol") { - if (!setup) { - await startEnrolment(password); - return; - } - const confirmed = await mfaEnrollConfirm(username, password, setup.secret, code); - if (confirmed.ok === false) { - setError(confirmed.message); - return; - } - // The server has minted a cookie, but this component does NOT open the - // frame yet. C12 requires the person to save this one clear response. - setIssued({ user: confirmed.user, recoveryCodes: confirmed.recoveryCodes }); - setSetup(null); - setPassword(""); - setCode(""); - setStep("codes"); + await accept(await login(username, newPassword)); } else { - await accept(await login(username, password, step === "mfa" ? code : undefined), password); + await accept(await login(username, password, step === "mfa" ? code : undefined)); } } finally { flight.current = false; @@ -231,14 +141,12 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs } function back() { - if (flight.current || step === "codes") return; + if (flight.current) return; setStep("credentials"); setPassword(""); setNewPassword(""); setConfirmPassword(""); setCode(""); - setSetup(null); - setIssued(null); setReveal(false); setError(""); } @@ -272,14 +180,21 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs ⚠ No dash of any kind (R6). A slogan is exactly where an em dash wants to appear. */}

Every workflow, automated.

- {/* One card and one anchor across all five steps. The extra content + {/* One card and one anchor across every step. The extra content grows downward; no alternate width, position or brand lockup. */} - {step !== "credentials" ? ( + {step === "expired" || step === "mfa" ? (

- {step === "expired" ? "New password" : step === "mfa" ? "Two step sign in" - : step === "enrol" ? "Authenticator setup" : "Recovery codes"} + {step === "expired" ? "New password" : "Two step sign in"}

) : null} + {step === "enrol" ? ( + { + setPassword(""); + setCode(""); + onSignedIn(user); + } }} /> + ) : ( + <>
{step === "credentials" ? ( <> @@ -373,45 +288,14 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs ) : null} - {step === "enrol" ? ( - <> - {/* Manual setup is deliberate: the pinned dependency set has no - QR encoder, and T36 adds no dependency or external QR service. */} -

Type this setup code into any authenticator app.

- {setup ? ( - <> - {setup.issuer} setup code - {setup.secret.match(/.{1,4}/g)?.join(" ")} - Authenticator setup link - {setup.otpauthUri} - - - - ) :

{busy ? "Preparing your setup code…" : "Try setup again to get a code."}

} - - ) : null} - - {step === "codes" && issued ? ( - { - const user = issued.user; - setIssued(null); - onSignedIn(user); - }} /> - ) : null} - {/* Disabled through THREE channels (the real attribute, the cursor and the opacity), never opacity alone — a control that only looks dead is a control that still takes the click. */} - {step !== "codes" ? ( - - ) : null} - {step !== "credentials" && step !== "codes" ? ( + + {step !== "credentials" ? ( @@ -424,6 +308,8 @@ export default function LoginPage({ onSignedIn }: { onSignedIn: (user: SessionUs

{error}

+ + )}
); diff --git a/web/src/shell/Shell.tsx b/web/src/shell/Shell.tsx index e1f084d956753fbad7a1e723f017d0bc237cb1fe..8072cfdc869373782c2f5760a2c9a4ada1c56135 100644 --- a/web/src/shell/Shell.tsx +++ b/web/src/shell/Shell.tsx @@ -315,6 +315,11 @@ function ApplicationWizard({ const [schema, setSchema] = useState>({ phase: "idle" }); const [fieldMap, setFieldMap] = useState>({}); const [name, setName] = useState(""); + const [shareEntries, setShareEntries] = useState([]); + const [sharePeople, setSharePeople] = useState([]); + const [shareQuery, setShareQuery] = useState(""); + const [sharePickRole, setSharePickRole] = useState("view"); + const [shareLoadPhase, setShareLoadPhase] = useState<"idle" | "loading" | "ready">("idle"); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); const schemaGeneration = useRef(0); @@ -370,8 +375,43 @@ function ApplicationWizard({ return () => { schemaGeneration.current += 1; }; }, [sourceDatabase]); + useEffect(() => { + const controller = new AbortController(); + setShareEntries([]); + setSharePeople([]); + setShareQuery(""); + setSharePickRole("view"); + if (!sourceDatabase) { + setShareLoadPhase("idle"); + return () => controller.abort(); + } + setShareLoadPhase("loading"); + void fetch(`${API_V1}/share/database/${encodeURIComponent(sourceDatabase)}`, { + credentials: CREDENTIALS, + signal: controller.signal, + }).then(async (response) => { + const body = await response.json().catch(() => ({})); + if (!controller.signal.aborted) { + setSharePeople(response.ok ? parseShare(body).people : []); + setShareLoadPhase("ready"); + } + }).catch(() => { + if (!controller.signal.aborted) { + setSharePeople([]); + setShareLoadPhase("ready"); + } + }); + return () => controller.abort(); + }, [sourceDatabase]); + useApplicationWizardOverlay(panelRef, onClose); + const shareOptions = addablePeople(sharePeople, shareEntries); + const shareRecipient = resolveShareRecipient(sharePeople, shareEntries, shareQuery); + const canAddEveryone = !shareEntries.some((entry) => entry.user === EVERYONE); + const shareNameOf = (user: string) => user === EVERYONE + ? EVERYONE_LABEL + : sharePeople.find((person) => person.user === user)?.name ?? user; const template = templates.phase === "ready" ? templates.value.find((candidate) => candidate.kind === templateKind) ?? null : null; @@ -393,7 +433,9 @@ function ApplicationWizard({ ? !!sourceDatabase && schema.phase === "ready" : step === 2 ? !missingRequired && !coordinatePairError && !routeLocationError - : !!name.trim(); + : step === 3 + ? true + : !!name.trim(); const chooseTemplate = (candidate: ApplicationTemplate) => { setTemplateKind(candidate.kind); @@ -429,6 +471,37 @@ function ApplicationWizard({ body?.detail?.message ?? body?.detail ?? "The Application was not saved.", )); } + if (shareEntries.length > 0) { + try { + // ⛔⛔ FIXED AT THE WAVE-59 COMBINE: this PATCHed `/nav/applications/{id}`, and NO + // SUCH ROUTE EXISTS -- `routes_nav.py` mounts only `POST /nav/applications`. Every + // create therefore fell into the catch below and reported "sharing did not save", + // so W59-T26's done-when ("creating the app persists those grants") could not hold. + // The reviewed door is `PATCH /applications/{id}` in `routes_assets.py`, which + // passes the body to `application_registry.mutate` -- and that is the writer that + // already understands `{share: {mode, users}}`, the exact shape sent below. + // ⚠ Only api_api's D-48 leg could see this: it is the one check that matches every + // decidable client fetch path against a route the app actually MOUNTED, it belongs + // to lane F, and lane E owned neither it nor `routes_nav.py`. + const shareResponse = await fetch(`${API_V1}/applications/${encodeURIComponent(body.application.id)}`, { + method: "PATCH", + credentials: CREDENTIALS, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + share: shareEntries.some((entry) => entry.user === EVERYONE) + ? { mode: "tenant", users: [] } + : { mode: "users", users: shareEntries.map((entry) => entry.user) }, + }), + }); + await shareResponse.json().catch(() => ({})); + if (!shareResponse.ok) throw new Error("Sharing did not save."); + } catch { + const sharingError = "The app was created, but sharing did not save. You can share it from the App module."; + setError(sharingError); + // Creation closes the wizard, so the shell must also carry the partial result. + window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: sharingError })); + } + } onCreated(body.application as CreatedTemplateApplication); } catch (reason: unknown) { setError(reason instanceof Error ? reason.message : "The Application was not saved."); @@ -437,7 +510,7 @@ function ApplicationWizard({ } }; - const headings = ["Choose a template", "Choose a database", "Map the fields", "Review and create"]; + const headings = ["Choose a template", "Choose a database", "Map the fields", "Choose who can open it", "Review and create"]; return (
    - {["Template", "Database", "Fields", "Review"].map((label, index) => ( + {["Template", "Database", "Fields", "Share", "Review"].map((label, index) => (
  1. {index + 1}{label} @@ -551,7 +624,83 @@ function ApplicationWizard({
) : null} - {step === 3 && template ? ( + {step === 3 ? ( + shareLoadPhase === "loading" ? ( +

Loading people...

+ ) : sharePeople.length === 0 ? ( +

Only you can open this app for now. You can share it from the App module later.

+ ) : ( + <> +
+ {shareEntries.length === 0 ? ( +
Not shared with anyone yet.
+ ) : null} + {shareEntries.map((entry) => ( +
+ {shareNameOf(entry.user)} + + +
+ ))} +
+
+ setShareQuery(event.target.value)} + /> + + {canAddEveryone ? + + +
+

Everyone you add can open this app. Only you can change it.

+ + ) + ) : null} + + {step === 4 && template ? (