diff --git a/RELEASES.json b/RELEASES.json index f3a1294dfb0151f0b2ec4b29a63b97ed638d47b1..640526b9815f259ec21a8e0f613ca2bc063eb510 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,431 +1,4 @@ { - "current": "25a941a", - "releases": [ - { - "version": "v71", - "sha": "96143dc", - "date": "2026-09-04", - "subject": "release v71 - wave 51, QA'd 24 PASS / 6 FAIL / 5 UNPROVEN" - }, - { - "version": "v70", - "sha": "eedfb6a", - "date": "2026-09-03", - "subject": "release v70 - wave 50 (grid analytics, the Sheet as a spreadsheet, view trash and audit)" - }, - { - "version": "v69", - "sha": "8598f56", - "date": "2026-09-02", - "subject": "release v69: wave48-continuation validated + hotfixes v67/v68/d4e3e31 + go-live QA repairs (Supplier option limit, optio" - }, - { - "version": "v68", - "sha": "62b065a", - "date": "2026-09-01", - "subject": "release v68: unified holiday date offsets" - }, - { - "version": "v67", - "sha": "3b22419", - "date": "2026-09-01", - "subject": "release v67" - }, - { - "version": "v66", - "sha": "73c94f0", - "date": "2026-08-31", - "subject": "release v66" - }, - { - "version": "v65", - "sha": "9f6cfed", - "date": "2026-08-31", - "subject": "release v65" - }, - { - "version": "v64", - "sha": "bb8985c", - "date": "2026-08-31", - "subject": "release v64" - }, - { - "version": "v63", - "sha": "ec75cb8", - "date": "2026-08-31", - "subject": "release v63" - }, - { - "version": "v62", - "sha": "2c8f702", - "date": "2026-08-31", - "subject": "release v62" - }, - { - "version": "v61", - "sha": "bbc4c07", - "date": "2026-08-30", - "subject": "v61 - wave 46 hotfix: D-590 schema overlay reaches the grid read (all three assemblies), D-599 route to zone association" - }, - { - "version": "v60", - "sha": "dd1b062", - "date": "2026-08-30", - "subject": "v60 - wave 46" - }, - { - "version": "v59", - "sha": "3cb73ce", - "date": "2026-08-29", - "subject": "release v59" - }, - { - "version": "v58", - "sha": "c526ced", - "date": "2026-08-29", - "subject": "release v58" - }, - { - "version": "v57", - "sha": "4859c03", - "date": "2026-08-29", - "subject": "release v57" - }, - { - "version": "v56", - "sha": "2970b72", - "date": "2026-08-28", - "subject": "release v56" - }, - { - "version": "v55", - "sha": "b766b90", - "date": "2026-08-28", - "subject": "release v55" - }, - { - "version": "v54", - "sha": "6a8e99a", - "date": "2026-08-27", - "subject": "release v54" - }, - { - "version": "v53", - "sha": "89ed2cb", - "date": "2026-08-24", - "subject": "release v53" - }, - { - "version": "v52", - "sha": "8263493", - "date": "2026-08-23", - "subject": "release v52" - }, - { - "version": "v51", - "sha": "5740d20", - "date": "2026-08-23", - "subject": "release v51" - }, - { - "version": "v50", - "sha": "8027c83", - "date": "2026-08-23", - "subject": "release v50" - }, - { - "version": "v49", - "sha": "9a14c01", - "date": "2026-08-23", - "subject": "release v49" - }, - { - "version": "v48", - "sha": "3c69fd9", - "date": "2026-08-23", - "subject": "release v48" - }, - { - "version": "v47", - "sha": "917f9a0", - "date": "2026-08-23", - "subject": "release v47" - }, - { - "version": "v46", - "sha": "a8dc531", - "date": "2026-08-23", - "subject": "release v46" - }, - { - "version": "v45", - "sha": "0b2b501", - "date": "2026-08-23", - "subject": "release v45" - }, - { - "version": "v44", - "sha": "a7877aa", - "date": "2026-08-23", - "subject": "Hotfix: allow renaming the active Map view" - }, - { - "version": "v43", - "sha": "e2ba461", - "date": "2026-08-23", - "subject": "release v43" - }, - { - "version": "v42", - "sha": "25b885e", - "date": "2026-08-23", - "subject": "release v42" - }, - { - "version": "v41", - "sha": "1d6adcc", - "date": "2026-08-22", - "subject": "v41 vertical route navigation" - }, - { - "version": "v40", - "sha": "1451837", - "date": "2026-08-22", - "subject": "release v40 route UI correction" - }, - { - "version": "v39", - "sha": "a78bdf0", - "date": "2026-08-21", - "subject": "W39 clean stale migration targets" - }, - { - "version": "v38", - "sha": "a556631", - "date": "2026-08-21", - "subject": "W39 reversible Live store cutover" - }, - { - "version": "v37", - "sha": "1ddd8cf", - "date": "2026-08-21", - "subject": "W39: coalesce derived workspace reads" - }, - { - "version": "v36", - "sha": "51db3b7", - "date": "2026-08-21", - "subject": "W39 share-field oid repair" - }, - { - "version": "v35", - "sha": "0f64212", - "date": "2026-08-21", - "subject": "W39 visual QA fixture" - }, - { - "version": "v34", - "sha": "cb4caa1", - "date": "2026-08-21", - "subject": "W39 Staging PostgreSQL egress controls" - }, - { - "version": "v33", - "sha": "2b115e5", - "date": "2026-08-21", - "subject": "W39: stop nonproduction Postgres background egress" - }, - { - "version": "v32", - "sha": "b47b27c", - "date": "2026-08-21", - "subject": "W39: repair Staging Postgres projection" - }, - { - "version": "v31", - "sha": "8880a49", - "date": "2026-08-21", - "subject": "v31: Wave 39 staging candidate" - }, - { - "version": "v30", - "sha": "71776ca", - "date": "2026-08-20", - "subject": "v30: W39-T29 Member-owned View first-share hotfix" - }, - { - "version": "v29", - "sha": "55ced50", - "date": "2026-08-18", - "subject": "v29: the build staging has been running since 2026-08-18 (55ced50)." - }, - { - "version": "v28", - "sha": "2793e4b", - "date": "2026-08-18", - "subject": "release v28" - }, - { - "version": "v27", - "sha": "c05a794", - "date": "2026-08-18", - "subject": "release v27" - }, - { - "version": "v25", - "sha": "bc24829", - "date": "2026-08-14", - "subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)" - }, - { - "version": "v26", - "sha": "6a642e1", - "date": "2026-08-14", - "subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)" - }, - { - "version": "v24", - "sha": "514d829", - "date": "2026-08-09", - "subject": "release v24" - }, - { - "version": "v23", - "sha": "44cc283", - "date": "2026-08-09", - "subject": "release v23" - }, - { - "version": "v22", - "sha": "f14e199", - "date": "2026-08-09", - "subject": "release v22" - }, - { - "version": "v21", - "sha": "5fdffce", - "date": "2026-08-09", - "subject": "release v21" - }, - { - "version": "v20", - "sha": "442e4a1", - "date": "2026-08-09", - "subject": "release v20" - }, - { - "version": "v19", - "sha": "94cb4a4", - "date": "2026-08-09", - "subject": "release v19" - }, - { - "version": "v18", - "sha": "0106ef9", - "date": "2026-08-09", - "subject": "release v18" - }, - { - "version": "v17", - "sha": "308240c", - "date": "2026-08-09", - "subject": "release v17" - }, - { - "version": "v16", - "sha": "a804df5", - "date": "2026-08-09", - "subject": "release v16" - }, - { - "version": "v15", - "sha": "6b84d72", - "date": "2026-08-09", - "subject": "release v15" - }, - { - "version": "v14", - "sha": "bee5eaf", - "date": "2026-08-07", - "subject": "release v14" - }, - { - "version": "v13", - "sha": "314c0e3", - "date": "2026-08-06", - "subject": "release v13" - }, - { - "version": "v12", - "sha": "4908f39", - "date": "2026-08-06", - "subject": "release v12" - }, - { - "version": "v11", - "sha": "3126617", - "date": "2026-08-05", - "subject": "release v11" - }, - { - "version": "v10", - "sha": "43f8042", - "date": "2026-08-05", - "subject": "release v10" - }, - { - "version": "v9", - "sha": "2e26f57", - "date": "2026-08-05", - "subject": "release v9" - }, - { - "version": "v8", - "sha": "8073b3e", - "date": "2026-08-05", - "subject": "release v8" - }, - { - "version": "v7", - "sha": "b1c5a7e", - "date": "2026-08-05", - "subject": "release v7" - }, - { - "version": "v6", - "sha": "a344020", - "date": "2026-08-05", - "subject": "release v6" - }, - { - "version": "v5", - "sha": "c0b0fbb", - "date": "2026-08-05", - "subject": "release v5" - }, - { - "version": "v4", - "sha": "de5037f", - "date": "2026-08-05", - "subject": "release v4" - }, - { - "version": "v3", - "sha": "6b9fa62", - "date": "2026-08-05", - "subject": "release v3" - }, - { - "version": "v2", - "sha": "fd05861", - "date": "2026-08-04", - "subject": "release v2" - }, - { - "version": "v1", - "sha": "5b4e2c4", - "date": "2026-08-04", - "subject": "release v1" - } - ] + "current": "d605a17", + "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index b89dc5168ae062104d0e9eabe0bc0c3642cf2518..e330110a52826d7bdf8ffdbec8967f660528cd6c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -25a941a +d605a17 diff --git a/api/routes_admin.py b/api/routes_admin.py index 7667db2b6a61c36e1c771203831e06d0ba8d67a4..e665c9b3f09c29f3c03460a86c241ef65040c000 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -75,10 +75,7 @@ def system_data_inspector(session: Session = Depends(require_session)): _UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$") #: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`. _UNAME_MAX = 254 -#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog -#: enforces nothing; this does, and the two are allowed to differ because only one of them is -#: reachable from the internet. -_MIN_PW = 8 +#: Password policy lives in core.users so this route and self-service rotation cannot disagree. _ROLES = ("user", "admin") @@ -131,6 +128,12 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None): # The session-revocation handle. An admin needs it: it is the only visible evidence # that "sign them out everywhere" actually happened. "epoch": int(rec.get("epoch") or 0), + # Administrators need the rotation state, never the credential history behind it. + "passwordChangedAt": users.password_changed_at(rec), + "passwordExpired": users.password_expired(rec), + # The reset controls need enrolment state, never the secret, hashes or code count. + "mfaEnrolled": users.mfa_enrolled(rec), + "mfaEnrolledAt": rec.get("mfa_enrolled_at") or None, # S3 ask 4 — one sentence for the roster's Access column, so the list does not need # a /perms round trip per row to fill one cell. Computed from the same record the # editor will open, so the two cannot disagree. @@ -213,10 +216,14 @@ def _clean_username(v): return uname -def _clean_password(v): +def _clean_password(v, username=None, name=None, email=None, record=None): pw = str(v or "") - if len(pw) < _MIN_PW: - raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") + failures = users.password_failures(pw, username=username, name=name, email=email, + record=record) + if failures: + # Name every broken rule so the caller can correct the value without guessing the policy. + raise err(400, "weak_password", + "That password does not meet the policy: " + str(users.PasswordPolicyError(failures))) return pw @@ -326,13 +333,13 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi body = body or {} _require_store() uname = _clean_username(body.get("username")) - pw = _clean_password(body.get("password")) + name = str(body.get("name") or "").strip() or uname + email = str(body.get("email") or "").strip() or None + pw = _clean_password(body.get("password"), username=uname, name=name, email=email) role = _clean_role(body.get("role") or "user") bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") - name = str(body.get("name") or "").strip() or uname agent = str(body.get("agent") or "").strip() or None - email = str(body.get("email") or "").strip() or None _reg_before = _registry() if uname in _reg_before: @@ -365,8 +372,11 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi f"address for this account.") # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — # the tenant is stamped from the session, not taken from the body. - users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, - agent=agent, email=email, tenant=_tenant_of(session.user)) + try: + users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, + agent=agent, email=email, tenant=_tenant_of(session.user)) + except users.PasswordPolicyError as exc: + raise err(400, "weak_password", "That password does not meet the policy: " + str(exc)) reg = _registry() if uname not in reg: # The store accepted the write and does not have it. Reporting 201 here would be the @@ -456,9 +466,15 @@ def set_password(username: str, body: dict = Body(default=None), # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — # 404, never 403, because "that account exists in another company" is itself a leak. raise err(404, "no_such_user", f"no account named {uname!r}") - pw = _clean_password((body or {}).get("password")) + pw = _clean_password((body or {}).get("password"), username=uname, + name=reg[uname].get("name"), email=reg[uname].get("email"), record=reg[uname]) before = int(reg[uname].get("epoch") or 0) - users.set_password(uname, pw) + try: + # This route just checked strength AND reuse against its record. Spend the history's + # PBKDF2 passes once; the core still merges the rotation into the fresh update document. + users.set_password(uname, pw, enforce=False) + except users.PasswordPolicyError as exc: + raise err(400, "weak_password", "That password does not meet the policy: " + str(exc)) after = int((_registry().get(uname) or {}).get("epoch") or 0) if after <= before: # The whole value of this route is the revocation. If the epoch did not move, the sessions @@ -468,6 +484,49 @@ def set_password(username: str, body: dict = Body(default=None), return Response(status_code=204) +@router.post("/admin/users/{username}/mfa/reset", status_code=204) +def reset_mfa(username: str, session: Session = Depends(admin_gate)): + """Clear a lost factor and revoke every session; the person enrols their own phone again. + + The administrator never sees or sets a user's secret. Removing it opens the enrolment door + at next sign-in; the atomic epoch bump closes every session that predates this reset. + """ + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # A cross-tenant username answers exactly like a missing one: naming another company's + # account, even only to refuse the reset, would itself disclose that the account exists. + raise err(404, "no_such_user", f"no account named {uname!r}") + before = int(reg[uname].get("epoch") or 0) + users.clear_mfa(uname) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # Revocation is the whole value of this door. A 204 without a moved epoch would claim + # the lost phone's sessions died when the store did not persist that change. + raise err(503, "store_unavailable", "the reset did not persist - outstanding sessions were NOT " + "revoked") + return Response(status_code=204) + + +@router.post("/admin/users/{username}/password/expire", status_code=204) +def expire_password(username: str, session: Session = Depends(admin_gate)): + """Force the account through password rotation without choosing its replacement secret.""" + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Same missing-account answer across tenants: the reset door cannot enumerate others. + raise err(404, "no_such_user", f"no account named {uname!r}") + before = int(reg[uname].get("epoch") or 0) + users.set_must_change_password(uname, True) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # A flag without revocation would leave the old sessions usable. Prove the write before + # telling an administrator the reset is in effect, just like the password setter above. + raise err(503, "store_unavailable", + "the forced password rotation did not persist. Outstanding sessions were NOT revoked") + return Response(status_code=204) + + # ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── #: The modules the permission editor governs — the GRID surfaces, the ones with a field schema #: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list diff --git a/api/routes_auth.py b/api/routes_auth.py index 5e919c397a0e3e52a8c10c2f50d2ba5e4d2e728a..5892c136a3c8fce0aa13cc8a70a7e00235f33b62 100644 --- a/api/routes_auth.py +++ b/api/routes_auth.py @@ -143,6 +143,7 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) body = body or {} uname = str(body.get("username") or "").strip().lower() pw = str(body.get("password") or "") + code = str(body.get("mfaCode") or "").strip() # Throttle on the IDENTIFIER alone: the tenant is not known until the credential resolves, # and a per-(tenant, uname) key would let an attacker reset their budget by rotating slugs. @@ -179,7 +180,48 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) _note_failure(throttle_key) raise err(401, "invalid_credentials", "that username and password do not match") - _clear_failures(throttle_key) + # W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or + # minting so an expired password gets the change door and no session, without counting a + # successful authentication against the guessing budget. Neither synthetic identity has a + # stored password clock, and the emergency master must keep its store-outage escape path. + if not (user.get("qa_runner") or user.get("emergency_master")): + try: + rec = (users.registry() or {}).get(user["username"]) + except Exception: + rec = None + if users.password_expired(rec): + raise err(403, "password_expired", "Your password has expired. Set a new one to continue.") + if rec and not users.password_changed_at(rec): + # Legacy records start their clock at first sign-in after this ships. The background + # merge is fail-silent like touch_login, so backfilling can never deny a good login. + try: + users.stamp_password_age(user["username"]) + except Exception: + pass + + # W57-T36 / C12: every STORED account needs its second factor before any cookie is minted. + # The same TWO identities as the epoch bump are exempt: qa_runner has no registry record + # at all; emergency_master must keep the break-glass path that exists for a store outage. + # Neither has a stored factor this route can require without defeating its existing purpose. + if not (user.get("qa_runner") or user.get("emergency_master")): + try: + rec = (users.registry() or {}).get(user["username"]) + except Exception: + rec = None + if not users.mfa_enrolled(rec): + # Correct credentials needing setup are not a guess. Counting them would lock the + # person out of the unauthenticated enrolment door that repairs their account. + raise err(403, "mfa_enrolment_required", "This account needs two step sign in. Set up your " + "authenticator app to continue.") + if not code: + # A missing factor is a prompt, not a failed attempt at guessing one. + raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.") + # TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's + # ordinary code already proves the factor, and never reset the guess budget on password. + if not (users.totp_valid(rec.get("mfa_secret"), code) + or users.consume_recovery_code(user["username"], code)): + _note_failure(throttle_key) + raise err(401, "mfa_invalid", _MFA_INVALID) # ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this # account already holds, so a forgotten sign-in on someone else's computer cannot lock the @@ -231,9 +273,226 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) import deps as _deps if not user.get("qa_runner"): _deps.note_active(user["username"]) + # MOVED from password success: only a WHOLE successful login clears the budget. Otherwise + # a known password would reset the counter before every wrong second-factor guess. + _clear_failures(throttle_key) return {"user": _public_user(user)} +# ------------------------------------------------------------------ second factor (W57-T36 / C12) +_MFA_INVALID = ("That code is not right. Check your authenticator app and try the next one.") +_MFA_ALREADY_ENROLLED = ("Two step sign in is already on for this account. Ask an administrator " + "to reset it if you lost your phone.") + + +def _mfa_credentials(uname, password): + """The MFA doors share login's credential check and guessing budget, including aliases.""" + throttle_key = ("*", uname) + wait = _throttled(throttle_key) + if wait: + raise err(429, "too_many_attempts", + f"too many failed attempts. Try again in {wait} seconds") + # Resolve the staging identity without touching tenant #0's store, then REFUSE it: no + # record exists to enrol. The emergency master is refused for the same no-stored-factor rule. + user = users.qa_identity(uname, password) if (uname and password) else None + if not user: + user = users.verify(uname, password) if (uname and password) else None + if not user: + try: + users._hash(password or "x", "00" * 16) + except Exception: + pass + _note_failure(throttle_key) + raise err(401, "invalid_credentials", "that username and password do not match") + if user.get("qa_runner") or user.get("emergency_master"): + raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.") + return user + + +def _mfa_record(uname): + """A stored identity was already proved. Failure to re-read it cannot assert absence.""" + try: + rec = (users.registry() or {}).get(uname) + except Exception: + rec = None + if not rec: + raise err(503, "store_unavailable", "the account could not be read. Try again") + return rec + + +def _mfa_session_tenant(user, rec, throttle_key): + """Both MFA cookie writers retain login's tenant and T35 password-expiry walls.""" + tenant = str(user.get("tenant") or "royal-imports").strip().lower() + from harness import runtime + try: + runtime.get_runtime(tenant) + except KeyError: + _note_failure(throttle_key) + raise err(401, "invalid_credentials", "that username and password do not match") + # Confirming a phone must not become a second login door that accepts an expired password. + # Check before writing the factor, so the person can rotate and resume the same setup. + if users.password_expired(rec): + raise err(403, "password_expired", "Your password has expired. Set a new one to continue.") + return tenant + + +def _mfa_set(user, secret, codes, *, require_unenrolled=False): + """Translate a raced credential change into a retry, never a claimed successful setup.""" + try: + # Use the RESOLVED username: the posted identifier can be an email alias with no record. + return users.set_mfa(user["username"], secret, codes, + require_unenrolled=require_unenrolled, + expected_epoch=int(user.get("epoch") or 0)) + except users.MFAStateError as exc: + if str(exc) == "already_enrolled": + raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED) + raise err(409, "mfa_changed", "This account changed during setup. Sign in again to continue.") + + +def _mfa_session(request, response, user, rec, tenant, landed, throttle_key): + """Mint only from the MFA write's landed epoch, never the stale login projection.""" + if landed is None: + # A missing record/no-op write cannot support a cookie or a promise that codes were saved. + raise err(503, "store_unavailable", "the second factor was not saved. Try again") + value, _claims = aios_session.mint(tenant, user["username"], int(landed)) + aios_session.set_cookie(response, request, value) + users.touch_login(user["username"]) + import deps as _deps + _deps.note_active(user["username"]) + if not users.password_changed_at(rec): + # This is a successful first sign-in too: preserve T35's legacy clock backfill. + users.stamp_password_age(user["username"]) + _clear_failures(throttle_key) + # Responses carrying a new secret or recovery codes must not survive in an HTTP cache. + response.headers["Cache-Control"] = "no-store" + + +@router.post("/mfa/enroll") +def mfa_enroll(response: Response, body: dict = Body(default=None)): + """Offer a candidate secret without changing the account or issuing recovery codes. + + Stateless by design: the client posts this candidate back with its password and phone code. + A pending-secret dict would belong to one worker and fail when confirmation reaches another. + """ + body = body or {} + uname = str(body.get("username") or "").strip().lower() + user = _mfa_credentials(uname, str(body.get("password") or "")) + rec = _mfa_record(user["username"]) + if users.mfa_enrolled(rec): + raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED) + secret = users.new_mfa_secret() + response.headers["Cache-Control"] = "no-store" + return {"secret": secret, "otpauthUri": users.otpauth_uri(secret, user["username"]), + "issuer": users.MFA_ISSUER, "digits": users.MFA_DIGITS, "period": users.MFA_PERIOD} + + +@router.post("/mfa/enroll/confirm") +def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(default=None)): + """Prove installation, enrol once and sign in under the new epoch. + + This response is the only time those codes exist in clear anywhere. The record holds SHA-256 + hashes. No other route returns them, no log line prints them, and there is no way to read them + back. Explicit regeneration issues a DIFFERENT set, never a second reading of this one. + """ + body = body or {} + uname = str(body.get("username") or "").strip().lower() + throttle_key = ("*", uname) + user = _mfa_credentials(uname, str(body.get("password") or "")) + rec = _mfa_record(user["username"]) + if users.mfa_enrolled(rec): + raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED) + secret = str(body.get("secret") or "").strip().upper() + if not users.totp_valid(secret, str(body.get("code") or "")): + _note_failure(throttle_key) + raise err(401, "mfa_invalid", _MFA_INVALID) + tenant = _mfa_session_tenant(user, rec, throttle_key) + codes = users.new_recovery_codes() + # The store guard repeats the check atomically: two workers cannot issue two first sets. + landed = _mfa_set(user, secret, codes, require_unenrolled=True) + _mfa_session(request, response, user, rec, tenant, landed, throttle_key) + return {"user": _public_user(user), "recoveryCodes": codes} + + +@router.get("/mfa") +def mfa_status(session: Session = Depends(require_session)): + """The calling account's state only: never a secret, code or hash.""" + # These two identities have no stored factor; do not read tenant #0 for the QA runner. + if session.user.get("qa_runner") or session.user.get("emergency_master"): + rec = {} + else: + rec = _mfa_record(session.uname) + return {"enrolled": users.mfa_enrolled(rec), "enrolledAt": rec.get("mfa_enrolled_at") or None, + "recoveryCodesRemaining": users.recovery_codes_remaining(rec)} + + +@router.post("/mfa/recovery-codes") +def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Issue a new set once after password re-authentication, keeping the installed phone. + + Regeneration is a credential change: set_mfa bumps the epoch and signs OTHER sessions out. + Re-mint the CALLING session from that landed epoch or the user's own action signs them out. + The clear codes appear only in this response; the store and all later reads hold only hashes. + """ + uname = session.uname + throttle_key = ("*", uname) + user = _mfa_credentials(uname, str((body or {}).get("password") or "")) + # The session supplies the identity. Body username/tenant fields can never choose a target. + rec = _mfa_record(uname) + if not users.mfa_enrolled(rec): + raise err(409, "mfa_not_enrolled", "Set up your authenticator app before making recovery codes.") + tenant = _mfa_session_tenant(user, rec, throttle_key) + codes = users.new_recovery_codes() + landed = _mfa_set(user, rec["mfa_secret"], codes) + _mfa_session(request, response, user, rec, tenant, landed, throttle_key) + return {"recoveryCodes": codes} + + +@router.get("/password-policy") +def password_policy(): + """An account-independent checklist the sign-in page can show before authentication.""" + return {"rules": users.password_rules(), "minLength": users.PASSWORD_MIN_LENGTH, + "maxAgeDays": users.PASSWORD_MAX_AGE_DAYS, "historyCount": users.PASSWORD_HISTORY} + + +@router.post("/password", status_code=204) +def change_password(body: dict = Body(default=None)): + """Rotate with the current credential, including when expiry refuses a login session.""" + body = body or {} + uname = str(body.get("username") or "").strip().lower() + pw = str(body.get("currentPassword") or "") + new_pw = str(body.get("newPassword") or "") + # Share login's identifier bucket: a second unauthenticated door must not buy a second + # password-guessing budget, and failures must not disclose whether an account exists. + throttle_key = ("*", uname) + wait = _throttled(throttle_key) + if wait: + raise err(429, "too_many_attempts", + f"too many failed attempts. Try again in {wait} seconds") + # Resolve the staging runner without registry I/O, exactly as login does. Its credential + # authenticates, but no stored password exists here, so reporting a rotation would be false. + user = users.qa_identity(uname, pw) if (uname and pw) else None + if not user: + user = users.verify(uname, pw) if (uname and pw) else None + if not user: + try: + users._hash(pw or "x", "00" * 16) + except Exception: + pass + _note_failure(throttle_key) + raise err(401, "invalid_credentials", "that username and password do not match") + if user.get("emergency_master") or user.get("qa_runner"): + raise err(400, "no_stored_password", "This account has no stored password to change here.") + try: + # verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed + # alias, or this door could return 204 without changing any password or revoking sessions. + users.set_password(user["username"], new_pw) + except users.PasswordPolicyError as exc: + raise err(400, "weak_password", "That password does not meet the policy: " + str(exc)) + _clear_failures(throttle_key) + return Response(status_code=204) + + @router.post("/logout", status_code=204) def logout(request: Request): """204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 3d2d3c9fc762949230d7bf92e20745e7ae1f2e1e..82ae5a844efa28172573a8166968008c3cc85fe4 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -21,6 +21,7 @@ that reported another user's Gmail as "connected" would be both wrong and a smal from __future__ import annotations import os +import time from fastapi import APIRouter, Body, Depends @@ -1376,6 +1377,38 @@ def _evict_odoo_product_pools(rt): rt.pool_cache.pop(cache_key, None) +# W57-T31 / R10 / C9: switching off starts a seven-day lock to prevent repeated toggling. +# The config document is shared with read_config, which ignores unknown keys: disabledAt +# belongs beside grids, not in a second document that could drift from enablement. +_RELOCK_DAYS = 7 +_RELOCK_SECONDS = _RELOCK_DAYS * 86400 + + +def _relock_stamps(rt): + """Read disable epochs without letting an absent or malformed map break a request.""" + try: + import odoo_relational + stamps = (rt.get(odoo_relational.CONFIG_KEY) or {}).get("disabledAt", {}) + if not isinstance(stamps, dict): + return {} + return {str(key): int(value) for key, value in stamps.items() + if isinstance(value, (int, float)) and not isinstance(value, bool)} + except Exception: # noqa: BLE001 + return {} + + +def _relock_days_left(stamp, now=None): + """Round any remaining part of a lock day up to a whole day.""" + if isinstance(stamp, bool) or not isinstance(stamp, (int, float)): + return 0 + try: + from math import ceil + remaining = stamp + _RELOCK_SECONDS - (time.time() if now is None else now) + return max(0, ceil(remaining / 86400)) + except Exception: # noqa: BLE001 + return 0 + + def _enable_odoo_database(table_key, session): """Turn on one declared Odoo database and materialise it before reporting success.""" if not session.admin: @@ -1389,6 +1422,20 @@ def _enable_odoo_database(table_key, session): raise err(404, "unknown_odoo_database", f"Odoo does not declare a database with key {key!r}") + # R10 / C9: only a transition back on is locked; an already-enabled database may retry + # materialisation. A successful enable clears its stamp so the next switch-off starts fresh. + # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. + if choices[key].get("enabled") is not True: + days = _relock_days_left(_relock_stamps(session.runtime).get(key)) + if days > 0: + label = str(choices[key].get("label") or key) + unit = "day" if days == 1 else "days" + raise err(409, "database_relocked", + f"{label} was switched off less than {_RELOCK_DAYS} days ago. " + f"It cannot be switched back on for another {days} {unit}. " + f"The {_RELOCK_DAYS} day lock is what stops a database being switched " + "on and off repeatedly.") + # ⭐ W56-T31: before this door, enabled meant only "will be included in the next sync". # `canRename` then correctly stayed false because there was no table. The flag remains in the # connector's one CONFIG_KEY document, but this door also creates the database before it can @@ -1396,6 +1443,8 @@ def _enable_odoo_database(table_key, session): def _up(cur): cur = cur if isinstance(cur, dict) else {} cur.setdefault("grids", {}).update({key: True}) + if isinstance(cur.get("disabledAt"), dict): + cur["disabledAt"].pop(key, None) return cur failure = None @@ -1510,6 +1559,52 @@ def odoo_database_enable(table_key: str, session: Session = Depends(require_sess return _enable_odoo_database(table_key, session) +def _disable_odoo_database(table_key, session): + """Turn off one declared Odoo database and persist its re-enable lock.""" + if not session.admin: + raise err(403, "not_admin", "only an administrator can disable an Odoo database") + + import odoo_relational + key = str(table_key or "").strip() + choices = {str(choice.get("key") or ""): choice + for choice in odoo_relational.grid_choices(session.runtime)} + if key not in choices: + raise err(404, "unknown_odoo_database", + f"Odoo does not declare a database with key {key!r}") + + # W57-T31 / R10 / C9: one config-only write starts the lock, with no sync or row deletion. + # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. + # Repeating a disable must preserve the original deadline. + was_enabled = choices[key].get("enabled") is True + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + grids = cur.get("grids") + grids = grids if isinstance(grids, dict) else {} + grids[key] = False + cur["grids"] = grids + if was_enabled: + if not isinstance(cur.get("disabledAt"), dict): + cur["disabledAt"] = {} + cur["disabledAt"][key] = int(time.time()) + return cur + + try: + session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", "the database setting was not saved. Try again.") + + stamp = _relock_stamps(session.runtime).get(key) + return {"key": key, "enabled": False, "relockDays": _RELOCK_DAYS, + "disabledAt": stamp, + "canEnableAt": stamp + _RELOCK_SECONDS if stamp is not None else None} + + +@router.post("/connectors/odoo/databases/{table_key}/disable") +def odoo_database_disable(table_key: str, session: Session = Depends(require_session)): + """Disable one declared Odoo database in this tenant's governed store.""" + return _disable_odoo_database(table_key, session) + + @router.patch("/connectors/odoo/schema/{field_key}") def odoo_schema_enable(field_key: str, body: dict = Body(default=None), session: Session = Depends(require_session)): diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 05ecc866f0d597577793c156ae1b1f909f1026c0..b40240a4b7cc63329cd64214f6dc0111f082da75 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -10,6 +10,7 @@ in the payload (`pausedNote`): pausing marks intent and warns; the source cutove the keychain cutover wave R3 staged. """ import os +import time from fastapi import Body, Depends from fastapi import APIRouter @@ -695,6 +696,132 @@ def _with_connector(rows): return out +_REGISTERED = {"done": False} +# A stale Remove must not inherit consent forever; ten minutes leaves time to read the warning +# and retry the existing client while binding that consent to the same user and database set. +REVOKE_CONFIRM_SECONDS = 600 +_REVOCATIONS_KEY = "keychain_revocations" + + +def _odoo_table_owner(): + """Odoo's per-connection table resolver, or None when it cannot be imported. + + ⛔ ITS ABSENCE MUST NOT TAKE THE REGISTRY DOWN WITH IT, and that is why this is a function + rather than one more name in the block below's `import`. `odoo_relational` is the heavy end of + this codebase (it pulls the mirror and the connector spec); a failure to import it inside one + `try` around all three families would leave AMAZON unregistered, and an unregistered Amazon + family is a revocation that purges nothing while reporting a clean `registry: True`. That is + the silent-no-op shape instruction 30 exists to remove. + ⚠ Falling back to None is a REAL degradation and a safe one: without the resolver every + `ut_odoo_*` table resolves to the tenant's designated default connection instead of its own, + which can only ever widen the KEEP list a disclosure prints. Odoo purges nothing (R10), so a + coarse answer here cannot delete a row. + """ + try: + import odoo_relational + return odoo_relational.connection_for_table_key + except Exception: # noqa: BLE001 + return None + + +def _register_credential_tables(): + """Declare revocation policy lazily; a failed import remains retryable and reportable. + + routes_connectors imports this module at boot, and odoo_relational is heavy. An import-time + registration would put a cycle on every route before anyone even opens Keychain. + """ + if _REGISTERED["done"]: + return True + try: + from core import user_tables as ut + import routes_connectors as rc + # Instruction 30 / C13 names the Amazon Data Protection Policy, not every connector. + ut.register_credential_tables( + "amazon_sp", rc.CONNECTED_PREFIXES["amazon_sp"], on_revoke=ut.REVOKE_PURGE, + why="The Amazon Data Protection Policy requires deleting the seller data " + "authorised by a revoked credential.") + # W32 R10's second sentence and D-259 require these grids to survive a reconnect. + ut.register_credential_tables( + "odoo", rc.CONNECTED_PREFIXES["odoo"], on_revoke=ut.REVOKE_KEEP, + owner=_odoo_table_owner(), + why="Wave 32 R10 says reconnecting resumes into the same tables. Use " + "POST /admin/connectors/odoo/disconnect to freeze the data " + "instead of deleting it.") + # No Meta ruling exists this wave. KEEP records the reversible direction explicitly. + ut.register_credential_tables( + "meta_ads", rc.CONNECTED_PREFIXES["meta_ads"], on_revoke=ut.REVOKE_KEEP, + why="No ruling has decided Meta Ads data deletion on revocation, so the reversible " + "direction is taken and its databases are kept.") + except Exception as exc: + _REGISTERED["error"] = type(exc).__name__ + return False + _REGISTERED["done"] = True + _REGISTERED.pop("error", None) + return True + + +def _credential_revocation(row, rt): + """A failed census is UNKNOWN, never evidence of an empty seller footprint.""" + registered = _register_credential_tables() + etype, cid = str(row.get("type") or ""), str(row["id"]) + if registered: + try: + from core import user_tables as ut + return registered, ut.credential_footprint(etype, cid, rt=rt, st=rt) + except Exception as exc: + why = f"The credential footprint could not be read ({type(exc).__name__})." + else: + failure = _REGISTERED.get("error", "unavailable") + why = f"The credential table registry could not be built ({failure})." + return registered, { + "type": etype, "connectionId": cid, "action": "none", "purge": [], "keep": [], + "rows": 0, "rowsKnown": False, + "why": why + " Data cleanup is not known. " + "Restore the store or registry and retry cleanup.", + } + + +def _revocation_residue(entry_type): + """Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear. + + amazon_store.py --help exposes sync/status/validate/selftest, but no deletion flag. + Clearing the app's materialised copy is not evidence that the raw reports disappeared. + """ + if entry_type != "amazon_sp": + return [] + return [{ + "where": "The tenant DuckDB mirror", + "why": "The API process opens the mirror read only, so a route cannot clear it.", + "fix": "An operator must delete the tenant's Amazon mirror tables. " + "platform/harness/amazon_store.py has no flag to do this today.", + }] + + +def _write_revocation_warning(rt, entry_id, warning=None): + """Prune expired consent on every write; failed persistence requires explicit confirmation.""" + now = int(time.time()) + + def _up(cur): + kept = {} + for key, value in (cur or {}).items(): + try: + age = now - int(value.get("at", 0)) + except (AttributeError, TypeError, ValueError): + continue + if 0 <= age < REVOKE_CONFIRM_SECONDS: + kept[key] = value + kept.pop(str(entry_id), None) + if warning is not None: + kept[str(entry_id)] = warning + return kept + + try: + rt.update(_REVOCATIONS_KEY, _up, flush="sync") + return True + except Exception: + return False + + @router.get("/admin/keychain") def list_keychain(session: Session = Depends(require_session)): """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. @@ -867,17 +994,112 @@ def update_key(entry_id: str, body: dict = Body(default=None), "owner": owner if scope == "personal" else ""}} +@router.get("/admin/keychain/{entry_id}/revocation") +def key_revocation(entry_id: str, session: Session = Depends(require_session)): + """Disclose what revocation removes, behind the delete's own visibility and ownership walls. + + Like table_footprint, only someone who could delete may case the joint. An unavailable + census carries rowsKnown=False, because an empty list alone would fabricate reassurance. + """ + row = _own_row(session, entry_id) + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to delete") + registered, footprint = _credential_revocation(row, session.runtime) + residue = (_revocation_residue(row["type"]) + if footprint["purge"] or not footprint["rowsKnown"] else []) + return {"entry": {key: row.get(key, "") for key in ("id", "label", "type")}, + **footprint, "registry": registered, "residue": residue} + + @router.delete("/admin/keychain/{entry_id}") -def delete_key(entry_id: str, session: Session = Depends(require_session)): +def delete_key(entry_id: str, session: Session = Depends(require_session), confirm: str = ""): + kc, rt = _kc(), session.runtime + entries = kc.list_entries(rt) + # Instruction 30 needs idempotent revocation. An absent id's 200 is distinguishable from + # another person's personal entry's 404, but ids have 12 random hex characters and are not + # enumerable. A 404 on a lost response's retry would break this policy path's idempotence. + if not any(entry["id"] == str(entry_id) for entry in entries): + return {"ok": True, "alreadyGone": True, + "purged": {"tables": [], "rows": 0, "rowsKnown": True, + "refused": [], "failed": []}} row = _own_row(session, entry_id) if not _may_touch(session, row): raise err(403, "not_yours", "this connection is not yours to delete") + registered, footprint = _credential_revocation(row, rt) + tables = sorted(table["key"] for table in footprint["purge"]) + purges = footprint["action"] == "purge" and bool(tables) + if purges and confirm != "purge": + now = int(time.time()) + try: + warning = (rt.get(_REVOCATIONS_KEY) or {}).get(str(entry_id)) or {} + repeated = (warning.get("user") == session.uname + and 0 <= now - int(warning.get("at", 0)) < REVOKE_CONFIRM_SECONDS + and warning.get("tables") == tables) + except Exception: + repeated = False + if not repeated: + # No lane owns KeychainTab.tsx this wave. Its dialog still says "Data already read + # stays; nothing new will be read." Purging silently behind that sentence would make + # the product lie on screen. Refusing once with the count makes disclosure true for + # the client that exists; a repeat by this user over this set is the second door. + stored = _write_revocation_warning(rt, entry_id, { + "user": session.uname, "at": now, "tables": tables, + }) + # ⚠ THE KIND COMES FROM THE CREDENTIAL, NOT FROM A LITERAL. Amazon is the only + # family that purges today, so "Amazon databases" would have read correctly and + # become a lie in the one change that flips a second family to REVOKE_PURGE. + kind = kc.kind_label(row.get("type") or "") + noun = "database" if len(tables) == 1 else "databases" + count = (f" holding {footprint['rows']:,} rows. " if footprint["rowsKnown"] + else ". The row count is not known. ") + next_step = ("Choose Remove again within 10 minutes to confirm, or pass confirm=purge." + if stored else "The warning could not be saved. " + "Pass confirm=purge to confirm explicitly.") + raise err(409, "confirm_purge", + f"Removing this connection also deletes {len(tables):,} {kind} {noun}" + + count + "That data cannot be recovered. " + next_step) + + class _Ownership: + # Revoking the default changes keychain's resolver immediately. Lend only metadata from + # BEFORE revocation so core can recompute ownership after the credential is gone. Table + # definitions and deletions still use the live tenant runtime; no caller-supplied plan + # and no encrypted credential is carried into the purge. + def get(self, name): + if name == kc.KEY: + return {"entries": {entry["id"]: entry for entry in entries}} + return rt.get(name) + + # Credential FIRST, data second. A crash leaves revoked access and residual tables that can + # be swept again; the reverse leaves live credentials over deleted data we cannot read back. + # The absent-id retry above is only a no-op receipt, not proof that a crashed sweep completed. try: - _kc().delete_entry(session.runtime, entry_id) - _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it + kc.delete_entry(rt, entry_id) + _write_scope(rt, entry_id, DEFAULT_SCOPE, "") except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} + raise err(503, "store_unavailable", "The delete did not land. Try again.") + if not purges: + _write_revocation_warning(rt, entry_id) + if footprint["rowsKnown"]: + return {"ok": True} + # Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes + # that incomplete cleanup explicit instead of returning an invented "nothing to purge". + return {"ok": True, "registry": registered, "footprint": footprint, + "purged": {"tables": [], "rows": 0, "rowsKnown": False, + "refused": [], "failed": []}, + "residue": _revocation_residue(row["type"])} + from core import user_tables as ut + try: + result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt) + except Exception as exc: + # Access is already revoked. Report every unresolved table without leaking a store path. + result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [], + "failed": [{"key": key, "error": type(exc).__name__} for key in tables]} + _write_revocation_warning(rt, entry_id) + return {"ok": True, + "purged": {"tables": result["purged"], "rows": result["rows"], + "rowsKnown": result["rowsKnown"], "refused": result["refused"], + "failed": result["failed"]}, + "residue": _revocation_residue(row["type"])} @router.post("/admin/keychain/{entry_id}/test") diff --git a/api/routes_sheet.py b/api/routes_sheet.py index c1b7e7b231b2d4ad0dc600b3c3e67f9aefb06342..98e8f80a8a4b01ba9dcf87d97452a27fac89b90f 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -1841,6 +1841,11 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non agg_refusal = sheet_mod.aggregate_refusal(binding["aggregate"], bound_field) if agg_refusal is not None: raise err(409, agg_refusal.code, agg_refusal.message) + if bound_field and bound_field.get("measure") and not binding.get("period"): + label = bound_field.get("label") or bound_field.get("key") + raise err(409, "metric_period_required", + f"{label} is measured over a period, so this cell needs one. " + "Open the cell's Period section and choose a period.") # ── the ROW wall, fail-closed ──────────────────────────────────────────────────────────── if wall_nodes: diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index 1e8957c1ae82bb852a9abcc376000663474ca9e0..9073470106cf5678964c36ad62db315246b1f678 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -118,6 +118,22 @@ "default": true, "description": "A Google Maps link to this customer's address." }, + { + "key": "lat", + "label": "Latitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Latitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, + { + "key": "lon", + "label": "Longitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Longitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, { "key": "customer_since", "label": "Customer since", @@ -166,17 +182,6 @@ "default": true, "description": "How many days late this customer is running against their own usual ordering rhythm." }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, { "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", "key": "ar_open", diff --git a/platform/core/measure_resolve.py b/platform/core/measure_resolve.py index 392db565c3ba3a873e67f311e9d7dac5dae9ae78..53633d1226c3177a498bfec750894b5cd1a6a3b2 100644 --- a/platform/core/measure_resolve.py +++ b/platform/core/measure_resolve.py @@ -19,6 +19,7 @@ Memo discipline (unchanged from the app.py originals): not leaks. """ import hashlib +from core import relations as _relations def offer(team_id=None, on_error=None): @@ -43,7 +44,7 @@ def _pool_key(allowed_pids): def condition_sets(view_configs, pending_cfg, team_id, allowed_pids, today, stamp, memo, - on_error=None): + on_error=None, related=None): """Every measure condition across `view_configs` (+ the optional pending config, which goes LAST and overwrites — the freshest statement of what the user wants) -> `{ruleId: [pid,…]}`. @@ -64,20 +65,24 @@ def condition_sets(view_configs, pending_cfg, team_id, allowed_pids, today, stam if not rules: return {} pk = _pool_key(allowed_pids) - out = {} + out = _relations.MeasureAnswer() if related is not None else {} + rkey = _relations.measure_pin_key(related) for rule in rules: if not _mf.rule_complete(rule): continue - key = (stamp, team_id, pk, _mf.signature(rule)) + key = (stamp, team_id, pk, _mf.signature(rule), rkey) if key not in memo: try: - memo[key] = _mf.resolve_rule(rule, today, team_id, allowed_pids) + memo[key] = _mf.resolve_rule(rule, today, team_id, allowed_pids, related=related) except Exception as e: # An unresolvable condition must NOT become "everything" or "nothing" silently. if on_error: on_error('measure-resolve', e) continue out[str(rule['id'])] = sorted(memo[key]) + if related is not None: + out.refusals.extend(dict(marker, rule=str(rule['id'])) + for marker in getattr(memo[key], 'refusals', ())) if len(memo) > 200: # a lifetime cache, not a leak memo.clear() return out @@ -137,13 +142,18 @@ def series_values(fields, bucket, buckets, team_id, allowed_pids, today, stamp, return out -def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None): +def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None): """Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`, handed to `rows_from_pool(derived=…)` beside the cohort cells. A column that cannot be computed (store down, BU scope on a company-level measure, unresolvable window) degrades to BLANK — the value dict simply omits that field key, and the client renders empty cells, never $0: blank is "could not compute", 0 is a real zero. + + `related` is the row-window {table, record, via} pin. The host binds its scoped readers + with relations.measure_pin_context and publishes the returned mapping's `refusals` beside + its cells. A refused pin retains unscoped values plus a per-field named marker, on memo + hits too. Every measure and its conditions use the same query scope and caller's pool. """ mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)] if not mfields: @@ -151,16 +161,17 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No from harness import measure_filter as _mf from harness import windows as _wn pk = _pool_key(allowed_pids) - out = {} + out = _relations.MeasureAnswer() if related is not None else {} + rkey = _relations.measure_pin_key(related) for f in mfields: spec = f['measure'] w = _wn.normalize(spec.get('window')) key = (stamp, team_id, pk, spec.get('key'), - None if w is None else tuple(sorted(w.items()))) + None if w is None else tuple(sorted(w.items())), rkey) if key not in memo: try: memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'), - today, team_id, allowed_pids) + today, team_id, allowed_pids, related=related) except Exception as e: if on_error: on_error('measure-column', e) @@ -183,6 +194,9 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No if vals is None: continue fkey = f['key'] + if related is not None: + out.refusals.extend(dict(marker, field=fkey) + for marker in getattr(vals, 'refusals', ())) for pid, v in vals.items(): out.setdefault(pid, {})[fkey] = v if len(memo) > 100: diff --git a/platform/core/perm_migrate.py b/platform/core/perm_migrate.py index a6865d876c37d740a418abf693041a57df1cc4f4..ab0f3e4060227664b23474fa8d5e62891f1d44cb 100644 --- a/platform/core/perm_migrate.py +++ b/platform/core/perm_migrate.py @@ -8,7 +8,7 @@ no-op before anything relies on it. ⛔ WHY THE PROOF IS ABOUT VALUES AND NOT ABOUT ROWS (C-PERM amendment 3). `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, LY, LTM) and into `cust._cadence_bulk`. It therefore decides what `rev`, `ly`, `ltm`, `aov`, -`est_missed` and the derived `status` MEAN — it is not a row filter. Swap it for a post-filter +and the derived `status` MEAN; it is not a row filter. Swap it for a post-filter and a Fisch-only user keeps a plausible row LIST while every number on it quietly becomes Fisch+Royal, worst for `dba = Both` customers, who are precisely the ones a BU filter admits. A pid-set reconciliation reports GREEN on that. So `reconcile()` compares VALUES. @@ -97,7 +97,7 @@ def plan(registry_dict, modules=(MODULE,)): #: The row members whose meaning depends on `team_id` — the ones amendment 3 is about. A #: reconciliation that skipped these would be the pid-only gate that cannot see the leak. -VALUE_KEYS = ('rev', 'ly', 'ltm', 'aov', 'est_missed', 'status', 'orders', 'last_order') +VALUE_KEYS = ('rev', 'ly', 'ltm', 'aov', 'status', 'orders', 'last_order') def reconcile(rec, pool_fn, modules=(MODULE,)): diff --git a/platform/core/relations.py b/platform/core/relations.py index d26215ba08af2d8899b9a314f602035e39e6f9d7..301c18461cfe04f98d3f886d6870b5a31b9a925c 100644 --- a/platform/core/relations.py +++ b/platform/core/relations.py @@ -56,6 +56,12 @@ asserted in a comment), and ANDed into the read ahead of the client predicate. booked in this ticket's handoff), ``pin_sql`` delegates to it and this note is deleted. """ +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +import json + + #: The three ways a database can be related to another. `via.direction` is always one of these. DIRECTIONS = ('forward', 'reverse', 'bridge') @@ -191,18 +197,21 @@ OPEN_INVOICE_WHERE = ("state = 'posted' AND move_type IN ('out_invoice','out_ref BRIDGES = ( {'key': 'bridge_customer_orders', 'label': 'Orders', + 'source_label': 'Customers', 'source': 'customer_data', 'target': 'ut_odoo_orders', 'source_col': IDENTITY_COL, 'target_col': 'partner_id', 'where': None, 'served': True, 'cardinality': 'one_to_many'}, {'key': 'bridge_customer_order_lines', 'label': 'Order lines', + 'source_label': 'Customers', 'source': 'customer_data', 'target': 'ut_odoo_order_lines', 'source_col': IDENTITY_COL, 'target_col': 'partner_id', 'where': None, 'served': True, 'cardinality': 'one_to_many'}, {'key': 'bridge_customer_open_invoices', 'label': 'Open invoices', + 'source_label': 'Customers', 'source': 'customer_data', 'target': 'ut_odoo_invoices', 'source_col': IDENTITY_COL, 'target_col': 'partner_id', # ⭐ W53-T19 / R5: `served` flipped to True when `ut_odoo_invoices` gained its read-through @@ -222,12 +231,14 @@ BRIDGES = ( # column to join to. {'key': 'bridge_product_order_lines', 'label': 'Order lines', + 'source_label': 'Products', 'source': 'product_data', 'target': 'ut_odoo_order_lines', 'source_col': 'product_id', 'target_col': 'product_id', 'where': None, 'served': True, 'cardinality': 'one_to_many'}, {'key': 'bridge_order_order_lines', 'label': 'Order lines', + 'source_label': 'Orders', 'source': 'ut_odoo_orders', 'target': 'ut_odoo_order_lines', 'source_col': IDENTITY_COL, 'target_col': 'order_id', 'where': None, 'served': True, 'cardinality': 'one_to_many'}, @@ -675,6 +686,184 @@ def pin_sql(id_sql, member_ids): return member_ids_tail(id_sql, member_ids) +class PinRefusal(ValueError): + """A named refusal that accompanies an explicitly unscoped measure answer.""" + + def __init__(self, code, message, path=()): + super().__init__(message) + self.marker = {'code': code, 'message': message, 'path': list(path), + 'scoped': False} + + +class MeasureAnswer(dict): + """Ordinary value mapping plus `refusals` for the host's response envelope. + + Hosts must serialize `refusals` alongside the values when accepting a related pin. + Keeping metadata out of the pid/field namespace preserves every existing mapping consumer. + """ + + def __init__(self, values=(), refusals=()): + super().__init__(values) + self.refusals = list(refusals) + + +class MeasureMatches(set): + """Condition membership with the same refusal channel as MeasureAnswer.""" + + def __init__(self, values=(), refusals=()): + super().__init__(values) + self.refusals = list(refusals) + + +_MEASURE_PIN_CONTEXT = ContextVar('measure_pin_context', default=None) + + +@contextmanager +def measure_pin_context(*, read, bindings, may_open, fields_for=None, defs=None): + """Bind the host's permission-checked relation reads to this request only. + + `read(table, column, values, where)` returns canonical rows, including pid and join + columns, through the SAME scoped reader used by row windows. It must AND the named + edge's predicate and membership with the account's wall, without a row-window limit. + `bindings[semantic_store_table]` declares `{table, column, sql}` rows mapping those + canonical ids to a metric's SQL. `sql` may contain `{alias}` from the semantic model. + Neither callbacks nor SQL bindings come from the client's related object. + + This adapter is necessary because core has no session and cannot import API readers. + Missing context refuses by name; it never guesses a join or reads an unscoped store. + """ + token = _MEASURE_PIN_CONTEXT.set(dict(read=read, bindings=bindings, may_open=may_open, + fields_for=fields_for, defs=defs, cache_key=object())) + try: + yield + finally: + _MEASURE_PIN_CONTEXT.reset(token) + + +def measure_pin_key(related): + """Wire identity plus the trusted read context; permission contexts never share answers.""" + if related is None: + return None + ctx = _MEASURE_PIN_CONTEXT.get() or {} + return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key') + + +@dataclass +class MeasurePin: + path: list + rows: dict + bindings: dict + refusal: dict = None + + def predicate(self, store): + """AND every declared, traversed member set into this metric's raw rows.""" + if self.refusal is not None: + return None, [] + parts, params = [], [] + for binding in self.bindings.get(store['table'], ()): + table, column = binding['table'], binding['column'] + if table not in self.rows: + continue + values = [] + for row in self.rows[table]: + if row.get(column) in (None, ''): + raise PinRefusal('relation_source_unavailable', + f"The related metric needs '{column}' on '{table}', " + "but that value could not be read. Values are unscoped.", + self.path) + values.append(row[column]) + try: + sql, args = pin_sql(binding['sql'].format(alias=store['alias']), values) + except (TypeError, ValueError) as exc: + raise PinRefusal('relation_binding_unavailable', + f"The metric binding on '{table}' cannot carry this pin. " + "Values are unscoped.", self.path) from exc + parts.append(f'({sql})') + params.extend(args) + if not parts: + raise PinRefusal('relation_binding_unavailable', + f"The metric source '{store['table']}' has no declared binding " + "to this record path. Values are unscoped.", self.path) + return ' AND '.join(parts), params + + +def measure_pin(related): + """Resolve only the requested named edges, retaining intermediate row membership. + + Retaining the intermediate is essential: customer -> lines -> products must not expand + back to every customer's lines for those products. Empty membership remains a real pin. + """ + if isinstance(related, MeasurePin): + return related + if not isinstance(related, dict): + raise PinRefusal('bad_relation', 'A related metric needs {table, record, via}.') + table, record, via = related.get('table'), related.get('record'), related.get('via') + path = via.get('path') if isinstance(via, dict) else via + if isinstance(path, str): + path = [path] + if isinstance(path, (list, tuple)) and len(path) > MAX_DEPTH: + raise PinRefusal('over_depth', over_depth_refusal(table, path), path) + if not isinstance(table, str) or not table or record in (None, ''): + raise PinRefusal('bad_relation', 'A related metric needs {table, record, via}.') + ctx = _MEASURE_PIN_CONTEXT.get() or {} + if ctx and not ctx['may_open'](table): + raise PinRefusal('forbidden_table', + f"'{table}' cannot be opened for this record scope. " + 'Metric values are unscoped.') + fields_for = ctx.get('fields_for') + fields = fields_for(table) if fields_for else None + entries = related_paths(table, fields=fields, defs=ctx.get('defs'), + may_open=ctx.get('may_open'), fields_for=fields_for) + if isinstance(via, dict) and path is None: + # Bag matching still requires an edge name, never just a join-column heuristic. + entry = pick([e for e in entries if e['hops'] == 1], via) if via.get('key') else None + else: + entry = pick_path(entries, path) + if entry is None: + if isinstance(path, (list, tuple)) and path: + diagnosis = diagnose_path(table, path, fields=fields, defs=ctx.get('defs'), + may_open=ctx.get('may_open'), fields_for=fields_for, + entries=entries) + raise PinRefusal(diagnosis['code'], diagnosis['message'], path) + raise PinRefusal('unknown_relation', unknown_via_refusal(table, via)) + path = entry['path'] + # Resolve each leg before reading. A composite's target is not necessarily the target + # whose binding is absent; the refusal must name the actual unserved leg. + hops = [pick_path(entries, path[:1])] + if len(path) == 2: + mid = hops[0]['target'] + hops.append(pick_path(related_paths( + mid, fields=fields_for(mid) if fields_for else None, defs=ctx.get('defs'), + depth=1, may_open=ctx.get('may_open'), fields_for=fields_for), path[1:])) + for hop in hops: + if hop is None: + raise PinRefusal('unknown_relation', unknown_via_refusal(table, path), path) + if not hop.get('served'): + raise PinRefusal('relation_not_served', + unserved_target_refusal(hop) + '. Metric values are unscoped.', path) + if not ctx: + raise PinRefusal('relation_context_required', + 'This request has no scoped relation reader for metrics. ' + 'Metric values are unscoped.', path) + rows = {table: list(ctx['read'](table, IDENTITY_COL, [record], None))} + if not rows[table]: + raise PinRefusal('relation_source_unavailable', + f"The origin record could not be read from '{table}'. " + 'Metric values are unscoped.', path) + source = table + for hop in hops: + frm = hop['via'].get('from') or IDENTITY_COL + values = [r.get(frm) for r in rows[source]] + if any(v in (None, '') for v in values): + raise PinRefusal('relation_source_unavailable', + unresolvable_source_refusal(source, hop), path) + target = hop['target'] + rows[target] = list(ctx['read'](target, hop['via'].get('on') or IDENTITY_COL, + values, hop.get('where'))) if values else [] + source = target + return MeasurePin(path, rows, ctx['bindings']) + + def compiled_pin_matches_filter_sql(id_sql, member_ids, columns=None): """Does `pin_sql` still spell the member tail exactly as `filter_sql` spells it? @@ -941,6 +1130,23 @@ def one_hop(table, fields=None, defs=None, include_retired=False): entry['cardinality'] = str(got or 'many_to_many') out.append(entry) seen = {edge_identity(e) for e in out} + # A product bridge is also the declared line -> product hop. Reverse the declared + # columns, including product_id, rather than inventing a pid join or searching a path. + for bridge in bridges_into(table): + entry = _entry(key=f"{bridge['source']}__{bridge['key']}", + label=bridge.get('source_label') or bridge['source'], + target=bridge['source'], direction='reverse', + on=bridge['source_col'], frm=bridge['target_col'], + served=bridge['served'], + cardinality=_INVERSE.get(_bridge_cardinality(bridge), 'many_to_many')) + # A predicate belongs to the forward target; reversing it needs an explicit binding. + # Keep such reverse edges absent instead of applying the predicate on the wrong table. + if bridge.get('where') or edge_identity(entry) in seen: + continue + if not _live([entry], include_retired): + continue + seen.add(edge_identity(entry)) + out.append(entry) for entry in spec_entries(table, include_retired=include_retired): if edge_identity(entry) in seen: continue diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index e574b223cfd0e2d5993fe19a1eea3e95265d0b16..ce6c401a41fc6e63bbbc4e7a7971e3f2567d4461 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -377,6 +377,149 @@ def limit_report(table_key, st=None): } +# W57-T37 / instruction 30 / C13: revocation policy belongs to a CREDENTIAL TYPE, not to +# every connected table. W32 R10 keeps Odoo data for reconnect; Meta has no purge ruling. +# The API declares its families here because core must never import the connector layer. +REVOKE_PURGE = 'purge' +REVOKE_KEEP = 'keep' +_CREDENTIAL_TABLES = {} + + +def register_credential_tables(entry_type, prefix, *, on_revoke=REVOKE_KEEP, why='', owner=None): + """Declare a credential's table family and revocation rule; repeat declarations overwrite. + + Empty types/prefixes and unknown policies raise ValueError: a typo must not silently KEEP + data that revocation promised to remove. Without an owner resolver, fixed-key families + belong to the tenant's designated default connection, as Amazon and Meta do today. + """ + etype = str(entry_type or '').strip().lower() + prefix = str(prefix or '').strip().lower() + if not etype or not prefix: + raise ValueError('credential type and table prefix must not be empty') + if on_revoke not in (REVOKE_PURGE, REVOKE_KEEP): + raise ValueError('credential revocation must be purge or keep') + if owner is not None and not callable(owner): + raise ValueError('credential table owner must be callable') + _CREDENTIAL_TABLES[etype] = { + 'prefix': prefix, 'on_revoke': on_revoke, 'why': str(why or ''), 'owner': owner, + } + return dict(_CREDENTIAL_TABLES[etype]) + + +def credential_table_types(): + """Copy the declarations; a disclosure caller must not be able to rewrite purge policy.""" + return {etype: dict(spec) for etype, spec in _CREDENTIAL_TABLES.items()} + + +def connection_of_table(table_key, rt=None, st=None): + """Return (credential type, connection id), or (None, None) outside declared families. + + The longest prefix wins, never registration order. A broken owner resolver falls back to + the designated default: calling its tables ownerless would quietly turn a purge into nothing. + """ + specs = sorted(_CREDENTIAL_TABLES.items(), key=lambda item: -len(item[1]['prefix'])) + for etype, spec in specs: + if not str(table_key).startswith(spec['prefix']): + continue + if rt is None: + return etype, None + if spec['owner'] is not None: + try: + return etype, spec['owner'](table_key, rt) + except Exception: + pass + # keychain imports the store too; importing sideways here avoids a boot-time cycle. + from core import keychain + return etype, keychain.default_connection_id(rt, etype) + return None, None + + +def credential_footprint(entry_type, connection_id, rt=None, st=None): + """Disclose what revoking this ONE credential purges and what its policy keeps. + + A read-through table's row count is UNKNOWN, not zero: its rows live in the mirror. + That distinction is the point of [[no-unverifiable-aggregates]] in a deletion dialog; + a fabricated zero invites consent under the false promise that no records will be lost. + """ + etype = str(entry_type or '').strip().lower() + cid = str(connection_id or '') + spec = credential_table_types().get(etype) + out = {'type': etype, 'connectionId': cid, 'action': 'none', 'purge': [], 'keep': [], + 'rows': 0, 'rowsKnown': True, 'why': 'This credential type spawns no databases.'} + if spec is None: + return out + out.update(action=spec['on_revoke'], why=spec['why']) + if not cid: + return out + + class _Reads: + # all_defs/all_tables serve display fallbacks on store errors. A revocation cannot + # mistake that empty fallback for proof there is nothing to delete. Projection failure + # may still recover through the whole read; only failure of that read is fatal here. + error = None + + def get_projection(self, name, drop=()): + return _st(st).get_projection(name, drop=drop) + + def get(self, name): + try: + return _st(st).get(name) + except Exception as exc: + self.error = exc + raise + + reads = _Reads() + defs = all_defs(reads) + if reads.error is not None: + raise reads.error + candidates = [] + for key, defn in defs.items(): + if (str(key).startswith(spec['prefix']) + and connection_of_table(key, rt, st)[1] == cid): + candidates.append((key, defn, materialises(key, st=st, defn=defn))) + # all_tables is a 28.6 MB deep copy on tenant #0. No materialised candidate means no row + # count needs that document, so the disclosure must not pay for it merely to list labels. + tables = all_tables(reads) if any(known for _, _, known in candidates) else {} + if reads.error is not None: + raise reads.error + for key, defn, known in candidates: + if known and key not in tables: + raise RuntimeError('credential table rows could not be read') + rows = len(tables[key].get('rows') or {}) if known else 0 + out[spec['on_revoke']].append({ + 'key': key, 'label': defn.get('label') or key, 'rows': rows, 'rowsKnown': known, + }) + for action in (REVOKE_PURGE, REVOKE_KEEP): + out[action].sort(key=lambda table: table['key']) + out['rows'] = sum(table['rows'] for table in out['purge']) + out['rowsKnown'] = all(table['rowsKnown'] for table in out['purge']) + return out + + +def purge_credential_tables(entry_type, connection_id, rt=None, st=None): + """Recompute this credential's footprint, then use the existing artifact-family sweep. + + A refused or failed delete is residue, never a success count. Exception text may contain + a store path, so the report carries only its type. A second sweep finds no deleted tables. + """ + footprint = credential_footprint(entry_type, connection_id, rt=rt, st=st) + out = {'purged': [], 'refused': [], 'failed': [], 'rows': 0, 'rowsKnown': True, + 'footprint': footprint} + for table in footprint['purge']: + key = table['key'] + try: + if delete(key, st=st) is False: + out['refused'].append(key) + continue + except Exception as exc: + out['failed'].append({'key': key, 'error': type(exc).__name__}) + continue + out['purged'].append(key) + out['rows'] += table['rows'] + out['rowsKnown'] = out['rowsKnown'] and table['rowsKnown'] + return out + + MAX_FIELDS = 60 #: the badge the nav paints for a table with no connector behind it BLANK_SOURCE = 'Blank' diff --git a/platform/core/users.py b/platform/core/users.py index f532cc96bf45ec3005eb25fe54151da45e511127..b39f3976cdb5a1c3fc8f8516731ae46e43a478ff 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -11,6 +11,12 @@ import os import hmac import hashlib import secrets +import re +import base64 +import binascii +import struct +import time +from urllib.parse import quote import core.store as store @@ -70,10 +76,309 @@ def _hash(pw, salt): return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex() +# ------------------------------------------------------------------ password policy (W57-T35 / C12) +# ONE policy for account creation and both rotation doors. Login only reads the age: checking +# history there would spend six PBKDF2 passes on every sign-in without improving authentication. +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 256 # bound PBKDF2 input, not a claim about strength +PASSWORD_HISTORY = 5 +PASSWORD_MAX_AGE_DAYS = 90 + + +class PasswordPolicyError(ValueError): + """Raised with FAILED RULES attached, never a bare message. + + `.failures` is a list of {"rule": , "message": }. + """ + def __init__(self, failures): + self.failures = [dict(failure) for failure in failures] + super().__init__('; '.join(failure['message'] for failure in self.failures)) + + +def password_rules(): + """The public checklist, including the rules enforced over an account's lifetime.""" + return [ + {'rule': 'length', 'message': f'it must be at least {PASSWORD_MIN_LENGTH} characters'}, + {'rule': 'max_length', + 'message': f'it must be no longer than {PASSWORD_MAX_LENGTH} characters'}, + {'rule': 'uppercase', 'message': 'it must contain at least one capital letter'}, + {'rule': 'lowercase', 'message': 'it must contain at least one small letter'}, + {'rule': 'digit', 'message': 'it must contain at least one number'}, + {'rule': 'symbol', + 'message': 'it must contain at least one character that is not a letter or a number'}, + {'rule': 'identifier', + 'message': 'it must not contain the username, the email local part, or any word of ' + 'the display name that is 3 characters or longer'}, + {'rule': 'reuse', + 'message': f'it must not repeat the current password or any of the last ' + f'{PASSWORD_HISTORY} passwords'}, + {'rule': 'expiry', 'message': f'it must be changed every {PASSWORD_MAX_AGE_DAYS} days'}, + ] + + +def password_failures(pw, username=None, name=None, email=None, record=None): + """Every rule this password breaks, in policy order. Empty list means it passes.""" + pw = str(pw or '') + # Case-fold identifiers, but never the credential being hashed. Name punctuation separates + # words too, so a hyphenated display name cannot slip through a whitespace-only split. + identifiers = [str(username or '').strip().casefold(), + str(email or '').strip().split('@', 1)[0].casefold()] + identifiers.extend(word.casefold() for word in re.findall(r'[^\W_]+', str(name or '')) + if len(word) >= 3) + folded = pw.casefold() + broken = { + 'length': len(pw) < PASSWORD_MIN_LENGTH, + 'max_length': len(pw) > PASSWORD_MAX_LENGTH, + 'uppercase': not any(c.isupper() for c in pw), + 'lowercase': not any(c.islower() for c in pw), + 'digit': not any(c.isdigit() for c in pw), + 'symbol': not any(not c.isalnum() for c in pw), + 'identifier': any(identifier and identifier in folded for identifier in identifiers), + } + # An oversized value is already refused; do not feed it to PBKDF2 merely to describe reuse. + # Every history entry owns its own salt, so comparing against only today's salt proves nothing. + if record is not None and len(pw) <= PASSWORD_MAX_LENGTH: + for credential in [record, *(record.get('password_history') or [])]: + if (credential.get('salt') and credential.get('hash') + and hmac.compare_digest(_hash(pw, credential['salt']), credential['hash'])): + broken['reuse'] = True + break + return [rule for rule in password_rules() if broken.get(rule['rule'])] + + +def password_changed_at(record): + """The stored ISO timestamp, or None when absent or unreadable.""" + import datetime as _dt + stamp = (record or {}).get('password_changed_at') + try: + parsed = _dt.datetime.fromisoformat(stamp) + # Our writer always includes an offset. A partial legacy/manual stamp must never raise + # while comparing it with UTC on the login path; treat it like an absent stamp instead. + if parsed.utcoffset() is None: + return None + except (TypeError, ValueError, OverflowError): + return None + return stamp + + +def password_age_days(record): + """Fractional days since the last password change, or None without a readable stamp.""" + import datetime as _dt + stamp = password_changed_at(record) + if stamp is None: + return None + return (_dt.datetime.now(_dt.timezone.utc) - _dt.datetime.fromisoformat(stamp)).total_seconds() / 86400 + + +def password_expired(record): + """Whether a forced rotation or the password's age requires the change door.""" + if (record or {}).get('must_change_password') is True: + return True + age = password_age_days(record) + # ABSENT STAMP IS NOT EXPIRED: pre-wave accounts have none, and deploying this policy must + # not force the whole tenant to reset. Their first successful sign-in starts the clock. + return age is not None and age > PASSWORD_MAX_AGE_DAYS + + +def generate_password(): + """A random credential that SATISFIES the policy above, for a caller that has none to offer. + + ⛔ IT EXISTS BECAUSE A GENERATOR AND A VALIDATOR THAT DISAGREE FAIL AT PROVISIONING TIME. + `provision_web_tenant.py` and `ops/provision_user.py` both defaulted to + `secrets.token_urlsafe(12)`, whose alphabet contains no symbol and need not contain an + uppercase letter or a digit in any given draw. Under the policy that is a tenant that + provisions on Monday and refuses on Tuesday, for no reason anyone could see. So the default + is built to the rules rather than checked against them, and asserted before it is returned. + + ⚠ NOT a password anybody types: it is handed to the operator to hand on, and the account is + expected to rotate it. Entropy is what matters here, which is why the bulk of it is + `token_urlsafe`. + """ + for _ in range(8): + candidate = f"A{secrets.token_urlsafe(16)}9!" + if not password_failures(candidate): + return candidate + # Unreachable in practice; a loop that can silently fall out of the bottom and return a + # policy-breaking string is exactly the defect this function exists to remove. + raise RuntimeError('could not generate a password that satisfies the policy') + + +# ------------------------------------------------------------------ second factor (W57-T36 / C12) +# RFC 6238: SHA-1, six digits and a 30-second step are what authenticator apps assume. +# They are constants, not account options: two parameter sets would make a scanned secret +# produce codes the server never accepts. Twenty random bytes is RFC 4226's reference length. +MFA_DIGITS = 6 +MFA_PERIOD = 30 +MFA_DRIFT = 1 # accept the previous and next window: phones drift +MFA_ISSUER = 'Loopable' +RECOVERY_CODE_COUNT = 10 + + +def new_mfa_secret() -> str: + return base64.b32encode(secrets.token_bytes(20)).decode('ascii').rstrip('=').upper() + + +def totp_code(secret, at=None, offset=0) -> str: + key = base64.b32decode(secret + '=' * (-len(secret) % 8), casefold=True) + if not key: + # Padding alone decodes to empty bytes, which is not an authenticator secret. + raise ValueError('missing authenticator secret') + counter = int(time.time() if at is None else at) // MFA_PERIOD + offset + digest = hmac.new(key, struct.pack('>Q', counter), hashlib.sha1).digest() + start = digest[-1] & 0x0f + number = struct.unpack('>I', digest[start:start + 4])[0] & 0x7fffffff + return str(number % (10 ** MFA_DIGITS)).zfill(MFA_DIGITS) + + +def totp_valid(secret, code, at=None) -> bool: + if not isinstance(secret, str) or not secret: + return False + code = str(code or '').replace(' ', '') + if len(code) != MFA_DIGITS or not code.isascii() or not code.isdigit(): + return False + try: + # Capture the time once: crossing a step boundary mid-loop must not change the window. + at = time.time() if at is None else at + valid = False + for offset in range(-MFA_DRIFT, MFA_DRIFT + 1): + # There is no counter before the Unix epoch; this only matters for boundary tests. + expected = (totp_code(secret, at=at, offset=offset) + if int(at) // MFA_PERIOD + offset >= 0 else '') + # Compare EVERY window, even after a match. Early return leaks which window matched. + valid |= hmac.compare_digest(expected, code) + return valid + except (ValueError, TypeError, binascii.Error, struct.error, OverflowError): + # A malformed stored/candidate secret is an invalid factor, never a login-path 500. + return False + + +def otpauth_uri(secret, username, issuer=MFA_ISSUER) -> str: + label = quote(f'{issuer}:{username}', safe='') + return (f'otpauth://totp/{label}?secret={quote(secret, safe="")}' + f'&issuer={quote(issuer, safe="")}&algorithm=SHA1' + f'&digits={MFA_DIGITS}&period={MFA_PERIOD}') + + +def new_recovery_codes(n=RECOVERY_CODE_COUNT) -> list[str]: + codes = [] + while len(codes) < n: + raw = secrets.token_hex(5).upper() + code = f'{raw[:5]}-{raw[5:]}' + # Even the unlikely duplicate must not make one printed code redeemable twice. + if code not in codes: + codes.append(code) + return codes + + +def _recovery_key(code) -> str: + # NOT PBKDF2: a recovery code is 40 bits of CSPRNG output with no user-chosen structure, + # so there is no password dictionary to run against it. Stretching adds no entropy; the + # codes use SHA-256 here. A PASSWORD is different and stays on PBKDF2-200k. + normal = str(code or '').strip().upper().replace('-', '').replace(' ', '') + return hashlib.sha256(normal.encode('utf-8')).hexdigest() + + +def mfa_enrolled(record) -> bool: + secret = (record or {}).get('mfa_secret') + return isinstance(secret, str) and bool(secret.strip()) + + +class MFAStateError(ValueError): + """A concurrent enrolment or credential change refused an MFA write, before mutation.""" + + +def set_mfa(username, secret, recovery_codes, *, require_unenrolled=False, expected_epoch=None): + """Store the factor and hashed codes together, returning the LANDED revocation epoch. + + The optional guards bind the HTTP door's decision to the actual write. A route-only + enrolment check lets two workers both issue a first set of codes; an epoch check prevents + regeneration racing an admin reset from silently reinstalling the lost factor. + """ + username = (username or '').strip().lower() + # The replay closure captures only hashes, never printable recovery codes: the HF store + # journals closures for conflict replay, and that journal must not retain clear codes. + hashes = [_recovery_key(c) for c in recovery_codes] + bumped = {} + + def _set(reg): + bumped.clear() # a replay may find the account gone + u = reg.get(username) + if u: + if require_unenrolled and mfa_enrolled(u): + raise MFAStateError('already_enrolled') + if expected_epoch is not None and int(u.get('epoch') or 0) != expected_epoch: + raise MFAStateError('changed') + u['mfa_secret'] = secret + u['recovery_codes'] = list(hashes) + u['mfa_enrolled_at'] = _now_iso() + # C12: enrolment and code rotation revoke old cookies in the SAME update as the + # secret. A separate bump could fail after enrolment and leave every old cookie live. + u['epoch'] = int(u.get('epoch') or 0) + 1 + bumped['epoch'] = u['epoch'] + return reg + landed = store.update('users', _set) + if 'epoch' not in bumped: + return None + # As with bump_epoch, a rebased document can land at a newer epoch than our first attempt. + try: + rec = (landed or {}).get(username) or {} + if rec.get('epoch') is not None: + return int(rec['epoch']) + except Exception: + pass + return int(bumped['epoch']) + + +def clear_mfa(username): + """Admin lost-factor reset: remove enrolment and revoke every session in one write.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + for key in ('mfa_secret', 'recovery_codes', 'mfa_enrolled_at'): + u.pop(key, None) + # Absence sends the next login through phone enrolment; the bump kills old cookies. + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def consume_recovery_code(username, code) -> bool: + username = (username or '').strip().lower() + wanted = _recovery_key(code) + consumed = [False] + + def _set(reg): + consumed[0] = False # report the final replay, not an earlier attempt + u = reg.get(username) + if u: + hashes = u.get('recovery_codes') or [] + matched = None + for i, stored in enumerate(hashes): + equal = hmac.compare_digest(stored, wanted) + # Compare EVERY entry, retaining only the first match; never leak its position. + if equal and matched is None: + matched = i + if matched is not None: + u['recovery_codes'] = hashes[:matched] + hashes[matched + 1:] + consumed[0] = True + # NO EPOCH BUMP: this runs during sign-in. Login owns the bump and the cookie it + # is about to mint; consuming a code must not independently invalidate that cookie. + return reg + store.update('users', _set) + return consumed[0] + + +def recovery_codes_remaining(record) -> int: + return len((record or {}).get('recovery_codes') or []) + + def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None, perms=None, tenant='royal-imports', platform_admin=False): salt = secrets.token_hex(16) rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role, + 'password_changed_at': _now_iso(), 'bus': bus, 'active': active, 'modules': modules, 'agent': agent or None, 'email': email or None, # Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every @@ -121,6 +426,8 @@ def ensure_bootstrap(): master = os.environ.get('APP_PASSWORD', '') if not master: return + # Keep the fresh-store break-glass seed exempt from the password policy. Rejecting whatever + # APP_PASSWORD holds here could leave a new deployment with nobody able to sign in. try: store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')}) except Exception: @@ -297,8 +604,12 @@ def create_user(username, pw, name, role='user', bus='all', modules='all', session-invalidating as a password change, and it usually IS one. """ username = (username or '').strip().lower() - if not username or not pw: + if not username: raise ValueError('username and password are required') + # Refuse before the update so even an empty password names its broken rules and writes nothing. + failures = password_failures(pw, username=username, name=name, email=email) + if failures: + raise PasswordPolicyError(failures) def _add(reg): prior = reg.get(username) or {} @@ -321,14 +632,36 @@ def create_user(username, pw, name, role='user', bus='all', modules='all', store.update('users', _add) -def set_password(username, pw): +def set_password(username, pw, enforce=True): + """Rotate the credential, bounded history, age and epoch in one write. + + `enforce=False` is only for a caller that already validated (or a future migration) to rotate + without re-reading. The admin password route is that one caller; all others enforce here. + """ username = (username or '').strip().lower() + if enforce: + try: + reg = store.get('users', fresh=True) or {} + except Exception: + reg = {} + rec = reg.get(username) or {} + # This must happen BEFORE store.update: refusing a rule may never leave half a rotation. + failures = password_failures(pw, username=username, name=rec.get('name'), + email=rec.get('email'), record=rec) + if failures: + raise PasswordPolicyError(failures) def _set(reg): u = reg.get(username) if u: + # Merge into the fresh record rather than rebuilding it: access and concurrent + # activity stamps must survive, and each retired credential keeps the salt it used. + u['password_history'] = ([{'salt': u['salt'], 'hash': u['hash'], 'at': _now_iso()}] + + (u.get('password_history') or []))[:PASSWORD_HISTORY] u['salt'] = secrets.token_hex(16) u['hash'] = _hash(pw, u['salt']) + u['password_changed_at'] = _now_iso() + u.pop('must_change_password', None) # X3: a password change revokes every outstanding API session for the account. Bumped # INSIDE the same read-modify-write as the hash so the two can never disagree — a # separate update() could rotate the password and leave old cookies live if the second @@ -338,6 +671,24 @@ def set_password(username, pw): store.update('users', _set) +def set_must_change_password(username, on): + """Require a new password at next login, revoking existing sessions in the same write.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + if on: + u['must_change_password'] = True + else: + u.pop('must_change_password', None) + # The flag blocks FUTURE logins; the epoch kills sessions already issued. Clearing + # the flag also bumps so it can never resurrect a cookie from before the forced reset. + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + def set_active(username, active): username = (username or '').strip().lower() @@ -454,6 +805,11 @@ def flush_stamps(timeout=10.0): return all(not t.is_alive() for t in pending) +def stamp_password_age(username, when=None): + """Backfill a legacy account's clock through the same fail-silent merge as login activity.""" + return _stamp(username, {'password_changed_at': when or _now_iso()}) + + def touch_login(username, when=None): """Stamp `last_login` (ISO-8601, UTC, OFFSET-BEARING) on a SUCCESSFUL login. diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index 12bd512f3321132368cd1b113f4bfe467ffb6b19..decd6ddd93aff927ad601d5e0d40ca0577a3f4f4 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -46,7 +46,10 @@ Ideation record, with every owner ruling: `AutoWiki/research/ideation/amazon-con """ from __future__ import annotations +import contextvars import json +import random +import threading import time import urllib.error import urllib.parse @@ -55,6 +58,33 @@ import urllib.request KEY = "amazon_sp" LABEL = "Amazon" +#: W57-T38/C13: five total attempts, including the first, bound one throttled operation. +RETRY_MAX_ATTEMPTS = 5 +#: W57-T38/C13: exponential backoff starts at one second after the first 429. +RETRY_BASE_SECONDS = 1.0 +#: W57-T38/C13: no individual retry sleep exceeds one minute, including Retry-After/jitter. +RETRY_MAX_SECONDS = 60.0 +#: W57-T38/C13: randomize each wait by up to half its value to separate competing clients. +RETRY_JITTER = 0.5 +#: W57-T38/C13: separate consecutive table report creations by at least five seconds. +REPORT_STAGGER_SECONDS = 5.0 +#: Amazon createReport usage plan: 15 initial tokens, shared by callers in this process. +#: https://developer-docs.amazon.com/sp-api/reference/createreport +REPORT_BURST = 15 +#: Amazon createReport usage plan: 0.0167 requests/second, approximately one token/minute. +#: https://developer-docs.amazon.com/sp-api/reference/createreport +REPORT_SUSTAINED_SECONDS = 60.0 + +# ⭐ One process-wide bucket and lock prevent concurrent syncs from spending the same token. +# The contextual wait belongs to the current report_document logger, never another sync's log. +_report_pacer = { + "tokens": float(REPORT_BURST), + "updated": None, + "last_create": None, + "lock": threading.Lock(), + "waited": contextvars.ContextVar("amazon_report_create_waited", default=0.0), +} + #: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data #: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the #: comments in `keychain.py` warn about it in as many words. @@ -165,13 +195,13 @@ REPORT_TYPES = { "report": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "report_incremental": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "validates_against": "order report totals vs our order rows", - "role": "Inventory and Order Management", + "role": "Inventory and Order Tracking", }, "order_items": { "report": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "report_incremental": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "validates_against": "order report line totals vs our item rows", - "role": "Inventory and Order Management", + "role": "Inventory and Order Tracking", }, "listings": { "report": "GET_MERCHANT_LISTINGS_ALL_DATA", @@ -203,7 +233,7 @@ REPORT_TYPES = { # ⛔⛔ O5 IS CLOSED AND THE ANSWER IS NO (researched 2026-09-04). All 101 report types were # enumerated and ZERO contain BUYBOX, PRICING, PRICE or COMPETITIVE. So this table SPLITS: # Buy Box SHARE -> GET_SALES_AND_TRAFFIC_REPORT.salesAndTrafficByAsin.buyBoxPercentage, - # which IS bulk. ⚠ needs the BrandAnalytics role, the same one R5 marks + # which IS bulk. ⚠ needs the Brand Analytics role, the same one R5 marks # UNPROVEN, so this row inherits that risk. # Buy Box PRICE -> Product Pricing entity endpoints ONLY. No bulk path exists. # ⛔ STANDING RULE 1'S SECOND HALF: the ceiling is Amazon's and cannot be removed, so it is @@ -218,7 +248,7 @@ REPORT_TYPES = { "report_covers": "buy box SHARE only; PRICE has no bulk path", "price_path": "productPricing/2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)", "validates_against": "sales and traffic report totals vs our rows", - "role": "Pricing + BrandAnalytics", + "role": "Pricing + Brand Analytics", }, "brand_analytics": { # ⚠ Returns data only for Brand Registry brand owners (owner ruling R5). Carried as the @@ -226,7 +256,7 @@ REPORT_TYPES = { # for any registered brand, else demote to a declared-but-unsynced stub. "report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", "validates_against": "report period totals vs our search-term rows", - "role": "BrandAnalytics", + "role": "Brand Analytics", }, } @@ -236,9 +266,9 @@ REPORT_TYPES = { #: architecture review over screen share at developer registration. #: · https://developer-docs.amazon.com/sp-api/docs/register-as-a-public-developer ROLES_TAKEN = ( - "Inventory and Order Management", "Amazon Fulfillment", + "Inventory and Order Tracking", "Amazon Fulfillment", "Amazon Warehousing and Distribution", "Finance and Accounting", - "Pricing", "Product Listing", "BrandAnalytics", "Selling Partner Insights", + "Pricing", "Product Listing", "Brand Analytics", "Selling Partner Insights", ) ROLES_REFUSED = ("Buyer Communication", "Buyer Solicitation") @@ -250,6 +280,17 @@ class AmazonError(RuntimeError): """Any refusal from LWA or SP-API, already stripped of anything secret.""" +class AmazonThrottled(AmazonError): + """An SP-API 429, with an optional Retry-After delay in seconds.""" + + def __init__(self, message, retry_after=None): + super().__init__(message) + try: + self.retry_after = max(0.0, float(retry_after)) if retry_after is not None else None + except (TypeError, ValueError): + self.retry_after = None + + def _safe(url): """A URL fit to appear in an error message: query string dropped entirely. @@ -264,6 +305,49 @@ def _safe(url): return "" +def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random): + """Retry only SP-API throttling; return successes and propagate other refusals unchanged.""" + for n in range(1, RETRY_MAX_ATTEMPTS + 1): + try: + return attempt_fn() + except AmazonThrottled as e: + if n == RETRY_MAX_ATTEMPTS: + # ⛔ probe_credential exposes str(e)[:200]; amazon_store.sync stores str(e)[:160]. + # Throttle, cause AND the complete fix must fit in the first 160 characters; + # this head is 152 with the named cap. Operation/URL detail must stay AFTER it. + raise AmazonThrottled( + f"Amazon throttled (429); {n - 1} retries spent. " + "Cause: SP-API limits per operation, app and seller. " + "Fix: sync fewer tables at once and widen the report window. " + f"Operation: {_safe(what)}", retry_after=e.retry_after) from None + wait = min(RETRY_MAX_SECONDS, RETRY_BASE_SECONDS * 2 ** (n - 1)) + floor = min(RETRY_MAX_SECONDS, e.retry_after or 0.0) + wait = max(wait, floor) + wait *= 1 - RETRY_JITTER + RETRY_JITTER * rand() * 2 + # ⭐ Negative jitter must not undo Amazon's Retry-After floor; the named cap wins. + sleep(min(RETRY_MAX_SECONDS, max(0.0, floor, wait))) + + +def _pace_report_create(*, sleep=time.sleep, now=time.monotonic): + """Reserve one report creation under the burst, refill rate and minimum gap; return its wait.""" + with _report_pacer["lock"]: + stamp = now() + updated = _report_pacer["updated"] + elapsed = max(0.0, stamp - updated) if updated is not None else 0.0 + tokens = min(REPORT_BURST, _report_pacer["tokens"] + elapsed / REPORT_SUSTAINED_SECONDS) + last = _report_pacer["last_create"] + gap = last + REPORT_STAGGER_SECONDS - stamp if last is not None else 0.0 + wait = max(0.0, gap, (1.0 - tokens) * REPORT_SUSTAINED_SECONDS) + if wait: + sleep(wait) + # ⭐ Count refill during the wait too. A late wake may refill up to the burst, never more. + issued = max(stamp + wait, now()) + _report_pacer.update( + tokens=min(REPORT_BURST, tokens + (issued - stamp) / REPORT_SUSTAINED_SECONDS) - 1.0, + updated=issued, last_create=issued) + return wait + + def _post_form(url, data, timeout=_TIMEOUT): body = urllib.parse.urlencode(data).encode("ascii") req = urllib.request.Request(url, data=body, method="POST") @@ -318,43 +402,51 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT, """ if str(method).upper() != "GET": raise AmazonError("this connector is read-only: only GET is permitted") - # ⚠ AN EXPLICIT FLAG, NOT A `fields` DICT. This function is handed a TOKEN, never the - # credential -- deliberately, so a secret cannot reach the request builder at all -- so the - # sandbox decision is made by the caller that does hold the credential and passed in as one - # boolean. A first draft reached for `fields` here, which does not exist in this scope. - base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( - str(region or DEFAULT_REGION).lower()) - if not base: - raise AmazonError(f"unknown region `{region}`") - clean = {k: v for k, v in (params or {}).items() if v not in (None, "")} - url = f"{base}/{str(path).lstrip('/')}" - if clean: - url += "?" + urllib.parse.urlencode(clean, doseq=True) - req = urllib.request.Request(url, method="GET") - req.add_header("x-amz-access-token", token) - req.add_header("Accept", "application/json") - req.add_header("User-Agent", _UA) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - return json.loads(r.read().decode("utf-8") or "{}") - except urllib.error.HTTPError as e: - detail = "" + + def attempt(): + """One raw read-only SP-API attempt, with no retry or pacing policy.""" + # ⚠ AN EXPLICIT FLAG, NOT A `fields` DICT. This function is handed a TOKEN, never the + # credential -- deliberately, so a secret cannot reach the request builder at all -- so the + # sandbox decision is made by the caller that does hold the credential and passed in as one + # boolean. A first draft reached for `fields` here, which does not exist in this scope. + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower()) + if not base: + raise AmazonError(f"unknown region `{region}`") + clean = {k: v for k, v in (params or {}).items() if v not in (None, "")} + url = f"{base}/{str(path).lstrip('/')}" + if clean: + url += "?" + urllib.parse.urlencode(clean, doseq=True) + req = urllib.request.Request(url, method="GET") + req.add_header("x-amz-access-token", token) + req.add_header("Accept", "application/json") + req.add_header("User-Agent", _UA) try: - payload = json.loads((e.read() or b"").decode("utf-8") or "{}") - errs = payload.get("errors") or [] - if errs: - detail = str((errs[0] or {}).get("message") or "")[:200] - except Exception: # noqa: BLE001 + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8") or "{}") + except urllib.error.HTTPError as e: detail = "" - # 429 is the rate limiter, and standing rule 1 wants it NAMED rather than swallowed. - if e.code == 429: - raise AmazonError( - f"{_safe(url)} was throttled (429). SP-API is token-bucket limited per operation; " - "the fix is the Reports API bulk path, not a slower page loop") from None - raise AmazonError(f"{_safe(url)} answered {e.code}" - + (f": {detail}" if detail else "")) from None - except Exception as e: # noqa: BLE001 - raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + try: + payload = json.loads((e.read() or b"").decode("utf-8") or "{}") + errs = payload.get("errors") or [] + if errs: + detail = str((errs[0] or {}).get("message") or "")[:200] + except Exception: # noqa: BLE001 + detail = "" + # 429 is the rate limiter, and standing rule 1 wants it NAMED rather than swallowed. + if e.code == 429: + raise AmazonThrottled( + f"{_safe(url)} was throttled (429). SP-API is token-bucket limited per operation; " + "the fix is the Reports API bulk path, not a slower page loop", + retry_after=(e.headers or {}).get("Retry-After")) from None + raise AmazonError(f"{_safe(url)} answered {e.code}" + + (f": {detail}" if detail else "")) from None + except Exception as e: # noqa: BLE001 + raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower(), "") + return _retry(attempt, f"{base}/{str(path).lstrip('/')}") #: ⛔⛔ THE ONE NON-GET PATH IN THIS MODULE, AND IT IS A CONSTANT RATHER THAN A PARAMETER. @@ -384,51 +476,60 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION, ⛔ NO `path` PARAMETER, ON PURPOSE. See `_REPORTS_CREATE_PATH` above: the whole safety argument is that this function can reach exactly one operation and a caller cannot redirect it. """ - base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( - str(region or DEFAULT_REGION).lower()) - if not base: - raise AmazonError(f"unknown region `{region}`") - body = {"reportType": str(report_type), - "marketplaceIds": list(marketplace_ids or [US_MARKETPLACE_ID])} - if start: - body["dataStartTime"] = str(start) - if end: - body["dataEndTime"] = str(end) - if options: - body["reportOptions"] = dict(options) - url = f"{base}/{_REPORTS_CREATE_PATH}" - req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST") - req.add_header("x-amz-access-token", token) - req.add_header("Content-Type", "application/json") - req.add_header("Accept", "application/json") - req.add_header("User-Agent", _UA) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - got = json.loads(r.read().decode("utf-8") or "{}") - except urllib.error.HTTPError as e: - detail = "" + def attempt(): + """One raw attempt at the declared report-creation POST.""" + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower()) + if not base: + raise AmazonError(f"unknown region `{region}`") + body = {"reportType": str(report_type), + "marketplaceIds": list(marketplace_ids or [US_MARKETPLACE_ID])} + if start: + body["dataStartTime"] = str(start) + if end: + body["dataEndTime"] = str(end) + if options: + body["reportOptions"] = dict(options) + url = f"{base}/{_REPORTS_CREATE_PATH}" + req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST") + req.add_header("x-amz-access-token", token) + req.add_header("Content-Type", "application/json") + req.add_header("Accept", "application/json") + req.add_header("User-Agent", _UA) try: - errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or []) - detail = str((errs[0] or {}).get("message") or "")[:200] if errs else "" - except Exception: # noqa: BLE001 + with urllib.request.urlopen(req, timeout=timeout) as r: + got = json.loads(r.read().decode("utf-8") or "{}") + except urllib.error.HTTPError as e: detail = "" - if e.code == 429: - raise AmazonError( - f"{_safe(url)} was throttled (429) creating a {report_type} report. Report " - "creation is limited to about one call every 60 seconds with a burst of 15; the " - "fix is fewer, wider report windows, not more frequent requests") from None - if e.code == 403: - raise AmazonError( - f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this " - "report needs a role the application was not granted") from None - raise AmazonError(f"{_safe(url)} answered {e.code}" - + (f": {detail}" if detail else "")) from None - except Exception as e: # noqa: BLE001 - raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None - rid = str(got.get("reportId") or "") - if not rid: - raise AmazonError(f"Amazon accepted the {report_type} request but returned no reportId") - return rid + try: + errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or []) + detail = str((errs[0] or {}).get("message") or "")[:200] if errs else "" + except Exception: # noqa: BLE001 + detail = "" + if e.code == 429: + raise AmazonThrottled( + f"{_safe(url)} was throttled (429) creating a {report_type} report. Report " + "creation is limited to about one call every 60 seconds with a burst of 15; the " + "fix is fewer, wider report windows, not more frequent requests", + retry_after=(e.headers or {}).get("Retry-After")) from None + if e.code == 403: + raise AmazonError( + f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this " + "report needs a role the application was not granted") from None + raise AmazonError(f"{_safe(url)} answered {e.code}" + + (f": {detail}" if detail else "")) from None + except Exception as e: # noqa: BLE001 + raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + rid = str(got.get("reportId") or "") + if not rid: + raise AmazonError(f"Amazon accepted the {report_type} request but returned no reportId") + return rid + + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower(), "") + # ⭐ Pace once per new report. Retrying that same report pays only the shared backoff. + _report_pacer["waited"].set(_pace_report_create()) + return _retry(attempt, f"{base}/{_REPORTS_CREATE_PATH}") #: Terminal processing states. `DONE` is the only one that yields a document. @@ -454,8 +555,15 @@ def report_document(fields, report_type, start=None, end=None, options=None, region = str((fields or {}).get("region") or DEFAULT_REGION) mkts = [str((fields or {}).get("marketplace_id") or US_MARKETPLACE_ID)] sbx = is_sandbox(fields) - rid = _create_report(token, report_type, mkts, region=region, start=start, end=end, - options=options, timeout=timeout, sandbox=sbx) + pace_context = _report_pacer["waited"].set(0.0) + try: + rid = _create_report(token, report_type, mkts, region=region, start=start, end=end, + options=options, timeout=timeout, sandbox=sbx) + finally: + paced = _report_pacer["waited"].get() + _report_pacer["waited"].reset(pace_context) + if paced: + say(f" {report_type}: waited {paced:.2f}s to stagger report creation") say(f" {report_type}: reportId {rid}") waited, state, doc_id = 0, "", "" diff --git a/platform/harness/measure_filter.py b/platform/harness/measure_filter.py index 7377dbb32670a4725d68e034da11ce63dbea8dd8..06389b14bb79590da1d076eccccbd7b4062ad1d9 100644 --- a/platform/harness/measure_filter.py +++ b/platform/harness/measure_filter.py @@ -46,6 +46,7 @@ count that looks authoritative. `ModelError` is the only safe failure. """ from harness import semantic as _sem from harness import windows as _win +from core import relations as _relations #: metric key -> how it reads in the condition builder. ONLY metrics with an independent #: validation contract (semantic.store_parity / a metric `validate:` block) appear here. @@ -299,7 +300,29 @@ def _base_topic(key): return topic -def _query_parts(key, window, today, team_id): +def _related_scope(related, keys): + """Prepare all bases before querying: a refused composite must be wholly unscoped.""" + if related is None or isinstance(related, _relations.MeasurePin): + return related + try: + pin = _relations.measure_pin(related) + bases, _ = _sem._expand_measures(list(keys)) + for key in bases: + _, _, store = _entity_sql(_base_topic(key)) + pin.predicate(store) + return pin + except _relations.PinRefusal as exc: + return _relations.MeasurePin(exc.marker['path'], {}, {}, exc.marker) + + +def _related_answer(values, related): + if related is None: + return values + cls = _relations.MeasureMatches if isinstance(values, set) else _relations.MeasureAnswer + return cls(values, [related.refusal] if related.refusal else ()) + + +def _query_parts(key, window, today, team_id, related=None): """The FROM / GROUP-BY id / aggregate / WHERE for one (measure, window), shared by the one-sided and the two-sided resolver so there is one definition of the scope.""" topic = _base_topic(key) @@ -325,6 +348,12 @@ def _query_parts(key, window, today, team_id): agg = f"COALESCE({_sem._measure_sql(m, s['alias'])}, 0)" where, params = [_sem.render_scope(s['scope_sql'].strip())], [] + related = _related_scope(related, [key]) + if related is not None: + pin_sql, pin_params = related.predicate(s) + if pin_sql is not None: + where.append(f'({pin_sql})') + params.extend(pin_params) if date_from: where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") params.append(f"{date_from} 00:00:00") @@ -352,7 +381,7 @@ def _scaled(key, value): return (value / scale) if scale else value -def resolve_stat(rule, today, team_id=None, allowed_pids=None): +def resolve_stat(rule, today, team_id=None, allowed_pids=None, related=None): """`Where [Sales] [this year] [>] [the 75th percentile]` — the top quartile of the pool. One totals query for the measure, the statistic taken over the CALLER'S POOL (see the STATS @@ -363,6 +392,7 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None): raise _sem.ModelError(f"{stat!r} is not a statistic. Offered: {sorted(STATS)}") key = rule.get('colId') topic_for(key) + related = _related_scope(related, [key]) op = rule.get('op') if op not in PAIR_OPS: raise _sem.ModelError( @@ -376,7 +406,7 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None): "which would be a different question with an authoritative-looking answer") con = _sem._store_con() try: - totals = _totals(con, key, rule.get('window'), today, team_id) + totals = _totals(con, key, rule.get('window'), today, team_id, related) finally: con.close() # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision, @@ -398,8 +428,8 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None): population = {pid: v for pid, v in totals.items() if pid in pool and v} cut = percentile(list(population.values()), stat) if cut is None: # nobody has this measure -> nobody qualifies - return set() - return {pid for pid, v in population.items() if _CMP[op](v, cut)} + return _related_answer(set(), related) + return _related_answer({pid for pid, v in population.items() if _CMP[op](v, cut)}, related) def _pool_of(allowed_pids): @@ -445,7 +475,7 @@ def rule_complete(rule): return v is not None and str(v).strip() != '' -def resolve_rule(rule, today, team_id=None, allowed_pids=None): +def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None): """One measure condition -> the SET OF CUSTOMER IDS satisfying it. `rule` = {colId: , window: , op: , value: }, @@ -467,11 +497,12 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None): 'refusing to answer a question that has not been fully asked') rhs = rule.get('rhs') if isinstance(rhs, dict) and rhs.get('kind') == 'measure': - return resolve_pair(rule, today, team_id, allowed_pids) + return resolve_pair(rule, today, team_id, allowed_pids, related) if isinstance(rhs, dict) and rhs.get('kind') == 'stat': - return resolve_stat(rule, today, team_id, allowed_pids) + return resolve_stat(rule, today, team_id, allowed_pids, related) key = rule.get('colId') topic_for(key) # raises for anything not admitted + related = _related_scope(related, [key]) op = rule.get('op') if op not in OPS: raise _sem.ModelError(f"{op!r} is not a numeric comparison ({sorted(OPS)})") @@ -492,17 +523,18 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None): # compare in Python — the same shape `resolve_pair` uses, and for the same reason. con = _sem._store_con() try: - totals = _totals(con, key, rule.get('window'), today, team_id) + totals = _totals(con, key, rule.get('window'), today, team_id, related) finally: con.close() # No pool means we can only speak about customers a base has seen. WITH a pool, everyone # else scores 0 through the ratio's guard — which is what makes the zero-group work for a # ratio exactly as it does for a sum ("AOV <= 250" includes those who bought nothing). universe = pool if pool is not None else set(totals) - return {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)} + return _related_answer( + {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)}, related) frm, id_col, agg, where_sql, params = _query_parts( - key, rule.get('window'), today, team_id) + key, rule.get('window'), today, team_id, related) sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} " f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?") @@ -524,10 +556,10 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None): con.close() if pool is not None: matched &= pool - return matched + return _related_answer(matched, related) -def _totals(con, key, window, today, team_id): +def _totals(con, key, window, today, team_id, related=None): """`{pid: value}` for one (measure, window), COMPOSITES included. For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived @@ -541,13 +573,13 @@ def _totals(con, key, window, today, team_id): """ if is_composite(key): bases, _ = _sem._expand_measures([key]) - parts = {b: _totals(con, b, window, today, team_id) for b in bases} + parts = {b: _totals(con, b, window, today, team_id, related) for b in bases} pids = set() for got in parts.values(): pids |= set(got) return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key) for pid in pids} - frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id) + frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id, related) sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}") # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in # COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and @@ -558,7 +590,7 @@ def _totals(con, key, window, today, team_id): for r in con.execute(sql, params).fetchall() if r[0] is not None} -def resolve_pair(rule, today, team_id=None, allowed_pids=None): +def resolve_pair(rule, today, team_id=None, allowed_pids=None, related=None): """CG-9 — `Where [Sales] [this year] [>] [Sales] [year to date, last year]`. TWO QUERIES, compared in Python. The obvious alternative — one query with two conditional @@ -576,6 +608,7 @@ def resolve_pair(rule, today, team_id=None, allowed_pids=None): rhs = rule.get('rhs') or {} lkey, rkey = rule.get('colId'), rhs.get('colId') topic_for(lkey), topic_for(rkey) # raises for anything not admitted + related = _related_scope(related, [lkey, rkey]) op = rule.get('op') if op not in PAIR_OPS: raise _sem.ModelError( @@ -587,14 +620,15 @@ def resolve_pair(rule, today, team_id=None, allowed_pids=None): pool = _pool_of(allowed_pids) con = _sem._store_con() try: - left = _totals(con, lkey, rule.get('window'), today, team_id) - right = _totals(con, rkey, rhs.get('window'), today, team_id) + left = _totals(con, lkey, rule.get('window'), today, team_id, related) + right = _totals(con, rkey, rhs.get('window'), today, team_id, related) finally: con.close() # With no pool we can only speak about customers at least one side has seen — the zero-vs- # zero group is unknowable without being told who exists. With a pool, it is every member. universe = pool if pool is not None else (set(left) | set(right)) - return {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))} + return _related_answer( + {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))}, related) def _zero_satisfies(op, value): @@ -602,7 +636,7 @@ def _zero_satisfies(op, value): return _CMP[op](0.0, value) -def resolve_values(key, window, today, team_id=None, allowed_pids=None): +def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None): """`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD). @@ -621,6 +655,10 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None): Raises for anything not admitted / an unresolvable window — the caller degrades that column to blank rather than to a number nobody computed. + + Related pins use the request's relations.measure_pin_context. A refused pin returns + unscoped numbers in a MeasureAnswer with named `refusals`; hosts must publish that channel. + A pin never replaces `allowed_pids`, including when its member set is empty. """ topic_for(key) # raises for anything not admitted # The window must RESOLVE before any query runs — same refusal as `_query_parts`, checked @@ -630,10 +668,11 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None): raise _sem.ModelError( f"unresolvable window {window!r} for the {key!r} column: refusing rather than " f"defaulting to all time") + related = _related_scope(related, [key]) pool = _pool_of(allowed_pids) con = _sem._store_con() try: - totals = _totals(con, key, window, today, team_id) + totals = _totals(con, key, window, today, team_id, related) finally: con.close() scale = ADMITTED.get(key, {}).get('input_scale') or 1.0 @@ -641,7 +680,9 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None): nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal universe = pool if pool is not None else set(totals) # round(x, None) returns an int — exactly the pool's `_round` convention for money/counts. - return {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe} + # The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction. + return _related_answer( + {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}, related) #: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to @@ -766,19 +807,23 @@ def resolve_series_slid(key, window_spec, buckets, today, team_id=None, allowed_ return {'values': values, 'agg': agg_kind} -def resolve_tree(rules, today, team_id=None, allowed_pids=None): +def resolve_tree(rules, today, team_id=None, allowed_pids=None, related=None): """{rule_id: sorted pids} for every measure condition in a tree. Keyed by the rule's client-generated `id` so the browser can map an answer back to the condition that asked it — positional matching would silently re-associate every answer the moment a user deleted a condition. """ - out = {} + out = _relations.MeasureAnswer() if related is not None else {} for r in rules or []: rid = str(r.get('id') or '') if not rid: raise _sem.ModelError('a measure condition needs a stable id to carry its answer back') - out[rid] = sorted(resolve_rule(r, today, team_id, allowed_pids)) + matched = resolve_rule(r, today, team_id, allowed_pids, related) + out[rid] = sorted(matched) + if related is not None: + out.refusals.extend(dict(marker, rule=rid) + for marker in getattr(matched, 'refusals', ())) return out diff --git a/platform/model/topics/odoo_customers.yml b/platform/model/topics/odoo_customers.yml index ca9c8a7c094d5c8a3c1b51e715ba78da1fa554dd..e5c6d7a4be1740361e5dcc4a507eab55b4cb53fa 100644 --- a/platform/model/topics/odoo_customers.yml +++ b/platform/model/topics/odoo_customers.yml @@ -144,6 +144,16 @@ fields: type: url kind: data means: "A Google Maps link to this customer's address." + - key: lat + label: "Latitude" + type: text + kind: data + means: "Latitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + - key: lon + label: "Longitude" + type: text + kind: data + means: "Longitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." - key: customer_since label: "Customer since" type: date @@ -174,11 +184,6 @@ fields: type: int kind: data means: "How many days late this customer is running against their own usual ordering rhythm." - - key: est_missed - label: "Est. missed $" - type: currency - kind: data - means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - key: ar_open label: "AR current $" type: currency diff --git a/platform/modules/customer_data.py b/platform/modules/customer_data.py index 3fbba8be521fa16660fcd48ae4738b8af85cf2c7..654b62bd5681caedc9c5ce7ea3126574489fa21d 100644 --- a/platform/modules/customer_data.py +++ b/platform/modules/customer_data.py @@ -1,21 +1,16 @@ -"""Customer List — build, filter and save customer lists (replaces My Day, owner IA 2026-07-23). - -The page is a customer-list WORKBENCH in two parts: - 1. BUILDER — the whole scoped book, one row per customer with a consistent metric surface - (YTD / LY revenue, at-risk $, cadence, overdue days, estimated missed $ …): filter, sort, - SELECT customers and add them to a list. - 2. LISTS — saved lists whose membership is a visible, editable FORMULA (a rule set over the - metric columns) plus hand-picked members. 'Call list' and 'Win-back' ship as TEMPLATES — - the exact formulas the old My Day queues used — and the user can change or reset them. - -Composes already-validated customers-module functions, so the page's counts derive FROM the -lists it renders (rule 8b). ⚠ **This header used to end "no separate validate()" and W30-T33 -made that false** — see `validate()` below. The claim held only while every column composed an -already-reconciled customers-module function; the ADDRESS family does not, so it needed its own -tie. Persistence: HF store key 'customer_lists' -({username: {list_name: {rules, sort, members, note}}}); templates are virtual until edited. - -The user↔agent link (users.py 'agent' field) scopes the pool to that agent's book; users with +"""Customer data: a scoped metric pool and saved table workspaces. + + 1. POOL: one row per customer, composing sales, cadence, receivables and address data. + 2. VIEWS: ordinary saved table views hold filters, sorts, layout and selected members in + 'customer_table_workspace'. The old rule-formula list readers and shipped list templates + were retired in wave 57 under R13. The 'customer_lists' bucket is retained without reads + or writes here so existing tenant data is preserved. + +The pool composes already-validated customers-module functions. Its own validate() also +reconciles the address family independently. The daily digest still uses queues(), which calls +the customers module's contact recommendations and at-risk calculations directly. + +The user-to-agent link (users.py 'agent' field) scopes the pool to that agent's book; users with no link (owner/CFO/admin) see the whole book. """ import re @@ -31,6 +26,7 @@ import modules.map as map_mod import modules.sales as S LIMIT = 500 # honest builder display cap — always shown with the full count, never silent +# Rule-formula readers retired in wave 57 under R13; keep the bucket so no tenant data is dropped. KEY = 'customer_lists' TABLE_KEY = 'customer_table_workspace' @@ -104,7 +100,6 @@ FIELDS = { 'days_since': ('Days since last order', 'num'), 'typical_gap_days': ('Typical gap (days)', 'num'), 'overdue_days': ('Overdue vs cadence (days)', 'num'), - 'est_missed': ('Est. missed $', 'num'), } NUM_OPS = ('>=', '<=', '>', '<', '=') TEXT_OPS = ('contains', '=') @@ -289,14 +284,14 @@ def pool(agent_name=None, team_id=None): buyers and anyone with an order in the 24-month cadence window — so lapsed/win-back accounts are in the pool, not just YTD actives. Reuses the customers module's validated building blocks (_cust_rev / _cadence_bulk / _partner_attrs) so every metric matches the - Customers page definitions; est_missed is the Call-list formula (min(cycles missed, 3) × AOV).""" + Customers page definitions.""" return _pool_build(agent_name, team_id, limit=None)['rows'] def pool_first(agent_name=None, team_id=None, limit=100): """Wave-7 W1 (C1 as amended): the cold-load FAST SLICE — the pool's first `limit` rows in its own default order + the true total, built from the CHEAP families only (revenue ×3, - cadence, status, est_missed, partner attributes ≈ 12s measured). The AR / product-mix / + cadence, status, partner attributes ≈ 12s measured). The AR / product-mix / salesperson / coords columns stay BLANK until the full build swaps in: `_mix_attrs` alone is a whole-book 20s read_group whose domain must never be pid-narrowed (502 — see its docstring), so "all columns on 100 rows" measured 44.6s vs 45.1s full — no win. A partial @@ -603,7 +598,6 @@ def _pool_build(agent_name, team_id, limit, fast=False): gap = c.get('typical_gap_days') overdue = c.get('overdue_days') aov = c.get('aov', 0.0) - est = (min(overdue / gap, 3.0) * aov) if (gap and overdue and overdue > 0) else 0.0 status = ('New' if rev > 0 and ly <= 0 else 'Lost' if ly > 0 and rev <= 0 else 'Declining' if 0 < rev < ly else @@ -671,7 +665,6 @@ def _pool_build(agent_name, team_id, limit, fast=False): 'last_order': c.get('last_order', ''), 'days_since': c.get('days_since'), 'typical_gap_days': gap, 'overdue_days': overdue, - 'est_missed': est, }) for r in rows: r['odoo_status'] = 'Active' @@ -760,7 +753,6 @@ def _reconcile_ledger(rows, total, agent_name, team_id): 'revenue_ytd': 0.0, 'revenue_ly': 0.0, 'yoy_pct': None, 'at_risk': 0.0, 'ltm_rev': 0.0, 'orders_24m': 0, 'aov': 0.0, 'last_order': '', 'days_since': None, 'typical_gap_days': None, 'overdue_days': None, - 'est_missed': 0.0, }) if store.available(): try: @@ -770,86 +762,6 @@ def _reconcile_ledger(rows, total, agent_name, team_id): return rows + revived, total + len(revived) -# ------------------------------------------------------------------ the rule engine -# A list's formula = [{'field','op','value'}, ...] over FIELDS + a 'sort' key ('-x' = desc). -# The two templates ARE the old My Day queue formulas, expressed as visible rules. -# -# 'note' is a SEED, not the source of truth (owner 2026-07-25). It is what a list reads -# BEFORE anyone edits it; the moment a user saves a description the stored one wins, and -# an empty stored description stays empty — the template must never resurrect. That -# precedence lives in aios_grid.views_from_defs (a saved view replaces its template-derived -# one wholesale) and is pinned by _qa_grid_view_note.py. Edit the prose here only to change -# what a NEW user starts with. -TEMPLATES = { - 'Call list': { - 'rules': [{'field': 'orders_24m', 'op': '>=', 'value': 3}, - {'field': 'overdue_days', 'op': '>', 'value': 0}, - {'field': 'days_since', 'op': '<=', 'value': 365}], - 'sort': '-est_missed', - 'note': 'Customers overdue against their OWN reorder cadence (3+ orders, a real gap ' - 'pattern, ordered within 365d), ranked by estimated missed revenue.'}, - 'Win-back': { - 'rules': [{'field': 'revenue_ly', 'op': '>=', 'value': 2000}, - {'field': 'at_risk', 'op': '>', 'value': 0}], - 'sort': '-at_risk', - 'note': 'Bought materially last year (over $2,000), down or gone this year, ranked by ' - 'dollars at risk.'}, -} - - -def _match(row, rule): - """One rule against one row. None/missing numeric values fail every numeric comparison - (a customer with no cadence is never 'overdue'); text ops are case-insensitive.""" - field, op, val = rule.get('field'), rule.get('op'), rule.get('value') - if field not in FIELDS: - return True # unknown field → rule is inert, not a crash - v = row.get(field) - if FIELDS[field][1] == 'text': - s, q = str(v or '').lower(), str(val or '').lower() - return q in s if op == 'contains' else s == q - try: - v, val = float(v), float(val) - except (TypeError, ValueError): - return False - return {'>=': v >= val, '<=': v <= val, '>': v > val, '<': v < val, - '=': v == val}.get(op, False) - - -def apply_rules(rows, rules, sort=None): - """Filter the pool through a formula (AND of all rules) and apply the list's sort.""" - out = [r for r in rows if all(_match(r, ru) for ru in (rules or []))] - if sort: - key = sort.lstrip('-') - out.sort(key=lambda r: (r.get(key) is None, - -(r.get(key) or 0) if sort.startswith('-') else (r.get(key) or 0)) - if FIELDS.get(key, ('', 'num'))[1] == 'num' - else str(r.get(key) or '').lower(), reverse=False) - return out - - -# ------------------------------------------------------------------ persistence (HF store) -def saved_lists(username): - """{list_name: {'rules','sort','members','note'}} for one user ({} when none / store down).""" - try: - return (store.get(KEY) or {}).get(username, {}) or {} - except Exception: - return {} - - -def save_list(username, name, definition): - def _up(d): - d.setdefault(username, {})[name] = definition - return d - store.update(KEY, _up) - - -def delete_list(username, name): - def _up(d): - (d.get(username) or {}).pop(name, None) - return d - store.update(KEY, _up) - - # ------------------------------------------------------------------ Airtable-style table workspace # This is intentionally separate from `customer_lists`: a LIST owns membership/formula # semantics, while a VIEW owns presentation/query state (filters, multi-sort, grouping, diff --git a/platform/modules/customers.py b/platform/modules/customers.py index ac7489d3c918568a1f1163568dc342534c9d1efc..af1f5fb6276962c132fe27aa6ec4ceea6aa3c3e1 100644 --- a/platform/modules/customers.py +++ b/platform/modules/customers.py @@ -527,7 +527,7 @@ def _cadence_bulk(t=None, team_id=None, months=24, agent_pids=None): def contact_recommendations(t=None, team_id=None, limit=40, months=24, agent_pids=None): """Prioritised follow-up list: customers overdue against their OWN purchase cadence, ranked by - estimated missed revenue (how far past due × their average order value). Reps' call list.""" + how far past their own reorder cadence they are. Reps' call list.""" t = t or P.today() cad = _cadence_bulk(t, team_id, months, agent_pids=agent_pids) attrs = _partner_attrs(list(cad)) @@ -546,17 +546,13 @@ def contact_recommendations(t=None, team_id=None, limit=40, months=24, agent_pid if c['days_since'] > 365: continue # long-dead → not a cadence follow-up ltm_rev = rev_map.get(pid, {}).get('rev', 0.0) - cycles_missed = overdue / gap # how many reorder cycles past due - # cap at 3 cycles so a long-churned account (belongs in win-back) doesn't dominate the - # "call them now" list; this rewards valuable regulars who are RECENTLY due. - est_missed = min(cycles_missed, 3.0) * c['aov'] rows.append({ 'pid': pid, 'customer': rev_map.get(pid, {}).get('name', '?'), 'last_order': c['last_order'], 'typical_gap_days': gap, 'overdue_days': overdue, - 'ltm_rev': ltm_rev, 'orders': c['n_orders'], 'est_missed': est_missed, + 'ltm_rev': ltm_rev, 'orders': c['n_orders'], 'agent': (attrs.get(pid) or {}).get('agent', '(none)'), }) - rows.sort(key=lambda x: -x['est_missed']) + rows.sort(key=lambda x: -(x['overdue_days'] or 0)) return rows[:limit] diff --git a/platform/modules/digest.py b/platform/modules/digest.py index 23ed2b3b702ca1743a654115dab9d9a79a77ddef..367a659079cb42f6d90cf833d42c3095079f6aaf 100644 --- a/platform/modules/digest.py +++ b/platform/modules/digest.py @@ -106,8 +106,7 @@ def build(user, t=None): rows.append(_sec('Call today — overdue vs their own cadence')) for r in calls: href = links.deeplink('customer_data', 'customer', r['pid'], src='digest') - meta = (f"{int(r.get('overdue_days') or 0)}d past cycle · " - f"est. missed {_m(r.get('est_missed'))}") + meta = f"{int(r.get('overdue_days') or 0)}d past cycle" rows.append(_line(r.get('customer', '?'), href, meta)) if risk: rows.append(_sec('Win-back — down vs last year')) diff --git a/platform/modules/outreach.py b/platform/modules/outreach.py index 573e01d6a06ccca3acede7d3644c1230a84f0e3f..cefaa5d142b2b5269f386ca2b2fb20de14914ba7 100644 --- a/platform/modules/outreach.py +++ b/platform/modules/outreach.py @@ -193,7 +193,7 @@ def build_audience(source, team_id=None, agent=None, limit=500): if source == 'reorder': rows = cust.contact_recommendations(team_id=team_id, limit=limit, agent_pids=pids_scope) return [{'pid': r['pid'], 'customer': r['customer'], 'agent': r.get('agent', ''), - 'metric': r.get('est_missed', 0.0), 'metric_label': 'est. missed $'} + 'metric': r.get('overdue_days', 0), 'metric_label': 'days past cadence'} for r in rows] if source == 'winback': rows = cust.at_risk(team_id=team_id, limit=limit, agent_pids=pids_scope) diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json index 5516d42f0966db62b821a36b72f6cd2165b61839..37d5b9ee33a9685fd715ba724ad40f2fdfd8a798 100644 --- a/web/public/sample_customers.json +++ b/web/public/sample_customers.json @@ -116,6 +116,22 @@ "default": true, "description": "A Google Maps link to this customer's address." }, + { + "key": "lat", + "label": "Latitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Latitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, + { + "key": "lon", + "label": "Longitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Longitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, { "key": "customer_since", "label": "Customer since", @@ -164,17 +180,6 @@ "default": true, "description": "How many days late this customer is running against their own usual ordering rhythm." }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, { "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", "key": "ar_open", @@ -323,7 +328,6 @@ "city": "Charlottesville", "state": "Virginia (US)", "last_order": "2026-07-21", - "est_missed": 0, "notes": "", "country": "United States", "zip": "02720", @@ -341,8 +345,8 @@ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", "days_to_pay": 34, "_created": "2023-01-15 09:10:00", - "lat": 25.7617, - "lon": -80.1918, + "lat": 38.029306, + "lon": -78.476678, "odoo_status": "Archived", "dba": "Royal", "ar_outstanding": 0 @@ -355,7 +359,6 @@ "city": "Portland", "state": "Oregon (US)", "last_order": "2026-07-19", - "est_missed": 0, "notes": "Expanding to a second storefront.", "country": "United States", "zip": "77041", @@ -373,8 +376,8 @@ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", "days_to_pay": null, "_created": "2023-02-15 09:11:00", - "lat": 27.9506, - "lon": -82.4572, + "lat": 45.515232, + "lon": -122.678385, "odoo_status": "Active", "dba": "Fisch", "ar_outstanding": 1240.5 @@ -387,7 +390,6 @@ "city": "Kansas City", "state": "Missouri (US)", "last_order": "2026-07-17", - "est_missed": 0, "notes": "", "country": "United States", "zip": "11219", @@ -405,8 +407,8 @@ "top_sku": "SATIN RIBBON 2IN", "days_to_pay": 61, "_created": "2023-03-15 09:12:00", - "lat": 28.5384, - "lon": -81.3789, + "lat": 39.099727, + "lon": -94.578567, "odoo_status": "Active", "dba": "Both", "ar_outstanding": 5120.25 @@ -419,7 +421,6 @@ "city": "Savannah", "state": "Georgia (US)", "last_order": "2026-05-30", - "est_missed": 41200, "notes": "Switched some volume to a local grower.", "country": "United States", "zip": "07649", @@ -437,8 +438,8 @@ "top_sku": "(none)", "days_to_pay": 12, "_created": "2023-04-15 09:13:00", - "lat": 30.3322, - "lon": -81.6557, + "lat": 32.083541, + "lon": -81.099834, "odoo_status": "Active", "dba": "Royal", "ar_outstanding": 9240 @@ -451,7 +452,6 @@ "city": "Seattle", "state": "Washington (US)", "last_order": "2026-07-22", - "est_missed": 0, "notes": "Top-10 account.", "country": "United States", "zip": "33125", @@ -469,8 +469,8 @@ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", "days_to_pay": 34, "_created": "2023-05-15 09:14:00", - "lat": 26.1224, - "lon": -80.1373, + "lat": 47.606209, + "lon": -122.332071, "odoo_status": "Active", "dba": "", "ar_outstanding": 0 @@ -483,7 +483,6 @@ "city": "Asheville", "state": "North Carolina (US)", "last_order": "2026-02-11", - "est_missed": 52400, "notes": "No spring order this year.", "country": "United States", "zip": "90210", @@ -501,8 +500,8 @@ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", "days_to_pay": null, "_created": "2023-06-15 09:15:00", - "lat": 27.3364, - "lon": -82.5307, + "lat": 35.595058, + "lon": -82.551487, "odoo_status": "Active", "dba": "Fisch", "ar_outstanding": 1240.5 @@ -515,7 +514,6 @@ "city": "Austin", "state": "Texas (US)", "last_order": "2025-11-04", - "est_missed": 78300, "notes": "Went with a competitor on freight terms.", "country": "United States", "zip": "08701", @@ -533,8 +531,8 @@ "top_sku": "SATIN RIBBON 2IN", "days_to_pay": 61, "_created": "2023-07-15 09:16:00", - "lat": null, - "lon": null, + "lat": 30.267153, + "lon": -97.743061, "odoo_status": "Active", "dba": "Both", "ar_outstanding": 5120.25 @@ -547,7 +545,6 @@ "city": "Denver", "state": "Colorado (US)", "last_order": "2026-07-14", - "est_missed": 0, "notes": "First order in April.", "country": "United States", "zip": "60614", @@ -565,8 +562,8 @@ "top_sku": "(none)", "days_to_pay": 12, "_created": "2023-08-15 09:17:00", - "lat": 33.749, - "lon": -84.388, + "lat": 39.739236, + "lon": -104.990251, "odoo_status": "Active", "dba": "Royal", "ar_outstanding": 9240 diff --git a/web/src/automation/AutomationSurface.tsx b/web/src/automation/AutomationSurface.tsx index 73e8cff89df7d877d81455122a7c24a8f0b1f03b..fb259b5fbf47d66bc868b3a2d64254f8dc9377a2 100644 --- a/web/src/automation/AutomationSurface.tsx +++ b/web/src/automation/AutomationSurface.tsx @@ -41,6 +41,7 @@ import { import type { VoiceAgent } from "./voiceAgentTypes"; import "../styles/wave44/voice-agent.css"; import "../styles/wave45/agent-operator.css"; +import "../styles/wave57/agents-rail.css"; /** * ⭐⭐ WAVE 30 · T21 (owner items 4 + 5) — THE EDITOR STOPS SHIPPING INSIDE THE LIST. @@ -248,6 +249,7 @@ export default function AutomationSurface() { const [activeId, setActiveId] = useState(""); const [activeVoiceId, setActiveVoiceId] = useState(""); const [railShut, setRailShut] = useState(false); + const [railQuery, setRailQuery] = useState(""); const [busy, setBusy] = useState(""); /** * ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to @@ -542,6 +544,9 @@ export default function AutomationSurface() { */ const fieldAgents = items.filter((agent) => agent.system === "field_agent"); const active = items.find((a) => a.id === activeId) || null; + const railNeedle = railQuery.trim().toLowerCase(); + const railMatches = (label: string) => + !railNeedle || label.toLowerCase().includes(railNeedle); const activeVoice = voiceAgents.find((agent) => agent.automationId === activeVoiceId) || null; /** @@ -641,38 +646,65 @@ export default function AutomationSurface() { -
+
+
+ + setRailQuery(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Escape") setRailQuery(""); + }} + /> +
+
+ {/* Keep both find guards inside the callbacks: verify_automation_ui.py's W44 + check reads the literal voiceAgents.map. Chaining a filter ahead of it + takes a green gate red even though it renders the same rows. */} {items.map((a) => { + if (!railMatches(a.name)) return null; return (
{ + if (!railMatches(agent.displayName)) return null; const isActive = agent.lifecycle === "active"; return (
{ } } +async function disableDatabase( + tableKey: string +): Promise<{ ok: true } | { ok: false; message: string }> { + try { + const res = await fetch( + `${API_V1}/connectors/odoo/databases/${encodeURIComponent(tableKey)}/disable`, + { method: "POST", credentials: CREDENTIALS } + ); + const payload = (await res.json().catch(() => null)) as + | { error?: { message?: string } } + | null; + if (res.ok) return { ok: true }; + return { + ok: false, + message: payload?.error?.message || `The server answered ${res.status}.`, + }; + } catch { + return { ok: false, message: "Cannot reach the server." }; + } +} + function sentenceList(values: string[]): string { if (values.length < 2) return values[0] || ""; if (values.length === 2) return `${values[0]} and ${values[1]}`; @@ -473,8 +495,9 @@ function DatabasePanel({ * remounted, and so this whole tab still paints under `renderToStaticMarkup`. * * ⛔ NO CONTROL AT ALL FOR A READER WHO MAY NOT RENAME (`canRename` is the server's own `may_open` - * answer). When `exists` is false, that is a different fact: the table is still waiting for a - * sync, so the cell says why the control is absent instead of presenting a permission refusal. + * answer). Enabling one Odoo database materialises it in the same request (W56-T31, + * `routes_connectors.py::_enable_odoo_database`), so there is no waiting state left to describe. + * A reader who may not rename gets no control at all, rather than a sentence about a wait. */ function NameCell({ database, @@ -516,8 +539,6 @@ function NameCell({ > {database.override ? "Change" : "Rename"} - ) : database.enabled && !database.exists ? ( - Available after the first sync. ) : null}
); @@ -589,6 +610,7 @@ const DatabasesTab = forwardRef reveal?.key ? { [reveal.key]: true } : {} ); const [busy, setBusy] = useState(""); + const [confirmOff, setConfirmOff] = useState(null); const [refusals, setRefusals] = useState>({}); const [autoEnableResults, setAutoEnableResults] = useState>( () => reveal?.key && reveal.status ? { [reveal.key]: reveal.status } : {} @@ -646,11 +669,22 @@ const DatabasesTab = forwardRef { + setPending((current) => { + const tables = { ...current.tables }; + if (next === database.enabled) delete tables[database.key]; + else tables[database.key] = next; + return { ...current, tables }; + }); + }, + [] + ); + /** - * ⛔⛔ R23's OFF ENFORCEMENT, AND TODAY IT LIVES HERE. Turning a database off keeps using - * `PUT /admin/connectors/odoo/config`, which knows nothing about dependents, so the ASK has to - * happen before the change enters the pending diff. Turning one on is queued for the - * materialising door; mixing the two would bypass the only dependents check on the off path. + * R23: the dependents check precedes staging because the disable door only writes config. + * W57-T31 / R10 / C9: Odoo also warns about the lock before accepting an off change. + * Provider-managed enablement keeps its existing staging path. * * ⛔ AND A FAILED CHECK IS NOT A PASSED ONE. Any non-answer leaves the table on and prints the * cause; proceeding would turn "REFUSED" into "usually refused". @@ -678,16 +712,16 @@ const DatabasesTab = forwardRef ({ ...current, [database.key]: answer.refusal })); return; } + if (!usesProviderEnablement) { + setBusy(""); + setConfirmOff(database); + return; + } } setBusy(""); - setPending((current) => { - const tables = { ...current.tables }; - if (next === database.enabled) delete tables[database.key]; - else tables[database.key] = next; - return { ...current, tables }; - }); + stageTable(database, next); }, - [] + [stageTable, usesProviderEnablement] ); const flipField = useCallback( @@ -776,10 +810,9 @@ const DatabasesTab = forwardRef + +

{confirmOff?.label} stops being materialised from Odoo when you save this change.

+

+ Switching a database off starts a 7 day lock. It cannot be switched back on until + the week is up, and nobody can shorten it. +

+ + } + onConfirm={() => { + if (confirmOff) stageTable(confirmOff, false); + setConfirmOff(null); + }} + onCancel={() => setConfirmOff(null)} + />
); }); diff --git a/web/src/customer-grid/CustomerGrid.tsx b/web/src/customer-grid/CustomerGrid.tsx index b25a7ac61f5fd0819091b8a85a84e581eb8c716c..8b619668066dad9d35e29ad62fb4a013f371dc49 100644 --- a/web/src/customer-grid/CustomerGrid.tsx +++ b/web/src/customer-grid/CustomerGrid.tsx @@ -1699,6 +1699,8 @@ interface CustomerGridProps { * and edit the reason the rows are here. Only meaningful with `cellView`. */ cellFilter?: FilterTree; + /** The Sheet cell's ordered visible field keys. Only meaningful with `cellView`. */ + cellFields?: string[]; /** Report a filter edit from the cell View data surface; its Sheet caller owns settling. */ onCellFilterChange?: (tree: FilterTree) => void; /** @@ -1759,12 +1761,15 @@ interface LinkGridModalProps { cellView?: boolean; /** W56-T25: the Sheet cell's own filter, shown in the grid's standard filter control. */ cellFilter?: FilterTree; + /** W57-T22: the Sheet cell's ordered visible field keys, shown in the grid's columns. */ + cellFields?: string[]; /** W56-T25: report a grid filter edit so the Sheet caller can settle and persist it. */ onCellFilterChange?: (tree: FilterTree) => void; /** W56-T24: chrome the caller owns, in the header, right of the title and left of Close. */ headerActions?: ReactNode; /** W56-T24: a statement under the rows. Only rendered on a non-editable modal. */ footer?: ReactNode; + onOpenRecord?: CustomerGridProps["onOpenRecord"]; onSave?: (recordIds: number[]) => void; onClose: () => void; } @@ -1774,8 +1779,8 @@ interface LinkGridModalProps { // the two surfaces cannot drift into two popups that behave almost alike. W56-T24 adds optional // chrome slots and a `cellView` modifier while leaving existing callers and the focus trap intact. export function LinkGridModal({ label, table, recordIds, editable = false, single = false, - excludeRecordId, cellView = false, cellFilter, onCellFilterChange, headerActions, footer, - onSave, onClose }: LinkGridModalProps) { + excludeRecordId, cellView = false, cellFilter, cellFields, onCellFilterChange, headerActions, footer, + onOpenRecord, onSave, onClose }: LinkGridModalProps) { const panelRef = useRef(null); const [selectedIds, setSelectedIds] = useState(() => [...recordIds]); const changeSelectedIds = useCallback((ids: number[]) => { @@ -1831,12 +1836,14 @@ export function LinkGridModal({ label, table, recordIds, editable = false, singl embedded cellView={cellView} cellFilter={cellFilter} + cellFields={cellFields} onCellFilterChange={onCellFilterChange} embeddedRecordIds={editable ? undefined : recordIds} embeddedExcludeRecordId={editable ? excludeRecordId : undefined} embeddedSelectable={editable} embeddedSelectedIds={selectedIds} onEmbeddedSelectionChange={changeSelectedIds} + onOpenRecord={onOpenRecord} />
{editable ? ( @@ -1983,6 +1990,7 @@ function CustomerGridSurface({ related, cellView = false, cellFilter, + cellFields, onCellFilterChange, onOpenRecord, relatedViewId, @@ -2620,6 +2628,7 @@ function CustomerGridSurface({ // A parent may rebuild an equivalent tree on every render. Depend on its value, not its identity, // so the initialization effect cannot re-seed over an edit still settling in this surface. const cellFilterKey = useMemo(() => JSON.stringify(cellFilter ?? null), [cellFilter]); + const cellFieldsKey = useMemo(() => JSON.stringify(cellFields ?? []), [cellFields]); // Initialize once per permission/data scope. Host state wins; local state // fills only missing objects and keeps the standalone path useful. @@ -2769,6 +2778,10 @@ function CustomerGridSurface({ ...active.config, filters: cellFilter.nodes ?? [], filterConj: cellFilter.conj ?? "and", + ...(cellFields?.length ? { + order: [...cellFields, ...active.config.order.filter((key) => !cellFields.includes(key))], + visible: cellFields, + } : {}), } : active.config); setWorkspaceReady(true); initializedKey.current = storageKey; @@ -2810,7 +2823,18 @@ function CustomerGridSurface({ reemittedRef.current = { ...(local?.reemitted ?? {}) }; } }, [payload, payloadFields, storageKey, hostWorkspace, scope, isQueryPreview, queryBinding, - cellView, cellFilterKey]); + cellView, cellFilterKey, cellFieldsKey]); + + // Initialization runs once per scope. Keep the columns current after a cell filter settles, + // retaining hidden keys in the order so the reader can show them again. + useEffect(() => { + if (!workspaceReady || !cellView || !cellFields?.length) return; + setConfig((current) => ({ + ...current, + order: [...cellFields, ...current.order.filter((key) => !cellFields.includes(key))], + visible: cellFields, + })); + }, [workspaceReady, cellView, cellFieldsKey]); /** * ⭐ W36-T03 — the view-preference stamp, and it is its OWN effect on purpose. diff --git a/web/src/customer-grid/RecordDetail.tsx b/web/src/customer-grid/RecordDetail.tsx index 19f63b9e70bef2e39fc912bda55fb212ea7ddb69..52127f4a4052f1f68da4a7ed2ec3f358924709ef 100644 --- a/web/src/customer-grid/RecordDetail.tsx +++ b/web/src/customer-grid/RecordDetail.tsx @@ -21,6 +21,9 @@ import { BodyPortal, useOverlayLayer } from "./OverlaySurface"; * who never opens a related tab never downloads a second copy of anything. */ const RelatedGrid = lazy(() => import("./CustomerGrid")); +const LinkedRecordGrid = lazy(() => import("./CustomerGrid").then((module) => ({ + default: module.LinkGridModal, +}))); import { automationDetail, automationState, @@ -59,7 +62,7 @@ import type { CustomerDoc, DisplayMode, Field, RelatedEntry, RelatedPin, Related /* ⭐ W53-T22 adds a FOURTH and a FIFTH, and for the same reason: `fetchTableWindow` + `WINDOW_ROWS` are the one windowed read this product owns, and the sibling set a pushed frame's chevrons page is that read with the frame's own pin on it. */ -import { fetchLinkTargets, fetchRelated, fetchTableWindow, fetchTopicRows, probeRelatedRead, +import { fetchLinkTargets, fetchRecordMeasureValues, fetchRelated, fetchTableWindow, fetchTopicRows, probeRelatedRead, readThroughTables, WINDOW_ROWS } from "./apiBridge"; import { API_V1, CREDENTIALS, UNAUTHORIZED_EVENT, checkTenant, signal } from "../apiContract"; @@ -723,6 +726,10 @@ interface RecordFrame { * chevrons go disabled and the caption says nothing, rather than paging a guess. */ pin: RelatedPin | null; + /** `pin` is the parent's pin for paging siblings; `originPin` is the record every + * measure on this frame must be counted within. It never changes the stored fields. */ + originPin: RelatedPin | null; + originReason: string; } /** @@ -989,6 +996,10 @@ export default function RecordDetail({ */ const [stack, setStack] = useState([]); const frame = stack.length > 0 ? stack[stack.length - 1] : null; + const [frameMeasures, setFrameMeasures] = useState<{ + frame: RecordFrame; values: Partial | null; reason: string; + } | null>(null); + const [openLinkField, setOpenLinkField] = useState(null); const navigationSequence = useRef(0); const [recordNavigation, setRecordNavigation] = useState(null); const beginRecordNavigation = useCallback((fromPid: number, targetPid?: number) => { @@ -1007,9 +1018,44 @@ export default function RecordDetail({ * way in, which is a filter that matches everything. */ const fields = frame ? frame.fields : baseFields; - const record = frame ? frame.record : baseRecord; + // The source row stays intact while enrichment is pending or fails. Only Metric keys from + // this exact frame's read may overlay it; a late response cannot paint another frame's values. + const record = useMemo(() => (frame + ? { ...frame.record, ...(frameMeasures?.frame === frame ? frameMeasures.values : null) } + : baseRecord), [baseRecord, frame, frameMeasures]); + useEffect(() => { + if (!frame?.originPin || !frame.fields.some((field) => field.measure)) return; + const asked = frame; + const pin = frame.originPin; + let live = true; + void (async () => { + const through = await readThroughTables(); + if (!live) return; + if (!through.has(asked.table)) { + setFrameMeasures({ frame: asked, values: null, + reason: "Metric values are unscoped because filtering support for this database could not be confirmed." }); + return; + } + const result = await fetchRecordMeasureValues( + asked.table, asked.pid, asked.fields, pin, () => live + ); + if (live) setFrameMeasures({ frame: asked, ...result }); + })().catch(() => { + if (live) setFrameMeasures({ frame: asked, values: null, + reason: "Metric values are unscoped because the related record read failed." }); + }); + return () => { live = false; }; + }, [frame]); + const metricScopeNote = frame && fields.some((field) => field.measure) + ? !frame.originPin + ? frame.originReason + : frameMeasures?.frame === frame + ? frameMeasures.reason + : "Metric values are unscoped while the values for the linked record load." + : ""; useEffect(() => { setStack((prev) => (prev.length ? [] : prev)); + setOpenLinkField(null); }, [baseRecord.pid, scope]); /** * ⭐⭐ W53-T22 (R4) — THE ONE PLACE THE STACK IS SHORTENED, and it is one place on purpose. @@ -1474,10 +1520,37 @@ export default function RecordDetail({ * upwards would be the same stale read this ticket exists to delete, one indirection along. */ const pushFrame = useCallback( - (pid: number, table: string, row?: Row, rowFields?: Field[]) => { + (pid: number, table: string, row?: Row, rowFields?: Field[], linkField?: Field) => { if (!row || !rowFields || rowFields.length === 0) return; const first = rowFields[0]; const named = first ? String(row[first.key] ?? "") : ""; + // Carry the parent's origin down the stack unless this hop establishes a nearer origin. + // An explicitly unnamed or unserved link breaks that inheritance: it cannot carry a pin. + let originPin = frame?.originPin ?? null; + let originReason = frame?.originReason + || "Metric values are unscoped because this link does not declare a relation path."; + if (linkField) { + // Match the actual Link field's declaration to a registry entry, never a target-only + // guess or an assembled path. Ambiguous declarations remain unscoped. + const link = linkField.link; + const matches = entries.filter((entry) => entry.target === table && link?.table === table + && (entry.via.field === linkField.key + || (link.on && entry.via.on === link.on && entry.via.from === link.from))); + const edge = matches.length === 1 ? matches[0] : null; + originPin = edge?.served ? { + table: recordTable, + record: Number(record.pid), + via: edge.hops === 2 ? edge.path : edge.via, + } : null; + originReason = edge && !edge.served + ? "Metric values are unscoped because this link cannot filter the destination database yet." + : "Metric values are unscoped because this link field does not declare one identifiable relation path."; + } else if (activeRelated?.entry.target === table && relatedPin) { + originPin = activeRelated.entry.served + ? { table: relatedPin.table, record: relatedPin.record, via: relatedPin.via } + : null; + originReason = "Metric values are unscoped because this link cannot filter the destination database yet."; + } setStack((prev) => [...prev, { table, pid, @@ -1490,9 +1563,11 @@ export default function RecordDetail({ pin: relatedPin ? { table: relatedPin.table, record: relatedPin.record, via: relatedPin.via } : null, + originPin, + originReason, }]); }, - [relatedPin] + [activeRelated, entries, frame, record.pid, recordTable, relatedPin] ); /* ═══════════════════════════════════════════════════════════════════════════════════════ @@ -1536,7 +1611,7 @@ export default function RecordDetail({ // ⛔ Not read through ⇒ no request at all. See the guard note above. if (!live || !through.has(asked.table)) return; const payload = await fetchTableWindow(asked.table, - { offset: 0, limit: WINDOW_ROWS, related: asked.pin }); + { offset: 0, limit: WINDOW_ROWS, related: asked.pin }, () => {}); if (!live || !payload) return; setSiblings({ key: framePinKey, rows: payload.rows, fields: payload.fields }); })(); @@ -1592,6 +1667,8 @@ export default function RecordDetail({ record: next, fields: siblings.fields, pin: frame.pin, + originPin: frame.originPin, + originReason: frame.originReason, }])); }, [beginRecordNavigation, frame, framePinKey, siblingIndex, siblingRows, siblings.fields] @@ -2074,6 +2151,7 @@ export default function RecordDetail({ // thing the viewer's parse-on-save exists to prevent, sitting one panel away from it. // It joins the constrained-control family: preview + "Open", and the viewer owns the edit. field.type !== "json" && + field.type !== "link" && field.type !== "pct"; const isControl = editable && !isEdit; const cls = @@ -2094,7 +2172,21 @@ export default function RecordDetail({ (dragKey === field.key ? " is-dragging" : ""); let slot: ReactNode; - if (isEdit) { + if (field.type === "link" && field.link) { + slot = ( + + {formatDisplay(field, record[field.key]) || "No linked records"} + + + ); + } else if (isEdit) { const raw = String(record[field.key] ?? ""); const href = actionHref(field, raw); slot = ( @@ -3087,6 +3179,11 @@ export default function RecordDetail({ <> {/* C-LAYOUT — ONE list, this user's order, every type interleaved. */}
+ {metricScopeNote && ( +

+ {metricScopeNote} +

+ )} {orderedKeys.map((key, index) => { const field = fieldByKey.get(key); return field ? renderRow(field, index) : null; @@ -3145,6 +3242,24 @@ export default function RecordDetail({ {/* ⭐ Wave-23 C7 — the document viewer, INSIDE this portal and therefore stacked above this modal. `useOverlayLayer`'s stack is what makes that work: it registers on top, so Escape closes the viewer and leaves the record open rather than dismissing both. */} + {openLinkField?.link && ( + + onNotesCommit(openLinkField.key, ids.join(","))} + onOpenRecord={(pid, table, row, rowFields) => { + pushFrame(pid, table, row, rowFields, openLinkField); + setOpenLinkField(null); + }} + onClose={() => setOpenLinkField(null)} + /> + + )} {jsonKey && (() => { const field = fields.find((f) => f.key === jsonKey); if (!field) return null; diff --git a/web/src/customer-grid/apiBridge.ts b/web/src/customer-grid/apiBridge.ts index aa83bf7e4e3b040a81f168f71d1ef8caf4065d30..4021ebd3b1148128d64b5817a5364c3c4b6eba10 100644 --- a/web/src/customer-grid/apiBridge.ts +++ b/web/src/customer-grid/apiBridge.ts @@ -496,12 +496,15 @@ export function readThroughTables(): Promise> { return readThroughMemo; } -/** `GET /api/v1/odoo-tables//rows?…` — one window, mapped. Null on any failure, having - * raised the matching signal (a null is a failure, never an empty table). */ +/** One window, mapped. A failure returns null and reports through the caller's handler or + * the default data-error signal; null is never an empty table. */ export async function fetchTableWindow( tableKey: string, - req: RowWindowRequest + req: RowWindowRequest, + onFailure?: (reason: string) => void ): Promise { + // Optional record enrichment reports its failure inline without replacing the open drawer. + const fail = onFailure ?? ((reason: string) => signal(DATA_ERROR_EVENT, reason)); const path = windowRowsPath(tableKey, req); const cached = rowsCache.get(path); if (cached && Date.now() - cached.at < CUSTOMERS_FRESH_MS) return cached.payload; @@ -509,7 +512,7 @@ export async function fetchTableWindow( try { res = await fetch(`${API_V1}/${path}`, { credentials: CREDENTIALS }); } catch { - signal(DATA_ERROR_EVENT, "Cannot reach the server."); + fail("Cannot reach the server."); return null; } if (handledSession(res)) return null; @@ -518,18 +521,59 @@ export async function fetchTableWindow( // condition has no SQL form and what to use instead; "The server answered 400." explains // nothing, which is the defect `refusalMessage` was extracted for one wave ago. const body = await readJson(res); - signal(DATA_ERROR_EVENT, refusalMessage(body, res.status)); + fail(refusalMessage(body, res.status)); return null; } const payload = windowPayload(await readJson(res)); if (!payload) { - signal(DATA_ERROR_EVENT, "The server sent an unreadable payload."); + fail("The server sent an unreadable payload."); return null; } rowsCache.set(path, { at: Date.now(), payload }); return payload; } +/** C10: find this exact record within the named pin, then return only its Metric values. + * No pid filter: undeclared filter keys can be dropped by the rows door. Each page uses the + * existing window path/cache, including `related`, and stops when the drawer leaves this frame. + */ +export async function fetchRecordMeasureValues( + tableKey: string, + pid: number, + fields: Field[], + related: RelatedPin, + isCurrent: () => boolean +): Promise<{ values: Partial | null; reason: string }> { + const measureKeys = fields.filter((field) => field.measure).map((field) => field.key); + let offset = 0; + while (isCurrent()) { + let reason = "Metric values are unscoped because the related record read failed."; + const payload = await fetchTableWindow(tableKey, { offset, limit: WINDOW_ROWS, related }, + (failure) => { reason = `Metric values are unscoped because the related record read failed: ${failure}`; }); + if (!payload || !isCurrent()) return { values: null, reason }; + // A successful HTTP response can still report a refused or incomplete calculation. + const limit = payload.limits?.find((item) => item.effect !== "none"); + if (limit) return { + values: null, + reason: `Metric values are unscoped because the related read is limited: ${limit.cause}`, + }; + const row = payload.rows.find((candidate) => Number(candidate.pid) === pid); + if (row) { + if (measureKeys.some((key) => !Object.prototype.hasOwnProperty.call(row, key))) { + return { values: null, + reason: "Metric values are unscoped because the related read did not return every metric field." }; + } + const values: Partial = {}; + for (const key of measureKeys) values[key] = row[key]; + return { values, reason: "" }; + } + offset += payload.rows.length; + if (!payload.rows.length || offset >= (payload.counts?.matched ?? offset)) break; + } + return { values: null, + reason: "Metric values are unscoped because this record was not found within the related records." }; +} + // ═══════════════════════════════════════════════════════════════════════════════════════════ // ⭐⭐ WAVE 50 · W50-T03 — CONTRACT C1: THE WINDOWED AGGREGATE (owner ruling R1, second half). // ═══════════════════════════════════════════════════════════════════════════════════════════ diff --git a/web/src/customer-grid/timeSeriesData.ts b/web/src/customer-grid/timeSeriesData.ts index e3d747cea5297479c1d89a0cf530f3cf1e40e769..4b145ddcb021e671f70e2b6363291aedb49bbc83 100644 --- a/web/src/customer-grid/timeSeriesData.ts +++ b/web/src/customer-grid/timeSeriesData.ts @@ -451,7 +451,8 @@ const TS_SNAPSHOT_TYPES: ReadonlySet = new Set(["currency", "int", "pct" * `overdue_days`, `days_since` and `typical_gap_days` are all `int`, and the sum of "days to pay" * across 1,500 customers is a number with no meaning that would nonetheless print, in bold, under * a month heading. Matched on the KEY because the contract does not distinguish them by type and - * mostly leaves `agg` unset (measured: only `est_missed` declares one). + * no contract field declares `agg` today: the last one was removed at the owner's instruction + * in wave 57. The `field.agg` branch supports user-created or future fields. * * A field's OWN declared `agg` wins when it has one — the contract saying so beats a heuristic. */ diff --git a/web/src/home/HomePage.tsx b/web/src/home/HomePage.tsx index a6453c28c6a64128f97a58026be540b4168e6bfd..31ac499cda1ba949322586569270527cec9e0d90 100644 --- a/web/src/home/HomePage.tsx +++ b/web/src/home/HomePage.tsx @@ -34,6 +34,7 @@ import { DatabaseMark } from "../customer-grid/iconShapes"; // `.home-db-glyph` box that replaces `.shell-db-chip` below, and Home never loads the grid chunk // that ViewSidebar's copy of this import lives in. import "../styles/wave50/rail.css"; +import "../styles/wave57/home-app.css"; // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOP MARK ON HOME'S FOURTH `.auto-dot` SITE. // The scout table named three call sites; this is the fourth, found by SESSION B and handed over // rather than reached across the fence. Same rule as the rail: consume the component that points @@ -374,11 +375,6 @@ export function HomeSurface({

{mode === "app" ? "App" : "Home"}

- {mode === "app" ? ( - - ) : null}
{mode === "home" ?
@@ -428,7 +424,23 @@ export function HomeSurface({ detail="Browse the connectors this workspace can use." onClick={onConnectors} /> -
: null} +
:
+ {/* ⭐ W57-T28 (owner instruction 26): *"The APP module, the 'Create new' should look + exactly like how the create new button looks like under Home page."* It was a filled + pill in the header while Home offered a card, and BOTH OPEN THE SAME DOOR: `Shell.tsx` + hands `openApplicationWizard` to each of them and says so in its own comment. One door + wearing two looks is the defect, so App now wears Home's card: same component, same + glyph, same tokens. The two ghosts below are layout, not content (see + `styles/wave57/home-app.css` for why they cannot be `display: none`). */} + } + title="Create new" + detail="Build an app over a database in this workspace." + onClick={onTemplates} + /> +
}
{/* The reference puts an "Opened anytime" filter here. We do not have one and do not diff --git a/web/src/settings/SecurityPane.tsx b/web/src/settings/SecurityPane.tsx new file mode 100644 index 0000000000000000000000000000000000000000..74cb94894443bcd3cc7aee412954cd9df385ed19 --- /dev/null +++ b/web/src/settings/SecurityPane.tsx @@ -0,0 +1,147 @@ +// --------------------------------------------------------------------------- +// settings / SecurityPane.tsx - W57-T36, R15 / C12. +// +// Every stored account owns its second factor, so this is a session-scoped +// room for every user. It reads status, never the installed secret. A new phone +// goes through the admin reset and the login enrolment door, not a second editor. +// --------------------------------------------------------------------------- + +import { useEffect, useRef, useState } from "react"; +import type { FormEvent } from "react"; +import { mfaStatus, regenerateRecoveryCodes } from "../shell/session"; +import type { MFAStatus } from "../shell/session"; +import { RecoveryCodes } from "../shell/LoginPage"; +import { useEscapeClose } from "../ui/useEscapeClose"; +import "../styles/wave57/login-mfa.css"; + +export default function SecurityPane() { + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + const [reload, setReload] = useState(0); + const [password, setPassword] = useState(""); + const [busy, setBusy] = useState(false); + const [codes, setCodes] = useState(null); + const [error, setError] = useState(""); + const flight = useRef(false); + const dialog = useRef(null); + const passwordField = useRef(null); + const locked = busy || codes !== null; + + useEffect(() => { + let current = true; + setLoading(true); + setError(""); + void mfaStatus().then((next) => { + if (!current) return; + setStatus(next); + setLoading(false); + if (!next) setError("Security status could not be loaded. Try again."); + }); + return () => { current = false; }; + }, [reload]); + + // Lock BEFORE issuing codes, not only after receiving them: closing Settings + // during the POST would throw away its only clear response. A native modal + // makes the rail, close button and backdrop inert without changing the parent. + // The shared Escape arbiter stops Settings' older window listener as well. + useEscapeClose(busy, () => setError("Wait for your recovery codes before leaving.")); + useEffect(() => { + if (!locked) return; + const node = dialog.current; + if (!node) return; + node.showModal(); + return () => { + node.close(); + passwordField.current?.focus(); + }; + }, [locked]); + + async function generate(event: FormEvent) { + event.preventDefault(); + if (flight.current || codes || !status?.enrolled || !password) return; + flight.current = true; + setBusy(true); + setError(""); + const result = await regenerateRecoveryCodes(password); + // No password survives a write attempt. The response is the new set only; + // do not re-read or cache it, and do not offer generation while it is shown. + setPassword(""); + if (result.ok === false) { + setError(result.message); + } else { + setCodes(result.recoveryCodes); + setStatus((previous) => previous + ? { ...previous, recoveryCodesRemaining: result.recoveryCodes.length } : null); + } + flight.current = false; + setBusy(false); + } + + return ( +
+

Security

+ {loading ?

Loading security status…

: status ? ( + <> +
+
Two step sign in
{status.enrolled ? "On" : "Off"}
+ {status.enrolled ? ( +
Turned on
{status.enrolledAt + ? + : "Date unavailable"}
+ ) : null} +
Recovery codes left
{status.recoveryCodesRemaining}
+
+

To enrol a new phone, ask an administrator to reset two step sign in, then sign in again.

+ {status.enrolled ? ( + <> +

Generating new recovery codes replaces the old codes and signs this account's other sessions out.

+ {!locked ? ( +
+ + { setPassword(event.target.value); setError(""); }} /> + +
+ ) : null} + + ) :

Sign in again to set up your authenticator app.

} + + ) : ( + + )} +

{locked ? "" : error}

+ + {/* R15/C12's explicit acknowledgement is the exception to ordinary + dismissible dialogs. Native cancel is prevented, outside clicks have + no close handler, and there is no close control or method=dialog form. */} + {locked ? ( + event.preventDefault()}> +

Recovery codes

+
+ {codes ? ( + { + setCodes(null); + setError(""); + // Count only, independently re-read after acknowledgement. + setReload((value) => value + 1); + }} /> + ) : ( + <> +

Generating your recovery codes…

+ + + )} +
+

{error}

+
+ ) : null} +
+ ); +} diff --git a/web/src/settings/SettingsModal.tsx b/web/src/settings/SettingsModal.tsx index 4516fc4613f067e4687bca8d7494779febf466c7..2e6e195a102ac9e85f0cbf8868043b3fe308212d 100644 --- a/web/src/settings/SettingsModal.tsx +++ b/web/src/settings/SettingsModal.tsx @@ -77,6 +77,7 @@ import { ManageDatabasePane } from "./ManageDatabase"; // client-local preference pane, self-contained like `ManageDatabasePane`; see the mount below. import ShortcutSettings from "./ShortcutSettings"; import ExternalAccessPane from "./ExternalAccessPane"; +import SecurityPane from "./SecurityPane"; import OperatorPage from "../operator/OperatorPage"; /* ⛔ `StatementsPane` LEFT WITH ITS RAIL ROW (wave 35 · T38): the file is DELETED, not merely unmounted. An orphaned component with no importer is the shape this repo books as @@ -148,6 +149,13 @@ const IconAccess = ( ); +/** Security - a shield protects the credential; access already wears the padlock. */ +const IconSecurity = ( + + + + +); /** Sheet shortcuts — a small keyboard: a row of keys above a spacebar, not the padlock * "Your access" already wears (two authority glyphs in one rail would drift together). */ const IconShortcuts = ( @@ -796,6 +804,7 @@ export function SettingsModal({ const rail: Array<[SettingsSection, string, ReactNode]> = [ ["account", "Account", IconAccount], ["scope", "Your access", IconAccess], + ["security", "Security", IconSecurity], // ⭐⭐ WAVE 52 · T19 (owner instruction 21) — NOT admin-gated, on the same footing as // `account` and `scope` immediately above: every user manages their own shortcut // preferences, and there is nothing behind this room a member's account cannot already do to @@ -909,6 +918,7 @@ export function SettingsModal({ `sheet/shortcuts.ts`'s own localStorage store directly and needs nothing passed in, because a keyboard preference is not server data. */} {active === "shortcuts" ? : null} + {active === "security" ? : null} {active === "external" ? : null} {active === "keychains" ? : null} {active === "connectors" ? : null} diff --git a/web/src/settings/permsModel.ts b/web/src/settings/permsModel.ts index 62bacf76f127468a41361e923f49f4ce96340e73..b1b0f99aa80a451cf1f6a348276c44932a94da00 100644 --- a/web/src/settings/permsModel.ts +++ b/web/src/settings/permsModel.ts @@ -40,6 +40,9 @@ import type { Field, FieldType, FilterTree } from "../customer-grid/types"; export type SettingsSection = | "account" | "scope" + /** W57-T36 / R15: every user enrols their own factor, so every user needs + * this room. Like shortcuts, it is NOT admin-only; the session scopes it. */ + | "security" /** * ⭐⭐ WAVE 52 (owner instruction 21, ruling R7) — SHEET SHORTCUTS: toggle any shortcut on or * off and rebind the eleven plain-Ctrl-letter ones. NOT admin-only — every user manages their diff --git a/web/src/sheet/BindingEditor.tsx b/web/src/sheet/BindingEditor.tsx index b7cadb14dc47a20bc841f7514d90df7d8c3f0710..7b14b14c18f71a856cec8ed8dc95d84a20e3ea7a 100644 --- a/web/src/sheet/BindingEditor.tsx +++ b/web/src/sheet/BindingEditor.tsx @@ -39,9 +39,10 @@ import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; import { fetchTopicRows, fetchWorkspace } from "../customer-grid/apiBridge"; import type { SurfaceScope } from "../customer-grid/apiBridge"; +import { FieldSelectButton } from "../customer-grid/FieldSelect"; import { FieldTypeIcon } from "../customer-grid/icons"; import { BodyPortal, OverlayProvider, useOverlayLayer } from "../customer-grid/OverlaySurface"; -import type { Field, FilterTree, Row } from "../customer-grid/types"; +import type { Field, FilterTree, GridWorkspace, Measure, Row, SavedView } from "../customer-grid/types"; import { choiceVocabulary, topicForScope } from "../customer-grid/types"; import type { WindowSpec } from "../customer-grid/windows"; import { normalizeWindow } from "../customer-grid/windows"; @@ -55,6 +56,14 @@ import { WindowPicker } from "../filter-kit/WindowPicker"; import { useEscapeClose } from "../ui/useEscapeClose"; import { previewCell } from "./sheetApi"; import { bindingDisplay, bindingFilterClauses } from "./types"; +import "../styles/wave57/binding-actions.css"; + +const BINDING_SECTIONS = [ + { key: "data", label: "Data" }, + { key: "rows", label: "Rows" }, + { key: "period", label: "Period" }, + { key: "copied", label: "When copied" }, +] as const; const AGGREGATES = [ { fn: "sum", label: "Sum" }, @@ -97,10 +106,6 @@ const NUMERIC_ROLLUP_FNS = [ * the field's `type` (the two live ones on `customer_data` are typed `currency` and `int`). */ const isMetricField = (f: Field): boolean => Boolean(f.measure); -/** ⭐ `custom`, not `measure`, identifies a Metric made in the table. `isMetricField` alone is the - * trap: it includes exactly the user-created Metric columns the cell picker must exclude. */ -const isUserMetricField = (f: Field): boolean => isMetricField(f) && f.custom === true; - /** ⭐ `custom`, not `measure`, separates supplied templates from table-made Metrics. * `isMetricField` alone is the trap because it includes both kinds. */ const isTemplateMetricField = (f: Field): boolean => isMetricField(f) && f.custom !== true; @@ -360,10 +365,21 @@ function savedViewOf(initial: Record | null | undefined): { }; } -/** The `{id, name}` pairs a workspace payload offers, and nothing else off it. */ -function viewList(raw: unknown): { id: string; name: string }[] { +type BindingView = Pick & { + config: Pick; +}; + +/** Workspace view names and the filter clauses a copied cell can step. */ +function viewList(raw: unknown): BindingView[] { if (!Array.isArray(raw)) return []; - return (raw as { id: string; name: string }[]).map((v) => ({ id: v.id, name: v.name })); + return (raw as GridWorkspace["views"]).map((v) => ({ + id: v.id, + name: v.name, + config: { + filterConj: v.config?.filterConj === "or" ? "or" : "and", + filters: Array.isArray(v.config?.filters) ? v.config.filters : [], + }, + })); } // ══ ⭐⭐ W54-T31 (ruling R8; owner instructions 6 and 8) — THE AGGREGATE'S COLUMN LIST ═════════ @@ -436,9 +452,6 @@ function ColumnPickerDialog({ }); const offersTemplateMetric = offered.some(isTemplateMetricField); - // ⛔ W56-T19 - A NARROWED LIST MUST SAY WHAT WAS WITHHELD. Without this cause, a database whose - // only usable-looking Metrics were made here would present a silent empty picker as if it had none. - const hasOnlyUnofferedUserMetrics = all.some(isUserMetricField) && !offersTemplateMetric; return ( @@ -541,9 +554,8 @@ function ColumnPickerDialog({
)} - {/* ⛔ AN EMPTY LIST SAYS WHY (R16). The causes need different acts from the reader, - and the user-Metric-only cause has its own honest explanation directly below. */} - {shown.length === 0 && (offered.length > 0 || !hasOnlyUnofferedUserMetrics) && ( + {/* AN EMPTY LIST SAYS WHY (R16). The causes need different acts from the reader. */} + {shown.length === 0 && (

{offered.length > 0 ? "No column matches that search." @@ -554,14 +566,6 @@ function ColumnPickerDialog({

)} - {hasOnlyUnofferedUserMetrics && ( -

- This database's Metric columns were created here rather than supplied as templates, - so they are not offered for a cell; a template metric is measured over the period this - panel sets. -

- )} - {/* ⭐ W56-T19, said where the choice is made: a template Metric is the one offered kind whose value is measured over the period the panel sets. */} {offersTemplateMetric && shown.length > 0 && ( @@ -633,6 +637,7 @@ export interface BindingEditorProps { export default function BindingEditor({ workbookId, addr, databases, initial, onSave, onCancel, }: BindingEditorProps) { + const [activeSection, setActiveSection] = useState("data"); const [database, setDatabase] = useState( String((initial?.database as string) ?? databases[0]?.key ?? ""), ); @@ -652,7 +657,9 @@ export default function BindingEditor({ // built out of groups is not undercounted into reading as empty. const conditionCount = useMemo(() => countConditions(tree.nodes ?? []), [tree]); const [fields, setFields] = useState([]); - const [views, setViews] = useState<{ id: string; name: string }[]>([]); + const [measures, setMeasures] = useState([]); + const [measureNotice, setMeasureNotice] = useState(""); + const [views, setViews] = useState([]); // ⛔ THE NOTICE BELOW IS GATED ON THIS FLAG AND NOT ON `views.length`. Before the first answer // arrives the list is legitimately empty, and a notice keyed on emptiness would flash "that view // is gone" at every perfectly valid binding for as long as the fetch took. @@ -726,6 +733,21 @@ export default function BindingEditor({ migrated?.start ?? normalizePeriodStart("month", todayYm()), ); const [locked, setLocked] = useState(migrated?.locked ?? false); + // ⛔⛔ W57-T17 - THE DATABASE IS SEEDED AT MOUNT OUT OF A LIST THAT ARRIVES LATER, AND AN EMPTY + // SEED NEVER HEALED. `databases` is fetched by `SheetPage` from `/api/v1/nav`; a panel opened + // before that answer lands seeds `database` to `""`, and nothing re-seeds it. A ` setDatabase(e.target.value)}> - {databases.map((d) => )} - - - -
- - {/* Count takes no column, and offering one would suggest count(column), which skips - blanks and answers a smaller number than the person meant. */} - {fn !== "count" && ( -
- Column - {/* ⭐⭐ W54-T31 (R8) — THE CONTROL SHOWS WHAT IS CHOSEN AND OPENS THE LIST; IT IS - NOT THE LIST. `cg-select cg-fsel` is the shipped footprint every replaced - ` setScopeKind("view")} /> - A saved view - - {scopeKind === "view" && ( - - )} - {/* ⭐ W53-T12 — SAID HERE, THE MOMENT THE LIST RESOLVES, rather than as a refusal after - a save or a refresh. `sheet-notice` with `role="alert"` is this panel's established - inline shape, so the sentence lands where the choice is made. */} - {savedViewMissing && ( -

- {`The saved view ${savedView.name || savedView.id} is not available any more. ` - + "Pick a view that still exists, or build a filter here."} -

- )} - - {/* ⭐⭐ W53-T15 (ruling R8) — A DOOR, NOT THE BUILDER. The same shared panel now opens - in a centred dialog: this column is `flex: 0 0 380px` and a condition row cannot fit - in it, so four conditions could only be reached by scrolling sideways. The button - says how many conditions the cell already carries, because the builder that used to - say so is now behind it. */} - {scopeKind === "filter" && ( - <> - - {/* ⛔⛔ MOUNTED ONLY WHILE OPEN, and that is what orders Escape correctly rather - than a flag inside the dialog. `useOverlayLayer` claims Escape as soon as it - runs and `ui/useEscapeClose` gives the key to the LAST entry pushed, so a - dialog mounted with the panel would have claimed Escape from the moment this - editor opened. Mounting it here, later in time, puts it ABOVE this panel's own - claim (pushed at mount, a few hundred lines up) and unmounting it hands Escape - straight back, so one press closes the filter and the next closes the panel. */} - {filterPopupOpen && ( - setFilterPopupOpen(false)} - /> - )} - - )} - - - {/* ══ R9: THE PERIOD ══════════════════════════════════════════════════════════════════ */} -
- Period - - {/* ⭐⭐ W53-T16 / D-660 - WHETHER THIS CELL HAS A PERIOD AT ALL, said out loud. The - pickers below show a unit and a month whether or not the cell is using them, so - without this switch an all-time total read as "September 2026" on screen and the next - save made that true. It is also the only way back to all time once a period is set. */} -
- -

- {chosenIsTemplateMetric - ? "The period is required because this column is measured over one." - : (periodOn - ? "Only rows dated inside the period below are counted." - : "All time, so every row this cell's filter matches is counted whatever its date. " - + "Saving leaves it that way.")} -

-
- - {periodOn && ( - <> - {/* ⭐ C5 - THE MODE LABELS SAY WHAT EACH CHOICE DOES. Internal dialect names - would make the reader guess which one can step and which one follows today. */} - - - {periodMode === "window" ? ( + ))} + +
+ {activeSection === "data" && ( + <> - ) : ( - <> + +
- -
-
+ )} +
+ + )} + {activeSection === "rows" && ( + <> +
+ Rows to include + + {scopeKind === "view" && ( + + )} + {/* ⭐ W53-T12 — SAID HERE, THE MOMENT THE LIST RESOLVES, rather than as a refusal after + a save or a refresh. `sheet-notice` with `role="alert"` is this panel's established + inline shape, so the sentence lands where the choice is made. */} + {savedViewMissing && ( +

+ {`The saved view ${savedView.name || savedView.id} is not available any more. ` + + "Pick a view that still exists, or build a filter here."} +

+ )} + + {/* ⭐⭐ W53-T15 (ruling R8) — A DOOR, NOT THE BUILDER. The same shared panel now opens + in a centred dialog: this column is `flex: 0 0 380px` and a condition row cannot fit + in it, so four conditions could only be reached by scrolling sideways. The button + says how many conditions the cell already carries, because the builder that used to + say so is now behind it. */} + {scopeKind === "filter" && ( + <> + + {/* ⛔⛔ MOUNTED ONLY WHILE OPEN, and that is what orders Escape correctly rather + than a flag inside the dialog. `useOverlayLayer` claims Escape as soon as it + runs and `ui/useEscapeClose` gives the key to the LAST entry pushed, so a + dialog mounted with the panel would have claimed Escape from the moment this + editor opened. Mounting it here, later in time, puts it ABOVE this panel's own + claim (pushed at mount, a few hundred lines up) and unmounting it hands Escape + straight back, so one press closes the filter and the next closes the panel. */} + {filterPopupOpen && ( + setFilterPopupOpen(false)} + /> + )} + + )} +
+ + )} + {activeSection === "period" && ( + <> + {/* ══ R9: THE PERIOD ══════════════════════════════════════════════════════════════════ */} +
+ Period + + {/* ⭐⭐ W53-T16 / D-660 - WHETHER THIS CELL HAS A PERIOD AT ALL, said out loud. The + pickers below show a unit and a month whether or not the cell is using them, so + without this switch an all-time total read as "September 2026" on screen and the next + save made that true. It is also the only way back to all time once a period is set. */} + {/* ⛔⛔ W57-T18 (R5, C3) - THIS SWITCH IS NO LONGER DISABLED ON A METRIC, AND + THAT IS THE TICKET RATHER THAN A REGRESSION. W56-T19 disabled it for a + template Metric, which made an invalid binding UNREACHABLE instead of + REFUSED. C3 asks for a refusal on both sides, and a control nobody can + move produces no refusal anybody can see: `ready` would be permanently + true for a Metric, the rail's Required tag could never paint, and the + client half of the contract would be satisfied only by the state being + impossible to enter. The effect above still raises the switch the moment a + Metric is chosen, so the DEFAULT is a valid binding; turning it off is + what the panel then refuses to save, and the rail says where to go. */} +
+ +

+ {chosenIsMetric + ? "The period is required because every Metric column is measured over one." + : (periodOn + ? "Only rows dated inside the period below are counted." + : "All time, so every row this cell's filter matches is counted whatever its date. " + + "Saving leaves it that way.")} +

+
+ + {periodOn && ( + <> + {/* ⭐ C5 - THE MODE LABELS SAY WHAT EACH CHOICE DOES. Internal dialect names + would make the reader guess which one can step and which one follows today. */} - )} - {unit === "quarter" && ( + + {periodMode === "window" ? ( + + ) : ( + <> + + +
+ + {unit === "month" && ( + + )} + {unit === "quarter" && ( + + )} +
+ + {/* W56-T21 (owner instruction 13) - THE QUICK PICKS ARE GONE. Unit, year and + month-or-quarter already reach every period the This and Last buttons + could, so a second control for the same value was a second way to say one + thing. `normalizePeriodStart` still normalises all three selects, and + `shiftPeriodStart` stays exported for the drag arithmetic. */} + + {/* ⭐ R9's LOCK. A switch rather than a checkbox: it has two states with names, and the + sentence under it says what each one costs when the cell is copied. */} + {chosenIsMetric ? ( +
+ +

+ {locked + ? "Locked, so any copy of this cell keeps this period." + : `Not locked, so a stepped copy moves by one ${UNIT_NOUN[unit]} per cell. ` + + "Right and down move forward. Left and up move backward."} +

+
+ ) : ( +

+ This period stays fixed when copied. The filter clause below is what + changes. +

+ )} + + {/* ⭐ "Use a cell" STAYS A LINK (R9). It reveals the A1 input rather than replacing the + picker, so the ordinary reader never meets a cell reference and the person who wants + one is one click away from it. */} + {!refOpen && ( + + )} + {refOpen && ( + + )} + + )} + + )} +
+ + )} + {activeSection === "copied" && ( + <> + {/* ══ R10: THE DRAG RULES ═════════════════════════════════════════════════════════════ */} +
+ When copied + + {chosenIsMetric && periodOn && periodMode === "calendar" && ( + <> - )} -
- - {/* W56-T21 (owner instruction 13) - THE QUICK PICKS ARE GONE. Unit, year and - month-or-quarter already reach every period the This and Last buttons - could, so a second control for the same value was a second way to say one - thing. `normalizePeriodStart` still normalises all three selects, and - `shiftPeriodStart` stays exported for the drag arithmetic. */} - - {/* ⭐ R9's LOCK. A switch rather than a checkbox: it has two states with names, and the - sentence under it says what each one costs when the cell is copied. */} - {chosenIsMetric ? ( -
-

- {locked - ? "Locked, so any copy of this cell keeps this period." - : `Not locked, so a stepped copy moves by one ${UNIT_NOUN[unit]} per cell. ` - + "Right and down move forward. Left and up move backward."} + Right and down move forward. Left and up move backward.

-
- ) : ( + + )} + + {/* ⛔ C5 - A ROLLING WINDOW HAS NO START TO SHIFT. Offering the calendar step here + would promise an action `dragBoundCell` cannot perform. */} + {periodOn && periodMode === "window" && (

- This period stays fixed when copied. The filter clause below is what - changes. + A rolling window is measured from today, so every copy of this cell answers the + same window. Use a calendar period when the period should step as the cell is copied.

)} - {/* ⭐ "Use a cell" STAYS A LINK (R9). It reveals the A1 input rather than replacing the - picker, so the ordinary reader never meets a cell reference and the person who wants - one is one click away from it. */} - {!refOpen && ( -
- - {/* ══ R10: THE DRAG RULES ═════════════════════════════════════════════════════════════ */} -
- When copied - - {chosenIsMetric && periodOn && periodMode === "calendar" && ( - <> - -

- Right and down move forward. Left and up move backward. -

- - )} - - {/* ⛔ C5 - A ROLLING WINDOW HAS NO START TO SHIFT. Offering the calendar step here - would promise an action `dragBoundCell` cannot perform. */} - {periodOn && periodMode === "window" && ( -

- A rolling window is measured from today, so every copy of this cell answers the - same window. Use a calendar period when the period should step as the cell is copied. -

- )} - - {/* ⚠ AN ALL-TIME METRIC SAYS WHY NO PERIOD PICKER IS PRESENT. Without this note the - absent control looks like a loading or configuration failure. */} - {chosenIsMetric && !periodOn && ( -

- This metric is all time, so copied cells keep the same all-time period. -

- )} - - - {savedDragDriverMissing && ( -

- The saved driver is not in this cell's filter, so copying this cell will not - change it. Pick a driver that is in the filter, or keep the filter. -

- )} - {scopeKind === "view" ? ( -

- These rows come from a saved view, so the view's filter clauses cannot be listed - here. This database's steppable columns are offered instead. A driver that is not - in the view's filter changes nothing, and the sheet says so when dragged. -

- ) : filterClauseOffers.length === 0 ? ( -

- This cell has no filter clauses to step. -

- ) : ( -

- Right and down choose later options. Left and up choose earlier options. -

- )} -
+ {filterClauseOffers.length === 0 ? ( +

+ This cell has no filter clauses to step. +

+ ) : ( +

+ Right and down choose later options. Left and up choose earlier options. +

+ )} + + + )} +
+
{/* ⛔⛔ W53-T16 (owner instruction 13) - THE FOUR SENTENCE SUMMARY IS DELETED, NOT HIDDEN. It restated the period, the lock and the two drag rules that the controls immediately @@ -1613,15 +1730,17 @@ export default function BindingEditor({ )}
+ {/* C7 and the owner's Apple principle: Save is this surface's ONE primary; + Preview and Cancel are deliberately quiet. */}
- - -
diff --git a/web/src/sheet/DrillModal.tsx b/web/src/sheet/DrillModal.tsx index 76e1a3f35f633285216f89c14e023989d4bc727d..335c9824c28df4985f9874f0791518d11450726f 100644 --- a/web/src/sheet/DrillModal.tsx +++ b/web/src/sheet/DrillModal.tsx @@ -38,9 +38,8 @@ // when we click 'View row' so its easy for user also to know what this filter is. In fact, they // should be able to change the filter there too. And if they make a mistake etc, they can just // CTRL + Z to return the Cell to its original filter."* Three clauses, and each one is a decision -// recorded beside the code that answers it: the BAND (`DrillFilterBand`) states it, the top-right -// Edit filter action opens the shared `SheetFilterDialog`, and one cell-scoped step of undo takes -// it back. +// recorded beside the code that answers it: the BAND (`DrillFilterBand`) states it, the grid's +// Filter control edits it, and one cell-scoped step of undo takes it back. // // ⭐⭐ W56-T24 / R4 / C4 — THE LINK MODAL NOW OWNS ORDINARY SLOTS FOR THE SHEET'S CHROME. // The toolbar and filter statement are children of the modal's header and footer, so they remain @@ -60,12 +59,10 @@ import type { SurfaceScope } from "../customer-grid/apiBridge"; import { dataPathHash } from "../customer-grid/dataPath"; import { AnchoredOverlay, OverlayProvider } from "../customer-grid/OverlaySurface"; import type { Field, FilterTree, GridWorkspace, Row } from "../customer-grid/types"; -import { choiceVocabulary, mayEditField, topicForScope } from "../customer-grid/types"; -import { - SheetFilterDialog, conditionCountLabel, filterSentence, -} from "../filter-kit/SheetFilterPopup"; +import { mayEditField, topicForScope } from "../customer-grid/types"; +import { conditionCountLabel, filterSentence } from "../filter-kit/SheetFilterPopup"; import { useEscapeClose } from "../ui/useEscapeClose"; -import { cellViewVisibleKeys, filterFieldKeys } from "./bindingRefs"; +import { cellForcedFieldKeys, cellViewVisibleKeys } from "./bindingRefs"; import { drillCell } from "./sheetApi"; import type { CellDrill } from "./sheetApi"; import type { BindingDisplay } from "./types"; @@ -182,6 +179,10 @@ export type DrillCellScope = ( & { /** W56-T24: accepted legacy cell display bag; the grid now owns presentation itself. */ display?: BindingDisplay; + /** The host supplies these from the current binding alongside its filter or saved view. */ + aggregateField?: string; + /** The Option step's binding.onDrag.down.field, when down is a field rule. */ + stepField?: string; }; /** What the drill's own answer says about the scope, for a caller that passed none. */ @@ -207,8 +208,8 @@ const sameTree = (a: FilterTree | null, b: FilterTree | null): boolean => * * The sentence is the whole of the owner's first clause, so it takes the width and truncates with * an ellipsis rather than being shortened into a count; the full text stays reachable as the - * element's `title`. W55-T07 moves Edit filter to the top-right toolbar and leaves this band one - * statement with one job. W56-T24 seats it inside `LinkGridModal`'s footer slot, so that shell now + * element's `title`. The grid's Filter control owns editing; this band states the cell's scope + * and any editing limit. W56-T24 seats it inside `LinkGridModal`'s footer slot, so that shell now * supplies the `cg-link-grid-foot` paint instead of this child impersonating the footer itself. */ function DrillFilterBand( @@ -256,7 +257,7 @@ export interface DrillModalProps { */ cellScope?: DrillCellScope | null; /** - * W54-T35: persist a new filter for this cell and recompute it. Absent means no Edit door. + * W54-T35: persist a new filter for this cell and recompute it. * * ⛔ IT RESOLVES WHEN THE WRITE HAS SETTLED, which is what lets the rows be re-read afterwards * rather than immediately: `GET …/drill` reads the STORED binding, so a re-read fired before @@ -291,18 +292,6 @@ export default function DrillModal( filterChangeRef.current = onFilterChange; // ── W54-T35 state ───────────────────────────────────────────────────────────────────────── - /** Is the centred filter dialog on screen? */ - const [filterOpen, setFilterOpen] = useState(false); - /** - * ⛔⛔ THE DRAFT IS THIS COMPONENT'S, NOT THE DIALOG'S, AND THE DIALOG'S OWN CONTRACT IS WHY. - * `SheetFilterPopup` holds no draft on purpose: `onChange` is the caller's setter and every - * keystroke commits. The binding panel can afford that because its setter is a `useState`. Here - * the setter would be a PUT plus a recompute, so a value typed into a condition box would be one - * save per character. So the keystrokes land here, and the dialog closing is the commit. - * ⚠ THAT IS ALSO WHY CLOSING IT IS NOT A CANCEL. Escape, an outside click and Done are the same - * event with three names in that dialog, and they stay the same event here: all three APPLY. - */ - const [draft, setDraft] = useState(null); /** * ⭐⭐ THE UNDO, AND IT IS ONE STEP SCOPED TO THIS CELL. Owner: *"if they make a mistake etc, * they can just CTRL + Z to return the Cell to its original filter."* @@ -311,21 +300,20 @@ export default function DrillModal( * array on every commit, so pushing a filter edit onto it would make one Ctrl+Z walk back * whatever else the reader had typed since - an undo that reaches past the surface it was * pressed on, into cells nobody was looking at. This holds ONE tree: the cell's filter as it - * stood when the dialog was opened. A second Ctrl+Z therefore has nothing to pop and does not + * stood before the latest grid filter edit. A second Ctrl+Z therefore has nothing to pop and does not * go further back, which is the done-when clause spelled as a data structure rather than as a * guard somebody has to remember to write. */ const [undoTree, setUndoTree] = useState(null); - /** The tree as it stood when the Edit door was last opened. See `openFilter`. */ + /** The tree before the current grid filter edit burst, retained for its undo step. */ const preEditRef = useRef(null); /** Latest grid-owned filter edit waiting for 800 ms of quiet before it reaches the cell. */ const pendingCellFilterRef = useRef(null); const cellFilterTimerRef = useRef(null); /** Bumped when a filter change has landed, so the rows are read again. */ const [reloadSeq, setReloadSeq] = useState(0); - /** The filter kit's two vocabularies for this database. Fetched once the drill has answered. */ + /** Fields for the filter sentence and export, fetched once the drill has answered. */ const [fields, setFields] = useState([]); - const [statusValues, setStatusValues] = useState>({}); /** Raw, permission-scoped rows back the export projection. */ const [rawRows, setRawRows] = useState([]); const [workspace, setWorkspace] = useState(null); @@ -411,7 +399,7 @@ export default function DrillModal( }, [workbookId, addr, reloadSeq]); /** - * ⭐⭐ W54-T35 — WHAT THE BAND SAYS, AND WHAT THE DIALOG EDITS. + * W54-T35: what the band says, and what the grid's Filter control edits. * * The page's own reading of the cell wins, because it is the one that MOVES: it is rebuilt from * the workbook after every commit, so the sentence changes the moment a filter is applied. The @@ -426,9 +414,7 @@ export default function DrillModal( shownTreeRef.current = shownTree; /** - * ⭐ THE FIELD AND VALUE VOCABULARIES, FETCHED THE WAY `BindingEditor` FETCHES THEM, and warmed - * on the drill's answer rather than on the Edit click for the reason `warmLinkChunk` is warmed - * at mount: the two doors would otherwise run back to back for one gesture. + * Fields and rows for the filter sentence and export are warmed on the drill's answer. * * ⛔ NEITHER CALL IS NEW TRAFFIC IN PRACTICE. `fetchWorkspace(scope, {allowCached: true})` is * served from `wsCache` inside its freshness window, and `fetchTopicRows` shares `rowsCache` @@ -458,22 +444,9 @@ export default function DrillModal( } const list = ((rowsPayload?.fields?.length ? rowsPayload.fields : ws?.fields) ?? []) as Field[]; const rows: Row[] = rowsPayload?.rows ?? []; - const options: string[] = rowsPayload?.userOptions ?? ws?.userOptions ?? []; - const map: Record = {}; - // The same rule `Toolbar.tsx` and `BindingEditor.tsx` compute, through the same pure - // function, so a server-computed select has a value list here as well. - for (const f of list) { - if (f.type === "user") { - map[f.key] = options; - continue; - } - if (f.type === "select" || f.type === "multiselect" || f.type === "status") - map[f.key] = choiceVocabulary(f, rows); - } setWorkspace(ws); setFields(list); setRawRows(rows); - setStatusValues(map); setToday(rowsPayload?.today); setRowsResolved(true); })(); @@ -518,34 +491,6 @@ export default function DrillModal( pendingCellFilterRef.current = null; }, []); - /** - * ⛔ CLOSING THE DIALOG IS THE COMMIT, and an unchanged tree commits nothing. Without that test - * every open-and-look would write the cell back to itself, which is a PUT, a recompute and a - * spurious undo step for a gesture that changed nothing. - */ - const closeFilter = useCallback(() => { - setFilterOpen(false); - const next = draft; - setDraft(null); - if (!next || sameTree(next, shownTree)) return; - // ⛔ THE UNDO STEP IS RECORDED HERE AND NOT AT THE OPEN, because a door that was opened and - // shut again changed nothing and must leave nothing to take back. Recording it on open would - // arm Ctrl+Z after a look, and the press would then write the cell back to itself: a PUT, a - // recompute and a stale mark, for a gesture that did nothing. - setUndoTree(preEditRef.current); - void applyTree(next); - }, [applyTree, draft, shownTree]); - - const openFilter = useCallback(() => { - if (!shownTree) return; - // "The filter it had before the edit" is the filter it has at the moment the door opens. It - // is held in a ref rather than in state because nothing renders it: it is read once, by the - // close above, and putting it in state would re-render the drill for no visible reason. - preEditRef.current = shownTree; - setDraft(shownTree); - setFilterOpen(true); - }, [shownTree]); - /** * ⭐⭐ WHERE THE KEYSTROKE ACTUALLY ARRIVES, WHICH IS NOT WHERE THE SHEET'S UNDO LIVES. * @@ -574,7 +519,7 @@ export default function DrillModal( */ const undoRef = useRef<() => void>(() => {}); undoRef.current = () => { - if (filterOpen || !undoTree) return; + if (!undoTree) return; setUndoTree(null); void applyTree(undoTree); }; @@ -609,9 +554,13 @@ export default function DrillModal( : savedView ? { conj: savedView.config.filterConj, nodes: savedView.config.filters } : null; - const forcedFieldKeys = filterFieldKeys(fieldFilter); + const forcedFieldKeys = cellForcedFieldKeys( + fieldFilter, shown?.aggregateField, shown?.stepField, + ); const defaultFieldKeys = fields - .filter((field) => field.default !== false && field.custom !== true && field.shared !== true) + .filter((field) => field.default !== false + && (field.custom !== true || Boolean(field.measure)) + && field.shared !== true) .map((field) => field.key); const visibleKeys = cellViewVisibleKeys( defaultFieldKeys, @@ -665,11 +614,18 @@ export default function DrillModal( const bandText = !shown ? "" : shown.kind === "view" - ? (shown.name ? `Rows come from the saved view ${shown.name}` : "Rows come from a saved view") + // ⛔⛔ W57-T21 - THE BAND CARRIES THE EDITING LIMIT NOW, because the control that used to + // carry it is gone. The deleted Edit filter button was DISABLED on this branch with a title + // saying where the filter is changed instead; without a replacement the grid's own Filter + // control would sit there looking editable and quietly write nothing back, which is standing + // rule 1's second sentence exactly: a limit that cannot be removed is REPORTED, never + // silently enforced. + ? (shown.name + ? `Rows come from the saved view ${shown.name}, so this cell's filter is changed in Data` + : "Rows come from a saved view, so this cell's filter is changed in Data") // ⛔ THE FALLBACK IS THE COUNT, NOT A SECOND SENTENCE. `filterSentence` answers "" when the // filter narrows nothing, which covers both an empty tree and one whose conditions are all - // still half typed; `conditionCountLabel` is the wording the dialog behind the Edit door - // already uses for that, so the two surfaces say the same thing about the same tree. + // still half typed; `conditionCountLabel` keeps the wording shared with the filter kit. : said ? `Rows where ${said}` : conditionCountLabel(shown.tree); if (error) { @@ -714,22 +670,21 @@ export default function DrillModal( // now would blink the screen back to empty at the last moment, which is the defect this ticket // exists to remove, arriving one step later. /** - * ⭐⭐ W54-T35 — ONE `OverlayProvider` OVER BOTH SURFACES, AND WITHOUT IT THE EDIT DOOR CLOSES - * THE DRILL. `useOverlayLayer` defers to a higher layer only when it can SEE one: - * `if (!dismissOnOutside || (stack && !stack.isTop(id))) return;`, where `stack` is - * `useContext(OverlayStackContext)`. The Sheet route mounts no provider, so `LinkGridModal` - * ran with `stack === null` and treated a pointerdown ANYWHERE outside its own card as a - * dismissal - including one inside a filter dialog opened on top of it. Both layers register - * here instead, the dialog registers later, and the card stands down while it is up. This is - * the same trap `filter-kit/SheetFilterPopup.tsx` documents at the top of its own file, one - * layer further out, which is why the dialog is mounted through `SheetFilterDialog` (no - * provider of its own) rather than through `SheetFilterPopup` (which brings one). + * ⭐⭐ ONE `OverlayProvider` OVER EVERY LAYER THIS CARD OPENS, AND IT IS STILL LOAD-BEARING + * AFTER W57-T21 DELETED THE EDIT DOOR. `useOverlayLayer` defers to a higher layer only when it + * can SEE one: `if (!dismissOnOutside || (stack && !stack.isTop(id))) return;`, where `stack` + * is `useContext(OverlayStackContext)`. The Sheet route mounts no provider of its own, so + * `LinkGridModal` would run with `stack === null` and read a pointerdown ANYWHERE outside its + * own card as a dismissal, including one inside a layer opened on top of it. The Export menu + * below is exactly such a layer, and so is every filter, field and sort popover the grid's own + * chrome opens now that the chrome is the only filter door. Both register here instead, the + * later one is topmost, and the card stands down while it is up. `filter-kit/SheetFilterPopup.tsx` + * documents the same trap one layer further out. * * ⛔ ESCAPE IS UNAFFECTED AND STILL RESOLVES TO EXACTLY ONE SURFACE. That is the module-level - * stack in `ui/useEscapeClose`, not this one: the card claims when it mounts, the dialog - * claims when it opens and is therefore topmost, so one press closes the filter and the next - * closes the drill. The band claims nothing: it is a statement inside the card, not an - * overlay, so the "exactly one claimant" property W51-T12 bought is unspent. + * stack in `ui/useEscapeClose`, not this one: the card claims when it mounts, a popover claims + * when it opens and is therefore topmost, so one press closes the popover and the next closes + * the drill. The band claims nothing: it is a statement inside the card, not an overlay. */ }> @@ -739,27 +694,12 @@ export default function DrillModal( recordIds={drill.record_ids} editable={false} cellView + cellFields={visibleKeys} // A view-bound cell passes neither prop: its rows remain the saved view's business. cellFilter={shownTree ?? undefined} onCellFilterChange={shownKind === "filter" ? settleCellFilter : undefined} headerActions={(
- {canEditRows && (
@@ -826,15 +904,42 @@ export default function Ribbon({ role="toolbar" aria-label={activeTab.toolbarLabel} > + {/* Scroll buttons stay outside the roving order because keyboard focus already scrolls controls into view. */} + {canScrollLeft && ( + + )} {/* ⛔ `min-width: 0` ON THE SCROLLER IS LOAD BEARING. A flex child will not scroll without it. Import, Refresh, Export and Full screen do not live in this row (W52-T15 moved them to the tab strip above), so the only thing that can walk off the right edge at a narrow window now is a group itself, which scrolling permits. */} -
+
{/* ⭐⭐ THE ACTIVE TAB'S OWN GROUPS. `ribbonHome.tsx` holds every one of them, in R8's order, and this is the whole of the shell's knowledge of what they are. */} {activeTab.groups(tabContext)}
+ {canScrollRight && ( + + )}
)} {/* ⭐⭐ W52-T17 (instruction 20, ruling R7) — mounted ONCE, absolutely positioned via its own diff --git a/web/src/sheet/SheetGrid.tsx b/web/src/sheet/SheetGrid.tsx index 9556606fe8284de4b478ffce20dc95a0fd301e57..3c1d9dcda8d123f293fcd9836aa72acdcd6ef64d 100644 --- a/web/src/sheet/SheetGrid.tsx +++ b/web/src/sheet/SheetGrid.tsx @@ -142,7 +142,7 @@ import { RulesDialog, ValidationDialog } from "./RulesDialog"; import ChartDialog from "./ChartDialog"; import SheetChartLayer from "./SheetChartLayer"; import ShortcutsPanel from "./ShortcutsPanel"; -import { colToIndex, indexToCol } from "./a1"; +import { colToIndex, columnRangeA1, indexToCol, rowRangeA1 } from "./a1"; // ⛔⛔ W51-T18 — THE RULE ENGINE AND THE VALIDATION CHECKER ARE PURE AND SHEET LOCAL, and that is // what lets `verify_api.py` compile them beside `sheetState.js` and EXECUTE the ten operators // rather than grep for their names. This component is the only place their answers meet a canvas. @@ -196,7 +196,7 @@ import type { SavedViewShadow } from "./bindingRefs"; // ⛔⛔ W51-T17 — ONE TABLE, TWO CONSUMERS. `shortcuts.ts` is the key map AND the panel's rows, and // it imports NOTHING: see its head note on tsc's emitted root. This file matches against it and // `ShortcutsPanel.tsx` renders it, so the two cannot advertise different keys. -import { arrowStep, columnBand, commitDirection, commitTarget, edgeIndex, edgeStart, +import { arrowStep, columnBand, commitDirection, commitTarget, currentRegion, edgeIndex, edgeStart, extendEdgeRect, isArrowKey, KEY_TIP_GRID_ARROW_EVENT, keyTipGridArrow, matchShortcut, rowBand, shortcutOwner, structureAxis, tabRunOrigin } from "./shortcuts"; import type { SelectionAxis, SelectionRect, ShortcutId } from "./shortcuts"; @@ -541,6 +541,23 @@ function pointRangeText(a: { col: number; row: number }, b: { col: number; row: return `${pointAddr({ col: col0, row: row0 })}:${pointAddr({ col: col1, row: row1 })}`; } +/** + * W57-T01: how far a whole-column or whole-row reference highlight paints, in one-based inclusive + * coordinates on both axes. + * + * ⛔ A CONSTANT RATHER THAN THE SHEET'S OWN EXTENT, AND THAT IS A DELIBERATE TRADE. The two memos + * that paint this band are pinned by `verify_formula_engine.py`'s RO-3 and RO-4 to close over the + * edit state and the selection and nothing else, so a bound read off `tab` would add a third input + * to both and cost the guarantee those legs exist to hold: that no outline can outlive the + * reference or the editor that made it. A fixed span costs nothing anybody can see instead. It is + * past `MAX_SHEET_COLS` (702), so a whole-ROW band is always complete; a whole-COLUMN band paints + * its first thousand rows, which is roughly thirty screenfuls at the contract 1440x900 viewport, so + * the column reads as highlighted at every scroll position a person reaches while editing one + * formula. And it stays under the 4000-cell paint cap below, so the band always paints rather than + * being skipped as pathological. + */ +const BAND_HIGHLIGHT_SPAN = 1000; + /** What the right-click menu is open ON: a whole column, or the row and column of one cell. */ interface SheetMenu { anchor: ReturnType; @@ -608,12 +625,17 @@ interface FormulaEditState { /** FA-23 #1: the dropdown is closed without touching `draft` (see `FormulaAssist`'s own note). */ dismissed: boolean; mode: "enter" | "point"; + /** W57-T04: F2 toggles native caret arrows; explicit Ctrl reference gestures still work. */ + caretOnly: boolean; /** PM-4's anchor: the EDITING cell, real 1-based coordinates. Never the selection. */ anchor: RealCell; /** Where in `draft` the live point-mode reference text starts, for PM-5's replace-not-append. */ insertStart: number | null; /** The single proposed cell while no Shift range is open. */ pointCell: RealCell | null; + /** W57-T01: set when the live proposal is a whole column or row rather than a cell or + * rectangle, so the dashed overlay can cover the band the same way the colour map does. */ + pointBand: "column" | "row" | null; /** PM-7's range anchor once Shift extends the proposal; null outside a range. */ rangeAnchor: RealCell | null; } @@ -1503,8 +1525,8 @@ export default function SheetGrid({ setMenu(null); setDropdown(null); setFormulaEdit({ - addr, inBand, viewCell, draft, caret, dropdownIndex: 0, dismissed: false, mode: "enter", - anchor, insertStart: null, pointCell: null, rangeAnchor: null, + addr, inBand, viewCell, draft, caret, dropdownIndex: 0, dismissed: false, mode: "enter", caretOnly: false, + anchor, insertStart: null, pointCell: null, pointBand: null, rangeAnchor: null, }); }, []); @@ -1588,21 +1610,40 @@ export default function SheetGrid({ * PM-4/PM-5/PM-7/PM-8: every point-mode proposal, arrived at from either the anchor (a fresh * arrow press) or the previous proposal (a continued one), goes through this ONE splice so * "insert" and "replace the last insert" are the same code path rather than two that can drift. + * W57-T01: a whole-column or whole-row proposal changes only the emitted text; it shares the + * same insertStart, replacement length and caret rule as a cell or rectangular range. */ - const proposeCell = useCallback((target: RealCell, shift: boolean) => { + const proposeCell = useCallback(( + target: RealCell, + shift: boolean, + wholeAxis?: "column" | "row", + /** W57-T02: where a NEW range starts, for a Shift-extended press made before point mode + * exists. Without it the anchor would default to `target` and a Ctrl+Shift+Arrow pressed as + * the first key after `=SUM(` would emit one cell instead of the run it selected. T02 only + * supplies this in enter mode; repeat presses keep the existing range anchor. W57-T04 + * also uses it to explicitly name a region's top-left corner while already pointing. */ + rangeFrom?: RealCell, + ) => { if (!formulaEdit) return; const insertStart = formulaEdit.mode === "point" && formulaEdit.insertStart !== null ? formulaEdit.insertStart : formulaEdit.caret; const oldLen = formulaEdit.mode === "point" ? formulaEdit.caret - insertStart : 0; + // W57-T04: an explicit rangeFrom wins so a region can name its anchor outright. This + // preserves T02: it only supplies rangeFrom in enter mode, where rangeAnchor is null by + // construction, so both orders answer identically for every existing T02 press. const rangeAnchor = shift - ? (formulaEdit.rangeAnchor ?? (formulaEdit.mode === "point" ? formulaEdit.pointCell : target)) + ? (rangeFrom ?? formulaEdit.rangeAnchor + ?? (formulaEdit.mode === "point" ? formulaEdit.pointCell : target)) : null; - const text = rangeAnchor ? pointRangeText(rangeAnchor, target) : pointAddr(target); + const text = wholeAxis === "column" ? columnRangeA1(target.col) + : wholeAxis === "row" ? rowRangeA1(target.row) + : rangeAnchor ? pointRangeText(rangeAnchor, target) : pointAddr(target); const draft = formulaEdit.draft.slice(0, insertStart) + text + formulaEdit.draft.slice(insertStart + oldLen); const caret = insertStart + text.length; setFormulaEdit({ ...formulaEdit, draft, caret, insertStart, mode: "point", pointCell: target, rangeAnchor, + pointBand: wholeAxis ?? null, }); // PM-6: the grid scrolls to keep the proposal visible. Best effort on the MAIN grid only -- // a proposal inside the frozen band is permanently on screen by construction (R17). @@ -1610,7 +1651,7 @@ export default function SheetGrid({ const viewRow = viewRowOf(target.row - 1); if (viewRow >= 0) gridRef.current?.scrollTo(viewIndexOf(visCols, target.col - 1), viewRow); } - }, [formulaEdit, viewRowOf, visCols]); + }, [columnRangeA1, formulaEdit, rowRangeA1, viewRowOf, visCols]); /** PM-9/PM-10/PM-11: `,`/`(`/`)` commit the live reference as literal text and leave point mode. */ const commitPointReference = useCallback((trailer: string) => { @@ -1619,7 +1660,7 @@ export default function SheetGrid({ + formulaEdit.draft.slice(formulaEdit.caret); setFormulaEdit({ ...formulaEdit, draft, caret: formulaEdit.caret + trailer.length, - mode: "enter", pointCell: null, rangeAnchor: null, insertStart: null, + mode: "enter", pointCell: null, pointBand: null, rangeAnchor: null, insertStart: null, }); }, [formulaEdit]); @@ -1693,17 +1734,80 @@ export default function SheetGrid({ const onFormulaKeyDown = useCallback((e: ReactKeyboardEvent) => { if (!formulaEdit) return; - const { draft, caret, mode } = formulaEdit; + const { draft, caret, mode, caretOnly } = formulaEdit; + const caretArrow = caretOnly && !e.ctrlKey && !e.metaKey && !e.altKey; + + // W57-T01 / C1: the formula input owns focus, so these keys cannot reach the grid handler. + // Use its SAME matcher and action names above point mode's early returns: both entry modes + // widen the pointed cell (or the editing anchor) through the ONE proposal splice. Passing + // false clears any rectangular range anchor; a repeated press replaces that live proposal, + // and the existing caret effect keeps focus just after the reference inside the formula. + // W57-T02: dispatch BEFORE the dropdown because it consumes Up/Down without checking Ctrl. + // Plain arrows and Tab do not match these actions; F2 can reserve the arrows for the caret + // below while the dropdown keeps its existing Tab behavior. + const hit: ShortcutId | null = matchShortcut(e); + if (hit === "select-column" || hit === "select-row") { + e.preventDefault(); + const target = mode === "point" ? (formulaEdit.pointCell ?? formulaEdit.anchor) + : formulaEdit.anchor; + proposeCell(target, false, hit === "select-column" ? "column" : "row"); + return; + } + + // W57-T04 / C1: the grid and formula editor share the same region in real coordinates. + // Only this boundary converts the zero-based rectangle to the one-based proposal splice. + if (hit === "select-region") { + e.preventDefault(); + const from = formulaEdit.pointCell ?? formulaEdit.anchor; + const region = currentRegion({ col: from.col - 1, row: from.row - 1 }, extent, + (c, r) => isFilled(tab, c, r)); + const topLeft: RealCell = { col: region.col0 + 1, row: region.row0 + 1 }; + const bottomRight: RealCell = { col: region.col1 + 1, row: region.row1 + 1 }; + proposeCell(bottomRight, true, undefined, topLeft); + return; + } + + // W57-T02 / C1: share the grid's edgeIndex walk and keep all A1 edits in proposeCell. + // RealCell is 1-based; edgeIndex and isFilled are 0-based. Convert once on entry and once + // on exit, or every reference silently lands one cell off. Point mode walks its active end; + // a NEW extended range starts at the clamped plain-arrow neighbour and walks from there. + // Enter mode only claims reference positions, preserving text navigation elsewhere. A jump + // clears rangeAnchor, while an extension keeps the existing anchor across repeat presses. + if (hit === "jump-edge" || hit === "extend-edge") { + const step = arrowStep(e.key); + if (step && (mode === "point" || isRefPosition(draft, caret))) { + e.preventDefault(); + const extending = hit === "extend-edge"; + const rangeFrom: RealCell | undefined = mode === "enter" && extending + ? { col: clampCol(formulaEdit.anchor.col + step.dCol), + row: clampRow(formulaEdit.anchor.row + step.dRow) } + : undefined; + const from = mode === "point" ? (formulaEdit.pointCell ?? formulaEdit.anchor) + : (rangeFrom ?? formulaEdit.anchor); + const fromColZero = from.col - 1; + const fromRowZero = from.row - 1; + const edgeZero = step.dRow !== 0 + ? { col: fromColZero, + row: edgeIndex(fromRowZero, extent.rows, step.dRow, + (i) => isFilled(tab, fromColZero, i)) } + : { col: edgeIndex(fromColZero, extent.cols, step.dCol, + (i) => isFilled(tab, i, fromRowZero)), + row: fromRowZero }; + const edgeCell: RealCell = { col: edgeZero.col + 1, row: edgeZero.row + 1 }; + proposeCell(edgeCell, extending, undefined, rangeFrom); + return; + } + } if (formulaDropdownOpen) { - if (e.key === "ArrowDown") { + if (e.key === "ArrowDown" && !caretArrow) { e.preventDefault(); setFormulaEdit({ ...formulaEdit, dropdownIndex: (formulaEdit.dropdownIndex + 1) % formulaSuggestions.length, }); return; } - if (e.key === "ArrowUp") { + if (e.key === "ArrowUp" && !caretArrow) { e.preventDefault(); setFormulaEdit({ ...formulaEdit, @@ -1747,8 +1851,29 @@ export default function SheetGrid({ return; } + // W57-T04: F2 switches Excel's Enter/Edit arrow behavior without changing the draft. + // Retire any live proposal before allowing native caret movement; its old insertStart + // must never be reused after the caret moves. Explicit Ctrl gestures are dispatched above. + if (e.key === "F2") { + e.preventDefault(); + setFormulaEdit({ + ...formulaEdit, caretOnly: !caretOnly, mode: "enter", pointCell: null, pointBand: null, + rangeAnchor: null, insertStart: null, + }); + return; + } + if (mode === "point") { if (ARROW_DELTA[e.key]) { + if (caretArrow) { + // A Ctrl gesture can propose while F2's caret mode is active. The next plain arrow + // retires that proposal, then the browser moves the caret without preventDefault. + setFormulaEdit({ + ...formulaEdit, mode: "enter", pointCell: null, pointBand: null, + rangeAnchor: null, insertStart: null, + }); + return; + } e.preventDefault(); const dir = ARROW_DELTA[e.key]; const from = formulaEdit.pointCell ?? formulaEdit.anchor; @@ -1772,13 +1897,18 @@ export default function SheetGrid({ // PM-14: any other typed character leaves point mode and is inserted literally. No // `preventDefault` -- the browser's own keystroke does the inserting. setFormulaEdit({ - ...formulaEdit, mode: "enter", pointCell: null, rangeAnchor: null, insertStart: null, + ...formulaEdit, mode: "enter", pointCell: null, pointBand: null, rangeAnchor: null, insertStart: null, }); return; } - if (e.key === "Backspace" || e.key === "Delete") { + // W57-T04 - MOVING THE CARET BY HAND LEAVES POINT MODE. insertStart records where the + // live proposal begins; after the reader moves elsewhere, reusing it would splice over + // unrelated characters. No preventDefault: the browser still moves or deletes, just as + // PM-14's sibling branch relies on the browser to insert the typed character. + if (e.key === "Backspace" || e.key === "Delete" + || e.key === "Home" || e.key === "End") { setFormulaEdit({ - ...formulaEdit, mode: "enter", pointCell: null, rangeAnchor: null, insertStart: null, + ...formulaEdit, mode: "enter", pointCell: null, pointBand: null, rangeAnchor: null, insertStart: null, }); } return; @@ -1786,7 +1916,7 @@ export default function SheetGrid({ // PM-4: an arrow key at a legal reference position starts point mode, anchored on the // EDITING cell (never the grid's own selection). - if (ARROW_DELTA[e.key] && isRefPosition(draft, caret)) { + if (!caretArrow && ARROW_DELTA[e.key] && isRefPosition(draft, caret)) { e.preventDefault(); const dir = ARROW_DELTA[e.key]; proposeCell( @@ -1802,8 +1932,9 @@ export default function SheetGrid({ // is capture phase on the document and stops propagation, so a branch here could never fire // anyway - it would be dead code that a later reader would trust. }, [ - acceptFormulaSuggestion, clampCol, clampRow, commitFormulaEditor, - commitPointReference, formulaDropdownOpen, formulaEdit, formulaSuggestions, proposeCell, + acceptFormulaSuggestion, arrowStep, clampCol, clampRow, commitFormulaEditor, + commitPointReference, currentRegion, edgeIndex, extent.cols, extent.rows, formulaDropdownOpen, formulaEdit, + formulaSuggestions, isFilled, matchShortcut, proposeCell, tab, ]); // The caret has to land where `formulaEdit.caret` says on every mutation, which React does not @@ -1844,11 +1975,18 @@ export default function SheetGrid({ : (selected?.cell?.kind === "formula" ? (selected.cell.formula ?? "") : ""); if (!source) return out; for (const ref of refsIn(source)) { - const c0 = colToIndex(ref.start.col); - const c1 = colToIndex(ref.end.col); - const r0 = ref.start.row; - const r1 = ref.end.row; - // A pathological typed range (a whole-column reference, say) must not stall a paint. + let c0 = colToIndex(ref.start.col); + let c1 = colToIndex(ref.end.col); + let r0 = ref.start.row; + let r1 = ref.end.row; + if (ref.band === "column") { + r0 = 1; + r1 = BAND_HIGHLIGHT_SPAN; + } else if (ref.band === "row") { + c0 = 1; + c1 = BAND_HIGHLIGHT_SPAN; + } + // A pathological typed range or a band with a huge used extent must not stall a paint. if ((Math.abs(c1 - c0) + 1) * (Math.abs(r1 - r0) + 1) > 4000) continue; const color = colorForSlot(ref.colorIndex, "light"); for (let c = Math.min(c0, c1); c <= Math.max(c0, c1); c += 1) { @@ -1880,11 +2018,20 @@ export default function SheetGrid({ * cannot say WHICH of its references is the live one: `=F6+F6` has two occurrences of the same * text and only one of them is under the arrow keys. */ + // W57-T01: a band inserted in point mode is still the live proposal; the next arrow replaces + // it with a cell reference. Dash the whole bounded band until it settles, then leave it solid. const formulaPointCells = useMemo(() => { const out = new Set(); if (!formulaEdit || formulaEdit.mode !== "point" || !formulaEdit.pointCell) return out; - const a = formulaEdit.rangeAnchor ?? formulaEdit.pointCell; - const b = formulaEdit.pointCell; + let a = formulaEdit.rangeAnchor ?? formulaEdit.pointCell; + let b = formulaEdit.pointCell; + if (formulaEdit.pointBand === "column") { + a = { col: b.col, row: 1 }; + b = { col: b.col, row: BAND_HIGHLIGHT_SPAN }; + } else if (formulaEdit.pointBand === "row") { + a = { col: 1, row: b.row }; + b = { col: BAND_HIGHLIGHT_SPAN, row: b.row }; + } // The same cap `formulaRefCells` uses: a pathological proposal must not stall a paint. if ((Math.abs(b.col - a.col) + 1) * (Math.abs(b.row - a.row) + 1) > 4000) return out; for (let c = Math.min(a.col, b.col); c <= Math.max(a.col, b.col); c += 1) { @@ -2233,6 +2380,10 @@ export default function SheetGrid({ // feature. The block is screened against the validation the TARGET already carries, the // allowed cells land, and every refusal is named in one sentence rather than forty. landFill(out, true); + if (out.tab) { + if (out.tab.charts !== tab.charts) writeCharts(out.tab.charts ?? []); + if (out.tab.rules !== tab.rules) saveRules(out.tab.rules ?? []); + } // ⭐ W52-T07: the WHOLE destination range, not just the anchor `target` glide handed over — // `values`' own shape (rows outer, cols inner) IS that range's size. setPasteHighlight({ col: realCol(target[0]), row: realRow(target[1]), @@ -2274,6 +2425,10 @@ export default function SheetGrid({ // ⛔ W51-T18 / C2 — AND THE FILL IS THE SECOND DOOR. Dragging a value down a column lands // it on every cell it passes, list or no list, so it is screened exactly as the paste is. landFill(out, true); + if (out.tab) { + if (out.tab.charts !== tab.charts) writeCharts(out.tab.charts ?? []); + if (out.tab.rules !== tab.rules) saveRules(out.tab.rules ?? []); + } }, [cellValueGetter, landFill, options, realCol, realRow, tab, viewShadows], ); @@ -2852,6 +3007,10 @@ export default function SheetGrid({ return; } landFill(out, mode === "values"); + if (out.tab) { + if (out.tab.charts !== tab.charts) writeCharts(out.tab.charts ?? []); + if (out.tab.rules !== tab.rules) saveRules(out.tab.rules ?? []); + } // ⭐ W52-T07: same highlight the glide door sets, from the ribbon's own paste path. setPasteHighlight({ col: realCol(range.col0), row: realRow(range.row0), cols: values[0]?.length ?? 0, rows: values.length }); @@ -3727,6 +3886,16 @@ export default function SheetGrid({ if (!cur) return; applySelection(rowBand(cur.cell[1], visCols.length), cur.cell); return; + case "select-region": { + if (!cur) return; + const region = currentRegion({ col: realCol(cur.cell[0]), row: realRow(cur.cell[1]) }, + extent, (c, r) => isFilled(tab, c, r)); + applySelection({ + col0: viewIndexOf(visCols, region.col0), row0: viewRowOf(region.row0), + col1: viewIndexOf(visCols, region.col1), row1: viewRowOf(region.row1), + }, cur.cell); + return; + } // -- entering and filling. Each one reaches the RIBBON's act, never a second copy. ----- case "fill-down": onRibbonAction({ kind: "fill-down" }); return; @@ -3759,6 +3928,10 @@ export default function SheetGrid({ return; } landFill(out, false); + if (out.tab) { + if (out.tab.charts !== tab.charts) writeCharts(out.tab.charts ?? []); + if (out.tab.rules !== tab.rules) saveRules(out.tab.rules ?? []); + } return; } case "line-break": { diff --git a/web/src/sheet/SheetPage.tsx b/web/src/sheet/SheetPage.tsx index 468719b0110719623c8ecc54c649a74cc7488ef5..08d8f224ecb30839b4ac8d897ee23e32f059dc48 100644 --- a/web/src/sheet/SheetPage.tsx +++ b/web/src/sheet/SheetPage.tsx @@ -40,6 +40,7 @@ import type { PointerEvent as ReactPointerEvent, ReactNode } from "react"; import "../styles/wave49/sheet.css"; import "../styles/wave50/sheet.css"; import "../styles/wave55/sheet.css"; +import "../styles/wave57/sheet-tabs.css"; // ⛔⛔ W51-T09 — THE LAUNCHER'S OWN STYLESHEET, AND IT IS NOT OPTIONAL. `.cg-chat-toggle` and // `.cg-chat-dock` live in `customer-grid/chatDock.css`, which until this ticket ONLY // `CustomerGrid.tsx` imported. On a direct load of `#/sheet` that chunk is never fetched, so the @@ -54,6 +55,7 @@ import BindingEditor from "./BindingEditor"; import DrillModal from "./DrillModal"; import type { DrillCellScope } from "./DrillModal"; import FormulaBar from "./FormulaBar"; +import { OPEN_CELL_DATA_EVENT } from "./Ribbon"; import SheetGridBase from "./SheetGrid"; import type { BoundEvaluationRequest, SheetGridProps } from "./SheetGrid"; import SheetRail from "./SheetRail"; @@ -89,6 +91,13 @@ import type { SheetHistory, StructureOp } from "./sheetState"; import { emptyIndex } from "./types"; import type { Cell, IndexRepair, SheetChartSpec, SheetRule, SheetTab, Workbook } from "./types"; +// W57-T19: the page supplies the ordered options and saved-view filters used by every copy door. +import { choiceVocabulary, topicForScope } from "../customer-grid/types"; +import { bindingOptionKey } from "./sheetState"; +import type { OptionCatalogue } from "./sheetState"; +import { bindingOnDrag } from "./types"; +import type { SavedViewShadow } from "./bindingRefs"; + /** * ⭐⭐ W51-T08 / CONTRACT C1 — THE FOUR SEAM PROPS, TYPED HERE BECAUSE `SheetGrid.tsx` IS LANE E'S. * @@ -506,6 +515,119 @@ export default function SheetPage() { /** The current drain, so the unmount flush can call it without re-arming its own effect. */ const drainRef = useRef<() => void>(() => {}); + // W57-T19: stable pair keys keep these background reads out of ordinary edits and paints. + const copyPairs = useMemo(() => { + const fields = new Set(); + const views = new Set(); + for (const sheet of book?.sheets ?? []) { + // ⛔⛔ W57-T19 - `cells` IS OPTIONAL ON A STORED TAB AND `Object.values(undefined)` + // THROWS. Measured: without this guard the whole Sheet route rendered its failure + // card on tenant #0's own workbook, and tsc, the gates and the production build were + // all green, because the shape is only wrong at RUNTIME on a tab saved without a + // cells map. A background read must never be able to take the page down with it. + for (const cell of Object.values(sheet?.cells ?? {})) { + if (cell?.kind !== "bound" || !cell.binding) continue; + const binding = cell.binding; + const database = typeof binding.database === "string" ? binding.database.trim() : ""; + if (!database) continue; + const down = bindingOnDrag(binding).down; + if (down !== "none") fields.add(bindingOptionKey(database, down.field)); + const scope = binding.scope as { view_id?: unknown } | undefined; + const viewId = typeof scope?.view_id === "string" ? scope.view_id.trim() : ""; + if (viewId) views.add(JSON.stringify([database, viewId])); + } + } + return { fields: JSON.stringify([...fields].sort()), views: JSON.stringify([...views].sort()) }; + }, [book?.sheets]); + const optionRequestKey = JSON.stringify([book?.id ?? "", copyPairs.fields]); + const shadowRequestKey = JSON.stringify([book?.id ?? "", copyPairs.views]); + const [copyOptions, setCopyOptions] = useState<{ key: string; value: OptionCatalogue }>( + { key: "", value: {} }, + ); + const [copyShadows, setCopyShadows] = useState<{ key: string; value: SavedViewShadow[] }>( + { key: "", value: [] }, + ); + const optionCatalogue = copyOptions.key === optionRequestKey ? copyOptions.value : undefined; + const viewShadows = copyShadows.key === shadowRequestKey ? copyShadows.value : undefined; + + useEffect(() => { + const [, pairsKey] = JSON.parse(optionRequestKey) as [string, string]; + const databases = new Map>(); + for (const pair of JSON.parse(pairsKey) as string[]) { + const [database, field] = JSON.parse(pair) as [string, string]; + const fields = databases.get(database) ?? new Set(); + fields.add(field); + databases.set(database, fields); + } + if (!databases.size) return; + let live = true; + const scopeOf = (database: string): SurfaceScope | null => { + if (database === "customer_data") return "customer"; + if (database === "product_data") return "product"; + return database.startsWith("ut_") ? (database as SurfaceScope) : null; + }; + void Promise.allSettled([...databases].map(async ([database, wanted]) => { + const scope = scopeOf(database); + if (!scope) return; + const workspace = await fetchWorkspace(scope); + if (!live || !workspace) return; + const fields = (workspace.fields ?? []).filter((field) => wanted.has(field.key)); + const needsRows = fields.some((field) => choiceVocabulary(field, []).length === 0); + const rows = needsRows ? (await fetchTopicRows(topicForScope(scope)))?.rows ?? [] : []; + if (!live) return; + const catalogue: OptionCatalogue = {}; + for (const field of fields) { + catalogue[bindingOptionKey(database, field.key)] = choiceVocabulary(field, rows); + } + setCopyOptions((current) => ({ + key: optionRequestKey, + value: { ...(current.key === optionRequestKey ? current.value : {}), ...catalogue }, + })); + })); + return () => { live = false; }; + }, [optionRequestKey]); + + useEffect(() => { + const [workbookId, pairsKey] = JSON.parse(shadowRequestKey) as [string, string]; + const wanted = new Set(JSON.parse(pairsKey) as string[]); + if (!workbookId || !wanted.size) return; + const bindings = new Map>(); + for (const sheet of bookRef.current?.sheets ?? []) { + // ⛔⛔ W57-T19 - `cells` IS OPTIONAL ON A STORED TAB AND `Object.values(undefined)` + // THROWS. Measured: without this guard the whole Sheet route rendered its failure + // card on tenant #0's own workbook, and tsc, the gates and the production build were + // all green, because the shape is only wrong at RUNTIME on a tab saved without a + // cells map. A background read must never be able to take the page down with it. + for (const cell of Object.values(sheet?.cells ?? {})) { + if (cell?.kind !== "bound" || !cell.binding) continue; + const binding = cell.binding; + const scope = binding.scope as { view_id?: unknown } | undefined; + const pair = JSON.stringify([String(binding.database ?? "").trim(), + String(scope?.view_id ?? "").trim()]); + if (wanted.has(pair) && !bindings.has(pair)) bindings.set(pair, binding); + } + } + const controller = new AbortController(); + void (async () => { + try { + const response = await fetch( + `/api/v1/sheets/${encodeURIComponent(workbookId)}/binding-shadows`, + { method: "POST", credentials: "same-origin", signal: controller.signal, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ bindings: [...bindings.values()] }) }, + ); + if (!response.ok) return; + const payload = await response.json() as { shadows?: SavedViewShadow[] }; + if (!controller.signal.aborted && Array.isArray(payload.shadows)) { + setCopyShadows({ key: shadowRequestKey, value: payload.shadows }); + } + } catch { + // An unavailable snapshot keeps the grid's existing missing-view copy notice. + } + })(); + return () => controller.abort(); + }, [shadowRequestKey]); + /** Write the ref and the state together. Nothing in this file calls `setBook` on its own. */ const applyLocal = useCallback((next: Workbook) => { bookRef.current = next; @@ -764,6 +886,16 @@ export default function SheetPage() { setEditAt(addr); }, []); + useEffect(() => { + const openCellData = () => { + if (!barAddr) return; + setNotice(""); + setEditAt(barAddr); + }; + document.addEventListener(OPEN_CELL_DATA_EVENT, openCellData); + return () => document.removeEventListener(OPEN_CELL_DATA_EVENT, openCellData); + }, [barAddr]); + /** * A 409 on one tab: re-read THAT TAB and say so, naming it. * @@ -2164,17 +2296,25 @@ export default function SheetPage() { const drillBindingScope = (drillBinding?.scope ?? null) as Record | null; const drillViewId = typeof drillBindingScope?.view_id === "string" ? drillBindingScope.view_id.trim() : ""; + const drillAggregate = drillBinding?.aggregate as Record | undefined; + const drillAggregateField = typeof drillAggregate?.field === "string" + ? drillAggregate.field.trim() : ""; + const drillStep = bindingOnDrag(drillBinding).down; const drillScope: DrillCellScope | null = !drillBindingScope ? null : drillViewId ? { kind: "view", name: typeof drillBindingScope.view_name === "string" ? drillBindingScope.view_name : "", + ...(drillAggregateField ? { aggregateField: drillAggregateField } : {}), + ...(drillStep !== "none" ? { stepField: drillStep.field } : {}), } : { kind: "filter", tree: (drillBindingScope.filter as FilterTree | null | undefined) ?? { conj: "and", nodes: [] }, + ...(drillAggregateField ? { aggregateField: drillAggregateField } : {}), + ...(drillStep !== "none" ? { stepField: drillStep.field } : {}), }; return ( @@ -2247,26 +2387,24 @@ export default function SheetPage() { ) : (
- {/* ⭐⭐ W51-T08 (owner instructions 8 and 10) — THE PAGE TOOLBAR IS GONE, TITLE AND ALL. - The owner's ask is that the Sheet carries no page title and that exactly ONE ribbon - sits above exactly ONE formula bar above the canvas. A `.sheet-toolbar` row holding an - `

` and a second Refresh button is a THIRD band of chrome above the ribbon, and the - workbook's name is already the selected row in the rail beside it, so the heading was - naming the page to a reader who had just clicked it. - - ⭐ REFRESH DID NOT DISAPPEAR, IT MOVED. R8 puts it on the ribbon's own row, so it is - handed down as `onRefresh` (C1) rather than drawn here. Wave-49 R12's reasoning still - holds and is why it is a ROW-level control rather than a cell one: a sheet is a - snapshot, so the one act that changes every number on screen is a page-level act. - - ⚠ `.sheet-notice` DELIBERATELY STAYS. It is a transient sentence about a refusal, not - chrome, and instructions 8 and 10 name a title and a duplicated bar. Deleting it would - take the only place a save conflict or a failed refresh is ever said out loud. */} - {notice &&

{notice}

} - - {/* ⭐⭐ W53-T08 — THE `.sheet-status` LINE THAT SAT HERE IS DELETED (instruction 14). Its - hook class went with it; `.sheet-notice` directly above is a DIFFERENT element and - survives deliberately. */} + {/* ⭐⭐ W51-T08 (owner instructions 8 and 10) — THERE IS NO PAGE TOOLBAR AND NO TITLE HERE, + AND THAT IS THE RULE, NOT AN OMISSION. Exactly ONE ribbon sits above exactly ONE formula + bar above the canvas; a `.sheet-toolbar` row holding an `

` and a second Refresh + button would be a THIRD band of chrome, and the workbook's name is already the selected + row in the rail beside it. Refresh did not disappear, it MOVED: R8 puts it on the + ribbon's own row, handed down as `onRefresh` (C1) rather than drawn here, because a + sheet is a snapshot and the one act that changes every number on screen is a page-level + act. + + ⭐⭐ W57-T15 (owner instructions 6 and 12) — AND THE NOTICE NO LONGER SITS HERE EITHER. + It used to render at the top of this column, so every refusal pushed the ribbon and the + whole grid down under the reader's cursor. It now renders just above the tab strip; see + `.sheet-tab-notices` below. + + ⭐⭐ W53-T08 — THE `.sheet-status` LINE THAT SAT HERE IS DELETED (instruction 14). Its + hook class went with it. ⚠ `verify_wiring.py:4124` still asserts that line's JSX is + present in this file and has therefore been RED since W53-T08 landed; the gate is what + is stale, not the code. */} {/* ⭐ THREE OUTCOMES, IN THE ORDER THEY CAN HAPPEN. A workbook is switching, so the spinner again; a workbook is open, so the grid; nothing opened and `loading` is over, @@ -2279,6 +2417,8 @@ export default function SheetPage() { )} + {/* `.sheet-notice` is a transient sentence about a refusal, not chrome. + Keep messages above the tab strip so they cannot move the ribbon or the grid's + origin (owner instructions 6 and 12). */} +
+ {notice &&

{notice}

} +
+ {/* ⭐⭐ THE TAB BAR (wave-49 R7), AND IT SITS AT THE BOTTOM OF THE WORKBOOK. R7 says one rail entry is a WORKBOOK WITH TABS, so the tabs belong INSIDE the workbook and never on the rail, which lists workbooks. The bottom is also Excel's own place for @@ -2396,10 +2543,22 @@ export default function SheetPage() {

diff --git a/web/src/sheet/a1.ts b/web/src/sheet/a1.ts index 1bed1872cccd429636da184f3ad796741a821601..528ef7151ee0e538a45d8d02cf9ffff2e58ffea2 100644 --- a/web/src/sheet/a1.ts +++ b/web/src/sheet/a1.ts @@ -29,6 +29,15 @@ export const indexToCol = (index: number): string => { return out; }; +/** W57-T01: a whole column from its 1-based index, e.g. 1 -> "A:A". */ +export const columnRangeA1 = (col: number): string => { + const column = indexToCol(col); + return `${column}:${column}`; +}; + +/** W57-T01: a whole row from its 1-based index, e.g. 1 -> "1:1". */ +export const rowRangeA1 = (row: number): string => `${row}:${row}`; + export interface CellRef { sheet: string; colAbs: boolean; diff --git a/web/src/sheet/bindingRefs.ts b/web/src/sheet/bindingRefs.ts index c7b0fc6f520e009c38262a4db81efce7fe609f91..ba33046de4a14b35db814aefad4c93ea1ed9b840 100644 --- a/web/src/sheet/bindingRefs.ts +++ b/web/src/sheet/bindingRefs.ts @@ -107,6 +107,84 @@ const asRecord = (value: unknown): Record | null => ? (value as Record) : null; +export type CopyAxis = "right" | "down" | "left" | "up"; + +export interface FilterStep { + binding: Record; + changed: boolean; + /** No leaf in the tree names that field. */ + noField: boolean; + /** The walk stopped at an option boundary rather than wrapping. */ + atEnd: boolean; +} + +const optionIndex = (options: readonly string[], value: string): number => + options.findIndex((option) => String(option) === value); + +/** An unknown value starts before the first option; walking past either end clamps. */ +const walkOption = ( + options: readonly string[], value: string, steps: number, +): { value: string; atEnd: boolean } => { + if (options.length === 0 || steps === 0) return { value, atEnd: false }; + const wanted = optionIndex(options, value) + steps; + const landed = Math.max(0, Math.min(options.length - 1, wanted)); + return { value: String(options[landed]), atEnd: landed !== wanted }; +}; + +/** Copy only changed paths through nested groups; unrelated clauses keep their exact records. */ +const walkFilterNodes = ( + nodes: readonly unknown[], field: string, next: (value: string) => string, +): { nodes: readonly unknown[]; found: boolean; changed: boolean } => { + let found = false; + let changed = false; + const out = nodes.map((node) => { + const record = asRecord(node); + if (!record) return node; + if (Array.isArray(record.children)) { + const inner = walkFilterNodes(record.children, field, next); + found = found || inner.found; + if (!inner.changed) return node; + changed = true; + return { ...record, children: inner.nodes }; + } + if (String(record.colId ?? "") !== field) return node; + found = true; + const value = String(record.value ?? ""); + const walked = next(value); + if (walked === value) return node; + changed = true; + return { ...record, value: walked }; + }); + return { nodes: changed ? out : nodes, found, changed }; +}; + +/** C4 / R8: right and down advance by n; left and up reverse that distance. */ +export const stepBindingFilter = ( + binding: Record, field: string, options: readonly string[], + axis: CopyAxis, n: number, +): FilterStep => { + const steps = n * (axis === "left" || axis === "up" ? -1 : 1); + const scope = asRecord(binding.scope); + const tree = asRecord(scope?.filter); + if (!tree || !Array.isArray(tree.nodes)) { + return { binding, changed: false, noField: true, atEnd: false }; + } + let atEnd = false; + const walked = walkFilterNodes(tree.nodes, field, (value) => { + const option = walkOption(options, value, steps); + atEnd = atEnd || option.atEnd; + return option.value; + }); + return { + binding: walked.changed + ? { ...binding, scope: { ...scope, filter: { ...tree, nodes: walked.nodes } } } + : binding, + changed: walked.changed, + noField: !walked.found, + atEnd, + }; +}; + /** Read the string at a path, trimmed. Empty when any step of the path is missing. */ const readAt = (root: Record, path: readonly string[]): string => { let node: Record | null = root; @@ -295,6 +373,20 @@ export const filterFieldKeys = (tree: unknown): string[] => { return [...seen]; }; +/** Fields that explain a cell's rows and value, regardless of the database's default columns. */ +export const cellForcedFieldKeys = ( + filterTree: unknown, + aggregateField?: string, + stepField?: string, +): string[] => { + const keys = new Set(filterFieldKeys(filterTree)); + for (const field of [aggregateField, stepField]) { + const key = field?.trim(); + if (key && !key.startsWith("__")) keys.add(key); + } + return [...keys]; +}; + /** * C6's visible-field law: a person's ordered choice (or the database defaults) UNION every * field the filter needs. Invalid/stale field keys are removed, and the identity key stays first. diff --git a/web/src/sheet/formulaAssist.ts b/web/src/sheet/formulaAssist.ts index 649adcaf414f1c2d4abd983b470e786a23fd74ae..63d9eb54188088557905faaabbb589bafa8ba4c6 100644 --- a/web/src/sheet/formulaAssist.ts +++ b/web/src/sheet/formulaAssist.ts @@ -198,12 +198,22 @@ export interface CellReference { end: CellRef; /** 0-5, a slot into the fixed six colour rotation in refColors.ts (CR-4). */ colorIndex: number; + /** W57-T01: a whole column or row. Endpoints carry only its axis position; a painter + * must bound the other axis against the sheet it is painting on. */ + band?: "column" | "row"; } /** One reference candidate, matched but not yet range-joined. */ const REF_TOKEN = "(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\\$?)([A-Za-z]{1,3})(\\$?)([1-9][0-9]{0,6})"; +// W57-T01: only identical axes form a band; matchBandAt checks equality after matching. +// The trailing boundary prevents A:A1 or 1:10 from being accepted as a shorter band. +const BAND_TOKEN = + "(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\\$?)" + + "([A-Za-z]{1,3}|[1-9][0-9]{0,6}):(\\$?)([A-Za-z]{1,3}|[1-9][0-9]{0,6})" + + "(?![A-Za-z0-9_$!])"; + /** * CR-1..CR-5, CR-9: every distinct cell or range reference in `text`, in order of FIRST * appearance, each carrying a colour slot from the fixed six colour rotation (CR-4) -- the same @@ -222,6 +232,7 @@ const REF_TOKEN = export function refsIn(text: string): CellReference[] { const src = String(text || ""); const scan = new RegExp(REF_TOKEN, "gy"); + const bandScan = new RegExp(BAND_TOKEN, "gy"); const out: CellReference[] = []; const slotByKey = new Map(); let nextSlot = 0; @@ -239,6 +250,36 @@ export function refsIn(text: string): CellReference[] { return { token: m[0], ref }; }; + const matchBandAt = (pos: number): { + token: string; ref: CellRef; end: CellRef; band: "column" | "row"; + } | null => { + bandScan.lastIndex = pos; + const m = bandScan.exec(src); + if (!m || m.index !== pos) return null; + if (m[4].toUpperCase() !== m[6].toUpperCase()) return null; + const after = src.slice(pos + m[0].length); + if (/^\s*\(/.test(after)) return null; + const isColumn = /^[A-Za-z]/.test(m[4]); + // parseRef stays cell-only: widening it would also change fill and structural rewrites. + const ref: CellRef = { + sheet: m[1] || m[2] || "", + colAbs: isColumn && m[3] === "$", + col: isColumn ? m[4].toUpperCase() : "A", + rowAbs: !isColumn && m[3] === "$", + row: isColumn ? 1 : Number(m[4]), + }; + return { + token: m[0], + ref, + end: { + ...ref, + colAbs: isColumn && m[5] === "$", + rowAbs: !isColumn && m[5] === "$", + }, + band: isColumn ? "column" : "row", + }; + }; + while (i < src.length) { const c = src[i]; if (quote) { @@ -262,13 +303,14 @@ export function refsIn(text: string): CellReference[] { } const prev = i ? src[i - 1] : ""; const boundary = !/[A-Za-z0-9_$]/.test(prev); - if (boundary && /[A-Za-z$']/.test(c)) { - const first = matchAt(i); + if (boundary && /[A-Za-z0-9$']/.test(c)) { + const band = matchBandAt(i); + const first = matchAt(i) ?? band; if (first) { let fullText = first.token; - let end = first.ref; + let end = band?.end ?? first.ref; let cursor = i + first.token.length; - if (src[cursor] === ":") { + if (!band && src[cursor] === ":") { const second = matchAt(cursor + 1); if (second) { fullText = `${first.token}:${second.token}`; @@ -289,6 +331,7 @@ export function refsIn(text: string): CellReference[] { start: first.ref, end, colorIndex: slot, + ...(band ? { band: band.band } : {}), }); i = cursor; continue; diff --git a/web/src/sheet/formulaEval.ts b/web/src/sheet/formulaEval.ts index 336d303df6e67d1f3695f0af6504a379ae18463b..510e532e68d50786ae6de66ecf6fa384f7e054b2 100644 --- a/web/src/sheet/formulaEval.ts +++ b/web/src/sheet/formulaEval.ts @@ -2,13 +2,13 @@ // // ⭐⭐ RULING R2: a formula that references only OTHER CELLS evaluates HERE, the instant it is // committed, and never touches the network. `evaluateFormula` is the whole of that: a small, -// self-contained engine over numeric literals, A1 cell references (with Excel's `$` tolerated), +// self-contained engine over literals, A1 cell references (with Excel's `$` tolerated), // the four operators, parentheses, and C1's five aggregate functions (SUM/COUNT/AVG/MIN/MAX) plus // COUNTA (W54-T27), each run over one simple range. It is NOT a reimplementation of the server's -// full grammar -// (`api/formula_runtime.py::evaluate_sheet`) — cross-sheet references are admitted only through -// an explicitly workbook-aware getter, and there is no function catalogue beyond the five -// aggregates. +// full grammar (`api/formula_runtime.py::evaluate_sheet`). W57-T06 adds scalar lookup results +// and comma-separated lookup arguments. Cross-sheet references are admitted only through +// an explicitly workbook-aware getter. W57-T07 adds five server-backed text/date functions +// through the same argument parser; unsupported forms still belong to Refresh. // // ⛔⛔ AND WHAT IT CANNOT ANSWER IS `unsupported`, NEVER AN ERROR. That distinction is the whole // safety of shipping a second, smaller engine beside the server's: `=IF(...)` and a qualified @@ -207,9 +207,9 @@ type Token = | { t: "fn"; v: string } | { t: "str"; v: string } | { t: "err"; v: FormulaErrorCode } - | { t: "op"; v: "+" | "-" | "*" | "/" | "(" | ")" | ":" }; + | { t: "op"; v: "+" | "-" | "*" | "/" | "(" | ")" | ":" | "," }; -const OPS = "+-*/():"; +const OPS = "+-*/():,"; /** * ⚠ A LETTER RUN IS A CELL REFERENCE ONLY WHEN DIGITS FOLLOW IT WITH NO GAP. `SUM` ahead of `(` @@ -227,7 +227,7 @@ const tokenize = (src: string, qualified = false): Token[] => { const ch = text[i]; if (/\s/.test(ch)) { i += 1; continue; } if (OPS.includes(ch)) { - tokens.push({ t: "op", v: ch as "+" | "-" | "*" | "/" | "(" | ")" | ":" }); + tokens.push({ t: "op", v: ch as "+" | "-" | "*" | "/" | "(" | ")" | ":" | "," }); i += 1; continue; } @@ -282,8 +282,8 @@ const tokenize = (src: string, qualified = false): Token[] => { continue; } // ⭐ W54-T26: A QUOTED RUN IS A TEXT LITERAL, so `=1+"x"` can say `#VALUE!` about the thing - // that is actually wrong with it. It is still only a literal: a comma, a `&` and a comparison - // all remain unreadable here, so every function call carrying one is still the server's. + // that is actually wrong with it. W57-T06 also preserves literals as lookup arguments; + // concatenation and comparisons still belong to the server. if (ch === '"' || ch === "'") { const close = text.indexOf(ch, i + 1); if (close < 0) throw new Unsupported("token"); @@ -291,7 +291,7 @@ const tokenize = (src: string, qualified = false): Token[] => { i = close + 1; continue; } - // A comma, an unrecognised `!`, a `%`, a `^`: every one of them is a formula the SERVER can read + // An unrecognised `!`, a `%`, a `^`: every one of them is a formula the SERVER can read // and this file cannot. See the header: that is `unsupported`, never an error. throw new Unsupported("token"); } @@ -856,6 +856,14 @@ export const qualifiedRecalcOrder = ( /** A number in progress, carrying whether any input it used was `pending` (last known, not fresh). */ interface Num { n: number; pending: boolean } +// W57-T06: scalar lookup answers keep their text until an operator asks for a number. +// Num and the aggregate arithmetic stay intact, including the gate's mutation anchors. +type ExpressionValue = Num | CellValue; +const expressionNumber = (value: ExpressionValue): Num => + "kind" in value ? toNum(value) : value; +const expressionCell = (value: ExpressionValue): CellValue => + "kind" in value ? value : { kind: "number", value: value.n, pending: value.pending }; + const toNum = (cv: CellValue): Num => { if (cv.kind === "number") return { n: cv.value, pending: cv.pending === true }; if (cv.kind === "text") { @@ -980,23 +988,401 @@ const rangeEndpoint = (tok: Token, name: string): string => { return tok.v; }; -const parseAtom = (c: Cursor, get: Getter): Num => { +// W57-T06: retain a rectangle's shape beside the shared row-major address expansion. +// Whole bands, named ranges and array-producing expressions still belong to Refresh. +interface LookupRange { kind: "range"; addrs: string[]; rows: number; cols: number; explicit: boolean } +interface LookupBoolean { kind: "boolean"; literal: "TRUE" | "FALSE" } +type LookupArgument = CellValue | LookupRange | LookupBoolean; +// W57-T07: one dispatch table and argument loop for lookups, text and dates. TEXTJOIN's +// variadic cap is formula_runtime.py's MANY (30); larger calls defer to its arity refusal. +const CALL_ARITY: Readonly> = { + XLOOKUP: [3, 5], VLOOKUP: [3, 4], INDEX: [2, 3], MATCH: [2, 3], CHOOSE: [2, 255], + TEXTJOIN: [3, 30], SUBSTITUTE: [3, 4], EOMONTH: [2, 2], WORKDAY: [2, 3], NETWORKDAYS: [2, 3], +}; + +const parseLookupArgument = ( + c: Cursor, get: Getter, booleanFlag: string | undefined, booleanLiterals = false, +): LookupArgument => { + const first = peekTok(c); + const next = c.tokens[c.i + 1]; + // W57-T07: reuse W57-T06's real-boolean flag for TEXTJOIN. Text arguments may also spell + // TRUE/FALSE for the server's _text conversion; a bare name outside these calls is unchanged. + if (booleanFlag || booleanLiterals) { + if (first?.t === "fn" && (first.v === "FALSE" || first.v === "TRUE") + && (isOp(next, ",") || isOp(next, ")"))) { + take(c); + return { kind: "boolean", literal: first.v }; + } + // W57-T06: the server takes a boolean and refuses a number. Numeric flags and + // expressions must reach the server, even when they would evaluate to zero here. + if (booleanFlag) throw new Unsupported(booleanFlag); + } + if (first?.t === "ref" && (isOp(next, ":") || isOp(next, ",") || isOp(next, ")"))) { + take(c); + let last = first.v; + if (isOp(peekTok(c), ":")) { + take(c); + last = rangeEndpoint(take(c), "lookup range"); + } + const a = splitRef(first.v); + const b = splitRef(last); + return { kind: "range", addrs: rangeAddresses(first.v, last), explicit: isOp(next, ":"), + rows: Math.abs(a.row - b.row) + 1, cols: Math.abs(a.col - b.col) + 1 }; + } + // W57-T06: recurse through the expression grammar; nested calls and 1+2 keep precedence. + return expressionCell(parseExpr(c, get)); +}; + +// W57-T06: match the server's _values_equal (numeric strings, then case-insensitive text). +// Do not use arithmetic's comma-stripping coercion for lookup keys: "1,000" is text there. +const lookupKey = (value: CellValue): number | string => { + if (value.kind === "number") { + if (!Number.isFinite(value.value)) throw new Unsupported("lookup number"); + return value.value; + } + if (value.kind === "empty") return ""; + if (value.kind === "pending") throw new PendingInput("lookup key"); + if (value.kind !== "text") { toNum(value); throw new Unsupported("lookup key"); } + const trimmed = value.value.trim(); + if (/^[+-]?(?:\d(?:_?\d)*(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?$/.test(trimmed)) { + const n = Number(trimmed.replace(/_/g, "")); + if (Number.isFinite(n)) return n; + } + // Python casefold and JavaScript lowercase differ for some Unicode text. Defer those + // comparisons instead of presenting an answer that can change at Refresh. + if (/[^\x00-\x7f]/.test(value.value)) throw new Unsupported("lookup Unicode comparison"); + return value.value.toLowerCase(); +}; + +// W57-T07: mirror _text without arithmetic coercion. Python's float spelling differs for +// tiny fractions and large integers; defer those instead of changing text after Refresh. +const functionText = (value: CellValue): string => { + if (value.kind === "text") return value.value; + if (value.kind === "empty") return ""; + if (value.kind === "pending") throw new PendingInput("text argument"); + if (value.kind === "number") { + const n = value.value; + if (!Number.isFinite(n) || (Number.isInteger(n) && !Number.isSafeInteger(n)) + || (n !== 0 && Math.abs(n) < 0.0001)) throw new Unsupported("text number spelling"); + return String(n); + } + toNum(value); + throw new Unsupported("text argument"); +}; + +// W57-T07: _num accepts decimal numeric text, including exponents and underscores, but +// never arithmetic's comma stripping, hex or blank-as-zero. Unicode numerals defer because +// Python float accepts spellings JavaScript Number does not. Null mirrors _num's no-number. +const functionNumber = (value: CellValue): number | null => { + if (value.kind === "pending") throw new PendingInput("numeric argument"); + if (value.kind === "empty") return null; + if (value.kind === "number") return Number.isFinite(value.value) ? value.value : null; + if (value.kind !== "text") { toNum(value); throw new Unsupported("numeric argument"); } + if (/[^\x00-\x7f]/.test(value.value)) throw new Unsupported("numeric Unicode argument"); + const trimmed = value.value.trim(); + if (!/^[+-]?(?:\d(?:_?\d)*(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?$/.test(trimmed)) return null; + const n = Number(trimmed.replace(/_/g, "")); + return Number.isFinite(n) ? n : null; +}; + +const MAX_DAY_SPAN = 20_000; +const CALENDAR_DAY_MS = 86_400_000; + +// W57-T07: dates are ISO strings, never serials or local time. Use UTC for every calendar +// operation so a reader west of Greenwich cannot see the previous date. setUTCFullYear also +// avoids Date.UTC's special 1900 offset for years 0 through 99. +const utcCalendarDate = (year: number, month: number, day: number): Date => { + const date = new Date(0); + date.setUTCFullYear(year, month - 1, day); + return date; +}; + +const functionDate = (value: CellValue): Date | null => { + if (value.kind === "pending") throw new PendingInput("date argument"); + if (value.kind === "error" || value.kind === "unsupported") toNum(value); + if (value.kind !== "text") return null; + // W57-T07: _date reads only the first ten characters, accepting timestamp suffixes. + // Other ISO spellings remain server-owned rather than using JavaScript's loose parser. + const prefix = value.value.slice(0, 10); + if (!/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/.test(prefix)) return null; + const year = Number(prefix.slice(0, 4)); + const month = Number(prefix.slice(5, 7)); + const day = Number(prefix.slice(8, 10)); + if (year < 1 || month < 1 || month > 12 || day < 1 || day > 31) return null; + const date = utcCalendarDate(year, month, day); + return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 + && date.getUTCDate() === day ? date : null; +}; + +const functionDateText = (date: Date): string => { + // W57-T07: Python dates stop at years 1..9999; its overflow outcome belongs to Refresh. + const year = date.getUTCFullYear(); + if (!Number.isFinite(year) || year < 1 || year > 9999) throw new Unsupported("date overflow"); + return date.toISOString().slice(0, 10); +}; + +const parseLookup = (name: string, c: Cursor, get: Getter): CellValue => { + const textOrDate = name === "TEXTJOIN" || name === "SUBSTITUTE" || name === "EOMONTH" + || name === "WORKDAY" || name === "NETWORKDAYS"; + let pending = false; + const observe = (value: CellValue): CellValue => { + if (value.kind === "pending" + || ((value.kind === "number" || value.kind === "text") && value.pending)) pending = true; + return value; + }; + // W57-T06: every actual read, including a preceding non-match or an argument expression, + // contributes its pending flag. Preserve workbook qualification on the wrapped getter. + const read: Getter = (addr) => observe(get(addr)); + if (get.workbookReferences) read.workbookReferences = true; + const args: LookupArgument[] = []; + take(c); // Opening parenthesis already checked by parseAtom. + if (isOp(peekTok(c), ")")) throw new Unsupported(name + " arguments"); + while (true) { + const booleanFlag = name === "VLOOKUP" && args.length === 3 ? "VLOOKUP approximate match" + : name === "TEXTJOIN" && args.length === 1 ? "TEXTJOIN ignore_empty" : undefined; + args.push(parseLookupArgument(c, read, booleanFlag, textOrDate)); + if (isOp(peekTok(c), ")")) { take(c); break; } + if (!isOp(peekTok(c), ",")) throw new Unsupported(name + " arguments"); + take(c); + } + const [min, max] = CALL_ARITY[name]; + if (args.length < min || args.length > max) throw new Unsupported(name + " arguments"); + const scalar = (index: number): CellValue => { + const arg = args[index]; + if (arg.kind === "boolean") throw new Unsupported(name + " scalar argument"); + if (arg.kind !== "range") return observe(arg); + // W57-T07: A1 is a scalar read, but even A1:A1 is explicit range syntax. Defer ranges + // at new scalar positions instead of flattening them and shifting argument meanings. + if (textOrDate && arg.explicit) throw new Unsupported(name + " scalar argument"); + if (arg.addrs.length !== 1) throw new Unsupported(name + " scalar argument"); + return read(arg.addrs[0]); + }; + const number = (index: number): number => { + const value = scalar(index); + if (value.kind === "pending") throw new PendingInput("lookup argument"); + const numeric = toNum(value); + pending = pending || numeric.pending; + if (!Number.isFinite(numeric.n)) throw new Unsupported(name + " numeric argument"); + return numeric.n; + }; + const range = (index: number): LookupRange => { + const arg = args[index]; + if (arg.kind !== "range") throw new Unsupported(name + " range argument"); + return arg; + }; + const finish = (value: CellValue): CellValue => { + observe(value); + // W57-T06: propagate a selected error here, after a nested aggregate has had its own + // chance to ignore error cells for COUNT/COUNTA. The tracking getter must not throw. + if (value.kind === "error" || value.kind === "unsupported") toNum(value); + if (pending && (value.kind === "number" || value.kind === "text")) { + return { ...value, pending: true }; + } + return pending && value.kind === "empty" ? { kind: "pending" } : value; + }; + const noMatch = (): never => { + if (pending) throw new PendingInput("lookup match"); + // W57-T06: Excel's #N/A is outside the shared four-code vocabulary. This deliberate + // deferral lets the server report no_match without inventing a fifth client error code. + throw new Unsupported(name + " no match (#N/A)"); + }; + + if (name === "TEXTJOIN" || name === "SUBSTITUTE") { + const text = (index: number): string => { + const arg = args[index]; + return arg.kind === "boolean" ? arg.literal : functionText(scalar(index)); + }; + if (name === "TEXTJOIN") { + const delimiter = text(0); + const flag = args[1]; + if (flag.kind !== "boolean") throw new Unsupported("TEXTJOIN ignore_empty"); + const parts: string[] = []; + for (let i = 2; i < args.length; i += 1) { + const part = text(i); + if (flag.literal === "FALSE" || part !== "") parts.push(part); + } + return finish({ kind: "text", value: parts.join(delimiter) }); + } + const source = text(0); + const old = text(1); + const replacement = text(2); + // W57-T07: Python inserts around characters for replace("", ...), but split("") raises + // for an instance request. Defer both empty separators rather than inventing that result. + if (old === "") throw new Unsupported("SUBSTITUTE empty old_text"); + const parts = source.split(old); + if (args.length === 3) return finish({ kind: "text", value: parts.join(replacement) }); + const instance = args[3].kind === "boolean" ? null : functionNumber(scalar(3)); + const occurrence = Math.trunc(instance ?? 0); + // W57-T07: reproduce the server's Python slices even for zero/negative occurrences; + // Excel's invalid-instance rules would give a different answer on the next Refresh. + const value = parts.slice(0, occurrence).join(old) + + (occurrence > 0 && occurrence < parts.length + ? replacement + parts.slice(occurrence).join(old) : ""); + return finish({ kind: "text", value }); + } + + if (name === "EOMONTH" || name === "WORKDAY" || name === "NETWORKDAYS") { + const date = (index: number): Date => { + const parsed = functionDate(scalar(index)); + // W57-T07: _date returns None without bad_value. Refresh owns the resulting blank or + // refusal; do not label an unparseable date #VALUE! just because JavaScript rejected it. + if (!parsed) throw new Unsupported(name + " date argument"); + return parsed; + }; + const strictNumber = (index: number): number => { + const n = functionNumber(scalar(index)); + // W57-T07: _strict_num distinguishes blank from bad_value. Defer either no-number + // case so arithmetic's zero coercion cannot turn a missing offset into a real date. + if (n === null) throw new Unsupported(name + " numeric argument"); + return Math.trunc(n); + }; + const start = date(0); + if (name === "EOMONTH") { + const months = strictNumber(1); + const index = start.getUTCMonth() + months; + const year = start.getUTCFullYear() + Math.floor(index / 12); + const month = ((index % 12) + 12) % 12 + 1; + // W57-T07: _month_shift clamps the original day, then EOMONTH discards it. Day zero + // of the following month gives that same last day, including leap years and negatives. + if (!Number.isSafeInteger(months) || year < 1 || year > 9999) { + throw new Unsupported("EOMONTH date overflow"); + } + return finish({ kind: "text", value: functionDateText(utcCalendarDate(year, month + 1, 0)) }); + } + const end = name === "NETWORKDAYS" ? date(1) : null; + const days = end ? (end.getTime() - start.getTime()) / CALENDAR_DAY_MS : strictNumber(1); + if (Math.abs(days) > MAX_DAY_SPAN) { + throw new FormulaError(ERROR_VALUE + " Working day span exceeds 20000 days."); + } + const holidays = new Set(); + if (args.length > 2) { + const arg = args[2]; + // W57-T07: the server accepts a holiday range or scalar and ignores invalid entries. + // Read every range member through read so even a pending holiday marks the answer. + if (arg.kind !== "boolean") { + const values = arg.kind === "range" ? arg.addrs.map(read) : [scalar(2)]; + for (const value of values) { + const parsed = functionDate(value); + // W57-T07: a non-ISO holiday (including an integer YYYYMMDD) might be a spelling + // Python accepts. Defer nonblank invalid entries instead of dropping a holiday. + if (!parsed && value.kind !== "empty" && !(value.kind === "text" && value.value === "")) { + throw new Unsupported(name + " holiday date"); + } + if (parsed) holidays.add(parsed.getTime()); + } + } + } + const working = (cursor: Date): boolean => cursor.getUTCDay() !== 0 + && cursor.getUTCDay() !== 6 && !holidays.has(cursor.getTime()); + const step = days >= 0 ? 1 : -1; + const cursor = new Date(start.getTime()); + if (end) { + let counted = 0; + // W57-T07: both ends count, including the start when dates run backwards. + while (true) { + if (working(cursor)) counted += 1; + if (cursor.getTime() === end.getTime()) break; + cursor.setUTCDate(cursor.getUTCDate() + step); + } + return finish({ kind: "number", value: counted === 0 ? 0 : step * counted }); + } + // W57-T07: WORKDAY steps before testing; zero days returns the start even on a weekend. + let remaining = Math.abs(days); + while (remaining > 0) { + cursor.setUTCDate(cursor.getUTCDate() + step); + if (cursor.getUTCFullYear() < 1 || cursor.getUTCFullYear() > 9999) { + throw new Unsupported("WORKDAY date overflow"); + } + if (working(cursor)) remaining -= 1; + } + return finish({ kind: "text", value: functionDateText(cursor) }); + } + + if (name === "CHOOSE") { + const index = Math.trunc(number(0)); + if (index < 1 || index >= args.length) { + throw new FormulaError(ERROR_VALUE + " CHOOSE index is outside its list."); + } + return finish(scalar(index)); + } + if (name === "INDEX") { + const array = range(0); + let row = Math.trunc(number(1)); + let col = args.length > 2 ? Math.trunc(number(2)) : 1; + if (args.length === 2) { + if (array.rows === 1) { col = row; row = 1; } + else if (array.cols !== 1) throw new Unsupported("INDEX array result"); + } + if (row < 1 || row > array.rows || col < 1 || col > array.cols) { + // W57-T06: `#VALUE!` rather than Excel's `#REF!`, on purpose. `formula_runtime.py` answers + // `bad_value` for a position outside the range and `types.ts` maps that to `#VALUE!`, and the + // server recomputes every formula at Refresh. Matching Excel here would make the token change + // under the reader a moment later. Excel parity is the SERVER's to fix, in one place, later. + throw new FormulaError(ERROR_VALUE + " INDEX position is outside its range."); + } + return finish(read(array.addrs[(row - 1) * array.cols + col - 1])); + } + + const wanted = lookupKey(scalar(0)); + const array = range(1); + let resultRange: LookupRange | undefined; + let col = 1; + // W57-T06: approximate modes require sorted-list and ordering parity beyond the worked + // cases. Keep VLOOKUP's TRUE/default, MATCH's +/-1/default and nonzero XLOOKUP modes on + // the server instead of silently treating an approximate request as an exact request. + if (name === "VLOOKUP") { + const flag = args[3]; + if (flag?.kind !== "boolean" || flag.literal !== "FALSE") throw new Unsupported("VLOOKUP approximate match"); + col = Math.trunc(number(2)); + if (col < 1) throw new FormulaError(ERROR_VALUE + " VLOOKUP column must be at least one."); + // W57-T06: the same server bad_value bounds mapping as INDEX above. + if (col > array.cols) throw new FormulaError(ERROR_VALUE + " VLOOKUP column is outside its table."); + } else { + if (array.rows !== 1 && array.cols !== 1) throw new Unsupported(name + " lookup vector"); + if (name === "MATCH") { + if (args.length < 3 || number(2) !== 0) throw new Unsupported("MATCH approximate match"); + } else { + resultRange = range(2); + if (resultRange.rows !== 1 && resultRange.cols !== 1) throw new Unsupported("XLOOKUP array result"); + if (array.addrs.length !== resultRange.addrs.length) { + throw new FormulaError(ERROR_VALUE + " XLOOKUP ranges must have the same size."); + } + if (args.length > 4 && number(4) !== 0) throw new Unsupported("XLOOKUP match mode"); + } + } + const count = name === "VLOOKUP" ? array.rows : array.addrs.length; + for (let i = 0; i < count; i += 1) { + const offset = name === "VLOOKUP" ? i * array.cols : i; + if (lookupKey(read(array.addrs[offset])) !== wanted) continue; + if (name === "MATCH") return finish({ kind: "number", value: i + 1 }); + const address = resultRange ? resultRange.addrs[i] : array.addrs[offset + col - 1]; + return finish(read(address)); + } + if (name === "XLOOKUP" && args.length > 3) { + const fallback = scalar(3); + if (fallback.kind !== "empty") return finish(fallback); + } + return noMatch(); +}; + +const parseAtom = (c: Cursor, get: Getter): ExpressionValue => { const tk = take(c); if (tk.t === "num") return { n: tk.v, pending: false }; - if (tk.t === "ref") return toNum(get(tk.v)); + if (tk.t === "ref") return get(tk.v); // ⭐⭐ W54-T26: AN ERROR PROPAGATES THROUGH ARITHMETIC, EXACTLY AS EXCEL DOES. Reading one is // the same as reading a cell that holds one, so it takes the same road out: a thrown // `FormulaError` whose message already states the code, which `errorValue` passes through. if (tk.t === "err") throw new FormulaError(tk.v); // ⭐ W54-T26: a text literal coerces the way a text CELL does, so `=1+"1"` is 2 and `=1+"x"` // is `#VALUE!`, which is Excel's own answer to both. - if (tk.t === "str") return toNum({ kind: "text", value: tk.v }); + if (tk.t === "str") return { kind: "text", value: tk.v }; if (tk.t === "fn") { const name = tk.v; // ⭐⭐ W54-T26: A BARE NAME IS NOT A CALL AND NOTHING CAN RESOLVE IT. `=FOO` names a thing, // and NEITHER engine has named ranges, so no round trip to the server can turn it into a // value. That makes it the one name this file may refuse on its own: `#NAME?`. if (!isOp(peekTok(c), "(")) throw new FormulaError(NAME_NOT_FOUND); + if (Object.prototype.hasOwnProperty.call(CALL_ARITY, name)) return parseLookup(name, c, get); // ⭐ EVERY OTHER FUNCTION IN THE CATALOGUE IS THE SERVER'S. `IF`, `ROUND`, `TEXT` and the // rest are real formulas this file does not speak, so they take the `unsupported` road home // and keep working exactly as they did before this ticket (see the header). @@ -1005,6 +1391,13 @@ const parseAtom = (c: Cursor, get: Getter): Num => { if (!AGGREGATE_SET.has(name)) throw new Unsupported(name); take(c); const from = rangeEndpoint(take(c), name); + // W57-T03 / P1 F2: one cell is a legal aggregate input in Excel and on the server. + // COUNT(A6) and SUM(A6) can compute here without waiting for Refresh. Only one cell or + // the existing range is admitted; argument lists and whole-band references stay unsupported. + if (isOp(peekTok(c), ")")) { + take(c); + return evalAggregate(name, [from], get); + } if (!isOp(peekTok(c), ":")) throw new Unsupported(name); take(c); const to = rangeEndpoint(take(c), name); @@ -1021,45 +1414,47 @@ const parseAtom = (c: Cursor, get: Getter): Num => { throw new Unsupported("token"); }; -const parseUnary = (c: Cursor, get: Getter): Num => { +const parseUnary = (c: Cursor, get: Getter): ExpressionValue => { if (isOp(peekTok(c), "-")) { take(c); - const inner = parseUnary(c, get); + const inner = expressionNumber(parseUnary(c, get)); return { n: -inner.n, pending: inner.pending }; } - if (isOp(peekTok(c), "+")) { take(c); return parseUnary(c, get); } + if (isOp(peekTok(c), "+")) { take(c); return expressionNumber(parseUnary(c, get)); } return parseAtom(c, get); }; -const parseTerm = (c: Cursor, get: Getter): Num => { - let left = parseUnary(c, get); +const parseTerm = (c: Cursor, get: Getter): ExpressionValue => { + let value = parseUnary(c, get); while (isOp(peekTok(c), "*") || isOp(peekTok(c), "/")) { const op = take(c); - const right = parseUnary(c, get); + const left = expressionNumber(value); + const right = expressionNumber(parseUnary(c, get)); if (op.t === "op" && op.v === "/") { // ⛔ A DIVISOR THAT IS ONLY ZERO BECAUSE IT HAS NOT LOADED IS NOT A DIVISION BY ZERO. It is // a value nobody has yet, so the whole formula waits rather than accusing the reader of // dividing by something they never wrote. if (right.n === 0 && right.pending) throw new PendingInput("divisor"); if (right.n === 0) throw new FormulaError("This formula divides by zero."); - left = { n: left.n / right.n, pending: left.pending || right.pending }; + value = { n: left.n / right.n, pending: left.pending || right.pending }; } else { - left = { n: left.n * right.n, pending: left.pending || right.pending }; + value = { n: left.n * right.n, pending: left.pending || right.pending }; } } - return left; + return value; }; -const parseExpr = (c: Cursor, get: Getter): Num => { - let left = parseTerm(c, get); +const parseExpr = (c: Cursor, get: Getter): ExpressionValue => { + let value = parseTerm(c, get); while (isOp(peekTok(c), "+") || isOp(peekTok(c), "-")) { const op = take(c); - const right = parseTerm(c, get); + const left = expressionNumber(value); + const right = expressionNumber(parseTerm(c, get)); const plus = op.t === "op" && op.v === "+"; - left = { n: plus ? left.n + right.n : left.n - right.n, + value = { n: plus ? left.n + right.n : left.n - right.n, pending: left.pending || right.pending }; } - return left; + return value; }; /** @@ -1078,8 +1473,8 @@ const cleanNumber = (n: number): number => { * accepted so a caller can hand this either the raw stored formula or its own trimmed draft. * * ⭐ A BARE SINGLE REFERENCE PASSES THROUGH UNCOERCED. `=F6` of a text cell is F6's text, not a - * refusal: only once an operator or a function is involved does this coerce through arithmetic, - * exactly as Excel's own distinction between "reads a cell" and "computes with a cell" works. + * refusal. W57-T06 also passes scalar lookup text through; only numeric arguments and + * arithmetic operators request coercion. * * ⛔ NEVER THROWS. Every outcome — a token this engine does not speak, a function that belongs to * the server, a division by zero, a reference to a cell held in a genuine error state — comes @@ -1100,8 +1495,13 @@ export const evaluateFormula = (src: string, get: Getter): CellValue => { return one.t === "ref" ? get(one.v) : { kind: "empty" }; } const cursor: Cursor = { tokens, i: 0 }; - const result = parseExpr(cursor, get); + const expression = parseExpr(cursor, get); if (!atEnd(cursor)) throw new Unsupported("trailing tokens"); + // W57-T06: a lookup's text or empty result reaches the cell without numeric coercion. + // Arithmetic has already called expressionNumber, which reuses the existing toNum rules. + if ("kind" in expression && (expression.kind === "text" || expression.kind === "empty" + || expression.kind === "pending")) return expression; + const result = expressionNumber(expression); if (!Number.isFinite(result.n)) { return { kind: "error", message: "This formula could not be calculated." }; } diff --git a/web/src/sheet/functionCatalog.ts b/web/src/sheet/functionCatalog.ts index 439446ff34e6cbd210c9883feea0e51b11028025..2b3a35160f4c8baa21bb1c315a664d2d80fca0f5 100644 --- a/web/src/sheet/functionCatalog.ts +++ b/web/src/sheet/functionCatalog.ts @@ -168,6 +168,18 @@ const FUNCTION_DEFS: FunctionDef[] = [ args: [{ name: "value" }, { name: "value_if_error" }], summary: "Returns a fallback value when the first argument is an error.", }, + { + name: "XLOOKUP", + category: "Lookup", + args: [ + { name: "lookup_value" }, + { name: "lookup_array" }, + { name: "return_array" }, + { name: "if_not_found", optional: true }, + { name: "match_mode", optional: true }, + ], + summary: "Finds a value in one range and returns the matching value from another, with an answer of your own when nothing matches.", + }, { name: "VLOOKUP", category: "Lookup", @@ -189,6 +201,13 @@ const FUNCTION_DEFS: FunctionDef[] = [ ], summary: "Returns a value at a given position in a range.", }, + { + name: "CHOOSE", + category: "Lookup", + args: [{ name: "index_num" }, { name: "value1" }, { name: "value2", optional: true }], + repeats: true, + summary: "Returns the value at the position you name in a list.", + }, { name: "CONCATENATE", category: "Text", @@ -196,6 +215,30 @@ const FUNCTION_DEFS: FunctionDef[] = [ repeats: true, summary: "Joins several text values into one text value.", }, + // W57-T07: offer only server-backed text functions, with hints built by buildSignature. + { + name: "TEXTJOIN", + category: "Text", + args: [ + { name: "delimiter" }, + { name: "ignore_empty" }, + { name: "text1" }, + { name: "text2", optional: true }, + ], + repeats: true, + summary: "Joins text values with a delimiter between them, optionally skipping the empty ones.", + }, + { + name: "SUBSTITUTE", + category: "Text", + args: [ + { name: "text" }, + { name: "old_text" }, + { name: "new_text" }, + { name: "instance_num", optional: true }, + ], + summary: "Replaces old text with new text in a text value.", + }, { name: "ROUND", category: "Math", @@ -232,6 +275,25 @@ const FUNCTION_DEFS: FunctionDef[] = [ args: [{ name: "text" }, { name: "num_chars", optional: true }], summary: "Returns the rightmost characters from a text value.", }, + // W57-T07: these date functions return ISO dates or counts on both sides of Refresh. + { + name: "EOMONTH", + category: "Date", + args: [{ name: "date" }, { name: "months" }], + summary: "Returns the last day of the month a number of months away from a date.", + }, + { + name: "WORKDAY", + category: "Date", + args: [{ name: "start_date" }, { name: "days" }, { name: "holidays", optional: true }], + summary: "Returns the date a number of working days away from a date, skipping weekends and any holidays you name.", + }, + { + name: "NETWORKDAYS", + category: "Date", + args: [{ name: "start_date" }, { name: "end_date" }, { name: "holidays", optional: true }], + summary: "Counts the working days between two dates, skipping weekends and any holidays you name.", + }, { name: "TODAY", category: "Date", diff --git a/web/src/sheet/ribbonData.tsx b/web/src/sheet/ribbonData.tsx index d60bbd38d6d0d813a3624641fd65be28cbbf1e61..9aa93abac3fd5956567278bffcb34f1d69e93b03 100644 --- a/web/src/sheet/ribbonData.tsx +++ b/web/src/sheet/ribbonData.tsx @@ -2,13 +2,280 @@ // stay within Sheet's existing executable actions; unavailable audit commands remain visible and // honest rather than becoming no-op affordances. -import type { RibbonTabContext } from "./Ribbon"; +import { useState } from "react"; +import { AnchoredOverlay } from "../customer-grid/OverlaySurface"; +import type { RibbonInsertDoor, RibbonTabContext } from "./Ribbon"; +import { usedRange } from "./sheetExport"; +import { addressOf, setLiteral } from "./sheetState"; +import { asNumber, MAX_SHEET_COLS } from "./types"; +import type { Cell, CellBinding } from "./types"; +import { listText, parseList, validationOf } from "./validation"; -const DATA = ["data-refresh"] as const; +const DATA = ["data-refresh", "data-queries-connections", "data-sort-asc", "data-sort-desc", "data-validation", "data-text-to-columns", "data-flash-fill", "data-remove-duplicates"] as const; const OUTLINE = ["data-hide-rows", "data-hide-columns", "data-unhide"] as const; export const DATA_KEYS = [...DATA, ...OUTLINE] as const; +type DataRows = (Cell | null)[][]; +type DataKey = number | string | null; + +const hasDataRange = (selection: RibbonInsertDoor["selection"] | undefined): boolean => + !!selection && (selection.col0 !== selection.col1 || selection.row0 !== selection.row1); + +/** + * Snapshot before writing, so moving a row cannot overwrite a later row's source. The door is + * already in sheet coordinates, including hidden rows and columns; no view-index conversion here. + * Excel would offer to widen the selection. Sort exactly the selected rectangle: silently widening + * it is the dangerous half of that dialog, because it moves cells the person did not select. + */ +function transformSelectionRows( + door: RibbonInsertDoor | undefined, + transform: (rows: DataRows) => DataRows, +): { before: number; after: number } | null { + const selection = door?.selection; + if (!door || !selection || !hasDataRange(selection)) return null; + const col0 = Math.min(selection.col0, selection.col1); + const row0 = Math.min(selection.row0, selection.row1); + const width = Math.abs(selection.col1 - selection.col0) + 1; + const height = Math.abs(selection.row1 - selection.row0) + 1; + const rows: DataRows = Array.from({ length: height }, (_, row) => + Array.from({ length: width }, (_, col) => door.cellAt(col0 + col, row0 + row))); + const next = transform(rows); + const edits: Record = {}; + for (let row = 0; row < height; row += 1) { + for (let col = 0; col < width; col += 1) { + // Include every address, especially the shortened tail: omitted edits leave stale cells. + edits[addressOf(col0 + col, row0 + row)] = next[row]?.[col] ?? null; + } + } + // The same batch door as paste makes the whole action one history step and one save. + door.onCells(edits); + return { before: height, after: next.length }; +} + +/** Sort and dedupe agree on values, never formula source, formatting, or binding definitions. */ +function dataKey(cell: Cell | null): DataKey { + if (!cell || cell.kind === "empty") return null; + // Bound cells use their last computed result, even when stale, then their displayed text. + // A missing value is not zero; an empty computed string also wins over old source text. + const value = cell.value ?? (cell.kind === "bound" + ? cell.display ?? cell.text + : cell.text ?? cell.display) ?? ""; + if (value === "") return null; + // Typed numbers are stored as strings in Sheet, so use the same numeric reader as its cells. + const number = asNumber(value); + return number === null ? String(value).toLowerCase() : number; +} + +/** dataKey normalizes comparisons; text operations need the same value precedence verbatim. */ +function dataText(cell: Cell | null): string { + if (!cell || cell.kind === "empty") return ""; + const value = cell.value ?? (cell.kind === "bound" + ? cell.display ?? cell.text + : cell.text ?? cell.display) ?? ""; + return String(value); +} + +function compareDataKeys(left: DataKey, right: DataKey, direction: 1 | -1): number { + // Blanks stay last in BOTH directions; only the nonblank ordering may be reversed. + if (left === null) return right === null ? 0 : 1; + if (right === null) return -1; + const order = typeof left === "number" + ? typeof right === "number" ? left - right : -1 + : typeof right === "number" ? 1 : left.localeCompare(right); + return direction * order; +} + +function sortRows(rows: DataRows, direction: 1 | -1): DataRows { + return rows.map((cells, index) => ({ cells, index, key: dataKey(cells[0] ?? null) })) + // Original indices break ties outside the direction multiplier, preserving equal-key order. + .sort((left, right) => compareDataKeys(left.key, right.key, direction) || left.index - right.index) + .map(({ cells }) => cells); +} + +function uniqueRows(rows: DataRows): DataRows { + const seen = new Set(); + return rows.filter((cells) => { + // JSON escapes every embedded NUL, so this separator cannot occur in an encoded cell key. + // It also keeps numeric and text keys distinct without mistaking column boundaries for data. + const key = cells.map((cell) => JSON.stringify(dataKey(cell))).join("\u0000"); + if (seen.has(key)) return false; + seen.add(key); + // Filtering in source order retains the first entire cell row, including its styles/bindings. + return true; + }); +} + +function splitTextToColumns(door: RibbonInsertDoor | undefined, delimiter: string): string { + const selection = door?.selection; + if (!door || !selection || !hasDataRange(selection)) return "Nothing was split; select a range of cells."; + const col0 = Math.min(selection.col0, selection.col1); + const row0 = Math.min(selection.row0, selection.row1); + const row1 = Math.max(selection.row0, selection.row1); + const splitRows = Array.from({ length: row1 - row0 + 1 }, (_, row) => { + const cell = door.cellAt(col0, row0 + row); + if (!cell || dataKey(cell) === null) return null; + const pieces = dataText(cell).split(delimiter); + return pieces.length > 1 ? pieces : null; + }); + const count = splitRows.filter((pieces) => pieces !== null).length; + if (count === 0) return "0 rows were split because no values contained the chosen delimiter."; + const width = splitRows.reduce((widest, pieces) => Math.max(widest, pieces?.length ?? 1), 1); + if (col0 + width > MAX_SHEET_COLS) return "Nothing was split because the result would exceed the sheet's column limit."; + + // Check the entire output rectangle before any write, including cells outside the selection. + for (let row = row0; row <= row1; row += 1) { + for (let col = col0 + 1; col < col0 + width; col += 1) { + const cell = door.cellAt(col, row); + // A blank formula or binding still owns content, even when its current result is empty. + if (cell && (dataKey(cell) !== null || cell.kind === "formula" || cell.kind === "bound" || cell.sparkline)) { + return `Nothing was split because ${addressOf(col, row)} already contains data.`; + } + } + } + + // Reuse the existing snapshot and batch writer with the checked output range in sheet coordinates. + transformSelectionRows({ ...door, selection: { col0, row0, col1: col0 + width - 1, row1 } }, (rows) => + rows.map((cells, row) => { + const pieces = splitRows[row]; + if (!pieces) return cells; + return cells.map((cell, col) => { + const piece = pieces[col]; + return piece === undefined ? cell : setLiteral(cell, piece); + }); + })); + return `${count} ${count === 1 ? "row was" : "rows were"} split into columns.`; +} + +type FlashFillExample = { sources: readonly string[]; target: string }; +type FlashFillTransform = (sources: readonly string[]) => string | null; + +/** Include overlapping occurrences and empty text, but stop at the end even for an empty match. */ +function* textPositions(text: string, value: string, from = 0): Generator { + let position = text.indexOf(value, from); + while (position !== -1 && position <= text.length) { + yield position; + if (position === text.length) return; + position = text.indexOf(value, position + 1); + } +} + +function fitFlashFillAffixes( + examples: readonly FlashFillExample[], + read: FlashFillTransform, +): FlashFillTransform | null { + const first = examples[0]; + if (!first) return null; + const value = read(first.sources); + if (value === null) return null; + // A repeated source can occur in an affix too; try every placement before refusing this rule. + for (const position of textPositions(first.target, value)) { + const prefix = first.target.slice(0, position); + const suffix = first.target.slice(position + value.length); + const transform: FlashFillTransform = (sources) => { + const part = read(sources); + return part === null ? null : prefix + part + suffix; + }; + if (examples.every((example) => transform(example.sources) === example.target)) return transform; + } + return null; +} + +function inferFlashFill(examples: readonly FlashFillExample[]): FlashFillTransform | null { + const first = examples[0]; + if (!first) return null; + const width = first.sources.length; + + // Family order is intentional: take the first rule that explains EVERY example exactly. + for (let col = 0; col < width; col += 1) { + const transform = fitFlashFillAffixes(examples, (sources) => sources[col] ?? null); + if (transform) return transform; + } + const delimiters = [" ", ",", ";", "\t", "-", "@", "/", "."] as const; + for (let col = 0; col < width; col += 1) { + for (const delimiter of delimiters) { + const pieces = first.sources[col]?.split(delimiter) ?? []; + for (let piece = 0; piece < pieces.length; piece += 1) { + const transform = fitFlashFillAffixes(examples, (sources) => + sources[col]?.split(delimiter)[piece] ?? null); + if (transform) return transform; + } + } + } + // Ordered pairs also permit a person to type the right source before the left source. + for (let left = 0; left < width; left += 1) { + const leftValue = first.sources[left]; + if (leftValue === undefined) continue; + for (let right = 0; right < width; right += 1) { + const rightValue = first.sources[right]; + if (left === right || rightValue === undefined) continue; + for (const start of textPositions(first.target, leftValue)) { + const end = start + leftValue.length; + for (const next of textPositions(first.target, rightValue, end)) { + const prefix = first.target.slice(0, start); + const separator = first.target.slice(end, next); + const suffix = first.target.slice(next + rightValue.length); + const transform: FlashFillTransform = (sources) => { + const leftPart = sources[left]; + const rightPart = sources[right]; + return leftPart === undefined || rightPart === undefined + ? null : prefix + leftPart + separator + rightPart + suffix; + }; + if (examples.every((example) => transform(example.sources) === example.target)) return transform; + } + } + } + } + return null; +} + +function flashFillSelection(door: RibbonInsertDoor | undefined): string { + const selection = door?.selection; + if (!door || !selection || !hasDataRange(selection)) return "Nothing was filled; select a range of cells."; + const col0 = Math.min(selection.col0, selection.col1); + const row0 = Math.min(selection.row0, selection.row1); + const row1 = Math.max(selection.row0, selection.row1); + if (col0 === 0) return "Nothing was filled because there is no source column to the left; select a target column to the right of your source data."; + + // Source columns can be outside the selection; snapshot them before inferring or writing. + const rows = Array.from({ length: row1 - row0 + 1 }, (_, index) => { + const cell = door.cellAt(col0, row0 + index); + return { + cell, + target: dataText(cell), + sources: Array.from({ length: col0 }, (_, col) => dataText(door.cellAt(col, row0 + index))), + }; + }); + const examples = rows.filter((row) => row.target !== ""); + if (examples.length === 0) return "Nothing was filled because there is no example; type an example in the target column within your selection."; + const pending = rows.filter((row) => row.target === ""); + if (pending.length === 0) return "Nothing was filled because there are no empty target cells left; include empty rows in your selection."; + // As with Text to columns, a blank computed result does not make a formula or binding disposable. + if (pending.some(({ cell }) => cell && (cell.kind === "formula" || cell.kind === "bound" || cell.sparkline))) { + return "Nothing was filled because an empty-looking target contains a formula, binding or sparkline; select empty target cells or clear their contents first."; + } + const transform = inferFlashFill(examples); + if (!transform) return "The examples do not agree on one rule, so nothing was changed; correct the examples or select a smaller range with a consistent pattern."; + + // Precompute ALL outputs: a missing piece in even one fill row must never cause a partial write. + const fills = new Map(); + for (const [index, row] of rows.entries()) { + if (row.target !== "") continue; + const value = transform(row.sources); + if (value === null || value === "") { + return "Nothing was filled because some rows lack the source text needed by the examples; complete those source cells or select a smaller range."; + } + fills.set(index, value); + } + // Only the target column is written; the shared batch helper preserves one history/save step. + transformSelectionRows({ ...door, selection: { col0, row0, col1: col0, row1 } }, (cells) => + cells.map((row, index) => { + const value = fills.get(index); + return value === undefined ? row : [setLiteral(row[0], value)]; + })); + return `${fills.size} ${fills.size === 1 ? "row was" : "rows were"} filled.`; +} + function IconRefresh() { return ( + {control( + "data-validation", "Data validation", <>{word("Data validation")}, + (event) => { + if (open === "data-validation") { closePop(); return; } + const selection = insert?.selection; + if (!insert || !selection) return; + const anchor = insert.cellAt(Math.min(selection.col0, selection.col1), Math.min(selection.row0, selection.row1)); + setText(listText(validationOf(anchor)?.list)); + setResult(null); + openPop("data-validation")(event); + }, + false, open === "data-validation", "V", "gallery", !insert?.selection, + )} + {open === "data-validation" && ( + +
{ event.preventDefault(); submit(); }}> + +