diff --git a/RELEASES.json b/RELEASES.json index d9de177f8432c59756c861426014a922d9400bf1..01906323953c9be51d549654505d33979282de04 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,4 +1,4 @@ { - "current": "9fce6c7e7", + "current": "b876ebd5f145", "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index 48cafc20ce2e86e4ddb9b9cc5f5a7907856df688..a10ff64346755f464c988d89aea59856f06bd50d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9fce6c7e7 +b876ebd5f145 diff --git a/api/amazon_relational.py b/api/amazon_relational.py index e16b31bec231a90f9443f3706f991a51381d8646..f9b8990489be5139c9ef46d3279d1cb7aecb78ec 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -734,7 +734,7 @@ def _read_through(cur, key, count, *, columns=None): if key in READ_THROUGH or count > ut.MAX_ROWS: return True if not count: - return key == PREFIX + "inventory" + return False names = [name for name, _dtype in (columns if columns is not None else mirror_columns(cur, SOURCE[key]))] sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall() @@ -1106,13 +1106,6 @@ def plan(cur, rt=None): n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0] except Exception: # noqa: BLE001 continue # absent = never synced - if key == PREFIX + "inventory" and not n: - # A measured empty inventory report remains a declared database with a source reason. - status = amazon_store.inventory_table_status(cur) - built[key] = {"readThrough": True, - "fields": _fields_for(cur, key, read_through=True), "count": n, - "sourceStatus": {name: status[name] for name in ("empty", "code", "reason")}} - continue if _read_through(cur, key, n): built[key] = {"readThrough": True, "fields": _fields_for(cur, key, read_through=True), "count": n} @@ -1419,7 +1412,7 @@ def apply_plan(rt, built, username="amazon", today=None): not_stored.clear() live_columns = {key: {field["key"] for field in table.get("fields", ())} for key, table in doc.items() if key in SOURCE and isinstance(table, dict)} - # Empty materialized snapshots reconcile existing definitions and create none. + # S78-T02: an empty population reconciles an EXISTING definition only; it creates none. unborn = {key for key, _label, _fields, rows in plans if isinstance(rows, list) and not rows and not isinstance(doc.get(key), dict)} live_columns.update({key: {field["key"] for field in fields} @@ -1513,12 +1506,6 @@ def apply_plan(rt, built, username="amazon", today=None): saved = stored.get(row["_id"]) if numeric and isinstance(saved, dict): saved.update({name: row[name] for name in numeric if name in row}) - if key == PREFIX + "inventory": - status = rows.get("sourceStatus") if isinstance(rows, dict) else None - if status is not None: - doc[key]["sourceStatus"] = dict(status) - else: - doc[key].pop("sourceStatus", None) # DPP 1.8 attribution is explicit in the stored definition, not inferred from a # mutable display label or from the process having imported this module. doc[key]["connector"] = source_attribution(key) diff --git a/api/application_registry.py b/api/application_registry.py index 878aa790f1238b776a98b99bae9bf14af4241218..3dada57fa0c0a27d53239e5f8ffb608c5ed87bdb 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -1415,16 +1415,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True) -def _nav_placement(app, index): - """Put default Project management entries first without rewriting stored placements.""" - placement = app.get("navPlacement") - if (app.get("templateKind") == "amazon_profit" - and (not placement or placement == {"section": "app", "order": 0})): - # A zero placement already asks for first place. Break its default tie in the rail. - return {"section": "app", "order": -1} - return placement or {"section": "app", "order": index} - - def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None): """The App home read: stored records the caller owns or was shared, never a code projection.""" return [{"id": app["id"], "name": app.get("name"), @@ -1435,7 +1425,7 @@ def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_da "origin": "tenant", "compatibleDatabases": app.get("compatibleDatabases") or [], "openTarget": app.get("openTarget"), - "navPlacement": _nav_placement(app, index), + "navPlacement": app.get("navPlacement") or {"section": "app", "order": index}, "capabilities": app.get("capabilities"), "starred": app.get("starred"), "isDefault": app.get("isDefault")} for index, app in enumerate(list_apps(runtime, user, is_admin, diff --git a/api/automation_engine.py b/api/automation_engine.py index f18bb3403cb3092ef0c6e318d750f2d18722b8cf..7a44755613cddab377b6c2da76d77c3676049a96 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -2808,12 +2808,9 @@ def alert_principal(rt, username): from core import account_rows, users resolve = getattr(rt, '_document_store', None) record = account_rows.get(username, st=resolve() if callable(resolve) else rt, fresh=True) - if not record or record.get('active', True) is False: - raise PermissionError('This alert owner is no longer active in this workspace.') - principal = users._public(username, record) - if principal['tenant'] != rt.key: + if not record or record.get('active', True) is False or record.get('tenant') != rt.key: raise PermissionError('This alert owner is no longer active in this workspace.') - return principal + return users._public(username, record) def alert_schema(rt, principal, database): @@ -14611,16 +14608,6 @@ def _mirror_table(rt, key, defn): return table, None -def _relation_native_rows(rt, key): - """Read omitted native rows only through the caller's tenant runtime.""" - if not _relation_tenant(rt): - raise RuntimeError('A tenant runtime is required to read relation rows.') - table = _ut().get_one(key, st=rt) - if table is not None and 'rows' not in table: - raise RuntimeError('The relation row reader returned only a definition.') - return table - - def _resolve_table(rt, store, key, memo, want_rows=False): """A database by key, from EITHER namespace — `(table_or_None, limit_or_None)`. @@ -14654,10 +14641,6 @@ def _resolve_table(rt, store, key, memo, want_rows=False): if cached is None: cached = memo[("mirror", key)] = _mirror_table(rt, key, hit) return cached - if want_rows and 'rows' not in hit: - if ('native', key) not in memo: - memo[('native', key)] = (_relation_native_rows(rt, key), None) - return memo[('native', key)] return hit, None return _module_table(rt, key, memo, want_rows=want_rows) @@ -14871,9 +14854,9 @@ def relation_cells(rt, table_key, tables=None): ifs = [f for f in fields if f.get('type') == 'if'] if not links and not rollups and not ifs: return {}, limits - rows = (t.get("rows") or {}) if t is not None and 'rows' in t else {} - if t is not None and ('rows' not in t or (not rows and ( - not t.get("_rows_built", True) or not _ut().materialises(table_key, defn=t)))): + rows = (t or {}).get("rows") or {} + if not rows and t is not None and (not t.get("_rows_built", True) + or not _ut().materialises(table_key, defn=t)): # A registry source: NOW the pool is worth its cost, because this table really does # declare relations. One fetch, memoised, shared with every link and rollup below. # ⭐⭐ W53-T40 (d) — AND A READ-THROUGH SOURCE TAKES THE SAME SECOND FETCH, for a reason @@ -15600,13 +15583,11 @@ class _RelationSnapshot(dict): if key not in self and key not in self.missing: if self.supplied is not None and key in self.supplied: table = copy.deepcopy(self.supplied[key]) - if table is not None and 'rows' not in table and _ut().materialises(key, defn=table): - table = _relation_native_rows(self.runtime, key) else: definition = self.definitions.get(key) if definition is None: definition = _ut().get_definition(key, st=self.runtime) - table = (_relation_native_rows(self.runtime, key) + table = (_ut().get_one(key, st=self.runtime) if definition and _ut().materialises(key, defn=definition) else copy.deepcopy(definition)) if table is None: @@ -15646,7 +15627,7 @@ def refresh_relations(rt, log=print, tables=None, persist=True, *, target=None): return local patches = {} for key, old in before.items(): - if not isinstance(old, dict) or 'rows' not in old: + if not isinstance(old, dict): continue fields = old.get('fields') or [] derived = {str(f['key']) for f in fields if f.get('key') diff --git a/api/deps.py b/api/deps.py index c23bf3a7369f3731898aaa96c8a30e21a67b9ee1..f8419b82aff2c7451afb7d936bd659b3567ff041 100644 --- a/api/deps.py +++ b/api/deps.py @@ -1809,62 +1809,17 @@ def _touch_active(uname): pass -def refuse_view_as_write(request: Request): - """Refuse every view-as method except GET and HEAD before any write.""" - if "X-View-As" in request.headers and request.method not in {"GET", "HEAD"}: - raise err(403, "view_as_read_only", "Viewing as a member is read-only.") - - -def _view_as_refused(): - return err(403, "view_as_refused", "This member view is unavailable.") - - -def _view_as_user(request, actor, tenant, rt): - """Resolve an active member of the actor's tenant and audit this request once.""" - if "X-View-As" not in request.headers: - return actor - if not perms.is_admin(actor): - raise _view_as_refused() - started_at = datetime.now(timezone.utc).isoformat() - target = request.headers["X-View-As"].strip().lower() - if not target: - raise _view_as_refused() - rec = copy.deepcopy(_session_registry(target).get(target)) - if (not rec or not rec.get("active", True) or rec.get("role", "user") != "user" - or str(rec.get("tenant") or "").strip().lower() != tenant): - raise _view_as_refused() - member = users._public(target, rec) - audit_key = (tenant, actor["username"], target) - if getattr(request.state, "_view_as_audited", None) != audit_key: - from core import activity_log - activity_log.append( - rt, kind="view_as", object={"app": "settings", "id": target}, - by=actor["username"], on_behalf_of=target, - changes={"member": {"before": None, "after": target}, - "started_at": {"before": None, "after": started_at}}) - request.state._view_as_audited = audit_key - return member - - def require_session(request: Request, response: Response) -> Session: """Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is refreshed on use; the 30-day absolute expiry is carried over, never extended).""" - refuse_view_as_write(request) - viewing_as = "X-View-As" in request.headers raw = request.cookies.get(aios_session.COOKIE_NAME) if not raw: - if viewing_as: - raise _view_as_refused() raise err(401, "no_session", "sign in to continue") claims = aios_session.read(raw) if not claims: - if viewing_as: - raise _view_as_refused() raise err(401, "invalid_session", "Your session ended. Sign in again.") user = _user_for(claims) if not user: - if viewing_as: - raise _view_as_refused() raise err(401, "invalid_session", "Your session ended. Sign in again.") identity = {_TENANT_HEADER: str(claims["t"]), _PRINCIPAL_HEADER: _principal(claims["t"], user["username"])} @@ -1880,12 +1835,8 @@ def require_session(request: Request, response: Response) -> Session: except KeyError: # An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this # deployment, and confirming which slugs exist would answer a question we were not asked. - if viewing_as: - raise _view_as_refused() raise err(401, "invalid_session", "Your session ended. Sign in again.") - actor = user - user = _view_as_user(request, actor, claims["t"], rt) - fresh, _ = aios_session.mint(claims["t"], actor["username"], int(actor.get("epoch") or 0), + fresh, _ = aios_session.mint(claims["t"], user["username"], int(user.get("epoch") or 0), absolute_expiry=claims["x"]) aios_session.set_cookie(response, request, fresh) # ⭐ STAMP THE TENANT THIS ANSWER WAS SERVED FOR (owner report 2026-08-09: "I can see @@ -1906,13 +1857,11 @@ def require_session(request: Request, response: Response) -> Session: # `session.tenant` read back from something the request could influence. response.headers[_TENANT_HEADER] = str(claims["t"]) response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"]) - if viewing_as: - response.headers["X-Viewing-As"] = user["username"] - # Ordinary reads only observe activity. View-as reads write their audit without - # changing either person's activity stamps. + # Observe only AFTER authentication admission. Read methods stay store-read-only; only an + # admitted mutation can launch the persisted stamp, without moving it outside request timing. # qa-runner has no global account record. Never let the normal activity telemetry create or # touch one through the default tenant-#0 users store. - if (not viewing_as and not user.get("qa_runner") + if (not user.get("qa_runner") and not (request.method == "GET" and request.url.path == "/api/v1/notifications")): _observe_active(claims["t"], user.get("username")) if request.method not in {"GET", "HEAD", "OPTIONS"}: @@ -1928,8 +1877,6 @@ def require_changes_session(request: Request, response: Response) -> Session | N verified signed cookie is enough to return that no-data response; resolving the account record would itself turn those legacy requests into a permanent Neon keepalive. """ - if "X-View-As" in request.headers: - return require_session(request, response) if store.backend() != "pg": return require_session(request, response) if request.headers.get("x-aios-change-check") == "focus-v1": @@ -1971,9 +1918,6 @@ def require_session_or_bearer(request: Request, response: Response) -> Session: `require_session` stays cookie only. A bearer session never reissues the cookie. """ - refuse_view_as_write(request) - if "X-View-As" in request.headers: - return _cookie_session(request, response) bearer = _bearer_header(request) if bearer is None or request.cookies.get(aios_session.COOKIE_NAME): try: diff --git a/api/loopy_catalog_tools.py b/api/loopy_catalog_tools.py index 93548b00b498339bb2545a48a91bd812c98557e2..3669273adc7e2c28a0f212b79d77bea42242c1d2 100644 --- a/api/loopy_catalog_tools.py +++ b/api/loopy_catalog_tools.py @@ -305,10 +305,8 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None): import contextvars import sys import threading - import time import catalog_store import app_templates - from core import request_timing render = render or sys.modules[__name__].catalog_pdf budget = export_budget() if budget is None else budget @@ -331,15 +329,8 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None): import logging logging.getLogger(__name__).warning("Catalog export result not saved (%s)", type(exc).__name__) - def run(): - # HTTP completion cancels its timing state, so the export needs its own bounded scope. - with request_timing.request_scope() as measured: - measured.deadline = time.monotonic() + catalog_store.EXPORT_STALE_SECONDS - with request_timing.thread_scope(): - work() - context = contextvars.copy_context() - worker = threading.Thread(target=context.run, args=(run,), daemon=True, name="catalog-pdf") + worker = threading.Thread(target=context.run, args=(work,), daemon=True, name="catalog-pdf") try: worker.start() except Exception: diff --git a/api/main.py b/api/main.py index 84f66c2ec133d859b5b0f72fac13672c6de14dbf..411b4d37051733e00988494b88b46bc02b98e9e2 100644 --- a/api/main.py +++ b/api/main.py @@ -269,32 +269,20 @@ def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen): @app.middleware("http") async def _freeze_response(request: Request, call_next): - """Enforce read-only member views and preserve maintenance refusals and verified response identity.""" - import deps - try: - deps.refuse_view_as_write(request) - except StarletteHTTPException as exc: - return _error_shape(request, exc) + """Keep a refused write readable even when a route catches its store error.""" if request.method in {"GET", "HEAD", "OPTIONS"} or request.url.path == "/api/v1/query/chat/stream": - response = await call_next(request) - else: - with freeze_notice.refusal_scope() as refused: - try: - response = await call_next(request) - except Exception: - if not refused[0]: - raise - return _data_input_frozen(request, freeze_notice.DataInputFrozen()) - if refused[0]: - # Replace the response, including any cookie minted from unpersisted auth state. - return _data_input_frozen(request, freeze_notice.DataInputFrozen()) - identity = getattr(request.state, "_view_as_audited", None) - if identity is not None: - tenant, _actor, member = identity - response.headers["X-Viewing-As"] = member - response.headers[deps._TENANT_HEADER] = str(tenant) - response.headers[deps._PRINCIPAL_HEADER] = deps._principal(tenant, member) - return response + return await call_next(request) + with freeze_notice.refusal_scope() as refused: + try: + response = await call_next(request) + except Exception: + if not refused[0]: + raise + return _data_input_frozen(request, freeze_notice.DataInputFrozen()) + if refused[0]: + # Replace the response, including any cookie minted from unpersisted auth state. + return _data_input_frozen(request, freeze_notice.DataInputFrozen()) + return response @app.exception_handler(data_binding.StoreWriteRefused) @@ -1674,17 +1662,16 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None): return sentence + suffix -def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, amazon_only=False): - """Drain admitted Sync now intents before their resident projection. +def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None): + """Drain Sync now intents on the resident resync thread, the mirror's single writer. - HTTP handlers queue intents. The slot loop owns Odoo and Meta, and its detached turn owns Amazon. - A failed request cannot discard another tenant's queued work. + D-29 / intake F6: HTTP handlers only queue intents. Provider calls stay here, before the + resident projection pass. A failed request cannot discard another tenant's queued work. `outcome`, when given, receives `{provider: "continuing" | "done" | "failed"}` so the loop can project only what a desk wake actually changed. """ from core import connector_sync - if amazon_only or globals().get("_AMAZON_DESK", {}).get("job") is None: - _drain_pending_amazon_revocations() + _drain_pending_amazon_revocations() def progress_log(provider, tenant, receipt, rt): """Publish only authored progress shapes, with one bounded detail per running receipt.""" @@ -1785,11 +1772,7 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, # A first-load continuation never runs inside an Odoo pass: it requeues itself without end. hold = during_odoo if hold_first_load is None else hold_first_load # Claim only work this turn executes. A later provider remains queued with an honest wait. - excluded = ({"odoo", "meta_ads"} if amazon_only else - ({"odoo"} if during_odoo else set())) - if not amazon_only and globals().get("_AMAZON_DESK", {}).get("managed"): - excluded.add("amazon_sp") - pending = connector_sync.take_sync_requests(limit=1, exclude_providers=excluded, + pending = connector_sync.take_sync_requests(limit=1, exclude_providers=("odoo",) if during_odoo else (), refresh_remote=True, exclude_requested_by=("first load",) if hold else ()) while True: @@ -2440,44 +2423,54 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out None, str(db_failure.get("stage") or "materialization"))["message"], **({"cause": _raw_cause(key)} if _raw_cause(key) else {})} for key, db_failure in blocking.items() if isinstance(db_failure, dict)] - if failed_databases is not None and table_keys is not None: - # Unassessed failures survive unless their exact missing-binding refusal is disproved. + record_failed = failed_databases + if table_keys is not None or safe_errors: try: prior = (_rt.get(_rel.CONFIG_KEY) or {}).get("failedDatabases") or [] except Exception: # noqa: BLE001 prior = [] - recovered_database_keys = set() - for item in prior: - if not isinstance(item, dict): - continue + try: + db_health = ((((_rt.get(routes_connectors._ODOO_HEALTH_STATUS) or {}) + .get("connections") or {}).get(cid) or {}) + .get("databases") or {}) + except Exception: # noqa: BLE001 + db_health = {} + + def _recovered(item): + # D-13512: a prior failure leaves once its database ran clean or reads rows again. key = item.get("key") - if (key in database_keys or key in database_failures - or item.get("code") != "materialization_failed" - or item.get("stage") != "materialization" - or item.get("cause") != - "Refused: the database has no readable mirror binding" - or not _rel.is_read_through_key(key)): - continue + if key in database_failures: + return False + if key in succeeded: + return True + stage = str(item.get("stage") or "materialization") + row = ((db_health.get(key) or {}).get("stages") or {}).get(stage) or {} + ran, failed_at = _rel._parse_iso(row.get("at")), _rel._parse_iso(item.get("at")) + if row.get("state") == "ready" and ran and failed_at and ran > failed_at: + return True + # Only a missing binding is cured by a binding that reads: a count mismatch reads rows too. + if (not _rel.is_read_through_key(key) + or "no readable mirror binding" not in str(item.get("cause") or "")): + return False try: - _base, owner = _rel.split_table_key(key) - if owner is not None or not _ut.get_definition(key, st=_rt): - continue - if not any(_key == key and bucket in _rel.enabled_buckets(_rt) - for bucket, _key, _label, _fields in _rel.TABLES): - continue - with _rt.mirror_handle().ro_cursor() as cur: - nrows = routes_odoo_tables.population(key, cur=cur, rt=_rt) - if isinstance(nrows, int) and nrows >= 0: - recovered_database_keys.add(key) + from routes_odoo_tables import population as _population + return (_population(key, rt=_rt) or 0) > 0 except Exception: # noqa: BLE001 - pass # A missing, filling or unreadable mirror cannot prove recovery. - failed_databases = [ - item for item in prior if isinstance(item, dict) - and item.get("key") not in database_keys - and item.get("key") not in recovered_database_keys] + failed_databases + return False + + kept = [item for item in prior if isinstance(item, dict) and not _recovered(item)] + if failed_databases is not None and table_keys is not None: + # D-2679: a changed-tables pass keeps the failures of databases it did not run. + failed_databases = [ + item for item in kept + if item.get("key") not in database_keys] + failed_databases + record_failed = failed_databases + elif len(kept) < len(prior): + # D-13512: a pass with errors still drops the failures that recovered. + record_failed = kept try: _rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at, - failed_databases=failed_databases) + failed_databases=record_failed) except Exception: # noqa: BLE001 pass # Owner 2026-10-29: a database that fails or recovers is Telegrammed at once. @@ -3277,77 +3270,15 @@ def _sheet_warm_after_pass(tenant, good, generation_before, rebuilt): def _sync_idle_wait(event, seconds): """The resync loop's wait. A `sync_worker.py --once` job exits here when it is idle (S82-T10).""" - if globals().get("_AMAZON_DESK", {}).get("job") is not None: - # An unclaimed or finishing desk turn still owns work, even between receipt states. - return event.wait(seconds) import sync_runtime return sync_runtime.idle_wait(event, seconds) -def _amazon_intraday_pass(tenant_key, previous_fingerprint): - """Run due Amazon tables and their projection without advancing the Odoo pass clock.""" - if globals().get("_AMAZON_DESK", {}).get("job") is not None: - return previous_fingerprint - try: - from harness import amazon_store, runtime - rt = runtime.get_runtime(tenant_key) - if not amazon_store.credential(tenant_key, rt=rt) or amazon_store.first_load_job_live(rt): - return previous_fingerprint - plan = amazon_store.scheduled_sync_plan(tenant_key, rt=rt) - if not plan["tables"]: - return previous_fingerprint - print("[aios-api] amazon intraday pass due: " + ", ".join(plan["tables"]), flush=True) - RESYNC_STATE["slotPass"] = False - _pull_amazon("intraday") - state = _rebuild_amazon_relational("intraday", rt, previous_fingerprint) - return state["fingerprint"] if state.get("known") else None - except Exception as exc: - print(f"[aios-api] amazon intraday pass failed ({type(exc).__name__}); " - "the next cadence check retries it", flush=True) - return None - - -def _amazon_desk_pass(tenant, *, project=True): - """Run one Amazon desk claim off the slot clock and collect its projection after publication.""" - import process_role - import time - if not process_role.may_write(): - return False - from core import connector_sync - state = globals().setdefault("_AMAZON_DESK", {"managed": True, "job": None, "outcome": {}}) - job = state["job"] - if job is not None: - if job.is_alive() or not project: - return True - job.join() - state["job"] = None - _project_desk(state["outcome"], tenant) - receipt = connector_sync.sync_request("amazon_sp", tenant) - if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load" - or max(receipt.get("notBefore") or 0, - connector_sync.sync_not_before("amazon_sp", tenant) or 0) > time.time()): - return False - outcome = state["outcome"] = {} - - def drain(): - try: - _run_sync_requests(outcome, hold_first_load=True, amazon_only=True) - except Exception as exc: - print(f"[aios-api] amazon desk turn failed ({type(exc).__name__})", flush=True) - finally: - connector_sync.sync_wake_event().set() - - # This turn inherits store-resync's heavy admission, including tenant-worker scope. - state["job"] = process_role.start("store-resync", "heavy", drain) - return state["job"] is not None - - def _store_resync_loop(): """Keep the analytical mirror current — the API-process stand-in for the re-sync the Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a refreshed pool re-reads the freshly synced store.""" - import process_role as _loop_role - if not _loop_role.may_write() or _freeze_holds("store resync"): + if _freeze_holds("store resync"): return import time as _t from core import connector_sync @@ -3383,7 +3314,6 @@ def _store_resync_loop(): last_full_pass_at = _t.time() # The least time between two desk-only passes of an Amazon first-load continuation. desk_min_spacing_seconds = 30.0 - amazon_check_seconds = 60.0 # H73: the start of the last daily-slot pass, and whether the boot restore already ran. last_slot_pass_at = _memory.get("slotPassAt") slot_restored = False @@ -3474,7 +3404,7 @@ def _store_resync_loop(): _slot = (globals().get("_resolve_resync_slot") or (lambda: None))() def _near_slot(): - # Completed desk projections wait inside the existing first-load slot quiet window. + # A first-load continuation does not start this close to the next Odoo slot. if _slot is None: return False import odoo_relational as _rel_near @@ -3489,7 +3419,6 @@ def _store_resync_loop(): # END drifted later every day and past the 26-hour stale line). _woken = bool(_idle_wait(connector_sync.sync_wake_event(), min( max(0.0, _every - (_t.time() - last_full_pass_at)), - amazon_check_seconds, float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time())))) connector_sync.sync_wake_event().clear() _slot_due = False @@ -3503,7 +3432,6 @@ def _store_resync_loop(): _woken = bool(_idle_wait(connector_sync.sync_wake_event(), min( 0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at) else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now), - amazon_check_seconds, float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now)))) connector_sync.sync_wake_event().clear() _slot_now = _t.time() @@ -3536,11 +3464,6 @@ def _store_resync_loop(): f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure") else: RESYNC_STATE.pop("nextPassAt", None) - _cadence_due = not passes or _t.time() - last_full_pass_at >= _every - _amazon_desk_busy = (_amazon_desk_pass(_cad["tenant"], project=not (_slot_due or _near_slot())) - if _slot is not None or globals().get("_AMAZON_DESK") else False) - if (not _slot_due if _slot is not None else not _cadence_due): - last_amzn_fingerprint = _amazon_intraday_pass(_cad["tenant"], last_amzn_fingerprint) # ⛔⛔ HOTFIX 2026-09-29 — A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner # capped syncing at once a day, and a deploy is not a reason to sync again. So the first # pass after a boot that lands within one cadence of the last scheduled pass only catches @@ -3583,16 +3506,13 @@ def _store_resync_loop(): else: _project_desk(_desk, _cad["tenant"]) if not slot_restored: - if not _amazon_desk_busy: - _pull_amazon("boot", False) + _pull_amazon("boot", False) # The restored mirror warms the workbooks once; Sheet warm reads only the mirror. _sheet_warm_after_pass(_cad["tenant"], True, None, True) slot_restored = True print(f"[aios-api] store resync: no Odoo pass outside the daily slot; the next one " f"runs at {RESYNC_STATE['nextPassAt']} ({_slot['at']} {_slot['tz']})") continue - if (_slot is None and not _cadence_due and _odoo_requested is None and not _continuation): - continue passes += 1 backfill_next_at = None if not _continuation: @@ -3839,10 +3759,9 @@ def _store_resync_loop(): except Exception as e: # noqa: BLE001 print(f"[aios-api] meta relational refresh failed (resync): " f"{type(e).__name__}: {e}") - # The independent desk retains Amazon's pull and projection while Odoo completes its pass. - if not _amazon_desk_busy and _resumed.get("amazon"): + if _resumed.get("amazon"): _pull_amazon("boot", False) - elif not _amazon_desk_busy: + else: # The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it. RESYNC_STATE["slotPass"] = _slot_claimed try: @@ -3856,8 +3775,7 @@ def _store_resync_loop(): try: from harness import runtime as _runtime _rt_resync = _runtime.get_runtime(_cad["tenant"]) - _amzn_state = ({} if _amazon_desk_busy else - _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)) + _amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous) last_amzn_fingerprint = (_amzn_state["fingerprint"] if _amzn_state.get("known") else None) except Exception as e: # noqa: BLE001 diff --git a/api/notify_destinations.py b/api/notify_destinations.py index ed12dcccef79f6d6c4809c5ce49f26a6b6ca7628..0d88a3041a6597332d9021c5eeb167f465505f38 100644 --- a/api/notify_destinations.py +++ b/api/notify_destinations.py @@ -1,14 +1,12 @@ """Authenticated preference routes and destination-bound notification delivery.""" import re -from datetime import datetime, timezone from fastapi import APIRouter, Body, Depends, Query from core import notify_destinations as rows from deps import Session, err, require_session import email_send -import dash_norm NotificationItem = rows.NotificationItem NotificationPayload = rows.NotificationPayload @@ -28,12 +26,6 @@ _REASONS = { 'notification_source_required': 'This notification needs source records so the Slack agent can check its read access.', 'transport_error': 'Slack delivery was not confirmed. Check Slack before retrying.', 'slack_readiness_unavailable': 'Slack readiness could not be checked. Check its connection and permissions.', - 'in_app_unavailable': 'In-app notifications are unavailable for this workspace.', - 'in_app_delivery_unconfirmed': 'The in-app notification could not be confirmed.', - 'webhook_not_configured': 'Choose one of your saved alert webhooks in connector settings.', - 'webhook_readiness_unavailable': 'Webhook readiness could not be checked. Check its saved configuration.', - 'webhook_delivery_unconfirmed': 'Webhook delivery was not confirmed. Check the destination before retrying.', - 'webhook_refused': 'The configured webhook refused this notification.', } @@ -51,56 +43,6 @@ def _own_session(session): raise err(401, 'notification_user_required', 'Sign in to edit notification destinations.') -def _in_app_readiness(rt): - try: - import assistant_log - ready = (callable(getattr(assistant_log, 'append', None)) - and callable(getattr(rt, 'get', None)) and callable(getattr(rt, 'update', None))) - available = getattr(rt, 'available', None) - if callable(available): - ready = ready and bool(available()) - except Exception: - ready = False - reason = None if ready else 'in_app_unavailable' - return {'configured': ready, 'available': ready, 'reason': reason, - 'message': '' if ready else _REASONS[reason]} - - -def _webhook_readiness(rt, user, identity): - try: - rows.webhook_fields(rt, user, identity) - import automation_engine - reason = None if callable(getattr(automation_engine, 'fetch_json', None)) else 'webhook_readiness_unavailable' - except ValueError: - reason = 'webhook_not_configured' - except Exception: - reason = 'webhook_readiness_unavailable' - return {'available': not reason, 'reason': reason, - 'message': '' if reason is None else _REASONS[reason]} - - -def _configured_webhooks(rt, user): - """Expose only safe metadata for this user's configured saved webhook references.""" - from core import keychain - destinations = [] - try: - if not callable(getattr(rt, 'get', None)): - return {'configured': False, 'destinations': [], 'reason': 'webhook_not_configured'} - for entry in keychain.list_entries(rt, 'generic'): - identity = entry.get('id') - if entry.get('createdBy') != user or not isinstance(identity, str) or not rows._WEBHOOK.fullmatch(identity): - continue - readiness = _webhook_readiness(rt, user, identity) - if readiness['available']: - label = entry.get('label') - destinations.append({'id': identity, 'label': dash_norm.plain(label[:200]) if isinstance(label, str) else identity, - 'destination': 'webhook:' + identity, **readiness}) - except Exception: - return {'configured': False, 'destinations': [], 'reason': 'webhook_readiness_unavailable'} - return {'configured': bool(destinations), 'destinations': destinations, - 'reason': None if destinations else 'webhook_not_configured'} - - @router.get('/destinations') def get_destinations(session: Session = Depends(require_session)): _own_session(session) @@ -109,7 +51,6 @@ def get_destinations(session: Session = Depends(require_session)): except Exception: raise err(503, 'notification_store_unavailable', 'Notification destinations are unavailable for this workspace.') mail_ready = email_send.availability(session.runtime) - in_app = _in_app_readiness(session.runtime) projected = [] for preference in destinations: readiness = { @@ -119,14 +60,8 @@ def get_destinations(session: Session = Depends(require_session)): } if preference['email']: readiness['email:' + preference['email']] = dict(mail_ready) - if preference.get('inApp'): - readiness['in-app:' + session.uname] = dict(in_app) - for identity in preference.get('webhooks', []): - readiness['webhook:' + identity] = _webhook_readiness(session.runtime, session.uname, identity) projected.append({**preference, 'label': rows.KIND_LABELS[preference['kind']], 'readiness': readiness}) - return {'destinations': projected, 'email': mail_ready, - 'inApp': {**in_app, 'destination': 'in-app:' + session.uname}, - 'webhooks': _configured_webhooks(session.runtime, session.uname)} + return {'destinations': projected, 'email': mail_ready} def _save(kind, body, session, *, replace): @@ -135,13 +70,9 @@ def _save(kind, body, session, *, replace): rows.kind_id(kind) patch = rows.clean_patch(body, replace=replace) except (ValueError, TypeError): - raise err(400, 'notification_destination_invalid', 'Choose a known notification kind and valid saved destinations.') - if patch.get('inApp') and not _in_app_readiness(session.runtime)['available']: - raise err(400, 'notification_destination_invalid', _REASONS['in_app_unavailable']) + raise err(400, 'notification_destination_invalid', 'Choose a known notification kind, Slack channel ids and one email address.') try: return rows.edit(session.runtime, session.uname, kind, patch) - except ValueError: - raise err(400, 'notification_destination_invalid', 'Choose one of your configured webhook destinations.') except Exception: raise err(503, 'notification_store_unavailable', 'Notification destinations could not be saved for this workspace.') @@ -253,36 +184,6 @@ def _dispatch_slack(rt, preference, channel, kind, items, *, dispatcher=None): return {'state': 'sent', 'reason': None, 'message': ''} -def _dispatch_in_app(rt, preference, kind, items): - ready = _in_app_readiness(rt) - if not ready['available']: - return _refused(ready['reason']) - import assistant_log - items = [{**item, 'title': dash_norm.plain(item['title']), 'body': dash_norm.plain(item['body'])} for item in items] - body = '\n\n'.join(item['title'] + '\n' + item['body'] for item in items) - turn = {'role': 'system', 'door': 'wake', 'kind': 'notice', 'text': body, - 'at': datetime.now(timezone.utc).isoformat(), - 'payload': {'title': rows.KIND_LABELS[kind], 'body': body, 'items': items, - 'notificationKind': kind, 'sender': 'Loopable'}} - receipt = assistant_log.append(rt, preference['user'], turn) - return ({'state': 'sent', 'reason': None, 'message': ''} - if isinstance(receipt, str) and receipt else _refused('in_app_delivery_unconfirmed')) - - -def _dispatch_webhook(rt, preference, identity, kind, items): - ready = _webhook_readiness(rt, preference['user'], identity) - if not ready['available']: - return _refused(ready['reason']) - fields = rows.webhook_fields(rt, preference['user'], identity) - import automation_engine - items = [{**item, 'title': dash_norm.plain(item['title']), 'body': dash_norm.plain(item['body'])} for item in items] - payload = {'version': rows.PAYLOAD_VERSION, 'tenant': rt.key, 'user': preference['user'], - 'kind': kind, 'items': items} - status, _body = automation_engine.fetch_json(fields['url'], payload, timeout=20, max_kb=64) - return ({'state': 'sent', 'reason': None, 'message': ''} - if 200 <= status < 300 else _refused('webhook_refused')) - - def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None): """Deliver trusted recipient-safe items to saved picks, optionally targeting one user.""" kind, items = rows.kind_id(kind), rows.clean_items(items) @@ -294,9 +195,6 @@ def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None) destinations = ['slack:' + channel for channel in preference['slackChannels']] if preference['email']: destinations.append('email:' + preference['email']) - if preference.get('inApp'): - destinations.append('in-app:' + preference['user']) - destinations.extend('webhook:' + identity for identity in preference.get('webhooks', [])) for destination in destinations: current = rows.get(rt, preference['user'], kind) if not rows.selected(current, destination): @@ -304,18 +202,12 @@ def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None) try: if destination.startswith('slack:'): outcome = _dispatch_slack(rt, current, destination[6:], kind, items, dispatcher=dispatcher) - elif destination.startswith('in-app:'): - outcome = _dispatch_in_app(rt, current, kind, items) - elif destination.startswith('webhook:'): - outcome = _dispatch_webhook(rt, current, destination[8:], kind, items) else: result = email_send.send_notification(rt, destination[6:], kind, items, transport=mail_transport) outcome = ({'state': 'sent', 'reason': None, 'message': ''} if result.get('sent') is True else _refused(result.get('reason'), message=result.get('message'))) except Exception: - reason = {'slack': 'transport_error', 'email': 'email_delivery_unconfirmed', - 'in-app': 'in_app_delivery_unconfirmed', 'webhook': 'webhook_delivery_unconfirmed'} - outcome = _refused(reason[destination.split(':', 1)[0]]) + outcome = _refused('transport_error' if destination.startswith('slack:') else 'email_delivery_unconfirmed') # Persistence failures propagate. An unrecorded refusal must never look successful. recorded = rows.record_delivery(rt, preference['user'], kind, destination, outcome) results.append({'user': preference['user'], 'destination': destination, diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 34b6728b408f08af68fe0e74e5053b60a6c32148..0bbfade6227b017e3428271283d206f2c2d85fbe 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -103,7 +103,7 @@ LANDED_COST_LINES_KEY = "ut_odoo_landed_cost_lines" from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PURCHASE_LINES_KEY, PICKINGS_KEY, LANDED_PICKINGS_FIELD, landed_picking_field, landed_picking_sql, require_landed_cost_mirror, - picking_fields as _picking_fields, picking_source_spec, + picking_fields, picking_source_spec, order_fields as purchase_order_fields, line_fields as _purchase_line_fields, source_specs as purchase_source_specs, @@ -112,77 +112,6 @@ from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PUR #: S78-T14: the databases whose row id is a `stock.picking` id. Their read wall admits a delivery photo. DELIVERY_KEYS = (SHIPMENTS_KEY, PICKINGS_KEY) -DELIVERY_PHOTO_REFERENCE = "delivery_photo_attachments" -_PHOTO_PRODUCER_MISSING = "delivery_photo_metadata_unavailable" - - -def delivery_photo_reference_field(): - """Declare a metadata-only attachment-reference array using the supported JSON field type.""" - return _preset({"key": DELIVERY_PHOTO_REFERENCE, "label": "Delivery photo attachments", - "type": "json", "source": "overlay", "default": False, - "description": "Attachment-reference array: id, res_model, res_id, name, mimetype, " - "file_size and create_date. Metadata only. No URLs or file contents. " - "An unavailable producer returns {error: {code, cause, fix}}.", - "attachmentReference": {"source": "odoo", "model": "ir.attachment", "multiple": True, - "fields": {"id": "int", "res_model": "text", "res_id": "int", "name": "text", - "mimetype": "text", "file_size": "int", "create_date": "datetime"}}}) - - -def picking_fields(): - """Add delivery attachment references to the existing picking field contract.""" - return _picking_fields() + [delivery_photo_reference_field()] - - -def _delivery_photo_references(value): - """Decode references and distinguish a missing producer from an empty attachment list.""" - if isinstance(value, str) and value != _PHOTO_PRODUCER_MISSING: - value = _json.loads(value) - if value is None or value == _PHOTO_PRODUCER_MISSING: - return {"error": {"code": "field_unavailable", - "cause": "Delivery photo attachments have no complete persisted metadata producer.", - "fix": "Run this tenant's Odoo source sync with delivery_photo_metadata, then publish " - "its complete observation snapshot."}} - return value - - -def _merge_delivery_photo_cols(sources): - """Project tenant-local attachment metadata through the existing delivery row walls.""" - out = dict(sources) - for key, spec in sources.items(): - if base_table_key(key) not in DELIVERY_KEYS: - continue - owner = (spec["id"].rsplit(".", 1)[0] + "." if "." in spec["id"] - else spec["table"] + ".") - missing = ("'" + _json.dumps(_PHOTO_PRODUCER_MISSING) + "'", _delivery_photo_references) - marker = ("CASE (SELECT legacy FROM odoo_connections pc WHERE pc.connection_id = " - + owner + "_connection_id) WHEN TRUE THEN 'delivery_photo_metadata' " - "WHEN FALSE THEN 'delivery_photo_metadata.connection.' || sha256(" - + owner + "_connection_id) END") - payload = ("SELECT to_json(list(struct_pack(id := coalesce(pa._source_id, pa.id), " - "res_model := pa.res_model, res_id := pa.res_id, name := pa.name, " - "mimetype := pa.mimetype, file_size := pa.file_size, create_date := pa.create_date) " - "ORDER BY coalesce(pa._source_id, pa.id))) FROM delivery_photo_metadata pa " - "WHERE pa.res_model = 'stock.picking' AND pa.res_id = coalesce(" - + owner + "_source_id, " + owner + "id) AND pa._connection_id " - "IS NOT DISTINCT FROM " + owner + "_connection_id") - sql = ("CASE WHEN EXISTS (SELECT 1 FROM _sync_state ps WHERE ps.entity = " + marker - + " AND ps.phase = 'live') THEN coalesce((" + payload + "), '[]') ELSE " - + missing[0] + " END") - cols = {**spec["cols"], DELIVERY_PHOTO_REFERENCE: (sql, _delivery_photo_references)} - optional = list(spec.get("optional") or ()) - from harness import datastore - requirements = [("delivery_photo_metadata", field) for field in - (*datastore.DELIVERY_PHOTO_FIELDS, "_source_id", "_connection_id")] - requirements += [("odoo_connections", "connection_id"), ("odoo_connections", "legacy"), - ("_sync_state", "entity"), ("_sync_state", "phase"), - ("stock_picking", "_source_id"), ("stock_picking", "_connection_id")] - for requirement in requirements: - fallback = {"absent": (requirement,), "without": {"cols": {DELIVERY_PHOTO_REFERENCE: missing}}} - if fallback not in optional: - optional.append(fallback) - out[key] = {**spec, "cols": cols, "optional": tuple(optional)} - return out - #: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). JOIN_KEY = "partner_id" #: The product-grain equivalent. @@ -780,7 +709,7 @@ def merge_audit_cols(sources): if optional: merged["optional"] = tuple(optional) out[key] = merged - return _merge_delivery_photo_cols(out) + return out #: Materialised grids read their audited columns beside their reader's own SQL. @@ -896,7 +825,7 @@ def shipment_fields(): "default": True, "description": "Elapsed days from the sale order date to the actual " "shipment completion. Blank when either date is missing."}, _refreshed_field(), - )] + audit_fields(SHIPMENTS_KEY) + [delivery_photo_reference_field()] + )] + audit_fields(SHIPMENTS_KEY) def agent_fields(): diff --git a/api/routes_account_health.py b/api/routes_account_health.py index 5dbace9bf5754b18a0e86e661166d77bcec09305..e3a9e1772aa240aa498a6f890d17245e6ea3b50c 100644 --- a/api/routes_account_health.py +++ b/api/routes_account_health.py @@ -5,8 +5,7 @@ from urllib.parse import urlencode from deps import Session, err, require_session TEMPLATE_KIND = "account_health" -router = APIRouter(tags=["account-health"]) -app_router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"]) +router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"]) SOURCE = "ut_amzn_account_health" @@ -102,7 +101,7 @@ def owners(session): return sorted((n for n in names if n), key=str.lower) -@app_router.get("/summary") +@router.get("/summary") def read_summary(app: str = Query(..., min_length=1), search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1), metric: str | None = Query(None), session: Session = Depends(_app_reader)): @@ -116,7 +115,7 @@ def read_summary(app: str = Query(..., min_length=1), search: str = Query(""), return {**out, "canEdit": _may_edit(session, _app(session, app))} -@app_router.get("/history") +@router.get("/history") def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None), limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)): ah = _module() @@ -134,7 +133,7 @@ def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Qu return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime) -@app_router.get("/items") +@router.get("/items") def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None), search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None), cursor: str | None = Query(None), limit: int = Query(50, ge=1), @@ -150,7 +149,7 @@ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(N return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} -@app_router.get("/items/{item_id}") +@router.get("/items/{item_id}") def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): ah = _module() @@ -164,7 +163,7 @@ def read_item(item_id: str, app: str = Query(..., min_length=1), "canEdit": _may_edit(session, _app(session, app))} -@app_router.patch("/items/{item_id}") +@router.patch("/items/{item_id}") def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None), due: str | None = Body(None), note: str | None = Body(None), session: Session = Depends(_app_writer)): @@ -177,13 +176,13 @@ def patch_item(item_id: str, owner: str | None = Body(None), status: str | None due=due, note=note, st=session.runtime) -@app_router.get("/settings") +@router.get("/settings") def read_settings(session: Session = Depends(_app_reader)): ah = _module() return _call(ah.settings, _tenant(session), st=session.runtime) -@app_router.put("/settings") +@router.put("/settings") def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None), riskOverdueDays: float | None = Body(None), session: Session = Depends(_app_writer)): @@ -192,7 +191,7 @@ def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: st default_owner=defaultOwner, risk_overdue_days=riskOverdueDays, st=session.runtime) -@app_router.get("/risks") +@router.get("/risks") def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(None), search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1), session: Session = Depends(_app_reader)): @@ -206,7 +205,7 @@ def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(N return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} -@app_router.patch("/risks/{item_id}") +@router.patch("/risks/{item_id}") def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None = Body(None), caseId: str | None = Body(None), appealDate: str | None = Body(None), outcome: str | None = Body(None), note: str | None = Body(None), @@ -219,22 +218,6 @@ def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime) -router.include_router(app_router) - - -@router.get("/api/v1/account-health/categories/{key}") -def read_category(key: str, session: Session = Depends(_app_reader)): - ah = _module() - context = _call(_narrowed, session) - - def allowed(row, snapshot): - evidence = _call(ah.category_source, _tenant(session), session.runtime, row, snapshot) - return _source_allowed({"sourceEvidence": evidence}, context) - - return _call(ah.category_detail, _tenant(session), key, user=session.user, - allowed=allowed if context is not None else None, st=session.runtime) - - # --------------------------------------------------------------------------- R62 source narrowing def _narrowed(session): diff --git a/api/routes_admin.py b/api/routes_admin.py index 2afc26c51340271f53fcc57c2bddc44010b0cda0..e3bad854d197f3f56cea6eb55ed419717197de18 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -143,7 +143,17 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None): # editor will open, so the two cannot disagree. "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st, surface_keys=surface_keys), - "perms_v": int(rec.get("perms_v") or 0)} + "perms_v": int(rec.get("perms_v") or 0), + # S79-T63: only a copied test account carries `test`, so a real account's keys stay exact. + **({"test": _test_marker(rec)} if _test_marker(rec) else {})} + + +def _test_marker(rec): + mark = (rec or {}).get(users.TEST_ACCOUNT_KEY) + if not isinstance(mark, dict): + return None + return {"copiedFrom": mark.get("copied_from"), "createdBy": mark.get("created_by"), + "createdAt": mark.get("created_at")} def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None): @@ -454,6 +464,9 @@ def update_user(username: str, body: dict = Body(default=None), if result == "last_admin": raise err(400, "last_admin", "an active tenant must retain at least one administrator") + if result in _TEST_ACCOUNT_REFUSALS: + # R42: a test account gets no email and no admin role, refused in the core writer. + raise err(400, result, _TEST_ACCOUNT_REFUSALS[result]) if result != "updated": raise err(404, "no_such_user", f"no account named {uname!r}") @@ -539,6 +552,164 @@ def expire_password(username: str, session: Session = Depends(admin_gate)): return Response(status_code=204) +# ── S79-T63 (R28, R42): test accounts that copy one member's permissions exactly ───────────────── +_TEST_ACCOUNT_REFUSALS = { + "test_account_no_email": "A test account cannot have an email address.", + "test_account_no_whatsapp": "A test account cannot connect WhatsApp.", + "test_account_not_admin": "A test account cannot be an administrator.", +} + + +def _copy_refusal(exc): + return err(exc.status, exc.code, str(exc)) + + +#: The en and em dash, built from code points so this file carries no dash character (rule 2). +def _copy_failure(exc, verb): + # A store error can carry any text; rule 2 keeps dashes off the screen. + import dash_norm + cause = dash_norm.plain(exc.cause).strip().rstrip(".") + message = f"The test account was not {verb}. {cause}." + if exc.leftovers: + message += (" These parts could not be removed: " + ", ".join(exc.leftovers) + + ". Remove the test account again to finish.") + elif verb == "created": + message += " Nothing was kept." + return err(503, "test_account_failed", message) + + +def _test_account_steps(session): + """Copy steps for grant stores outside `core`: Connector grants, through their own route.""" + import core.perm_scope as perm_scope + import routes_keychain as keychain + + def held(): + doc = session.runtime.get(perm_scope.CONNECTOR_GRANT_KEY, fresh=True) + if doc is not None and not isinstance(doc, dict): + raise RuntimeError("connector access could not be read") + return doc or {} + + def copy_grants(username, source): + known = {kind["key"] for kind in keychain.connector_grant_kinds()} + kinds = sorted(k for k in (held().get(source) or ()) if k in known) + if kinds: + keychain.put_connector_grants(username, {"granted": kinds}, session) + + def clear_grants(username): + if username in held(): + keychain.put_connector_grants(username, {"granted": []}, session) + + return [("connector access", copy_grants, clear_grants)] + + +@router.get("/admin/users/{username}/test-account/preview") +def preview_test_account(username: str, session: Session = Depends(admin_gate)): + """What a test account copied from this member would carry, before anything is written.""" + from core import test_accounts, user_tables + reg = _registry() + try: + plan = test_accounts.plan(username, reg, tenant=_tenant_of(session.user), + st=session.runtime) + lent = user_tables.lend_defs(session.runtime, schema_only=True) + except test_accounts.CopyRefused as exc: + raise _copy_refusal(exc) + except Exception as exc: # noqa: BLE001 + raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc + limits = _source_limits(plan["source"], reg.get(plan["source"]), session) + body = test_accounts.preview(plan, st=lent, limits=limits) + _label_grants(session, body["copy"]["grants"]) + return body + + +def _source_limits(uname, rec, session): + """The source's row limits, phrased for the administrator; None if they cannot be read. + + Field walls are lifted for the phrasing only, so a condition on a hidden field is named + instead of reading "Limited by your access". The filters themselves are the source's. + """ + import core.perm_scope as perm_scope + try: + reader = deepcopy(users._public(uname, rec)) + for entry in (reader.get("perms") or {}).values(): + if isinstance(entry, dict): + entry.pop("hiddenFields", None) + return perm_scope.scope_limits(reader, st=session.runtime) + except Exception: # noqa: BLE001 + return None + + +#: What a preview row says when a shared item's name cannot be read. Never a raw id. +_GRANT_FALLBACK = {"view": "A shared view", "folder": "A shared folder", "database": "A database", + "field": "A shared field", "cohort": "A locked view", "workbook": "A shared sheet"} +#: Naming stops after this many seconds; every grant is still listed, the rest by kind. +_LABEL_BUDGET_S = 3.0 + + +def _label_grants(session, grants): + """Name each grant through the share door's own resolver (`routes_shares._object_ref`).""" + import time + try: + from routes_shares import _object_ref + except Exception: # noqa: BLE001 + _object_ref = None + deadline = time.monotonic() + _LABEL_BUDGET_S + for grant in grants: + if grant.get("label"): + continue + label = None + if time.monotonic() < deadline: + try: + if grant["kind"] == "workbook": + # `_object_ref` has no workbook branch; the workbook's own name is the label. + import core.sheets as sheets + label = str(sheets.get_workbook(session.runtime, grant["oid"]).get("name") or "") + elif _object_ref is not None: + label = str(_object_ref(session, grant["kind"], grant["oid"])[0] or "") + except Exception: # noqa: BLE001 + label = None + label = (label or "").strip() + grant["label"] = label or _GRANT_FALLBACK.get(grant["kind"], "A shared item") + + +@router.post("/admin/test-accounts", status_code=201) +def create_test_account(body: dict = Body(default=None), session: Session = Depends(admin_gate)): + """Create a test account whose permissions copy `source` exactly. The password shows once.""" + from core import test_accounts + body = body if isinstance(body, dict) else {} + _require_store() + try: + made = test_accounts.create(body.get("source"), admin=session.uname, + tenant=_tenant_of(session.user), st=session.runtime, + steps=_test_account_steps(session)) + except test_accounts.CopyRefused as exc: + raise _copy_refusal(exc) + except test_accounts.CopyFailed as exc: + raise _copy_failure(exc, "created") + except Exception as exc: # noqa: BLE001 + raise err(503, "store_unavailable", "The test account was not created. Try again.") from exc + rec = _registry().get(made["username"]) + if not isinstance(rec, dict): + raise err(503, "store_unavailable", "The test account was not saved. Try again.") + return {"user": _view(made["username"], rec), "password": made["password"]} + + +@router.delete("/admin/test-accounts/{username}", status_code=204) +def delete_test_account(username: str, session: Session = Depends(admin_gate)): + """Remove a test account, its sign in and every grant naming it, in one call.""" + from core import test_accounts + _require_store() + try: + test_accounts.remove(username, tenant=_tenant_of(session.user), st=session.runtime, + granter=session.uname, steps=_test_account_steps(session)) + except test_accounts.CopyRefused as exc: + raise _copy_refusal(exc) + except test_accounts.CopyFailed as exc: + raise _copy_failure(exc, "fully removed") + except Exception as exc: # noqa: BLE001 + raise err(503, "store_unavailable", "The test account was not removed. Try again.") from exc + return Response(status_code=204) + + # ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── #: The modules the permission editor governs — the GRID surfaces, the ones with a field schema #: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list @@ -2140,7 +2311,7 @@ def _leaf_count(nodes): return n -def _fold_legacy_scope(rec, cleaned, session=None): +def _fold_legacy_scope(rec, cleaned): """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` @@ -2189,8 +2360,7 @@ def _fold_legacy_scope(rec, cleaned, session=None): return cleaned # nothing to preserve: the account was unrestricted out = {} for key, e in cleaned.items(): - legacy_here = _prune_to_module( - legacy, {f["key"] for f in _module_fields(key, session=session)}) + legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) tree = e.get("filter") if not legacy_here: # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than @@ -2607,7 +2777,7 @@ def grant_database_access(session, username, database, role): is_table = key.startswith(user_tables.KEY_PREFIX) if role not in ('view', 'edit') or (not is_table and role != 'view'): raise err(409, 'unsupported_role', 'This database supports only viewing access.') - tenant = str(session.tenant).strip().lower() + tenant = str(session.runtime.tenant.slug).strip().lower() if _tenant_of(session.user) != tenant: raise err(403, 'forbidden', 'Database access could not be granted.') # Read the account store strictly rather than the directory's fail-silent projection. @@ -2632,7 +2802,7 @@ def grant_database_access(session, username, database, role): candidate = dict(prior or {}, access=True) cleaned = _clean_perms({key: candidate}, governed_keys=governed, session=session, stored=rec.get('perms')) - folded = _fold_legacy_scope(rec, cleaned, session=session) + folded = _fold_legacy_scope(rec, cleaned) _guard_bu_widening(rec, folded, False) # Validation may prune retired fields. This click changes access, so retain stored walls. if folded[key].get('filter') != cleaned[key].get('filter'): @@ -2740,7 +2910,7 @@ def set_database_block(session, username, database, entry, *, expect, confirm_wi raise err(400, "bad_perms", "The entry needs access set to true or false.") if not isinstance(expect, (dict, str)) or not expect: raise err(400, "missing_expect", "Read the account permissions first, then send its access state.") - tenant = str(session.tenant).strip().lower() + tenant = str(session.runtime.tenant.slug).strip().lower() if _tenant_of(session.user) != tenant: raise err(403, "forbidden", "Database access could not be changed.") # Read the account store strictly rather than the directory's fail-silent projection. @@ -2760,7 +2930,7 @@ def set_database_block(session, username, database, entry, *, expect, confirm_wi if field in entry} cleaned = _clean_perms({key: submitted}, governed_keys=governed, session=session, stored=rec.get("perms")) - folded = _fold_legacy_scope(rec, cleaned, session=session) + folded = _fold_legacy_scope(rec, cleaned) _guard_bu_widening(rec, folded, bool(confirm_widen)) candidate = {field: copy.deepcopy(folded[key][field]) for field in users.DATABASE_ENTRY_KEYS} prior = copy.deepcopy((rec.get("perms") or {}).get(key)) @@ -2841,8 +3011,7 @@ def put_perms(username: str, body: dict = Body(default=None), governed = {m["key"] for m in _mods} opaque = {key: entry for key, entry in cleaned.items() if key not in governed} known = _fold_legacy_scope(reg[uname], - {key: entry for key, entry in cleaned.items() if key in governed}, - session=session) + {key: entry for key, entry in cleaned.items() if key in governed}) _guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen"))) cleaned = dict(known, **opaque) users.set_access(uname, perms=cleaned) diff --git a/api/routes_amazon_projects.py b/api/routes_amazon_projects.py index 83d1fa7a6ece23be66c8a99585c3c9881abbd4fa..1c2a8a46f2416b4a4e3d3c760467953cf111a33f 100644 --- a/api/routes_amazon_projects.py +++ b/api/routes_amazon_projects.py @@ -12,7 +12,7 @@ from pathlib import Path import sys from typing import Any -from fastapi import APIRouter, Body, Depends, HTTPException, Query +from fastapi import APIRouter, Body, Depends, HTTPException from core import amazon_projects as projects from core import amazon_project_results as project_results @@ -219,7 +219,7 @@ def _distinct(found, pairs): def cell_values(records): - """Store average progress with Completed at 100 and the completion count, blank when none count.""" + """R19 and R67: the average progress as a stored pct and "N of M completed", blank with no counted project.""" counted = projects.summary(records) if not counted["total"]: return {PROGRESS_KEY: None, LABEL_KEY: None} @@ -343,34 +343,28 @@ def _result_sources(session, targets): return available -class ProjectMirror(sku_action_results.Mirror): - """Read a deduplicated project SKU union and retain source evidence for every target day.""" - - def __init__(self, con, skus, available): - super().__init__(con, {}, available) - self.skus, self.coverage = tuple(skus), [] - - def entity(self): - self.field(sku_action_results.ECONOMICS, "msku") - return '"msku" IN (' + ", ".join("?" for _ in self.skus) + ")", self.skus - - def raw(self, table, days, values, **kwargs): - plan = super().raw(table, days, values, **kwargs) - sku = self.field(table, *kwargs["sku_fields"]) - head, tail = plan.sql.split(" FROM ", 1) - coverage = sku_action_results.Plan( - head + ", " + sku_action_results._q(sku) + ' AS "__project_member" FROM ' + tail, - plan.args, plan.sources) - self.coverage.append((table, tuple(days), coverage)) - return plan - - def _result_figure(con, config, bounds, skus, available): """Reuse daily metric definitions and reconcile their union against separate source totals.""" if not skus: return {"state": "unavailable", "reason": "Assign an Amazon target before measuring results."} - mirror = ProjectMirror(con, skus, available) + class ProjectMirror(sku_action_results.Mirror): + def entity(self): + self.field(sku_action_results.ECONOMICS, "msku") + return '"msku" IN (' + ", ".join("?" for _ in skus) + ")", tuple(skus) + + def raw(self, table, days, values, **kwargs): + plan = super().raw(table, days, values, **kwargs) + sku = self.field(table, *kwargs["sku_fields"]) + head, tail = plan.sql.split(" FROM ", 1) + coverage = sku_action_results.Plan( + head + ", " + sku_action_results._q(sku) + ' AS "__project_member" FROM ' + tail, + plan.args, plan.sources) + self.coverage.append((table, tuple(days), coverage)) + return plan + + mirror = ProjectMirror(con, {}, available) + mirror.coverage = [] try: mirror.select_scope(bounds) except sku_action_results.SourceUnavailable as exc: @@ -412,81 +406,6 @@ def _complete_result_membership(con, coverage, bounds, sources, skus, names=("cu return True -_SERIES_UNAVAILABLE = { - "margin": "Margin requires daily net profit with verified Odoo or typed COGS. " - "amzn_economics alone cannot reconcile that cost.", - "returns": "Returns require FBA and MFN customer return events. " - "amzn_economics has no independent returned-unit aggregate.", - "returns_pct": "Returns % requires returned units over shipped units. " - "amzn_economics cannot independently reconcile those quantities.", - "refunds_pct": "Refunds % requires deduplicated refund units over shipped units. " - "amzn_economics cannot independently reconcile those quantities.", - "roas": "ROAS requires attributed ad sales over ad spend. " - "amzn_economics has no independent attributed ad-sales aggregate.", -} - - -def _series_figure(key, figure, bounds): - """Adapt existing sales and units definitions without publishing partial source windows.""" - if key in _SERIES_UNAVAILABLE: - return {"reason": _SERIES_UNAVAILABLE[key]} - if figure.get("reason"): - return {"reason": figure["reason"]} - current = figure.get("current") - if not current or current["status"] != "available": - return {"reason": (current or {}).get("reason") or "Some target source days are unavailable."} - if key != "sales_yoy": - return {"total": current["value"], "points": current["series"], "unit": figure["unit"], - "reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None} - prior = figure["comparison"] - if prior["status"] != "available" or prior["value"] is None or prior["value"] <= 0: - return {"reason": prior.get("reason") or "Sales YoY requires positive sales in the matching prior window."} - points = [] - for now, before in zip(current["series"], prior["series"]): - if before["value"] is None or before["value"] <= 0: - return {"reason": "Sales YoY requires positive sales on every matching prior day."} - points.append({"date": now["date"], "value": 100 * (now["value"] - before["value"]) / before["value"]}) - return {"total": 100 * (current["value"] - prior["value"]) / prior["value"], "points": points, - "reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None} - - -def _series_economics_total(con, key, bounds, skus, available): - """Aggregate original economics rows without the display plan, daily rollup or display total.""" - if key in _SERIES_UNAVAILABLE: - return {"value": None, "reason": _SERIES_UNAVAILABLE[key]} - if not skus or not bounds["days"]: - return {"value": None, "reason": "No completed target days are available in this range."} - mirror = ProjectMirror(con, skus, available) - try: - mirror.select_scope(bounds) - field = mirror.field(sku_action_results.ECONOMICS, - "sales_net_units_sold" if key == "units" else "sales_net_product_sales_amount") - number = f'TRY_CAST({sku_action_results._q(field)} AS DOUBLE)' - day = sku_action_results._day("start_date") - marks = ", ".join("?" for _ in skus) - - def total(days): - row = con.execute( - f'SELECT SUM({number}), count(*), count({number}), count(DISTINCT (msku, {day})) ' - f'FROM amzn_economics WHERE msku IN ({marks}) AND marketplace_id = ? ' - f'AND sales_net_product_sales_currency_code = ? AND {day} >= ? AND {day} < ?', - (*skus, *mirror.scope, *days)).fetchone() - if (not row[1] or row[1] != row[2] or row[3] != len(skus) * bounds["days"] - or not project_results._finite(row[0])): - raise sku_action_results.SourceUnavailable("amzn_economics: some target values or days are unavailable.") - return float(row[0]) - - value = total(bounds["current"]) - if key == "sales_yoy": - prior = total(bounds["comparison"]) - if prior <= 0: - raise sku_action_results.SourceUnavailable("Sales YoY requires positive prior sales.") - value = 100 * (value - prior) / prior - return {"value": value, "reason": None} - except sku_action_results.SourceUnavailable as exc: - return {"value": None, "reason": str(exc)} - - @router.get("") def row_projects(grain: str | None = None, key: str | None = None, session: Session = Depends(require_session)): @@ -589,61 +508,6 @@ def project_result_figures(project_id: str, session: Session = Depends(require_s con.close() -@router.get("/{project_id}/series") -def project_result_series(project_id: str, start: str | None = Query(None, alias="from"), - end: str | None = Query(None, alias="to"), scope: str = Query("all"), - session: Session = Depends(require_session)): - """Serve the preset daily charts within the intersection of project targets and the modal filter.""" - runtime = _reader(session) - record = _call(projects.get, runtime, project_id) - targets = _call(projects.targets, runtime, project_id) - con = None - try: - bounds = project_results.series_window(record, start, end, today=projects._today()) - selected = project_results.series_scope(scope) - selected_targets = ([{"grain": "parent_asin" if selected["grain"] == "parent" else selected["grain"], - "key": key} for key in selected["keys"]] if selected else []) - available = _result_sources(session, targets + selected_targets) - con = routes_sku_actions._results_mirror(runtime) - reason = None - try: - skus = projects.resolve_target_skus(con, targets) - if selected: - selected_skus = set(projects.resolve_target_skus(con, selected_targets)) - skus = [sku for sku in skus if sku in selected_skus] - if not skus: - reason = "No project targets match the selected scope." - elif not bounds["days"]: - reason = "Today is incomplete. No completed days are available in this range." - except projects.MirrorUnreadable as exc: - skus, reason = [], str(exc) - figures = {} - - def read(key): - if reason: - return {"reason": reason} - metric = "sales" if key == "sales_yoy" else key - if metric not in _SERIES_UNAVAILABLE and metric not in figures: - figures[metric] = _result_figure(con, {"metric": metric}, bounds, skus, available) - return _series_figure(key, figures.get(metric, {}), bounds) - - return project_results.series(record, bounds=bounds, read=read, - independent=lambda key: _series_economics_total(con, key, bounds, skus, available)) - except HTTPException: - raise - except (project_results.ValidationFailed, sku_action_results.ValidationFailed): - raise err(409, "project_results_not_validated", - "A series total does not tie to its independent Amazon economics aggregate.") from None - except ValueError as exc: - raise err(400, "invalid_project_results", str(exc)) from None - except Exception as exc: # noqa: BLE001 - print(f"[amazon-projects] series failed: {type(exc).__name__}: {exc}", file=sys.stderr) - raise err(503, "project_results_unavailable", "Project result sources are unavailable.") from None - finally: - if con is not None: - con.close() - - @router.patch("/{project_id}") def edit_project(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)): """Change the named fields. Targets named here are added.""" @@ -658,11 +522,8 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi record = _call(projects.update, runtime, project_id, values, by=_by(session)) if targets: record = _call(projects.add_targets, runtime, project_id, targets, by=_by(session))["project"] - # A status or progress edit repaints every assigned target, including when a project reopens. - if {"status", "progress"} & set(values): - _repaint(runtime, _target_pairs(runtime, project_id)) - elif targets: - _repaint(runtime, targets) + # Only status and progress move a cell: a name or note edit repaints only the targets it adds. + _repaint(runtime, _target_pairs(runtime, project_id) if {"status", "progress"} & set(values) else targets) return {"project": _project(record, projects._today())} diff --git a/api/routes_assets.py b/api/routes_assets.py index 0b065111cb434e1f9f4c1bc493075d8ff2f8d3df..38da9a688520fb76317073888a5b475cb5e043c3 100644 --- a/api/routes_assets.py +++ b/api/routes_assets.py @@ -145,18 +145,6 @@ def code_access(session, application_id): return True -def alerts_access(session, application_id): - """Admit a source-free Alerts App through its registry and Automation permission walls.""" - app = applications.load(session.runtime)["applications"].get(application_id) - if not isinstance(app, dict) or app.get("templateKind") != "alerts" or not applications._database_free(app): - return False - if (app.get("tenant") != session.tenant or session.tenant != session.runtime.key - or app.get("retired") is True or not applications.may_read(app, session.uname, session.admin) - or not _session_wall(session)("automation")): - raise applications.ApplicationError("application does not exist") - return True - - # Tokens and the embedded Inter font come from the same stylesheet as the shell. _CODE_KIT_COMPONENTS = """ :root { @@ -440,8 +428,7 @@ def delete_application(application_id: str, body: dict = Body(default=None), session: Session = Depends(require_session)): try: # A database-free App still needs its own read and edit permissions. - if (finance_access(session, application_id) or code_access(session, application_id) - or alerts_access(session, application_id)): + if finance_access(session, application_id) or code_access(session, application_id): allowed = set() else: current = _application_databases(session.runtime, application_id) diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 90934cc21b0a170373992543bd73c814386a9d2f..c739077679dd4701c6bf3914c8bea6e90c19c7f5 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -65,7 +65,7 @@ from core import schema_overlay #: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")` #: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a #: second correct one here would leave the same two-copies shape that produced the defect. -from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available, stored_business_tz +from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available #: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a #: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any #: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router` @@ -1095,25 +1095,15 @@ def _contact_rows(rt, uname, is_admin, incomplete): def read(source, keys, fn): try: fn() - except Exception as exc: # noqa: BLE001 - logging.getLogger(__name__).warning("contact_check=%s exception=%s", - source, type(exc).__name__) + except Exception: # noqa: BLE001 for key in keys: by_key[key]["state"] = "unavailable" - label = {"whatsapp": "WhatsApp", "slack": "Slack", "mcp": "MCP", - "loopable-api": "Loopable API"}[source] - incomplete.append({"source": source, "cause": f"{label} connection status unavailable.", + incomplete.append({"source": source, "cause": "Connection status unavailable.", "recommendation": "Try again."}) def whatsapp(): - from core import account_rows, users - from whatsapp_identity import WhatsAppRefusal - # Directory status reads the reader's canonical account, never registration challenges. - record = account_rows.get(uname, st=users.store, fresh=True) - if not isinstance(record, dict) or str(record.get("tenant") or "royal-imports") != rt.key: - raise WhatsAppRefusal("This account is not available in this workspace.") - connection = record.get("whatsapp") - if isinstance(connection, dict) and connection.get("phone"): + import whatsapp_identity + if whatsapp_identity.status(rt.key, uname).get("state") == "connected": by_key["whatsapp"]["state"] = "connected" def slack(): @@ -3066,88 +3056,6 @@ def _amazon_mirror_completion(rt): return None, False -def _amazon_table_schedule(rt, connection_id=None, now=None): - """Read account cadence metadata without restoring a mirror or reading source rows.""" - import duckdb - from core import keychain - from harness import amazon_store as az, datastore - - now = now or datetime.now(timezone.utc) - result = {"tableSchedule": [], "scheduleState": "unavailable", - "displayTimezone": stored_business_tz(rt) or "UTC"} - - def read(cursor, entries): - schedule = [] - for entry in entries: - cid = entry["connection_id"] - prefix = az._account_prefix(cid) - keys = [prefix + az.entity_key(family + table) - for table in az.TABLES for family in ("", "sync_tick_", az.WATERMARK_PREFIX)] - rows = cursor.execute( - "SELECT entity, phase, rows, updated_at, " - "CASE WHEN phase = 'throttled' THEN cursor_wd ELSE NULL END FROM _sync_state " - "WHERE entity IN (SELECT unnest(?))", [keys]).fetchall() - markers = {entity: {"state": phase, "rows": count, "updated_at": _sync_stamp(at), - "detail": _sync_stamp(detail)} - for entity, phase, count, at, detail in rows} - states = {table: state for table in az.TABLES - if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))} - if any(not state["updated_at"] for state in states.values()): - raise ValueError("Amazon cadence metadata has an invalid timestamp") - # C1 owns every due time, including throttle retries and calendar periods. - for row in az.sync_schedule(states, now): - table = row["table"] - tick = states.get(table) or {} - watermark = markers.get(prefix + az.entity_key(az.WATERMARK_PREFIX + table)) or {} - landed = markers.get(prefix + az.entity_key(table)) or {} - completed = (tick if tick.get("state") == "done" - and landed.get("state") not in ("refused", "failed", "throttled") else watermark) - schedule.append({**row, "connectionId": cid, "account": entry["label"], - "lastSyncAt": (completed.get("updated_at") - if completed.get("state") == "done" else None), - "state": (landed.get("state") - if landed.get("state") in ("refused", "failed", "throttled") - else tick.get("state") or "never_run")}) - return schedule - - try: - path = az._tenant_mirror_path(rt.key, rt=rt) - if not keychain.list_entries(rt, "amazon_sp"): - return {**result, "scheduleState": "empty"} - entries = az.account_entries(rt.key, rt) - cid = str(connection_id or "").strip() - if cid and cid != "default": - entries = [entry for entry in entries if entry["connection_id"] == cid] - if not entries: - return result - if not path.is_file(): - return result - if path.resolve() == Path(datastore.DB_PATH).resolve(): - schedule = read(datastore.ro_cursor(), entries) - else: - with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor: - schedule = read(cursor, entries) - return {**result, "tableSchedule": schedule, "scheduleState": "ready"} - except Exception as exc: - raise_if_resource_error(exc) - return result - - -def _amazon_sync_staleness(completed, schedule, now=None): - """Use the fastest Amazon table's cadence and Odoo's existing grace period.""" - import odoo_relational as rel - fastest = max(amazon_client.CADENCE[table]["perDay"] - for table in amazon_client.GROUP_A) - allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS - stamps = [row.get("lastSyncAt") for row in schedule if row["perDay"] == fastest] - if not stamps: - stamps = [completed] - now = now or datetime.now(timezone.utc) - stale = any((now - datetime.fromisoformat(stamp)).total_seconds() > allowed - for value in stamps if (stamp := _sync_stamp(value))) - return {"stale": stale, "staleAfterSeconds": allowed} - - def _daily_completed_sync(rt, connector_key): """Read the stored last good sync; never create a mirror or contact a provider.""" if connector_key == "amazon_sp": @@ -3246,11 +3154,8 @@ def connector_sync_status(connector_key: str, connection_id: str = None, if connector_key not in registered: raise err(404, "sync_provider_unknown", f"{connector_key} is not registered for connector sync") - from core import keychain - import odoo_relational as rel rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo" - else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY, - connector_sync.STATUS_KEY, keychain.KEY, rel.CONFIG_KEY) + else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY) if connector_key == "amazon_sp" else ())) resolved_connection = (_odoo_connection_slot(rt, connection_id) if connector_key == "odoo" else connection_id) @@ -3274,15 +3179,9 @@ def connector_sync_status(connector_key: str, connection_id: str = None, if connector_key == "amazon_sp" and completed is None and known and _amazon_restore_pending(rt): completed = _amazon_saved_completion(rt) answer["restoring"] = True - answer.update(lastCompletedAt=completed, lastSyncAt=completed, - lastCompletedKnown=known, canSync=bool(session.admin)) - if connector_key == "amazon_sp": - now = datetime.now(timezone.utc) - schedule = _amazon_table_schedule(rt, resolved_connection, now) - answer.update(schedule="per_table", **schedule, - **_amazon_sync_staleness(completed, schedule["tableSchedule"], now)) - else: - answer.update(schedule="daily", **_daily_sync_staleness(completed)) + answer.update(schedule="daily", lastCompletedAt=completed, lastSyncAt=completed, + lastCompletedKnown=known, canSync=bool(session.admin), + **_daily_sync_staleness(completed)) return _safe_connector_status(connector_key, answer) diff --git a/api/routes_external.py b/api/routes_external.py index 79f0dc7a7ce6f0f219e93f3746aea28224471cf6..65dd1d58b022b28ec0fa83e3216a61609e4260a8 100644 --- a/api/routes_external.py +++ b/api/routes_external.py @@ -153,20 +153,6 @@ def _activity_store_view(real, insert): return view -def _sheet_row_revision(tx, key): - """Read the header's store revision on the same transaction as its sheet rows.""" - from core import store_pg, view_audit - if isinstance(tx, sheets._PgSheetRows): - row = tx.con.execute(store_pg._q( - "SELECT rev, updated_at FROM {tbl} WHERE key = %s", "store_kv", tx.slug), - (tx.namespace + key,)).fetchone() - return store_pg._revision_value(row[0], row[1]) if row else store_pg._revision_value(None, None) - reader = getattr(type(tx), "revision", None) - if callable(reader): - return reader(tx, key) - raise view_audit.RevisionUnverifiable("The Sheet transaction cannot verify its header revision.") - - class _PreviewRuntime: """Lazy copy-on-write view of one tenant runtime; no method may reach a real writer. @@ -194,20 +180,11 @@ class _PreviewRuntime: if key not in self._docs: import application_registry value = application_registry._read_strict(self._real, key) - revision = None - if key.startswith(sheets.WORKBOOK_KEY_PREFIX) and sheets.is_row_workbook(value): - # Hydrate through the reader, never delegate a native writer to this preview. - with sheets._sheet_rows_transaction(self._real) as tx: - if tx is None: - raise err(503, "workbook_unavailable", "The workbook snapshot is unavailable. Try again.") - head = tx.get(key) - value = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head - revision = _sheet_row_revision(tx, key) value = copy.deepcopy(value) self._initial[key] = copy.deepcopy(value) self._docs[key] = value try: - revision = revision if revision is not None else self._real.revision(key) + revision = self._real.revision(key) except Exception: revision = None self._base_revisions[key] = copy.deepcopy(revision) @@ -351,70 +328,6 @@ class _ConfirmationRuntime: def tenant(self): return getattr(self._real, "tenant", None) - @contextmanager - def sheet_rows_transaction(self, *, write=False): - """Guard native Sheet writes under the header lock with the preview's store revision.""" - from core import view_audit - with sheets._sheet_rows_transaction(self._real, write=write) as tx: - if tx is None: - yield tx - return - owner = self - held = {} - - class GuardedRows: - def get(self, key, *, lock=False, **selection): - if lock and not write: - raise view_audit.RevisionUnverifiable("A Sheet read cannot acquire write authority.") - value = tx.get(key, lock=lock, **selection) - if lock: - expected = owner._expected.get(key) - rev = expected.get("rev") if isinstance(expected, dict) else None - if (not key.startswith(sheets.WORKBOOK_KEY_PREFIX) or type(rev) is not int - or held): - raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed header.") - current = _sheet_row_revision(tx, key) - if current.get("rev") != rev or ("token" in expected and current.get("token") != expected["token"]): - raise view_audit.ViewRevisionStale(rev_current=current.get("rev"), rev_sent=rev) - before = (sheets._verify_row_layout(tx, value) if sheets.is_row_workbook(value) - else tx.get(key)) - held.update(key=key, before=copy.deepcopy(before), head=value) - return value - - def revision(self, key): - return _sheet_row_revision(tx, key) - - def put(self, key, value): - header = held.get("key") - if not header or (key != header and key != sheets.sheet_row_key( - header[len(sheets.WORKBOOK_KEY_PREFIX):], value.get("id", ""))): - raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed workbook.") - if key == header and value and value.get("id") != header[len(sheets.WORKBOOK_KEY_PREFIX):]: - raise view_audit.RevisionUnverifiable("The Sheet header identity changed.") - tx.put(key, value) - - def delete(self, key): - head = held.get("head") or {} - if key not in {sheets.sheet_row_key(head["id"], ref["id"]) - for ref in head.get("rowSheets", [])}: - raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed sheet.") - tx.delete(key) - - yield GuardedRows() - if held: - key = held["key"] - head = tx.get(key) - after = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head - produced = _sheet_row_revision(tx, key) - if held: - self._expected[key] = copy.deepcopy(produced) - if held["before"] != after: - previous = self._committed_documents.get(key) - self._committed_documents[key] = { - "before": previous["before"] if previous else held["before"], - "after": copy.deepcopy(after), - } - def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None): from core import view_audit key = str(name) @@ -1990,22 +1903,6 @@ def list_keys(session: Session = Depends(require_session)): return {"keys": keys, "can_mint": _can_use_analyst(session)} -@router.post("/keys/verify") -def verify_owned_key(response: Response, body=Body(default=None), session: Session = Depends(require_session)): - """Return a private diagnosis for this session owner's key without accepting bearer sign-in.""" - require_external_analyst(session) - token = body.get("token") if isinstance(body, dict) else None - if not isinstance(token, str) or len(token) > 1024: - token = "" - try: - result = external_access.verify_for_owner( - session.runtime, session.uname, session.tenant, token) - except external_access.ExternalAccessError as exc: - _management_refusal(exc) - response.headers["Cache-Control"] = "no-store" - return result - - @router.patch("/keys/{key_id}") def edit_key(key_id: str, body=Body(default=None), session: Session = Depends(require_session)): """Edit metadata and narrowing scope without ever reading or replacing the secret.""" @@ -3327,8 +3224,11 @@ def _validate_personal_view_draft(session: Session, database: str, raw: dict) -> if any(key not in visible for key in _view_referenced_fields(config)): raise err(400, "assistant_unknown_field", "Every requested field must be readable in this database.") - # Saved predicates use Table's cleaner. The reader still validates permanent permission filters. - _external_read_scope(session, database, fields=[], filters=None) + # The ordinary Assistant validator owns filter syntax and operands. fields=[] keeps values + # out of this validation result while still applying the live database/field/row walls. + filters = config.get("filters") + _external_read_scope( + session, database, fields=[], filters=None if filters == [] else filters) def _saved_view_filters(config): @@ -3337,9 +3237,9 @@ def _saved_view_filters(config): _assert_query_filter_bounds([value] if isinstance(value, dict) else value) import aios_grid - def refuse(message=None): + def refuse(): raise err(400, "view_filters_unsupported", - message or "These filters cannot be saved without changing their meaning. " + "These filters cannot be saved without changing their meaning. " "Use colId, op and value leaves, with conj and children for nested groups.") def node(raw): @@ -3351,17 +3251,8 @@ def _saved_view_filters(config): or not isinstance(raw[key], list) or not raw[key]): refuse() return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]} - if any(key in raw for key in ("valueRef", "value2Ref")): - refuse("Cell references cannot be saved through this tool because workbook and cell access " - "cannot be verified here. Use literal values or a readable rhs field instead.") - op = raw.get("op") - if not isinstance(op, str) or op not in aios_grid.FILTER_OPS: - name = json.dumps(op[:120]) if isinstance(op, str) else "(missing or invalid)" - refuse(f"Unsupported filter operator {name}. Use a supported operator: " - + ", ".join(sorted(aios_grid.FILTER_OPS)) + ".") - if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked", - "dateWindow", "dateMode", "rhs"} - or not isinstance(raw.get("colId"), str)): + if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked"} + or not isinstance(raw.get("colId"), str) or raw.get("op") not in aios_grid.FILTER_OPS): refuse() for key in ("value", "value2"): value = raw.get(key) diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index c0312620ac51eee064dfd723aa7b6da93971e543..e5a51eb38389a12b737c33b9aa80e6ca8e331acd 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -1393,22 +1393,6 @@ def require_amazon_mirror(table_key, session, definition=None, st=None): cur.close() -def require_dynamic_odoo_mirror(table_key, session, st=None): - """Admit a saved dynamic definition before resolving its publication or refusal.""" - if not str(table_key).startswith('ut_odoo_model_'): - return None - from routes_tables import _defn_or_refuse - try: - _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) - except HTTPException as exc: - if exc.status_code == 404: - raise err(404, 'odoo_definition_not_published', - 'This Odoo database has no published definition. Enable it in connection ' - 'settings and complete its first sync before reading rows.') from exc - raise - return not _materialised_table(table_key, st=st if st is not None else session.runtime) - - register_source_provider(_odoo_sources) register_source_provider(_amazon_sources) register_source_provider(_meta_sources) @@ -1625,25 +1609,19 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku """Resolve one admitted Amazon target and keep query failures distinct from stored tables.""" import amazon_relational as amazon - if (definition or {}).get('oversize'): - definition = {**definition, 'readThrough': True} try: if binding: columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True) spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups) else: spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups) - if key == amazon.PREFIX + "inventory" and spec is not None and not spec.get("missing"): - status = amazon.amazon_store.inventory_table_status(cur) - spec = {**spec, 'sourceStatus': { - name: status[name] for name in ('empty', 'code', 'reason') if name in status}} except Exception as exc: # noqa: BLE001 from core.request_timing import raise_if_resource_error raise_if_resource_error(exc) raise err(503, "amazon_source_unavailable", "This Amazon database could not be resolved. Retry the request.") from exc if spec is not None and spec.get("missing"): - raise err(409, "amazon_mirror_not_loaded", spec["missing"]) + raise err(503, "amazon_mirror_not_loaded", spec["missing"]) if spec is not None and '_connection_id' in { name for name, _kind in amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)}: spec = {**spec, 'cols': {**spec['cols'], '_connection_id': ( @@ -1679,10 +1657,7 @@ def _source_for(cur, table_key, *, allow_lookups=True): if module_cache_source(key) is not None: return None if key.startswith("ut_odoo_model_"): - try: - return _rel().dynamic_model_source(cur, key) - except _rel().Refused as exc: - raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc + return _rel().dynamic_model_source(cur, key) if key.startswith('ut_meta_'): return _meta_source(cur, key) if key.startswith("ut_amzn_"): @@ -1698,7 +1673,7 @@ def _source_for(cur, table_key, *, allow_lookups=True): if not spec: return None if spec.get("missing"): - raise err(409, "amazon_mirror_not_loaded", spec["missing"]) + raise err(503, "amazon_mirror_not_loaded", spec["missing"]) from harness import datastore if _rel().base_table_key(table_key) == _rel().BILLS_KEY: try: @@ -1737,7 +1712,7 @@ def _source_for(cur, table_key, *, allow_lookups=True): if not cols: if key.startswith("ut_amzn_"): import amazon_relational - raise err(409, "amazon_mirror_not_loaded", + raise err(503, "amazon_mirror_not_loaded", amazon_relational.missing_source(key)["missing"]) return None # the mirror has no such table on this deployment have_by_alias[alias] = cols @@ -2667,33 +2642,16 @@ def _materialised_table(table_key, *, cur=None, st=None): if module_cache_source(table_key) is not None: return True if str(table_key).startswith("ut_odoo_model_"): - definition = _ut().get_definition(table_key, st=st) if st is not None else {} - mandatory = bool((definition or {}).get('readThrough') or (definition or {}).get('oversize')) - if st is not None and not mandatory: + if st is not None and (_ut().get_definition(table_key, st=st) or {}).get("readThrough") is not True: return True own_cursor = cur is None if own_cursor: if st is None: return True - try: - cur = _table_cursor(st, table_key) - except _rt().DatastoreMismatch as exc: - raise err(409, 'cross_tenant_store', str(exc)) from exc - except _rel().Refused as exc: - raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc - except RuntimeError as exc: - raise err(409, 'odoo_mirror_not_loaded', str(exc)) from exc + cur = _table_cursor(st, table_key) try: - try: - spec = _rel().dynamic_model_source(cur, table_key, - tenant_key=getattr(st, "key", None)) - except _rel().Refused as exc: - raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc - if mandatory and spec is None: - raise err(409, 'odoo_mirror_not_loaded', - 'This Odoo database has no complete mirror publication. Complete its ' - 'sync in connection settings before reading rows.') - return spec is None + return _rel().dynamic_model_source(cur, table_key, + tenant_key=getattr(st, "key", None)) is None finally: if own_cursor and cur is not None: cur.close() @@ -2705,7 +2663,7 @@ def _materialised_table(table_key, *, cur=None, st=None): own_cursor = cur is None cur = datastore.ro_cursor() if own_cursor else cur try: - definition = _ut().get_definition(table_key, st=st) if st is not None else None + definition = _ut().get(table_key, st=st) if st is not None else None spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False) return spec is None finally: @@ -4051,8 +4009,6 @@ def resolved_fields(table_key, defn): if str(table_key).startswith("ut_amzn_"): import amazon_relational as amazon connected = table_key in amazon.SOURCE - elif str(base).startswith('ut_odoo_model_'): - connected = True else: connected = table_key in _sources() if not connected: @@ -4152,7 +4108,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= # try, not inside it. try: cur, mismatch = (_mirror_cur(session, table_key) - if (table_key.startswith(("ut_amzn_", "ut_meta_", "ut_odoo_model_")) + if (table_key.startswith(("ut_amzn_", "ut_meta_")) or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY or rel.base_table_key(table_key) in rel.stock_source_specs()) else _mirror_cur(session)) @@ -4858,11 +4814,7 @@ def _window_principal(session): """Bind pagination to identity and permissions without volatile activity stamps.""" import registry_source policy = {k: v for k, v in session.user.items() if k not in registry_source.USER_NOISE} - claims = getattr(session, 'claims', {}) - if isinstance(claims, dict): - # Authentication renews idle expiry on use without changing the caller's access. - claims = {key: value for key, value in claims.items() if key != 'i'} - return [session.tenant, session.uname, claims, policy] + return [session.tenant, session.uname, getattr(session, 'claims', {}), policy] def _window_cursor(session, query, position=None, token=None): @@ -5165,10 +5117,8 @@ def odoo_table_rows(table_key: str, ids = _record_ids(body) # Admission resolves this Amazon database without publishing a process-wide binding. - page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None + page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None) - if str(table_key).startswith('ut_odoo_model_'): - admitted = True if require_dynamic_odoo_mirror(table_key, session, st=page[0]) else None if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)): return _audit_personal_response(table_key, _materialised_window( table_key, session, offset, limit, filters, filterConj, @@ -5188,11 +5138,6 @@ def odoo_table_rows(table_key: str, read = _restrict_row_ids(read, ids) defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"] cols, limits, frm = read["cols"], read["limits"], read["frm"] - source_status = spec.get('sourceStatus') or {} - if source_status.get('empty') and source_status.get('reason'): - limits = [*limits, {'subject': table_key, 'effect': 'source_empty', - 'code': source_status.get('code'), 'cause': source_status['reason'], - 'recommendation': ''}] base_where, wall_params = read["base_where"], read["wall_params"] where, params, sort_spec = read["where"], read["params"], read["sort_spec"] _gsql, _gparams = _group_pin(cols, groupBy, groupKey) @@ -5337,7 +5282,6 @@ def odoo_table_rows(table_key: str, if str(row["pid"]) in notes: row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])] return _audit_personal_response(table_key, {"ok": True, "fields": fields, "rows": rows, - **source_status, # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`. "total": win["total"], "totalUnfiltered": unfiltered, "offset": win["offset"], "limit": win["limit"], "limits": limits, @@ -5802,10 +5746,8 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None), if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS): raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}") - page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None + page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None) - if str(table_key).startswith('ut_odoo_model_'): - admitted = True if require_dynamic_odoo_mirror(table_key, session, st=page[0]) else None if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)): with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read: diff --git a/api/routes_tables.py b/api/routes_tables.py index 0d85992b6d44ef2a4391a5c1041a84267d713620..29dcf635aa5d1b5b020d36ed42ad6b2b5852ab05 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -4476,8 +4476,6 @@ def delete_table_view(table_key: str, view_id: str, request: Request, "rev_current": stale.rev_current}}) except view_audit.RevisionUnverifiable as unknown: raise err(503, "view_revision_unverifiable", str(unknown)) - from core import shares - shares.drop_objects([('view', vid)], st=ops.st) return {"deleted": vid, "owner": owner} diff --git a/api/sync_worker.py b/api/sync_worker.py index 184b55e7cb9d7af42ef9575c3ceddd593152fe38..76112aa19527e381ab387f23a1ac3862bf1def6b 100644 --- a/api/sync_worker.py +++ b/api/sync_worker.py @@ -35,10 +35,9 @@ def configure(env=None): worker_db = worker_dir / f"{safe}.duckdb" env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir) env["AIOS_DUCKDB_PATH"] = str(worker_db) - # Leave room for Python and the web processes when a full Brand Analytics week is parsed. - env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "6GB" + # The launcher (request_class.py --env) sizes these to the box. Unset: the 2 vCPU values. + env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB" env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1" - env["AIOS_DUCKDB_MAX_TEMP_SIZE"] = env.get("AIOS_WORKER_DUCKDB_MAX_TEMP_SIZE") or "16GB" # The worker holds its own file read-write. Mirror versions are for the web workers only. env["AIOS_MIRROR_VERSIONS"] = "0" env["AIOS_UVICORN_WORKERS"] = "1" @@ -130,7 +129,6 @@ def main(argv=None): print(f"[sync-worker] pid={os.getpid()} tenant={tenant} " f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} " f"memory_limit={os.environ['AIOS_DUCKDB_MEMORY']} threads={os.environ['AIOS_DUCKDB_THREADS']} " - f"spill_limit={os.environ['AIOS_DUCKDB_MAX_TEMP_SIZE']} " f"nice={niceness} job={mode or 'none'}", flush=True) print("[sync-worker] amazon_store.sync includes Brand Analytics reports in the worker role", flush=True) diff --git a/api/tool_groups/apps.py b/api/tool_groups/apps.py index 1705bf2e0317c732c0398639878535cd7fcae447..443f14c988adbe9913c92da1bfdf30442242ad70 100644 --- a/api/tool_groups/apps.py +++ b/api/tool_groups/apps.py @@ -470,15 +470,7 @@ def apply_template(ctx, args): allowed_databases=allowed, semantic_records=records, field_contracts=_fields(ctx, allowed), **attribution) return _result(result) - except applications.ApplicationError as exc: - # Keep typed refusals and inaccessible-App responses on their original path. - if type(exc) is applications.ApplicationError and "does not exist" not in str(exc): - kind = json.dumps(args["templateKind"], ensure_ascii=True) - exc = applications.ApplicationError( - f"Cannot apply templateKind {kind}. {str(exc).rstrip('.')}. " - "Check this template's inputs with apps.templates and retry.") - _refuse(exc) - except PermissionError as exc: + except (applications.ApplicationError, PermissionError) as exc: _refuse(exc) diff --git a/api/whatsapp_identity.py b/api/whatsapp_identity.py index 72e6f44cbab5554b08a28ae20de472d24cc4e02f..8c782b104e8a64ae03670995f32e2216121d0971 100644 --- a/api/whatsapp_identity.py +++ b/api/whatsapp_identity.py @@ -8,11 +8,9 @@ import re import secrets import time from datetime import datetime, timezone -from types import SimpleNamespace import core.store as store import core.users as users -from core import account_rows import telnyx_whatsapp _store = store @@ -50,16 +48,8 @@ def _ownership_read(source, key): return reader(key) if callable(reader) else source.get(key, fresh=True) -def _account_source(): - source = users.store - if account_rows._pg_source(source): - return source - # Compatibility stores must refuse stale fallback for canonical ownership reads too. - return SimpleNamespace(get=lambda key, fresh=False: _ownership_read(source, key)) - - def _account(tenant, username): - record = account_rows.get(username, st=_account_source(), fresh=True) + record = (_ownership_read(users.store, "users") or {}).get(username) if not isinstance(record, dict) or str(record.get("tenant") or "royal-imports") != tenant: raise WhatsAppRefusal("This account is not available in this workspace.") return record @@ -68,7 +58,7 @@ def _account(tenant, username): def _phone_conflict(phone, tenant, username, index=None): index = (_ownership_read(_store, INDEX_KEY) or {}) if index is None else index # The registry proves ownership. An interrupted index write proves no binding. - registry = dict((account_rows.registry(st=_account_source()) or {}).items()) + registry = _ownership_read(users.store, "users") or {} caller = registry.get(username) if not isinstance(caller, dict) or str(caller.get("tenant") or "royal-imports") != tenant: raise WhatsAppRefusal("This account is not available in this workspace.") @@ -91,7 +81,7 @@ def lookup(phone_e164): entry = (_ownership_read(_store, INDEX_KEY) or {}).get(phone_e164) if not isinstance(entry, dict): return None - record = account_rows.get(entry.get("username"), st=_account_source(), fresh=True) + record = (_ownership_read(users.store, "users") or {}).get(entry.get("username")) if not isinstance(record, dict): return None whatsapp = record.get("whatsapp") diff --git a/platform/core/activity_log.py b/platform/core/activity_log.py index d1719094ee2000d041272d39bb9f3a67aaf2dee1..5836c5a607de954d84c3e3f2137868d9be0306dc 100644 --- a/platform/core/activity_log.py +++ b/platform/core/activity_log.py @@ -12,7 +12,7 @@ import re DATABASE_KEY = 'ut_activity' SYSTEM_KIND = 'activity' -KINDS = frozenset(('record', 'field', 'cell', 'file', 'app', 'view', 'view_as')) +KINDS = frozenset(('record', 'field', 'cell', 'file', 'app', 'view')) OBJECT_TYPES = frozenset(('database', 'workbook', 'file', 'app')) CHANGES_MAX_BYTES = 4096 RETENTION_MONTHS = 13 @@ -242,7 +242,7 @@ def seal_copy_batch(rt, events, outbox, *, backup_path, definitions=None): def append(rt, *, kind, object, by, via=None, on_behalf_of=None, changes=None, _delivery=None): """Append one C5 event and remove at most PRUNE_BATCH expired events in the same transaction.""" if kind not in KINDS: - raise ValueError('kind must be record, field, cell, file, app, view or view_as.') + raise ValueError('kind must be record, field, cell, file, app or view.') object_type, object_key, object_id = _object(object) by = _text(by, 'by') via = _text(via, 'via', optional=True) @@ -522,13 +522,12 @@ class PgActivityStore: needed = {f'{self.name}_{suffix}' for suffix in ( 'delivery', 'time', 'object_time', 'object_id_time', 'person_time')} if (set(_COLUMNS) | {'delivery_key', 'delivery_version'} <= {r[0] for r in columns} - and needed <= {r[0] for r in indexes} and constraint - and "'view_as'" in constraint[0]): + and needed <= {r[0] for r in indexes} and constraint and "'view'" in constraint[0]): return con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} ( id BIGSERIAL PRIMARY KEY, occurred_at TIMESTAMPTZ NOT NULL, - kind TEXT NOT NULL CHECK (kind IN ('record','field','cell','file','app','view','view_as')), + kind TEXT NOT NULL CHECK (kind IN ('record','field','cell','file','app','view')), object_type TEXT NOT NULL, object_key TEXT NOT NULL, object_id TEXT NOT NULL, actor TEXT NOT NULL, via TEXT, on_behalf_of TEXT, changes JSONB NOT NULL, changes_truncated BOOLEAN NOT NULL, @@ -542,11 +541,11 @@ class PgActivityStore: 'SELECT pg_get_constraintdef(oid) FROM pg_constraint ' 'WHERE conrelid = %s::regclass AND conname = %s', (self.table, constraint)).fetchone() - if prior is None or "'view_as'" not in prior[0]: + if prior is None or "'view'" not in prior[0]: con.execute(f'ALTER TABLE {self.table} DROP CONSTRAINT IF EXISTS "{constraint}"') # The former constraint already admitted every existing event kind. con.execute(f'ALTER TABLE {self.table} ADD CONSTRAINT "{constraint}" ' - "CHECK (kind IN ('record','field','cell','file','app','view','view_as')) NOT VALID") + "CHECK (kind IN ('record','field','cell','file','app','view')) NOT VALID") for suffix, columns in ( ('time', 'occurred_at DESC, id DESC'), ('object_time', 'object_type, object_key, occurred_at DESC, id DESC'), diff --git a/platform/core/amazon_project_results.py b/platform/core/amazon_project_results.py index 8276e5e66a96ead2fe731511b9ab37fbb8aa9cf7..da529b7db92725bfe49e237360ce6587f51fb088 100644 --- a/platform/core/amazon_project_results.py +++ b/platform/core/amazon_project_results.py @@ -1,4 +1,4 @@ -"""Typed project results and daily series with independent source reconciliation.""" +"""Typed project result configurations and tenant-day completion windows.""" from datetime import date, timedelta import hashlib @@ -13,121 +13,6 @@ METRICS = ('sales', 'units', 'returns', 'return_rate', 'refund_rate', 'roas') GRAINS = ('sku', 'asin', 'parent_asin') MAX_CONFIGS = 20 BOUNDARY = '[completedOn, endExclusive) in the tenant timezone. Today is excluded.' -SERIES_METRICS = ( - ('sales', 'Sales', 'currency'), ('units', 'Units', 'units'), - ('margin', 'Margin', 'percent'), ('returns', 'Returns', 'units'), - ('returns_pct', 'Returns %', 'percent'), ('refunds_pct', 'Refunds %', 'percent'), - ('roas', 'ROAS', 'ratio'), ('sales_yoy', 'Sales YoY', 'percent'), -) -SERIES_BOUNDARY = 'Inclusive requested dates in the tenant timezone. Today is incomplete and excluded from totals.' - - -class ValidationFailed(RuntimeError): - """A daily project series does not reconcile to its independent source aggregate.""" - - -def series_scope(value='all'): - """Parse the modal filter without allowing it to widen project membership.""" - if value == 'all': - return None - if not isinstance(value, str) or ':' not in value: - raise ValueError('scope must be all, sku:, asin: or parent:.') - grain, text = value.split(':', 1) - keys = [key.strip() for key in text.split(',')] - if (grain not in ('sku', 'asin', 'parent') or not all(keys) - or (grain != 'sku' and len(keys) != 1)): - raise ValueError('scope must be all, sku:, asin: or parent:.') - return {'grain': grain, 'keys': list(dict.fromkeys(keys))} - - -def series_window(project, start=None, end=None, *, today=None): - """Default to thirty days before start through today and aggregate only completed tenant days.""" - today = today or periods.today() - - def parsed(value): - if not isinstance(value, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}', value): - raise ValueError('Series dates must use YYYY-MM-DD.') - return date.fromisoformat(value) - - first = (parsed(start) if start is not None else - parsed(project.get('start_on') or today.isoformat()) - timedelta(days=30)) - last = parsed(end) if end is not None else today - if first > last or last > today: - raise ValueError('Series dates must be ordered and cannot end after today.') - stop = min(last + timedelta(days=1), today) - lo, hi, rule = periods.yoy_window(first.isoformat(), stop.isoformat(), 'day') - return {'from': first.isoformat(), 'to': last.isoformat(), 'days': (stop - first).days, - 'current': (first, stop), 'comparison': (date.fromisoformat(lo), date.fromisoformat(hi)), - 'compare_rule': rule} - - -def _finite(value): - return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) - - -def validate(answer, *, independent): - """Requery independent totals and return failing checks for any changed or unproved series.""" - checks = [] - first, last = date.fromisoformat(answer['from']), date.fromisoformat(answer['to']) - dates = [(first + timedelta(days=i)).isoformat() for i in range((last - first).days + 1)] - expected_keys = [key for key, _label, _unit in SERIES_METRICS] - metrics = answer['metrics'] - if [metric['key'] for metric in metrics] != expected_keys: - return [{'check': 'series total ties to economics', 'ok': False, - 'reason': 'Series metrics do not follow the project result contract.'}] - for metric in metrics: - ours = metric.get('total') - source = independent(metric['key']) - expected, source_reason = source['value'], source.get('reason') - points = metric.get('points', []) - ok = [point.get('date') for point in points] == dates - if ours is None: - ok = ok and bool(metric.get('reason')) and all(point.get('value') is None for point in points) - else: - ok = (ok and _finite(ours) and _finite(expected) - and math.isclose(ours, expected, rel_tol=1e-9, abs_tol=1e-6) - and all(point.get('value') is None or _finite(point['value']) for point in points)) - if metric['key'] in ('sales', 'units', 'returns'): - values = [point['value'] for point in points if point.get('value') is not None] - ok = ok and bool(values) and math.isclose(math.fsum(values), ours, rel_tol=1e-9, abs_tol=1e-6) - tie = metric.get('tie', {}) - if tie.get('ok') or tie.get('ours') is not None or tie.get('independent') is not None: - ok = (ok and ours is not None and tie.get('ok') is True - and _finite(tie.get('ours')) and _finite(tie.get('independent')) - and math.isclose(tie['ours'], ours, rel_tol=1e-9, abs_tol=1e-6) - and _finite(expected) - and math.isclose(tie['independent'], expected, rel_tol=1e-9, abs_tol=1e-6)) - checks.append({'check': 'series total ties to economics', 'metric': metric['key'], 'ok': bool(ok), - 'ours': ours, 'independent': expected, - 'reason': metric.get('reason') or source_reason}) - return checks - - -def series(project, *, bounds, read, independent): - """Publish the fixed chart order and project markers after independent economics reconciliation.""" - first, last = date.fromisoformat(bounds['from']), date.fromisoformat(bounds['to']) - dates = [(first + timedelta(days=i)).isoformat() for i in range((last - first).days + 1)] - answer = {'projectId': project['id'], 'from': bounds['from'], 'to': bounds['to'], - 'timezone': timezone_name(), 'boundary': SERIES_BOUNDARY, - 'markers': {'start': project.get('start_on'), 'target': project.get('target_on'), - 'completed': project.get('completed_on')}, 'metrics': []} - for key, label, unit in SERIES_METRICS: - measured = read(key) - total, reason = measured.get('total'), measured.get('reason') - if total is None and not reason: - reason = 'An independent daily source for this metric is unavailable.' - values = {point['date']: point['value'] for point in measured.get('points', [])} if total is not None else {} - answer['metrics'].append({'key': key, 'label': label, 'unit': measured.get('unit', unit), - 'points': [{'date': day, 'value': values.get(day)} for day in dates], - 'total': total, 'reason': reason, - 'tie': {'ok': False, 'ours': None, 'independent': None}}) - checks = validate(answer, independent=independent) - if any(not check['ok'] for check in checks): - raise ValidationFailed('A series total does not tie to its independent Amazon economics aggregate.') - for metric, check in zip(answer['metrics'], checks): - if metric['total'] is not None: - metric['tie'] = {'ok': True, 'ours': check['ours'], 'independent': check['independent']} - return answer def configurations(values): diff --git a/platform/core/amazon_projects.py b/platform/core/amazon_projects.py index 6004b0298213b0752ee8b2d9df37dd7e33b4d512..ba62b9bb31ae7eab8c5c749d9e3e49f40beaed3c 100644 --- a/platform/core/amazon_projects.py +++ b/platform/core/amazon_projects.py @@ -250,11 +250,10 @@ def health(project, today): def summary(projects): - """Average progress with Completed at 100 and Canceled excluded.""" + """R19 and R67: average progress and "N of M completed", with Canceled left out of M.""" counted = [p for p in projects if p.get('status') != CANCELED] total = len(counted) - return {'average': round(sum(100 if p.get('status') == COMPLETED else p['progress'] - for p in counted) / total) if total else None, + return {'average': round(sum(p['progress'] for p in counted) / total) if total else None, 'completed': sum(1 for p in counted if p.get('status') == COMPLETED), 'total': total} diff --git a/platform/core/connector_sync.py b/platform/core/connector_sync.py index 2cb3c1e6ef9235b150f1560bf79abe33586b38a3..5c83fb151e9f1f0f273b83b24407d14488d19b06 100644 --- a/platform/core/connector_sync.py +++ b/platform/core/connector_sync.py @@ -162,23 +162,6 @@ def sync_not_before(provider, tenant): return _SYNC_NOT_BEFORE.get((provider, tenant)) -def _clear_sync_defer(provider, tenant): - """Remove one obsolete provider hold under the desk lock, retaining explicit continuation waits.""" - key = (provider, tenant) - held = _SYNC_NOT_BEFORE.pop(key, None) - receipt = _SYNC_REQUESTS.get(key) - if (held is not None and receipt and receipt["state"] == "queued" - and receipt.get("requestedBy") != "first load" and receipt.get("notBefore") == held): - receipt["notBefore"] = None - - -def clear_sync_defer(provider, tenant): - """Release one provider hold after its producer adopts partition-specific waits.""" - with _SYNC_REQUEST_LOCK: - _clear_sync_defer(provider, tenant) - _wake_sync_requests(time.time()) - - def register_request_listener(fn): """Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock.""" if fn not in _REQUEST_LISTENERS: @@ -238,13 +221,7 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non _wake_sync_requests(stamp) return {**result, "accepted": accepted_id == identity, "duplicate": accepted_id != identity} with _SYNC_REQUEST_LOCK: - if provider == "amazon_sp" and requested_by != "first load": - # Amazon's history floor belongs to owed tables, never a loaded-table refresh. - _clear_sync_defer(provider, tenant) previous = _SYNC_REQUESTS.get((provider, tenant)) - replaces_amazon_history = (provider == "amazon_sp" and previous - and previous.get("requestedBy") == "first load" - and requested_by != "first load") if previous and previous["state"] in ("queued", "running"): if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load" and requested_by != "first load"): @@ -253,7 +230,7 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non "requestedAt": time.time() if now is None else now, "notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0), float(previous.get("notBefore") or 0) if previous - and previous["state"] == "queued" and not replaces_amazon_history else 0) or None, + and previous["state"] == "queued" else 0) or None, "startedAt": None, "finishedAt": None, "detail": "", "requestedBy": str(requested_by or ""), "accepted": True} if requested_by == "first load" and previous and previous.get("firstLoad"): @@ -494,9 +471,6 @@ def poll_remote_requests(rt, *, now=None): if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS: pruned.append(identity) continue - manual_amazon = key[0] == "amazon_sp" and entry.get("requestedBy") != "first load" - if manual_amazon: - _clear_sync_defer(*key) receipt = _SYNC_REQUESTS.get(key) # Promote under the same lock as claims, so a continuation cannot win a manual hand-off race. if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load" @@ -504,7 +478,7 @@ def poll_remote_requests(rt, *, now=None): receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "", requestedAt=entry["requestedAt"], connectionId=entry.get("connectionId"), - notBefore=max(0 if manual_amazon else float(receipt.get("notBefore") or 0), + notBefore=max(float(receipt.get("notBefore") or 0), float(entry.get("notBefore") or 0), _SYNC_NOT_BEFORE.get(key, 0)) or None) admitted.append(deepcopy(entry)) diff --git a/platform/core/external_access.py b/platform/core/external_access.py index 686f038fd3f02fd86f9c53c4f295342494d67a7c..66f143a4ca730a8014d8c6e4314be3a1dffb443c 100644 --- a/platform/core/external_access.py +++ b/platform/core/external_access.py @@ -11,8 +11,6 @@ from dataclasses import dataclass from datetime import datetime, timezone import hashlib import hmac -import logging -import os import secrets import time @@ -20,7 +18,6 @@ import core.users as users from harness import runtime -_log = logging.getLogger(__name__) BUCKET = "external_access" ACCESS_READ = "read" ACCESS_READWRITE = "readwrite" @@ -60,10 +57,6 @@ class ExternalAccessError(RuntimeError): class ExternalAuthError(ExternalAccessError): """A uniform authentication refusal that reveals no credential or tenant detail.""" - def __init__(self, code: str, *, cause: str = "unknown_key"): - super().__init__(code) - self.cause = cause - def normalise_access(value, *, allow_refresh=False) -> str: if value is None or value == "": @@ -146,8 +139,6 @@ class VerifiedCredential: access: str = ACCESS_READWRITE databases: object = ALL_DATABASES workbooks: tuple = () - warning: str | None = None - origin_recorded: bool = False def _now() -> str: @@ -338,7 +329,6 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None "rotated_at": None, "expires_at": expiry, "test_connection": test_connection, - "minted_space": os.environ.get("SPACE_ID") or None, } book[key_id] = record minted["record"] = record @@ -434,7 +424,6 @@ def rotate(st, username: str, tenant: str, key_id: str) -> tuple[dict, str]: record.update({ "salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM, "rotated_at": _now(), "last_used": None, - "minted_space": os.environ.get("SPACE_ID") or None, }) book[target] = record rotated["record"] = record @@ -525,63 +514,31 @@ def _comparison_material(record) -> tuple[bool, bytes, str]: return False, _DUMMY_SALT, _DUMMY_HASH -def _recorded_space(record: dict) -> str | None: - """Treat missing or malformed origin metadata as unproved.""" - value = record.get("minted_space") - return value if isinstance(value, str) and value.strip() else None - - -def _credential_cause(record, tenant: str, key_id: str, secret: str, *, owner=None): - """Inspect status only after the secret and optional session owner match the stored record.""" - material_ok, salt, expected = _comparison_material(record) - matched = hmac.compare_digest(_digest(salt, secret), expected) - if (not material_ok or not matched or record.get("id") != key_id - or record.get("tenant") != tenant or not record.get("username") - or (owner is not None and record.get("username") != owner)): - return "unknown_key" - origin, current = _recorded_space(record), os.environ.get("SPACE_ID") - if origin and current and origin != current: - return "minted_elsewhere" - if record.get("revoked_at"): - return "revoked" - if record.get("expires_at"): - expiry = _epoch(record["expires_at"]) - if expiry is None or expiry <= time.time(): - return "expired" - return None - - -def verify_for_owner(st, username: str, tenant: str, token: str) -> dict: - """Diagnose only a session owner's secret inside that session's tenant runtime.""" - owner, tenant_key = _normalise_owner(username, tenant) - _assert_bound_runtime(st, tenant_key) - try: - verified = verify(token, owner_runtime=st, owner=owner) - except ExternalAuthError as exc: - return {"valid": False, "cause": exc.cause, "warning": None, "origin_recorded": None} - return {"valid": True, "cause": None, "warning": verified.warning, - "origin_recorded": verified.origin_recorded, "key_id": verified.key_id} - - -def verify(token: str, *, owner_runtime=None, owner=None) -> VerifiedCredential: +def verify(token: str) -> VerifiedCredential: """Verify a bearer token from fresh state and resolve its ordinary public-user principal.""" try: tenant_key, key_id, secret = _parse_token(token) - if owner_runtime is not None: - if not owner or getattr(owner_runtime, "key", None) != tenant_key: - raise ExternalAuthError(AUTH_ERROR) - st = owner_runtime - else: - try: - st = runtime.get_runtime(tenant_key) - except KeyError as exc: - raise ExternalAuthError(AUTH_ERROR) from exc + try: + st = runtime.get_runtime(tenant_key) + except KeyError as exc: + raise ExternalAuthError(AUTH_ERROR) from exc book = _book(st) record = book.get(key_id) - cause = _credential_cause(record, tenant_key, key_id, secret, owner=owner) - if cause: - raise ExternalAuthError(AUTH_ERROR, cause=cause) + material_ok, salt, expected = _comparison_material(record) + matched = hmac.compare_digest(_digest(salt, secret), expected) + record_ok = ( + material_ok + and isinstance(record, dict) + and record.get("id") == key_id + and record.get("tenant") == tenant_key + and bool(record.get("username")) + and not record.get("revoked_at") + and (_epoch(record.get("expires_at")) is None + if not record.get("expires_at") else _epoch(record.get("expires_at")) > time.time()) + ) + if not record_ok or not matched: + raise ExternalAuthError(AUTH_ERROR) owner = str(record["username"]).strip().lower() try: @@ -592,8 +549,7 @@ def verify(token: str, *, owner_runtime=None, owner=None) -> VerifiedCredential: access, databases = scope_of(record) if not _stamp_is_due(record.get("last_used"), time.time()): return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st, - access=access, databases=databases, workbooks=workbooks_of(record), - origin_recorded=bool(_recorded_space(record))) + access=access, databases=databases, workbooks=workbooks_of(record)) used_at = _now() @@ -601,32 +557,26 @@ def verify(token: str, *, owner_runtime=None, owner=None) -> VerifiedCredential: if not isinstance(value, dict): raise ExternalAuthError(AUTH_ERROR) current = value.get(key_id) - cause = _credential_cause(current, tenant_key, key_id, secret, owner=owner) - if cause: - raise ExternalAuthError(AUTH_ERROR, cause=cause) + if (not isinstance(current, dict) or current.get("revoked_at") + or current.get("username") != owner + or current.get("tenant") != tenant_key + or current.get("hash") != record.get("hash")): + raise ExternalAuthError(AUTH_ERROR) book_now = dict(value) stamped = dict(current) stamped["last_used"] = used_at book_now[key_id] = stamped return book_now - warning = None try: st.update(BUCKET, _stamp, flush="sync") except ExternalAuthError: raise except Exception as exc: - # Store diagnostics must not include credential material or exception text. - _log.warning("external_access last_used stamp failed: %s", type(exc).__name__) - warning = "store_write_refused" + raise ExternalAuthError(AUTH_ERROR) from exc return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st, - access=access, databases=databases, workbooks=workbooks_of(record), - warning=warning, origin_recorded=bool(_recorded_space(record))) + access=access, databases=databases, workbooks=workbooks_of(record)) except ExternalAuthError: raise - except ExternalAccessError as exc: - if owner_runtime is not None: - raise - raise ExternalAuthError(AUTH_ERROR) from exc except Exception as exc: raise ExternalAuthError(AUTH_ERROR) from exc diff --git a/platform/core/notify_destinations.py b/platform/core/notify_destinations.py index 447e90b8df02ddfdea40d0673a6cf27e48461680..298b2386517fd22d405531fd23e584cdb215c42d 100644 --- a/platform/core/notify_destinations.py +++ b/platform/core/notify_destinations.py @@ -3,7 +3,6 @@ from contextlib import contextmanager from datetime import datetime, timezone import hashlib -import ipaddress import json import re from typing import NotRequired, TypedDict @@ -24,7 +23,6 @@ PAYLOAD_VERSION = 1 PAGE_SIZE = 100 _CHANNEL = re.compile(r'[CG][A-Z0-9]{1,79}\Z') _EMAIL = re.compile(r'[^\s@,;<>]+@[^\s@,;<>]+\.[^\s@,;<>]+\Z') -_WEBHOOK = re.compile(r'k_[0-9a-f]{12}\Z') class NotificationSource(TypedDict): @@ -75,9 +73,9 @@ def email_address(value): def clean_patch(patch, *, replace=False): - if not isinstance(patch, dict) or set(patch) - {'slackChannels', 'email', 'inApp', 'webhooks'}: - raise ValueError('Only slackChannels, email, inApp and webhooks may be changed.') - out = {'slackChannels': [], 'email': None, 'inApp': False, 'webhooks': []} if replace else {} + if not isinstance(patch, dict) or set(patch) - {'slackChannels', 'email'}: + raise ValueError('Only slackChannels and email may be changed.') + out = {'slackChannels': [], 'email': None} if replace else {} if 'slackChannels' in patch: channels = patch['slackChannels'] if (not isinstance(channels, list) @@ -86,49 +84,9 @@ def clean_patch(patch, *, replace=False): out['slackChannels'] = list(dict.fromkeys(channels)) if 'email' in patch: out['email'] = None if patch['email'] is None else email_address(patch['email']) - if 'inApp' in patch: - if type(patch['inApp']) is not bool: - raise ValueError('Use a boolean in-app preference.') - out['inApp'] = patch['inApp'] - if 'webhooks' in patch: - hooks = patch['webhooks'] - if (not isinstance(hooks, list) - or any(not isinstance(hook, str) or not _WEBHOOK.fullmatch(hook) for hook in hooks)): - raise ValueError('Use a list of saved webhook ids.') - out['webhooks'] = list(dict.fromkeys(hooks)) return out -def webhook_fields(rt, user, identity): - """Resolve an owner-bound saved alert webhook without accepting inline credentials.""" - from core import keychain - _backend(rt) - user = _text(user, 'User') - if not isinstance(identity, str) or not _WEBHOOK.fullmatch(identity): - raise ValueError('Choose a saved webhook id.') - entry = next((entry for entry in keychain.list_entries(rt, 'generic') - if entry['id'] == identity and entry.get('createdBy') == user), None) - if entry is None: - raise ValueError('Choose one of your configured webhooks.') - fields = keychain.read_fields(rt, identity) - if not isinstance(fields, dict) or fields.get('purpose') != 'alert_webhook': - raise ValueError('Choose a saved alert webhook.') - url = _text(fields.get('url'), 'Webhook URL', 2000) - parsed = urlsplit(url) - host = (parsed.hostname or '').rstrip('.').lower() - if (parsed.scheme != 'https' or not host or parsed.username or parsed.password - or parsed.fragment or parsed.port not in (None, 443) - or '.' not in host or host.endswith(('.localhost', '.local', '.internal'))): - raise ValueError('The webhook requires a public HTTPS URL.') - try: - address = ipaddress.ip_address(host) - except ValueError: - address = None - if address is not None and not address.is_global: - raise ValueError('The webhook requires a public HTTPS URL.') - return fields - - def clean_items(items): if not isinstance(items, list) or not items: raise ValueError('Supply at least one notification item.') @@ -191,8 +149,7 @@ def seed_definition(tenant, user, kind, channel): def empty_row(tenant, user, kind): return {'tenant': tenant, 'user': user, 'kind': kind, 'slackChannels': [], - 'email': None, 'inApp': False, 'webhooks': [], - 'revision': 0, 'delivery': {}, 'slackAutomations': {}} + 'email': None, 'revision': 0, 'delivery': {}, 'slackAutomations': {}} def _backend(rt): @@ -222,14 +179,7 @@ def preferences(rt, user): def edit(rt, user, kind, patch, *, replace=False): - backend, user, kind = _backend(rt), _text(user, 'User'), kind_id(kind) - patch = clean_patch(patch, replace=replace) - prior = backend.read(user, kind) or empty_row(rt.key, user, kind) - for identity in patch.get('webhooks', []): - # Existing picks remain removable while their configured destination is unavailable. - if identity not in prior.get('webhooks', []): - webhook_fields(rt, user, identity) - return backend.edit(user, kind, patch) + return _backend(rt).edit(_text(user, 'User'), kind_id(kind), clean_patch(patch, replace=replace)) def recipients(rt, kind, *, user=None): @@ -249,14 +199,8 @@ def recipients(rt, kind, *, user=None): def selected(row, destination): - if not isinstance(destination, str): - return False if destination.startswith('slack:'): return destination[6:] in row['slackChannels'] - if destination.startswith('in-app:'): - return row.get('inApp') is True and destination[7:] == row['user'] - if destination.startswith('webhook:'): - return destination[8:] in row.get('webhooks', []) return destination.startswith('email:') and destination[6:] == row['email'] @@ -288,7 +232,6 @@ class PgRows: raise ValueError('Invalid tenant schema.') self.table = f'"{schema}"."notification_destinations"' self.seeds = f'"{schema}"."notification_slack_automations"' - self.picks = f'"{schema}"."notification_channel_preferences"' @contextmanager def _connection(self, *, write=False): @@ -310,7 +253,7 @@ class PgRows: return con.execute('SELECT to_regclass(%s)', (self.table,)).fetchone()[0] is not None def _ensure(self, con): - if self._exists(con) and self._picks_exist(con): + if self._exists(con): return con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} ( @@ -323,42 +266,23 @@ class PgRows: id TEXT PRIMARY KEY, user_id TEXT NOT NULL, kind TEXT NOT NULL, channel TEXT NOT NULL, FOREIGN KEY (user_id, kind) REFERENCES {self.table} (user_id, kind) ON DELETE CASCADE, UNIQUE (user_id, kind, channel))''') - con.execute(f'''CREATE TABLE IF NOT EXISTS {self.picks} ( - user_id TEXT NOT NULL, kind TEXT NOT NULL, in_app BOOLEAN NOT NULL DEFAULT FALSE, - webhooks JSONB NOT NULL DEFAULT '[]', PRIMARY KEY (user_id, kind), - FOREIGN KEY (user_id, kind) REFERENCES {self.table} (user_id, kind) ON DELETE CASCADE)''') - - def _picks_exist(self, con): - return con.execute('SELECT to_regclass(%s)', (self.picks,)).fetchone()[0] is not None - - def _projection(self, con): - base = f'user_id, kind, slack_channels, email, revision, delivery FROM {self.table}' - if not self._picks_exist(con): - return 'SELECT ' + base, '' - return (f'SELECT n.user_id, n.kind, n.slack_channels, n.email, n.revision, n.delivery, ' - f'COALESCE(p.in_app, FALSE), COALESCE(p.webhooks, \'[]\') FROM {self.table} n ' - f'LEFT JOIN {self.picks} p ON p.user_id = n.user_id AND p.kind = n.kind', 'n.') def _row(self, values): if values is None: return None - user, kind, channels, email, revision, delivery = values[:6] - in_app, webhooks = values[6:] if len(values) == 8 else (False, []) - webhooks = json.loads(webhooks) if isinstance(webhooks, str) else webhooks + user, kind, channels, email, revision, delivery = values channels = json.loads(channels) if isinstance(channels, str) else channels delivery = json.loads(delivery) if isinstance(delivery, str) else delivery row = {**empty_row(self.tenant_slug, user, kind), 'slackChannels': channels, - 'email': email, 'inApp': bool(in_app), 'webhooks': webhooks, 'revision': revision} + 'email': email, 'revision': revision} row['delivery'] = {key: state for key, state in delivery.items() if selected(row, key)} row['slackAutomations'] = {c: automation_id(self.tenant_slug, user, kind, c) for c in channels} return row def _read(self, con, user, kind, *, lock=False): - projection, prefix = self._projection(con) return self._row(con.execute( - f'{projection} WHERE {prefix}user_id = %s AND {prefix}kind = %s' - + (' FOR UPDATE OF n' if lock and prefix else ' FOR UPDATE' if lock else ''), - (user, kind)).fetchone()) + f'SELECT user_id, kind, slack_channels, email, revision, delivery FROM {self.table} ' + 'WHERE user_id = %s AND kind = %s' + (' FOR UPDATE' if lock else ''), (user, kind)).fetchone()) def read(self, user, kind): with self._connection() as con: @@ -368,9 +292,9 @@ class PgRows: with self._connection() as con: if not self._exists(con): return [] - projection, prefix = self._projection(con) - rows = con.execute(f'{projection} WHERE {prefix}kind = %s AND {prefix}user_id > %s ' - f'ORDER BY {prefix}user_id LIMIT %s', (kind, after, limit)).fetchall() + rows = con.execute(f'SELECT user_id, kind, slack_channels, email, revision, delivery ' + f'FROM {self.table} WHERE kind = %s AND user_id > %s ' + 'ORDER BY user_id LIMIT %s', (kind, after, limit)).fetchall() return [self._row(row) for row in rows] def edit(self, user, kind, patch): @@ -383,9 +307,6 @@ class PgRows: con.execute(f'UPDATE {self.table} SET slack_channels = %s::jsonb, email = %s, ' 'revision = revision + 1, delivery = %s::jsonb WHERE user_id = %s AND kind = %s', (json.dumps(current['slackChannels']), current['email'], json.dumps(kept), user, kind)) - con.execute(f'INSERT INTO {self.picks} (user_id, kind, in_app, webhooks) VALUES (%s, %s, %s, %s::jsonb) ' - 'ON CONFLICT (user_id, kind) DO UPDATE SET in_app = EXCLUDED.in_app, webhooks = EXCLUDED.webhooks', - (user, kind, current['inApp'], json.dumps(current['webhooks']))) con.execute(f'DELETE FROM {self.seeds} WHERE user_id = %s AND kind = %s ' 'AND NOT (channel = ANY(%s::text[]))', (user, kind, current['slackChannels'])) for channel in current['slackChannels']: diff --git a/platform/core/sheets.py b/platform/core/sheets.py index 3254a752f7e8b947db193635547826c12d0775fb..322911e11628afe6d88f459738fc28a59325bdc6 100644 --- a/platform/core/sheets.py +++ b/platform/core/sheets.py @@ -2477,10 +2477,9 @@ def _publish_cell_activity(st, events): for event in events: try: activity_log.append(st, **event) - except Exception as exc: + except Exception: # A notification failure must not present a committed save as a failed edit. - logging.getLogger(__name__).error("Sheet activity notification failed after persistence (%s)", - type(exc).__name__) + logging.getLogger(__name__).error("Sheet activity notification failed after persistence") def _read_workbook_doc(st, key): diff --git a/platform/core/table_store.py b/platform/core/table_store.py index 6073dc3464a6e7415032fb23ab2ad80c342c8de8..34cb57e247a4acb4623de89ca50bb0d10bad9800 100644 --- a/platform/core/table_store.py +++ b/platform/core/table_store.py @@ -105,9 +105,10 @@ TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60 # browser re-reads only after some writes. So "stamp differs" alone refuses one tab's own second # write, and the outbox then resends that refused write forever. # -# Each view write records its author, landed revision and the member digests from before it. -# The next before-image isolates each write's changes. Known requester writes are skipped. -# A foreign or unknown writer's changed member stays protected when this write would replace it. +# The record below tells the two apart. Each view write appends the view's member digests as +# they stood BEFORE it, tagged with the revision it landed as. For a stamp `R`, the earliest +# entry after `R` is the view as that sender saw it. A write is refused only when a member +# changed after `R` and this write would set it to something else. # # ⛔ The record lives in the owner's own stratum beside `views`, so it rides the same update and # the same settings row. `workspace()` never projects it. @@ -120,7 +121,7 @@ TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60 VIEW_WRITES_KEY = 'viewWrites' # Folder ids are surface-scoped and keep history separate from view ids. FOLDER_WRITES_KEY = 'folderWrites' -#: Writes kept per view. Older senders need a matching trimmed-author boundary. +#: Writes kept per view. A sender older than the oldest kept write cannot be judged and is refused. VIEW_WRITES_KEEP = 8 #: Views kept per stratum. The least recently written record goes first. VIEW_WRITES_VIEWS = 400 @@ -149,7 +150,7 @@ def view_members(view): return out -def _note_write(ws, view_id, at, kind, before, book_key, *, author=None): +def _note_write(ws, view_id, at, kind, before, book_key): """Append one write to the selected bounded history in this owner's stratum.""" if at is None or not view_id or not isinstance(ws, dict): return @@ -158,21 +159,11 @@ def _note_write(ws, view_id, at, kind, before, book_key, *, author=None): book = ws[book_key] = {} entry = book.get(view_id) if isinstance(book.get(view_id), dict) else {} log = [row for row in entry.get('log') or () if isinstance(row, dict)] - author = str(author or '').strip().lower() or None - log.append({'rev': int(at), 'kind': kind, 'before': before, 'author': author}) + log.append({'rev': int(at), 'kind': kind, 'before': before}) since = int(entry.get('since') or 0) - trimmed = entry.get('trimmed') - trimmed = dict(trimmed) if isinstance(trimmed, dict) else {} while len(log) > VIEW_WRITES_KEEP: - removed = log.pop(0) - removed_author = removed.get('author') - # Only a known, uninterrupted author run can excuse writes outside the ring. - if not removed_author or removed_author != trimmed.get('author'): - trimmed = {'author': removed_author, 'after': since} - since = max(since, int(removed.get('rev') or 0)) + since = max(since, int(log.pop(0).get('rev') or 0)) book[view_id] = {'since': since, 'log': log} - if trimmed: - book[view_id]['trimmed'] = trimmed if len(book) > VIEW_WRITES_VIEWS: # jsonb keeps no key order, so recency is read off each record's newest revision. def newest(vid): @@ -183,9 +174,9 @@ def _note_write(ws, view_id, at, kind, before, book_key, *, author=None): book.pop(vid, None) -def note_view_write(ws, view_id, at, kind, before, *, author=None): +def note_view_write(ws, view_id, at, kind, before): """Append one view, gone or place write to this stratum's record.""" - _note_write(ws, view_id, at, kind, before, VIEW_WRITES_KEY, author=author) + _note_write(ws, view_id, at, kind, before, VIEW_WRITES_KEY) def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY): @@ -200,31 +191,25 @@ def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY): return later, int(sent) >= int(entry.get('since') or 0) -def stale_view_conflict(ws, view_id, mine, sent, *, requester=None): +def stale_view_conflict(ws, view_id, mine, sent): """Whether saving `mine` over `view_id` undoes a change made after revision `sent`.""" later, judgeable = _later_writes(ws, view_id, sent, ('view', 'gone')) - requester = str(requester or '').strip().lower() or None + if not later: + return False if not judgeable: - entry = (ws.get(VIEW_WRITES_KEY) or {}).get(view_id) or {} - trimmed = entry.get('trimmed') - trimmed = trimmed if isinstance(trimmed, dict) else {} - if not (requester and trimmed.get('author') == requester - and int(sent) >= int(trimmed.get('after', entry.get('since') or 0))): - return True + return True + if later[0].get('kind') == 'view' and later[0].get('before') is None: + # Created after `sent`, so the sender is the tab that created it. Judge only later writes. + later = later[1:] + if not later: + return False current = ((ws.get('views') or {}).get(view_id)) - now, want = view_members(current) or {}, view_members(mine) or {} - for index, row in enumerate(later): - if requester and row.get('author') == requester: - continue - if not isinstance(current, dict): - return True - before = row.get('before') or {} - after = (later[index + 1].get('before') or {}) if index + 1 < len(later) else now - # The next before-image isolates this writer's changes from the requester's later edits. - if any(before.get(member) != after.get(member) and want.get(member) != now.get(member) - for member in set(before) | set(after)): - return True - return False + if not isinstance(current, dict): + return True + base = later[0].get('before') or {} + now, want = view_members(current), view_members(mine) or {} + return any(now.get(member) != base.get(member) and want.get(member) != now.get(member) + for member in set(base) | set(now)) def stale_place_conflict(ws, view_id, folder, sent): @@ -1406,15 +1391,14 @@ class TableStore: for folder_id in sorted(set(before_parents) | set(parents)): if before_parents.get(folder_id) != parents.get(folder_id): _note_write(ws, folder_id, landing.get('at'), 'parent', - {'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY, - author=actor or username) + {'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY) ws['folders'] = tree ws['itemFolders'] = placed now = placed.get('views') or {} for view_id in sorted(set(was) | {str(k) for k in now}): if was.get(view_id) != now.get(view_id): note_view_write(ws, view_id, landing.get('at'), 'place', - {'folderId': was.get(view_id)}, author=actor or username) + {'folderId': was.get(view_id)}) def _audit(ws, record): now = (ws.get('itemFolders') or {}).get('views') or {} @@ -2010,8 +1994,7 @@ class TableStore: prior = (ws.get('views') or {}).get(view_id) activity_before['value'] = view_audit.summary(prior) ws.setdefault('views', {})[view_id] = payload - note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior), - author=actor or username) + note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior)) audit(action=action or view_audit.derive_action(prior, payload), owner=username, view_id=view_id, actor=actor, origin=origin, before=view_audit.summary(prior), after=view_audit.summary(payload)) @@ -2027,8 +2010,7 @@ class TableStore: if rev is None or not rebase: raise self._land_stale(username, rev, stale, - lambda ws, sent: stale_view_conflict( - ws, view_id, requested, sent, requester=actor or username), + lambda ws, sent: stale_view_conflict(ws, view_id, requested, sent), _write) self.log_view_activity(view_id, 'saved', by=actor or username, before=activity_before.get('value'), @@ -2073,8 +2055,7 @@ class TableStore: removed.clear() if isinstance(before, dict): removed['value'] = view_audit.summary(before) - note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before), - author=who) + note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before)) views.pop(vid, None) audit(action='delete' if isinstance(before, dict) else None, owner=owner, view_id=vid, actor=actor, origin=origin, diff --git a/platform/core/test_accounts.py b/platform/core/test_accounts.py new file mode 100644 index 0000000000000000000000000000000000000000..883092499cb042bbd162ff4e3470053c30b70826 --- /dev/null +++ b/platform/core/test_accounts.py @@ -0,0 +1,398 @@ +"""Test accounts: an exact copy of one member's permissions that takes no seat and sends nothing. + +S79-T63 (QI34, rulings R28 and R42). The copy carries the source's role, business units, modules, +agent, permission block (row filters and field walls stay locks, rule 3) and every share entry +that names the source. It never carries a password, MFA, email, WhatsApp, avatar or epoch. +Creation is all or nothing: a failure removes every part already written. +""" +import copy +import datetime as _dt +import re + +import core.shares as shares +import core.store as store +import core.users as users + +PREFIX = 'test-' +#: `routes_admin._UNAME_RE` admits 2 to 32 characters; a generated name stays inside it. +_NAME_MAX = 32 +_RETRIES = 3 + +#: The record fields a copy takes verbatim from its source. +COPIED_FIELDS = ('role', 'bus', 'modules', 'agent', 'perms', 'perms_v') + + +class CopyRefused(ValueError): + """The request names a source or target this door does not copy or remove.""" + + def __init__(self, code, message, status=400): + self.code, self.status = code, status + super().__init__(message) + + +class CopyFailed(RuntimeError): + """A copy or removal stopped part way. `leftovers` names what cleanup could not remove.""" + + def __init__(self, cause, leftovers=()): + self.cause, self.leftovers = cause, list(leftovers) + super().__init__(cause) + + +def is_test(record): + """Whether this stored record is a copied test account.""" + return users.is_test_account(record) + + +def _tenant_of(record): + return str((record or {}).get('tenant') or 'royal-imports').strip().lower() + + +def paid_seats(registry, tenant): + """Active accounts in `tenant` that take a paid seat. A test account takes none (R42).""" + want = str(tenant or '').strip().lower() + return sum(1 for record in (registry or {}).values() + if isinstance(record, dict) and record.get('active', True) + and _tenant_of(record) == want and not is_test(record)) + + +def test_username(source, registry): + """`test-` cut to fit the username rule, with a numeric suffix on a collision.""" + base = re.sub(r'[^a-z0-9._-]+', '-', str(source or '').strip().lower()).strip('.-_') + stem = (PREFIX + base)[:_NAME_MAX].rstrip('.-_') + taken = set(registry or {}) + if stem not in taken: + return stem + for n in range(2, 1000): + suffix = f'-{n}' + candidate = stem[:_NAME_MAX - len(suffix)].rstrip('.-_') + suffix + if candidate not in taken: + return candidate + raise CopyRefused('no_free_username', 'No free name is left for another test account.', 409) + + +class _Doc: + """A read-only store over one already-read shares document.""" + + def __init__(self, doc): + self._doc = doc if isinstance(doc, dict) else {} + + def get(self, name, *args, **kwargs): + return self._doc if name == shares.SHARES_KEY else None + + +class _Batch: + """Stage `shares.set_grants` writes in memory, then replay them all in ONE store update. + + Every staged closure keeps its `expect` check, so the replay refuses if any record changed. + """ + + def __init__(self, st, doc): + self._st, self._doc, self._fns = st, copy.deepcopy(doc or {}), [] + + def get(self, name, *args, **kwargs): + if name != shares.SHARES_KEY: + raise RuntimeError(f'a staged grant write cannot read {name!r}') + return self._doc + + def update(self, name, fn, flush='sync'): + if name != shares.SHARES_KEY: + raise RuntimeError(f'a staged grant write cannot write {name!r}') + self._doc = fn(self._doc) + self._fns.append(fn) + return self._doc + + def commit(self): + if not self._fns: + return + fns = list(self._fns) + + def replay(data): + data = data if isinstance(data, dict) else {} + for fn in fns: + data = fn(data) + return data + + self._st.update(shares.SHARES_KEY, replay, flush='sync') + + +def _read_shares(st): + """The tenant's shares document, read strictly: an unreadable bucket stops the copy.""" + doc = st.get(shares.SHARES_KEY, fresh=True) + if doc is None: + return {} + if not isinstance(doc, dict): + raise RuntimeError('the sharing records could not be read') + return doc + + +def grants_naming(username, doc): + """`[{kind, oid, role}]` for every share entry that names `username`, from one document.""" + username = str(username or '').strip().lower() + found = [] + view = _Doc(doc) + for kind in shares.KINDS: + for oid, record in sorted(shares.grants_bulk(kind, st=view).items()): + for entry in record['entries']: + if entry['user'] == username: + found.append({'kind': kind, 'oid': oid, 'role': entry['role']}) + return found + + +def _check_source(source, registry, tenant): + uname = str(source or '').strip().lower() + record = (registry or {}).get(uname) + if not uname or not isinstance(record, dict) or _tenant_of(record) != tenant: + raise CopyRefused('no_such_user', f'No account named {uname or "(blank)"}.', 404) + if is_test(record): + raise CopyRefused('source_is_test', 'A test account cannot be copied. Copy a member.') + if str(record.get('role') or 'user').strip().lower() != 'user': + raise CopyRefused('source_is_admin', + 'An administrator cannot be copied. Copy a member account.') + if not record.get('active', True): + raise CopyRefused('source_inactive', 'This account is turned off. Turn it on first.') + return uname, record + + +def plan(source, registry, *, tenant, st): + """What a copy of `source` would carry, read now: the access fields and every share entry.""" + tenant = str(tenant or '').strip().lower() + uname, record = _check_source(source, registry, tenant) + access = {key: copy.deepcopy(record[key]) for key in COPIED_FIELDS if key in record} + return {'source': uname, + 'sourceName': str(record.get('name') or uname), + 'username': test_username(uname, registry), + 'name': f'Test copy of {record.get("name") or uname}', + 'access': access, + 'grants': grants_naming(uname, _read_shares(st))} + + +def _password(username, name): + for _ in range(8): + candidate = users.generate_password() + if not users.password_failures(candidate, username=username, name=name): + return candidate + raise RuntimeError('could not generate a password for the test account') + + +def copy_grants(username, grants, *, st, granter): + """Add `username` beside each listed share entry, same role, in one store update.""" + batch = _Batch(st, _read_shares(st)) + for grant in grants: + current = shares.grants(grant['kind'], grant['oid'], st=batch) + if any(entry['user'] == username for entry in current['entries']): + continue + entries = current['entries'] + [{'user': username, 'role': grant['role']}] + shares.set_grants(grant['kind'], grant['oid'], entries, st=batch, + granter=granter, expect=current['entries']) + batch.commit() + + +def remove_grants(username, *, st, granter): + """Remove every share entry naming `username`, retrying if a record changes meanwhile.""" + for attempt in range(_RETRIES): + doc = _read_shares(st) + batch = _Batch(st, doc) + for grant in grants_naming(username, doc): + current = shares.grants(grant['kind'], grant['oid'], st=batch) + entries = [entry for entry in current['entries'] if entry['user'] != username] + shares.set_grants(grant['kind'], grant['oid'], entries, st=batch, + granter=granter, expect=current['entries']) + try: + batch.commit() + return + except shares.GrantsChanged: + if attempt == _RETRIES - 1: + raise + + +def _fresh_registry(): + registry = store.get('users', fresh=True) + if not isinstance(registry, dict): + raise RuntimeError('the account registry could not be read') + return registry + + +def _confirm(username, plan_, marker, st): + record = _fresh_registry().get(username) or {} + if record.get(users.TEST_ACCOUNT_KEY) != marker: + raise RuntimeError('the account was not saved') + if any(record.get(key) != plan_['access'].get(key) for key in plan_['access']): + raise RuntimeError('the copied permissions were not saved') + held = {(g['kind'], g['oid'], g['role']) for g in grants_naming(username, _read_shares(st))} + missing = [g for g in plan_['grants'] if (g['kind'], g['oid'], g['role']) not in held] + if missing: + raise RuntimeError(f'{len(missing)} shared items were not saved') + + +def _cleanup(username, tenant, marker, st, granter, steps): + """Undo every part of a copy. Returns the names of the parts that could not be removed.""" + leftovers = [] + for name, _do, undo in reversed(list(steps)): + try: + undo(username) + except Exception: # noqa: BLE001 + leftovers.append(name) + try: + remove_grants(username, st=st, granter=granter) + except Exception: # noqa: BLE001 + leftovers.append('shared items') + try: + users.delete_test_account(username, tenant=tenant, marker=marker) + left = _fresh_registry().get(username) + if is_test(left) and left.get(users.TEST_ACCOUNT_KEY) == marker: + leftovers.append('sign in') + except Exception: # noqa: BLE001 + leftovers.append('sign in') + return leftovers + + +def _cause(exc): + text = str(exc).strip() or type(exc).__name__ + return text[:1].upper() + text[1:] + + +def create(source, *, admin, tenant, st, steps=(), now=None): + """Copy `source` into a new test account. Returns `{username, password, plan}`. + + `steps` are `(name, do(username, source), undo(username))` for grant stores outside core. + Any failure removes what was written and raises `CopyFailed`. + """ + tenant = str(tenant or '').strip().lower() + admin = str(admin or '').strip().lower() + for attempt in range(_RETRIES): + plan_ = plan(source, _fresh_registry(), tenant=tenant, st=st) + username = plan_['username'] + marker = {'copied_from': plan_['source'], 'created_by': admin, + 'created_at': (now or _dt.datetime.now(_dt.timezone.utc)).isoformat( + timespec='seconds')} + password = _password(username, plan_['name']) + try: + users.create_test_account(username, password, name=plan_['name'], + access=plan_['access'], marker=marker, tenant=tenant) + except users.TestAccountExists: + if attempt == _RETRIES - 1: + raise CopyFailed('Another account took the test account name. Try again.') + continue + except Exception as exc: # noqa: BLE001 + # The write may have landed before it reported failure: clean up as for any step. + leftovers = _cleanup(username, tenant, marker, st, admin, ()) + raise CopyFailed(_cause(exc), leftovers) from exc + try: + copy_grants(username, plan_['grants'], st=st, granter=admin) + for _name, do, _undo in steps: + do(username, plan_['source']) + _confirm(username, plan_, marker, st) + except Exception as exc: # noqa: BLE001 + leftovers = _cleanup(username, tenant, marker, st, admin, steps) + raise CopyFailed(_cause(exc), leftovers) from exc + return {'username': username, 'password': password, 'plan': plan_} + raise CopyFailed('The test account could not be created. Try again.') + + +def remove(username, *, tenant, st, granter, steps=()): + """Delete a test account and every grant naming it. Refuses a real account.""" + tenant = str(tenant or '').strip().lower() + uname = str(username or '').strip().lower() + record = _fresh_registry().get(uname) + if not isinstance(record, dict) or _tenant_of(record) != tenant: + raise CopyRefused('no_such_user', f'No account named {uname or "(blank)"}.', 404) + if not is_test(record): + raise CopyRefused('not_a_test_account', + 'This is a real account. Only a test account can be removed here.') + leftovers = _cleanup(uname, tenant, record.get(users.TEST_ACCOUNT_KEY), st, granter, steps) + if leftovers: + raise CopyFailed('Some parts are still saved', leftovers) + + +# ------------------------------------------------------------------ preview +def _label(key, defs): + import core.perms as perms + import core.registry as registry + definition = (defs or {}).get(key) or {} + # The editor's surface rows (Assistant, Agents, AI apps (MCP)) are named by perms, not the registry. + surfaces = {m['key']: m['label'] for m in perms.governable_surface_modules()} + return str(definition.get('label') or (registry.BY_KEY.get(key) or {}).get('label') + or surfaces.get(key) or key) + + +def _plural(n, one, many): + return f'{n} {one if n == 1 else many}' + + +def preview(plan_, *, st=None, limits=None): + """The contract body for GET .../test-account/preview, built from `plan`. + + `st` is a definitions store (`user_tables.lend_defs`) for the database lock checks. + `limits` is `perm_scope.scope_limits` for the source; it names each row limit, business units included. + """ + from core import user_tables + access = plan_['access'] + try: + defs = user_tables.all_defs(st=st, schema_only=True) or {} + except Exception: # noqa: BLE001 + defs = {} + block = access.get('perms') if isinstance(access.get('perms'), dict) else {} + locks, reached = [], set() + said = {} + for limit in limits or (): + text = str(limit.get('text') or '').strip().rstrip('.') + if limit.get('database') and text: + said[str(limit['database'])] = text + for key in sorted(said): + locks.append({'kind': 'row_scope', 'label': f'{_label(key, defs)}: {said[key]}'}) + for key, entry in sorted(block.items()): + if not isinstance(entry, dict) or not entry.get('access', True): + continue + reached.add(key) + if entry.get('filter') and key not in said: + locks.append({'kind': 'row_scope', 'label': f'Rows are limited in {_label(key, defs)}'}) + hidden = [f for f in entry.get('hiddenFields') or () if f] + if hidden: + locks.append({'kind': 'field_wall', + 'label': f'{_plural(len(hidden), "field is", "fields are")} hidden in ' + f'{_label(key, defs)}'}) + if not access.get('perms_v') and not said: + bus = access.get('bus', 'all') + if bus != 'all' and isinstance(bus, list): + names = ', '.join(users.BU_LABELS.get(b, str(b)) for b in bus) + locks.append({'kind': 'row_scope', 'label': f'Rows are limited to {names}'}) + if access.get('agent'): + locks.append({'kind': 'row_scope', 'label': "Rows are limited to one salesperson's book"}) + for grant in plan_['grants']: + if grant['kind'] == 'cohort': + scope, _cid = shares.split_cohort_oid(grant['oid']) + where = _label(scope, defs) if scope else 'a database' + locks.append({'kind': 'locked_view', 'label': f'A locked view in {where}'}) + if grant['kind'] == 'database': + reached.add(grant['oid']) + for key in sorted(k for k in reached if str(k).startswith(user_tables.KEY_PREFIX)): + definition = defs.get(key) + if not isinstance(definition, dict): + continue + if not user_tables.records_mutable(key, st=st): + locks.append({'kind': 'locked_database', + 'label': f'{_label(key, defs)} is locked for record changes'}) + preset = [f for f in definition.get('fields') or () + if isinstance(f, dict) and isinstance(f.get('automation'), dict) + and f['automation'].get('preset') is True + and not user_tables.preset_editable(f)] + if preset: + locks.append({'kind': 'preset_field', + 'label': f'{_plural(len(preset), "pre-set field", "pre-set fields")} ' + f'in {_label(key, defs)}'}) + grants = [] + for grant in plan_['grants']: + row = dict(grant) + if grant['kind'] == 'database': + row['label'] = _label(grant['oid'], defs) + grants.append(row) + database_access = [{'key': key, 'label': _label(key, defs), + 'access': bool(entry.get('access', True))} + for key, entry in sorted(block.items()) if isinstance(entry, dict)] + return {'source': {'username': plan_['source'], 'name': plan_['sourceName']}, + 'testUsername': plan_['username'], + 'copy': {'role': access.get('role', 'user'), 'bus': access.get('bus', 'all'), + 'buLabels': users.allowed_bus_labels(access), + 'modules': access.get('modules', 'all'), 'agent': access.get('agent'), + 'grants': grants, 'databaseAccess': database_access, 'locks': locks}, + 'counts': {'grants': len(grants), 'locks': len(locks)}} diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 92d11fbd4c11a4bc98ee7458ea6e3ff14b78c20c..c1d61a5a5cc75f348af334bb3a87011f94b4201f 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -529,8 +529,6 @@ def oversize_of(table_key, st=None, defn=None): defn = get_definition(str(table_key or '').strip(), st) or {} except Exception: # noqa: BLE001 defn = {} - if not materialises(table_key, defn=defn): - return None stamp = (defn or {}).get(OVERSIZE_FIELD) return stamp if isinstance(stamp, dict) else None diff --git a/platform/core/users.py b/platform/core/users.py index 74a94c6c3ae7671d13ee5ec164d7f6f965ac56eb..e30968c5bb10535e4bdf0a92d840c1b48197c373 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -1176,6 +1176,8 @@ def set_whatsapp(username, value): u = reg.get(username) if not isinstance(u, dict): raise ValueError('This account is not available.') + # R42: a test account has no outbound channel, so no writer may add one. + refuse_test_account_change(u, whatsapp=value) if value is None: u.pop('whatsapp', None) else: @@ -1198,9 +1200,110 @@ def set_whatsapp_opt_out(username, on): return account_rows.update(username, _set, st=store) +# ------------------------------------------------------------------ test accounts (S79-T63, R42) +#: The record key that marks a copied test account: `{copied_from, created_by, created_at}`. +TEST_ACCOUNT_KEY = 'test_account' + + +class TestAccountRefused(ValueError): + """A writer refused to give a test account an outbound channel or the admin role.""" + + def __init__(self, code, message): + self.code = code + super().__init__(message) + + +class TestAccountExists(ValueError): + """The chosen test username is already a key in the registry.""" + + def is_test_account(record): - """Read the legacy marker until release confirms live has no copied accounts.""" - return isinstance(record, dict) and isinstance(record.get('test_account'), dict) + """Whether this stored record is a copied test account.""" + return isinstance(record, dict) and isinstance(record.get(TEST_ACCOUNT_KEY), dict) + + +def refuse_test_account_change(record, *, email=None, whatsapp=None, role=None): + """Raise `TestAccountRefused` if a change would give a test account email, WhatsApp or admin.""" + if not is_test_account(record): + return + if email: + raise TestAccountRefused('test_account_no_email', + 'A test account cannot have an email address.') + if whatsapp: + raise TestAccountRefused('test_account_no_whatsapp', + 'A test account cannot connect WhatsApp.') + if role is not None and str(role).strip().lower() != 'user': + raise TestAccountRefused('test_account_not_admin', + 'A test account cannot be an administrator.') + + +def create_test_account(username, pw, *, name, access, marker, tenant): + """Write one test account in one update: `access` fields verbatim, no email, no WhatsApp. + + Raises `TestAccountExists` when the key is taken by any record other than this exact write. + """ + import copy + freeze_notice.check_write() + original_username = (username or '').strip() + username = original_username.lower() + failures = password_failures(pw, username=username, name=name) + if failures: + raise PasswordPolicyError(failures) + marker = dict(marker) + salt = secrets.token_hex(16) + digest = _hash(pw, salt) + + def _add(reg): + prior = reg.get(username) + if prior is not None: + # A replayed closure finds its own landed write; anything else is a real collision. + if isinstance(prior, dict) and prior.get(TEST_ACCOUNT_KEY) == marker: + return reg + raise TestAccountExists(username) + rec = {'salt': salt, 'hash': digest, 'name': name, 'role': 'user', + 'password_changed_at': _now_iso(), + 'bus': copy.deepcopy(access.get('bus', 'all')), 'active': True, + 'modules': copy.deepcopy(access.get('modules', 'all')), + 'agent': access.get('agent') or None, 'email': None, + 'tenant': str(tenant or 'royal-imports').strip().lower(), + 'username_original': original_username, TEST_ACCOUNT_KEY: dict(marker)} + # The permission block and its marker travel together or not at all (see `_record`). + for key in ('perms', 'perms_v'): + if key in access: + rec[key] = copy.deepcopy(access[key]) + reg[username] = rec + return reg + + account_rows.update(username, _add, st=store) + + +def delete_test_account(username, *, tenant, marker=None): + """Delete one test account record. Returns deleted, missing or not_test; never a real account. + + `marker`, when given, must equal the stored marker, so a rollback never removes a copy + another administrator made under the same name. + """ + freeze_notice.check_write() + username = (username or '').strip().lower() + tenant = str(tenant or '').strip().lower() + outcome = ['missing'] + + def _del(reg): + outcome[0] = 'missing' + rec = reg.get(username) + if (not isinstance(rec, dict) + or str(rec.get('tenant') or 'royal-imports').strip().lower() != tenant): + return reg + if not is_test_account(rec) or (marker is not None + and rec.get(TEST_ACCOUNT_KEY) != dict(marker)): + outcome[0] = 'not_test' + return reg + reg.pop(username, None) + outcome[0] = 'deleted' + return reg + + account_rows.update(username, _del, st=store) + return outcome[0] #: The keys of one database entry that `set_database_access` writes. @@ -1319,6 +1422,7 @@ def set_access(username, role=None, bus=None, modules=None, agent=None, email=No def _set(reg): u = reg.get(username) if u: + refuse_test_account_change(u, email=email, role=role) if name is not None: u['name'] = name if role is not None: @@ -1361,6 +1465,12 @@ def update_access_with_active_admin_guard(username, role=None, bus=None, modules if not isinstance(current, dict): outcome[0] = 'missing' return reg + try: + refuse_test_account_change(current, email=email, role=role) + except TestAccountRefused as exc: + outcome[0] = exc.code + return reg + # Build the whole candidate before touching reg. On a refused transition the original # record remains byte-for-byte intact, including its epoch. candidate = dict(current) diff --git a/platform/core/windows.py b/platform/core/windows.py index fd7c4a6de5daaa9fb2bea1abec44210728a02693..f65b63bf8191823723d836baf9e6c09c016cc265 100644 --- a/platform/core/windows.py +++ b/platform/core/windows.py @@ -286,10 +286,7 @@ def ending_bounds(kind, mode, value, today, step="month", offset=0, n=0, unit="m """Keep the kind's own start. An ending before it refuses instead of reversing bounds.""" if kind not in TO_DATE_KINDS: return None - ending_today = today - if kind == "ytd_last_year" and mode not in ("fixed", "cell"): - ending_today = resolve({"kind": kind}, today)[1] - end = resolve_ending(mode, value, ending_today, step, offset, n, unit) + end = resolve_ending(mode, value, today, step, offset, n, unit) if end is None: return None start = resolve({"kind": kind}, today)[0] diff --git a/platform/harness/amazon_ads_store.py b/platform/harness/amazon_ads_store.py index f645b4489e68b576641cc973ff6831097cd53180..e29e470f1c5780050b4aa36f330c67f886aad94d 100644 --- a/platform/harness/amazon_ads_store.py +++ b/platform/harness/amazon_ads_store.py @@ -7,8 +7,9 @@ One table per ad type and report, `amzn_ads__`, spawned as `ut ⛔ HISTORY IS KEPT FROM THE CONNECT DAY. The first sync stamps the connect day once and reaches back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_BACKFILL_DAYS`). -A later sync re-reads the table's daily or intraday cadence window and upserts by row identity. -No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here. +A later sync re-reads the last `RESTATEMENT_DAYS`, because Amazon restates attributed sales, and +upserts by row identity. No sync deletes a day an earlier sync stored, so a day Amazon has aged +out stays here. ⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as @@ -34,7 +35,6 @@ if str(_HERE.parent) not in sys.path: sys.path.insert(0, str(_HERE.parent)) from core import connector_spec # noqa: E402 -from harness import amazon_client # noqa: E402 from harness import amazon_ads_client as client # noqa: E402 from harness import amazon_ads_schema as schema # noqa: E402 from harness import amazon_store # noqa: E402 @@ -45,8 +45,8 @@ from harness import datastore # noqa: E ADS_AFTER_SYNC = ("modules.amazon_operator:on_ads_sync",) TABLES = tuple(schema.TABLE_SPECS) -#: Keep the exported default compatible. Requests use each table's own cadence. -RESTATEMENT_DAYS = amazon_client.CADENCE["ads_sp_campaigns"]["lookbackDays"] +#: Amazon restates attributed sales for about 14 days, so each sync re-reads them. +RESTATEMENT_DAYS = 14 CONNECT_DAY_KEY = amazon_store.entity_key("ads_connect_day") BUDGET_KEY = amazon_store.entity_key("ads_report_budget") INSERT_BATCH_ROWS = 500 @@ -95,51 +95,11 @@ def _utc_today(now=None): return now.date() if isinstance(now, datetime) else now -def _utc_now(now=None): - """Normalize datetime and date callers to the same UTC cadence clock.""" - now = now or datetime.now(timezone.utc) - if not isinstance(now, datetime): - now = datetime(now.year, now.month, now.day, tzinfo=timezone.utc) - return now.replace(tzinfo=timezone.utc) if now.tzinfo is None else now.astimezone(timezone.utc) - - def _state_key(key, connection_id=None): """Keep the public table markers and give each connection its own continuation markers.""" return f"{key}::connection::{connection_id}" if connection_id else key -def _cadence_due(con, table, now, connection_id=None): - """Only this connection's completed pass can delay its next cadence interval.""" - cadence = amazon_client.CADENCE["ads_" + table] - tick = amazon_store._read_state(con, _state_key( - amazon_store.entity_key("sync_tick_" + _marker_table(table)), connection_id)) - if not tick or tick.get("state") != "done": - return True - try: - stamp = amazon_store._utc(tick.get("detail") or tick["updated_at"]) - except (ValueError, TypeError, AttributeError, OverflowError): - return True - return now >= stamp + timedelta(days=1) / cadence["perDay"] - - -def _intraday(con, table, today, connection_id=None): - """Retry the daily lookback until every request for this connection succeeds.""" - day = amazon_store._read_state(con, _state_key( - amazon_store.entity_key("lookback_day_" + _marker_table(table)), connection_id)) - return bool(day and day.get("state") == "done" and day.get("detail") == today.isoformat()) - - -def _record_cadence(con, table, now, connection_id=None): - """Persist successful empty and nonempty passes with their daily lookback completion.""" - tick = _state_key(amazon_store.entity_key("sync_tick_" + _marker_table(table)), connection_id) - stamp = now.isoformat() - amazon_store._record_state(con, tick, "done", detail=stamp) - con.execute("UPDATE _sync_state SET updated_at = ? WHERE entity = ?", [stamp, tick]) - amazon_store._record_state(con, _state_key( - amazon_store.entity_key("lookback_day_" + _marker_table(table)), connection_id), - "done", detail=now.date().isoformat()) - - def _connect_day(con, today, connection_id=None, *, legacy=False): """The connect day, stamped once on the first sync and never moved.""" key = _state_key(CONNECT_DAY_KEY, connection_id) @@ -170,10 +130,7 @@ def _day_marker(con, key): def windows(con, table, connect_day, today, connection_id=None): """The report windows one sync asks for, oldest first, each at most 31 days.""" - table = schema.short_name(table) - spec = schema.TABLE_SPECS[table] - cadence = amazon_client.CADENCE["ads_" + table] - lookback = cadence["intradayLookbackDays" if _intraday(con, table, today, connection_id) else "lookbackDays"] + spec = schema.TABLE_SPECS[schema.short_name(table)] last = today - timedelta(days=1) target_floor = max(connect_day, today) - timedelta(days=spec["backfill_days"]) mark_name = _marker_table(table) @@ -185,7 +142,7 @@ def windows(con, table, connect_day, today, connection_id=None): else: if oldest > target_floor: spans.append((target_floor, oldest - timedelta(days=1))) - spans.append((max(target_floor, min(newest, last - timedelta(days=lookback - 1))), last)) + spans.append((max(target_floor, min(newest, last) - timedelta(days=RESTATEMENT_DAYS - 1)), last)) out = [] for start, end in spans: cursor = start @@ -347,7 +304,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None profile_id=None): """Pull every offered Ads report into the tenant's Amazon mirror. -> a report dict. - When this call opens the connection and has a runtime, it saves landed windows and cadence + When this call opens the connection and has a runtime, it saves the tables that landed rows through `amazon_store.persist_mirror`. A caller that lends `con` persists itself. """ key = amazon_store._tenant_key(tenant_key) @@ -366,10 +323,8 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None con, _path = amazon_store._connect_tenant_mirror(key, rt) try: datastore._ensure_sync_state(con) - now = _utc_now(now) today = _utc_today(now) synced = [] - checkpointed = [] for cid, label, profile, legacy in connections: connect_day = _connect_day(con, today, cid, legacy=legacy) report.setdefault("connectDay", connect_day.isoformat()) @@ -384,8 +339,6 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None pulled, state, mark, reason = 0, "done", _marker_table(table), "" if not spec["offered"]: state, reason = "not_offered", spec["not_offered_reason"] - elif not _cadence_due(con, table, now, cid): - state = "skipped" else: for start, end in windows(con, table, connect_day, today, cid): if used >= client.REPORTS_PER_DAY: @@ -416,8 +369,6 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None _row_key=_row_key(cid, row["_row_key"])) stored.append(row) pulled += _upsert(con, table, stored) - if table_name(table) not in checkpointed: - checkpointed.append(table_name(table)) # Only this connection's landed window advances its continuation. for prefix, day, later in ((amazon_store.WATERMARK_PREFIX, end, True), (amazon_store.BACKFILL_FLOOR_PREFIX, start, False)): @@ -426,8 +377,6 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None previous = _day_marker(con, marker) if previous is None or (day > previous if later else day < previous): amazon_store._record_state(con, marker, "done", _count(con, table), day.isoformat()) - if state == "done": - _record_cadence(con, table, now, cid) if reason: _record_limit(con, table, reason, cid, state=state) shown = f"{label}: {reason}" if cid else reason @@ -436,20 +385,17 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None _clear_limit(con, table, cid) total = _count(con, table) entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)} - if state != "skipped": - amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid), - "done" if state == "done" else "partial", _count(con, table, cid), state) + amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid), + "done" if state == "done" else "partial", _count(con, table, cid), state) previous = report["tables"].get(table, {}) - severity = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3} - aggregate = max((state, previous.get("state", state)), key=severity.get) + severity = {"done": 0, "not_offered": 1, "deferred": 2, "failed": 3} + aggregate = max((state, previous.get("state", "done")), key=severity.get) report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled, "in_mirror": total} if (pulled or origin_changed) and table_name(table) not in synced: synced.append(table_name(table)) log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}") for table, state in report["tables"].items(): - if state["state"] == "skipped": - continue amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)), "done" if state["state"] == "done" else "partial", state["in_mirror"], state["state"]) @@ -457,8 +403,8 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None _record_limit(con, table, report["limits"][table], state=state["state"]) else: _clear_limit(con, table) - if own and rt is not None and (synced or checkpointed): - _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log) + if own and rt is not None and synced: + _persist(key, rt, con, synced, report, log) finally: if own: con.close() diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index f66da1a5436131318798110e7f058f8b0732892d..19493bb6e32c8b2b57036e3a9fe1c01eb4d57597 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -416,7 +416,7 @@ REPORT_TYPES = { }, "storage_fees": { "report": "GET_FBA_STORAGE_FEE_CHARGES_DATA", "monthly": True, - "min_interval_s": 86400, + "min_interval_s": 30 * 86400, "validates_against": "Amazon document data-row count vs stored rows for its latest calendar month", "role": "Amazon Fulfillment", }, @@ -458,7 +458,6 @@ GROUP_A_TABLES = { "report": "GET_FBA_REIMBURSEMENTS_DATA", "role": "Amazon Fulfillment", "min_interval_s": 14400, - "restatement_keys": ("reimbursement_id",), "cadence_note": "Date windows; refresh no sooner than 14400 seconds.", "validates_against": "Amazon document or API data-row count vs stored rows for the latest period", }, @@ -467,7 +466,6 @@ GROUP_A_TABLES = { "report": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", "role": "Amazon Fulfillment", "min_interval_s": 14400, - "restatement_keys": ("order_id", "sku", "fnsku", "asin", "return_date"), "cadence_note": "Date windows; refresh no sooner than 14400 seconds.", "validates_against": "Amazon document or API data-row count vs stored rows for the latest period", }, @@ -678,8 +676,8 @@ GROUP_A_TABLES = { "label": "Amazon account health", "report": "GET_V2_SELLER_PERFORMANCE_REPORT", "role": "Selling Partner Insights", - "min_interval_s": 43200, - "cadence_note": "Current snapshot; refresh no sooner than 43200 seconds.", + "min_interval_s": 86400, + "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.", "snapshot": True, "json_root": "performanceMetrics", "json_schema": "accountHealthReport-2020-11-18.json", @@ -736,7 +734,6 @@ GROUP_A_TABLES = { "report": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE", "role": "Inventory and Order Tracking", "min_interval_s": 14400, - "restatement_keys": ("order_id", "sku", "return_request_date"), "cadence_note": "Date windows; refresh no sooner than 14400 seconds.", "max_window_days": 60, "validates_against": "Amazon document or API data-row count vs stored rows for the latest period", @@ -1018,40 +1015,6 @@ GROUP_A = { GROUP_A.update({key: {"key": key, **spec} for key, spec in DECLARED_DATASETS.items()}) -# Source: AutoWiki/research/datadoe-sync-cadence.md. Unlisted tables keep the daily default. -CADENCE = { - table: {"perDay": 1, "lookbackDays": 1, "intradayLookbackDays": 1, "period": "day"} - for table in GROUP_A -} -for _table, _spec in REPORT_TYPES.items(): - if _spec.get("monthly") or _spec.get("period") == "MONTH": - CADENCE[_table].update(period="month", lookbackDays=60, intradayLookbackDays=60) - elif _spec.get("weekly") or _spec.get("min_interval_s", 0) >= 7 * 86400: - CADENCE[_table].update(period="week", lookbackDays=14, intradayLookbackDays=14) -for _tables, _passes, _daily, _intraday in ( - (("orders", "order_items"), 6, 8, 1), - (("settlements", "transactions"), 3, 8, 2), - (("listings", "inventory_planning"), 4, 1, 1), - (("account_health", "manage_inventory", "restock_recommendations"), 2, 1, 1), - (("pricing", "fba_fee_estimates", "referral_fees"), 1, 4, 4), - (("reimbursements", "long_term_storage_fees"), 1, 30, 30), - (("fba_customer_returns", "mfn_returns"), 1, 60, 60), - (("storage_fees",), 1, 1, 1), -): - for _table in _tables: - CADENCE[_table].update(perDay=_passes, lookbackDays=_daily, - intradayLookbackDays=_intraday, period="day") - -# Ads uses namespaced mirror marker names, so its tables cannot collide with SP-API tables. -from harness import amazon_ads_schema as _ads_schema - -CADENCE.update({ - "ads_" + table: {"perDay": 4, "lookbackDays": 21, "intradayLookbackDays": 5, "period": "day"} - for table in _ads_schema.TABLE_SPECS -}) -del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema - - # Inventory section 2.5: immutable refusal even if a future report roster admits one. EXCLUDED_REPORT_TYPES = frozenset({ "GET_ORDER_REPORT_DATA_SHIPPING", "GET_ORDER_REPORT_DATA_TAX", "GET_ORDER_REPORT_DATA_ALL", @@ -3418,8 +3381,7 @@ def report_document(fields, report_type, start=None, end=None, options=None, # a role-shaped refusal on screen and left the window to retry forever. An empty # document lets the window complete and the connector detail name the empty period. say(f" {report_type}: Amazon cancelled report {rid}; no data for the requested period") - return "", {"reportId": rid, "reportDocumentId": None, "processingStatus": state, - "cancelled": True, + return "", {"reportId": rid, "reportDocumentId": None, "cancelled": True, "dataStartTime": got.get("dataStartTime"), "dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"), "compression": None, @@ -3457,7 +3419,7 @@ def report_document(fields, report_type, start=None, end=None, options=None, raise AmazonError(f"the {report_type} report document carried no download URL") text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True) say(f" {report_type}: {len(text):,} chars") - return text, {"reportId": rid, "reportDocumentId": doc_id, "processingStatus": state, + return text, {"reportId": rid, "reportDocumentId": doc_id, "dataStartTime": got.get("dataStartTime"), "dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"), "compression": meta.get("compressionAlgorithm"), "marketplaceIds": got.get("marketplaceIds")} diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index e7a7338bce015b1ab2bceb7e895f7a4236b4d990..5dfc5e3da9f0ab4ceaa4d362823e35a05ae020f9 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -463,48 +463,27 @@ def read_database_enablement(rt): return {"available": True, "databases": {key: stored.get(key, True) for key in keys}} -# Day length for per-table cadence. Manual sync keeps the report's own refresh floor. +#: ⛔⛔ HOTFIX 2026-09-29 (owner): *"We should only sync daily at the fastest."* A scheduled pass +#: refreshes each table at most once a day. The two hours of grace keep a table stamped late in +#: yesterday's pass from sliding to 48 hours because today's pass reached it a little earlier. +#: An operator's Sync now keeps only Amazon's own report floors (owner ruling, same day). SCHEDULED_FLOOR_SECONDS = 86400 -SCHEDULED_FLOOR_GRACE_SECONDS = 0 - - -def _cadence_seconds(table, state=None): - """Return the table's scheduled interval, using calendar months for monthly reports.""" - cadence = _client().CADENCE[table] - if cadence["period"] == "week": - if state: - stamp = _utc(state["updated_at"]) - return int((_week_start(stamp) + timedelta(days=7) - stamp).total_seconds()) - return 7 * SCHEDULED_FLOOR_SECONDS - if cadence["period"] == "month": - stamp = _utc(state["updated_at"]) if state else None - if stamp is None: - return 30 * SCHEDULED_FLOOR_SECONDS - year, month = stamp.year + (stamp.month == 12), stamp.month % 12 + 1 - due = stamp.replace(year=year, month=month, day=1, - hour=0, minute=0, second=0, microsecond=0) - return int((due - stamp).total_seconds()) - return SCHEDULED_FLOOR_SECONDS // cadence["perDay"] - - -def sync_schedule(states, now=None): - """Return every SP-API table's last sync, next sync and configured passes per day.""" - now = _utc(now) - schedule = [] - for table in TABLES: - state = states.get(table) - seconds = _cadence_seconds(table, state) - _wait, due, _seconds, _scheduled = _interval_wait(table, state, now, floor_s=seconds) - if due is None and state and state.get("state") == "done": - vendor = _client().REPORT_TYPES[table].get("min_interval_s", 0) - due = _utc(state["updated_at"]) + timedelta(seconds=max(vendor, seconds)) - schedule.append({"table": table, "lastSyncAt": state.get("updated_at") if state else None, - "nextSyncAt": _iso(due or now), "perDay": _client().CADENCE[table]["perDay"]}) - return schedule +SCHEDULED_FLOOR_GRACE_SECONDS = 2 * 3600 def sync_plan(states, tables=None, *, now=None, slot_pass=False): - """Plan every due table, oldest first, with the same per-table cadence inside and outside the daily slot.""" + """Pure cadence decision: EVERY due table, oldest first, with no stagger sleep. + + `slot_pass` is the tenant's daily slot pass: it IS the once-a-day sync, so only Amazon's own + report floors defer a table. Any other scheduled pass keeps the daily floor. + + Attempt markers survive in the tenant's saved mirror. Failed tables rotate behind peers. + Settlement LIST discovery runs separately on every resync and consumes no report slot. + The client still owns pacing/retries within a table. + ⛔ HOTFIX 2026-09-29: this picked ONE table per tick. At a once-a-day cadence that is one + table a day, about seven weeks between two refreshes of the same table, so a scheduled pass + now takes every table whose daily floor (and Amazon's own floor) has passed. + """ from harness.amazon_client import REPORT_STAGGER_SECONDS now = _utc(now) want = list(TABLES if tables is None else tables) @@ -512,8 +491,10 @@ def sync_plan(states, tables=None, *, now=None, slot_pass=False): if state and table != "settlements"} deferred = {} due = [] + floor = 0 if slot_pass else SCHEDULED_FLOOR_SECONDS - SCHEDULED_FLOOR_GRACE_SECONDS for table in want: - floor = _cadence_seconds(table, states.get(table)) + if table == "settlements": + continue if (reason := _interval_reason(table, states.get(table), now, floor_s=floor)): deferred[table] = reason else: @@ -527,37 +508,8 @@ def sync_plan(states, tables=None, *, now=None, slot_pass=False): for table in due: deferred[table] = ("table stagger: waiting for a later cadence tick; " f"minimum spacing {REPORT_STAGGER_SECONDS:g}s") - return {"tables": selected, "deferred": deferred} - - -def scheduled_sync_plan(tenant_key, *, rt, now=None): - """Combine enabled tables' due plans from each connection's own resident cadence ticks.""" - path = _tenant_mirror_path(tenant_key, rt) - if path.resolve() != Path(datastore.DB_PATH).resolve(): - raise RuntimeError("Amazon scheduling requires the tenant's resident mirror") - enabled = read_database_enablement(rt) - tables = [table for table in TABLES - if enabled["databases"].get(amazon_schema.DATABASE_PREFIX + table)] - if not tables: - return {"tables": [], "deferred": {}} - now = _utc(now) - con, _path = _connect_tenant_mirror(tenant_key, rt) - try: - due, deferred = {}, {} - for entry in account_entries(tenant_key, rt): - prefix = _account_prefix(entry["connection_id"]) + entity_key("sync_tick_") - ticks = {entity[len(prefix):]: {"state": phase, "rows": rows, "updated_at": at, "detail": detail} - for entity, phase, rows, at, detail in con.execute( - "SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state " - "WHERE starts_with(entity, ?)", [prefix]).fetchall()} - plan = sync_plan(ticks, tables, now=now) - due.update(dict.fromkeys(plan["tables"])) - for table, reason in plan["deferred"].items(): - deferred.setdefault(table, []).append(reason) - return {"tables": list(due), "deferred": { - table: "; ".join(dict.fromkeys(reasons)) for table, reasons in deferred.items() if table not in due}} - finally: - con.close() + return {"tables": selected + (["settlements"] if "settlements" in want else []), + "deferred": deferred} def _readable_duration(seconds): @@ -573,12 +525,10 @@ def _readable_duration(seconds): def _interval_wait(table, state, now, floor_s=None): - """Return the wait and deciding refresh floor, preserving the vendor floor on manual sync.""" - if state and state.get("state") == "throttled" and state.get("detail"): - due = _utc(state["detail"]) - wait = (due - _utc(now)).total_seconds() - if wait > 0: - return wait, due, int(wait), False + """Seconds until `table` may be requested again (0 when due) and which floor decides it. + + `floor_s` is the scheduled daily floor; Amazon's own `min_interval_s` always applies too. + """ vendor = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s") or 0 seconds = max(int(vendor), int(floor_s or 0)) if seconds and state and state["state"] == "done": @@ -590,13 +540,11 @@ def _interval_wait(table, state, now, floor_s=None): def _interval_reason(table, state, now, floor_s=None): - wait, due, seconds, scheduled = _interval_wait(table, state, now, floor_s=floor_s) + wait, due, seconds, daily = _interval_wait(table, state, now, floor_s=floor_s) if wait <= 0: return None - if state and state.get("state") == "throttled": - return f"Amazon is limiting requests; next retry {_iso(due)}" - if scheduled: - return (f"This Amazon table refreshes every {_readable_duration(seconds)}; " + if daily: + return (f"Scheduled syncs refresh each Amazon table at most once a day; " f"next due {_iso(due)}") return (f"Amazon allows this report at most once every {_readable_duration(seconds)}; " f"next due {_iso(due)}") @@ -4016,7 +3964,7 @@ def _restored_state_rows(rows, tables, refused): return rows refused = set(refused) names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name)) - families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "resume_", "reconcile_", "limit_", "refused_", + families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "reconcile_", "limit_", "refused_", RESYNC_PREFIX) shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)} @@ -4787,7 +4735,7 @@ def _record_state(con, key, phase, rows=0, detail=None): con.execute( "INSERT OR REPLACE INTO _sync_state " "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", - [str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_", "amzn_resume_", + [str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_", "amzn_resync_", "amzn_waiting_")) else str(detail)[:300]) if detail else None, int(rows or 0), _iso(datetime.now(timezone.utc))]) @@ -4859,25 +4807,31 @@ def _first_load_due(con): def _restore_first_load_wait(tenant_key, con): - """Keep the history floor in the producer and release legacy connector-wide desk holds.""" from core import connector_sync due = _first_load_due(con) - connector_sync.clear_sync_defer("amazon_sp", tenant_key) + if _account_scope() is not None: + return due + connector_sync.defer_sync("amazon_sp", tenant_key, due.timestamp() if due else None) return due def _loaded_refresh_hold(con, table, now, *, slot_pass=False): - """Return a loaded table's cadence hold while its first load continues, or None when due.""" + """`(next due, reason)` while a loaded table's own floor holds it during a first load, else None. + + A weekly table is due when a new week has closed after its watermark (D-11328). Every other + table, settlement discovery included, keeps the daily floor and Amazon's own floor. + """ now = _utc(now) tick = _read_state(con, entity_key("sync_tick_" + table)) # A queued resync is a repair: only Amazon's own floor holds it. repair = queued_resync(con, table) is not None - floor = 0 if repair else _cadence_seconds(table, tick) - wait, due, _seconds, _scheduled = _interval_wait(table, tick, now, floor_s=floor) + floor = 0 if slot_pass or repair else SCHEDULED_FLOOR_SECONDS - SCHEDULED_FLOOR_GRACE_SECONDS + wait, due, _seconds, daily = _interval_wait(table, tick, now, floor_s=floor) reason = None if wait > 0: # The card parses "next due " out of a reason, so every hold keeps that phrase. - reason = _interval_reason(table, tick, now, floor_s=floor) + reason = (f"Each loaded Amazon table refreshes at most once a day while the first load continues. " + f"It is next due {_iso(due)}" if daily else _interval_reason(table, tick, now, floor_s=floor)) mark = read_watermark(con, table) if _client().REPORT_TYPES.get(table, {}).get("weekly") and mark and not repair: week = _week_start(mark) + timedelta(days=7) @@ -5175,8 +5129,6 @@ def first_load_status(tenant_key, rt=None): remaining = int(values["remaining"]) # S82-T14: one pass sees only its own tables. The job's store record counts the whole load. job = first_load_job_summary(rt) if _store_documents(rt) else None - if job: - job["tables"]["inventory"] = inventory_table_status(con) if job and isinstance(job.get("remaining"), int) and state["state"] != "done": remaining = job["remaining"] out = {"state": state["state"], **first_load_payload(markers, remaining, @@ -5461,7 +5413,7 @@ def _sync_writer(tenant_key, rt, log): def _sync_delegate(tenant_key, rt, log): """Every table but the job's own takes one bounded first-load pass: settlements, period, Group A, SQP. - The same pass refreshes each loaded table whose cadence is due, so loaded data keeps moving. + The same pass refreshes each loaded table whose daily floor is due, so loaded data keeps moving. """ def delegate(exclude, budget): tables = None if exclude is None else [t for t in TABLES if t not in set(exclude)] @@ -5521,91 +5473,7 @@ def _status_stamp(status, connector): return max(stamps, key=lambda item: item[0])[1] if stamps else None -def inventory_table_status(con): - """Read inventory counts and account-scoped source proof without changing the mirror or requesting reports.""" - rows = (int(con.execute("SELECT COUNT(*) FROM amzn_inventory").fetchone()[0]) - if _columns_of(con, "amzn_inventory") else 0) - out = {"rows": rows, "empty": rows == 0} - if rows: - return out - keys = ("amzn_inventory", "amzn_reconcile_inventory", "amzn_refused_inventory", "amzn_inventory_awd") - clauses = " OR ".join("entity=? OR ends_with(entity, ?)" for _ in keys) - saved = con.execute( - "SELECT entity, phase, cursor_wd, rows, updated_at FROM _sync_state WHERE " + clauses - + " OR contains(entity, 'amzn_pending_report_')", - [value for key in keys for value in (key, "__" + key)]).fetchall() - accounts, pending, unknown_request = {}, [], False - for entity, phase, detail, count, stamp in saved: - prefix, key = (str(entity).split("__", 1) if str(entity).startswith("amzn_account_") - and "__" in str(entity) else ("", str(entity))) - if key.startswith("amzn_pending_report_"): - try: - request = json.loads(detail or "{}") - except (ValueError, TypeError): - request = {} - if not isinstance(request, dict): - request = {} - if request.get("table") == "inventory": - pending.append({"type": request.get("reportType"), "state": phase, - "reportId": request.get("reportId"), - "processingStatus": request.get("processingStatus"), "updatedAt": stamp}) - elif not request.get("table"): - unknown_request = True - continue - accounts.setdefault(prefix, {})[key] = {"state": phase, "detail": detail, - "rows": count, "updated_at": stamp} - if pending: - return {**out, "code": "first_load_pending", "reason": - "Amazon is preparing the inventory report, so keep sync enabled and retry after it completes.", - "reports": pending} - # Account markers own their proof once account scoping exists. - scoped = {key: state for key, state in accounts.items() if key} - accounts = scoped or accounts - reports = [] - for states in accounts.values(): - marker = states.get("amzn_inventory") or {} - refused = refused_reports(states) - if refused: - return {**out, "code": "refused_by_amazon", "reason": refused[0]["reason"]} - proof = states.get("amzn_reconcile_inventory") or {} - try: - detail = json.loads(proof.get("detail") or "{}") - except (ValueError, TypeError): - detail = {} - if not isinstance(detail, dict): - detail = {} - dates = detail.get("dates") - valid_dates = (isinstance(dates, dict) and bool(dates) - and all(isinstance(day, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", day) - and type(count) is int and count == 0 for day, count in dates.items())) - awd = states.get("amzn_inventory_awd") or {} - if (marker.get("state") != "done" or marker.get("rows") != 0 or proof.get("state") != "done" - or type(detail.get("count")) is not int or detail["count"] != 0 - or not isinstance(detail.get("document"), str) or not detail["document"].strip() - or detail.get("column") != "_snapshot_date" or not valid_dates - or detail.get("start") != min(dates) or detail.get("end") != max(dates) - or awd.get("state") != "done" or awd.get("rows") != 0): - return {**out, "code": "first_load_pending", "reason": - "Inventory source checks are incomplete, so retry Amazon sync to finish the FBA and AWD inventory checks."} - reports.append({"type": _client().REPORT_TYPES["inventory"]["report"], "state": proof["state"], - "reportId": detail.get("reportId"), "processingStatus": detail.get("processingStatus"), - "documentId": detail["document"], "start": detail["start"], "end": detail["end"], - "rows": detail["count"], "updatedAt": proof.get("updated_at")}) - if not reports: - if unknown_request: - return {**out, "code": "first_load_pending", "reason": - "A saved Amazon request has no table identity, so retry sync to refresh the inventory report status."} - return {**out, "code": "not_requested", "reason": - "No inventory report request is saved, so start Amazon sync to request the FBA ledger."} - periods = {(report["start"], report["end"]) for report in reports} - start, end = next(iter(periods)) - period = (start if start == end else f"{start} to {end}") if len(periods) == 1 else "the loaded dates" - return {**out, "code": "no_source_rows", "reason": - f"Amazon returned no FBA ledger rows for {period} and no AWD inventory, so check seller stock and retry Amazon sync.", - "reports": reports} - - -def first_load_table_status(tasks, finished, failed, counts, *, table_status=None): +def first_load_table_status(tasks, finished, failed, counts): """D-2643, D-2642, D-2649: a row count or a named reason per table, and why orders are behind.""" out = {} for table in dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES): @@ -5619,25 +5487,17 @@ def first_load_table_status(tasks, finished, failed, counts, *, table_status=Non f"The first load is still loading this table. {len(left)} report windows remain." if left else "Amazon returned no rows for this seller in the loaded period." if own else "This table holds no rows and the first load owes it no windows.") - if table == "inventory": - row.update(empty=True, code="first_load_pending", reason= - "Inventory source proof is not available in this job record, so retry Amazon sync to refresh its status.") if table in BEHIND_TABLES and left: done = [t for t in own if t["id"] in finished or t.get("landed")] oldest = min((t["start"] for t in done if t.get("start")), default=None) row["behind"] = (f"The first load is still loading this table. {len(left)} of {len(own)} report " "windows remain" + (f", and history before {oldest[:10]} is not loaded yet." if oldest else ", starting with the most recent month.")) - if table_status and table in table_status: - row.update(table_status[table]) - if not row.get("empty"): - row.pop("reason", None) - row.pop("code", None) out[table] = row return out -def first_load_job_summary(docs, *, table_status=None): +def first_load_job_summary(docs): """The latest generation record and the cursor's remaining count, read from the store.""" try: record = _doc(docs, FIRST_LOAD_GENERATIONS_KEY) @@ -5651,7 +5511,7 @@ def first_load_job_summary(docs, *, table_status=None): "remaining": record.get("remaining", last.get("remainingAfter")), "tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"), "operation": last.get("operation"), "resume_at": last.get("resume_at"), - "tables": {**(last.get("tables") or {}), **(table_status or {})}} + "tables": last.get("tables") or {}} def _record_generation(docs, row): @@ -5927,15 +5787,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES))) except Exception: # noqa: BLE001 row_counts = {} - table_status = None - if plan is None: - con, _path = _connect_tenant_mirror(key, rt) - try: - table_status = {"inventory": inventory_table_status(con)} - finally: - con.close() - record["tables"] = first_load_table_status(after, finished, failed, row_counts, - table_status=table_status) + record["tables"] = first_load_table_status(after, finished, failed, row_counts) record["state"] = "done" if not record["remainingAfter"] else "partial" status_after = read_connector_status(docs) record["odooLastGoodSyncAt"] = {"start": _status_stamp(status_before, "odoo"), @@ -5960,7 +5812,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon pass -def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=False): +def sync_window(table, mark, *, now, since=None, backfill_days=None): """Pure request decision for one table: start, end, report, incremental and cause.""" end = _utc(now) days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days @@ -5978,11 +5830,6 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=Fa start, incremental = end - timedelta(days=days), False cause = f"first full load: {days} days of backfill" spec = _client().REPORT_TYPES.get(table) or {} - if incremental and not spec.get("snapshot") and not _periodic(spec): - cadence = _client().CADENCE[table] - lookback = cadence["intradayLookbackDays" if intraday else "lookbackDays"] - start = min(start, end - timedelta(days=lookback)) - cause += f"; re-read {lookback} days ({'intraday' if intraday else 'daily'})" rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report") rtype = rtype or spec.get("api") if table in ("economics", "order_metrics"): @@ -6011,22 +5858,7 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=Fa # The watermark filters issued reports by creation time; it never requests one. rtype = spec.get("report_discover") return {"start": start, "end": end, "report": rtype, - "incremental": incremental, "intraday": intraday, "cause": cause} - - -def _lookback_intraday(con, table, now, *, slot_pass=False): - """Use the daily window until this account has completed it, including after a failed attempt.""" - day = _read_state(con, entity_key("lookback_day_" + table)) - return bool(not slot_pass and day and day.get("state") == "done" - and day.get("detail") == now.date().isoformat()) - - -def _record_lookback_day(con, table, plan, now): - """Only a completed incremental daily request earns this account's intraday window.""" - if (plan["incremental"] and not plan.get("intraday") - and plan["end"] >= now - timedelta(minutes=2) - and not _lookback_intraday(con, table, now)): - _record_state(con, entity_key("lookback_day_" + table), "done", detail=now.date().isoformat()) + "incremental": incremental, "cause": cause} def _sync_event(kind, tenant, table, detail, rt, report, log): @@ -6123,21 +5955,10 @@ def _pending_report_key(table, report_type, **kwargs): return entity_key("pending_report_" + hashlib.sha256(identity.encode()).hexdigest()) -def _save_throttled_resume(con, table, exc, plan, completed, *, first_load, since): - """Keep the continuation beside the account's markers and leave the retry tick as an ISO deadline.""" - continuation = {**exc.as_resume(), "plan": {**plan, "start": _iso(plan["start"]), "end": _iso(plan["end"])}, - "completed": completed, "first_load": first_load, - "since": _iso(_utc(since)) if since is not None else None} - _record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation)) - due = datetime.now(timezone.utc) + timedelta(seconds=max(60.0, float(exc.retry_after or 60))) - _record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due)) - return max(0, (due - datetime.now(timezone.utc)).total_seconds()) - - @contextmanager def _report_resume(con, table, report_type, **kwargs): """Resume the same report/window for up to 24 hours instead of paying createReport again.""" - from harness.amazon_client import AmazonReportError, AmazonReportPending, AmazonThrottled + from harness.amazon_client import AmazonReportError, AmazonReportPending window = {key: kwargs.get(key) for key in ("start", "end", "options")} key = _pending_report_key(table, report_type, **kwargs) state = _read_state(con, key) @@ -6152,15 +5973,7 @@ def _report_resume(con, table, report_type, **kwargs): yield kwargs except AmazonReportPending as exc: _record_state(con, key, "pending", detail=json.dumps({ - "reportId": exc.report_id, "createdAt": saved.get("createdAt", stamp), "window": window, - "table": table, "reportType": report_type, "processingStatus": exc.status})) - raise - except AmazonThrottled as exc: - if exc.report_id is not None: - _record_state(con, key, "throttled", detail=json.dumps({ - **exc.as_resume(), "reportId": str(exc.report_id), - "createdAt": saved.get("createdAt", stamp), "window": window, - "table": table, "reportType": report_type})) + "reportId": exc.report_id, "createdAt": saved.get("createdAt", stamp), "window": window})) raise except AmazonReportError: con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) @@ -6199,21 +6012,6 @@ def _record_reconciliation(con, table, marker): _record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":"))) -def _restatement_may_overlap(con, mirror, predicate, params, start, end, *, unsafe="TRUE"): - """Treat absent or invalid legacy period metadata as unknown when checking retained rows.""" - columns = set(_columns_of(con, mirror)) - def period(name): - return f'TRY_CAST("{name}" AS TIMESTAMPTZ)' if name in columns else "NULL::TIMESTAMPTZ" - old_start, old_end = period("_period_start"), period("_period_end") - return bool(con.execute( - f'SELECT 1 FROM {mirror} WHERE NOT COALESCE(({predicate}), FALSE) AND ({unsafe}) ' - f'AND ({old_start} IS NULL OR TRY_CAST(? AS TIMESTAMPTZ) IS NULL ' - f'OR {old_start} < TRY_CAST(? AS TIMESTAMPTZ)) ' - f'AND ({old_end} IS NULL OR {old_start} IS NULL OR {old_end} <= {old_start} ' - f'OR {old_end} > TRY_CAST(? AS TIMESTAMPTZ)) LIMIT 1', - [*params, end, end, start]).fetchone()) - - def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False): """Atomic range/date replacement and proof; an empty restatement removes old rows too. @@ -6243,44 +6041,10 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N if column != "_snapshot_date" or end: raise ValueError("a snapshot replacement is keyed by one _snapshot_date") predicate, params = "TRUE", [] - source_keys = _client().REPORT_TYPES[table].get("restatement_keys", ()) - staged = None - if source_keys: - overlap_end = (marker or {}).get("end") or end - missing = set(source_keys) - set(columns) - if missing: - # A same-window replay remains safe without source keys; a moving overlap must refuse. - if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end): - raise ValueError(f"{table}: overlapping restatement omitted source keys: " - + ", ".join(sorted(missing)) + "; request the complete source report") - elif rows: - staged = "amazon_restatement_" + uuid.uuid4().hex - names = ", ".join(_quoted_identifier(c) for c in columns) - con.execute(f"CREATE TEMP TABLE {staged} AS SELECT {names} FROM {mirror} WHERE FALSE") - try: - _replace(con, staged, columns, types, rows, source_table=mirror) - required = [c for c in source_keys if c not in ("fnsku", "asin")] - invalid = " OR ".join(f'NULLIF(TRIM(CAST({_quoted_identifier(c)} AS VARCHAR)), \'\') IS NULL' - for c in required) - if con.execute(f"SELECT 1 FROM {staged} WHERE {invalid} LIMIT 1").fetchone(): - raise ValueError(f"{table}: restatement omitted a source key value; request the complete source report") - if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end, unsafe=invalid): - raise ValueError(f"{table}: overlapping legacy rows omitted a source key value; " - "restore the complete source report before restating") - except BaseException: - con.execute(f"DROP TABLE {staged}") - raise con.execute("BEGIN TRANSACTION") try: con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params) - if staged: - matches = " AND ".join(f'{mirror}.{_quoted_identifier(c)} IS NOT DISTINCT FROM ' - f'k.{_quoted_identifier(c)}' for c in source_keys) - con.execute(f'DELETE FROM {mirror} WHERE EXISTS (SELECT 1 FROM {staged} k WHERE {matches})') - con.execute(f"INSERT INTO {mirror} ({names}) SELECT {names} FROM {staged}") - count = con.execute(f"SELECT COUNT(*) FROM {staged}").fetchone()[0] - else: - count = _replace(con, mirror, columns, types, rows) + count = _replace(con, mirror, columns, types, rows) if marker is not None: _record_reconciliation(con, table, marker) if table == "listing_items": @@ -6290,9 +6054,6 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N except Exception: con.execute("ROLLBACK") raise - finally: - if staged: - con.execute(f"DROP TABLE {staged}") def document_row_count(text, roots=()): @@ -6435,7 +6196,6 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count): dates = sorted(counts) marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date", "count": source_count, "dates": counts, "document": meta.get("reportDocumentId")} - marker.update({key: meta[key] for key in ("reportId", "processingStatus") if meta.get(key)}) # Every earlier date and any undated pre-migration copy goes; AWD rows are another leg. predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE" con.execute("BEGIN TRANSACTION") @@ -7710,8 +7470,7 @@ def _sync_accounts_candidate(tenant_key="royal-imports", tables=None, since=None for entry in entries: cid = entry["connection_id"] try: - with _account_workspace(con, tenant_key, rt, entry) as scope, \ - _client().rate_limit_scope(tenant_key, cid): + with _account_workspace(con, tenant_key, rt, entry) as scope: _sync_credential(tenant_key, rt, connection_id=cid) result = _sync_candidate(tenant_key, tables=tables, since=since, log=log, rt=rt, max_wait=max_wait, scheduled=scheduled, first_load=first_load, @@ -7882,7 +7641,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri ticks.update({name: None for name in (table, pairs.get(table)) if name in want}) for table in want: tick = ticks.get(table) - # A queued resync is a repair: only Amazon's own floor and the stagger hold it. + # A queued resync is a repair: only Amazon's own floor and the stagger hold it, never the daily floor. if (tick and tick["state"] == "done" and queued_resync(con, table) is not None and read_watermark(con, table) and not _interval_reason(table, tick, end)): ticks[table] = {**tick, "state": "resync"} @@ -8016,15 +7775,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri else: for table in pending: mark = read_watermark(con, table) - if table == "settlements" and since is None: - # Empty discovery earns a request cutoff, never a stored source watermark. - tick = _read_state(con, entity_key("sync_tick_" + table)) - if tick and tick.get("state") == "done": - mark = _utc(tick["updated_at"]) - elif not mark: - day = _read_state(con, entity_key("lookback_day_" + table)) - if day and day.get("state") == "done": - mark = _utc(day["updated_at"]) if not mark and since is None and table != "settlements" and table in completed: # A local fallback cannot reopen old daily history or invent a stored watermark. recent = end - timedelta(days=FIRST_LOAD_RECENT_DAYS) @@ -8034,8 +7784,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri mark = recent if not recent <= mark <= end: mark = recent - plan = plans[table] = sync_window(table, mark, now=end, since=since, - intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass)) + plan = plans[table] = sync_window(table, mark, now=end, since=since) if table == SQP_TABLE: months = _month_windows(plan["start"], plan["end"]) if not plan["incremental"]: @@ -8049,16 +7798,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri else: work.append((table, None, None, None)) - resumes = {} - for table in pending: - saved = _read_state(con, entity_key("resume_" + table)) - continuation = json.loads(saved["detail"]) if saved and saved.get("detail") else {} - if (continuation and continuation.get("first_load") == first_load - and continuation.get("since") == (_iso(_utc(since)) if since is not None else None)): - resumes[table] = continuation - plans[table] = {**continuation["plan"], "start": _utc(continuation["plan"]["start"]), - "end": _utc(continuation["plan"]["end"])} - if sqp_limit is not None: def continuation_first(task): _table, w0, w1, batch = task @@ -8095,14 +7834,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri and (task_end is not None or table == "settlements" or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)): break - tick = _read_state(con, entity_key("sync_tick_" + table)) - if first_load and (tick or {}).get("state") == "throttled": - reason = _interval_reason(table, tick, datetime.now(timezone.utc)) - if reason: - report["tables"][table] = {"state": "deferred", "reason": reason} - wait = _interval_wait(table, tick, datetime.now(timezone.utc))[0] - report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) - continue if table == SQP_TABLE and task_start is not None and _sqp_period(task_start, task_end) == "WEEK": weekly_limit = {**sqp_limit, "refusedAsins": {}, "errors": []} causes = [] @@ -8125,16 +7856,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900) _record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"]) except Exception as exc: - if isinstance(exc, cx.AmazonThrottled): - problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log) - report["tables"][table] = problem - failed.add(table) - pending_reports.add(table) - wait = _save_throttled_resume(con, table, exc, plans[table], [], - first_load=first_load, since=since) - report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) - _record_state(con, entity_key(table), "throttled", detail=problem["reason"]) - continue causes.append(str(exc)) if isinstance(exc, cx.AmazonError) and not isinstance(exc, (cx.AmazonReportPending, cx.AmazonThrottled)): marker = _sqp_month(con, task_start, task_end) @@ -8151,6 +7872,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri continue if first_load and (table in GROUP_A_TABLES or (table in prior_refusals and cx.REPORT_TYPES[table].get("min_interval_s"))): + tick = _read_state(con, entity_key("sync_tick_" + table)) reason = _interval_reason(table, tick, datetime.now(timezone.utc)) if reason: report["tables"][table] = {"state": "deferred", "reason": reason} @@ -8210,7 +7932,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri proof_key = entity_key("reconcile_" + table) previous_proof = _read_state(con, proof_key) if table in invalid_proofs else None resync, resync_count = [], 0 - completed_windows = list(resumes.get(table, {}).get("completed", [])) try: if previous_proof: # The recorder accepts an empty object; only a parsed document can replace it with proof. @@ -8220,9 +7941,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri _record_state(con, entity_key("sync_tick_" + table), "attempted") if sibling and not first_load: _record_state(con, entity_key("sync_tick_" + sibling), "attempted") - sibling_written, sibling_cleared = 0, bool(completed_windows) - written, cols, cleared = 0, [], bool(completed_windows) - landed_window = bool(completed_windows) + sibling_written, sibling_cleared = 0, False + written, cols, cleared = 0, [], False + landed_window = False pricing_asins = [] operations = {} if first_load: @@ -8272,17 +7993,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri "state": "done", "reports": len(discovered), "downloaded": downloaded, "skipped": skipped} if not discovered: - con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)]) report["tables"][table] = {"rows": 0, "operations": operations, "state": "empty", "reason": "No issued settlement reports found."} landed.add((table, task_start, task_end, task_batch)) if first_load: _record_state(con, key, "done", 0, "No issued settlement reports found.") - # A successful empty scheduled discovery earns cadence, but no source cursor or document marker. - if scheduled and not first_load: - _record_state(con, entity_key("sync_tick_" + table), "done", 0) - if not first_load and since is None: - _record_lookback_day(con, table, {**plan, "incremental": True}, end) + # Empty discovery earns no cursor, attempt stamp or document marker. continue if first_load and settlement_remaining: break @@ -8331,8 +8047,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri plan["cause"] = (f"resync of {len(margin_windows)} stored windows " f"from {_iso(margin_windows[0][0])}") log(f"{table}: {plan['cause']}") - elif (table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1 - and not spec.get("restatement_keys")): + elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1: # One due window per table tick respects the vendor floor. The watermark # advances only to that window's end, so the following tick loses no history. remaining_windows = len(margin_windows) - 1 @@ -8348,9 +8063,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if not reason.startswith("Amazon keeps this report back to "): _record_state(con, entity_key("limit_" + table), "done", detail="{}") for w0, w1 in margin_windows: - window = [_iso(w0) if w0 is not None else None, _iso(w1) if w1 is not None else None] - if window in completed_windows: - continue # One detail window is one budget task, including its reconciliation query. probe = _DocumentProbe(table, spec.get("report") or operation) documents += 1 @@ -8366,7 +8078,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri spend_rows(count) written += count landed_window = True - completed_windows.append(window) if table in LEGACY_SNAPSHOT_TABLES: # The dated replacement already landed; AWD must not clear that # document or historical FBA/AWD rows on an explicit first load. @@ -8390,9 +8101,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri plan["end"] = windows[0][1] weekly = bool(table != "settlements" and cx.REPORT_TYPES[table].get("weekly")) for w0, w1 in (() if table == "settlements" else windows): - window = [_iso(w0), _iso(w1)] - if window in completed_windows: - continue # W59-T43 / C2 has no amazon.access kind. Record each credential-backed # vendor attempt as a sync ACCESS phase, distinct from the completion event. # Lane F/combine own the vocabulary decision; no credential or row enters C2. @@ -8471,7 +8179,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1") cleared = True else: - completed_windows.append(window) continue if sibling: sibling_mirror = table_name(sibling) @@ -8512,7 +8219,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri _clear_week_wait(con, table, w0) probe.finish(len(rows) if cols else 0) spend_rows(len(rows) if cols else 0) - completed_windows.append(window) if first_load: for reached in (table, sibling): if reached: @@ -8562,8 +8268,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, " f"{updated:,} mirror rows enriched") except Exception as e: # noqa: BLE001 - if isinstance(e, cx.AmazonThrottled): - raise problem = _sync_problem(e, table, "getCompetitiveSummary", tenant_key, rt, report, log) _record_state(con, state_key, problem["state"], 0, problem["reason"]) @@ -8603,7 +8307,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["tables"][table]["nextWeek"] = _iso(period_repair_next) _record_state(con, key, "done", total_rows) _record_state(con, entity_key("sync_tick_" + table), "done", total_rows) - con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)]) if weekly and landed_window: # A landed week closes any earlier refusal of this table. con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("refused_" + table)]) @@ -8619,10 +8322,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri or landed_window or (first_load and floors[table] is not None)) if mark is not None and has_window: record_watermark(con, table, mark, total_rows) - if not first_load and not resync: - completed_plan = ({**plan, "incremental": True} - if table == "settlements" and since is None else plan) - _record_lookback_day(con, table, completed_plan, end) _sync_event("amazon.sync", tenant_key, table, {"state": "done", "rows": total_rows, "operations_ok": all(v["state"] == "done" for v in operations.values())}, @@ -8642,10 +8341,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri "incremental": sibling_plan["incremental"], "operations": {}} _record_state(con, entity_key(sibling), "done", sibling_total) _record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total) - con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + sibling)]) record_watermark(con, sibling, sibling_plan["end"], sibling_total) - if not first_load: - _record_lookback_day(con, sibling, sibling_plan, end) _sync_event("amazon.sync", tenant_key, sibling, {"state": "done", "rows": sibling_total, "operations_ok": True}, rt, report, log) @@ -8667,17 +8363,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri # A failed pass retains its previous watermark so its window will be retried. # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN # ticket answering itself, and it must not cost the six tables that did work. - problem = _sync_problem(e, table, getattr(e, "operation", None) or operation, tenant_key, rt, report, log) + problem = _sync_problem(e, table, operation, tenant_key, rt, report, log) failed.add(table) report["tables"][table] = {"rows": 0, **problem} - if problem["state"] == "throttled": - landed.discard((table, task_start, task_end, task_batch)) - pending_reports.add(table) - if _columns_of(con, mirror): - report["tables"][table]["rows"] = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] - wait = _save_throttled_resume(con, table, e, plan, completed_windows, - first_load=first_load, since=since) - report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) if problem["state"] == "pending": pending_reports.add(table) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900) @@ -8695,13 +8383,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri "reason": problem["reason"][:500], "amazonStatus": problem["amazonStatus"], "diagnostic": problem.get("diagnostic", "")[:500], "error": problem.get("error", ""), "attempts": attempts})) - if problem["state"] == "throttled": - if refused != table: - _save_throttled_resume(con, refused, e, sibling_plan, completed_windows, - first_load=first_load, since=since) - else: - _record_state(con, entity_key("sync_tick_" + refused), - "attempted" if problem["state"] == "pending" else "done") + _record_state(con, entity_key("sync_tick_" + refused), + "attempted" if problem["state"] == "pending" else "done", 0) except Exception: # noqa: BLE001 pass _sync_event("amazon.sync", tenant_key, table, @@ -10929,8 +10612,8 @@ def group_a_selftest(log=print, check=None): clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"] + 60) idle = sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None, scheduled=True)["tables"]["long_term_storage_fees"] - ck("NC: with the queue empty the table cadence defers it again", - idle.get("state") == "deferred" and "next due " in idle.get("reason", "")) + ck("NC: with the queue empty the daily floor defers the table again", + idle.get("state") == "deferred" and "once a day" in idle.get("reason", "")) finally: con.close() diff --git a/platform/harness/canonical_source.py b/platform/harness/canonical_source.py index e47f52df13309cf1f694289b69bf454cc9fddfe4..cb032c5b68ea58a31f864db48d794fca4f34904c 100644 --- a/platform/harness/canonical_source.py +++ b/platform/harness/canonical_source.py @@ -956,14 +956,14 @@ class MirrorSource(Source): else: ids = [int(i) for i in product_ids if i is not None] out = {} - # Explicit selected ids include archived products; automatic enumeration stays active-only. + # Odoo's search_read on product.product skips archived ids, so they stay absent here too. result = self.cur.execute( "SELECT p.id, coalesce(f.qty, 0) FROM product_product p LEFT JOIN (" "SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q " "JOIN stock_location l ON l.id = q.location_id " "WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) " "GROUP BY 1) f ON f.product_id = p.id " - "WHERE p.id IN (SELECT unnest($2::BIGINT[]))", + "WHERE p.id IN (SELECT unnest($2::BIGINT[])) AND coalesce(p.active, false)", [view[1], ids]) while True: batch = result.fetchmany(BATCH) diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index d4fdf6bcb84cc32649f04a5031effa19065be5e5..64434ba54c6eac8ddd0d1f253eabd371ea76b248 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -513,8 +513,7 @@ def odoo_dynamic_model_identity(tenant_key, connection_id, model, table_key): or key not in (base, base + "__" + cid) or len(key) > 60 or not re.fullmatch(r"[a-z0-9_]+", key)): raise ValueError("Invalid dynamic Odoo database identity") - source = next((name for name, spec in ENTITIES.items() - if spec["model"] == native and not spec.get("metadata_only")), None) + source = next((name for name, spec in ENTITIES.items() if spec["model"] == native), None) if source is None: token = hashlib.sha256(json.dumps([tenant, cid, native], separators=(",", ":")).encode()).hexdigest() source = "odoo_dynamic_" + token @@ -3669,20 +3668,6 @@ ENTITIES.update({ }, }) -# Delivery references contain metadata only and share the tenant's publication boundary. -DELIVERY_PHOTO_FIELDS = ("id", "res_model", "res_id", "name", "mimetype", "file_size", "create_date") -DELIVERY_PHOTO_TYPES = ("image/jpeg", "image/png", "image/gif", "image/webp") -# An ID cursor disables Odoo's implicit res_field filter, so the gallery scope must be explicit. -DELIVERY_PHOTO_DOMAIN = [("res_model", "=", "stock.picking"), - ("type", "=", "binary"), - ("res_field", "=", False), - ("mimetype", "in", list(DELIVERY_PHOTO_TYPES))] -ENTITIES["delivery_photo_metadata"] = { - "model": "ir.attachment", "archivable": False, "optional": True, - "fields": list(DELIVERY_PHOTO_FIELDS[1:]), "domain": DELIVERY_PHOTO_DOMAIN, "metadata_only": True, -} -INT_FIELDS += ("file_size",) - # The product pool's pricing and tags. Each table holds exactly Odoo's default (active_test) read, # so it carries no `active` column and a reader needs no active filter. ENTITIES.update({ @@ -3708,7 +3693,6 @@ NUMERIC_FIELDS += ("volume", "incoming_qty", "free_qty", "virtual_available", "f #: cannot see an Odoo delete, or a quant quantity that moved without a write_date change. #: Scope `internal` replaces the internal-location quants only. Any other scope is the whole table. REPLACE_EACH_PASS = { - "delivery_photo_metadata": {"domain": DELIVERY_PHOTO_DOMAIN}, "stock_quant": {"domain": [("location_id.usage", "=", "internal")], "scope": "internal"}, "product_pricelist": {"domain": []}, "product_pricelist_item": {"domain": []}, @@ -3998,7 +3982,7 @@ def _save_state(con, key, phase, last_id, cursor_wd): def _base_domain(spec): - return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else []) + return [("active", "in", [True, False])] if spec["archivable"] else [] def sync_entity(key, max_batches=200, batch=BATCH, log=print): @@ -4271,9 +4255,6 @@ def _already(con, key, rec): def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch=BATCH, connection_id=None, client_factory=None): selected = list(ENTITIES) if entities is None else list(entities) - if ("stock_picking" in selected and "delivery_photo_metadata" in ENTITIES - and "delivery_photo_metadata" not in selected): - selected.append("delivery_photo_metadata") if entities is not None and not selected: return {} # ⭐ D-10. `{}` rather than a dict of paused phases, and the caller decides why that matters: diff --git a/platform/modules/account_health.py b/platform/modules/account_health.py index e6ab99ff75deecbf6ca8550eb8fb0a2151cd0217..7df140ce5a08b6a924fd4103a7ff3b0aaac46094 100644 --- a/platform/modules/account_health.py +++ b/platform/modules/account_health.py @@ -24,7 +24,6 @@ import hashlib import json import math from pathlib import Path -from urllib.parse import urlencode APP_LABEL = "Account health" SOURCE_KIND = "account_health" @@ -56,11 +55,6 @@ SETTLEMENT_TABLE = "amzn_settlements" COMPLAINT_METRICS = ("received_intellectual_property_complaints", "suspected_intellectual_property_violations", "product_authenticity_customer_complaints") SOURCE = "ut_amzn_account_health" -LISTING_SOURCE = "ut_amzn_listing_items" -CATEGORY_SOURCE_NOTE = ( - "Amazon's account health report provides category totals, targets and reporting windows, but no violation records. " - "Related listings show listing issues. Amazon does not link them to this category." -) NOTIFICATION_KIND = "account_health_new_items" _sync_delivery = None @@ -469,7 +463,7 @@ def _page_bounds(offset, limit, total): return start, size -def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=None, category=None, st=None): +def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=None, st=None): """Read one metric page or one exact metric without loading the complete snapshot.""" st = _rt(tenant_key, st) distance = settings(tenant_key, st=st)["alertDistancePct"] @@ -501,8 +495,6 @@ def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=Non "ELSE CASE WHEN CONTAINS(UPPER(condition), 'EQUAL') OR target = 0 THEN 'At most' ELSE 'Below' END END) " "|| ' ' || CASE WHEN unit = 'percent' THEN printf('%.2f%%', target) ELSE CAST(target AS VARCHAR) END END, " "condition, window_from, window_to)), ?)", [str(search or "").strip().lower()]) - if category is not None: - where, filters = f"({where}) AND key = ?", [*filters, str(category)] folded = ", ".join(f"'{s}'" for s in FOLDED) total, folded_count = cur.execute( sql + f" SELECT COUNT(*), COUNT(*) FILTER (WHERE status IN ({folded})) FROM metrics WHERE {where}", @@ -538,110 +530,6 @@ def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=Non "cursor": str(start + size) if start + size < out["metricTotal"] else None} -def _listing_array(value): - """Decode a connector JSON array without inventing missing entries.""" - if isinstance(value, str): - try: - value = json.loads(value) - except (TypeError, ValueError): - return None - return value if isinstance(value, list) else None - - -def _related_listings(tenant_key, st, user, market): - """Read all permitted listing issue pages through the listing source's own wall.""" - from core import perm_scope - st = _rt(tenant_key, st) - if user is None: - return [], "Related listings need access to the listing source." - related, cursor, incomplete = [], None, False - try: - while True: - page = perm_scope.scoped_table_page(user, LISTING_SOURCE, st=st, cursor=cursor, limit=1024) - if "issues" not in {field.get("key") for field in page["fields"]}: - return [], "Listing issues are unavailable with your current source access." - for row in page["rows"]: - if market and row.get("marketplace_id") and str(row["marketplace_id"]) != market: - continue - issues = _listing_array(row.get("issues")) - if issues is None: - incomplete = True - continue - summaries = _listing_array(row.get("summaries")) or [] - summaries = [item for item in summaries if isinstance(item, dict)] - matching = [item for item in summaries if not market or item.get("marketplaceId") == market] - if market and summaries and not matching: - continue - asin = row.get("asin") or next((item.get("asin") for item in matching if item.get("asin")), None) - for issue in issues: - if not isinstance(issue, dict) or not (issue.get("message") or issue.get("code")): - incomplete = True - continue - related.append({"asin": asin, "sku": row.get("sku"), - "issue": issue.get("message") or issue["code"], - "severity": issue.get("severity")}) - after = page["nextCursor"] - if after is None: - break - if cursor is not None and int(after) <= int(cursor): - raise RuntimeError("The listing source cursor did not advance.") - cursor = after - except perm_scope.Denied: - return [], "Related listings are unavailable with your current source access." - except (perm_scope.UnknownTable, perm_scope.Unresolvable): - return [], "Related listing data is unavailable." - return related, "Some listing issues are missing or unreadable." if incomplete else None - - -def category_detail(tenant_key, key, *, user=None, allowed=None, st=None): - """Serve one permitted category and separate listing evidence without claiming violation records.""" - st = _rt(tenant_key, st) - category, _sep, _market = str(key).partition("@") - labels = dict(COUNT_METRICS) - if category not in labels: - raise KeyError(key) - kept, offset, total = [], 0, 0 - while True: - page = summary_page(tenant_key, category=category, metric=key if _sep else None, - offset=offset, limit=PAGE_SIZE, st=st) - total = page["metricTotal"] - kept.extend(row for row in page["metrics"] if allowed is None or allowed(row, page["snapshotDate"])) - if len(kept) > 1: - raise ValueError("Choose a marketplace with category@marketplace.") - if page["cursor"] is None: - break - offset = int(page["cursor"]) - if not kept and (total or allowed is not None or _sep): - raise KeyError(key) - row = kept[0] if kept else {} - note = CATEGORY_SOURCE_NOTE - if not row: - note += " This category is missing from the saved report." - if row and row.get("value") is None: - note += " The category count is missing." - if row and row.get("target") is None: - note += " Amazon did not provide a target." - rule = str(row.get("targetCondition") or "").upper() - known_rule = "LESS" in rule or "GREATER" in rule - if row.get("target") is not None and not known_rule: - note += " The target rule is missing or unsupported." - if row and (row.get("windowFrom") is None or row.get("windowTo") is None): - note += " The reporting window is missing or incomplete." - related, listing_note = _related_listings(tenant_key, st, user, row.get("marketplaceId")) if row else ([], None) - if listing_note: - note += " " + listing_note - query = urlencode({"marketplaceID": row["marketplaceId"]}) if row.get("marketplaceId") else "" - status = row.get("status", "No data") - if row.get("value") is not None and row.get("target") is not None and not known_rule: - status = "No target" - return {"key": category, "label": labels[category], "count": row.get("value"), - "status": status, "target": row.get("condition") if known_rule else None, - "window": {"from": row.get("windowFrom"), "to": row.get("windowTo")}, - "recordList": False, "sourceNote": note, - "sellerCentralUrl": "https://sellercentral.amazon.com/performance/dashboard" + ("?" + query if query else ""), - "related": related} - - # --------------------------------------------------------------------------- settings and owner #: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit). @@ -1031,7 +919,7 @@ def _title(row): return f"{row['label']} is {'at risk' if row['status'] == 'At risk' else 'near target'}" -def _alert_source(tenant_key, st, row, snapshot_date, *, extra_fields=()): +def _alert_source(tenant_key, st, row, snapshot_date): """Bind an alert to the actual mirror row supplying its metric value.""" from harness.amazon_schema import IDENTITY_PID column = row["key"] + ("_defects_count" if row["key"] in dict(COUNT_METRICS) else "_rate") @@ -1042,7 +930,6 @@ def _alert_source(tenant_key, st, row, snapshot_date, *, extra_fields=()): return {"source": None, "fields": fields} fields += [name for name in (row["key"] + "_target_value", row["key"] + "_target_condition") if name in cols] - fields += [name for name in extra_fields if name in cols and name not in fields] order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else "" row_ids = [] for field in (name for name in fields if name not in {"_snapshot_date", "marketplace_id"}): @@ -1058,14 +945,6 @@ def _alert_source(tenant_key, st, row, snapshot_date, *, extra_fields=()): "fields": fields} -def category_source(tenant_key, st, row, snapshot_date): - """Bind every category detail field to the report rows that supplied it.""" - key = row["key"] - return _alert_source(tenant_key, st, row, snapshot_date, extra_fields=( - key + "_status", key + "_reporting_date_range_reporting_date_from", - key + "_reporting_date_range_reporting_date_to")) - - def raise_alerts(tenant_key, view=None, *, st=None, now=None): """Raise one owned alert per metric that is near or past its target, never a second while one is open.""" st = _rt(tenant_key, st) @@ -1852,7 +1731,7 @@ def validate_dashboard(tenant_key, figures=None, *, start=None, end=None, st=Non return {"ok": bool(detail) and all(check["ok"] for check in detail), "detail": detail} def validate(tenant_key, *, st=None, **period): - """Tie the rating, rates and category detail counts to the raw account health snapshot.""" + """Tie the rating and each rate in `summary` to the raw `ut_amzn_account_health` snapshot.""" st = _rt(tenant_key, st) view = summary(tenant_key, st=st) with _reader(tenant_key, st) as cur: @@ -1863,33 +1742,17 @@ def validate(tenant_key, *, st=None, **period): date = cur.execute(f'SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {AH_TABLE}').fetchone()[0] \ if "_snapshot_date" in cols else None wanted = [("rating", "account_health_rating_ahr_score", 1.0)] + \ - [(p, f"{p}_rate", None) for p, _l, _d in RATE_METRICS] + \ - [(p, f"{p}_defects_count", 1.0) for p, _l in COUNT_METRICS] + [(WARNINGS[0], WARNINGS[2], 1.0)] + [(p, f"{p}_rate", None) for p, _l, _d in RATE_METRICS] for key, col, _s in wanted: if col not in cols: continue where = ' WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date is not None else "" order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else "" - q = (f'SELECT CAST("marketplace_id" AS VARCHAR), ' - f'FIRST(TRY_CAST(REPLACE(TRIM(CAST("{col}" AS VARCHAR)), \',\', \'\') AS DOUBLE){order}) ' + q = (f'SELECT CAST("marketplace_id" AS VARCHAR), FIRST(TRY_CAST("{col}" AS DOUBLE){order}) ' f'FILTER (WHERE "{col}" IS NOT NULL) ' f'FROM {AH_TABLE}{where} GROUP BY 1') for market, value in cur.execute(q, [date] if date is not None else []).fetchall(): independent[(key, str(market or ""))] = value - inbound_cols = _columns(cur, INBOUND_TABLE) - if inbound_cols: - column = next((name for name in ("issue_reported_date", "_period_start") if name in inbound_cols), None) - where, args = "", [] - if column: - try: - end = dt.date.fromisoformat(str(date)[:10]) - except (TypeError, ValueError): - end = _now().date() - where = (f' WHERE TRY_CAST(SUBSTR(CAST("{column}" AS VARCHAR), 1, 10) AS DATE) ' - f'> ? AND TRY_CAST(SUBSTR(CAST("{column}" AS VARCHAR), 1, 10) AS DATE) <= ?') - args = [end - dt.timedelta(days=INBOUND_DAYS), end] - independent[(INBOUND[0], "")] = cur.execute( - f"SELECT COUNT(*) FROM {INBOUND_TABLE}{where}", args).fetchone()[0] app = {} for m in view["marketplaces"]: app[("rating", m["marketplaceId"] or "")] = m["rating"] @@ -1898,8 +1761,6 @@ def validate(tenant_key, *, st=None, **period): # The shown percent ties back through its scale; a no orders row ties its raw rate (Amazon's 0). raw = row["value"] / row.get("scale", 100.0) if row["value"] is not None else row.get("raw") app[(row["key"], row["marketplaceId"] or "")] = raw - else: - app[(row["key"], row["marketplaceId"] or "")] = row["value"] detail, app_total, ind_total = [], 0.0, 0.0 for key in sorted(set(app) | set(independent)): a, b = app.get(key), independent.get(key) @@ -1909,24 +1770,9 @@ def validate(tenant_key, *, st=None, **period): detail.append({"metric": key[0], "marketplaceId": key[1] or None, "app": a, "independent": b, "ok": tied}) ok = bool(detail) and all(d["ok"] for d in detail) and str(view.get("snapshotDate")) == str(date) groups = _validate_groups(tenant_key, st, view) - categories = [] - for key, market in sorted(set(app) | set(independent)): - if key not in dict(COUNT_METRICS): - continue - try: - shown = category_detail(tenant_key, key + ("@" + market if market else ""), st=st)["count"] - source = independent.get((key, market)) - tied = (shown is None and source is None) or (shown is not None and source is not None - and abs(shown - source) <= 1e-7) - except KeyError: - shown, source = None, independent.get((key, market)) - tied = (key, market) not in app and source is None - categories.append({"metric": key, "marketplaceId": market or None, - "app": shown, "independent": source, "ok": tied}) - category_check = {"ok": all(row["ok"] for row in categories), "detail": categories} - return {"ok": ok and reserves["ok"] and groups["ok"] and category_check["ok"], "app_total": round(app_total, 9), + return {"ok": ok and reserves["ok"] and groups["ok"], "app_total": round(app_total, 9), "independent_total": round(ind_total, 9), "detail": detail, "snapshotDate": date, - "message": None if detail else EMPTY, "reserves": reserves, "groups": groups, "categories": category_check} + "message": None if detail else EMPTY, "reserves": reserves, "groups": groups} def _validate_groups(tenant_key, st, view): diff --git a/platform/modules/amazon_profit.py b/platform/modules/amazon_profit.py index a4fb9a4e7f8436ae9ae75cf53316310af55a46d6..d733d83f6e86d60b38cc00f3e379b89ba8a41aa9 100644 --- a/platform/modules/amazon_profit.py +++ b/platform/modules/amazon_profit.py @@ -10,9 +10,9 @@ survive every period switch. `ut_amazon_asin_profit` and `ut_amazon_parent_profit` roll the SKU rows up on every refresh. Projects retain their own verified period totals and Impact in the tenant projects store. -`validate()` ties each net proceeds component to independent Amazon amounts within 2% over settled -days, with order charges on purchase dates and monthly storage in its service month. -A refresh runs it for the last complete month and writes no figures when it +`validate()` ties a month's net proceeds to Amazon settlements within 2% over the days its issued +settlements cover, excluding loan repayments and withheld tax, and proves a negative control that +drops fees goes red. A refresh runs it for the last complete month and writes no figures when it fails. A passing refresh stores the verified range and the day the rest is due to settle. Owned items (R10, R31): a refresh opens one item on a Losing or Thin SKU with no status, for the @@ -615,16 +615,11 @@ def reported_net_proceeds(con, start, end): def economics_currency(con): if "currency_code" not in _columns(con, ECONOMICS): return None - currency, count, missing = con.execute( - f"SELECT max(currency_code), count(DISTINCT currency_code), " - f"count(*) FILTER (WHERE NULLIF(trim(currency_code), '') IS NULL) FROM {ECONOMICS}").fetchone() - if count > 1 or missing: - raise SourceUnavailable("Amazon economics needs one known currency.") - return currency + return con.execute(f"SELECT any_value(currency_code) FROM {ECONOMICS}").fetchone()[0] def settlements_total(con, start, end, currency=None): - """Sum posted settlements with both collected and withheld sales tax included. + """Settlements posted in the window, excluding loan repayment and withheld tax. `currency` keeps settlements whose header row names it: Amazon.ca and Amazon.com.mx settle in their own currency, outside the economics marketplace. Their reports write DD.MM.YYYY. @@ -634,188 +629,18 @@ def settlements_total(con, start, end, currency=None): stream = ("JOIN (SELECT settlement_id FROM {t} WHERE currency IS NOT NULL GROUP BY settlement_id " "HAVING any_value(currency) = ?) h USING (settlement_id)").format(t=SETTLEMENTS) if scoped else "" total, rows, undated = con.execute( - "SELECT COALESCE(SUM(TRY_CAST(amount AS DECIMAL(38, 2))) " - "FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), 0), " + "SELECT COALESCE(SUM(amount) FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), 0), " "count(*) FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), " "count(*) FILTER (WHERE d IS NULL AND posted_date IS NOT NULL) " "FROM (SELECT amount, posted_date, COALESCE(TRY_CAST(left(posted_date, 10) AS DATE), " "CAST(try_strptime(left(posted_date, 10), '%d.%m.%Y') AS DATE)) AS d " - f"FROM {SETTLEMENTS} s {stream} WHERE COALESCE(transaction_type, '') <> ?)", - [start, end, start, end] + ([currency] if scoped else []) + [LOAN_REPAYMENT]).fetchone() + f"FROM {SETTLEMENTS} s {stream} WHERE COALESCE(transaction_type, '') <> ? " + "AND COALESCE(amount_type, '') <> ?)", + [start, end, start, end] + ([currency] if scoped else []) + [LOAN_REPAYMENT, WITHHELD_TAX]).fetchone() return {"total": float(total), "rows": int(rows), "undated": int(undated), "currency": currency if scoped else None} -def settlement_components(con, start, end, currency=None): - """Tie purchase-date order charges, service-month storage and daily ads to independent sources.""" - month = dt.date.fromisoformat(start).strftime("%B %Y") - parts, problems = [], [] - - def problem(component, reason): - problems.append(f"Not verified: {component} in {month}: {reason}.") - - def tie(component, ours, source, *, reason=None): - ours = float(ours) - source = None if source is None else float(source) - ok, gap = _tie(ours, source) if source is not None else (False, math.inf) - basis = {"product sales": "order purchase date", "order fees": "order purchase date", - "monthly storage": "storage service month", "ads": "daily ad report date"} - parts.append({"component": component, "month": start[:7], "ok": ok and not reason, - "a": ours, "b": source, "gap": gap, "tolerance": SETTLEMENT_TOLERANCE, - "dateBasis": basis.get(component, "order purchase date when present, otherwise posted date"), - "sourceTables": ([ECONOMICS] + [f"amzn_ads_{kind}_advertised_products" for kind in ("sp", "sb", "sd")] - if component == "ads" else [ECONOMICS, SETTLEMENTS, "amzn_orders"])}) - if reason: - problem(component, reason) - elif not ok: - problem(component, f"independent Amazon amounts differ by {gap:.1%}") - - sc = _columns(con, SETTLEMENTS) - ec = _columns(con, ECONOMICS) - required = set(_SETTLEMENT_REQUIRED) | {"settlement_id", "currency", "order_id", "amount_description"} - if not required <= sc: - problem("settlement components", "missing source columns " + ", ".join(sorted(required - sc))) - return {"total": None, "parts": parts, "reason": problems[0], "rows": 0} - if currency is None: - problem("settlement components", "economics currency unavailable") - return {"total": None, "parts": parts, "reason": problems[0], "rows": 0} - if not {"amazon_order_id", "purchase_date"} <= _columns(con, "amzn_orders"): - problem("order charges", "missing amzn_orders purchase dates") - return {"total": None, "parts": parts, "reason": problems[0], "rows": 0} - unscoped = con.execute(f"""SELECT count(*) FROM {SETTLEMENTS} s - LEFT JOIN (SELECT settlement_id FROM {SETTLEMENTS} GROUP BY settlement_id - HAVING count(DISTINCT currency) = 1) h USING (settlement_id) - WHERE h.settlement_id IS NULL AND (s.amount IS NOT NULL OR NULLIF(s.amount_type, '') IS NOT NULL) - AND (CAST({_moment_sql('posted_date')} AS DATE) BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) - OR {_moment_sql('posted_date')} IS NULL)""", [start, end]).fetchone()[0] - if unscoped: - problem("settlement currency", "missing or conflicting settlement header currency") - # Aggregate in SQL before reading: duplicate order lines must not multiply settlement amounts. - cte = f"""WITH orders AS ( - SELECT amazon_order_id, min(CAST({_moment_sql('purchase_date')} AS DATE)) AS bought, - count(DISTINCT CAST({_moment_sql('purchase_date')} AS DATE)) AS dates, - count(*) FILTER (WHERE {_moment_sql('purchase_date')} IS NULL) AS bad_dates - FROM amzn_orders GROUP BY amazon_order_id), - streams AS (SELECT settlement_id FROM {SETTLEMENTS} GROUP BY settlement_id - HAVING count(DISTINCT currency) = 1 AND max(currency) = ?), - lines AS (SELECT s.*, TRY_CAST(amount AS DECIMAL(38, 2)) AS currency_amount, - CAST({_moment_sql('s.posted_date')} AS DATE) AS posted, - o.bought, o.dates, o.bad_dates, - lower(regexp_replace(COALESCE(amount_description, ''), '[^a-zA-Z0-9]', '', 'g')) AS description, - lower(regexp_replace(COALESCE(amount_type, ''), '[^a-zA-Z0-9]', '', 'g')) AS kind, - lower(regexp_replace(COALESCE(transaction_type, ''), '[^a-zA-Z0-9]', '', 'g')) AS transaction - FROM {SETTLEMENTS} s JOIN streams USING (settlement_id) - LEFT JOIN orders o ON trim(s.order_id) = o.amazon_order_id - WHERE (amount IS NOT NULL OR NULLIF(amount_type, '') IS NOT NULL) - AND COALESCE(transaction_type, '') <> ?), - classified AS (SELECT *, CASE - WHEN concat(kind, description, transaction) LIKE '%inbounddefect%' THEN 'Inbound Defect Fee' - WHEN concat(kind, description, transaction) LIKE '%mcf%' THEN 'MCF credits' - WHEN kind = 'itemwithheldtax' OR description IN ('tax', 'shippingtax', 'giftwraptax') - OR description LIKE 'marketplacefacilitatortax%' THEN 'tax' - WHEN (kind = 'itemprice' AND (description IN ('shipping', 'giftwrap') - OR description LIKE '%promotion%')) OR kind IN ('promotion', 'shipping', 'shippingfees') - THEN 'shipping and promotions' - WHEN 'costofadvertising' IN (kind, description, transaction) - OR description IN ('advertising', 'advertisingfee') THEN 'ads billing' - WHEN description IN ('storagefee', 'fbamonthlystoragefee', 'monthlystoragefee') - OR 'fbainventorystoragefee' IN (kind, description, transaction) THEN 'monthly storage' - WHEN concat(kind, description, transaction) LIKE '%reimbursement%' THEN 'reimbursements' - WHEN description IN ('subscription', 'subscriptionfee', 'fbaremovalfee', 'fbadisposalfee', - 'fbainboundtransportationfee', 'couponredemptionfee', 'dealfee') - OR kind IN ('dealfee', 'lightningdealfee', 'dealparticipationfee', 'dealperformancefee', - 'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee', - 'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee') - OR description IN ('lightningdealfee', 'dealparticipationfee', 'dealperformancefee', - 'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee', - 'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee') - OR transaction IN ('dealfee', 'lightningdealfee', 'dealparticipationfee', 'dealperformancefee', - 'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee', - 'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee') THEN 'account fees' - WHEN NULLIF(trim(order_id), '') IS NOT NULL AND kind = 'itemprice' - AND description = 'principal' THEN 'product sales' - WHEN NULLIF(trim(order_id), '') IS NOT NULL AND kind IN ('itemfee', 'itemfees') THEN 'order fees' - ELSE 'unmatched component' END AS component FROM lines), - dated AS (SELECT *, CASE - WHEN component = 'monthly storage' THEN CAST(date_trunc('month', posted) - INTERVAL '1 month' AS DATE) - WHEN NULLIF(trim(order_id), '') IS NOT NULL THEN bought - ELSE posted END AS basis FROM classified) - """ - groups = con.execute(cte + """SELECT component, COALESCE(SUM(currency_amount), 0), count(*), - count(*) FILTER (WHERE currency_amount IS NULL OR posted IS NULL), - count(*) FILTER (WHERE NULLIF(trim(order_id), '') IS NOT NULL - AND (bought IS NULL OR dates <> 1 OR bad_dates <> 0)) - FROM dated WHERE basis BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) - OR ((basis IS NULL OR dates <> 1 OR bad_dates <> 0) - AND (posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL)) - GROUP BY component ORDER BY component""", - [currency, LOAN_REPAYMENT, start, end, start, end]).fetchall() - amounts = {name: total for name, total, *_ in groups} - if not groups: - problem("settlement components", "no settlement source rows on the purchase or service dates") - for name, _total, _n, bad, unmatched in groups: - if bad or unmatched: - problem(name, "unmatched purchase date, unreadable posting date or unavailable amount") - if name in ("unmatched component", "Inbound Defect Fee", "MCF credits"): - problem(name, "settlement category has no economics mapping") - fee_columns = sorted(c for c in ec if c.startswith("fee_") and c.endswith("_amount") and "_charge_" not in c) - storage_columns = [c for c in fee_columns if "storage" in c] - reimbursement_columns = [c for c in fee_columns if "reimbursement" in c] - account_columns = [c for c in fee_columns if any(name in c for name in - ("subscription", "removal", "disposal", "inbound_transportation", "inbound_placement", - "coupon", "deal_", "customer_return", "return_processing", "returns_processing"))] - # Storage fee columns are accruals in economics; only monthly settlement charges use the prior month. - sums = ["sales_net_product_sales_amount", "fees_total_amount", *ads_columns(ec), - *storage_columns, *reimbursement_columns, *account_columns] - observed = con.execute("SELECT " + ", ".join(f'COALESCE(SUM(TRY_CAST("{c}" AS DECIMAL(38, 2))), 0)' for c in sums) - + ", count(*) FILTER (WHERE " - + " OR ".join(f'("{c}" IS NOT NULL AND TRY_CAST("{c}" AS DECIMAL(38, 2)) IS NULL)' - for c in sums) + ")" - + f" FROM {ECONOMICS} WHERE left(start_date, 10) BETWEEN ? AND ?", - [start, end]).fetchone() - if observed[-1]: - problem("settlement components", "unavailable economics amount") - values = dict(zip(sums, observed[:-1])) - sales, fees = values[sums[0]], values[sums[1]] - storage = sum(values[c] for c in storage_columns) - reimbursements = sum(values[c] for c in reimbursement_columns) - account_fees = sum(values[c] for c in account_columns) - ads = sum(values[c] for c in ads_columns(ec)) - tie("product sales", sales, amounts.get("product sales", 0.0)) - tie("order fees", fees - storage - reimbursements - account_fees, -amounts.get("order fees", 0.0)) - tie("monthly storage", storage, -amounts.get("monthly storage", 0.0)) - tie("reimbursements", reimbursements, -amounts.get("reimbursements", 0.0)) - tie("account fees", account_fees, -amounts.get("account fees", 0.0)) - tie("sales tax", 0.0, amounts.get("tax", 0.0)) - # Shipping and promotions lie outside net product sales; ad invoices use a separate billing period. - if ads or "ads billing" in amounts: - ad_total = 0 - for kind in ("sp", "sb", "sd"): - table = f"amzn_ads_{kind}_advertised_products" - if not {"date", "cost"} <= _columns(con, table): - problem("ads", f"missing source {table}.cost") - ad_total = None - break - total, count, present = con.execute( - f"SELECT SUM(TRY_CAST(cost AS DECIMAL(38, 2))), count(*), " - f"count(TRY_CAST(cost AS DECIMAL(38, 2))) FROM {table} " - "WHERE left(CAST(date AS VARCHAR), 10) BETWEEN ? AND ?", [start, end]).fetchone() - if not count or count != present or total is None or not math.isfinite(float(total)): - problem("ads", f"unavailable daily amounts in {table}") - ad_total = None - break - ad_total += total - tie("ads", ads, ad_total) - else: - ad_total = 0 - target = (float(sum(amounts.get(name, 0) for name in - ("product sales", "order fees", "monthly storage", "reimbursements", "account fees"))) - - float(ad_total)) if ad_total is not None else None - return {"total": target, "parts": parts, "reason": problems[0] if problems else None, - "rows": sum(row[2] for row in groups), - "excluded": {k: float(amounts[k]) for k in ("shipping and promotions", "ads billing") if k in amounts}} - - _COVERAGE_REQUIRED = ("settlement_id", "total_amount", "settlement_start_date", "settlement_end_date") @@ -2163,8 +1988,6 @@ def refusal(checks, month): ) for index, check in enumerate(checks): if not check.get("ok"): - if index == 0 and check.get("reason"): - return check["reason"] if index < len(sentences): return sentences[index](check) return f"Not verified: the Amazon figures do not tie in {label}." @@ -2862,9 +2685,7 @@ def _mirror_has_economics(con): return False -def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders", - "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products", - "amzn_ads_sd_advertised_products")): +def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS)): """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back.""" key = (tenant, tuple(tables)) if key in _SNAPSHOT: @@ -2999,7 +2820,7 @@ def typed_cogs_checks(con, computed, typed, usage=None, bounds=None): def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None, typed_values=None, typed_period=None): - """Tie net proceeds components to independent Amazon sources within 2% over settled days. + """validate.py convention: net proceeds tie to settlements within 2% over the month's settled days. Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.""" month = month or last_complete_month() bounds = period_bounds(month) @@ -3008,35 +2829,25 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No if con is None: con, source = validation_cursor() checks = [] - try: - _require(con, ECONOMICS, _ECONOMICS_REQUIRED) - currency = economics_currency(con) - span = settlement_coverage(con, start, end, currency) - settled = settlements_total(con, start, end, currency) - except SourceUnavailable as exc: - label = dt.date.fromisoformat(start).strftime("%B %Y") - raise SourceUnavailable(f"Not verified: settlement components in {label}: {exc}") from None + currency = economics_currency(con) + span = settlement_coverage(con, start, end, currency) + # Both sides tie the same covered days: economics by start_date, settlement lines by posted_date. lo, hi = span["start"] or start, span["end"] or end proceeds = net_proceeds_total(con, lo, hi) - aligned = settlement_components(con, lo, hi, currency) + settled = settlements_total(con, lo, hi, currency) if span["start"] is None: - aligned = dict(aligned, total=None, rows=0, - reason=f"Not verified: settlement components in {bounds['label']}: no settled days.") - target = aligned["total"] - ok, gap = _tie(proceeds, target) if target is not None else (False, math.inf) + settled = dict(settled, total=0.0, rows=0) + ok, gap = _tie(proceeds, settled["total"]) covered = {k: span[k] for k in ("start", "end", "label", "partial", "clear", "sentence")} checks.append({"check": f"amazon_profit {month}: net proceeds tie to settlements within 2% over " f"{span['label'] or 'no settled days'} ({gap:.2%} apart, {source})", - "ok": ok and not aligned["reason"], "a": round(proceeds, 2), - "b": None if target is None else round(target, 2), "reason": aligned["reason"], - "components": aligned["parts"], "excluded": aligned.get("excluded", {}), - "gap": round(gap, 4), "covered": covered, "tolerance": SETTLEMENT_TOLERANCE}) + "ok": ok, "a": round(proceeds, 2), "b": round(settled["total"], 2), + "gap": round(gap, 4), "covered": covered}) dropped = net_proceeds_total(con, lo, hi, drop_fees=True) - control_ok, control_gap = _tie(dropped, target) if target is not None else (False, math.inf) + control_ok, control_gap = _tie(dropped, settled["total"]) checks.append({"check": f"amazon_profit {month}: NC net proceeds without fees miss settlements " f"({control_gap:.2%} apart)", - "ok": not control_ok, "a": round(dropped, 2), - "b": None if target is None else round(target, 2), + "ok": not control_ok, "a": round(dropped, 2), "b": round(settled["total"], 2), "gap": round(control_gap, 4)}) checks.append({"check": f"amazon_profit {month}: every {settled['currency'] or 'settlement'} " "settlement row carries a readable date", diff --git a/platform/modules/amazon_profit_account.py b/platform/modules/amazon_profit_account.py index d42a19ca1622ba1d3f7573adf878b1e17f56b321..31ac978dcd2b56791976d7e44b619287c450e3aa 100644 --- a/platform/modules/amazon_profit_account.py +++ b/platform/modules/amazon_profit_account.py @@ -487,11 +487,9 @@ def validate(con, view): span = profit.settlement_coverage(con, start, end, currency) # The tie reads only the days issued settlements cover, on both sides. lo, hi = span["start"] or start, span["end"] or end - settled = profit.settlement_components(con, lo, hi, currency) - why = settled["reason"] + settled = profit.settlements_total(con, lo, hi, currency) if span["start"] is None: settled = dict(settled, rows=0) - why = f"Not verified: settlement components in {dt.date.fromisoformat(start):%B %Y}: no settled days." elif (lo, hi) != (start, end) and bridge is not None: proceeds = profit.net_proceeds_total(con, lo, hi) except profit.SourceUnavailable as exc: @@ -503,14 +501,12 @@ def validate(con, view): gap = gap if gap is None or math.isfinite(gap) else None covered = {k: span[k] for k in ("start", "end", "label", "partial", "clear", "sentence")} if span else None checks.append({"check": "Net proceeds tie to settlements within 2%", - "ok": tied and not why, + "ok": tied and not why and not settled["undated"], "a": proceeds, "b": target, "gap": gap, "tolerance": profit.SETTLEMENT_TOLERANCE, - "reason": why or ("No settlement source rows for window" if target is None else None), - "components": [{**part, "gap": part["gap"] if math.isfinite(part["gap"]) else None} - for part in settled["parts"]] if settled else [], - "excluded": settled.get("excluded", {}) if settled else {}, + "reason": why or ("No settlement source rows for window" if target is None else + "Settlement dates unavailable" if settled["undated"] else None), "covered": covered, "note": covered and covered["sentence"], - "sourceTables": [ECONOMICS, profit.SETTLEMENTS, "amzn_orders", *ADS_TABLES]}) + "sourceTables": [ECONOMICS, profit.SETTLEMENTS]}) values = [_cents(s["netProfit"]["value"]) for s in view["skus"]] sku_net = sum(values) / 100 if values and all(c is not None for c in values) else None check("Account net profit equals sum of SKU profit after COGS", metrics["netProfit"]["value"], sku_net, diff --git a/platform/modules/procurement.py b/platform/modules/procurement.py index 18269c67201e1ae5ca0c02a51daf031d6b9e98fc..46b1464b2ed257646789d4a5f9ebccf70e4564be 100644 --- a/platform/modules/procurement.py +++ b/platform/modules/procurement.py @@ -20,7 +20,6 @@ READ-ONLY on Odoo; the team's Order Qty + master-data edits persist in the platf """ import sys import json -import hashlib import math import calendar import datetime as dt @@ -154,22 +153,6 @@ def _preorders(cutoff_iso): return qty, need -def _fba_refusal(product_id, quantity, report): - """Classify unknown FBA stock without treating missing or invalid quantities as zero.""" - if not product_id: - return {'cause_code': 'no_product_id', 'cause': 'no product id for this SKU'} - if report.get('available') is False: - cause_code = ('non_numeric_fba' if report.get('cause_code') == 'non_numeric_fba' - else 'report_unavailable') - return {'cause_code': cause_code, - 'cause': report.get('cause') or 'FBA stock report unavailable'} - if (not isinstance(quantity, (int, float)) or isinstance(quantity, bool) - or not math.isfinite(quantity)): - return {'cause_code': 'non_numeric_fba', - 'cause': 'FBA quantity is missing or is not a finite number for this product'} - return None - - def recommendations(t=None): """Base per-SKU table (curated master data + live Odoo demand/stock/PO + alt-vendors). The UI overlays the team's vendor/country/cost edits on top; reorder methods + CBM/container are here.""" @@ -208,13 +191,13 @@ def recommendations(t=None): try: import modules.warehouse as warehouse fba_by_pid, fba_report = warehouse.fba_qty_by_product_id( - [p.get('id') for p in by_code.values()]) + [p['id'] for p in by_code.values()]) except Exception as exc: fba_by_pid, fba_report = None, { 'available': False, 'cause': 'the FBA warehouse read failed: ' + type(exc).__name__} rows = [] for code, p in by_code.items(): - pid = p.get('id') + pid = p['id'] meta = sup.get(code, {}) lead = meta.get('lead') or 0 tmpl = O.m2o_id(p.get('product_tmpl_id')) @@ -228,8 +211,7 @@ def recommendations(t=None): on_hand = p.get('qty_available') or 0.0 po = inpo.get(pid, 0.0) fba = (fba_by_pid or {}).get(pid) - fba_refusal = _fba_refusal(pid, fba, fba_report) - fba_known = fba_refusal is None + fba_known = isinstance(fba, (int, float)) and fba_report.get('available') is not False avail = on_hand - fba + po if fba_known else None days_cover = ((avail / avg_day) if avg_day > 0 else (None if avail <= 0 else 9999.0)) if avail is not None else None @@ -263,7 +245,8 @@ def recommendations(t=None): }) if not fba_known: rows[-1]['_coverage_report'] = {**fba_report, 'available': False, - **fba_refusal} + 'cause': fba_report.get('cause') + or 'FBA stock unavailable for this product'} return rows @@ -666,13 +649,13 @@ def validate_coverage(rows, t=None, oracle=None, demand=None): unavailable, wrong_days, wrong_buy = [], [], [] for r in rows: pid = r.get('_product_id') - report = r.get('_coverage_report', {}) - refusal = _fba_refusal(pid, r.get('_fba'), report) + known = (pid is not None and isinstance(r.get('_fba'), (int, float)) + and r.get('_coverage_report', {}).get('available') is not False) expected_days, expected_buy = None, None - if refusal is not None: + if not known: unavailable.append({'code': r['code'], - 'sku_ref': hashlib.sha256(r['code'].encode('utf-8')).hexdigest(), - **refusal}) + 'cause': r.get('_coverage_report', {}).get('cause') + or 'FBA stock unavailable for this product'}) else: effective = stock['non_fba_by_pid'].get(pid, 0.0) + float(r.get('incoming_po') or 0.0) daily = demand.get(pid, 0.0) / window_days diff --git a/platform/modules/returns_refunds.py b/platform/modules/returns_refunds.py index d22de26b16bd7ef980a7d1a7370821e29e188be7..d790012385aef56cf61d299b7274a953ecce7ac1 100644 --- a/platform/modules/returns_refunds.py +++ b/platform/modules/returns_refunds.py @@ -1987,33 +1987,6 @@ def _return_source(cur, table, date, quantity, sku="sku", reason="reason"): + (f" FROM {table}" if cols else " WHERE false") + ") captured") -def _refund_overlap_relations(): - """Match wildcard identities through distinct projections with at most eight probes per identity.""" - fields = ('order_id', 'sku', 'settlement_id') - mask = '+'.join(f'CASE WHEN {field} IS NULL THEN 0 ELSE {1 << i} END' - for i, field in enumerate(fields)) - projection = ','.join(f"CASE WHEN (p.mask & {1 << i})!=0 THEN {field} ELSE '' END AS {field}" - for i, field in enumerate(fields)) - matches = ' AND '.join( - f"b.{field}=CASE WHEN (a.mask & m.mask & {1 << i})!=0 THEN a.{field} ELSE '' END" - for i, field in enumerate(fields)) - identities = ' AND '.join(f'a.{field} IS NOT DISTINCT FROM o.{field}' for field in fields) - return f"""refund_identities AS ( - SELECT DISTINCT source,account,order_id,sku,settlement_id,{mask} AS mask FROM refund_candidates), - refund_overlap_keys AS ( - SELECT DISTINCT source,account,i.mask AS identity_mask,p.mask AS projection_mask,{projection} - FROM refund_identities i CROSS JOIN range(8) p(mask) WHERE (i.mask & p.mask)=p.mask), - refund_overlaps AS ( - SELECT a.source,a.account,a.order_id,a.sku,a.settlement_id,bool_or(b.source IS NOT NULL) AS overlap - FROM refund_identities a CROSS JOIN range(8) m(mask) - LEFT JOIN refund_overlap_keys b ON a.source!=b.source AND a.account=b.account - AND b.identity_mask=m.mask AND b.projection_mask=(a.mask & m.mask) AND {matches} - GROUP BY a.source,a.account,a.order_id,a.sku,a.settlement_id), - refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key,o.overlap - FROM refund_candidates a JOIN refund_overlaps o ON a.source=o.source AND a.account=o.account - AND {identities})""" - - def _refund_relations(cur): """Keep source identities, capture deduplication and ambiguous overlap inside SQL.""" sc, tc = (_columns(cur, table) for table in FINANCE_TABLES) @@ -2073,20 +2046,24 @@ def _refund_relations(cur): SELECT 'finance',account,settlement_id,event_id,item_id,order_id,sku,asin,day,NULL,money,currency, conflict,event_id IS NULL OR position IS NULL OR NOT isfinite(position) OR position<0 OR position!=trunc(position) OR sku IS NULL AND asin IS NULL, true FROM finance_events), - {_refund_overlap_relations()}""" + refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key, + EXISTS(SELECT 1 FROM refund_candidates b WHERE a.source!=b.source AND a.account=b.account + AND (a.order_id=b.order_id OR a.order_id IS NULL OR b.order_id IS NULL) + AND (a.sku=b.sku OR a.sku IS NULL OR b.sku IS NULL) + AND (a.settlement_id=b.settlement_id OR a.settlement_id IS NULL OR b.settlement_id IS NULL)) AS overlap + FROM refund_candidates a)""" -def _refund_totals_sql(first, last, group=False, *, windows=False): +def _refund_totals_sql(first, last, group=False): """Aggregate only event-grain refunds after source capture deduplication.""" - return ("SELECT " + ("window_id," if windows else "key," if group else "") + "COALESCE(sum(units),0) AS refunded," + return ("SELECT " + ("key," if group else "") + "COALESCE(sum(units),0) AS refunded," "sum(money) AS refund_cost,min(currency) AS refund_currency,count(DISTINCT currency) AS currencies," "count(*) FILTER (WHERE currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') " "OR money IS NULL OR NOT isfinite(money) OR money<0) AS money_bad," "count(*) FILTER (WHERE identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad," "count(*) FILTER (WHERE unit_bad) AS unit_bad,count(*) FILTER (WHERE source='finance') AS finance_units," "count(*) FILTER (WHERE overlap) AS overlaps FROM refund_events " - + ("JOIN windows ON day BETWEEN first AND last OR day IS NULL GROUP BY window_id" if windows else - f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL" + (" GROUP BY key" if group else ""))) + f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL" + (" GROUP BY key" if group else "")) def _rate_values(row, ready): @@ -3896,51 +3873,25 @@ def _dashboard_windows(first, last): def _dashboard_rates(cur, first, last, independent): - return _dashboard_rate_windows(cur, [(first, last)], independent)[0] - - -def _dashboard_rate_windows(cur, windows, independent=False, *, ready=None): - """Fold total, comparison and monthly windows in one query over complete source captures.""" - bounds = ','.join(f"({i},DATE '{a.isoformat()}',DATE '{b.isoformat()}')" for i, (a, b) in enumerate(windows)) + if not independent: + return _aggregate(cur, _relations(cur, first, last, {'risingRatio': 1.5, 'risingMinPct': 5})) + # Independent scalar folds avoid the serving membership and formatter projections. merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku' physical = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source( cur, MFN_TABLE, 'return_request_date', 'return_quantity', merchant_sku, 'return_reason') shipments = _return_source(cur, SHIPPED_TABLE, 'shipment_date', 'quantity') - prefix = f"""WITH windows(window_id,first,last) AS (VALUES {bounds}), - returns AS ({physical}),shipments AS ({shipments}),{_refund_relations(cur)},""" - if independent: - # Separate scalar source folds do not reuse the serving aggregate or formatter. - sql = prefix + """r AS (SELECT window_id,COALESCE(sum(units),0) AS returned - FROM windows LEFT JOIN returns ON day>=first AND day<=last GROUP BY window_id), - s AS (SELECT window_id,COALESCE(sum(units),0) AS shipped - FROM windows LEFT JOIN shipments ON day>=first AND day<=last GROUP BY window_id), - f AS (SELECT window_id,COALESCE(sum(units),0) AS refunded,sum(money) AS refund_cost, + operands = _mirror_one(cur, f"""WITH returns AS ({physical}),shipments AS ({shipments}), + {_refund_relations(cur)} SELECT * FROM + (SELECT COALESCE(sum(units),0) AS returned FROM returns WHERE day BETWEEN ? AND ?) r CROSS JOIN + (SELECT COALESCE(sum(units),0) AS shipped FROM shipments WHERE day BETWEEN ? AND ?) s CROSS JOIN + (SELECT COALESCE(sum(units),0) AS refunded,sum(money) AS refund_cost, min(currency) AS refund_currency,count(DISTINCT currency) AS currencies, count(*) FILTER (WHERE money IS NULL OR money<0 OR NOT isfinite(money) - OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}')) AS money_bad, + OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{{3}}')) AS money_bad, bool_or(identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad, - bool_or(unit_bad) AS unit_bad FROM refund_events JOIN windows - ON day>=first AND day<=last OR day IS NULL GROUP BY window_id) - SELECT r.*,s.shipped,f.* EXCLUDE(window_id) FROM r JOIN s USING(window_id) - LEFT JOIN f USING(window_id) ORDER BY window_id""" - else: - sql = prefix + """events AS ( - SELECT day,units AS returned,0.0 AS shipped FROM returns - UNION ALL SELECT day,0.0,units FROM shipments), - quantities AS (SELECT window_id,COALESCE(sum(returned),0) AS returned, - COALESCE(sum(shipped),0) AS shipped FROM windows LEFT JOIN events - ON day BETWEEN first AND last GROUP BY window_id), - refunds AS (""" + _refund_totals_sql(None, None, windows=True) + """) - SELECT quantities.*,refunds.* EXCLUDE(window_id) FROM quantities - LEFT JOIN refunds USING(window_id) ORDER BY window_id""" - ready = _rate_availability(cur) if ready is None else ready - result = cur.execute(sql) - names = [field[0] for field in result.description] - rows = [] - while batch := result.fetchmany(MAX_PAGE_SIZE): - rows.extend(dict(zip(names, row)) for row in batch) - formatter = _independent_rate_values if independent else _rate_values - return [formatter(row, ready) for row in rows] + bool_or(unit_bad) AS unit_bad FROM refund_events + WHERE day>=? AND day<=? OR day IS NULL) f""", [first, last, first, last, first, last]) + return _independent_rate_values(operands, _rate_availability(cur)) def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independent=False): @@ -3950,12 +3901,12 @@ def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independen total, previous, points, reasons, reasons_value = {}, {}, [], [], None if allowed: with _screen_reader(tenant_key, st) as (cur, _present): + total = _dashboard_rates(cur, first, last, independent) + previous = _dashboard_rates(cur, first - (last - first) - dt.timedelta(days=1), + first - dt.timedelta(days=1), independent) + points = [(a.isoformat(), _dashboard_rates(cur, a, b, independent)) + for a, b in _dashboard_windows(first, last)] ready = _rate_availability(cur) - months = list(_dashboard_windows(first, last)) - windows = [(first, last), (first - (last - first) - dt.timedelta(days=1), - first - dt.timedelta(days=1)), *months] - total, previous, *monthly = _dashboard_rate_windows(cur, windows, independent, ready=ready) - points = [(a.isoformat(), values) for (a, _), values in zip(months, monthly)] if ready['returns'] and ready['merchantReturns']: merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku' source = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source( @@ -4035,19 +3986,19 @@ def validate_figure(tenant_key, figure, *, start, end, st=None, user=None): series += [dict(label=reason_label(reason), value=units) for reason, units in batch] value, unit, prior = sum(row['value'] for row in series), 'returns', None else: - months = list(_dashboard_windows(first, last)) - windows = [(first, last), (first - (last - first) - dt.timedelta(days=1), - first - dt.timedelta(days=1)), *months] - total, previous, *monthly = _dashboard_rate_windows(cur, windows, True) + total = _dashboard_rates(cur, first, last, True) value = total[key] unit = total['currency'] if key == 'refundCost' else '%' if value is None or not unit: return failed + previous = _dashboard_rates(cur, first - (last - first) - dt.timedelta(days=1), + first - dt.timedelta(days=1), True) prior = previous[key] if key == 'refundCost' and previous['currency'] != unit: prior = None series = [] - for (begin, _), row in zip(months, monthly): + for begin, stop in _dashboard_windows(first, last): + row = _dashboard_rates(cur, begin, stop, True) if key == 'refundCost' and row[key] is not None and row['currency'] != unit: return failed series.append(dict(x=begin.isoformat(), y=row[key])) diff --git a/platform/modules/warehouse.py b/platform/modules/warehouse.py index fbe064439f830746bd21c6c97dd6da3841b0d7c6..9c41985b6005903b30d2516f572ad700ab288198 100644 --- a/platform/modules/warehouse.py +++ b/platform/modules/warehouse.py @@ -13,7 +13,6 @@ reconciles row-built aggregates to independent read_group/search_count paths. """ import sys import datetime as dt -import math from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import core.odoo as O @@ -793,14 +792,9 @@ def fba_qty_by_product_id(product_ids=None, wh=None): for i in range(0, len(ids), _FBA_BATCH): batches += 1 chunk = ids[i:i + _FBA_BATCH] - # Explicit selected ids include archived products; invalid quantities stay unknown. - for r in o.execute_kw('product.product', 'search_read', - [[('id', 'in', chunk), ('active', 'in', [True, False])]], + for r in o.execute_kw('product.product', 'search_read', [[('id', 'in', chunk)]], {'fields': ['id', 'qty_available'], 'context': ctx}): - quantity = r.get('qty_available') - if (isinstance(quantity, (int, float)) and not isinstance(quantity, bool) - and math.isfinite(quantity)): - out[r['id']] = quantity + out[r['id']] = r.get('qty_available') or 0.0 return out, fba_report(wh, len(ids), len(out), batches, 'stock.warehouse.code at runtime, never a pinned id') diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index 8566769b1de7b8178c60d4ab7da64d916aff5d97..20c07e850b2178e2628c61e27a7bcdeedb3774d2 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -97c1e86cb8eca3e160554e40319e1c35021e3f3502e9f144089c052593a8e4d0 +07e8d9ecae1d7a64e32ba15267a878944ef225627eed362f47d4c656e681f706 diff --git a/web/src/apps/AppRail.tsx b/web/src/apps/AppRail.tsx index 52622d18674a4c32802b7cb6847816d981547207..5bd2c248bb47c766edbf2ff1944827219bac565d 100644 --- a/web/src/apps/AppRail.tsx +++ b/web/src/apps/AppRail.tsx @@ -13,7 +13,6 @@ interface RailApplication { deepLink?: string; templateKind?: string; templateConnector?: string; - navPlacement?: { order?: number }; openTarget?: { database?: string }; compatibleDatabases?: string[]; capabilities?: Record; @@ -32,8 +31,7 @@ function railApplications(body: unknown): RailApplication[] { || row.capabilities?.read === false) return false; seen.add(row.id); return true; - }).sort((a, b) => (a.navPlacement?.order ?? 0) - (b.navPlacement?.order ?? 0) - || a.name.localeCompare(b.name, undefined, { sensitivity: "base" }) || a.id.localeCompare(b.id)); + }).sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: "base" }) || a.id.localeCompare(b.id)); } /** An Amazon App's rail row draws its own mark, never the Amazon logo (QI37). */ diff --git a/web/src/apps/account-health/AppView.tsx b/web/src/apps/account-health/AppView.tsx index 2249c123dff1fe8b9465188230e9442cecead53f..72d50fd6969c8f91440ceec624a242f49c83043e 100644 --- a/web/src/apps/account-health/AppView.tsx +++ b/web/src/apps/account-health/AppView.tsx @@ -1,6 +1,5 @@ -import { useCallback, useEffect, useId, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { API_V1, CREDENTIALS } from "../../apiContract"; -import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; import { useEscapeClose } from "../../ui/useEscapeClose"; import { AppTabs } from "../AppTabs"; import { LineChart } from "../../viz/kit"; @@ -8,7 +7,7 @@ import Dashboard from "./Dashboard"; import "../../styles/sdlc79/account-health.css"; import "../../styles/sdlc81/account-health-risk.css"; -// The dispatcher owns the App frame. This view owns its lists and detail overlays. +// The dispatcher owns the App frame. This view owns its lists and right-side detail panel. /** C2: the App as the dispatcher passes it. Structurally equal to lane A's AppViewProps application. */ interface AccountHealthApplication { @@ -94,30 +93,16 @@ interface RiskPage { interface Point { day: string; value: number | null; target: number | null; status: string } interface HistoryPage { points: Point[]; cursor: string | null; label: string; unit: string } -interface Category { - key: string; label: string; count: number | null; status: string; target: string; - window: { from: string; to: string }; sourceNote: string; sellerCentralUrl: string; - related: { asin: string; sku: string; issue: string; severity: string }[]; -} -type CategoryLoad = Load | { phase: "refused"; message: string }; type Load = { phase: "loading"; value?: T } | { phase: "error"; message: string; value?: T } | { phase: "ready"; value: T }; -type Picked = { kind: "metric" | "category" | "alert" | "risk"; id: string; metric?: Metric; alert?: Alert; risk?: Risk; linked?: boolean }; +type Picked = { kind: "metric" | "alert" | "risk"; id: string; metric?: Metric; alert?: Alert; risk?: Risk; linked?: boolean }; type Tab = "dashboard" | "metrics" | "alerts" | "risks"; const EMPTY = "No account health data"; const ALERT_STATUSES = ["Open", "Investigating", "Resolved", "Dismissed"]; const PAGE = 50; const HISTORY_PAGE = 50; -const CATEGORY_KEYS = new Set([ - "listing_policy_violations", "product_authenticity_customer_complaints", "product_condition_customer_complaints", - "product_safety_customer_complaints", "food_and_product_safety_issues", "received_intellectual_property_complaints", - "suspected_intellectual_property_violations", "restricted_product_policy_violations", - "customer_product_reviews_policy_violations", "other_policy_violations", "document_requests", -]); -const categoryKey = (row: Metric) => row.key || row.id.split("@")[0]; -const isCategory = (row: Metric) => CATEGORY_KEYS.has(categoryKey(row)); const text = (value: unknown) => (typeof value === "string" ? value : ""); const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); @@ -248,21 +233,6 @@ function parseHistory(raw: unknown): HistoryPage { }; } -function parseCategory(raw: unknown, key: string): Category { - const body = record(raw); - if (text(body.key) !== key || body.recordList !== false) throw new Error("This category could not be read."); - const window = record(body.window); - return { - key, label: text(body.label), count: num(body.count), status: text(body.status) || "No data", - target: text(body.target), window: { from: text(window.from), to: text(window.to) }, - sourceNote: text(body.sourceNote), sellerCentralUrl: text(body.sellerCentralUrl), - related: list(body.related).map((raw) => { - const row = record(raw); - return { asin: text(row.asin), sku: text(row.sku), issue: text(row.issue), severity: text(row.severity) }; - }), - }; -} - /** "Oct 22" from an ISO day. A calendar day is read in UTC. */ function day(iso: string) { if (!iso) return ""; @@ -324,135 +294,6 @@ function Failed({ message, onRetry }: { message: string; onRetry: () => void })

; } -function CategoryPopup({ metric, appId, onClose }: { metric: Metric; appId: string; onClose: () => void }) { - const panel = useRef(null); - const titleId = useId(); - const noteId = useId(); - const [load, setLoad] = useState({ phase: "loading" }); - const [attempt, setAttempt] = useState(0); - useOverlayLayer({ panelRef: panel, onDismiss: onClose, dismissOnOutside: false, - initialFocus: "[data-category-close]", restoreFocus: true, trapFocus: true }); - - useEffect(() => { - const controller = new AbortController(); - setLoad({ phase: "loading" }); - const query = new URLSearchParams({ app: appId }); - void fetch(`${API_V1}/account-health/categories/${encodeURIComponent(metric.id)}?${query.toString()}`, - { credentials: CREDENTIALS, signal: controller.signal }) - .then(async (response) => { - if (response.status === 401 || response.status === 403) { - const message = await readJson(response, "You do not have access to this category.") - .then(() => "You do not have access to this category.") - .catch((reason: unknown) => failure(reason, "You do not have access to this category.")); - if (!controller.signal.aborted) setLoad({ phase: "refused", message }); - return; - } - const category = parseCategory(await readJson(response, "This category could not be read."), categoryKey(metric)); - if (!controller.signal.aborted) setLoad({ phase: "ready", value: category }); - }) - .catch((reason: unknown) => { - if (!controller.signal.aborted) setLoad({ phase: "error", message: failure(reason, "This category could not be read.") }); - }); - return () => controller.abort(); - }, [appId, metric, attempt]); - - const category = load.phase === "ready" ? load.value : null; - const relatedIncomplete = Boolean(category - && /unavailable|need access|missing or unreadable|missing from the saved report/i.test(category.sourceNote)); - const count = category ? amount(category.count, "count") : ""; - const dataState = load.phase !== "ready" ? load.phase === "refused" ? "no-permission" : load.phase - : category?.count === null ? "na" : category?.count === 0 ? "empty" : "ready"; - - return -
-
-
- Account health - -
-
-
-
-
-

{category?.label || metric.label}

- - Amazon account health - - - - -
- {category && {count}} -
-

Amazon sends no record list for this category

- {load.phase === "loading" && } - {load.phase === "error" && { - panel.current?.querySelector("[data-category-close]")?.focus(); - setAttempt((n) => n + 1); - }} />} - {load.phase === "refused" &&

{load.message}

} - {category && <> -
- -

{category.count === 0 ? "No violations reported" : "Record list unavailable"}

-
-
-
Status
-
Amazon's target
{category.target || "No target"}
-
-
-
Reporting window - {category.window.from && category.window.to - ? <>{" to "} - - : "No data"} -
- {category.sellerCentralUrl && Open in Seller Central} -
- } -
- {category && -
-
-

Flagged listings

Related -
-

{category.sourceNote}

- {category.related.length ?
    - {category.related.map((listing, index) => { - const severity = listing.severity.toLowerCase().replace(/_/g, " "); - const severityTone = /error|critical|fatal/.test(severity) ? "is-risk" - : /warn/.test(severity) ? "is-near" : "is-quiet"; - return
  • -
    -
    - {listing.sku && Sku: {listing.sku}} - {listing.asin && Asin: {listing.asin}} - {!listing.sku && !listing.asin && Listing} -
    - {severity && {severity[0].toUpperCase() + severity.slice(1)}} -
    -

    {listing.issue || "No issue description"}

    -
  • ; - })} -
:
- {relatedIncomplete ? "Related listings unavailable" : "No flagged listings"} -
} -
- } -
-
-
-
; -} - // Busy buttons stay focusable so Enter keeps its place without a focus restore. function Pager({ page, previous, next, busy, onPrevious, onNext }: { page: number; previous: boolean; next: boolean; busy: boolean; onPrevious: () => void; onNext: () => void; @@ -797,11 +638,7 @@ function MetricCell({ column, row, markets, picked, groups, onPick }: { switch (column) { case "label": return + onClick={() => onPick(row)}>{row.label} {markets > 1 && row.marketplaceId && {row.marketplaceId}} ; case "status": return ; @@ -1054,7 +891,7 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) { }, [url, tab, riskStatusFilter, riskOffset, query, risksAttempt]); useEffect(() => { - if (!picked || picked.kind === "risk" || picked.kind === "category") return; + if (!picked || picked.kind === "risk") return; const controller = new AbortController(); setDetailAccess({ owners: [], canEdit: false }); if (picked.kind === "alert" && picked.linked) { @@ -1101,7 +938,7 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) { const pick = (next: Picked) => { setDetailMetric(next.metric ?? null); setDetailAlert(next.alert ?? null); - setDetailLoading(next.kind !== "risk" && next.kind !== "category"); + setDetailLoading(next.kind !== "risk"); setDetailError(""); opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; setPicked(next); @@ -1132,9 +969,9 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) { const alert = detailAlert; const alerting = (view?.metrics ?? []).filter((row) => !FOLDED.includes(row.status)); const folded = (view?.metrics ?? []).filter((row) => FOLDED.includes(row.status)); - const metricPick = (row: Metric) => pick({ kind: isCategory(row) ? "category" : "metric", id: row.id, metric: row }); + const metricPick = (row: Metric) => pick({ kind: "metric", id: row.id, metric: row }); - const open = Boolean(picked && picked.kind !== "category"); + const open = Boolean(picked); const busy = tab === "risks" ? risks.phase !== "ready" : tab === "metrics" ? summary.phase !== "ready" : alerts.phase !== "ready"; const shownOffset = (tab === "risks" ? riskPage?.offset : tab === "metrics" ? view?.offset : page?.offset) ?? 0; @@ -1181,13 +1018,13 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) { &&

No matching metrics

} {summary.phase === "ready" && view?.state === "ready" && alerting.length > 0 && } + picked={picked?.kind === "metric" ? picked.id : ""} onPick={metricPick} />} {summary.phase === "ready" && view?.state === "ready" && view.folded > 0 && } {summary.phase === "ready" && view?.state === "ready" && foldOpen && folded.length > 0 && } + picked={picked?.kind === "metric" ? picked.id : ""} onPick={metricPick} />} } {tab === "alerts" &&
@@ -1235,8 +1072,6 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) { onPrevious={() => movePage(Math.max(0, shownOffset - shownLimit))} onNext={() => movePage(shownOffset + shownLimit)} />}
- {picked?.kind === "category" && picked.metric && } {open && import("../../customer-grid/CustomerGrid")); -const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP }; -const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]); - -/** Admission belongs to the selected tab's fresh probe. */ -type TableProbe = { table: string; status: "ready"; tables: Record } - | { table: string; status: "failed" } | null; +/** Which Command Center tables exist, keyed by table; "failed" mounts the grid as before. */ +type TableProbe = Record | "failed" | null; // The Command Center App body: the Table module's own Grid, hosted under the App frame. // The tab strip switches `table` through the context, so the URL stays on the App. @@ -33,17 +28,6 @@ export default function ProjectManagementView(_props: AppViewProps) { const [projects, setProjects] = useState(null); const [competitors, setCompetitors] = useState(null); const [gridRevision, setGridRevision] = useState(0); - const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null); - const openTable = useCallback((next: string) => { - if (next === table) return; - // Clear synchronously: returning to a tab must not reuse its earlier admission. - setProbe(null); - setEntryEmpty(null); - setPicture(null); - setProjects(null); - setCompetitors(null); - setTable(next); - }, [table]); const reloadGrid = useCallback(() => { for (const key of [AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]) { clearTopicRowsCache(`tables/${key}/rows`); @@ -56,27 +40,23 @@ export default function ProjectManagementView(_props: AppViewProps) { useEffect(() => { const controller = new AbortController(); void fetch(`${API_V1}/amazon-profit/tables`, { credentials: CREDENTIALS, cache: "no-store", - signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20000)]) }) + signal: controller.signal }) .then(async (res) => { if (!res.ok) throw new Error("probe refused"); const body = await res.json() as { tables?: Record }; - if (!body.tables || typeof body.tables !== "object" || Array.isArray(body.tables) - || Object.values(body.tables).some(value => typeof value !== "boolean")) throw new Error("invalid probe"); - if (!controller.signal.aborted) setProbe({ table, status: "ready", tables: body.tables }); + setProbe(body.tables ?? {}); }) - .catch(() => { if (!controller.signal.aborted) setProbe({ table, status: "failed" }); }); + .catch(() => { if (!controller.signal.aborted) setProbe((last) => last ?? "failed"); }); return () => controller.abort(); }, [table]); - const currentProbe = probe?.table === table ? probe : null; - const loading = currentProbe === null; - const probeFailed = currentProbe?.status === "failed"; - const notBuilt = currentProbe?.status === "ready" && currentProbe.tables[table] === false; - const inaccessible = currentProbe?.status === "ready" && !Object.hasOwn(currentProbe.tables, table); - const admitted = currentProbe?.status === "ready" && currentProbe.tables[table] === true; + const notBuilt = probe !== null && probe !== "failed" && probe[table] === false; + const inaccessible = probe !== null && probe !== "failed" && !Object.hasOwn(probe, table); const dataEntry = table === AMAZON_DATA_ENTRY_TABLE; // Only an empty, permitted row response proves every SKU has its data. + const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null); useEffect(() => { - if (!dataEntry || !admitted) { setEntryEmpty(null); return; } + if (!dataEntry) { setEntryEmpty(null); return; } + if (probe === null || notBuilt || inaccessible) return; const controller = new AbortController(); void fetch(`${API_V1}/tables/${AMAZON_DATA_ENTRY_TABLE}/rows`, { credentials: CREDENTIALS, cache: "no-store", signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20000)]) }) @@ -87,32 +67,27 @@ export default function ProjectManagementView(_props: AppViewProps) { }) .catch(() => { if (!controller.signal.aborted) setEntryEmpty({ table, empty: false }); }); return () => controller.abort(); - }, [dataEntry, admitted, table]); + }, [dataEntry, probe, notBuilt, inaccessible, table]); const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table; - const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty; + const allComplete = dataEntry && !notBuilt && entryEmpty?.table === table && entryEmpty.empty; // A new key remounts the grid per table, as the Table module does on a database switch. - return + return - {dataEntry && admitted && } - {!admitted || allComplete - ?
} + {table === "ut_amazon_projects" ?
+ + +
: probe === null ? : inaccessible || notBuilt || allComplete + ?
- - {loading ? - :

{inaccessible ? ACCESS_REFUSED_MESSAGE : probeFailed ? "This table could not be checked. Switch tabs to try again." - : notBuilt ? "This table is not built yet. An admin Refresh builds it." : "Every SKU has its data."}

} -
- : table === "ut_amazon_projects" ?
- - + +

{inaccessible ? ACCESS_REFUSED_MESSAGE : notBuilt ? "This table is not built yet. An admin Refresh builds it." + : "Every SKU has its data."}

: !entryChecked ? : }> {/* C3 App-only props (S82-T68). Projects keeps New record. */} setProjects(projectsTargetFromRow(table, row)), competitors: row => setCompetitors(competitorsTargetFromRow(table, row)), diff --git a/web/src/apps/amazon-profit/ProjectResults.tsx b/web/src/apps/amazon-profit/ProjectResults.tsx index 66ca482b4231d0d0cfa1024a87d8b025b90ab6fb..efcffe1ce6b2b677986e7a57a0faded84f6e1ce1 100644 --- a/web/src/apps/amazon-profit/ProjectResults.tsx +++ b/web/src/apps/amazon-profit/ProjectResults.tsx @@ -1,8 +1,5 @@ import { useEffect, useRef, useState } from "react"; -import { API_V1, CREDENTIALS, PRINCIPAL_HEADER, TENANT_HEADER, checkTenant, currentTenant } from "../../apiContract"; -import { refusalMessage } from "../../customer-grid/apiBridge"; -import { LineChart, formatMeasure } from "../../viz/kit"; -import "../../styles/sdlc84/project-results.css"; +import { API_V1, CREDENTIALS } from "../../apiContract"; export type ResultMetric = "sales" | "units" | "returns" | "return_rate" | "refund_rate" | "roas"; export interface ResultConfig { @@ -26,11 +23,73 @@ interface Result { deltaPercent: number | null; } interface Answer { timezone: string; anchor: string | null; results: Result[] } +const METRICS: Array<{ key: string; metric: ResultMetric; label: string; compare: "prior" | "yoy" }> = [ + { key: "sales", metric: "sales", label: "Sales", compare: "prior" }, + { key: "sales-yoy", metric: "sales", label: "Sales YoY", compare: "yoy" }, + { key: "return_rate", metric: "return_rate", label: "Returns %", compare: "prior" }, + { key: "units", metric: "units", label: "Units", compare: "prior" }, + { key: "returns", metric: "returns", label: "Returns", compare: "prior" }, + { key: "refund_rate", metric: "refund_rate", label: "Refunds %", compare: "prior" }, + { key: "roas", metric: "roas", label: "ROAS", compare: "prior" }, +]; +const configKey = (config: ResultConfig) => config.metric === "sales" && config.compare === "yoy" ? "sales-yoy" : config.metric; const periodLabel = (period: WindowResult) => { const last = new Date(`${period.end_exclusive}T00:00:00Z`); last.setUTCDate(last.getUTCDate() - 1); return `${period.start} to ${last.toISOString().slice(0, 10)}`; }; +function figure(value: number | null, unit: string) { + if (value === null || !Number.isFinite(value)) return "Unavailable"; + if (/^[A-Z]{3}$/.test(unit)) return value.toLocaleString("en-US", { style: "currency", currency: unit, maximumFractionDigits: 2 }); + return `${value.toLocaleString("en-US", { maximumFractionDigits: 2 })}${unit === "percent" ? "%" : unit === "ratio" ? "×" : ""}`; +} + +function Trend({ result }: { result: Result }) { + const current = result.current, comparison = result.comparison; + if (!current || !comparison || result.state !== "ready") return

+ {result.state === "waiting_for_completion" ? "Complete the project to view results" + : result.state === "waiting_for_window" || result.state === "waiting_for_completed_day" + ? "Results appear when this period ends" : "Results unavailable"} +

; + const values = [...current.series, ...comparison.series].map(point => point.value) + .filter((value): value is number => value !== null && Number.isFinite(value)); + if (!values.length) return

Results unavailable

; + const low = Math.min(0, ...values), high = Math.max(1, ...values); + const count = Math.max(current.series.length, comparison.series.length, 2); + const x = (index: number) => 70 + index * 700 / (count - 1); + const y = (value: number) => 180 - (value - low) * 150 / (high - low); + const path = (points: Point[]) => { + let joined = false; + return points.map((point, index) => { + if (point.value === null || !Number.isFinite(point.value)) { joined = false; return ""; } + const command = joined ? "L" : "M"; + joined = true; + return `${command}${x(index)},${y(point.value)}`; + }).join(" "); + }; + return <> +
+
After completion
{figure(current.value, result.unit)}
+
{result.config.compare === "yoy" ? "Prior year" : "Before completion"}
{figure(comparison.value, result.unit)}
+
Change
{figure(result.delta, result.unit)}
+
+ + {result.label}: {periodLabel(current)} compared with {periodLabel(comparison)} + {[low, (high + low) / 2, high].map((value, index) => + + {figure(value, result.unit)} + )} + + + Day 1Day {count} + +
+ After completion · {periodLabel(current)} + {result.config.compare === "yoy" ? "Prior year" : "Before completion"} · {periodLabel(comparison)} +
+ ; +} + export function ProjectResultPreview({ projectId, configs, revision, completedOn, onOpen }: { projectId: string; configs: ResultConfig[]; revision: string; completedOn: string | null; onOpen: () => void; }) { @@ -80,135 +139,91 @@ export function ProjectResultPreview({ projectId, configs, revision, completedOn
; } -const CHARTS = [ - { key: "sales", label: "Sales" }, - { key: "units", label: "Units" }, - { key: "margin", label: "Margin" }, - { key: "returns", label: "Returns" }, - { key: "returns_pct", label: "Returns %" }, - { key: "refunds_pct", label: "Refunds %" }, - { key: "roas", label: "ROAS" }, - { key: "sales_yoy", label: "Sales YoY" }, -] as const; -type ChartKey = typeof CHARTS[number]["key"]; -interface SeriesMetric { - key: ChartKey; - label: string; - unit: string; - points: Point[]; - total: number | null; - tie: { ok: boolean; ours: number | null; independent: number | null }; - reason: string | null; -} -interface SeriesAnswer { - projectId: string; - from: string; - to: string; - markers: { start: string | null; target: string | null; completed: string | null }; - metrics: SeriesMetric[]; -} -interface ChartVisibility { projectId: string; storageKey: string; hidden: ChartKey[] } - -function readHiddenCharts(storageKey: string): ChartKey[] { - try { - const value: unknown = JSON.parse(localStorage.getItem(storageKey) ?? "[]"); - return Array.isArray(value) ? CHARTS.filter(chart => value.includes(chart.key)).map(chart => chart.key) : []; - } catch { return []; } -} - -export default function ProjectResults({ projectId, configs, completedOn, updatedAt, scope }: { +export default function ProjectResults({ projectId, configs, completedOn, updatedAt, canEdit, onSaved }: { projectId: string; configs: ResultConfig[]; completedOn: string | null; updatedAt?: string; canEdit: boolean; onSaved: () => void; - scope?: { grain: "sku" | "asin" | "parent"; keys: string[] }; }) { - const [retry, setRetry] = useState(0); - const [loaded, setLoaded] = useState<{ request: string; answer: SeriesAnswer | null; error: string } | null>(null); - const [visibility, setVisibility] = useState(null); - const [storageError, setStorageError] = useState(""); - const host = useRef(null); - const scopeValue = scope ? `${scope.grain}:${scope.keys.join(",")}` : "all"; - const request = JSON.stringify([projectId, configs, completedOn, updatedAt, scopeValue, retry]); + const [draft, setDraft] = useState(configs); + const [answer, setAnswer] = useState(null); + const [error, setError] = useState(""); + const [busy, setBusy] = useState(false); + const [customPeriod, setCustomPeriod] = useState(false); + const [moreMetrics, setMoreMetrics] = useState(false); + const [days, setDays] = useState(configs[0]?.days ?? 7); + const [window, setWindow] = useState(configs[0]?.window ?? "fixed"); + const configStamp = JSON.stringify(configs); + useEffect(() => { setDraft(JSON.parse(configStamp) as ResultConfig[]); }, [configStamp]); useEffect(() => { const controller = new AbortController(); - // C3 owns the default dates and the tenant's completed-day boundary. - const query = new URLSearchParams({ scope: scopeValue }); - void fetch(`${API_V1}/amazon-projects/${encodeURIComponent(projectId)}/series?${query}`, { credentials: CREDENTIALS, + setAnswer(null); + setError(""); + void fetch(`${API_V1}/amazon-projects/${encodeURIComponent(projectId)}/results`, { credentials: CREDENTIALS, signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20000)]) }) .then(async response => { - if (controller.signal.aborted || checkTenant(response)) return; - const storageKey = `aios-project-result-charts:${JSON.stringify([ - response.headers.get(TENANT_HEADER) ?? currentTenant(), response.headers.get(PRINCIPAL_HEADER), projectId, - ])}`; - setVisibility(previous => previous?.storageKey === storageKey ? previous - : { projectId, storageKey, hidden: readHiddenCharts(storageKey) }); - const body = await response.json().catch(() => null); - if (!response.ok) throw new Error(refusalMessage(body, response.status, "Results could not be loaded")); - if (!body || body.projectId !== projectId || !Array.isArray(body.metrics)) throw new Error("Results could not be loaded"); - if (!controller.signal.aborted) setLoaded({ request, answer: body as SeriesAnswer, error: "" }); - }).catch(error => { - if (!controller.signal.aborted) setLoaded({ request, answer: null, - error: error instanceof Error ? error.message : "Results could not be loaded" }); - }); + if (!response.ok) throw new Error("Results could not be loaded."); + const body = await response.json() as Answer; + if (!controller.signal.aborted) setAnswer(body); + }).catch(() => { if (!controller.signal.aborted) setError("Results could not be loaded."); }); return () => controller.abort(); - }, [projectId, scopeValue, request]); - const current = loaded?.request === request ? loaded : null; - const answer = current?.answer; - const loading = current === null; - const hidden = visibility?.projectId === projectId ? visibility.hidden : []; - const charts = CHARTS.filter(chart => !hidden.includes(chart.key)); - const period = answer ? `${answer.from} to ${answer.to}` : ""; - const changeHidden = (next: ChartKey[], focusIndex = 0) => { - if (!visibility || visibility.projectId !== projectId) return; - setVisibility({ ...visibility, hidden: next }); + }, [projectId, configStamp, completedOn, updatedAt]); + const dirty = JSON.stringify(draft) !== configStamp; + const period = (nextWindow: ResultConfig["window"], nextDays: number) => { + setWindow(nextWindow); setDays(nextDays); + setDraft(current => current.map(config => ({ ...config, window: nextWindow, days: nextDays }))); + }; + const save = async () => { + if (busy) return; + setBusy(true); setError(""); try { - localStorage.setItem(visibility.storageKey, JSON.stringify(next)); - setStorageError(""); - } catch { setStorageError("Chart visibility lasts for this visit"); } - requestAnimationFrame(() => { - const controls = host.current?.querySelectorAll("[data-chart-visibility]"); - controls?.[Math.min(focusIndex, controls.length - 1)]?.focus(); - }); + const response = await fetch(`${API_V1}/amazon-projects/${encodeURIComponent(projectId)}`, { + method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + signal: AbortSignal.timeout(20000), body: JSON.stringify({ resultConfigs: draft }), + }); + if (!response.ok) throw new Error("The results were not saved."); + onSaved(); + } catch { setError("The results were not saved."); } + finally { setBusy(false); } }; - return
-
-

Results

- {hidden.length > 0 && charts.length > 0 ? : null} - {current?.error ? : null} + return
+

Results

+
+ + + {customPeriod ? : null}
- {storageError ?

{storageError}

: null} - {charts.length === 0 ?
-

All charts are hidden

- -
: null} - {charts.length > 0 ?
- {charts.map((chart, index) => { - const metric = answer?.metrics.find(item => item.key === chart.key); - const label = metric?.label || chart.label; - const available = metric !== undefined && metric.total !== null; - const reason = current?.error || metric?.reason || (!metric && !loading ? "This metric was not returned" : ""); - return
-
-
-
{label}
- {loading ?
- -
- {loading ?
- : <> - {available ? : null} - {reason || !available ?

- {reason || "This metric is unavailable"} -

: null} - } -
; - })} +
+ {METRICS.filter(metric => moreMetrics || draft.some(config => configKey(config) === metric.key)).map(metric => )} + {canEdit ? : null} +
+ {dirty && canEdit ?
+ +
: null} + {error ?

{error}

: null} + {answer?.results.map(result =>
+
{result.label}{result.config.compare === "yoy" ? " YoY" : ""}
+ +
)}
; } diff --git a/web/src/apps/amazon-profit/Projects.tsx b/web/src/apps/amazon-profit/Projects.tsx index c010320f8976be70bd9a01dddd23d5804da4bb34..cff007d7781513912a464cf7370e74713e9b6583 100644 --- a/web/src/apps/amazon-profit/Projects.tsx +++ b/web/src/apps/amazon-profit/Projects.tsx @@ -1,10 +1,10 @@ import { createContext, useContext, useCallback, useEffect, useId, useMemo, useRef, useState, type FormEvent, type ReactNode } from "react"; import { createPortal } from "react-dom"; +import { ArrowRight } from "@phosphor-icons/react/dist/csr/ArrowRight"; import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh"; import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium"; import { CellSignalLow } from "@phosphor-icons/react/dist/csr/CellSignalLow"; import { Check } from "@phosphor-icons/react/dist/csr/Check"; -import { CaretDown } from "@phosphor-icons/react/dist/csr/CaretDown"; import { CircleDashed } from "@phosphor-icons/react/dist/csr/CircleDashed"; import { Cube } from "@phosphor-icons/react/dist/csr/Cube"; import { DotsThree } from "@phosphor-icons/react/dist/csr/DotsThree"; @@ -21,7 +21,7 @@ import { SidebarSimple } from "@phosphor-icons/react/dist/csr/SidebarSimple"; import { API_V1, CREDENTIALS, TOAST_EVENT, signal } from "../../apiContract"; import { refusalMessage } from "../../customer-grid/apiBridge"; import { assetUrl } from "../../customer-grid/catalogData"; -import { AnchoredOverlay, BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; +import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; import "../../styles/sdlc82/amazon-projects.css"; import ProjectResults, { ProjectResultPreview } from "./ProjectResults"; import type { ResultConfig } from "./ProjectResults"; @@ -34,11 +34,6 @@ export const PROJECT_PRIORITIES = ["No priority", "Urgent", "High", "Medium", "L type Status = (typeof PROJECT_STATUSES)[number]; type Priority = (typeof PROJECT_PRIORITIES)[number]; type Health = "On track" | "At risk" | "Off track" | null; -const STATUS_GROUPS: Array<{ label: string; statuses: readonly Status[] }> = [ - { label: "Not started", statuses: ["Backlog", "Planned"] }, - { label: "Active", statuses: ["In progress"] }, - { label: "Done", statuses: ["Completed", "Canceled"] }, -]; export interface Project { id: string; @@ -108,7 +103,6 @@ interface ProjectInput { ownerUsername?: string | null; startOn?: string | null; targetOn?: string | null; - completedOn?: string | null; progress?: number; notes?: string | null; targets?: Array<{ grain: ProjectGrain; key: string }>; @@ -117,9 +111,8 @@ interface ProjectInput { type ProjectDraft = Required>; function draftOf(project: Project): ProjectDraft { - const { name, status, priority, owner, startOn, targetOn, completedOn, progress, notes } = project; - return { name, status, priority, owner, ownerUsername: project.ownerUsername ?? null, - startOn, targetOn, completedOn, progress, notes }; + const { name, status, priority, owner, startOn, targetOn, progress, notes } = project; + return { name, status, priority, owner, ownerUsername: project.ownerUsername ?? null, startOn, targetOn, progress, notes }; } const BASE = `${API_V1}/amazon-projects`; @@ -136,40 +129,27 @@ const FIELD_LABEL: Record = { }; const enc = encodeURIComponent; -function OwnerSelect({ value, legacy, disabled, onChange, label = "Owner", emptyLabel = "Unassigned" }: { +function OwnerSelect({ value, legacy, disabled, onChange }: { value: string | null; legacy?: string | null; disabled: boolean; onChange: (value: string | null) => void; - label?: string; emptyLabel?: string; }) { const [owners, setOwners] = useState>([]); const [failed, setFailed] = useState(false); - const [loading, setLoading] = useState(true); - const [attempt, setAttempt] = useState(0); useEffect(() => { const controller = new AbortController(); - setLoading(true); - setFailed(false); void call<{ owners: Array<{ username: string; name: string }> }>("/owners", { signal: controller.signal }) .then(answer => { if (!controller.signal.aborted) setOwners(answer.owners); }) - .catch(() => { if (!controller.signal.aborted) setFailed(true); }) - .finally(() => { if (!controller.signal.aborted) setLoading(false); }); + .catch(() => { if (!controller.signal.aborted) setFailed(true); }); return () => controller.abort(); - }, [attempt]); + }, []); const selected = value || (legacy ? "__legacy__" : ""); - return <> - {failed ?
- Assignees did not load. - -
: null} - ; + ; } class ProjectRequestError extends Error { @@ -279,9 +259,9 @@ function StatusRing({ status, progress }: { status: Status; progress: number }) const share = clampPercent(progress) / 100; if (status === "Completed" || status === "Canceled" || status === "Backlog") { const Icon = status === "Completed" ? Check : status === "Canceled" ? Prohibit : CircleDashed; - return
; } -// ---- One project: Results, the existing Setup form, and Log ---- - -const PROJECT_PANES = [ - { key: "results", label: "Results" }, - { key: "setup", label: "Setup" }, - { key: "log", label: "Log" }, -] as const; -type ProjectPane = (typeof PROJECT_PANES)[number]["key"]; - -function SetupRow({ field, label, error, children, readOnly = false }: { - field: string; label: string; error?: string; children: ReactNode; readOnly?: boolean; -}) { - const labelId = useId(); - return
- {label} -
- {children} - {error ?

{error}

: null} -
-
; -} - -function SetupDate({ label, value, canEdit, busy, onChange, max, hint }: { - label: string; value: string | null; canEdit: boolean; busy: boolean; - onChange: (value: string | null) => void; max?: string; hint?: string; -}) { - const [editing, setEditing] = useState(false); - const caption = value ? new Date(`${value}T00:00:00`).toLocaleDateString("en-US", - { month: "short", day: "numeric", year: "numeric" }) : "Not set"; - if (!canEdit) return value ? : Not set; - return editing ? onChange(event.target.value || null)} onBlur={() => setEditing(false)} /> - : ; -} +// ---- One project: the detail reference's dates pair and progress, then notes, SKUs, activity ---- function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }: { id: string; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void; @@ -727,19 +607,22 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }: const [busy, setBusy] = useState(false); const [adding, setAdding] = useState(false); const [confirming, setConfirming] = useState(false); - const [editingDescription, setEditingDescription] = useState(false); - const [fieldErrors, setFieldErrors] = useState>>({}); - const [saved, setSaved] = useState(false); - const [pane, setPane] = useState("results"); + const [editingDates, setEditingDates] = useState(false); + const dateEditorId = useId(); const formId = useId(); - const railRef = useRef(null); + const startDateRef = useRef(null); const writing = useRef(false); + const mainRef = useRef(null); const loaded = detail !== null; useEffect(() => { - if (loaded) railRef.current?.querySelector('[aria-selected="true"]')?.focus(); + if (loaded) mainRef.current?.querySelector("input, h3")?.focus(); }, [loaded]); + useEffect(() => { + if (editingDates) startDateRef.current?.focus(); + }, [editingDates]); + useEffect(() => { const controller = new AbortController(); call(`/${enc(id)}`, { signal: controller.signal }, "This project did not load.") @@ -753,8 +636,7 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }: return () => controller.abort(); }, [id, tick]); - const write = useCallback(async (path: string, method: string, body: unknown, after?: () => void, - fields?: Array) => { + const write = useCallback(async (path: string, method: string, body: unknown, after?: () => void) => { if (writing.current || !mayEdit || !detail?.canEdit) return; writing.current = true; setBusy(true); @@ -766,19 +648,10 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }: if (method === "PATCH") setDraft(draftOf(project)); } setError(""); - setFieldErrors({}); - if (method === "PATCH") setSaved(true); if (after) after(); else setTick((value) => value + 1); onChanged(); } catch (reason) { - const message = messageOf(reason, "The change was not saved."); - if (fields?.length) { - const namedField = fields.find(field => message.toLowerCase().includes( - ({ startOn: "start_on", targetOn: "target_on", completedOn: "completed_on", - ownerUsername: "owner" } as Partial>)[field] ?? field.toLowerCase())); - setFieldErrors(Object.fromEntries((namedField ? [namedField] : fields).map(field => [field, message]))); - setError(""); - } else setError(message); + setError(messageOf(reason, "The change was not saved.")); if (reason instanceof ProjectRequestError && reason.status === 403) { setDetail((current) => current ? { ...current, canEdit: false } : current); onChanged(); @@ -789,158 +662,105 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }: } }, [detail?.canEdit, mayEdit, onChanged]); - const patch = useCallback((change: ProjectInput, fields: Array) => - write(`/${enc(id)}`, "PATCH", change, undefined, fields), [id, write]); - - const rail = ; - - if (!detail || !draft) return
- {rail} -
- {PROJECT_PANES.map(({ key }) => )} -
-
; - const { project, targets } = detail; - const activity = [...detail.activity].sort((a, b) => Date.parse(b.at) - Date.parse(a.at)); + const patch = useCallback((change: ProjectInput) => write(`/${enc(id)}`, "PATCH", change), [id, write]); + + if (!detail || !draft) return error ?

{error}

: ; + const { project, targets, activity } = detail; const canEdit = mayEdit && detail.canEdit; const display = canEdit ? draft : project; const progress = Number.isFinite(display.progress) ? clampPercent(display.progress) : 0; const dirty = JSON.stringify(draft) !== JSON.stringify(draftOf(project)); - const set = (key: K, value: ProjectDraft[K]) => { - setSaved(false); - setFieldErrors(current => ({ ...current, [key]: undefined })); - setDraft(current => current ? { ...current, [key]: value, - ...(key === "status" && value !== "Completed" ? { completedOn: null } : {}), - ...(key === "completedOn" && value ? { status: "Completed" as Status } : {}), - } : current); - }; + const set = (key: K, value: ProjectDraft[K]) => + setDraft((current) => current ? { ...current, [key]: value } : current); const save = (event: FormEvent) => { event.preventDefault(); if (!canEdit || !dirty) return; const name = draft.name.trim(); - if (!name) { setFieldErrors({ name: "Name the project." }); return; } + if (!name) { setError("Name the project."); return; } if (!Number.isInteger(draft.progress) || draft.progress < 0 || draft.progress > 100) { - setFieldErrors({ progress: "Enter a whole percent from 0 to 100." }); return; - } - const today = new Date().toISOString().slice(0, 10); - if (draft.completedOn && draft.completedOn > today) { - setFieldErrors({ completedOn: "Choose today or an earlier date." }); return; - } - if (project.completedOn && draft.status === "Completed" && !draft.completedOn) { - setFieldErrors({ completedOn: "Choose a completed date, or change Status to reopen the project." }); return; + setError("Enter a whole percent from 0 to 100."); return; } - const original = draftOf(project); - const changed = (Object.keys(draft) as Array).filter(key => draft[key] !== original[key]); - const fields = Object.fromEntries(changed.filter(key => key !== "owner").map(key => [key, draft[key]])) as ProjectInput; - if (changed.includes("owner")) fields.ownerUsername = draft.ownerUsername; - if (changed.includes("name")) fields.name = name; - if (changed.includes("notes")) fields.notes = draft.notes?.trim() || null; - setSaved(false); - setFieldErrors({}); - void patch(fields, changed); + const { owner, ownerUsername, ...fields } = draft; + void patch({ ...fields, name, notes: draft.notes?.trim() || null, + ...(!ownerUsername && owner ? {} : { ownerUsername }) }); }; - const setup =
-
-
-

- {canEdit ? set("name", event.target.value)} /> : project.name} -

- {fieldErrors.name ? : null} -
- -
- - {canEdit - ? set("status", value)} /> - : } - - -
+ return +
+ {canEdit + ? set("name", event.target.value)} /> + :

{project.name}

} + +
+ Properties +
+ + {canEdit + ? + : {project.status}} + + {canEdit - ? : {project.priority}} -
- - - {canEdit ?
- { set("ownerUsername", value); set("owner", null); }} /> -
: project.owner ? : Unassigned} -
- {([ ["startOn", "Start date"], ["targetOn", "Target date"], ["completedOn", "Completed date"] ] as const) - .map(([field, label]) => - set(field, value)} /> - )} - - {canEdit ?
- set("progress", event.target.valueAsNumber)} />% -
: {progress}%} -
- - {project.health ? : Not set} - - - {canEdit ? draft.notes || editingDescription - ?