diff --git a/Dockerfile b/Dockerfile index a4e842430bad4dfb1924eeb45095024f75d06691..8c7054f37e9a3f88a67ade4eec2306b259719db9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,4 +120,12 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ # nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other # proxies in this class use, so it is the safest available floor without over-fitting to a number # we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise. -CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"] +# +# S79-T37 (R25): the launcher sizes the web tier to the container. `request_class.py --env` reads +# the cgroup CPU and memory limits (never os.cpu_count(), which reports the host) and exports the +# uvicorn worker count, heavy concurrency and DuckDB caps. An env value set on the Space wins. +# Several workers start only with AIOS_MIRROR_VERSIONS=1, which opens the mirror read-only in each +# of them, and the shell holds W at 1 without it. 2 vCPU keeps one worker and today's caps. +# Several workers load the app through a factory: uvicorn imports a plain app string in its +# supervisor too, and that process would run the boot threads without serving. +CMD ["sh", "-c", "eval \"$(python api/request_class.py --env)\"; W=1; if [ \"${AIOS_MIRROR_VERSIONS:-0}\" = 1 ]; then W=\"${AIOS_UVICORN_WORKERS:-1}\"; fi; export AIOS_UVICORN_WORKERS=\"$W\"; APP=api.main:app; if [ \"$W\" -gt 1 ]; then APP=\"--factory api.request_class:web_app\"; fi; exec uvicorn $APP --host 0.0.0.0 --port ${PORT} --workers \"$W\" --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"] diff --git a/RELEASES.json b/RELEASES.json index 3ac530ffbc38da0badef22dc4a41a102f9d15038..e279c569edf63df0d1a9a940fb92a45315b12f98 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,4 +1,4 @@ { - "current": "f75ce5527c", + "current": "fb1493505", "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index d06b29668f935e171f6e8326721ef1d27e01c49f..9ccf9f7e898c1c67eb7f4caace5855342b4844eb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -f75ce5527c +fb1493505 diff --git a/api/agent_runtime.py b/api/agent_runtime.py index dcaf2d7da54fd1b309a1729155c3a67daf2118ef..66a98ced1e0772e2e87a10b8dacb3287e7602db6 100644 --- a/api/agent_runtime.py +++ b/api/agent_runtime.py @@ -742,7 +742,18 @@ def _hold(session, name, args, effect, held, turn): resolved = _fill(args, refs, len(held) + 1) if _PLACEHOLDER.search(json.dumps(resolved)): raise ValueError("A change can only use an earlier step's created object.") - preview = assistant_write_preview.prepare(session, name, resolved, prior=held) + if name == "apps_confirm": + import routes_external as external + payload = external._read_confirmation_token(resolved.get("confirmation_token"), session=session) + if payload.get("tool") != "apps": + raise ValueError("Choose an App preview to confirm.") + external._load_operation(session, payload["operation_id"], confirming=True) + result = external._preview_external_write( + session, "apps", payload["arguments"], issue_token=False, execution=payload["execution"], + purpose=payload.get("purpose"), about=payload.get("about")) + preview = {"documents": {}, "result": result, "diff": result.get("diff") or []} + else: + preview = assistant_write_preview.prepare(session, name, resolved, prior=held) _observe_sources(session, turn, name, resolved, preview["result"]) held.append({"tool": name, "args": copy.deepcopy(args), "effect": effect, "ref": ref, "resolvedArgs": resolved, "preview": preview, @@ -826,7 +837,8 @@ def _apply_change(session, entry): import assistant_write_preview if args != step.get("resolvedArgs"): raise ValueError("An earlier step no longer matches this preview. Ask again.") - result = assistant_write_preview.apply(session, step) + result = (_dispatch(session, "apps_confirm", args) if step["tool"] == "apps_confirm" + else assistant_write_preview.apply(session, step)) _observe_sources(session, turn, step["tool"], args, result) finally: _PRODUCING_TURN.reset(token) diff --git a/api/amazon_relational.py b/api/amazon_relational.py index 3c14f9b6def4601c7b4eb783192efa6c98725079..0ee4f8fbb41d93c41b14023bba40c6a866f92a1e 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -47,6 +47,7 @@ if str(_API.parent.parent / "platform") not in sys.path: import odoo_relational as _odoo # noqa: E402 from core import connector_spec # noqa: E402 from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402 +from harness import amazon_ads_store # noqa: E402 Refused = _odoo.Refused @@ -57,7 +58,10 @@ PREFIX = "ut_amzn_" #: `ut_*` key -> the mirror table `amazon_store` writes. Derived from ONE list so a table cannot #: exist on one side and not the other. -SOURCE = {PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_TABLES} +#: S79-T13: the Ads tables (`ut_amzn_ads_*`, connector `amazon_ads`) live in the same mirror. +_ADS_SPEC = {s.key: s for s in connector_spec.tables_for("amazon_ads")} +SOURCE = {**{PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_TABLES}, + **{k: amazon_ads_store.table_name(s.source) for k, s in _ADS_SPEC.items()}} #: ⭐⭐ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE** #: (W53-T37, contract C2). `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their module-level @@ -66,7 +70,20 @@ SOURCE = {PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_ #: longer be stated twice and say two things. Every comment that used to justify a literal here #: (the two spines, R12, the report-vs-API spellings, the cardinalities) travelled with the values #: into `connector_spec.AMAZON_TABLES`; read it there. -_SPEC = {s.key: s for s in connector_spec.tables_for("amazon")} +_SPEC = {**{s.key: s for s in connector_spec.tables_for("amazon")}, **_ADS_SPEC} + + +def _identity_columns(table, columns=()): + """An Ads row is identified by its stored `_row_key`; every other table by amazon_schema.""" + if PREFIX + table in _ADS_SPEC: + return ("_row_key",) + return amazon_schema.row_key_columns_for(table, columns) + + +def _fallback_columns(table): + if PREFIX + table in _ADS_SPEC: + return ("_report_id",) + return amazon_schema.row_fallback_columns_for(table) #: ⛔ A `ut_*` KEY THAT `amazon_store` SPAWNS BUT THE SPEC DOES NOT DECLARE IS ABSENT FROM ALL FOUR #: DICTS BELOW, ON PURPOSE. `verify_amazon.section_relational` asserts the four key sets equal @@ -106,8 +123,10 @@ def source_attribution(key): if key not in SOURCE or key not in _SPEC: raise KeyError(f"unknown Amazon database {key!r}") return { - "key": "amazon_sp", - "origin": "Apify Amazon search dataset" if key == PREFIX + "keyword_ranks" else "Amazon Services API", + "key": "amazon_ads" if key in _ADS_SPEC else "amazon_sp", + "origin": ("Amazon Ads API" if key in _ADS_SPEC + else "Apify Amazon search dataset" if key == PREFIX + "keyword_ranks" + else "Amazon Services API"), "rawTable": SOURCE[key], "native": _SPEC[key].native, "readOnly": True, @@ -621,8 +640,8 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True): return missing_source(key) names = {name for name, _dtype in columns} table = key.removeprefix(PREFIX) - identity = amazon_schema.row_key_columns_for(table, names) - fallback = sorted(set((*identity, *amazon_schema.row_fallback_columns_for(table)))) + identity = _identity_columns(table, names) + fallback = sorted(set((*identity, *_fallback_columns(table)))) source = _quoted(SOURCE[key]) def value(name, owner=source): @@ -713,14 +732,14 @@ def grid_sources(cur=None): def _row_id(row, key, occurrence=0): """Return a legacy row key from declared identity, lineage and equal-key occurrence.""" table = key.removeprefix(PREFIX) - columns = amazon_schema.row_key_columns_for(table, row) + columns = _identity_columns(table, row) identity = tuple(row.get(name) for name in columns) if all(value not in (None, "") for value in identity): if table in _LEGACY_IDENTITY_TABLES and len(identity) == 1: return str(identity[0]) payload = json.dumps(identity, ensure_ascii=True, default=str) else: - fallback = sorted(set((*columns, *amazon_schema.row_fallback_columns_for(table)))) + fallback = sorted(set((*columns, *_fallback_columns(table)))) values = tuple(None if row.get(name) in (None, "") else str(row[name]) for name in fallback) payload = json.dumps(("__amazon_missing_identity__", *values, str(occurrence)), @@ -1100,6 +1119,9 @@ def apply_plan(rt, built, username="amazon", today=None): for key, _label, _fields, _rows in plans} listing_ids = (_listing_identity_rows(cur_mirror) if PREFIX + "listing_items" in built else ()) + # S79-T13: each Ads table's description states its history window and any recorded limit. + ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror) + for key, *_rest in plans if key in _ADS_SPEC} for key, _label, _fields, rows in plans: if isinstance(rows, dict): continue @@ -1116,7 +1138,8 @@ def apply_plan(rt, built, username="amazon", today=None): candidates = ( _row_id(raw, key, occurrence), hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]) - declared = amazon_schema.TABLE_SPECS[key.removeprefix(PREFIX)]["row_identity"] + declared = (("_row_key",) if key in _ADS_SPEC + else amazon_schema.TABLE_SPECS[key.removeprefix(PREFIX)]["row_identity"]) if all(raw.get(name) not in (None, "") for name in declared): old_composite = json.dumps(tuple(raw[name] for name in declared), ensure_ascii=True, default=str) @@ -1235,6 +1258,8 @@ def apply_plan(rt, built, username="amazon", today=None): # DPP 1.8 attribution is explicit in the stored definition, not inferred from a # mutable display label or from the process having imported this module. doc[key]["connector"] = source_attribution(key) + if ads_descriptions.get(key): + doc[key]["description"] = ads_descriptions[key] if key == PREFIX + "listing_items": doc[key]["amazonListingIdentity"] = "sku" return doc diff --git a/api/app_routes.py b/api/app_routes.py new file mode 100644 index 0000000000000000000000000000000000000000..ec9c2ce19045f7569a7edfe3bb66d190bc5371e5 --- /dev/null +++ b/api/app_routes.py @@ -0,0 +1,58 @@ +"""C3: mount every C1 App router by name, recording a failed import instead of raising it.""" +from __future__ import annotations + +import sys +from importlib import import_module + +import app_templates + + +def _failures_record(): + main = sys.modules.get("main") + record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None + return record if isinstance(record, dict) else app_templates.ROUTER_MOUNT_FAILURES + + +def _flat(routes): + """Every route, descending into a router FastAPI keeps as one included entry.""" + for route in routes: + yield route + inner = getattr(route, "original_router", None) + if inner is not None: + yield from _flat(inner.routes) + + +def _included(app, router): + endpoints = {getattr(route, "endpoint", None) for route in _flat(router.routes)} + endpoints.discard(None) + return any(getattr(route, "original_router", None) is router + or getattr(route, "endpoint", None) in endpoints for route in _flat(app.routes)) + + +def mount_app_routers(app, failures=None): + """Include each C1 `router` once, with main.py `_mount_router` semantics; returns those mounted.""" + record = failures if isinstance(failures, dict) else _failures_record() + mounted = [] + for row in app_templates.TEMPLATES: + name = row["router"] + if not name: + continue + try: + router = import_module(name).router + # An already included router (Competitor watch, mounted by main.py) is not added twice. + if not _included(app, router): + app.include_router(router) + except Exception as exc: # Boot must answer even when a router cannot load. + failure = {"error": type(exc).__name__, + "missing_module": exc.name if isinstance(exc, ModuleNotFoundError) else ""} + record.setdefault(name, []).append(failure) + if record is not _failures_record(): + # The offer predicate reads main.py's record and this one, never a caller's own dict. + app_templates.ROUTER_MOUNT_FAILURES.setdefault(name, []).append(failure) + # Exception messages can contain configuration values; report the class/module only. + print(f"[aios-api] ROUTER MOUNT FAILED: {name}: {failure}; " + "its App template is withheld until the router mounts") + continue + app_templates.MOUNTED_ROUTERS.add(name) + mounted.append(name) + return mounted diff --git a/api/app_rules.py b/api/app_rules.py index a7386e636c26eadbd9ca2f9d640b6bbb53a2153c..5a8461e2ba551bd70f0a3cd9ad870cec78855cd8 100644 --- a/api/app_rules.py +++ b/api/app_rules.py @@ -441,8 +441,7 @@ def _catalog_selftest(): source = rt.get("product_table_workspace__shared") contracts = registry.chart_field_contracts(session, ["product_data"]) app = registry.create(rt, tenant, "member", { - "name": "Template fixture", "templateKind": "catalog", "sourceDatabase": "product_data", - "fieldMap": {"key": "code", "title": "product", "image": "catalog_photo"}, + "name": "Template fixture", "templateKind": "catalog", "navPlacement": {"section": "app", "order": 0}}, allowed_databases={"product_data"}, field_contracts=contracts) api = FastAPI() diff --git a/api/app_templates.py b/api/app_templates.py new file mode 100644 index 0000000000000000000000000000000000000000..82da7017698d861019e088fc1657d0016503e9b6 --- /dev/null +++ b/api/app_templates.py @@ -0,0 +1,195 @@ +"""C1: the shipped App template catalogue, one row per App template, each bundled with its connector. + +The registry's template contracts and `core.external_tools`' templateKind enum derive from or +mirror `TEMPLATES`; `verify_app_templates.py` holds all three name for name. +""" +from __future__ import annotations + +import sys +from importlib import import_module +from types import MappingProxyType + +PROCESSES = ("Create value", "Acquire", "Convert", "Deliver", "Retain", "Finance and cash", + "Risk and compliance", "None") + +# A C1 connector key is not a keychain entry type: Amazon SP-API is stored as `amazon_sp` (F25). +CONNECTOR_ENTRY_TYPES = MappingProxyType({"odoo": "odoo", "amazon": "amazon_sp", "amazon_ads": "amazon_ads"}) + + +def _amazon(kind, label, purpose, process, sources, *, connector="amazon", proposes=True, tools=()): + return {"kind": kind, "label": label, "purpose": purpose, "connector": connector, + "sources": tuple(sources), "process": process, "mark": kind, + "module": "modules." + kind, "router": "routes_" + kind, + "view": kind.replace("_", "-"), "tools": tuple(tools), "proposes": proposes} + + +TEMPLATES = ( + # Routing and Catalog render through Map mode; their routes are mounted by main.py. + # They read FIELD_BINDINGS over their source, so an App of either kind stores no field map. + {"kind": "route", "label": "Routing", "purpose": "Plan delivery routes for your customers.", + "connector": "odoo", "sources": ("customer_data",), "process": "None", "mark": "route", + "module": None, "router": None, "view": None, "tools": (), "proposes": False}, + {"kind": "catalog", "label": "Catalog", "purpose": "Build and print product catalogs.", + "connector": "odoo", "sources": ("product_data",), "process": "None", "mark": "catalog", + "module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"), + "proposes": False}, + _amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.", + "Acquire", ("ut_amzn_brand_analytics",), proposes=False), + _amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.", + "Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics", + "ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings", + "ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False), + _amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.", + "Acquire", (), connector="amazon_ads", proposes=False), + _amazon("restock_planner", "Restock planner", "Send stock to Amazon before it runs out.", + "Deliver", ("ut_amzn_restock_recommendations", "ut_amzn_inventory_planning", + "ut_amzn_fba_inventory")), + _amazon("reimbursements", "Reimbursements", "Find FBA money Amazon owes you.", + "Create value", ("ut_amzn_reimbursements", "ut_amzn_inventory_ledger_detail", + "ut_amzn_fba_customer_returns")), + _amazon("aged_stock", "Aged stock", "Move old stock before storage fees grow.", + "Create value", ("ut_amzn_inventory_planning", "ut_amzn_long_term_storage_fees", + "ut_amzn_storage_fees", "ut_amzn_recommended_removals", + "ut_amzn_stranded_inventory")), + _amazon("opportunity_finder", "Opportunity finder", "Find rising searches you do not serve yet.", + "Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings")), + _amazon("returns_repeat", "Returns and repeat buyers", "See why buyers return and who buys again.", + "Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_repeat_purchase", + "ut_amzn_fba_shipped_sales", "ut_amzn_market_basket")), + _amazon("payouts", "Payouts", "Tie each payout to its orders and fees.", + "Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions")), + _amazon("account_health", "Account health", "Catch account risks before Amazon acts.", + "Risk and compliance", ("ut_amzn_account_health",)), +) +BY_KIND = MappingProxyType({row["kind"]: row for row in TEMPLATES}) +KINDS = tuple(BY_KIND) + +#: R33: the fixed role to field binding each kind reads from its first C1 source. +#: Catalog price is Royal 1 tier 1: the pooled `price_1` is not in the typed product_data schema. +FIELD_BINDINGS = MappingProxyType({ + "route": MappingProxyType({"title": "customer", "address": "street", + "latitude": "lat", "longitude": "lon"}), + "catalog": MappingProxyType({"key": "code", "title": "product", "image": "image", + "sku": "code", "price": "price_royal1_t1", "category": "category"}), + "competitor_watch": MappingProxyType({"term": "search_term"}), +}) +#: Kinds whose App record stores no source or field map: the registry projects C1 on read. +BOUND_KINDS = ("route", "catalog") +_BINDING_KEYS = ("sourceDatabase", "fieldMap") + + +def source(kind): + """The C1 source table a kind reads, or None.""" + row = BY_KIND.get(kind) + return row["sources"][0] if row and row["sources"] else None + + +def field_binding(kind): + """A copy of the kind's fixed role to field map; empty when it names none.""" + return dict(FIELD_BINDINGS.get(kind) or {}) + + +def bound(app): + """Read projection: a copy of a bound kind's App with the C1 source and field map filled in. + + A stored App that still carries its own map keeps it until the S79-T04 migration. + """ + if not isinstance(app, dict) or app.get("templateKind") not in BOUND_KINDS: + return app + row = dict(app) + projection = {"sourceDatabase": source(row["templateKind"]), + "fieldMap": field_binding(row["templateKind"])} + if row.get("fieldMap"): + projection = {"sourceDatabase": row.get("sourceDatabase") or projection["sourceDatabase"]} + row.update(projection) + return row + + +def own_binding(app): + """The binding keys a stored App carries that differ from what `bound` projects.""" + if not isinstance(app, dict) or app.get("templateKind") not in BOUND_KINDS: + return {key: app[key] for key in _BINDING_KEYS if isinstance(app, dict) and key in app} + kind = app["templateKind"] + fixed = field_binding(kind) + held = app.get("fieldMap") or {} + if (app.get("sourceDatabase") in (None, source(kind)) + and all(fixed.get(role) == field for role, field in held.items())): + return {} + return {key: app[key] for key in _BINDING_KEYS if key in app} + +#: Routers `app_routes.mount_app_routers` failed to mount when main.py's record is absent. +ROUTER_MOUNT_FAILURES = {} +#: Routers `app_routes.mount_app_routers` mounted or found already included. +MOUNTED_ROUTERS = set() +# A failed seam import is printed once per process and withholds its kind. +_SEAM_FAILURES = {} + + +def entry_type(connector): + """The keychain entry type that proves a C1 connector key is connected.""" + return CONNECTOR_ENTRY_TYPES.get(connector, connector) + + +def connectors_present(session): + """C1 connector keys this reader has. Any read failure counts as nothing connected.""" + try: + from routes_connectors import _entry_types + types, _scopes, failure = _entry_types(session.runtime, session.uname) + have = set() if failure else set(types or ()) + except Exception as exc: # noqa: BLE001 + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + have = set() + if "odoo" not in have: + try: + from routes_keychain import env_odoo_available + if env_odoo_available(session.runtime): + have.add("odoo") + except Exception: # noqa: BLE001 + pass + return {key for key in CONNECTOR_ENTRY_TYPES if entry_type(key) in have} + + +def connector_present(connector, session): + return connector in connectors_present(session) + + +def router_failures(): + """Every recorded router mount failure: main.py's record joined with this module's own.""" + merged = {name: list(rows) for name, rows in ROUTER_MOUNT_FAILURES.items()} + main = sys.modules.get("main") + record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None + if isinstance(record, dict) and record is not ROUTER_MOUNT_FAILURES: + for name, rows in record.items(): + merged.setdefault(name, []).extend(rows) + return merged + + +def _importable(name, attribute=None): + if name in _SEAM_FAILURES: + return False + try: + module = import_module(name) + if attribute and not hasattr(module, attribute): + raise ImportError(f"{name} has no {attribute}") + return True + except Exception as exc: # noqa: BLE001 + _SEAM_FAILURES[name] = type(exc).__name__ + print(f"[aios-api] APP TEMPLATE WITHHELD: {name}: {type(exc).__name__}") + return False + + +def seams_ready(kind): + """True when the kind's module imports and its router imports and never failed to mount.""" + row = BY_KIND.get(kind) + if row is None: + return False + if row["module"] and not _importable(row["module"]): + return False + router = row["router"] + return not router or (_importable(router, "router") and router not in router_failures()) + + +def reset_seam_cache(): + """Forget recorded seam import failures (test fixtures only).""" + _SEAM_FAILURES.clear() diff --git a/api/application_registry.py b/api/application_registry.py index 5358e1b3fa962b2efdf2a5f38aea0070f3484345..beb9663dc2df522b791f7bd322a8a62b3ac277a7 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -8,6 +8,7 @@ import json from core import shares, write_execution from core.request_timing import raise_if_resource_error import app_rules +import app_templates as _app_templates from modules.finance_apps import (finance_application as _finance_application, finance_applications as _finance_applications, @@ -19,6 +20,7 @@ APPLICATION_TYPES = ("chart", "route", "map", "code") # Finance descriptors remain available to the governed fact routes. FINANCE_TEMPLATE_KIND = "finance" _KEY = "applications_v1" +#: D-8392 (R58): the most live Apps one account may create. The cap counts per account, never per tenant. _MAX = 500 _MAX_DISPLAY_BYTES = 512 * 1024 @@ -66,28 +68,58 @@ _TEMPLATE_CONTRACTS = { }, } +# C1 (S79-T01): every contract carries its catalogue row's wire metadata. +_C1_WIRE_KEYS = ("connector", "purpose", "process", "mark", "view") +# R33: Routing and Catalog read their C1 source themselves, so the wizard asks for no database. +_BUNDLED_SOURCE_LABELS = {"customer_data": "Odoo customers", "product_data": "Odoo products"} +for _kind, _contract in _TEMPLATE_CONTRACTS.items(): + _contract.update({key: _app_templates.BY_KIND[_kind][key] for key in _C1_WIRE_KEYS}) + _contract.update(sourceDatabase=_app_templates.source(_kind), + sourceLabel=_BUNDLED_SOURCE_LABELS[_app_templates.source(_kind)]) + # Keep stored copy identities readable after a template stops accepting new Apps. APPLICATION_WIZARD_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS) APPLICATION_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS) #: S78-T17 (C4, R20, R21): templates offered only to a tenant that passes the kind's gate. #: They stay out of the ungated tuples above, which other gates pin to Routing and Catalog. +#: S79-T01 (C1): every connector-gated kind in `app_templates.TEMPLATES` is one of them. BRAND_ANALYTICS_DATABASE = "ut_amzn_brand_analytics" -_GATED_TEMPLATE_CONTRACTS = { +# Wire fields a C1 row does not carry, per gated kind. +_GATED_WIRE = { "competitor_watch": { - "kind": "competitor_watch", - "label": "Competitor watch", - # The screen reads the competitor watch routes; the source names the Amazon database. - "applicationType": "map", - "displayMode": "map", "fieldRoles": [ {"key": "term", "label": "Search term", "required": True, "types": []}, ], - "defaultPlacement": {"section": "app", "order": 0}, - "sourceDatabase": BRAND_ANALYTICS_DATABASE, "sourceLabel": "Amazon brand analytics", }, } + + +def _source_label(database): + """The connector table's label, so a refusal never names a `ut_` key.""" + try: + from core import connector_spec + return connector_spec.spec_for(database).label + except Exception: # noqa: BLE001 + return None + + +def _gated_contract(row): + # The screen reads the kind's own routes; the source names its connector database. + contract = {"kind": row["kind"], "label": row["label"], + "applicationType": "map", "displayMode": "map", "fieldRoles": [], + "defaultPlacement": {"section": "app", "order": 0}, + **{key: row[key] for key in _C1_WIRE_KEYS}} + if row["sources"]: + contract["sourceDatabase"] = row["sources"][0] + contract["sourceLabel"] = _source_label(row["sources"][0]) + contract.update(copy.deepcopy(_GATED_WIRE.get(row["kind"], {}))) + return contract + + +_GATED_TEMPLATE_CONTRACTS = {row["kind"]: _gated_contract(row) for row in _app_templates.TEMPLATES + if row["kind"] not in _TEMPLATE_CONTRACTS} GATED_TEMPLATE_KINDS = tuple(_GATED_TEMPLATE_CONTRACTS) @@ -140,13 +172,56 @@ def brand_analytics_offered(session): return False -_TEMPLATE_GATES = {"competitor_watch": brand_analytics_offered} +def _connector_gate(connector): + """C1 gate: the reader has the kind's connector (F25: the connector key, never a `ut_` prefix).""" + def gate(session): + return _app_templates.connector_present(connector, session) + gate.connector = connector + return gate + + +# Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21). +_TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch" + else _connector_gate(contract["connector"]) + for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()} +#: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo. +_BUNDLED_GATES = {kind: _connector_gate(contract["connector"]) + for kind, contract in _TEMPLATE_CONTRACTS.items()} + + +def template_offered(kind, session): + """C1 offer predicate: the module imports, the router mounted and the kind's gate holds.""" + gate = _TEMPLATE_GATES.get(kind) or _BUNDLED_GATES.get(kind) + if gate is None or not _app_templates.seams_ready(kind): + return False + try: + return bool(gate(session)) + except Exception as exc: # noqa: BLE001 + raise_if_resource_error(exc) + return False class ApplicationError(ValueError): pass +def account_capacity(): + """The per-account App cap. `AIOS_APP_CAPACITY` lowers it for one server, as a lane proof does.""" + import os + raw = str(os.environ.get("AIOS_APP_CAPACITY") or "").strip() + return min(int(raw), _MAX) if raw.isdigit() else _MAX + + +def capacity_refusal(doc, user): + """D-8392: the sentence naming the cap when `user` already holds it, else None.""" + cap = account_capacity() + held = sum(1 for app in (doc.get("applications") or {}).values() + if isinstance(app, dict) and app.get("createdBy") == user and app.get("retired") is not True) + if held < cap: + return None + return f"Your account can hold up to {cap} App{'' if cap == 1 else 's'}. Delete an App to add another." + + class ApplicationRuleError(ApplicationError): """Expose an App rule refusal through every writer's existing error handler.""" code = "app_rule" @@ -176,7 +251,8 @@ def template_identity(app): if kind == FINANCE_TEMPLATE_KIND: key = str(app.get("financeKey") or "") return f"{FINANCE_TEMPLATE_KIND}:{key}" if key else None - return kind if kind in APPLICATION_WIZARD_TEMPLATE_KINDS else None + # S79-T05: a connector-gated copy is grouped too, so the App directory can show Added. + return kind if kind in APPLICATION_WIZARD_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS else None def _own_template_app(applications, user, identity): @@ -208,12 +284,15 @@ def own_template_apps(runtime, user): def template_contracts(session=None): """The trusted renderer contracts offered by the Application creation flow. - A gated template joins only for a session whose tenant passes its gate (R21). + With a session, every template is offered only when the reader has its connector (instr 20). """ - rows = _ungated_template_contracts() - if session is not None: - rows.extend(copy.deepcopy(contract) for kind, contract in _GATED_TEMPLATE_CONTRACTS.items() - if _TEMPLATE_GATES[kind](session)) + if session is None: + return _ungated_template_contracts() + have = _app_templates.connectors_present(session) + rows = [copy.deepcopy(contract) for contract in _TEMPLATE_CONTRACTS.values() + if contract["connector"] in have] + rows.extend(copy.deepcopy(contract) for kind, contract in _GATED_TEMPLATE_CONTRACTS.items() + if template_offered(kind, session)) return rows @@ -366,8 +445,12 @@ def _clean_doc(raw): def load(runtime): - try: return _clean_doc(runtime.get(_KEY)) + """The read door: a Routing or Catalog App carries its C1 binding (`app_templates.bound`).""" + try: doc = _clean_doc(runtime.get(_KEY)) except Exception: return blank_doc() + doc["applications"] = {key: _app_templates.bound(app) + for key, app in doc["applications"].items()} + return doc def _server_origin(origin_door=None, via=None): @@ -436,6 +519,14 @@ def _database_free_code(app): and not app.get("compatibleDatabases")) +def _database_free(app): + """A code App, or an App whose C1 kind names no source, opens without a database binding.""" + if _database_free_code(app): + return True + row = _app_templates.BY_KIND.get(str(app.get("templateKind") or "")) if isinstance(app, dict) else None + return bool(row and not row["sources"] and not app.get("compatibleDatabases")) + + def _finance_permission(app): try: return _finance_application(str(app.get("financeKey") or ""))["permissionKey"] @@ -711,66 +802,45 @@ def _refuse_legacy_catalog_changes(app, config): def _template_definition(kind, payload, *, semantic_records=None, field_contracts=None, allowed_databases=None): + """R33: a template App binds its C1 source and fixed field map; the caller names neither.""" contract = _TEMPLATE_CONTRACTS.get(kind) or _GATED_TEMPLATE_CONTRACTS.get(kind) if contract is None: raise ApplicationError("application template kind is not recognized") template_kind = str(payload.get("templateKind") or "").strip().lower() if template_kind != kind: raise ApplicationError(f"{kind} applications require templateKind {kind!r}") - source = str(payload.get("sourceDatabase") or "").strip()[:120] - if not source: + named = str(payload.get("sourceDatabase") or "").strip()[:120] + source = _app_templates.source(kind) or named + # S79-T05: a C1 kind whose `sources` is empty (amazon_operator) reads its own routes, no database. + sourceless = kind in _app_templates.BY_KIND and not _app_templates.BY_KIND[kind]["sources"] + if sourceless: + if named or payload.get("compatibleDatabases"): + raise ApplicationError(f"{contract['label']} reads no database. Remove the source database.") + elif not source: raise ApplicationError("sourceDatabase is required") - if contract.get("sourceDatabase") and source != contract["sourceDatabase"]: + elif named and named != source: raise ApplicationError(f"{contract['label']} reads " - f"{contract.get('sourceLabel') or contract['sourceDatabase']} only. " + f"{contract.get('sourceLabel') or source} only. " "Choose it as the source database.") - if source not in set(allowed_databases or ()): + elif source not in set(allowed_databases or ()): raise PermissionError("an Application cannot bind a database you cannot open") - if "compatibleDatabases" in payload: + if not sourceless and "compatibleDatabases" in payload: compatible = _compat(payload.get("compatibleDatabases")) if compatible != [source]: raise ApplicationError("a template Application has exactly one sourceDatabase") - - raw_map = payload.get("fieldMap") - if not isinstance(raw_map, dict): - raise ApplicationError("fieldMap must be an object") - roles = {row["key"]: row for row in contract["fieldRoles"]} - extras = sorted(set(raw_map) - set(roles)) - if extras: - raise ApplicationError("fieldMap contains roles outside the template contract: " - + ", ".join(extras) + ". Accepted roles: " + ", ".join(roles)) - missing = sorted(key for key, role in roles.items() - if role["required"] and not str(raw_map.get(key) or "").strip()) - if missing: - raise ApplicationError("fieldMap is missing required roles: " + ", ".join(missing)) - available = _contract_fields(source, semantic_records, field_contracts) - if not available: - raise ApplicationError("the source database has no readable field contract") - field_map = {} - for role, raw_field in raw_map.items(): - if raw_field is not None and not isinstance(raw_field, str): - raise ApplicationError(f"fieldMap role {role!r} must name a source field with a text key") - field = str(raw_field or "").strip() - if not field: - continue - if len(field) > 160 or field not in available: - raise ApplicationError(f"fieldMap role {role!r} does not name a source field") - allowed_types = set(roles[role].get("types") or ()) - if allowed_types and available[field] not in allowed_types: - raise ApplicationError( - f"fieldMap role {role!r} cannot use field type {available[field]!r}") - field_map[role] = field - latitude, longitude = field_map.get("latitude"), field_map.get("longitude") - if bool(latitude) != bool(longitude): - raise ApplicationError("Routing needs both coordinate fields or neither") - if latitude and latitude == longitude: - raise ApplicationError("Routing latitude and longitude must use different fields") - # A new Route must name at least one placement source. Stored definitions are not revalidated - # on read, so this create-time refusal does not retire older Applications with an empty map. - if kind == "route" and not (latitude and longitude) and not field_map.get("address"): - raise ApplicationError("Routing needs both coordinate fields or an address field") - - raw_placement = payload.get("navPlacement") + # A legacy caller may still send a field map: the fixed binding replaces it. + binding = {} if sourceless else _app_templates.field_binding(kind) + if binding: + available = _contract_fields(source, semantic_records, field_contracts) + if not available: + raise ApplicationError("the source database has no readable field contract") + unreadable = sorted(role["label"] for role in contract["fieldRoles"] + if role["required"] and binding.get(role["key"]) not in available) + if unreadable: + raise ApplicationError(f"{contract['label']} needs fields your account cannot read: " + + ", ".join(unreadable) + ". Ask an administrator for access.") + + raw_placement = payload.get("navPlacement") or contract["defaultPlacement"] if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}: raise ApplicationError("navPlacement must contain section and order") section = str(raw_placement.get("section") or "").strip().lower() @@ -779,8 +849,33 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract raise ApplicationError("template Applications must be placed in App") if isinstance(order, bool) or not isinstance(order, int) or not 0 <= order <= 999: raise ApplicationError("navPlacement order must be an integer from 0 to 999") - return {"templateKind": kind, "sourceDatabase": source, "fieldMap": field_map, - "navPlacement": {"section": "app", "order": order}} + definition = {"templateKind": kind, "compatibleDatabases": [] if sourceless else [source], + "navPlacement": {"section": "app", "order": order}} + if kind not in _app_templates.BOUND_KINDS and not sourceless: + definition.update(sourceDatabase=source, fieldMap=binding) + return definition + + +def bound_application(app, available): + """The App with its field map read against the caller's `{field: type}` schema. + + A C1 binding drops an optional role the caller cannot read. A stored map stays strict. + """ + stored = bool(isinstance(app, dict) and app.get("fieldMap")) + row = _app_templates.bound(app) + contract = _template_contract(str(row.get("templateKind") or "")) or {} + roles = {role["key"]: role for role in contract.get("fieldRoles") or ()} + resolved = {} + for role, field in (row.get("fieldMap") or {}).items(): + spec = roles.get(role) + if spec is None: + raise ApplicationError(f"fieldMap role {role!r} is outside the template contract") + types = set(spec.get("types") or ()) + if field in available and (not types or available[field] in types): + resolved[role] = field + elif stored or spec["required"]: + raise ApplicationError(f"fieldMap role {role!r} does not name a readable source field") + return {**row, "fieldMap": resolved} def validate_application(payload, tenant, *, semantic_records=None, field_contracts=None, @@ -830,7 +925,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None, is_default=False): - row=copy.deepcopy(app) + row=copy.deepcopy(_app_templates.bound(app)) row.setdefault("origin", None) if row.get("type") == "code": # Only the authenticated code door returns the body. @@ -844,6 +939,11 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas "default":bool(is_admin)} row["starred"] = user in (app.get("starredBy") or ()) row["isDefault"] = bool(is_default) + catalogue = _app_templates.BY_KIND.get(str(row.get("templateKind") or "")) + if catalogue and catalogue["view"]: + # C2: the client mounts apps//AppView.tsx and fills the two marks. + row.update(templateView=catalogue["view"], templateMark=catalogue["mark"], + templateConnector=catalogue["connector"]) if row.get("templateKind") == FINANCE_TEMPLATE_KIND: # A Finance App is a user's own copy: owner controls apply. Defaults bind a database. row["capabilities"]["default"] = False @@ -864,6 +964,8 @@ def dependent_view_counts(runtime, database): """S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY user's Views in that database's workspace. The Delete App dialog names this count before confirm. None when the workspace cannot be read, so the dialog never states a guessed 0. + + D-2233: it walks the Views `delete` clears, nested and `presentation` ones too, trash excepted. """ from core.view_templates import workspace_key key = workspace_key(database) @@ -874,16 +976,11 @@ def dependent_view_counts(runtime, database): except Exception: return None counts = {} - for member, bucket in (doc.items() if isinstance(doc, dict) else ()): - if member in _RESERVED_WORKSPACE_MEMBERS or not isinstance(bucket, dict): - continue - views = bucket.get("views") - for view in (views.values() if isinstance(views, dict) else ()): - config = view.get("config") if isinstance(view, dict) else None - ref = config.get("application") if isinstance(config, dict) else None - app_id = str(ref.get("id") or "") if isinstance(ref, dict) else "" - if app_id: - counts[app_id] = counts.get(app_id, 0) + 1 + users = {member: bucket for member, bucket in (doc.items() if isinstance(doc, dict) else ()) + if member not in _RESERVED_WORKSPACE_MEMBERS} + for view in _workspace_views(users, trash=False): + for app_id in _view_app_ids(view): + counts[app_id] = counts.get(app_id, 0) + 1 return counts @@ -900,7 +997,7 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre if not may_read(app,user,is_admin): continue if database not in (app.get("compatibleDatabases") or ()): continue elif not _listed(app, user): continue - if (not _database_free_code(app) + if (not _database_free(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue if kind and app.get("type") != kind: continue if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue @@ -947,7 +1044,7 @@ def _build_application(tenant, user, payload, *, semantic_records=None, field_co name=" ".join(str(payload.get("name") or "").split())[:120] if not name: raise ApplicationError("application name is required") app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat() - compatible = ([definition["sourceDatabase"]] if definition + compatible = (list(definition["compatibleDatabases"]) if definition else _compat(payload.get("compatibleDatabases"), allow_empty=kind == "code")) if _allowed(compatible, allowed_databases) != set(compatible): raise PermissionError("an Application cannot bind a database you cannot open") @@ -1035,7 +1132,8 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr identity = template_identity(app) def apply(raw): doc=_clean_doc(raw) - if len(doc["applications"])>=_MAX: raise ApplicationError("application limit reached") + refusal = capacity_refusal(doc, user) + if refusal: raise ApplicationError(refusal) # R216 and R247 allow every template to have several independent copies. doc["applications"][app_id]=app; return doc _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of) @@ -1066,10 +1164,14 @@ def ensure_route_application(runtime, tenant, *, accounts=None, wall=None): """ result = _ensure_route_seed(runtime, tenant) copies = ensure_user_app_copies(runtime, tenant, accounts=accounts, wall=wall) + bindings = migrate_c1_bindings(runtime, tenant) folded = dict(result) folded["userCopies"] = copies + folded["c1Bindings"] = bindings note = "user copies " + copies["outcome"] + ( ": " + copies["reason"] if copies.get("reason") else "") + note += "; C1 bindings " + bindings["outcome"] + ( + ": " + bindings["reason"] if bindings.get("reason") else "") folded["reason"] = result["reason"] + "; " + note if result.get("reason") else note return folded @@ -1106,13 +1208,12 @@ def _ensure_route_seed(runtime, tenant): app_id = existing.get("id") if existing is not None else None outcome = "existing" if existing is not None else "no_routes" if existing is None and route_count: - if len(doc["applications"]) >= _MAX: - raise ApplicationError("application limit reached") + refusal = capacity_refusal(doc, "loopable") + if refusal: + raise ApplicationError(refusal) app = _build_application( tenant, "loopable", {"name": "Routing", "templateKind": "route", "type": "route", - "sourceDatabase": "customer_data", - "fieldMap": {"title": "customer", "latitude": "lat", "longitude": "lon"}, "navPlacement": {"section": "app", "order": 0}}, field_contracts={"customer_data": [ {"key": "customer", "type": "text"}, {"key": "lat", "type": "text"}, @@ -1130,6 +1231,96 @@ def _ensure_route_seed(runtime, tenant): return result +#: Marker of the R33 migration: the binding each rewritten App held before, and the Apps it kept. +C1_BINDINGS_MARKER = "c1Bindings" + + +def _c1_rewrite(app): + """A Routing or Catalog App rewritten to its C1 binding, or None when it must stay as stored. + + An App whose source or field map differs from C1 (`own_binding` non-empty) is never rewritten. + """ + if not isinstance(app, dict) or app.get("templateKind") not in _app_templates.BOUND_KINDS: + return None + if "sourceDatabase" not in app and "fieldMap" not in app: + return None + if _app_templates.own_binding(app): + return None + source = _app_templates.source(app["templateKind"]) + row = copy.deepcopy(app) + row.pop("sourceDatabase", None) + row.pop("fieldMap", None) + row["compatibleDatabases"] = [source, *(database for database in app.get("compatibleDatabases") or () + if database != source)] + return row + + +def _c1_plan(doc, app_ids): + """`{id: rewritten App}` plus the stored Apps that keep a binding of their own, for one document.""" + rewrites, kept = {}, [] + for app_id, app in sorted(doc["applications"].items()): + if app_ids is not None and app_id not in app_ids: + continue + if not isinstance(app, dict) or app.get("templateKind") not in _app_templates.BOUND_KINDS: + continue + rewritten = _c1_rewrite(app) + if rewritten is not None: + rewrites[app_id] = rewritten + elif _app_templates.own_binding(app): + kept.append({"id": app_id, "templateKind": app["templateKind"], "name": app.get("name"), + "sourceDatabase": app.get("sourceDatabase"), + "fieldMap": copy.deepcopy(app.get("fieldMap"))}) + return rewrites, kept + + +def migrate_c1_bindings(runtime, tenant, *, app_ids=None): + """R33: rewrite every stored Routing and Catalog App to its C1 binding. + + Writes only when an App still stores a binding equal to C1, so a second run changes nothing. + An App with its own source or field map is kept and listed. A proof run + passes `app_ids` to touch only the named Apps; the boot call never passes it. + """ + scope = None if app_ids is None else set(app_ids) + try: + doc = _clean_doc(_read_strict(runtime, _KEY)) + except Exception as exc: + return {"outcome": "skipped", "reason": f"registry read failed: {type(exc).__name__}: {exc}", + "rewritten": [], "kept": []} + rewrites, kept = _c1_plan(doc, scope) + result = {"outcome": "unchanged", "rewritten": [], "kept": kept} + if rewrites: + def apply(raw): + current = _clean_doc(raw) + planned, held = _c1_plan(current, scope) + result["kept"] = held + if not planned: + return raw + marker = current.setdefault("migrations", {}).setdefault( + C1_BINDINGS_MARKER, {"rewritten": {}, "kept": []}) + for app_id, app in planned.items(): + before = current["applications"][app_id] + marker["rewritten"][app_id] = {key: copy.deepcopy(before[key]) for key in + ("sourceDatabase", "fieldMap", "compatibleDatabases") + if key in before} + current["applications"][app_id] = app + marker["kept"] = sorted({row["id"] for row in held} | set(marker.get("kept") or ())) + marker["at"] = write_execution.utcnow().isoformat() + result.update(outcome="migrated", rewritten=sorted(planned)) + return current + _update_document(runtime, apply) + if result["rewritten"]: + try: + from catalog_store import migrate_snapshots + result["catalogSnapshots"] = migrate_snapshots(runtime, result["rewritten"]) + except Exception as exc: + result["catalogSnapshots"] = f"skipped: {type(exc).__name__}" + parts = [f"{len(result['rewritten'])} rewritten"] if result["rewritten"] else [] + if result["kept"]: + parts.append("kept with their own binding: " + ", ".join(row["id"] for row in result["kept"])) + result["reason"] = "; ".join(parts) + return result + + _COPIES = "userAppCopies" _APP_INDEX_KEY = "module_index_app" @@ -1331,7 +1522,7 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie doc=_clean_doc(raw); app=doc["applications"].get(app_id) if not app: raise ApplicationError("application does not exist") if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases): - if not _database_free_code(app): + if not _database_free(app): raise ApplicationError("application does not exist") if (app.get("templateKind") in APPLICATION_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS and "compatibleDatabases" in patch): @@ -1366,14 +1557,9 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie _refuse_legacy_catalog_changes(app, config) app["config"] = config if "fieldMap" in patch: - kind = app.get("templateKind") - if kind not in APPLICATION_WIZARD_TEMPLATE_KINDS: - raise ApplicationError("only a field-mapped template Application may change fieldMap") - definition = _template_definition( - kind, {**app, "fieldMap": patch["fieldMap"]}, - semantic_records=semantic_records, field_contracts=field_contracts, - allowed_databases=allowed_databases) - app["fieldMap"] = definition["fieldMap"] + # R33: a template App reads the fields its template names. + raise ApplicationError("This App reads the fields its template names. " + "Its field map cannot change.") compatible = (_compat(patch["compatibleDatabases"], allow_empty=app["type"] == "code") if "compatibleDatabases" in patch else app.get("compatibleDatabases") or []) if _allowed(compatible, allowed_databases) != set(compatible): @@ -1419,13 +1605,16 @@ def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records app=load(runtime)["applications"].get(app_id) if not may_read(app,user,is_admin): raise ApplicationError("application does not exist") if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases): - if not _database_free_code(app): + if not _database_free(app): raise ApplicationError("application does not exist") payload={"type":app["type"],"name":f"{app['name']} copy", "compatibleDatabases":app.get("compatibleDatabases"), "semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")} - for key in ("templateKind", "sourceDatabase", "fieldMap", "navPlacement", "financeKey"): + # R33: a template copy binds C1 itself, so neither the source nor the field map is copied. + for key in ("templateKind", "navPlacement", "financeKey"): if key in app: payload[key]=copy.deepcopy(app[key]) + if app.get("templateKind") not in _app_templates.BOUND_KINDS and "sourceDatabase" in app: + payload["sourceDatabase"] = app["sourceDatabase"] return create(runtime,tenant,user,payload, semantic_records=semantic_records, field_contracts=field_contracts, allowed_databases=allowed_databases, via=via, on_behalf_of=on_behalf_of, @@ -1435,11 +1624,10 @@ def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records _ANY_NAME = object() -def _strip_view_refs(doc, app_id): - """Remove `config.application` naming `app_id` from every View in one workspace document. +def _workspace_views(doc, *, trash=True): + """Every View in one workspace document: each member's Views, reserved members included. - Walks every top-level member, reserved ones included, and each owner's trash entries. - Returns the number of Views changed. Mutates `doc` in place. + Descends into nested folders. With `trash`, each owner's trash entries are walked too. """ def nested_views(rows): for row in (rows.values() if isinstance(rows, dict) else rows if isinstance(rows, list) else ()): @@ -1448,20 +1636,38 @@ def _strip_view_refs(doc, app_id): elif isinstance(row, (dict, list)): yield from nested_views(row) - def views_of(bucket): - views = bucket.get("views") - yield from nested_views(views) - trash = bucket.get("trash") - for entry in (trash.values() if isinstance(trash, dict) - else trash if isinstance(trash, list) else ()): + for bucket in (doc.values() if isinstance(doc, dict) else ()): + if not isinstance(bucket, dict): + continue + yield from nested_views(bucket.get("views")) + held = bucket.get("trash") if trash else None + for entry in (held.values() if isinstance(held, dict) + else held if isinstance(held, list) else ()): if isinstance(entry, dict) and isinstance(entry.get("view"), dict): yield entry["view"] + +def _view_app_ids(view): + """The distinct App ids a View's `config` or `presentation` names.""" + ids = [] + for key in ("config", "presentation"): + config = view.get(key) if isinstance(view, dict) else None + ref = config.get("application") if isinstance(config, dict) else None + app_id = str(ref.get("id") or "") if isinstance(ref, dict) else "" + if app_id and app_id not in ids: + ids.append(app_id) + return ids + + +def _strip_view_refs(doc, app_id): + """Remove `config.application` naming `app_id` from every View in one workspace document. + + Walks every top-level member, reserved ones included, and each owner's trash entries. + Returns the number of Views changed. Mutates `doc` in place. + """ cleared = 0 - for bucket in (doc.values() if isinstance(doc, dict) else ()): - if not isinstance(bucket, dict): - continue - for view in views_of(bucket): + if isinstance(doc, dict): + for view in _workspace_views(doc): changed = False for key in ("config", "presentation"): config = view.get(key) if isinstance(view, dict) else None @@ -1711,7 +1917,7 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None, doc=_clean_doc(raw); app=doc["applications"].get(app_id) if not app: raise ApplicationError("application does not exist") if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases): - if not _database_free_code(app): + if not _database_free(app): raise ApplicationError("application does not exist") if not may_edit(app,user,is_admin): raise PermissionError("application belongs to another user") if confirm_name is not _ANY_NAME and ( @@ -1732,6 +1938,20 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None, result["viewsCleared"] = cleared if failed: result["viewsNotCleared"] = "; ".join(failed) + kind = removed.get("templateKind") + contract = _app_templates.BY_KIND.get(kind) + if contract and contract.get("module"): + try: + if not any(row.get("templateKind") == kind for row in load(runtime)["applications"].values()): + from importlib import import_module + module = import_module(contract["module"]) + cleanup = next((getattr(module, name) for name in + ("remove_app_data", "on_app_removed", "remove") + if callable(getattr(module, name, None))), None) + if cleanup is not None: + cleanup(removed.get("tenant") or runtime.key, st=runtime) + except Exception as exc: + result["appDataNotCleared"] = type(exc).__name__ return result diff --git a/api/assistant_stream.py b/api/assistant_stream.py index a421bdd0ee562743a1533c901d4e933bfc90f1de..d53552794162905c0b54b61660bfa6283f8309a4 100644 --- a/api/assistant_stream.py +++ b/api/assistant_stream.py @@ -1,10 +1,10 @@ """Expose one Query turn as server-sent events with same-turn error recovery.""" import asyncio import contextvars -import gzip -import io +import inspect import json import threading +import zlib import anyio @@ -39,34 +39,24 @@ class StreamGZipMiddleware: await responder(scope, receive, send) -from starlette.middleware.gzip import GZipResponder, IdentityResponder +from starlette.middleware.gzip import IdentityResponder -class _OffLoopGZipResponder(GZipResponder): +class _OffLoopGZipResponder(IdentityResponder): """Retain Starlette's header and chunk behavior with compression outside the event loop.""" + content_encoding = "gzip" + def __init__(self, app, minimum_size, compresslevel): - IdentityResponder.__init__(self, app, minimum_size) + super().__init__(app, minimum_size) self.compresslevel = compresslevel - self.gzip_file = None - - async def __call__(self, scope, receive, send): - try: - await IdentityResponder.__call__(self, scope, receive, send) - finally: - if self.gzip_file is not None: - def close(): - self.gzip_file.close() - self.gzip_buffer.close() - with anyio.CancelScope(shield=True): - await anyio.to_thread.run_sync(close) + self._compressor = None def _compress(self, body, more): - if self.gzip_file is None: - self.gzip_buffer = io.BytesIO() - self.gzip_file = gzip.GzipFile(mode="wb", fileobj=self.gzip_buffer, - compresslevel=self.compresslevel) - return GZipResponder.apply_compression(self, body, more_body=more) + if self._compressor is None: + self._compressor = zlib.compressobj(self.compresslevel, zlib.DEFLATED, 16 + zlib.MAX_WBITS) + out = self._compressor.compress(body) + return out if more else out + self._compressor.flush() async def send_with_compression(self, message): if (message["type"] == "http.response.body" @@ -77,8 +67,13 @@ class _OffLoopGZipResponder(GZipResponder): lambda: self._compress(body, more)) await super().send_with_compression(message) - def apply_compression(self, body, *, more_body): - return self._compressed_body + # Starlette 1.4 awaits apply_compression and 1.3 calls it, so match the installed base. + if inspect.iscoroutinefunction(IdentityResponder.apply_compression): + async def apply_compression(self, body, *, more_body): + return self._compressed_body + else: + def apply_compression(self, body, *, more_body): + return self._compressed_body def _event(name, data): diff --git a/api/catalog_store.py b/api/catalog_store.py index f02c4225151fd221fb6fdc8488373cba9dd3ce39..7c5fff4c02798f045d065c42a79bacabf863e9ca 100644 --- a/api/catalog_store.py +++ b/api/catalog_store.py @@ -7,6 +7,7 @@ import hashlib import json import app_rules +import app_templates import application_registry as applications from core import write_execution from deps import err @@ -116,7 +117,33 @@ def normalize_pages(pages, *, legacy=False, identity=""): def _snapshot(app): - return {key: copy.deepcopy(value) for key, value in app.items() if key in _SNAPSHOT_KEYS} + # R33: a snapshot keeps a source or field map only when it differs from the C1 binding. + kept = _SNAPSHOT_KEYS - {"sourceDatabase", "fieldMap"} + row = {key: copy.deepcopy(value) for key, value in app.items() if key in kept} + row.update(copy.deepcopy(app_templates.own_binding(app))) + return row + + +def migrate_snapshots(runtime, app_ids): + """R33: drop a C1 binding from the named Apps' catalog snapshots. Writes only on a change.""" + wanted = set(app_ids) + + def stale(doc): + return sorted(app_id for app_id, row in doc["appSnapshots"].items() + if app_id in wanted and isinstance(row, dict) + and applications._c1_rewrite(row) is not None) + + if not stale(_load(runtime)): + return "unchanged" + + def apply(doc): + changed = stale(doc) + for app_id in changed: + row = applications._c1_rewrite(doc["appSnapshots"][app_id]) + doc["appSnapshots"][app_id] = {key: value for key, value in row.items() if key in _SNAPSHOT_KEYS} + return changed + + return f"{len(_update(runtime, apply))} rewritten" def _insert(doc, record): @@ -194,7 +221,8 @@ def require_app(session, app_id, *, write=False, allow_orphan=False): may = applications.may_edit if write else applications.may_read if not may(app, session.uname, session.admin): raise err(403, "catalog_forbidden", "Your account cannot access this Catalog App. Ask its owner for access.") - source = app.get("sourceDatabase") + # R33: Catalog reads Odoo products through its C1 binding unless the record holds its own. + source = app_templates.bound(app).get("sourceDatabase") _assert_database_scope(session, source) session.require(source) if not perm_scope.may_read(session.user, source, st=runtime): @@ -202,12 +230,12 @@ def require_app(session, app_id, *, write=False, allow_orphan=False): if write: _assert_write_scope(session, "catalogs") contracts = applications.chart_field_contracts(session, [source]) + fields = applications._contract_fields(source, field_contracts=contracts) try: - applications._template_definition("catalog", app, field_contracts=contracts, - allowed_databases={source}) + app = applications.bound_application(app, fields) except applications.ApplicationError: raise err(403, "catalog_fields_forbidden", "The Catalog App binds unavailable fields. Ask its owner to rebind readable fields.") from None - return app, applications._contract_fields(source, field_contracts=contracts) + return app, fields def _binding_wall(record, app, fields): diff --git a/api/connectors_amazon_ads.py b/api/connectors_amazon_ads.py new file mode 100644 index 0000000000000000000000000000000000000000..15f7155206469dffef32da0765a0f8e83b8f1ff1 --- /dev/null +++ b/api/connectors_amazon_ads.py @@ -0,0 +1,130 @@ +"""connectors_amazon_ads.py: the Amazon Ads connector's API layer (S79-T13): the probe and the card state. + +The transport is `platform/harness/amazon_ads_client.py`. This module adds the liveness probe the +keychain calls through `register_prober`, and the card state the Connectors directory reads. + +⛔ READ ONLY. No route here writes. The probe makes one Login with Amazon token exchange and one +profile listing GET, and only when an administrator tests a stored credential. +⛔ `AMAZON_ADS_FIXTURE=1` answers from the fixture tokens below and opens no socket. +""" +from __future__ import annotations + +import json +import sys +import urllib.request +from pathlib import Path + +_PLATFORM = Path(__file__).resolve().parents[2] / "platform" +if str(_PLATFORM) not in sys.path: + sys.path.insert(0, str(_PLATFORM)) + +from harness import amazon_ads_client as client # noqa: E402 + +KEY = client.KEY +LABEL = client.LABEL +#: Must be a `core.keychain.ENTRY_TYPES` member, or storing the credential 400s. +ENTRY_TYPE = client.ENTRY_TYPE +CREDENTIAL_FIELDS = client.CREDENTIAL_FIELDS + +#: The one read the probe makes after the token exchange. GET only, one path. +PROFILES_PATH = "v2/profiles" + +#: Fixture refresh tokens. A token with the prefix passes; the expired one fails. +FIXTURE_TOKEN_PREFIX = "Atzr|fixture-ads" +FIXTURE_REFRESH_TOKEN = FIXTURE_TOKEN_PREFIX + "-ok" +FIXTURE_EXPIRED_TOKEN = FIXTURE_TOKEN_PREFIX + "-expired" +FIXTURE_PROFILE_ID = "1234567890" + +#: Messages a person reads. No dash, no provider text. +OK_MESSAGE = "Amazon Ads confirmed profile access." +EXPIRED_MESSAGE = "The Amazon Ads token expired. Connect again." +NO_PROFILE_MESSAGE = "The token does not reach this profile. Check the profile ID." +NOT_FIXTURE_MESSAGE = "Fixture mode accepts only the fixture token." +SIGN_IN_MESSAGE = "Amazon Ads did not accept the sign in. Check the fields and connect again." + + +def fixture_fields(*, expired=False): + """The fixture credential an administrator pastes on staging. -> the four fields.""" + return {"client_id": "amzn1.application-oa2-client.fixture", + "client_secret": "fixture-ads-secret", + "refresh_token": FIXTURE_EXPIRED_TOKEN if expired else FIXTURE_REFRESH_TOKEN, + "profile_id": FIXTURE_PROFILE_ID} + + +def missing_fields(fields): + """The credential fields that are absent or blank, in declaration order.""" + f = fields if isinstance(fields, dict) else {} + return [key for key in CREDENTIAL_FIELDS if not isinstance(f.get(key), str) or not f[key].strip()] + + +def _fixture_probe(fields): + token = fields["refresh_token"].strip() + if token == FIXTURE_EXPIRED_TOKEN: + return {"ok": False, "message": EXPIRED_MESSAGE} + if not token.startswith(FIXTURE_TOKEN_PREFIX): + return {"ok": False, "message": NOT_FIXTURE_MESSAGE} + return {"ok": True, "message": OK_MESSAGE} + + +def _profiles(access_token, fields, timeout): + """The advertiser's profile ids. One GET; any failure raises `client.AdsError`.""" + host = client.HOSTS.get(str(fields.get("region") or client.DEFAULT_REGION).lower(), + client.HOSTS[client.DEFAULT_REGION]) + req = urllib.request.Request(f"{host}/{PROFILES_PATH}", method="GET", headers={ + "Authorization": f"Bearer {access_token}", + "Amazon-Advertising-API-ClientId": fields["client_id"], "Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + body = json.loads(r.read().decode("utf-8", "replace") or "[]") + except Exception as exc: # noqa: BLE001 + raise client.AdsError(f"Amazon Ads profiles could not be read ({type(exc).__name__})") from None + return {str(p.get("profileId")) for p in body if isinstance(p, dict)} if isinstance(body, list) else set() + + +def probe_credential(fields, timeout=8): + """`{ok, message}` for a stored `amazon_ads` credential: the keychain's liveness probe.""" + f = fields if isinstance(fields, dict) else {} + missing = missing_fields(f) + if missing: + return {"ok": False, "message": "Missing " + ", ".join(missing) + ". Enter every field."} + if client.fixture_mode(): + return _fixture_probe(f) + try: + profiles = _profiles(client._token(f), f, timeout) + except client.AdsError: + return {"ok": False, "message": SIGN_IN_MESSAGE} + if f["profile_id"].strip() not in profiles: + return {"ok": False, "message": NO_PROFILE_MESSAGE} + return {"ok": True, "message": OK_MESSAGE} + + +def register(): + """Fill the keychain's probe slot for `amazon_ads`. Never raises.""" + try: + import core.keychain as kc + return kc.register_prober(ENTRY_TYPE, probe_credential) + except Exception: # noqa: BLE001 + return False + + +def held(rt, entry_id): + """True when this entry's current ciphertext carries a passing probe proof. No decrypt, no I/O. + + A failed test, an expired token or a replaced secret clears the proof, so the card reads Connect. + """ + try: + from core import keychain as kc + return bool(kc.entry_verified(rt, entry_id)) + except Exception: # noqa: BLE001 + return False + + +def card_state(verified): + """The directory state: `connected` (Settings) only with a verified credential, else `available`.""" + return "connected" if verified else "available" + + +__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "CREDENTIAL_FIELDS", "PROFILES_PATH", "FIXTURE_TOKEN_PREFIX", + "FIXTURE_REFRESH_TOKEN", "FIXTURE_EXPIRED_TOKEN", "FIXTURE_PROFILE_ID", "OK_MESSAGE", + "EXPIRED_MESSAGE", "fixture_fields", "missing_fields", "probe_credential", "register", + "held", "card_state"] diff --git a/api/deps.py b/api/deps.py index bc9e9c720cd9b2677ccc75e7233edd1dc7c4ecd5..27453c6c05de46f90922ec297ca918418fb43716 100644 --- a/api/deps.py +++ b/api/deps.py @@ -1661,3 +1661,60 @@ def module_gate(module_key): session.require(module_key) return session return _dep + + +def _bearer_header(request: Request): + value = request.headers.get("authorization") + return value if isinstance(value, str) and value.strip() else None + + +def _cookie_session(request: Request, response: Response) -> Session: + """`require_session`, or the app's override of it, so a gate's override still applies here.""" + app = request.scope.get("app") + override = (getattr(app, "dependency_overrides", None) or {}).get(require_session) + if override is None: + return require_session(request, response) + import inspect + names = inspect.signature(override).parameters + return override(**{key: value for key, value in (("request", request), ("response", response)) + if key in names}) + + +def require_session_or_bearer(request: Request, response: Response) -> Session: + """The cookie session when a cookie is sent, else an external bearer under its own key's grants. + + `require_session` stays cookie only. A bearer session never reissues the cookie. + """ + bearer = _bearer_header(request) + if bearer is None or request.cookies.get(aios_session.COOKIE_NAME): + try: + return _cookie_session(request, response) + except HTTPException as exc: + if bearer is None or exc.status_code != 401: + raise + import routes_external # routes_external imports deps, so this import stays lazy. + return routes_external.require_external_analyst( + routes_external.require_external_session(bearer, request)) + + +def is_bearer(session) -> bool: + """True for a session an external API key or OAuth token resolved.""" + return (getattr(session, "claims", None) or {}).get("external") is True + + +def require_bearer_write(session, tool_name): + """Refuse a read-only or refresh-only credential on a write route. A cookie session passes.""" + if is_bearer(session): + import routes_external + routes_external._assert_write_scope(session, tool_name) + + +def module_gate_or_bearer(module_key, database=None): + """`module_gate` that also admits a bearer, walled to its key's database scope.""" + def _dep(session: Session = Depends(require_session_or_bearer)) -> Session: + session.require(module_key) + if is_bearer(session): + import routes_external + routes_external._assert_database_scope(session, database or module_key) + return session + return _dep diff --git a/api/error_sink.py b/api/error_sink.py new file mode 100644 index 0000000000000000000000000000000000000000..a72dfa0ffba84875c314cad559cd914297481148 --- /dev/null +++ b/api/error_sink.py @@ -0,0 +1,151 @@ +"""Store secret-free exception groups and send one daily operations digest.""" +from __future__ import annotations + +from datetime import date, datetime, timedelta, timezone +from functools import partial +import hashlib +import json +import logging +from pathlib import Path +import re +import sys +from uuid import uuid4 + +LOG = logging.getLogger(__name__) + + +def utc_now(): + return datetime.now(timezone.utc) + + +def store_backend(): + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "platform")) + import core.store as store + return store + + +def day_key(day): + return "ops_error_sink_" + date.fromisoformat(day).isoformat() + + +def stack_hash(error): + frames = [] + trace = error.__traceback__ + while trace: + code = trace.tb_frame.f_code + frames.append((code.co_filename, code.co_name, trace.tb_lineno)) + trace = trace.tb_next + signature = (type(error).__module__, type(error).__qualname__, frames) + return hashlib.sha256(json.dumps(signature).encode()).hexdigest() + + +def verified_tenant(scope): + from starlette.requests import Request + request = Request(scope) + authorization = request.headers.get("authorization") + if authorization: + external = sys.modules.get("api.routes_external") or sys.modules.get("routes_external") + if external is None: + return "unknown" + try: + value = external.require_external_session(authorization=authorization, request=request).tenant + except Exception: + return "unknown" + else: + import aios_session + claims = aios_session.read(request.cookies.get(aios_session.COOKIE_NAME, "")) + value = (claims or {}).get("t", "unknown") + return value if isinstance(value, str) and re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", value) else "unknown" + + +def record_error(scope, error, *, status=500, backend=None, now=None, tenant_resolver=None): + backend = backend if backend is not None else store_backend() + now = now or utc_now() + if now.utcoffset() is None: + raise ValueError("error timestamps must include a timezone") + now = now.astimezone(timezone.utc) + route = getattr(scope.get("route"), "path", None) or scope.get("_timing_route") or "" + tenant = (tenant_resolver or verified_tenant)(scope) + event = {"route": route, "status": int(status), "tenant": tenant, "stack_hash": stack_hash(error)} + group = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest() + stamp = now.isoformat() + + def update(document): + document = document or {"events": {}} + events = document.setdefault("events", {}) + previous = events.get(group, {}) + events[group] = {**event, "count": previous.get("count", 0) + 1, + "first": previous.get("first", stamp), "last": stamp} + return document + + backend.update(day_key(now.date().isoformat()), update) + return group + + +class ErrorSinkMiddleware: + def __init__(self, app, *, backend=None, tenant_resolver=None, clock=None): + self.app, self.backend = app, backend + self.tenant_resolver, self.clock = tenant_resolver, clock or utc_now + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + return await self.app(scope, receive, send) + try: + return await self.app(scope, receive, send) + except Exception as error: + if not scope.get("_error_sink_recorded"): + scope["_error_sink_recorded"] = True + try: + from starlette.concurrency import run_in_threadpool + await run_in_threadpool(partial(record_error, scope, error, backend=self.backend, + now=self.clock(), tenant_resolver=self.tenant_resolver)) + except Exception: + LOG.error("Error sink persistence failed; original request exception preserved") + raise + + +def daily_digest(*, sender, day=None, backend=None, now=None): + backend = backend if backend is not None else store_backend() + now = (now or utc_now()).astimezone(timezone.utc) + day = day or (now.date() - timedelta(days=1)).isoformat() + key, claim = day_key(day), uuid4().hex + + def acquire(document): + document = document or {"events": {}} + previous = document.get("digest") or {} + if not document.get("events") or previous.get("state") in {"sent", "sending", "uncertain"}: + return document + document["digest"] = {"state": "sending", "claim": claim, "at": now.isoformat()} + return document + + document = backend.update(key, acquire) + receipt = document.get("digest") or {} + if receipt.get("claim") != claim: + pending = receipt.get("state") in {"sending", "uncertain"} + return {"ok": not pending, "sent": False, "day": day, + "reason": "delivery_needs_reconciliation" if pending else "empty_or_already_sent"} + events = sorted(document["events"].values(), key=lambda event: (-event["count"], event["stack_hash"])) + total = sum(event["count"] for event in events) + lines = [f"Loopable errors for {day}: {total} events in {len(events)} groups"] + for event in events: + line = f"{event['count']} x {event['status']} {event['route']} [{event['tenant']}] {event['stack_hash'][:12]}" + if len(("\n".join(lines) + "\n" + line).encode()) > 3500: + lines.append(f"Remaining groups: {len(events) - len(lines) + 1}. Full groups are in the error sink.") + break + lines.append(line) + + def finish(current, state): + if (current.get("digest") or {}).get("claim") == claim: + current["digest"] = {"state": state, "claim": claim, "at": now.isoformat()} + return current + + try: + delivered = sender("\n".join(lines)) + except Exception: + backend.update(key, lambda current: finish(current, "uncertain")) + raise + if delivered is False: + backend.update(key, lambda current: finish(current, "failed")) + raise RuntimeError("digest sender refused delivery") + backend.update(key, lambda current: finish(current, "sent")) + return {"sent": True, "day": day, "groups": len(events), "events": total} diff --git a/api/external_freshness.py b/api/external_freshness.py index 17809296326c5cb23874a9b7d41a352da2aef74a..9cc5d3e41a4824b03e05f76e7ae481add0894523 100644 --- a/api/external_freshness.py +++ b/api/external_freshness.py @@ -67,6 +67,21 @@ _AMAZON_DERIVED_INPUTS = {"sku_keywords": ("listing_items", "search_query_perfor _AMAZON_DERIVED_RECEIPTS = {"keyword_ranks": "amzn_apify_keyword_ranks"} +def _amazon_raw(table_key, source=None): + """The mirror table behind one `ut_amzn_*` key: the relational map, else an Ads spec (S79-T13).""" + if source is None: + import amazon_relational + source = amazon_relational.SOURCE + raw = source.get(table_key) + if raw: + return raw + from core import connector_spec + spec = connector_spec.spec_for(table_key) + if spec is not None and connector_spec.connector_of(table_key) == connector_spec.AMAZON_ADS_CONNECTOR: + return spec.source + return None + + def amazon_sync_state(cur): """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read.""" return {str(entity): at for entity, at in cur.execute( @@ -94,9 +109,7 @@ def amazon_sync_stamp(cur, table_key, state=None): R11: a refusal overwrites the raw receipt and a pass with no window writes no mark, so the reconciliation proofs also count. The time is the receipt's own, never a restore or read time. """ - import amazon_relational - - raw = amazon_relational.SOURCE.get(table_key) + raw = _amazon_raw(table_key) if not raw or (cur is None and state is None): return None if state is None: @@ -161,7 +174,7 @@ def amazon_nav(session, pages): bases = {str(name) for (name,) in cur.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' " "AND table_type = 'BASE TABLE' AND starts_with(table_name, 'amzn_')").fetchall()} - raws = {page["key"]: amazon_relational.SOURCE.get(page["key"]) for page in rows} + raws = {page["key"]: _amazon_raw(page["key"], amazon_relational.SOURCE) for page in rows} probed = sorted({raw for raw in raws.values() if raw in bases}) held = {} if probed: diff --git a/api/external_openapi.py b/api/external_openapi.py index 12fe81d9954ce8608a0730e7198bbf73ea804ec1..2b07983798c663f7be94d198a459ab8c3378753b 100644 --- a/api/external_openapi.py +++ b/api/external_openapi.py @@ -43,8 +43,8 @@ def _field_schema(field, source_kind): kind = _JSON_TYPE.get(field.get("type"), "string") types = [kind] # Materialised table writes retain numeric cells as strings; reads preserve those bytes. - if source_kind == "materialised_user_table" and kind in {"number", "integer"}: - types.append("string") + if kind in {"number", "integer"}: + types = ["number", "string"] return {"type": [*types, "null"]} diff --git a/api/loopy_catalog_tools.py b/api/loopy_catalog_tools.py index 9fd7c0aeee5eb39a786ef259ce6b7736dc956dd2..3669273adc7e2c28a0f212b79d77bea42242c1d2 100644 --- a/api/loopy_catalog_tools.py +++ b/api/loopy_catalog_tools.py @@ -11,6 +11,7 @@ def _build_refusal(code, message): def _build_app(ctx, args): import application_registry as applications import catalog_store + import app_templates from fastapi import HTTPException source = args["sourceDatabase"] @@ -23,8 +24,8 @@ def _build_app(ctx, args): if app["sourceDatabase"] != source: _build_refusal("catalog_source_mismatch", "Choose a Catalog App over the requested source database.") else: - candidates = [app for app in applications.load(ctx.session.runtime)["applications"].values() - if app.get("templateKind") == "catalog" and app.get("sourceDatabase") == source + candidates = [app_templates.bound(app) for app in applications.load(ctx.session.runtime)["applications"].values() + if app.get("templateKind") == "catalog" and app_templates.bound(app).get("sourceDatabase") == source and app.get("retired") is not True and applications.may_read(app, ctx.session.uname, ctx.session.admin) and applications.may_edit(app, ctx.session.uname, ctx.session.admin)] @@ -40,8 +41,8 @@ def _build_app(ctx, args): if app is None: _build_refusal("catalog_app_unavailable", ( "No editable Catalog App is available for this source. Create one with apps.apply_template " - "using templateKind catalog and a fieldMap that maps key and title to field keys of this source, " - "for example {\"key\": \"sku\", \"title\": \"name\"}. apps.templates lists the optional roles.")) + "using templateKind catalog. " + "The App binds its Odoo products automatically.")) fields = {field["key"] for field in schema["fields"]} if any(field not in fields for field in app["fieldMap"].values()): _build_refusal("catalog_fields_forbidden", "The Catalog App uses unavailable fields. Rebind it to readable source fields.") @@ -116,6 +117,7 @@ def _build_rows(ctx, source, fields, filters, sorts): def catalog_build(ctx, args): """Build a C1 record through the caller's staged runtime and permissioned query callbacks.""" import catalog_store + import app_templates from core import external_tools, write_execution from loopy_file_tools import _secure_link from loopy_links import application_link @@ -220,6 +222,22 @@ def build_schema(): "required": ["sourceDatabase"], "additionalProperties": False} +def _register_render_read(session, source): + """Record the render's product read in the caller's producing turn, so the PDF keeps its source (D-9854). + + `assistant_files.save` keeps only sources the turn proved it read. The render reads the source + under this caller even when the producing turn holds another session object for the same user. + """ + import agent_runtime + + producing = agent_runtime._PRODUCING_TURN.get() + if producing is None: + return + caller, turn = producing + if caller.runtime is session.runtime and caller.uname == session.uname: + agent_runtime._merge_sources(caller, turn, [source]) + + def catalog_pdf(session, args): import assistant_files import catalog_render @@ -234,6 +252,7 @@ def catalog_pdf(session, args): _secure_link(links.app_link("#/assistant")) _secure_link(links.artifact_link("catalog-preview")) data, record, app = catalog_render.render_catalog(session, args["catalog_id"], ai=True) + _register_render_read(session, app["sourceDatabase"]) title = _no_dashes(record["name"]).strip()[:200] saved = assistant_files.save(session.runtime, session.uname, title=title, data=data, mime="application/pdf", door=door, @@ -287,6 +306,7 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None): import sys import threading import catalog_store + import app_templates render = render or sys.modules[__name__].catalog_pdf budget = export_budget() if budget is None else budget diff --git a/api/main.py b/api/main.py index 67a6f97aca0213df12b89eb5a5773b97b32971e8..ade82abc6c9498a9879571e4efa952b4bba1dfc2 100644 --- a/api/main.py +++ b/api/main.py @@ -146,7 +146,8 @@ from request_timing import RequestTiming # noqa: E402 class _TimedFastAPI(FastAPI): def build_middleware_stack(self): # Include error responses, freeze refusals, admission and compressed body sends. - return RequestTiming(super().build_middleware_stack()) + from error_sink import ErrorSinkMiddleware + return ErrorSinkMiddleware(RequestTiming(super().build_middleware_stack())) app = _TimedFastAPI(title="AIOS web API") @@ -208,7 +209,23 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): async def _resource_unavailable(request: Request, exc: Exception): from request_timing import resource_response - return resource_response(exc) + return _with_holder(resource_response(exc), exc) + + +def _with_holder(response, exc): + seen = set() + while exc is not None and id(exc) not in seen: + seen.add(id(exc)) + holder = getattr(exc, "aios_holder", None) + if isinstance(holder, dict) and response is not None: + import json + body = json.loads(response.body) + body["error"]["holder"] = holder + body["error"]["message"] = ( + f"Another change is still saving ({holder.get('label', 'another request')}). Try again shortly.") + return JSONResponse(body, status_code=503, headers={"Retry-After": "1"}) + exc = exc.__cause__ or exc.__context__ + return response @app.exception_handler(Exception) @@ -216,28 +233,31 @@ async def _unexpected_error(request: Request, exc: Exception): from request_timing import resource_response busy = resource_response(exc) if busy is not None: - return busy + return _with_holder(busy, exc) return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}}, status_code=500) -from core.odoo import OdooPoolTimeout -from core.store_pg import SharedReadTimeout +from core.odoo import OdooPoolTimeout, OdooRequestTimeout +from core.store_pg import SharedReadTimeout, PostgresOperationTimeout from core.request_timing import RequestDeadlineExceeded from request_timing import DeadlineCapacityExceeded from duckdb import OutOfMemoryException -for _resource_error in (OdooPoolTimeout, SharedReadTimeout, RequestDeadlineExceeded, - DeadlineCapacityExceeded, OutOfMemoryException): +# Each busy cause needs its own handler: the catch-all `Exception` one answers, then re-raises to uvicorn. +for _resource_error in (OdooPoolTimeout, OdooRequestTimeout, SharedReadTimeout, PostgresOperationTimeout, + RequestDeadlineExceeded, DeadlineCapacityExceeded, OutOfMemoryException): app.add_exception_handler(_resource_error, _resource_unavailable) try: from psycopg_pool import PoolTimeout, TooManyRequests from psycopg import OperationalError, InterfaceError from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout + from psycopg.errors import ConnectionTimeout except ImportError: pass else: for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable, - OperationalError, InterfaceError, IdleInTransactionSessionTimeout): + OperationalError, InterfaceError, IdleInTransactionSessionTimeout, + ConnectionTimeout): app.add_exception_handler(_resource_error, _resource_unavailable) @@ -381,6 +401,35 @@ async def health(challenge: str = ""): return {"ok": True} +def _mirror_answers(): + """The resident Odoo mirror's live state for readiness. Never raises.""" + try: + from harness import datastore as _ready_ds + return bool(_ready_ds.ready()) + except Exception: # noqa: BLE001 + return False + + +@app.get("/api/v1/ready") +def readiness_check(): + """C9: 200 only when the mirror is installed, the pools are open and the first reads answered. + + Unauthenticated, because the deploy switch polls it before anyone signs in. It carries the + version on purpose (C9), unlike /api/health. + """ + report = process_role.readiness(mirror_live=_mirror_answers()) + body = {"ready": report["ready"], "version": os.environ.get("AIOS_VERSION") or "unknown", + "checks": report["checks"]} + headers = {"Cache-Control": "no-store"} + if report["ready"]: + return JSONResponse(body, headers=headers) + body["failing"] = report["failing"] + body["error"] = {"code": "not_ready", + "message": "Not ready: " + ", ".join(report["failing"]) + ". " + + " ".join(report["detail"].values())} + return JSONResponse(body, status_code=503, headers=headers) + + # W59-T43: boot stays available while a cross-lane module is absent, but never silently. # verify_wiring.W59_PENDING_CROSS_LANE_HALVES owns the shrink-only declarations and rejects # any failure outside them, including a missing dependency INSIDE a declared router. @@ -609,6 +658,8 @@ app.include_router(routes_competitor_watch.router) # S78-T16 / S78-T17 competit _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount. _mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors. _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount. +import app_routes +app_routes.mount_app_routers(app) # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) @@ -1041,7 +1092,7 @@ def _restore_amazon_mirror_once(why, tenant_key): # Publish measured definitions before any Odoo read can block the resident writer. import process_role as _process_role # The relational rebuild writes the store, so only the sync worker runs it. - if (result.get("state") != "absent" and _process_role.allows("heavy") and + if (result.get("state") != "absent" and _process_role.may_write() and rt.datastore_path.resolve() == _ds.DB_PATH.resolve()): state = _rebuild_amazon_relational(why, rt) if state.get("changed") and not state.get("applied") and not state.get("known"): @@ -2914,10 +2965,11 @@ def _prewarm(): if _process_role.role() == "worker": # The warm below fills per-process memos, which the sync worker never serves. return + # D-273: the shared read warm fills both default pools in runtime.pool_cache. + _warm_read_pools(attempts=1) try: from harness import runtime as _runtime rt = _runtime.get_runtime("royal-imports") - routes_customers.warm_default(rt) import pages as _pages _pages.warm_default(rt) # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent @@ -3098,8 +3150,8 @@ def _restore_sync_request_desks(): from core import connector_sync from harness import runtime import process_role as _process_role - # The web role never claims, so it must not requeue the worker's in-flight claims. - requeue = _process_role.role() != "web" + # Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease. + requeue = _process_role.may_write() for tenant_key in runtime.known_tenants(): try: connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue) @@ -3110,30 +3162,90 @@ def _restore_sync_request_desks(): def _boot_mirrors(): import process_role as _role_boot _restore_sync_request_desks() - _seed_mirror_if_absent() + if _role_boot.role() != "web" or _role_boot.claim_primary(): + # With several web workers only the primary restores the mirror. The others read it. + _seed_mirror_if_absent() if _role_boot.role() == "web": + # Readiness also waits for the worker's boot hand-offs and the mirror's own completeness. + _role_boot.mark_ready("mirror", True) # Every connector but Odoo reaches the web process through the worker's hand-off. print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker") # The biggest read caches are this process's own, so they warm here, before the first reader pays. _role_boot.start("read-warm", "light", _warm_read_pools) return + if _role_boot.role() == "worker": + try: + _restore_amazon_mirror("boot") + _restore_declared_mirrors("boot") + finally: + # Every boot hand-off is on disk now; the web process's readiness waits for this line. + _role_boot.note_boot_restore() + return _restore_amazon_mirror("boot") _restore_declared_mirrors("boot") + # all and reader have no worker: their own restores finish before readiness and the warm. + _role_boot.mark_ready("mirror", True) + if _role_boot.role() == "reader" or data_binding.is_space(): + # A local `all` run keeps its old boot: AIOS_PREWARM=1 warms it. + _warm_read_pools() -def _warm_read_pools(): - """Build the consolidated customer and product pools once after boot, off the request path.""" - import time as _t - t0 = _t.time() +#: A failed boot warm retries, so one slow first read cannot leave readiness red until a restart. +_READ_WARM_ATTEMPTS = 40 +_READ_WARM_RETRY_SECONDS = 15 + + +def _probe_read_pools(rt): + """Open the mirror read cursor and the tenant store once. '' when both answer, else the cause.""" + from harness import datastore as _probe_ds + if not _probe_ds.DB_PATH.exists(): + return "The mirror file is missing." + cur = _probe_ds.ro_cursor() try: - from harness import runtime as _runtime - import routes_products - rt = _runtime.get_runtime("royal-imports") - routes_customers.warm_default(rt) - routes_products._pool_for(rt, None) - print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s") - except Exception as exc: # noqa: BLE001 - print(f"[aios-api] read warm skipped ({type(exc).__name__}: {exc})") + cur.execute("SELECT 1").fetchone() + finally: + cur.close() + if rt.available() is False: + return "The tenant store did not answer." + return "" + + +def _warm_read_pools(attempts=None, retry_seconds=None): + """Open the read pools, then build the consolidated customer and product pools, off the request path. + + Marks the `pools` and `first_reads` readiness checks. A failure never clears a check that passed. + """ + import time as _t + import process_role as _warm_role + attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts)) + retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds + for attempt in range(1, attempts + 1): + t0 = _t.time() + stage = "pools" + try: + from harness import runtime as _runtime + import routes_products + rt = _runtime.get_runtime("royal-imports") + problem = _probe_read_pools(rt) + if problem: + raise RuntimeError(problem) + _warm_role.mark_ready("pools", True) + stage = "first_reads" + routes_customers.warm_default(rt) + routes_products._pool_for(rt, None) + _warm_role.mark_ready("first_reads", True) + routes_customers.warm_bu_pools(rt) + print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s") + return True + except Exception as exc: # noqa: BLE001 + if not _warm_role._READINESS.get(stage, {}).get("ok"): + _warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} " + f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.") + print(f"[aios-api] read warm failed at {stage} ({type(exc).__name__}: {exc}); " + f"attempt {attempt} of {attempts}") + if attempt < attempts: + _t.sleep(retry_seconds) + return False def _restore_declared_mirrors(why): @@ -3183,7 +3295,7 @@ if (not data_binding.gate_process() if (not data_binding.gate_process() and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0" - and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1") + and (bool(os.environ.get("SPACE_ID")) or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1") and not _freeze_holds("app migrations")): import process_role as _role_apps _role_apps.thread(target=_boot_app_migrations, args=("boot",), daemon=True, kind="heavy", diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 8098f83bfabefb5054fdd0419fc45d73e428382a..952f938588075244c0e3ba72d05a3080f8c12ba3 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -2572,26 +2572,33 @@ class Refused(Exception): #: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole #: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. #: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. +#: (key, label, type, converter) per payment fact, in column order. +_PAYMENT_FACTS = ( + ("amount_company_currency_signed", "Company currency amount", "currency", + lambda value: float(value or 0)), + ("date", "Date", "date", _as_date), ("state", "State", "text", str), + ("payment_type", "Payment type", "text", str), ("partner_type", "Partner type", "text", str), + ("is_internal_transfer", "Internal transfer", "checkbox", bool), + ("partner_id", "Customer ID", "int", lambda value: int(value or 0))) + + def payment_fields(): """Declare company-currency payment facts without materializing source rows.""" - return [_preset({"key": key, "label": label, "type": kind, "source": "overlay", - "default": True}) for key, label, kind in ( - ("amount_company_currency_signed", "Company currency amount", "currency"), - ("date", "Date", "date"), ("state", "State", "text"), - ("payment_type", "Payment type", "text"), ("partner_type", "Partner type", "text"), - ("is_internal_transfer", "Internal transfer", "checkbox"), - ("partner_id", "Customer ID", "int"))] + # The pinned identity column is what ODOO_SPEC's payments row names (pinned and identity). + return [_preset({"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": True, "pinned": True, + "description": "The `account.payment` id. Also this row's id."})] + [ + _preset({"key": key, "label": label, "type": kind, "source": "overlay", "default": True}) + for key, label, kind, _convert in _PAYMENT_FACTS] def payment_source_specs(): """Bind the payment database to its native payment identity and gross receipt amount.""" + cols = {"odoo_id": ("pay.id", lambda value: int(value or 0))} + cols.update({key: ("pay." + key, convert) for key, _label, _kind, convert in _PAYMENT_FACTS}) return {PAYMENTS_KEY: { "table": "account_payment", "from_sql": "account_payment pay", "tables": {"pay": "account_payment"}, - "alias": "pay", "id": "pay.id", "where": "", - "cols": {field["key"]: ("pay." + field["key"], converter) - for field, converter in zip(payment_fields(), - (lambda value: float(value or 0), _as_date, str, str, str, bool, - lambda value: int(value or 0)))}}} + "alias": "pay", "id": "pay.id", "where": "", "cols": cols}} TABLES = ( @@ -2716,12 +2723,19 @@ def _selection_field_contracts(fields, catalogue_row): return out -def discover_readable_models(client): +def _read_access(client, model): + """Odoo's own read verdict for this credential; raises when Odoo gives no answer.""" + return bool(client.execute_kw( + model, "check_access_rights", ["read"], {"raise_exception": False})) + + +def discover_readable_models(client, verdicts=None): """Read-only introspection over an injected Odoo client. ``ir.model`` supplies candidates, ``check_access_rights('read')`` narrows them to this API account, and ``fields_get`` supplies only fields that account may inspect. No method in this path can mutate Odoo; the production client independently blocks write-shaped method names. + A ``verdicts`` dict receives every candidate: True, False (an explicit refusal) or None. """ errors = [] try: @@ -2734,13 +2748,16 @@ def discover_readable_models(client): model = str((item or {}).get("model") or "").strip().lower() if not model: continue + if verdicts is not None: + verdicts[model] = None try: - readable = client.execute_kw( - model, "check_access_rights", ["read"], {"raise_exception": False}) + readable = _read_access(client, model) except Exception as exc: # noqa: BLE001 errors.append(f"{model}: access: {type(exc).__name__}: {exc}") continue - if not bool(readable): + if verdicts is not None: + verdicts[model] = readable + if not readable: continue try: fields = client.fields_get( @@ -2764,28 +2781,125 @@ def _catalog_slot(connection_id): return str(connection_id or DEFAULT_CONNECTION) -def readable_model_catalog(rt, connection_id=None): - """The last complete tenant-local catalogue; reading it never contacts Odoo.""" +def _read_catalog_slot(rt, connection_id=None): + """One read of this connection's raw catalogue slot, or {} before any discovery.""" try: cached = rt.get(MODEL_CATALOG_KEY) if rt is not None else None except Exception: # noqa: BLE001 cached = None if not isinstance(cached, dict) or cached.get("version") != MODEL_CATALOG_VERSION: - return {"version": MODEL_CATALOG_VERSION, "complete": False, "capturedAt": "", - "models": [], "errors": []} - cached = (cached.get("connections") or {}).get(_catalog_slot(connection_id)) or {} + return {} + slot = (cached.get("connections") or {}).get(_catalog_slot(connection_id)) + return slot if isinstance(slot, dict) else {} + + +def _catalog_view(slot): + slot = slot if isinstance(slot, dict) else {} return {"version": MODEL_CATALOG_VERSION, - "complete": cached.get("complete") is True, - "capturedAt": str(cached.get("capturedAt") or ""), - "models": [dict(row) for row in (cached.get("models") or ()) + "complete": slot.get("complete") is True, + "capturedAt": str(slot.get("capturedAt") or ""), + "models": [dict(row) for row in (slot.get("models") or ()) if isinstance(row, dict) and row.get("model")], - "errors": [str(value) for value in (cached.get("errors") or ())][:50]} + "errors": [str(value) for value in (slot.get("errors") or ())][:50]} + + +def _model_names(values): + return sorted({str(value).strip().lower() for value in (values or ()) + if str(value or "").strip()}) + + +def _access_view(slot): + """Who answered this slot, and which models Odoo refused or does not expose.""" + slot = slot if isinstance(slot, dict) else {} + return {"source": str(slot.get("source") or ""), + "denied": _model_names(slot.get("denied")), + "absent": _model_names(slot.get("absent"))} + + +def readable_model_catalog(rt, connection_id=None): + """The last complete tenant-local catalogue; reading it never contacts Odoo.""" + return _catalog_view(_read_catalog_slot(rt, connection_id)) + + +def model_access(rt, connection_id=None): + """This slot's explicit access facts (D-748, R70); reading them never contacts Odoo.""" + return _access_view(_read_catalog_slot(rt, connection_id)) + + +def _credential_identity(rt, connection_id=None): + """The stored credential that answers for this slot, or "" when unknown. Decrypts nothing.""" + slot = _catalog_slot(connection_id) + if slot != DEFAULT_CONNECTION: + return "keychain:" + slot + default = default_connection_id(rt) + if default: + return "keychain:" + str(default) + return "env" if getattr(rt, "key", None) == "royal-imports" else "" + + +def _same_credential(stamped, current): + """Evidence stamped by another credential is not this credential's; an unknown side matches.""" + return not stamped or not current or stamped == current + + +def _merged_access(previous, verdicts, identity): + """Explicit verdicts replace earlier ones; a model Odoo did not answer keeps its last verdict.""" + previous = previous or {"source": "", "denied": [], "absent": []} + denied = set(previous["denied"]) + for model, verdict in verdicts.items(): + if verdict is True: + denied.discard(model) + elif verdict is False: + denied.add(model) + # An answered ir.model listing is the exposure fact; without one the last fact stands. + absent = ({model for model in ODOO_TABLE_MODELS.values() if model and model not in verdicts} + if verdicts else set(previous["absent"])) + return {"source": identity or previous["source"], "denied": sorted(denied), + "absent": sorted(absent)} + + +def _write_catalog_slot(rt, connection_id, fn): + """Rewrite one connection's slot through `fn`; every other slot is carried unchanged.""" + slot_key = _catalog_slot(connection_id) + + def _cache(current_cache): + current_cache = dict(current_cache) if isinstance(current_cache, dict) else {} + connections = dict(current_cache.get("connections") or {}) + connections[slot_key] = fn(dict(connections.get(slot_key) or {})) + return {"version": MODEL_CATALOG_VERSION, "connections": connections} + + rt.update(MODEL_CATALOG_KEY, _cache, flush="sync") + + +def _narrow_catalog(rt, connection_id, current, previous, access): + """Withdraw refused and unexposed models from a retained slot; nothing here widens it.""" + withdrawn = set(access["denied"]) | set(access["absent"]) + models = [row for row in current["models"] if row.get("model") not in withdrawn] + if (models == current["models"] and access["denied"] == previous["denied"] + and access["absent"] == previous["absent"]): + return current + + def _narrowed(stored): + # A narrowing is not a capture: complete, capturedAt, errors and the stamp stay as stored. + if not stored: + stored = {"source": access["source"]} + stored["models"] = [row for row in (stored.get("models") or ()) + if not (isinstance(row, dict) and row.get("model") in withdrawn)] + stored["denied"], stored["absent"] = access["denied"], access["absent"] + return stored + + _write_catalog_slot(rt, connection_id, _narrowed) + return {**current, "models": models} def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False): """Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure.""" - current = readable_model_catalog(rt, connection_id) - if (current["capturedAt"] and not force + slot = _read_catalog_slot(rt, connection_id) + current, previous = _catalog_view(slot), _access_view(slot) + identity = _credential_identity(rt, connection_id) + same = _same_credential(previous["source"], identity) + # A slot another credential answered is never fresh for this one, whatever its age. + if (current["capturedAt"] and not force and same and not any(_missing_selection_metadata(row) for row in current["models"])): try: captured = _dt.datetime.fromisoformat(current["capturedAt"]) @@ -2802,46 +2916,42 @@ def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=Fa source = None if source is None: return current + verdicts = {} try: if client is None: from core.odoo import get_odoo with source._bound(): # tenant credential + scope - rows, errors = discover_readable_models(get_odoo()) + rows, errors = discover_readable_models(get_odoo(), verdicts) else: - rows, errors = discover_readable_models(client) + rows, errors = discover_readable_models(client, verdicts) except Exception as exc: # noqa: BLE001 rows, errors = [], [f"catalogue: {type(exc).__name__}: {exc}"] + access = _merged_access(previous if same else None, verdicts, identity) # A partial retry is not evidence that a previously readable model disappeared. Preserve a # complete, stamped catalogue in full whenever the new pass reports any error; otherwise one # refused model could silently shrink the enablement roster. A first-ever partial pass may be # retained for diagnostics, but an entirely failed pass still has no catalogue to publish. - if errors and current["complete"] and current["capturedAt"]: - return current - if not rows and errors: + # R70: Odoo's explicit refusal, or a model it does not expose, still withdraws that model. + if errors and same and (not rows or (current["complete"] and current["capturedAt"])): + return _narrow_catalog(rt, connection_id, current, previous, access) + if errors and not rows: return current captured = _dt.datetime.now(_dt.timezone.utc).isoformat() result = {"version": MODEL_CATALOG_VERSION, "complete": not errors, - "capturedAt": captured, "models": rows, "errors": errors[:50]} - slot = _catalog_slot(connection_id) - - def _cache(current_cache): - current_cache = dict(current_cache) if isinstance(current_cache, dict) else {} - connections = dict(current_cache.get("connections") or {}) - connections[slot] = result - return {"version": MODEL_CATALOG_VERSION, "connections": connections} - - rt.update(MODEL_CATALOG_KEY, _cache, flush="sync") - return result + "capturedAt": captured, "models": rows, "errors": errors[:50], **access} + _write_catalog_slot(rt, connection_id, lambda _stored: result) + return _catalog_view(result) def refresh_model_catalog_entry(rt, model, connection_id=None, client=None): """Refresh one model's fields with two read-only calls, without enumerating other models.""" - current = readable_model_catalog(rt, connection_id) + slot = _read_catalog_slot(rt, connection_id) + current, previous = _catalog_view(slot), _access_view(slot) + identity = _credential_identity(rt, connection_id) + same = _same_credential(previous["source"], identity) def _read_fields(bound_client): - readable = bound_client.execute_kw( - model, "check_access_rights", ["read"], {"raise_exception": False}) - if not bool(readable): + if not _read_access(bound_client, model): return None fields = bound_client.fields_get( model, attributes=_ODOO_FIELD_ATTRIBUTES) @@ -2867,28 +2977,40 @@ def refresh_model_catalog_entry(rt, model, connection_id=None, client=None): # No answer (including unavailable credentials) is not evidence of denied model access. return current if fields is None: + # R70: this credential's explicit refusal withdraws the model from its own offer. + if same and (model not in previous["denied"] + or any(row.get("model") == model for row in current["models"])): + def _deny(stored): + stored = stored or {**current, "source": identity} + stored["models"] = [row for row in (stored.get("models") or ()) + if not (isinstance(row, dict) and row.get("model") == model)] + stored["denied"] = _model_names([*(stored.get("denied") or ()), model]) + return stored + + _write_catalog_slot(rt, connection_id, _deny) raise Refused(f"This Odoo account cannot read the model {model}. " "Ask your Odoo administrator to grant read access in Odoo, not here.") - slot = _catalog_slot(connection_id) - def _cache(current_cache): - current_cache = dict(current_cache) if isinstance(current_cache, dict) else {} - connections = dict(current_cache.get("connections") or {}) - result = dict(connections.get(slot) or current) + def _merge(stored): + result = stored or {**current, "source": identity} rows = list(result.get("models") or ()) - previous = next((row for row in rows if row.get("model") == model), None) + prior = next((row for row in rows if row.get("model") == model), None) row = {"model": model, - "name": " ".join(str((previous or {}).get("name") or model).split()), + "name": " ".join(str((prior or {}).get("name") or model).split()), "fields": fields} result["models"] = [row if item.get("model") == model else item for item in rows] - if previous is None: + if prior is None: result["models"].append(row) # One model neither captures a whole catalogue nor changes its completeness. Preserve # complete and capturedAt exactly, along with every unrelated row and connection slot. - connections[slot] = result - return {"version": MODEL_CATALOG_VERSION, "connections": connections} - - rt.update(MODEL_CATALOG_KEY, _cache, flush="sync") + # Its explicit read verdict does clear an earlier refusal for this credential. + if same: + for name in ("denied", "absent"): + if name in result: + result[name] = [value for value in _model_names(result[name]) if value != model] + return result + + _write_catalog_slot(rt, connection_id, _merge) return readable_model_catalog(rt, connection_id) @@ -3255,6 +3377,10 @@ ODOO_SPEC = ( "pinned": "invoice_no", "identity": (_ROW_ID_COL, "odoo_id"), "links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),)}, + # Read-through like the line grains; payments name customers and vendors alike, so no link. + {"key": PAYMENTS_KEY, "source": "account_payment", "label": "Odoo payments", + "pinned": "odoo_id", "identity": (_ROW_ID_COL, "odoo_id"), "links": ()}, + # ⚠ RETIRED FROM `TABLES` BY W33-T44, DECLARED HERE ANYWAY, AND THAT IS THE POINT OF A SPEC. # `RETIRED_KEYS` pops this key and `PRODUCTS_KEY` from a tenant's document on every sync, but # the key constants, the field builders and the readers in `_READERS` all still exist, the @@ -5341,27 +5467,37 @@ def grid_choices(rt, connection_id=None): by a curated table is covered there; every remaining readable model receives a stable raw-model key and defaults off. Stored choices survive catalogue refreshes because config is keyed by the stable database key, never by its position. + + S79-T66 / R70 (D-748): a model is withdrawn only on this credential's own evidence. Odoo's + explicit read refusal, or a curated model it does not expose, withdraws it at any catalogue + age or completeness; no answer keeps the curated default. A slot another credential answered + offers no discovered model at all. """ chosen = read_config(rt)["grids"] namespace = connection_namespace(rt, connection_id)[1] if connection_id else None - catalogue = readable_model_catalog(rt, namespace) + slot = _read_catalog_slot(rt, namespace) + catalogue, access = _catalog_view(slot), _access_view(slot) + own = _same_credential(access["source"], _credential_identity(rt, namespace)) + withdrawn = set(access["denied"]) | set(access["absent"]) if own else set() fixed = [{"key": key, "label": label, "model": ODOO_TABLE_MODELS.get(key, ""), "relevant": True, "defaultEnabled": True, "enabled": bool(chosen.get(key, True))} for _bucket, key, label, _fields in TABLES] + # A row with no model names nothing the connector exposes, so it is never offered. + fixed = [row for row in fixed if row["model"] and row["model"] not in withdrawn] # Before discovery, keep the established curated defaults. A complete stamped catalogue is # authoritative for this API account, so a curated database whose model is absent must not be # offered: relevance never widens access. - if catalogue["complete"] and catalogue["capturedAt"]: + if own and catalogue["complete"] and catalogue["capturedAt"]: readable = {str(row.get("model") or "") for row in catalogue["models"]} - fixed = [row for row in fixed if not row["model"] or row["model"] in readable] + fixed = [row for row in fixed if row["model"] in readable] covered = {row["model"] for row in fixed if row["model"]} dynamic = [] try: from core import connector_spec - for item in catalogue["models"]: + for item in (catalogue["models"] if own else ()): model = str(item.get("model") or "") - if not model or model in covered: + if not model or model in covered or model in withdrawn: continue key = connector_spec.odoo_model_table_key(model) dynamic.append({"key": key, @@ -5795,7 +5931,7 @@ def plan(cur, rt=None, connection_id=None, buckets=None): existing = {} if rt is not None: try: - existing = dict(rt.get(ut.STORE_KEY) or {}) + existing = rt.get(ut.STORE_KEY) or {} except Exception: # noqa: BLE001 existing = {} # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and @@ -5814,12 +5950,10 @@ def plan(cur, rt=None, connection_id=None, buckets=None): # be eligible FOR), but the law is written without an exception, so this lends the evaluator # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so # what is lent is the document as it stands the moment this plan applies. - lent = _LentDoc({**existing, - **{table_key_for(k, connection_id): { - **(existing.get(table_key_for(k, connection_id)) or {}), - "readThrough": True} - for k in READ_THROUGH_KEYS}}, - ut.STORE_KEY) + for k in READ_THROUGH_KEYS: + tk = table_key_for(k, connection_id) + existing[tk] = {**(existing.get(tk) or {}), "readThrough": True} + lent = _LentDoc(existing, ut.STORE_KEY) built, limits = {}, {} caps = {} @@ -6192,9 +6326,14 @@ def apply_plan(rt, built, username="automation", today=None, report=None, generated = connector_spec.system_link_fields("odoo", live_columns) for key, label, fields, rows in plans: existing = {field["key"] for field in fields} + # A declared link on the same column to the same target already serves that edge. + declared = {((field.get("link") or {}).get("table"), (field.get("link") or {}).get("from")) + for field in fields if field.get("type") == "link"} fields = [*fields, *namespaced_fields( [_preset(field) for field in generated.get(base_table_key(key), ()) - if field["key"] not in existing], connection_id)] + if field["key"] not in existing + and (field["link"]["table"], field["link"]["from"]) not in declared], + connection_id)] written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) from core import connector_sync cur[key]["connector"] = { @@ -6590,7 +6729,7 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None, if not selected: return {} ut = _ut() - existing = dict(rt.get(ut.STORE_KEY) or {}) + existing = rt.get(ut.STORE_KEY) or {} # ⛔ HOTFIX 2026-09-24: a model the store's byte budget already refused is NOT pulled again. # Without this, every resync re-read all of it from Odoo (account.invoice.line.agent: 250,000 # rows) only for the store door to discard the rows. The refusal and its fix are on the wire diff --git a/api/process_role.py b/api/process_role.py index 5cb322e5cbea916d2d6a1aa406726ce9c0b540ca..0d6da5888c88b3a3a4773e792eebe08a7396e264 100644 --- a/api/process_role.py +++ b/api/process_role.py @@ -1,8 +1,15 @@ """One process rule for every database: heavy background work runs only in the sync worker. Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs every heavy job, -`all` does both in one process (local runs and gates). Kinds: `heavy` moves source data or writes -the store in the background; `light` serves this process's own reads. +`all` does both in one process (local runs and gates), `reader` serves reads, spawns no worker and +never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the +store in the background; `light` serves this process's own reads. + +One sync writer per store: a worker runs its heavy jobs only while it holds the store's writer +lease, so a second Space's worker waits instead of becoming a second writer. + +Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock and +supervises the sync worker and installs hand-offs. The others only read the mirror read-only. """ from __future__ import annotations @@ -10,6 +17,7 @@ import atexit import json import os import shutil +import socket import subprocess import sys import threading @@ -17,13 +25,16 @@ import time import uuid from pathlib import Path -ROLES = ("web", "worker", "all") +ROLES = ("web", "worker", "all", "reader") KINDS = ("heavy", "light") +SERVING_ROLES = ("web", "all", "reader") API_DIR = Path(__file__).resolve().parent POLL_SECONDS = 15 HEARTBEAT_SECONDS = 30 HEARTBEAT_FILE = "worker.heartbeat.json" READY_FILE = "READY.json" +PRIMARY_LOCK = "web-primary.lock" +PRIMARY_FILE = "web-primary.json" INSTALL_ATTEMPTS = 3 INCOMPLETE_GRACE_SECONDS = 3600 @@ -45,7 +56,7 @@ ROLE, ROLE_SOURCE = resolve_role() os.environ["AIOS_PROCESS_ROLE"] = ROLE if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web": print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True) -if ROLE == "web": +if ROLE in ("web", "reader"): os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB") os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2") @@ -95,9 +106,14 @@ class _Refused: def thread(*, target, name, kind, daemon=True, args=(), kwargs=None, delay=None, role_name=None): - """An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in.""" + """An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in. + + While the writer lease is armed, a heavy job's target first waits until this process holds it. + """ if not admit(name, kind, role_name): return _Refused(name) + if kind == "heavy" and _WRITER["armed"]: + target = _after_writer_lease(target, name) if delay is None: job = threading.Thread(target=target, args=tuple(args), kwargs=dict(kwargs or {}), daemon=daemon, name=name) @@ -130,6 +146,305 @@ def started_jobs(): return {name: dict(row) for name, row in _STARTED.items()} +# ----------------------------------------------------------------------------- the writer lease +WRITER_LEASE_DOC = "sync_writer_lease" +WRITER_LEASE_SECONDS = 180 +WRITER_POLL_SECONDS = 15 +_WRITER = {"armed": False, "holder": None, "renewedAt": None, "lostAt": None, "waitingSince": None, + "error": None} +_WRITER_HELD = threading.Event() + + +def writer_holder(env=None): + """This container's lease identity, so a restarted worker in it takes its predecessor's lease.""" + env = os.environ if env is None else env + return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}" + + +def acquire_writer(rt, holder=None, *, now=None, ttl=None): + """Take or renew the store's writer lease. True when `holder` holds it after the call. + + The store's update is the compare-and-set: Postgres locks the row for the read-modify-write. + """ + holder = holder or writer_holder() + stamp = time.time() if now is None else float(now) + ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl) + + def take(doc): + doc = dict(doc or {}) + current = doc.get("holder") + expired = stamp - float(doc.get("renewedAt") or 0) > ttl + if current in (None, "", holder) or expired: + if current != holder: + doc.update(acquiredAt=stamp, previous=current) + doc.update(holder=holder, renewedAt=stamp, pid=os.getpid()) + return doc + + doc = rt.update(WRITER_LEASE_DOC, take) + if not isinstance(doc, dict): + doc = rt.get(WRITER_LEASE_DOC, fresh=True) + return isinstance(doc, dict) and doc.get("holder") == holder + + +def release_writer(rt, holder=None): + """Give the lease up at a clean exit, so the next writer need not wait for it to expire.""" + holder = holder or writer_holder() + + def drop(doc): + doc = dict(doc or {}) + if doc.get("holder") == holder: + doc.update(holder=None, renewedAt=0, previous=holder) + return doc + + rt.update(WRITER_LEASE_DOC, drop) + + +def holds_writer(): + return _WRITER_HELD.is_set() + + +def may_write(role_name=None): + """allows('heavy'), plus the writer lease when armed. For a store write outside a heavy thread.""" + return allows("heavy", role_name) and (not _WRITER["armed"] or _WRITER_HELD.is_set()) + + +def _after_writer_lease(target, name): + """Wrap a heavy job so it starts only after this process holds the writer lease.""" + def gated(*args, **kwargs): + if not _WRITER_HELD.is_set(): + print(f"[process-role] {name} waits for the sync writer lease", flush=True) + _WRITER_HELD.wait() + return target(*args, **kwargs) + gated.__name__ = getattr(target, "__name__", name) + return gated + + +def writer_lease_tick(rt_factory, holder, *, now=None, ttl=None): + """One acquire or renew. Returns 'held', 'waiting' or 'lost'. 'lost' means stop writing now.""" + stamp = time.time() if now is None else float(now) + ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl) + try: + held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl) + except Exception as exc: # noqa: BLE001 + held = None + if _WRITER.get("error") != type(exc).__name__: + # Fail closed: no lease, no heavy job. Set AIOS_SYNC_WRITER_TENANT to a writable tenant. + print(f"[process-role] sync writer lease unreadable ({type(exc).__name__}); heavy jobs wait", + flush=True) + _WRITER["error"] = type(exc).__name__ + else: + _WRITER["error"] = None + if held: + if not _WRITER_HELD.is_set(): + print(f"[process-role] sync writer lease held by {holder}", flush=True) + _WRITER.update(renewedAt=stamp, waitingSince=None) + _WRITER_HELD.set() + return "held" + if _WRITER_HELD.is_set(): + # Another holder took it, or renewals failed long enough that one may take it next. + if held is False or stamp - float(_WRITER["renewedAt"] or 0) > ttl - 2 * WRITER_POLL_SECONDS: + _WRITER_HELD.clear() + _WRITER["lostAt"] = stamp + print("[process-role] sync writer lease lost; this worker stops writing", flush=True) + return "lost" + return "held" + if _WRITER["waitingSince"] is None: + _WRITER["waitingSince"] = stamp + print("[process-role] another process holds the sync writer lease; heavy jobs wait", flush=True) + return "waiting" + + +def _writer_lease_loop(rt_factory, holder, exit_fn): + while True: + if writer_lease_tick(rt_factory, holder) == "lost": + exit_fn(3) + return + if _STOP.wait(WRITER_POLL_SECONDS): + return + + +def _writer_runtime(): + from harness import runtime + return runtime.get_runtime(os.environ.get("AIOS_SYNC_WRITER_TENANT") or "royal-imports") + + +def _release_at_exit(): + if _WRITER_HELD.is_set(): + try: + release_writer(_writer_runtime(), _WRITER["holder"]) + except Exception: # noqa: BLE001 + pass + + +def arm_writer_lease(rt_factory=None, holder=None, exit_fn=None): + """Worker only: every later heavy job waits for the lease. Call before any heavy job starts.""" + if _WRITER["armed"]: + return False + _WRITER.update(armed=True, holder=holder or writer_holder()) + # Light: it must run before the lease is held, and it writes only the lease document. + start("sync-writer-lease", "light", _writer_lease_loop, + args=(rt_factory or _writer_runtime, _WRITER["holder"], exit_fn or os._exit)) + if rt_factory is None: + atexit.register(_release_at_exit) + return True + + +# ----------------------------------------------------------------------------- readiness (C9) +READY_CHECKS = ("mirror", "pools", "first_reads") +_READY_DEFAULT = {"mirror": "The mirror is not installed yet.", + "pools": "The read pools are not open yet.", + "first_reads": "The first reads have not answered yet."} +_READINESS = {} +_BOOT_RESTORE = {} +# The first full pass latches the boot conditions: a later worker restart or a fresh hand-off +# must not pull a warm web process out of rotation. +_READY_LATCH = {"at": None} + + +def reset_readiness(): + _READY_LATCH["at"] = None + _READINESS.clear() + _READINESS.update({name: {"ok": False, "detail": text, "at": None} for name, text in _READY_DEFAULT.items()}) + + +reset_readiness() + + +def mark_ready(check, ok, detail=""): + """Record one readiness check. A check passes only after its own step finished in this process.""" + if check not in _READY_DEFAULT: + raise ValueError(f"unknown readiness check {check!r}") + _READINESS[check] = {"ok": bool(ok), "detail": "" if ok else (detail or _READY_DEFAULT[check]), + "at": time.time()} + + +def note_boot_restore(state="done"): + """Worker side: its boot restore finished, so every hand-off it made is already on disk.""" + _BOOT_RESTORE.update(state=state, at=time.time(), pid=os.getpid()) + try: + write_heartbeat() + except Exception as exc: # noqa: BLE001 + print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True) + + +def worker_boot_restored(root=None): + """Web side: the current worker's heartbeat says its boot restore finished.""" + beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {} + restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {} + # A secondary web worker supervises nothing: the primary names the current sync worker. + pid = _WORKER.get("pid") if is_primary() else primary_state(root).get("workerPid") + return bool(restore.get("state")) and pid is not None and beat.get("pid") == pid + + +def _installed_map(root=None): + if is_primary(): + return dict(_INSTALLED) + installed = primary_state(root).get("installed") or {} + return {tuple(key.split("/", 1)): snap for key, snap in installed.items() if "/" in key} + + +def pending_handoffs(root=None): + """`tenant/connector` for each newest READY hand-off not installed yet (by the primary).""" + installed = _installed_map(root) + return sorted(f"{key[0]}/{key[1]}" for key, rows in ready_handoffs(root).items() + if installed.get(key) != (rows[-1][2].get("snapshotId") or rows[-1][1].name)) + + +# ----------------------------------------------------------------------------- several web workers +_PRIMARY = {"fd": None} + + +def web_workers(env=None): + """How many uvicorn workers serve this container, from AIOS_UVICORN_WORKERS. 1 when unset.""" + env = os.environ if env is None else env + try: + return max(1, int(env.get("AIOS_UVICORN_WORKERS") or 1)) + except ValueError: + return 1 + + +def is_primary(): + """True when this web process supervises the sync worker and installs. Always for one worker.""" + return web_workers() == 1 or _PRIMARY["fd"] is not None + + +def claim_primary(root=None): + """Take the container's primary lock without waiting. True when this process holds it. + + The lock is a kernel flock: it drops when its holder dies, so a standby can take over. + """ + if is_primary(): + return True + try: + import fcntl + except ImportError: + return False + path = Path(root or handoff_dir()) / PRIMARY_LOCK + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + return False + _PRIMARY["fd"] = fd + print(f"[process-role] web worker pid={os.getpid()} is the primary: it supervises the sync worker " + f"and installs hand-offs", flush=True) + write_primary_state(root) + return True + + +def write_primary_state(root=None): + """Primary only: the sync worker pid and every installed snapshot, for the secondaries.""" + if web_workers() == 1 or _PRIMARY["fd"] is None: + return False + try: + from harness import datastore + version = datastore.serving_file()[1] + except Exception: # noqa: BLE001 + version = None + _write_json(Path(root or handoff_dir()) / PRIMARY_FILE, { + "pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(), + "installed": {f"{key[0]}/{key[1]}": snap for key, snap in _INSTALLED.items()}, + "mirrorVersion": version}) + return True + + +def primary_state(root=None): + state = _read_json(Path(root or handoff_dir()) / PRIMARY_FILE) + return state if isinstance(state, dict) else {} + + +def _standby_loop(): + while not _STOP.wait(POLL_SECONDS): + if claim_primary(): + start_web_services() + return + + +def readiness(mirror_live=None, role_name=None, root=None): + """{ready, checks: {name: bool}, failing: [names], detail: {name: text}} for GET /api/v1/ready.""" + role_name = role_name or ROLE + checks, detail = {}, {} + for name in READY_CHECKS: + row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")} + checks[name], detail[name] = bool(row["ok"]), row["detail"] + boot = role_name == "web" and _READY_LATCH["at"] is None + if "mirror" in checks and checks["mirror"]: + if mirror_live is False: + checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading." + elif boot and not worker_boot_restored(root): + checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore." + elif boot and pending_handoffs(root): + checks["mirror"] = False + detail["mirror"] = "Mirror hand-offs are not installed yet: " + ", ".join(pending_handoffs(root)) + "." + failing = [name for name in checks if not checks[name]] + if not failing and _READY_LATCH["at"] is None: + _READY_LATCH["at"] = time.time() + return {"ready": not failing, "checks": checks, "failing": failing, + "detail": {name: detail[name] for name in failing}} + + # ----------------------------------------------------------------------------- the hand-off dir def handoff_dir(): return Path(os.environ.get("AIOS_MIRROR_HANDOFF_DIR") or "/tmp/aios-mirror-handoff") @@ -200,9 +515,15 @@ def ready_handoffs(root=None): def _install_default(connector, tenant, ready): from harness import mirror_store, runtime rt = runtime.get_runtime(tenant) + from harness import datastore paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()} - installed = mirror_store.install_handoff(connector, tenant, rt, ready["manifest"], - ready.get("stateRows") or [], paths) + resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve() + with (datastore.versioned_write(tenant) if resident else _no_version()) as version: + installed = mirror_store.install_handoff(connector, tenant, rt, ready["manifest"], + ready.get("stateRows") or [], paths) + if version is not None: + # Already resident: keep serving the current version, publish nothing. + version["publish"] = installed is not False try: import routes_odoo_tables routes_odoo_tables.invalidate_sources() @@ -211,6 +532,14 @@ def _install_default(connector, tenant, ready): return installed +class _no_version: + def __enter__(self): + return None + + def __exit__(self, *exc): + return False + + def _remove_dir(path): shutil.rmtree(path, ignore_errors=True) @@ -336,6 +665,7 @@ def _web_watch_loop(): while not _STOP.wait(POLL_SECONDS): try: install_ready() + write_primary_state() except Exception as exc: # noqa: BLE001 print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True) try: @@ -359,7 +689,7 @@ def write_heartbeat(now=None): resync = dict(getattr(app_main, "RESYNC_STATE", None) or {}) _write_json(handoff_dir() / HEARTBEAT_FILE, {"pid": os.getpid(), "at": time.time() if now is None else now, - "catchUp": catch_up, "resync": resync}) + "catchUp": catch_up, "resync": resync, "bootRestore": dict(_BOOT_RESTORE)}) def _heartbeat_loop(parent_pid): @@ -382,6 +712,8 @@ def start_worker_services(): mirror_store.register_publish_listener(write_handoff) parent = int(os.environ.get("AIOS_PARENT_PID") or 0) start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,)) + # Before the first heavy job: each one now waits until this worker holds the writer lease. + arm_writer_lease() start("sync-desk-poll", "heavy", _worker_desk_loop) @@ -391,6 +723,8 @@ def _worker_env(): AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1") for name in ("AIOS_DUCKDB_MEMORY", "AIOS_DUCKDB_THREADS"): env.pop(name, None) # the worker sets its own caps + # The worker writes its own file read-write and is never a web worker. + env.update(AIOS_MIRROR_VERSIONS="0", AIOS_UVICORN_WORKERS="1") return env @@ -406,6 +740,7 @@ def _supervise(): if proc is not None: _WORKER.update(proc=proc, pid=proc.pid, startedAt=started) print(f"[process-role] sync worker started pid={proc.pid}", flush=True) + write_primary_state() code = proc.wait() _WORKER["lastExit"] = code else: @@ -435,11 +770,21 @@ def stop_worker(): def start_web_services(): - """Web role only: forward Sync now, supervise the worker, install its snapshots.""" + """Web role only: forward Sync now, supervise the worker, install its snapshots. + + With several uvicorn workers only the primary does this. The others stand by and read. + """ if ROLE != "web" or _WORKER.get("supervisor") is not None: return False from core import connector_sync + # Every web worker forwards the Sync now it accepts: the desk lives in the store. connector_sync.register_request_listener(_forward_request) + if not claim_primary(): + if _WORKER.get("standby") is None: + _WORKER["standby"] = start("web-primary-standby", "light", _standby_loop) + print(f"[process-role] web worker pid={os.getpid()} serves reads only; " + f"the primary supervises the sync worker", flush=True) + return False _WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise) start("mirror-handoff-watcher", "light", _web_watch_loop) atexit.register(stop_worker) @@ -460,7 +805,8 @@ def status(): age = None if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"): age = round(max(0.0, time.time() - float(beat["at"])), 1) - return {"role": ROLE, "roleSource": ROLE_SOURCE, + return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(), "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age, "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None, - "lastInstallError": dict(LAST_INSTALL_ERROR) or None}} + "lastInstallError": dict(LAST_INSTALL_ERROR) or None}, + "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}} diff --git a/api/providers.py b/api/providers.py index 258a3d4ad90719d64792158eba6dd6d05a2855ee..90624a7593c92f8e314c1e4d106876c9ebb0f7ce 100644 --- a/api/providers.py +++ b/api/providers.py @@ -34,7 +34,9 @@ lets a tenant be moved off a vendor without a release. from __future__ import annotations import json +import math import os +import threading import time from dataclasses import dataclass, field from datetime import datetime, timezone @@ -573,7 +575,86 @@ def llm_chain(capability="llm_tool_calling"): raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() names = [x.strip() for x in raw.split(",") if x.strip() in LLM_DEFAULT_ORDER] or list(LLM_DEFAULT_ORDER) return [LLM_PROVIDERS[n] for n in names - if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] + if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n) + and not llm_breaker_open(n)] + + +# ═══════════ LLM CIRCUIT BREAKER, ONE PER PROVIDER NAME (S79-T43) ═══════════════════════════════ +# After N consecutive failures (transport, 408, 5xx) a provider opens for a cooldown and is skipped. +# After the cooldown it half-opens: the next recorded outcome closes it or re-opens it. +# ⛔ 400, 401, 402, 404, 413 and 429 are ANSWERS, so they never count. Credit has `_NO_CREDIT`. +# ⚠ Process-local like `_NO_CREDIT`: a latency guard, not a record. +_monotonic = time.monotonic # Gates patch this clock so no leg sleeps. +_LLM_BREAKERS: dict[str, dict] = {} # name -> {"failures": int, "opened_at": float | None, "trial": bool} +_LLM_BREAKER_LOCK = threading.Lock() + + +def _llm_breaker_setting(name, default, minimum): + try: + value = float(os.environ.get(name) or default) + return max(minimum, value) if math.isfinite(value) else float(default) + except ValueError: + return float(default) + + +def _llm_breaker_cooling(state): + """Is an open breaker still inside its cooldown?""" + cooldown = _llm_breaker_setting("AIOS_LLM_BREAKER_COOLDOWN_S", 30, 0) + return _monotonic() - state["opened_at"] < cooldown + + +def llm_breaker_record(name, status): + """Record one call outcome. 0 is a transport exception, 2xx closes, 0, 408 and 5xx count.""" + code = int(status or 0) + with _LLM_BREAKER_LOCK: + if 200 <= code <= 299: + _LLM_BREAKERS.pop(str(name), None) + return + if not (code == 0 or code == 408 or 500 <= code <= 599): + if str(name) in _LLM_BREAKERS: + _LLM_BREAKERS[str(name)]["trial"] = False # An answer ends a trial, count unchanged. + return + state = _LLM_BREAKERS.setdefault(str(name), {"failures": 0, "opened_at": None, "trial": False}) + state["failures"] += 1 + limit = _llm_breaker_setting("AIOS_LLM_BREAKER_FAILURES", 5, 1) + if state["opened_at"] is not None or state["failures"] >= limit: + state["opened_at"] = _monotonic() + state["trial"] = False + + +def llm_breaker_open(name): + """Is this provider refused right now? False while closed or while a half-open trial is allowed.""" + with _LLM_BREAKER_LOCK: + state = _LLM_BREAKERS.get(str(name)) + if state is None or state["opened_at"] is None: + return False + return bool(state["trial"] or _llm_breaker_cooling(state)) + + +def _llm_breaker_admit(name): + """Admit one call: True for the half-open trial, False when closed, None while open.""" + with _LLM_BREAKER_LOCK: + state = _LLM_BREAKERS.get(str(name)) + if state is None or state["opened_at"] is None: + return False + if state["trial"] or _llm_breaker_cooling(state): + return None + state["trial"] = True + return True + + +def llm_breaker_reset(name=None): + """Close one breaker, or all of them. For a gate.""" + with _LLM_BREAKER_LOCK: + if name is None: + _LLM_BREAKERS.clear() + else: + _LLM_BREAKERS.pop(str(name), None) + + +def _llm_max_retries(): + """SDK retries per call. 0 bounds one call by its own timeout.""" + return int(_llm_breaker_setting("AIOS_LLM_MAX_RETRIES", 0, 0)) #: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND @@ -743,6 +824,20 @@ def anthropic_send(req, timeout=None, use_sdk=None): later check in that process running on the wrong path and passing for the wrong reason. """ sdk = None if use_sdk is False else anthropic_sdk() + transport = "http" if sdk is None else "sdk" + if _llm_breaker_admit("anthropic") is None: + return 503, {"error": {"type": "llm_breaker_open", "message": ( + "Anthropic failed several times in a row, so calls pause briefly before the next try.")}}, transport + status = 0 + try: + status, body, transport = _anthropic_post(sdk, req, timeout) + return status, body, transport + finally: + llm_breaker_record("anthropic", status) + + +def _anthropic_post(sdk, req, timeout): + """One Anthropic call on the chosen transport. A raised exception reaches the breaker as 0.""" payload = dict(req.get("json") or {}) if sdk is None: r = requests.post(req["url"], headers=req["headers"], json=payload, @@ -754,7 +849,8 @@ def anthropic_send(req, timeout=None, use_sdk=None): key = (req.get("headers") or {}).get("x-api-key") or "" try: - client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) + client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60), + max_retries=_llm_max_retries()) msg = client.messages.create(**payload) # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same # wire-shaped dict the raw POST parses out of the response body. diff --git a/api/request_class.py b/api/request_class.py index 084377b25e21283e96345511b1b9a9f793cd6393..a6530004109345763c728a9f322f0f31cb28ae08 100644 --- a/api/request_class.py +++ b/api/request_class.py @@ -1,18 +1,27 @@ -"""Heavy request admission: a bounded queue so CPU-heavy reads never starve light requests. - -One web process on 2 vCPU serves everyone. `classify` names the heavy doors in ONE registry, and -`HeavyGate` lets at most `AIOS_HEAVY_CONCURRENCY` of them run at once. A heavy request that waits -longer than `AIOS_HEAVY_QUEUE_SECONDS` gets a 503 `server_busy`. Light requests never wait. +"""Heavy request admission: bounded queues so CPU-heavy reads never starve light requests. + +`classify` names the heavy doors in ONE registry, each in a named request class (grid rows, +workspace, sheet, export, query). `HeavyGate` gives every class its own limit and queue bound in +each web process: no permit is shared between classes, so a full class answers a 503 +`server_busy` naming itself while another class keeps admitting. Inside a class, waiting requests +are admitted round robin by tenant, so one tenant's burst queues behind itself and another +tenant's read is admitted within its queue bound. Light requests never wait. + +Box sizing lives here too: `python request_class.py --env` reads the container's CPU and memory +limits and prints the launch values (uvicorn workers, heavy concurrency, DuckDB caps). +Run as a script it imports only the standard library. """ from __future__ import annotations import asyncio +import hashlib import json import os import re import sys import threading import time +from collections import deque from urllib.parse import parse_qs BUSY_CODE = "server_busy" @@ -36,23 +45,32 @@ def _sheet_read_heavy(query): return not (tabs == "meta" and not _flag(query, "opening")) -# (methods, path regex, query predicate or None). A request is heavy when one row matches. +#: The heavy request classes. Each has its own limit (`AIOS_HEAVY_CONCURRENCY_`, default +#: `AIOS_HEAVY_CONCURRENCY` for grid_rows and half of it for the others, see `class_limit`) and +#: queue bound (`AIOS_HEAVY_QUEUE_SECONDS_`, default `AIOS_HEAVY_QUEUE_SECONDS`). +CLASSES = ("grid_rows", "workspace", "sheet", "export", "query") +#: The class a heavy request with no registry row lands in (a caller that names no class). +DEFAULT_CLASS = "grid_rows" + +# (class, methods, path regex, query predicate or None). A request is heavy when one row matches. HEAVY = ( - ({"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")), - ({"GET"}, r"/api/v1/products", None), - ({"GET"}, r"/api/v1/customers", None), - ({"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None), - ({"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None), - ({"GET"}, r"/api/v1/tables/[^/]+/rows", None), - ({"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy), - ({"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None), + ("workspace", {"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")), + ("grid_rows", {"GET"}, r"/api/v1/products", None), + ("grid_rows", {"GET"}, r"/api/v1/customers", None), + ("grid_rows", {"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None), + ("grid_rows", {"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None), + ("grid_rows", {"GET"}, r"/api/v1/tables/[^/]+/rows", None), + ("sheet", {"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy), + ("sheet", {"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None), # In the web role a refresh only queues for the sync worker; it computes only in worker or all. - ({"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"), - ({"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None), - ({"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None), - ({"POST"}, r"/api/v1/external/explorer/run", None), + ("sheet", {"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"), + ("export", {"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None), + ("query", {"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None), + ("query", {"POST"}, r"/api/v1/external/explorer/run", None), ) -_HEAVY = tuple((methods, re.compile(rx), pred) for methods, rx, pred in HEAVY) +_HEAVY = tuple((cls, methods, re.compile(rx), pred) for cls, methods, rx, pred in HEAVY) +#: The class of a heavy MCP tools/call (sheet reads and database queries). +MCP_CLASS = "query" # MCP tools/call names that read sheets or query databases. MCP_HEAVY_TOOLS = {"sheet", "table", "data"} @@ -88,18 +106,50 @@ def _mcp_heavy(body): return False -def classify(method, path, query="", body=None): - """Return "heavy" or "light" for one request. `query` is a raw query string or a parsed dict.""" +def request_class(method, path, query="", body=None): + """The heavy class of one request, or None when it is light.""" method = (method or "").upper() if isinstance(query, bytes): query = query.decode("latin-1") q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else (query or {}) - for methods, rx, pred in _HEAVY: + for cls, methods, rx, pred in _HEAVY: if method in methods and rx.fullmatch(path or "") and (pred is None or pred(q)): - return "heavy" + return cls if method == "POST" and path == MCP_PATH and _mcp_heavy(body): - return "heavy" - return "light" + return MCP_CLASS + return None + + +def classify(method, path, query="", body=None): + """Return "heavy" or "light" for one request. `query` is a raw query string or a parsed dict.""" + return "heavy" if request_class(method, path, query, body) else "light" + + +def request_tenant(scope): + """The fairness key of one request: the signed session's tenant, else a hash of its API key. + + No store read: the cookie is verified with its HMAC only. An unsigned request shares one key. + """ + cookie, auth = "", "" + for key, value in scope.get("headers") or (): + if key == b"cookie": + cookie = value.decode("latin-1") + elif key == b"authorization": + auth = value.decode("latin-1") + if cookie: + try: + import aios_session + for part in cookie.split(";"): + name, _, value = part.strip().partition("=") + if name == aios_session.COOKIE_NAME and value: + claims = aios_session.read(value) + if claims: + return "tenant:" + str(claims["t"]).strip().lower() + except Exception: # noqa: BLE001 + pass + if auth: + return "key:" + hashlib.sha256(auth.encode("utf-8", "replace")).hexdigest()[:16] + return "anonymous" class _Counters: @@ -120,17 +170,40 @@ class _Counters: _COUNTERS = _Counters() +#: The same counters for each request class. +_CLASS_COUNTERS = {name: _Counters() for name in CLASSES} def status(): """Snapshot of heavy request admission for a settings or health surface.""" c = _COUNTERS + workers = web_workers() + classes = {} + for name, cc in _CLASS_COUNTERS.items(): + with cc.lock: + limit, queue = class_limit(name) + classes[name] = {"limit": cc.limit if cc.limit is not None else limit, + "queueSeconds": cc.queue_seconds if cc.queue_seconds is not None else queue, + "inFlight": cc.in_flight, "queued": cc.queued, + "busyRefusals": cc.busy_refusals, "admitted": cc.admitted} with c.lock: - return {"limit": c.limit if c.limit is not None else _env_int("AIOS_HEAVY_CONCURRENCY", 2), + limit = c.limit if c.limit is not None else _env_int("AIOS_HEAVY_CONCURRENCY", 2) + # Counters are per process: N uvicorn workers admit N times `limit` across the box. + return {"limit": limit, "workers": workers, "boxLimit": limit * workers, "queueSeconds": (c.queue_seconds if c.queue_seconds is not None else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)), "inFlight": c.in_flight, "queued": c.queued, - "busyRefusals": c.busy_refusals, "admitted": c.admitted} + "busyRefusals": c.busy_refusals, "admitted": c.admitted, + "tenantCap": tenant_cap_env() or None, "classes": classes} + + +def web_workers(env=None): + """How many uvicorn workers share this box, from AIOS_UVICORN_WORKERS. 1 when unset.""" + env = os.environ if env is None else env + try: + return max(1, int(env.get("AIOS_UVICORN_WORKERS") or 1)) + except ValueError: + return 1 def _env_int(name, default): @@ -147,8 +220,32 @@ def _env_float(name, default): return default -async def _send_busy(send, cause="heavy_queue_timeout"): - body = json.dumps({"error": {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE}}).encode() +def tenant_cap_env(): + """AIOS_HEAVY_TENANT_CAP: most permits of one class one tenant holds. 0 or unset: the class limit.""" + try: + return max(0, int(os.environ.get("AIOS_HEAVY_TENANT_CAP") or 0)) + except ValueError: + return 0 + + +def class_limit(name): + """(limit, queue seconds) of one request class from its env, else from the shared heavy env. + + Unset, grid_rows takes the shared heavy limit and every other class half of it (at least 1), + so five independent classes do not multiply the heavy work one process admits. + """ + shared = _env_int("AIOS_HEAVY_CONCURRENCY", 2) + limit = shared if name == DEFAULT_CLASS else max(1, shared // 2) + queue = _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0) + return (_env_int(f"AIOS_HEAVY_CONCURRENCY_{name.upper()}", limit), + _env_float(f"AIOS_HEAVY_QUEUE_SECONDS_{name.upper()}", queue)) + + +async def _send_busy(send, cause="heavy_queue_timeout", request_class=None): + error = {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE} + if request_class: + error["requestClass"] = request_class + body = json.dumps({"error": error}).encode() await send({"type": "http.response.start", "status": 503, "headers": [(b"content-type", b"application/json"), (b"content-length", str(len(body)).encode()), @@ -157,25 +254,143 @@ async def _send_busy(send, cause="heavy_queue_timeout"): await send({"type": "http.response.body", "body": body, "more_body": False}) +class _FairSlots: + """One class's permits in one event loop. A free permit goes to the waiting tenant served + least recently, so tenants take turns. A tenant holds at most `tenant_cap` permits. + + FIFO would let one tenant's burst hold every other tenant's read past the queue bound. + """ + + TURNS_KEPT = 1024 + + def __init__(self, limit, tenant_cap=None): + self.limit = max(1, int(limit)) + self.tenant_cap = max(1, min(self.limit, int(tenant_cap or self.limit))) + self.running = 0 + self.by_tenant = {} + self.waiting = {} # tenant -> deque of futures, oldest first + self.turn = {} # tenant -> the grant count at its last grant + self.grants = 0 + + @property + def _value(self): + """Free permits, named as asyncio.Semaphore names them.""" + return self.limit - self.running + + def _dispatch(self): + while self.running < self.limit: + best = None + for tenant in list(self.waiting): + queue = self.waiting[tenant] + while queue and queue[0].done(): # a waiter that left (timeout, disconnect) + queue.popleft() + if not queue: + del self.waiting[tenant] + continue + if self.by_tenant.get(tenant, 0) >= self.tenant_cap: + continue + if best is None or self.turn.get(tenant, -1) < self.turn.get(best, -1): + best = tenant + if best is None: + return + queue = self.waiting[best] + self.running += 1 + self.by_tenant[best] = self.by_tenant.get(best, 0) + 1 + self.grants += 1 + self.turn[best] = self.grants + queue.popleft().set_result(True) + if not queue: + del self.waiting[best] + if len(self.turn) > self.TURNS_KEPT: + for tenant in [t for t in self.turn if t not in self.by_tenant and t not in self.waiting]: + del self.turn[tenant] + + async def acquire(self, tenant): + fut = asyncio.get_running_loop().create_future() + self.waiting.setdefault(tenant, deque()).append(fut) + self._dispatch() + try: + await fut + except asyncio.CancelledError: + if fut.done() and not fut.cancelled(): + self.release(tenant) # granted in the same tick as the cancel + else: + fut.cancel() + raise + return True + + def release(self, tenant): + self.running -= 1 + held = self.by_tenant.get(tenant, 0) - 1 + if held > 0: + self.by_tenant[tenant] = held + else: + self.by_tenant.pop(tenant, None) + self._dispatch() + + +class _Permit: + """One tenant's handle on a class's permits: acquire() and release() as a Semaphore has them.""" + + def __init__(self, slots, tenant): + self.slots, self.tenant = slots, tenant + + async def acquire(self): + return await self.slots.acquire(self.tenant) + + def release(self): + self.slots.release(self.tenant) + + @property + def _value(self): + return self.slots._value + + class HeavyGate: - """Pure ASGI middleware. Heavy requests take a permit; light ones pass straight through.""" + """Pure ASGI middleware. Heavy requests take a permit of their class; light ones pass through. - def __init__(self, app, concurrency=None, queue_seconds=None, counters=None): + `concurrency` and `queue_seconds` set every class at once (tests). Unset, each class reads + its own env, then the shared heavy env. `classes` overrides one class: {name: {limit, queueSeconds}}. + """ + + def __init__(self, app, concurrency=None, queue_seconds=None, counters=None, classes=None, + tenant_cap=None): self.app = app self.concurrency = concurrency or _env_int("AIOS_HEAVY_CONCURRENCY", 2) self.queue_seconds = (queue_seconds if queue_seconds is not None else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)) self.counters = counters or _COUNTERS self.counters.limit, self.counters.queue_seconds = self.concurrency, self.queue_seconds - self._sem = None + # A gate given its own counters (a test) keeps its own class counters too. + self.class_counters = ({name: _Counters() for name in CLASSES} if counters is not None + else _CLASS_COUNTERS) + self.tenant_cap = tenant_cap if tenant_cap is not None else tenant_cap_env() + self.limits = {} + for name in CLASSES: + limit, queue = class_limit(name) + if concurrency: + limit = concurrency + if queue_seconds is not None: + queue = queue_seconds + override = (classes or {}).get(name) or {} + limit = max(1, int(override.get("limit") or limit)) + queue = float(override["queueSeconds"]) if override.get("queueSeconds") is not None else queue + self.limits[name] = (limit, queue) + cc = self.class_counters[name] + cc.limit, cc.queue_seconds = limit, queue + self._slots = {} self._loop = None - def _semaphore(self): - # A Semaphore binds to the first loop that waits on it. One per loop. + def _semaphore(self, cls=DEFAULT_CLASS, tenant="anonymous"): + # Futures bind to the loop they wait on. One set of permits per loop. loop = asyncio.get_running_loop() - if self._sem is None or self._loop is not loop: - self._sem, self._loop = asyncio.Semaphore(self.concurrency), loop - return self._sem + if self._loop is not loop: + self._slots, self._loop = {}, loop + slots = self._slots.get(cls) + if slots is None: + limit = self.limits.get(cls, (self.concurrency, self.queue_seconds))[0] + slots = self._slots[cls] = _FairSlots(limit, self.tenant_cap or None) + return _Permit(slots, tenant) async def __call__(self, scope, receive, send): if scope.get("type") != "http": @@ -189,23 +404,34 @@ class HeavyGate: return if classify(method, path, scope.get("query_string", b""), peek) != "heavy": return await self.app(scope, _replay(body_msgs, receive), send) + cls = request_class(method, path, scope.get("query_string", b""), peek) or DEFAULT_CLASS if body_msgs is None: body_msgs, _ = await _read_body(receive, None) if body_msgs is None: return - await self._run_heavy(scope, body_msgs, receive, send) + await self._run_heavy(scope, body_msgs, receive, send, cls=cls) - async def _run_heavy(self, scope, body_msgs, receive, send): + async def _run_heavy(self, scope, body_msgs, receive, send, cls=None, tenant=None): queued_at = time.perf_counter() - sem = self._semaphore() - c = self.counters - c.add("queued", 1) + if cls is None: + cls = request_class(scope.get("method", ""), scope.get("path", ""), + scope.get("query_string", b"")) or DEFAULT_CLASS + tenant = tenant if tenant is not None else request_tenant(scope) + queue_seconds = self.limits.get(cls, (self.concurrency, self.queue_seconds))[1] + sem = self._semaphore(cls, tenant) + cc = self.class_counters.get(cls) or _Counters() + + def add(name, n): + self.counters.add(name, n) + cc.add(name, n) + + add("queued", 1) timing = sys.modules.get("core.request_timing") budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError) acquire = asyncio.ensure_future(sem.acquire()) gone = asyncio.ensure_future(_wait_disconnect(receive)) try: - timeout = timing.remaining_budget(self.queue_seconds) if timing else self.queue_seconds + timeout = timing.remaining_budget(queue_seconds) if timing else queue_seconds done, _ = await asyncio.wait({acquire, gone}, timeout=timeout, return_when=asyncio.FIRST_COMPLETED) if timing: @@ -213,8 +439,8 @@ class HeavyGate: except budget_error: if acquire.done() and not acquire.cancelled() and acquire.exception() is None: sem.release() - c.add("busy_refusals", 1) - await _send_busy(send, "request_deadline") + add("busy_refusals", 1) + await _send_busy(send, "request_deadline", cls) return except BaseException: # Cancellation can race a successful acquire before the handler owns the permit. @@ -226,7 +452,7 @@ class HeavyGate: timing = sys.modules.get("core.request_timing") if timing is not None: timing.record_queue((time.perf_counter() - queued_at) * 1000) - c.add("queued", -1) + add("queued", -1) if not acquire.done(): acquire.cancel() if not gone.done(): @@ -240,15 +466,15 @@ class HeavyGate: except budget_error: if got: sem.release() - c.add("busy_refusals", 1) - await _send_busy(send, "request_deadline") + add("busy_refusals", 1) + await _send_busy(send, "request_deadline", cls) return if acquire not in done or gone in done: if got: sem.release() if gone not in done: - c.add("busy_refusals", 1) - await _send_busy(send) + add("busy_refusals", 1) + await _send_busy(send, f"{cls}_queue_timeout", cls) return released = False @@ -256,7 +482,7 @@ class HeavyGate: nonlocal released if not released: released = True - c.add("in_flight", -1) + add("in_flight", -1) sem.release() async def send_wrapped(message): @@ -264,8 +490,8 @@ class HeavyGate: if message.get("type") == "http.response.body" and not message.get("more_body"): release() - c.add("in_flight", 1) - c.add("admitted", 1) + add("in_flight", 1) + add("admitted", 1) try: await self.app(scope, _replay(body_msgs, receive), send_wrapped) finally: @@ -307,3 +533,180 @@ def _replay(msgs, receive): return pending.pop(0) return await receive() return replay + + +# ----------------------------------------------------------------------------- box sizing (R25) +#: The 2 vCPU tier: today's values, exactly. Staging (cpu-basic, 2 vCPU, 16 GB) runs these. +TODAY = {"workers": 1, "heavyConcurrency": 2, "webDuckdbMemory": "6GB", "webDuckdbThreads": 2, + "workerDuckdbMemory": "4GB", "workerDuckdbThreads": 1} +#: Launch env name for each sizing value. An explicit env value always wins over the derived one. +SIZING_ENV = {"workers": "AIOS_UVICORN_WORKERS", "heavyConcurrency": "AIOS_HEAVY_CONCURRENCY", + "webDuckdbMemory": "AIOS_WEB_DUCKDB_MEMORY", "webDuckdbThreads": "AIOS_WEB_DUCKDB_THREADS", + "workerDuckdbMemory": "AIOS_WORKER_DUCKDB_MEMORY", + "workerDuckdbThreads": "AIOS_WORKER_DUCKDB_THREADS"} +VERSIONS_ENV = "AIOS_MIRROR_VERSIONS" +MAX_WEB_WORKERS = 4 +#: Share of box memory all DuckDB instances may take together (10 of 16 GB today). +DUCKDB_MEMORY_SHARE = 0.625 + + +def _read(path): + try: + with open(path, encoding="ascii") as handle: + return handle.read().strip() + except (OSError, UnicodeError): + return None + + +def _cgroup_cpus(root): + raw = _read(os.path.join(root, "cpu.max")) # cgroup v2 + if raw: + parts = raw.split() + if len(parts) == 2 and parts[0] != "max": + try: + return max(1, int(int(parts[0]) / int(parts[1]))), "cgroup v2 cpu.max" + except (ValueError, ZeroDivisionError): + pass + for sub in ("cpu", "cpu,cpuacct", "cpuacct,cpu"): # cgroup v1 + quota = _read(os.path.join(root, sub, "cpu.cfs_quota_us")) + period = _read(os.path.join(root, sub, "cpu.cfs_period_us")) + try: + if quota and period and int(quota) > 0: + return max(1, int(int(quota) / int(period))), "cgroup v1 cpu.cfs_quota_us" + except (ValueError, ZeroDivisionError): + pass + return None, None + + +def _cgroup_memory(root): + raw = _read(os.path.join(root, "memory.max")) # cgroup v2 + if raw and raw != "max" and raw.isdigit(): + return int(raw), "cgroup v2 memory.max" + raw = _read(os.path.join(root, "memory", "memory.limit_in_bytes")) # cgroup v1 + # v1 reports "no limit" as a huge page-aligned number. + if raw and raw.isdigit() and int(raw) < 1 << 60: + return int(raw), "cgroup v1 memory.limit_in_bytes" + return None, None + + +def _meminfo_bytes(path): + raw = _read(path) or "" + for line in raw.splitlines(): + if line.startswith("MemTotal:"): + try: + return int(line.split()[1]) * 1024 + except (IndexError, ValueError): + return None + return None + + +def detect_box(cgroup_root="/sys/fs/cgroup", meminfo="/proc/meminfo", affinity=None): + """{cpus, memoryBytes, cpuSource, memorySource} for this container. + + `os.cpu_count()` reports the HOST inside a container, so the cgroup limit is read first. + """ + if affinity is None: + try: + affinity = len(os.sched_getaffinity(0)) + except (AttributeError, OSError): + affinity = os.cpu_count() or 1 + cpus, cpu_source = _cgroup_cpus(cgroup_root) + if cpus is None or cpus > affinity: + cpus, cpu_source = affinity, "sched_getaffinity" + physical = _meminfo_bytes(meminfo) + memory, memory_source = _cgroup_memory(cgroup_root) + if memory is None or (physical and memory > physical): + memory, memory_source = physical, "/proc/meminfo" + return {"cpus": int(cpus), "memoryBytes": int(memory or 0), "cpuSource": cpu_source, + "memorySource": memory_source} + + +def sizing(cpus, memory_bytes, workers=None): + """Launch values for a box. Under 4 CPUs with no forced worker count: TODAY exactly. + + `workers` forces the uvicorn worker count, and the per-worker values follow it. + """ + cpus = max(1, int(cpus)) + memory_gb = max(0.0, float(memory_bytes or 0) / 1e9) + if workers is None and cpus < 4: + return {"tier": "2 vCPU", **TODAY} + if workers is None: + workers = max(1, min(MAX_WEB_WORKERS, cpus // 2, int(memory_gb // 8))) + workers = max(1, int(workers)) + worker_memory = max(4, int(memory_gb // 8)) + duckdb_total = int(memory_gb * DUCKDB_MEMORY_SHARE) + return {"tier": "derived", "workers": workers, + # One heavy request per CPU across the box, split over the workers. + "heavyConcurrency": max(1, cpus // workers), + "webDuckdbMemory": f"{max(2, (duckdb_total - worker_memory) // workers)}GB", + "webDuckdbThreads": max(2, cpus // workers), + "workerDuckdbMemory": f"{worker_memory}GB", + "workerDuckdbThreads": max(1, cpus // 4)} + + +def launch_role(environ=None): + """The role the app will resolve, as `process_role.resolve_role` does: a valid + AIOS_PROCESS_ROLE, else web on a Space (SPACE_ID set), else all.""" + environ = os.environ if environ is None else environ + raw = str(environ.get("AIOS_PROCESS_ROLE") or "").strip().lower() + if raw in ("web", "worker", "all", "reader"): + return raw + return "web" if str(environ.get("SPACE_ID") or "").strip() else "all" + + +def launch_env(environ=None, box=None): + """{env name: value} the launcher exports. Explicit env wins, except it never buys workers + without the read-only mirror: AIOS_MIRROR_VERSIONS=0 or a role other than web holds one worker. + """ + environ = os.environ if environ is None else environ + box = box or detect_box() + explicit = {key for key, name in SIZING_ENV.items() if str(environ.get(name) or "").strip()} + forced = web_workers(environ) if "workers" in explicit else None + role = launch_role(environ) + if str(environ.get(VERSIONS_ENV) or "").strip() == "0" or role != "web": + # Only the web role versions its mirror writes. Every other role restores in place. + forced = 1 + values = sizing(box["cpus"], box["memoryBytes"], workers=forced) + out = {SIZING_ENV[key]: str(values[key]) for key in SIZING_ENV + if key not in explicit and key != "workers"} + workers = int(values["workers"]) + out["AIOS_UVICORN_WORKERS"] = str(workers) + # Several workers each open the mirror read-only through its version pointer. + out[VERSIONS_ENV] = "1" if workers > 1 else "0" + return out + + +def web_app(): + """`uvicorn --factory` target for several workers. The supervisor imports only this module. + + uvicorn imports a plain `api.main:app` in its supervisor process too, which would run the + app's boot threads in a process that never serves. + """ + import importlib + return importlib.import_module("api.main").app + + +def main(argv=None): + import argparse + import shlex + parser = argparse.ArgumentParser(description="Print launch values sized to this box.") + parser.add_argument("--env", action="store_true", help="print shell export lines") + # The env defaults let a gate point the launcher at a fixture box. + parser.add_argument("--cgroup-root", default=os.environ.get("AIOS_CGROUP_ROOT") or "/sys/fs/cgroup") + parser.add_argument("--meminfo", default=os.environ.get("AIOS_MEMINFO") or "/proc/meminfo") + args = parser.parse_args(argv) + box = detect_box(args.cgroup_root, args.meminfo) + env = launch_env(os.environ, box) + print(f"[box-sizing] cpus={box['cpus']} ({box['cpuSource']}) " + f"memory={box['memoryBytes'] / 1e9:.1f}GB ({box['memorySource']}) " + + " ".join(f"{k}={v}" for k, v in env.items()), file=sys.stderr, flush=True) + if args.env: + for name, value in env.items(): + print(f"export {name}={shlex.quote(value)}") + else: + print(json.dumps({"box": box, "env": env}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/api/request_timing.py b/api/request_timing.py index a99e4c507e6e93c0a9cc9cc03c550693820bae9c..e295bdf9cd228f6d35b1f1fba1a98d8a0b58a4c4 100644 --- a/api/request_timing.py +++ b/api/request_timing.py @@ -29,6 +29,7 @@ def request_timeout_s(): _BUSY_CAUSES = { + "OdooBreakerOpen": "odoo_breaker_open", "PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full", "QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout", "SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout", diff --git a/api/routes_account_health.py b/api/routes_account_health.py new file mode 100644 index 0000000000000000000000000000000000000000..9daec2157a9857b26f710fa22d79819396c58436 --- /dev/null +++ b/api/routes_account_health.py @@ -0,0 +1,142 @@ +"""Account health App routes: summary, metric history, owned alerts and settings, each App scoped (C3).""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +TEMPLATE_KIND = "account_health" +router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"]) +SOURCE = "ut_amzn_account_health" + + +def _module(): + from modules import account_health + return account_health + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need read access to the account health database.""" + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "account_health_forbidden", "Your account cannot read Amazon account health.") + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "account_health_read_only", "Your account can view account health but not change it.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "account_health_read_only", str(exc)) from exc + except ValueError as exc: + raise err(400, "account_health_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, "account_health_not_found", "That alert or metric does not exist.") from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "account_health_unavailable", "Account health could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Account health App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "account_health_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "account_health_app_not_found", "That Account health App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session: + """A read names the App, and the caller must be able to read that App.""" + _app(session, app) + return session + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session: + """A write names the App, and only an editor of that App may change it.""" + if not _may_edit(session, _app(session, app)): + raise err(403, "account_health_read_only", "You can view this App but not change it.") + return session + + +def owners(session): + """People who may own an alert: active, non-service, non-viewer accounts of this tenant, by display name.""" + ah = _module() + try: + users = session.runtime.get("users") + except Exception: # noqa: BLE001 + users = None + want = _tenant(session).strip().lower() + names = set() + for name, rec in (users.items() if isinstance(users, dict) else ()): + if (not isinstance(rec, dict) or not rec.get("active", True) or ah._is_service(name, rec) + or str(rec.get("role") or "").strip().lower() == "viewer" + or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)): + continue + names.add(str(rec.get("name") or name).strip()) + return sorted((n for n in names if n), key=str.lower) + + +@router.get("/summary") +def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): + ah = _module() + out = _call(ah.summary, _tenant(session), st=session.runtime) + return {**out, "canEdit": _may_edit(session, _app(session, app))} + + +@router.get("/history") +def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None), + limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)): + ah = _module() + return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime) + + +@router.get("/items") +def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None), + cursor: str | None = Query(None), limit: int = Query(200, ge=1), + session: Session = Depends(_app_reader)): + ah = _module() + listed = _call(ah.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit, + st=session.runtime) + return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} + + +@router.patch("/items/{item_id}") +def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None), + due: str | None = Body(None), note: str | None = Body(None), + session: Session = Depends(_app_writer)): + ah = _module() + return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status, + due=due, note=note, st=session.runtime) + + +@router.get("/settings") +def read_settings(session: Session = Depends(_app_reader)): + ah = _module() + return _call(ah.settings, _tenant(session), st=session.runtime) + + +@router.put("/settings") +def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None), + session: Session = Depends(_app_writer)): + ah = _module() + return _call(ah.set_settings, _tenant(session), alert_distance_pct=alertDistancePct, + default_owner=defaultOwner, st=session.runtime) diff --git a/api/routes_admin.py b/api/routes_admin.py index 6f72740a3b75f86f7d5ae20c33156a1d9f5d430c..c2ec5451f4101a0bd80534a7532449b1527ac999 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -143,7 +143,17 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None): # editor will open, so the two cannot disagree. "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st, surface_keys=surface_keys), - "perms_v": int(rec.get("perms_v") or 0)} + "perms_v": int(rec.get("perms_v") or 0), + # S79-T63: only a copied test account carries `test`, so a real account's keys stay exact. + **({"test": _test_marker(rec)} if _test_marker(rec) else {})} + + +def _test_marker(rec): + mark = (rec or {}).get(users.TEST_ACCOUNT_KEY) + if not isinstance(mark, dict): + return None + return {"copiedFrom": mark.get("copied_from"), "createdBy": mark.get("created_by"), + "createdAt": mark.get("created_at")} def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None): @@ -454,6 +464,9 @@ def update_user(username: str, body: dict = Body(default=None), if result == "last_admin": raise err(400, "last_admin", "an active tenant must retain at least one administrator") + if result in _TEST_ACCOUNT_REFUSALS: + # R42: a test account gets no email and no admin role, refused in the core writer. + raise err(400, result, _TEST_ACCOUNT_REFUSALS[result]) if result != "updated": raise err(404, "no_such_user", f"no account named {uname!r}") @@ -539,6 +552,166 @@ def expire_password(username: str, session: Session = Depends(admin_gate)): return Response(status_code=204) +# ── S79-T63 (R28, R42): test accounts that copy one member's permissions exactly ───────────────── +_TEST_ACCOUNT_REFUSALS = { + "test_account_no_email": "A test account cannot have an email address.", + "test_account_no_whatsapp": "A test account cannot connect WhatsApp.", + "test_account_not_admin": "A test account cannot be an administrator.", +} + + +def _copy_refusal(exc): + return err(exc.status, exc.code, str(exc)) + + +#: The en and em dash, built from code points so this file carries no dash character (rule 2). +_DASHES = re.compile("[" + chr(0x2013) + chr(0x2014) + "]") + + +def _copy_failure(exc, verb): + # A store error can carry any text; rule 2 keeps dashes off the screen. + cause = _DASHES.sub(",", str(exc.cause or "")).strip().rstrip(".") + message = f"The test account was not {verb}. {cause}." + if exc.leftovers: + message += (" These parts could not be removed: " + ", ".join(exc.leftovers) + + ". Remove the test account again to finish.") + elif verb == "created": + message += " Nothing was kept." + return err(503, "test_account_failed", message) + + +def _test_account_steps(session): + """Copy steps for grant stores outside `core`: Connector grants, through their own route.""" + import core.perm_scope as perm_scope + import routes_keychain as keychain + + def held(): + doc = session.runtime.get(perm_scope.CONNECTOR_GRANT_KEY, fresh=True) + if doc is not None and not isinstance(doc, dict): + raise RuntimeError("connector access could not be read") + return doc or {} + + def copy_grants(username, source): + known = {kind["key"] for kind in keychain.connector_grant_kinds()} + kinds = sorted(k for k in (held().get(source) or ()) if k in known) + if kinds: + keychain.put_connector_grants(username, {"granted": kinds}, session) + + def clear_grants(username): + if username in held(): + keychain.put_connector_grants(username, {"granted": []}, session) + + return [("connector access", copy_grants, clear_grants)] + + +@router.get("/admin/users/{username}/test-account/preview") +def preview_test_account(username: str, session: Session = Depends(admin_gate)): + """What a test account copied from this member would carry, before anything is written.""" + from core import test_accounts, user_tables + reg = _registry() + try: + plan = test_accounts.plan(username, reg, tenant=_tenant_of(session.user), + st=session.runtime) + lent = user_tables.lend_defs(session.runtime, schema_only=True) + except test_accounts.CopyRefused as exc: + raise _copy_refusal(exc) + except Exception as exc: # noqa: BLE001 + raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc + limits = _source_limits(plan["source"], reg.get(plan["source"]), session) + body = test_accounts.preview(plan, st=lent, limits=limits) + _label_grants(session, body["copy"]["grants"]) + return body + + +def _source_limits(uname, rec, session): + """The source's row limits, phrased for the administrator; None if they cannot be read. + + Field walls are lifted for the phrasing only, so a condition on a hidden field is named + instead of reading "Limited by your access". The filters themselves are the source's. + """ + import core.perm_scope as perm_scope + try: + reader = deepcopy(users._public(uname, rec)) + for entry in (reader.get("perms") or {}).values(): + if isinstance(entry, dict): + entry.pop("hiddenFields", None) + return perm_scope.scope_limits(reader, st=session.runtime) + except Exception: # noqa: BLE001 + return None + + +#: What a preview row says when a shared item's name cannot be read. Never a raw id. +_GRANT_FALLBACK = {"view": "A shared view", "folder": "A shared folder", "database": "A database", + "field": "A shared field", "cohort": "A locked view", "workbook": "A shared sheet"} +#: Naming stops after this many seconds; every grant is still listed, the rest by kind. +_LABEL_BUDGET_S = 3.0 + + +def _label_grants(session, grants): + """Name each grant through the share door's own resolver (`routes_shares._object_ref`).""" + import time + try: + from routes_shares import _object_ref + except Exception: # noqa: BLE001 + _object_ref = None + deadline = time.monotonic() + _LABEL_BUDGET_S + for grant in grants: + if grant.get("label"): + continue + label = None + if time.monotonic() < deadline: + try: + if grant["kind"] == "workbook": + # `_object_ref` has no workbook branch; the workbook's own name is the label. + import core.sheets as sheets + label = str(sheets.get_workbook(session.runtime, grant["oid"]).get("name") or "") + elif _object_ref is not None: + label = str(_object_ref(session, grant["kind"], grant["oid"])[0] or "") + except Exception: # noqa: BLE001 + label = None + label = (label or "").strip() + grant["label"] = label or _GRANT_FALLBACK.get(grant["kind"], "A shared item") + + +@router.post("/admin/test-accounts", status_code=201) +def create_test_account(body: dict = Body(default=None), session: Session = Depends(admin_gate)): + """Create a test account whose permissions copy `source` exactly. The password shows once.""" + from core import test_accounts + body = body if isinstance(body, dict) else {} + _require_store() + try: + made = test_accounts.create(body.get("source"), admin=session.uname, + tenant=_tenant_of(session.user), st=session.runtime, + steps=_test_account_steps(session)) + except test_accounts.CopyRefused as exc: + raise _copy_refusal(exc) + except test_accounts.CopyFailed as exc: + raise _copy_failure(exc, "created") + except Exception as exc: # noqa: BLE001 + raise err(503, "store_unavailable", "The test account was not created. Try again.") from exc + rec = _registry().get(made["username"]) + if not isinstance(rec, dict): + raise err(503, "store_unavailable", "The test account was not saved. Try again.") + return {"user": _view(made["username"], rec), "password": made["password"]} + + +@router.delete("/admin/test-accounts/{username}", status_code=204) +def delete_test_account(username: str, session: Session = Depends(admin_gate)): + """Remove a test account, its sign in and every grant naming it, in one call.""" + from core import test_accounts + _require_store() + try: + test_accounts.remove(username, tenant=_tenant_of(session.user), st=session.runtime, + granter=session.uname, steps=_test_account_steps(session)) + except test_accounts.CopyRefused as exc: + raise _copy_refusal(exc) + except test_accounts.CopyFailed as exc: + raise _copy_failure(exc, "fully removed") + except Exception as exc: # noqa: BLE001 + raise err(503, "store_unavailable", "The test account was not removed. Try again.") from exc + return Response(status_code=204) + + # ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── #: The modules the permission editor governs — the GRID surfaces, the ones with a field schema #: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list diff --git a/api/routes_aged_stock.py b/api/routes_aged_stock.py new file mode 100644 index 0000000000000000000000000000000000000000..94a5e2165474b2296c01b0ff40ee5e516e029199 --- /dev/null +++ b/api/routes_aged_stock.py @@ -0,0 +1,167 @@ +"""Aged stock App routes (C3): summary, items, one SKU, settings, refresh and the rule 8 tie.""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +TEMPLATE_KIND = "aged_stock" +router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["aged-stock"]) +SOURCE = "ut_amzn_storage_fees" + + +def _module(): + from modules import aged_stock + return aged_stock + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need read access to the Amazon storage fees database.""" + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "aged_stock_forbidden", "Your account cannot read Amazon storage fees.") + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "aged_stock_read_only", "You can view this App but not change it.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "aged_stock_read_only", "You can view this App but not change it.") from exc + except ValueError as exc: + raise err(400, "aged_stock_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, "aged_stock_item_not_found", "That aged stock item does not exist.") from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "aged_stock_unavailable", "The aged stock view could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Aged stock App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "aged_stock_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session: + """A read names the App, and the caller must be able to read that App.""" + _app(session, app) + return session + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session: + """A write names the App, and only an editor of that App may change it.""" + if not _may_edit(session, _app(session, app)): + raise err(403, "aged_stock_read_only", "You can view this App but not change it.") + return session + + +def owners(session): + """People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name.""" + mod = _module() + try: + users = session.runtime.get("users") + except Exception: # noqa: BLE001 + users = None + want = _tenant(session).strip().lower() + names = set() + for name, rec in (users.items() if isinstance(users, dict) else ()): + if (not isinstance(rec, dict) or not rec.get("active", True) or mod._is_service(name, rec) + or str(rec.get("role") or "").strip().lower() == "viewer" + or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)): + continue + names.add(str(rec.get("name") or name).strip()) + return sorted((n for n in names if n), key=str.lower) + + +#: The item fields a Next tier row carries, so the screen lists it with no second read. +ROW_FIELDS = ("action", "recommended_action", "owner", "status", "due", "units") + + +def _summary_rows(tenant_key, st): + """The module summary with each Next tier row joined to its item's action, owner, status and due.""" + mod = _module() + out = mod.summary(tenant_key, st=st) + stored = mod._doc(st, mod.KEY_ITEMS).get("items") or {} + for row in out.get("nextTier") or (): + item = stored.get(row.get("id")) or {} + row["item"] = {key: item.get(key) for key in ROW_FIELDS} + return out + + +@router.get("/summary") +def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): + out = _call(_summary_rows, _tenant(session), session.runtime) + return {**out, "canEdit": _may_edit(session, _app(session, app))} + + +@router.get("/items") +def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None), + action: str | None = Query(None), cursor: str | None = Query(None), + limit: int = Query(200, ge=1), q: str | None = Query(None), + session: Session = Depends(_app_reader)): + listed = _call(_module().items, _tenant(session), session.uname, status=status, action=action, + cursor=cursor, limit=limit, q=q, st=session.runtime) + return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} + + +@router.get("/items/{item_id}") +def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): + detail = _call(_module().item_detail, _tenant(session), item_id, st=session.runtime) + return {**detail, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} + + +@router.patch("/items/{item_id}") +def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None), + due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None), + session: Session = Depends(_app_writer)): + return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due, + action=action, note=note, user=session.uname or "unknown", st=session.runtime) + + +@router.get("/settings") +def read_settings(session: Session = Depends(_app_reader)): + return _call(_module().settings, _tenant(session), st=session.runtime) + + +@router.patch("/settings") +def write_settings(adsFromDays: int | None = Body(None), priceDownFromDays: int | None = Body(None), + removeFromDays: int | None = Body(None), feeHorizonDays: int | None = Body(None), + dueDays: int | None = Body(None), defaultOwner: str | None = Body(None), + session: Session = Depends(_app_writer)): + return _call(_module().set_settings, _tenant(session), ads_from_days=adsFromDays, + price_down_from_days=priceDownFromDays, remove_from_days=removeFromDays, + fee_horizon_days=feeHorizonDays, due_days=dueDays, default_owner=defaultOwner, + st=session.runtime) + + +@router.post("/refresh") +def refresh(session: Session = Depends(_app_writer)): + return _call(_module().refresh, _tenant(session), st=session.runtime) + + +@router.get("/validate") +def validate(start: str | None = Query(None), end: str | None = Query(None), + session: Session = Depends(_app_reader)): + return _call(_module().validate, _tenant(session), start=start, end=end, st=session.runtime) diff --git a/api/routes_amazon_operator.py b/api/routes_amazon_operator.py new file mode 100644 index 0000000000000000000000000000000000000000..6326065aedcdc649548bec19033e116611517ddf --- /dev/null +++ b/api/routes_amazon_operator.py @@ -0,0 +1,133 @@ +"""Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss. + +Every route names its App with `?app=`. Approve and dismiss call `amazon_operator.set_status`, the +same path an MCP approval takes. The router never calls the Ads client. +""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1/app-kinds/amazon_operator", tags=["amazon-operator"]) +TEMPLATE_KIND = "amazon_operator" + + +def _module(): + from modules import amazon_operator + return amazon_operator + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need a signed in session. The App grant is checked per request.""" + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "amazon_operator_read_only", "Your account can view these proposals but not change them.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "amazon_operator_read_only", str(exc)) from exc + except ValueError as exc: + raise err(400, "amazon_operator_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, "amazon_operator_not_found", "That proposal does not exist.") from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "amazon_operator_unavailable", "The Amazon operator could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Amazon operator App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "amazon_operator_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "amazon_operator_app_not_found", "That Amazon operator App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session: + """C3: a read names the App, and the caller must be able to read that App.""" + _app(session, app) + return session + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session: + """C3: a write names the App, and only an editor of that App may change it.""" + if not _may_edit(session, _app(session, app)): + raise err(403, "amazon_operator_read_only", "You can view this App but not change it.") + return session + + +@router.get("/today") +def read_today(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): + op = _module() + out = _call(op.today, _tenant(session), st=session.runtime) + return {**out, "canEdit": _may_edit(session, _app(session, app))} + + +@router.get("/history") +def read_history(status: str | None = Query(None), cursor: str | None = Query(None), + limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)): + op = _module() + page = _call(op.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit, + st=session.runtime) + if cursor: + return page + # The first page carries the full count for the History tab. Uncapped: every row is counted (rule 1). + every = _call(op.items, _tenant(session), session.user, status=status, cursor=None, limit=None, + st=session.runtime) + return {**page, "total": len(every["items"])} + + +@router.get("/settings") +def read_settings(session: Session = Depends(_app_reader)): + op = _module() + return _call(op.settings, _tenant(session), st=session.runtime) + + +@router.put("/settings") +def write_settings(stop_cover_days: float | None = Body(None), raise_cover_days: float | None = Body(None), + pause_below_profit: float | None = Body(None), bid_step_pct: float | None = Body(None), + budget_step_pct: float | None = Body(None), lookback_days: float | None = Body(None), + negative_min_clicks: float | None = Body(None), negative_min_cost: float | None = Body(None), + defaultOwner: str | None = Body(None), session: Session = Depends(_app_writer)): + op = _module() + return _call(op.set_settings, _tenant(session), st=session.runtime, default_owner=defaultOwner, + stop_cover_days=stop_cover_days, raise_cover_days=raise_cover_days, + pause_below_profit=pause_below_profit, bid_step_pct=bid_step_pct, + budget_step_pct=budget_step_pct, lookback_days=lookback_days, + negative_min_clicks=negative_min_clicks, negative_min_cost=negative_min_cost) + + +@router.post("/approve") +def approve(item_ids: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)): + """Approve the ticked proposals as one batch. Returns each row's outcome.""" + op = _module() + return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime) + + +@router.post("/dismiss") +def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length=1), + session: Session = Depends(_app_writer)): + op = _module() + return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason, + st=session.runtime) diff --git a/api/routes_assets.py b/api/routes_assets.py index a0c083b671aae0392062d6520c867e2158f76da3..8efaef0eb05c75e3e92f87bf4dda588a7672a7df 100644 --- a/api/routes_assets.py +++ b/api/routes_assets.py @@ -36,7 +36,7 @@ from fastapi import Body, Depends from fastapi import APIRouter from fastapi.responses import FileResponse, Response -from deps import Session, err, module_gate, require_session +from deps import Session, err, is_bearer, module_gate_or_bearer, require_session, require_session_or_bearer from core import platform_admin from routes_admin import admin_gate import application_registry as applications @@ -467,6 +467,13 @@ _EXT_MEDIA = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"} #: closed charset — no slashes, no backslashes, no colons, so a code can never be a path. _CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$") _IMMUTABLE = "public, max-age=31536000, immutable" +#: A bearer answer stays out of shared caches: a CDN must never serve a key-fetched image to anyone. +_BEARER_CACHE = "private, max-age=300" + + +def _cache_control(session): + """The cookie answer keeps its immutable header. A bearer answer is private.""" + return _BEARER_CACHE if is_bearer(session) else _IMMUTABLE #: per-process index of the local master dir: rebuilt when the dir path or its mtime changes. _DIR_INDEX = {"path": None, "mtime": None, "map": {}} @@ -548,8 +555,16 @@ def _assets(): return assets +def product_image_codes(): + """The upper-cased codes that have a master image, from the dir or the repo manifest.""" + d = _local_dir() + if d is not None: + return set(_dir_index(d)) + return {_norm(code) for code in (_assets().manifest() or {})} + + @router.get("/assets/products") -def asset_manifest(session: Session = Depends(module_gate(MODULE))): +def asset_manifest(session: Session = Depends(module_gate_or_bearer(MODULE))): """The codes that HAVE an image — so the catalog designer can mark gaps instead of rendering broken tiles. `source` names which backend answered.""" d = _local_dir() @@ -563,7 +578,7 @@ def asset_manifest(session: Session = Depends(module_gate(MODULE))): @router.get("/assets/products/{code}") def product_asset(code: str, q: str = "web", - session: Session = Depends(module_gate(MODULE))): + session: Session = Depends(module_gate_or_bearer(MODULE))): if q not in _QUALITIES: raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}") norm = _norm(code) @@ -583,7 +598,7 @@ def product_asset(code: str, q: str = "web", media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream") return FileResponse(str(path), media_type=media, - headers={"Cache-Control": _IMMUTABLE}) + headers={"Cache-Control": _cache_control(session)}) # ------------------------------------------------------------------ editorial (DEBT-5) @@ -626,7 +641,7 @@ def _ed_dir_index(ed): @router.get("/assets/editorial") -def editorial_manifest(session: Session = Depends(module_gate(MODULE))): +def editorial_manifest(session: Session = Depends(module_gate_or_bearer(MODULE))): """The slugs that HAVE an editorial image — the designer's picker reads this.""" if _local_dir() is not None: ed = _ed_dir() @@ -637,7 +652,7 @@ def editorial_manifest(session: Session = Depends(module_gate(MODULE))): @router.get("/assets/editorial/{slug}") def editorial_asset(slug: str, q: str = "web", - session: Session = Depends(module_gate(MODULE))): + session: Session = Depends(module_gate_or_bearer(MODULE))): if q not in _QUALITIES: raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}") slug = str(slug or "").strip().lower() @@ -656,7 +671,7 @@ def editorial_asset(slug: str, q: str = "web", raise err(404, "asset_not_found", f"no editorial image named {slug}") media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream") return FileResponse(str(path), media_type=media, - headers={"Cache-Control": _IMMUTABLE}) + headers={"Cache-Control": _cache_control(session)}) @router.post("/assets/editorial", status_code=201) @@ -825,7 +840,7 @@ def record_upload(body: dict = Body(default=None), @router.get("/assets/records/{asset_id}") def record_asset(asset_id: str, q: str = "web", - session: Session = Depends(require_session)): + session: Session = Depends(require_session_or_bearer)): """Serve one record image for THIS session's tenant. Any signed-in user of the tenant may read it — a picture on a row is visible to whoever can open the row, and the row walls already decided that.""" @@ -853,4 +868,4 @@ def record_asset(asset_id: str, q: str = "web", raise err(404, "asset_not_found", "no image for that reference") media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream") return FileResponse(str(path), media_type=media, - headers={"Cache-Control": _IMMUTABLE}) + headers={"Cache-Control": _cache_control(session)}) diff --git a/api/routes_automation.py b/api/routes_automation.py index c100952dcec45c352e3bedf66a6f603c8567c9ef..953d33cdce8d3b078c9418c2f29fb77c698e373b 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -1350,12 +1350,14 @@ def list_automations(session: Session = Depends(_GATE)): retired = [] tables = None if session.runtime.available() and session.tenant not in _BOARD_RETIRED: - tables = engine.ut_all(session.runtime) _BOARD_RETIRED.add(session.tenant) # Idempotent Board retirement removes only engine-marked stage fields and legacy Board # state. User-created Status/Stage columns remain intact. - engine.retire_automation_board_state(session.runtime, tables=tables) - retired = engine.retire_preset_automations(session.runtime)["removed"] + if not (session.runtime.get("automation_migrations") or {}).get("board_retired_v1"): + tables = engine.ut_all(session.runtime) + engine.retire_automation_board_state(session.runtime, tables=tables) + retired = engine.retire_preset_automations(session.runtime)["removed"] + session.runtime.update("automation_migrations", lambda d: {**(d or {}), "board_retired_v1": True}) # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds # it present and returns immediately. See `engine.seed_statements_agent` for why this one is diff --git a/api/routes_catalogs.py b/api/routes_catalogs.py index 2d56fc7221bc304ae1c0f0b8e9cc55dded8a9a62..c3602d2d6964bfb4601006c6aedcbaa68af09da2 100644 --- a/api/routes_catalogs.py +++ b/api/routes_catalogs.py @@ -1,10 +1,13 @@ -"""Authenticated catalog record routes for Catalog Apps.""" +"""Authenticated catalog record routes for Catalog Apps. + +Every route admits the cookie or an external bearer under its own key's grants (S79-T09). +""" from fastapi import APIRouter, Body, Depends, Query from fastapi.responses import Response import app_rules import catalog_store -from deps import Session, err, require_session +from deps import Session, err, require_bearer_write, require_session_or_bearer router = APIRouter(prefix="/api/v1") @@ -20,29 +23,30 @@ def _call(operation, *args, **kwargs): @router.get("/apps/{app_id}/catalogs") def catalogs(app_id: str, cursor: str | None = None, limit: int = Query(catalog_store.DEFAULT_LIMIT, ge=1, le=catalog_store.MAX_LIMIT), - session: Session = Depends(require_session)): + session: Session = Depends(require_session_or_bearer)): return _call(catalog_store.list_catalogs, session, app_id, cursor=cursor, limit=limit) @router.post("/apps/{app_id}/catalogs", status_code=201) def create_catalog(app_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): + session: Session = Depends(require_session_or_bearer)): + require_bearer_write(session, "catalogs") return {"catalog": _call(catalog_store.create_catalog, session, app_id, body)} # Static paths must precede the catalog id route. @router.get("/catalogs/templates") -def catalog_templates(session: Session = Depends(require_session)): +def catalog_templates(session: Session = Depends(require_session_or_bearer)): return app_rules.catalog_templates() @router.get("/catalogs/{catalog_id}") -def read_catalog(catalog_id: str, session: Session = Depends(require_session)): +def read_catalog(catalog_id: str, session: Session = Depends(require_session_or_bearer)): return {"catalog": _call(catalog_store.read_catalog, session, catalog_id)} @router.get("/catalogs/{catalog_id}/pdf") -def catalog_pdf(catalog_id: str, session: Session = Depends(require_session)): +def catalog_pdf(catalog_id: str, session: Session = Depends(require_session_or_bearer)): import catalog_render data, _record, _app = _call(catalog_render.render_catalog, session, catalog_id) @@ -53,16 +57,19 @@ def catalog_pdf(catalog_id: str, session: Session = Depends(require_session)): @router.patch("/catalogs/{catalog_id}") def update_catalog(catalog_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): + session: Session = Depends(require_session_or_bearer)): + require_bearer_write(session, "catalogs") return {"catalog": _call(catalog_store.update_catalog, session, catalog_id, body)} @router.post("/catalogs/{catalog_id}/duplicate", status_code=201) def duplicate_catalog(catalog_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): + session: Session = Depends(require_session_or_bearer)): + require_bearer_write(session, "catalogs") return {"catalog": _call(catalog_store.duplicate_catalog, session, catalog_id, body)} @router.delete("/catalogs/{catalog_id}") -def delete_catalog(catalog_id: str, session: Session = Depends(require_session)): +def delete_catalog(catalog_id: str, session: Session = Depends(require_session_or_bearer)): + require_bearer_write(session, "catalogs") return _call(catalog_store.delete_catalog, session, catalog_id) diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 00fab138bb433c6bce2d92a4895a5781a3468664..16a61af892ec65c0a89a05441ab6084db096064d 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -54,6 +54,7 @@ from core import connector_sync from core.request_timing import raise_if_resource_error from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD from harness import amazon_client +import connectors_amazon_ads #: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default, #: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already #: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module @@ -499,6 +500,12 @@ TOKEN_CONNECTORS = [ "marketplace_id.", # R22: status comes from the stored verification state, without mechanism copy. "note": ""}, + # S79-T13 (R7, R9): Amazon Ads reads SP, SB and SD reports. Settings only with a verified + # credential; an expired or failed token reads Connect again (`connectors_amazon_ads.card_state`). + {"key": "amazon_ads", "label": "Amazon Ads", "entry": "amazon_ads", + "desc": "Sponsored ads reports", + "hint": "Paste the four values from your Amazon Ads app", + "note": ""}, ] #: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many @@ -602,7 +609,7 @@ def _register_probers(): store a credential and never the API's ability to boot. """ ok = False - for mod in ("connectors_meta", "connectors_amazon"): + for mod in ("connectors_meta", "connectors_amazon", "connectors_amazon_ads"): try: ok = __import__(mod).register() or ok except Exception: # noqa: BLE001 @@ -986,7 +993,8 @@ def _entry_types(rt, uname=""): types, scopes = set(), {} for e in rows: t = str(e.get("type") or "") - if t == "amazon_sp": + # Amazon SP and Amazon Ads count only with a passing probe proof for the current secret. + if t in ("amazon_sp", "amazon_ads"): from core import keychain as kc if kc.entry_verified(rt, e.get("id")): types.add(t) @@ -1238,7 +1246,9 @@ def directory(rt, uname, is_admin=False, incomplete=None): rows.append({ "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"], "hint": t["hint"], - "state": ("expired" if t["entry"] == "amazon_sp" and amazon_expired + "state": (connectors_amazon_ads.card_state(t["entry"] in have) + if t["entry"] == connectors_amazon_ads.ENTRY_TYPE + else "expired" if t["entry"] == "amazon_sp" and amazon_expired else "connected" if t["entry"] in have else "reconnect" if t["entry"] == "amazon_sp" and t["entry"] in scopes else "available"), @@ -1268,8 +1278,10 @@ def directory(rt, uname, is_admin=False, incomplete=None): # 400s — the exact failure R6 removed from the Google card. Today that means # Stripe and Shopify open for a member and Meta Ads does not # ([[one-question-two-normalizers]]). + # S79-T13: Amazon Ads is business wide, before and after the keychain learns it. "manage": ("keychain" if is_admin else "" if t["entry"] in TENANT_WIDE_TYPES + or t["entry"] == connectors_amazon_ads.ENTRY_TYPE else "keychain_personal"), # Said once, on the row, rather than in a paragraph somewhere (R13): storing the # key is real and useful, and it is not the same as the data being on screen. @@ -3519,6 +3531,14 @@ def _entry_type_for(connector_key): _AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret", "refresh_token": "Refresh token", "seller_id": "Seller ID", "marketplace_id": "Marketplace ID"} +_AMAZON_ADS_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret", + "refresh_token": "Refresh token", "profile_id": "Profile ID"} +_AMAZON_ADS_FIELD_HINTS = { + "client_id": "Your Amazon Ads app's Login with Amazon client ID.", + "client_secret": "The same app's client secret.", + "refresh_token": "The refresh token returned when you authorize the app.", + "profile_id": "The advertising profile for your marketplace.", +} _CONNECT_FIELDS = { "odoo": [{"key": "url", "label": "Server URL", "secret": False, "hint": "Your Odoo address, starting with https://."}, @@ -3545,6 +3565,14 @@ _CONNECT_FIELDS = { **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})} for key in amazon_client.CREDENTIAL_FIELDS ], + # S79-T13: keys and order derived from `amazon_ads_client.CREDENTIAL_FIELDS`, like Amazon SP. + "amazon_ads": [ + {"key": key, + "label": _AMAZON_ADS_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()), + "secret": key in ("client_secret", "refresh_token"), + "hint": _AMAZON_ADS_FIELD_HINTS.get(key, "")} + for key in connectors_amazon_ads.CREDENTIAL_FIELDS + ], "stripe": [{"key": "api_key", "label": "API key", "secret": True, "hint": "A restricted key with read access."}], "shopify": [{"key": "api_key", "label": "Admin API token", "secret": True, @@ -3669,6 +3697,16 @@ _CONNECT_TUTORIALS = { "and check its marketplace access. Correct the saved key in Settings.", "testNote": "Checks marketplace access. Saves the key before testing.", }, + "amazon_ads": { + "need": "An Amazon Ads API app and access to your advertising account.", + "steps": ["Open your Amazon Ads API app's Login with Amazon security profile.", + "Copy its client ID and client secret.", + "Authorize the app for your advertising account and copy the refresh token.", + "Copy the profile ID of the marketplace you advertise in."], + "failureAction": "Authorize the app again for your advertising account, then check " + "that the profile ID belongs to it.", + "testNote": "Checks access to your advertising profile. Saves the key before testing.", + }, "stripe": { "need": "Access to your Stripe dashboard's API keys.", "steps": ["Open the Stripe dashboard, then Developers and API keys.", @@ -3695,12 +3733,12 @@ _CONNECT_TUTORIALS = { }, "webhooks": { "need": "An automation and a system that can send an HTTP POST.", - "steps": ["Go to Automations and create or open an automation.", - "Choose the Webhooks connector trigger. Saving it creates its token.", - "Use your Loopable host followed by /api/v1/automations/{automation_id}/hook/{token}. " - "Use this automation's ID and trigger token; keep the token private.", - "Paste that URL into your sending system."], - "failureAction": "Check the automation ID and token in the URL, then confirm the trigger is enabled.", + # The client lists each automation's generated URL. No step names a credential or a keychain Test. + "steps": ["Open Automations and create or open an automation.", + "Choose the Webhooks trigger and save it. Saving creates its own URL.", + "Copy its URL below. Keep it private, because anyone with it can run the automation.", + "Send an HTTP POST to that URL from your other system."], + "failureAction": "Copy the URL again from this guide. Check that the trigger is on.", "testNote": "Sending to the hook runs the automation. Check its run history after an intentional send.", }, } @@ -3905,7 +3943,8 @@ def _detail_connect(entry_type, is_admin): if not entry_type: return None fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD) - tenant_wide = entry_type in TENANT_WIDE_TYPES + tenant_wide = (entry_type in TENANT_WIDE_TYPES + or entry_type == connectors_amazon_ads.ENTRY_TYPE) hint = "" try: hint = next((str(token.get("hint") or "") for token in TOKEN_CONNECTORS diff --git a/api/routes_customers.py b/api/routes_customers.py index cf3931819a8979f6ec68103b645c242c4022a4e9..b30e4e612b719fdd9231c46b17bd4369f52c3f90 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -229,6 +229,22 @@ def warm_default(rt): _pool_for(rt, None, None) +def warm_bu_pools(rt): + """Warm each permitted business-unit pool outside the request path.""" + from core import context, perms + for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}): + if label == "All" or label not in context.BRANDS: + continue + team = perms.scope_team_id({"bus": [context.BRANDS[label]], "role": "user"}) + if team is None: + continue + try: + _pool_for(rt, team, None) + print(f"[aios-api] customer read pool warm: {label}") + except Exception as exc: + print(f"[aios-api] customer read pool warm refused: {label} ({type(exc).__name__})") + + def _team_agent(session: Session): """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree. diff --git a/api/routes_external.py b/api/routes_external.py index fbef8ff941271e8ad6f07682bf7a8fcff5206872..2a1952f22b52d9e1d54b33f33db480d038356f43 100644 --- a/api/routes_external.py +++ b/api/routes_external.py @@ -27,6 +27,9 @@ import aios_session from deps import Session, err, require_session from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets from harness import runtime +from tool_groups.data import (QUERY_SYNTAX_CODES, UNKNOWN_DATABASE_FIX, bad_filter_message, + query_syntax_section, with_query_syntax) +from tool_groups.sheet import WORKBOOK_MISSING_FIX EXTERNAL_PREFIX = "/api/v1/external" @@ -228,6 +231,9 @@ class _PreviewRuntime: key = str(name) self._load(key) base = self._base_revisions.get(key) + if isinstance(base, dict) and not self._local_revisions.get(key, 0): + # D-8920: an unchanged copy answers its source revision, so a preview staged inside a preview pins it. + return copy.deepcopy(base) rev = int((base or {}).get("rev") or 0) + self._local_revisions.get(key, 0) return {"rev": rev, "token": _json_digest({"key": key, "rev": rev, "base": base})} @@ -851,6 +857,12 @@ def _assert_query_filter_bounds(raw) -> None: stack.extend((child, depth + 1) for child in children) +def _is_query_call(name, arguments): + """Tell a query_database call at its bare, table or data address.""" + return name == "query_database" or ( + name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database") + + def _assert_tool_query_filter_bounds(name, arguments): """Name query limits before the finite tool schema reports a generic shape error.""" if name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database": @@ -1616,6 +1628,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str: "", ]) + lines.extend(query_syntax_section()) if public: lines.extend([ "## Your read scope", @@ -2381,7 +2394,7 @@ def external_setup(request: Request): @router.get("/skill.md") -def portable_skill(request: Request, session: Session = Depends(require_session)): +def portable_skill(request: Request, session: Session = Depends(deps.require_session_or_bearer)): """Download current governed external read and build instructions.""" require_external_analyst(session) document = _portable_skill_document(session, str(request.base_url)) @@ -2483,11 +2496,13 @@ def query_database( database: str, body: dict | None = Body(default=None), session: Session = Depends(require_external_session), + request: Request = None, ): """Read one freshly permissioned, bounded and cursor-stable Assistant snapshot.""" try: require_external_analyst(session) - return _query_database(session, database, body if body is not None else {}) + with _external_image_base(_mcp_public_origin(request) if request is not None else ""): + return _query_database(session, database, body if body is not None else {}) except HTTPException as exc: external_activity.record_refused(session, database, exc) raise @@ -2557,6 +2572,11 @@ def _query_database(session: Session, database: str, payload: dict): "assistant_store_not_ready", "store_not_ready", "amazon_mirror_not_loaded"}: raise err(503, "query_source_unavailable", "This database is temporarily unavailable. Try the same query again shortly.") from exc + if info.get("code") == "assistant_bad_filter": + # S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules. + info["message"] = bad_filter_message(info.get("message")) + if info.get("code") in QUERY_SYNTAX_CODES and isinstance(info.get("message"), str): + info["message"] = with_query_syntax(info["message"]) if info.get("code") == "query_cursor_stale": fix = _MCP_FIXES["query_cursor_stale"] if fix not in info["message"]: @@ -2612,6 +2632,37 @@ def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggrega return result +@contextmanager +def _external_image_base(base_url): + """Give product rows image URLs on this origin for one external query, and only that query.""" + from modules import product_data + token = product_data.EXTERNAL_IMAGE_BASE.set(str(base_url or "").rstrip("/")) + try: + yield + finally: + product_data.EXTERNAL_IMAGE_BASE.reset(token) + + +def _external_image_urls(session, database, rows, records): + """Replace each product row's Image cell with a URL the caller's own credential fetches.""" + if database != "product_data" or (getattr(session, "claims", None) or {}).get("external") is not True: + return records + from modules import product_data + base = product_data.EXTERNAL_IMAGE_BASE.get() + key = product_data.IMAGE_FIELD_KEY + if base is None or not records or key not in records[0]: + return records + import routes_assets + try: + masters = routes_assets.product_image_codes() + except Exception: # noqa: BLE001 - an unreadable manifest leaves empty cells empty, never fails the read + masters = set() + for row, record in zip(rows, records): + path = product_data.image_path(record.get(key), row.get("code"), masters) + record[key] = base + path if path.startswith("/") else path + return records + + def _query_database_page(session: Session, database: str, payload: dict): if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS: raise err(400, "query_arguments_invalid", @@ -2784,6 +2835,7 @@ def _query_database_page(session: Session, database: str, payload: dict): records = [{"pid": copy.deepcopy(row.get("pid")), **{key: copy.deepcopy(row.get(key)) for key in selected_keys}} for row in page_rows] + records = _external_image_urls(session, database, page_rows, records) next_offset = offset + limit check_publication() next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset, publication) @@ -2849,6 +2901,7 @@ def _query_database_page(session: Session, database: str, payload: dict): **{key: copy.deepcopy(row.get(key)) for key in selected_keys}} for row in page_rows ] + records = _external_image_urls(session, database, page_rows, records) next_offset = offset + len(page_rows) check_publication() next_cursor = (_encode_query_cursor(session, query_digest, snapshot_digest, next_offset, publication) @@ -3259,6 +3312,11 @@ def _application_upsert(session, args): allowed, _ = _application_access(session, current) if not current or not allowed: raise applications.ApplicationError("application does not exist") + stored = applications.load(session.runtime)["applications"].get(app_id) + from tool_groups.apps import is_template_app, template_sealed + if is_template_app(stored): + # R35: a template App's definition is sealed over tools. Admins rename and share it in web. + raise template_sealed(stored) proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current proposed_allowed, records = _application_access(session, proposed, strict=True) return {"application": applications.mutate( @@ -4052,6 +4110,11 @@ def _affected_for(name, args, diff, result=None): ranges.append({"workbook_id": arguments.get("workbook_id"), "tab": arguments.get("tab") or arguments.get("worksheet"), "range": address}) collect(args) + inner = args.get("arguments") if name == "apps" and isinstance(args.get("arguments"), dict) else {} + # C8: an App item change names its App and every item id it changes. + for value in [inner.get("application"), *(inner.get("item_ids") or [])]: + if value is not None and str(value) not in values: + values.append(str(value)) return {"ids": values[:100], "count": max(len(values), len(diff)), "ranges": ranges} @@ -4617,7 +4680,7 @@ def _confirmation_claim(session, operation_id): if item["status"] != "preview": raise _OperationAlreadyClaimed() if item["payload"]["expiry"] <= time.time(): - raise err(409, "confirmation_expired", "This confirmation expired. Preview the change again.") + raise err(409, "confirmation_expired", _expired_fix(item["payload"])) item["status"] = "claimed" item["retain_until"] = time.time() + _OPERATION_RETENTION_SECONDS return raw @@ -4746,6 +4809,24 @@ def _finish_website_note(session, payload, plan, result): _STALE_FIX = "so repeat the original call for a new preview and confirm with its new confirmation_token." +def _expired_fix(payload): + """Name the call that previews an expired change again and the confirm that follows it.""" + import tool_groups + payload = payload if isinstance(payload, dict) else {} + tool, arguments = payload.get("tool"), payload.get("arguments") + address = "the original tool" + if isinstance(tool, str) and isinstance(arguments, dict): + try: + group, grouped = tool_groups.legacy_target(tool, arguments) + address = group if group == tool else group + " with action " + grouped["action"] + except (KeyError, TypeError, ValueError): + if tool in tool_groups.groups() and isinstance(arguments.get("action"), str): + address = tool + " with action " + arguments["action"] + return ("This confirmation token expired. Call " + address + " again with the same arguments for a new " + "preview. Then call confirm_change with its new confirmation_token within " + + str(_CONFIRMATION_TTL_SECONDS // 60) + " minutes.") + + def _confirm_fix(args): """Name what a confirm_change call got wrong and the shape that succeeds.""" allowed = ("confirmation_token", "confirmation_phrase", "intent") @@ -4778,8 +4859,7 @@ def _confirm_change(session, args): if payload.get("environment") != _write_environment(): raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX) if int(payload.get("expiry") or 0) <= int(time.time()): - raise err(409, "confirmation_expired", - "This confirmation token expired. Preview the change again.") + raise err(409, "confirmation_expired", _expired_fix(payload)) if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE: raise err(400, "confirmation_phrase_required", "Enter the exact stronger confirmation phrase shown in the preview.") @@ -4905,7 +4985,7 @@ def _confirm_change(session, args): _MCP_CODE_GROUPS = ( ("The arguments do not match this action. Read its guide and correct them.", """ tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type - query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid + assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid bad_application bad_entries bad_value bad_values bad_request bad_recurrence invalid_field_contract external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid """), @@ -5000,6 +5080,12 @@ MCP_RESULT_CODES.update({ "confirmation_session_required": "Read this confirmation through its authenticated connection.", "link_selection_refused": "Choose a readable linked record that matches the field's selection condition.", "app_rule": "Correct the App rule named in the refusal before saving it.", + "template_app_sealed": "A Loopable template App cannot be changed over tools. Use its own tools. An Administrator can rename or share it in Loopable.", + "app_unavailable": "No App with this id is shared with this account. Call apps.list for the Apps it can open.", + "app_items_unavailable": "This App has no items to list or change over tools. Open it in Loopable.", + "app_items_forbidden": "This account cannot change this App's items. Ask its owner or an Administrator.", + "app_item_status_invalid": "Choose a status this App allows. See allowed.", + "app_item_unknown": "Choose item ids that apps.items returned for this App.", "audience_not_allowed": "A person must choose sharing with everyone on screen.", "not_owner": "You cannot administer this item's audience.", "unknown_people": "Choose active people in this workspace.", @@ -5148,6 +5234,8 @@ _MCP_RESULT_OBJECTS = frozenset({ "applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools", "annotations", "serverInfo", "environment", }) +# A link descriptor names its target database key. It is an identifier, never prose (D-9307). +_MCP_RESULT_DESCRIPTORS = frozenset({"link"}) _MCP_AUTHORED_DESCRIPTIONS = frozenset({ "fields", "file", "files", "template", "templates", "application", "applications", "view", "views", }) @@ -5174,6 +5262,8 @@ def _mcp_result_copy(value, labels=None, *, field=None, metadata=False): result[key] = copy.deepcopy(child) elif key == "about": result[key] = copy.deepcopy(child) + elif key in _MCP_RESULT_DESCRIPTORS and isinstance(child, dict): + result[key] = copy.deepcopy(child) elif key in _MCP_RESULT_OPAQUE and not metadata: result[key] = copy.deepcopy(child) elif key in {"schema", "inputSchema"}: @@ -5944,7 +6034,10 @@ def _record_mcp_method(session, req, reply): or audit.redact_intent(method) != method): method = "[redacted]" error = reply.get("error", {}).get("code") - code = "request_method_unknown" if error == -32601 else "request_invalid" if error else "" + if error == -32601: + # D-2667: a probe for a method the server does not offer is no refused query and crowds out history. + return + code = "request_invalid" if error else "" external_activity.record_protocol(session, method, code=code) except Exception: pass @@ -6175,6 +6268,20 @@ def _dispatch_mcp(req, session: Session, base_url: str = ""): return reply +#: Raw refusal codes whose MCP answer names the call that succeeds next (S79-T11). +_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX} + + +def _mcp_next_call(code, message): + """Append the call that succeeds next for a refusal that names a missing item (S79-T11).""" + text = str(message or "").rstrip() + fix = _MCP_NEXT_CALL.get(code, "") + if code == "assistant_unknown_database": + # The read path's lowercase reason is replaced: the fix sentence names the missing key itself. + return fix + return text if not fix or fix in text else (text + " " + fix).strip() + + def _dispatch_mcp_call(req, session: Session, base_url: str = ""): """One HTTP request object; the stdio precedent's JSON-RPC and content envelopes.""" protocol_error = _mcp_protocol_error @@ -6326,6 +6433,9 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): wiki_log.record_build(session, kind=kind, object_id=arguments["arguments"]["path"], event_id="mcp-wiki:" + _json_digest({"path": arguments["arguments"]["path"], "version": data["version"]}), occurred_at=data["updated"], purpose=purpose, about=about, build_kind=kind) + elif name == "query_database": + with _external_image_base(base_url): + data = _dispatch_external_tool(_tool_context(session), name, arguments) else: data = _dispatch_external_tool(_tool_context(session), name, arguments) if isinstance(data, dict) and data.get("refused"): @@ -6353,6 +6463,10 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): import routes_sheet if exc.code == "tool_arguments_invalid" and name == "confirm_change": exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments)) + if exc.code == "tool_arguments_invalid" and _is_query_call(name, arguments): + exc = sheets.SheetRefusal(exc.code, with_query_syntax(exc.message)) + if exc.code in _MCP_NEXT_CALL: + exc = sheets.SheetRefusal(exc.code, _mcp_next_call(exc.code, exc.message)) translated = routes_sheet._refuse(exc) info = translated.detail["error"] out = _tool_refusal(name, info["code"], info["message"], translated.status_code, @@ -6377,6 +6491,8 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): message = info.get("message", "The app refused this operation.") if info.get("code") == "build_about_required" and isinstance(message, str): message += _mcp_about_fix(called) + if info.get("code") in _MCP_NEXT_CALL: + message = _mcp_next_call(info["code"], message) out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry) except (TypeError, ValueError, KeyError, AttributeError): out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.") diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 1dfcfc66699c04bf7e7c1ad9d6f3c2472db6b5db..f51c39b057c6e58a0c894836d533a1b85cc8bc9c 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -122,7 +122,7 @@ SCOPE_KEY = "keychain_scopes" #: (the resolver-side guard held), but a door that stores a credential the product then ignores, #: which is its own kind of wrong. `routes_connectors.py::_tokens` imports this tuple rather than #: keeping its own, so its `manage` field for the Amazon card corrects itself in the same change. -TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp") +TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp", "amazon_ads") #: ⭐⭐ W53-T29 / OWNER RULING R6 — WHICH SCOPES ONE SOURCE MAY EVER CARRY, SAID AS DATA. diff --git a/api/routes_nav.py b/api/routes_nav.py index 6ecd171d3fc9719c5c59acc01296dc3e5c2b119b..bea38aea80c1f26ab7c54949d482cbcb985cadd3 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -531,12 +531,24 @@ def _granted_view_ids(session): #: of the reserved words below is still a second connection, and its sibling must still be named. #: #: These two reserved words are the rail's OWN vocabulary, not a connection's to take: -#: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the -#: "Loopable" group; the legacy "Custom" group is reserved too. A connection someone named +#: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` / +#: empty into the one Custom group (S79-T60, key `CUSTOM_SOURCE_GROUP`). A connection someone named #: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside #: somebody's scratch table, so such a label falls back to the generic Odoo folder instead. _RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"}) +#: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "Custom"; +#: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`. +CUSTOM_SOURCE_GROUP = "custom" +#: Native provenance tokens, the legacy "Loopable" from `user_tables` included, fold into the key. +_NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation"}) + + +def _native_group(group): + """The `sourceGroup` to send: a native token becomes `CUSTOM_SOURCE_GROUP`, a connector keeps its name.""" + text = str(group or "").strip() + return CUSTOM_SOURCE_GROUP if text.lower() in _NATIVE_GROUP_TOKENS else text + #: The registry modules that ARE the designated default connection's data rather than a namespaced #: copy of it: `odoo_relational.namespaced_fields` says so in as many words ("`customer_link` #: targets the compiled registry module `customer_data`, which holds the DEFAULT connection's @@ -712,7 +724,7 @@ def nav(session: Session = Depends(require_session)): continue pages.append({"key": e["key"], "label": e["label"], "source": e.get("source") or "Blank", - "sourceGroup": e.get("sourceGroup") or "Loopable", "chrome": "main"}) + "sourceGroup": _native_group(e.get("sourceGroup")), "chrome": "main"}) except Exception: # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. # @@ -771,8 +783,8 @@ def nav(session: Session = Depends(require_session)): _folder = _odoo_folder_label(key, _conn_labels, _conn_default) if _folder: page["sourceGroup"] = _folder - elif page.get("user_table") or source in {"", "Blank", "Custom"}: - page.setdefault("sourceGroup", "Loopable") + elif page.get("user_table") or source.strip().lower() in _NATIVE_GROUP_TOKENS: + page.setdefault("sourceGroup", CUSTOM_SOURCE_GROUP) # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. # # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS @@ -1498,17 +1510,19 @@ def create_nav_application(body: dict = Body(default=None), wizard_kinds = {row["kind"] for row in application_registry.template_contracts(session=session)} if template_kind not in wizard_kinds: raise err(400, "bad_application", "application template kind is not recognized") - source = str(payload.get("sourceDatabase") or "").strip() - if not source: + import app_templates + source = str(payload.get("sourceDatabase") or app_templates.source(template_kind) or "").strip() + sourceless = template_kind in app_templates.BY_KIND and not app_templates.BY_KIND[template_kind]["sources"] + if not source and not sourceless: raise err(400, "bad_application", "sourceDatabase is required") - fields = application_registry.chart_field_contracts(session, [source])[source] + fields = application_registry.chart_field_contracts(session, [source])[source] if source else [] if not session.runtime.available(): raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.") try: application = application_registry.create( session.runtime, session.tenant, session.uname, payload, - field_contracts={source: fields if isinstance(fields, list) else []}, - allowed_databases={source}) + field_contracts={source: fields if isinstance(fields, list) else []} if source else {}, + allowed_databases={source} if source else set()) except PermissionError as exc: raise err(403, "forbidden", str(exc)) except application_registry.ApplicationExistsError as exc: diff --git a/api/routes_odoo_attachments.py b/api/routes_odoo_attachments.py index a5a57121c22be7a812c693255d83997341fb932e..9b3fd3ecf83a0aecfb197705676dbeb49adec937 100644 --- a/api/routes_odoo_attachments.py +++ b/api/routes_odoo_attachments.py @@ -1,28 +1,62 @@ """Contract C5 (S78-T14): the delivery photo list and the photo bytes, behind the delivery read wall. The work lives in `odoo_attachments.py`. This router decides who may ask and shapes the response. +Both routes admit the cookie or an external bearer, walled to the key's database scope (D-9926). Mounted by `main.py` through `_mount_router("routes_odoo_attachments")`. """ from fastapi import APIRouter, Depends, Path from fastapi.responses import Response import odoo_attachments -from deps import Session, require_session +from deps import Session, is_bearer, require_session_or_bearer router = APIRouter(prefix="/api/v1") _MAX_ID = 2 ** 31 - 1 +def _key_allowed(session): + """The delivery key filter for a bearer: its database scope. None for a cookie session.""" + if not is_bearer(session): + return None + from routes_external import _assert_database_scope + + def allowed(key): + _assert_database_scope(session, key) + return allowed + + +def _bearer_photo_wall(session, attachment_id, key_allowed): + """Check a bearer's database scope on the photo's delivery before `read_photo` reads bytes.""" + rows = odoo_attachments._search_read(session, odoo_attachments.ATTACHMENT_MODEL, + [("id", "=", int(attachment_id))], ["res_model", "res_id"]) + row = (rows or [None])[0] + res_id = row.get("res_id") if row else None + # A malformed attachment falls through to read_photo, which answers its own 404. + if (row and row.get("res_model") == odoo_attachments.PICKING_MODEL + and type(res_id) is int and res_id > 0): + odoo_attachments.require_delivery_read(session, res_id, key_allowed) + + @router.get("/odoo/deliveries/{picking_id}/photos") -def delivery_photos(picking_id: int = Path(..., ge=1, le=_MAX_ID), session: Session = Depends(require_session)): +def delivery_photos(picking_id: int = Path(..., ge=1, le=_MAX_ID), + session: Session = Depends(require_session_or_bearer)): """`[{id, name, mimetype, url}]`: every photo attachment of one delivery.""" - return odoo_attachments.list_photos(session, picking_id) + key_allowed = _key_allowed(session) + if key_allowed is None: + return odoo_attachments.list_photos(session, picking_id) + from loopy_links import public_base + return odoo_attachments.list_photos(session, picking_id, key_allowed=key_allowed, + base=public_base() or "") @router.get("/odoo/attachments/{attachment_id}") -def delivery_photo(attachment_id: int = Path(..., ge=1, le=_MAX_ID), session: Session = Depends(require_session)): +def delivery_photo(attachment_id: int = Path(..., ge=1, le=_MAX_ID), + session: Session = Depends(require_session_or_bearer)): """The photo bytes, streamed through and never stored.""" + key_allowed = _key_allowed(session) + if key_allowed is not None: + _bearer_photo_wall(session, attachment_id, key_allowed) body, mimetype, _name = odoo_attachments.read_photo(session, attachment_id) return Response(content=body, media_type=mimetype, headers={ "Cache-Control": "private, max-age=300", diff --git a/api/routes_opportunity_finder.py b/api/routes_opportunity_finder.py new file mode 100644 index 0000000000000000000000000000000000000000..6794171074f15dd21dd0b7e3b4198ce3002bf717 --- /dev/null +++ b/api/routes_opportunity_finder.py @@ -0,0 +1,159 @@ +"""Opportunity finder App routes (C3): rising terms, one item, status, a term's trend, settings, validate.""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1/app-kinds/opportunity_finder", tags=["opportunity_finder"]) +SOURCE = "ut_amzn_brand_analytics" +TEMPLATE_KIND = "opportunity_finder" +#: Weeks in a list row's sparkline. Each week is one store read per /screen call. +SPARK_WEEKS = 8 +SETTING_NAMES = {"min_rise_pct": "min_rise_pct", "minRisePct": "min_rise_pct", "departments": "departments"} + + +def _module(): + from modules import opportunity_finder + return opportunity_finder + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need read access to the Amazon brand analytics database.""" + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "opportunity_finder_forbidden", "Your account cannot read Amazon brand analytics.") + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _user(session): + return {**session.user, "username": session.uname} + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.") from exc + except KeyError as exc: + raise err(404, "opportunity_finder_not_found", "That search term is not in the opportunity finder.") from exc + except ValueError as exc: + raise err(400, "opportunity_finder_invalid", str(exc)) from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "opportunity_finder_unavailable", + "The opportunity finder could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Opportunity finder App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "opportunity_finder_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "opportunity_finder_app_not_found", "That Opportunity finder App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session: + """C3: a read names the App, and the caller must be able to read that App.""" + _app(session, app) + return session + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session: + """C3: a write names the App, and only an editor of that App may change it.""" + if not _may_edit(session, _app(session, app)): + raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.") + return session + + +def _with_spark(of, st, items): + """Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first.""" + weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else [] + held = [(week, of.week_point(None, week, st=st).get("terms") or {}) for week in weeks] + for item in items: + term = of.norm(item.get("term")) + item["spark"] = [{"week": week, "rank": terms[term]} for week, terms in held if term in terms] + return items + + +@router.get("/screen") +def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None), + cursor: str | None = Query(None), session: Session = Depends(_app_reader)): + """The rising list for the newest week, with counts, departments, settings and any named cause.""" + of = _module() + record = _app(session, app) + st = session.runtime + listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st) + listed["items"] = _call(_with_spark, of, st, listed["items"]) + return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"], + "weeks": listed["weeks"], "departments": listed["departments"], "counts": listed["counts"], + "total": listed["total"], "items": listed["items"], "cursor": listed["cursor"], + "problems": listed["problems"], "statuses": list(of.STATUSES), + "settings": _call(of.settings, _tenant(session), st=st)} + + +@router.get("/items/{item_id}") +def item_detail(item_id: str, session: Session = Depends(_app_reader)): + of = _module() + return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime) + + +@router.patch("/items/{item_id}") +def update_item(item_id: str, status: str = Body(...), note: str | None = Body(None), + owner: str | None = Body(None), due: str | None = Body(None), + session: Session = Depends(_app_writer)): + of = _module() + return _call(of.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner, + due=due, st=session.runtime) + + +@router.get("/terms/{item_id}") +def term_trend(item_id: str, session: Session = Depends(_app_reader)): + """A term's weekly rank trend and its newest top 3 clicked ASINs.""" + of = _module() + return _call(of.term_detail, _tenant(session), item_id, st=session.runtime) + + +@router.get("/settings") +def read_settings(session: Session = Depends(_app_reader)): + of = _module() + return _call(of.settings, _tenant(session), st=session.runtime) + + +@router.put("/settings") +def write_settings(body: dict = Body(...), session: Session = Depends(_app_writer)): + of = _module() + body = dict(body or {}) + owner = body.pop("defaultOwner", None) + unknown = sorted(k for k in body if k not in SETTING_NAMES) + if unknown: + raise err(400, "opportunity_finder_invalid", "Unknown setting: " + ", ".join(unknown) + ".") + values = {SETTING_NAMES[k]: v for k, v in body.items()} + return _call(of.set_settings, _tenant(session), _user(session), default_owner=owner, st=session.runtime, + **values) + + +@router.get("/validate") +def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None), + session: Session = Depends(_app_reader)): + of = _module() + return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime) diff --git a/api/routes_payouts.py b/api/routes_payouts.py new file mode 100644 index 0000000000000000000000000000000000000000..db44b7632035f0a4a27826d6e0cf71b3df8ade55 --- /dev/null +++ b/api/routes_payouts.py @@ -0,0 +1,228 @@ +"""Payouts App routes (C3): the overview, settlements with their waterfall and tie, owned items and settings. + +Every route names the App (`?app=`) and checks the caller may read it and the settlements database. +Writes also need edit rights on that App and a role other than viewer. `main.py` does not mount this +router: `app_routes.mount_app_routers` does, from the App contract. +""" +from fastapi import APIRouter, Body, Depends, HTTPException, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1/app-kinds/payouts", tags=["payouts"]) +SOURCE = "ut_amzn_settlements" +TEMPLATE_KIND = "payouts" + + +def _module(): + from modules import payouts + return payouts + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need read access to the Amazon settlements database.""" + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "payouts_forbidden", "Your account cannot read Amazon settlements.") + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "payouts_read_only", "Your account can view Payouts but not change it.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _app(session, app_id): + """The Payouts App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "payouts_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "payouts_app_not_found", "That Payouts App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session: + """A read names the App the caller can read (404 first, so no App is confirmed), then the database.""" + _app(session, app) + return _reader(session) + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session: + """A write names the App, and only an editor of that App may change it.""" + found = _app(session, app) + _writer(_reader(session)) + if not _may_edit(session, found): + raise err(403, "payouts_read_only", "You can view this App but not change it.") + return session + + +def _call(fn, *args, missing="payouts_not_found", missing_message="That settlement does not exist.", **kwargs): + pay = _module() + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "payouts_read_only", str(exc)) from exc + except pay.SourceUnavailable as exc: + raise err(503, "payouts_no_source", str(exc)) from exc + except ValueError as exc: + raise err(400, "payouts_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, missing, missing_message) from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "payouts_unavailable", "Payouts could not be read. Try again shortly.") from exc + + +@router.get("/overview") +def read_overview(session: Session = Depends(_app_reader)): + pay = _module() + return _call(pay.overview, _tenant(session), st=session.runtime) + + +@router.get("/settlements") +def list_settlements(cursor: str | None = Query(None), limit: int = Query(200, ge=1), + currency: str | None = Query(None), start: str | None = Query(None), + end: str | None = Query(None), session: Session = Depends(_app_reader)): + pay = _module() + return _call(pay.settlements, _tenant(session), st=session.runtime, currency=currency, start=start, end=end, + cursor=cursor, limit=limit) + + +@router.get("/settlements/{settlement_id}") +def read_settlement(settlement_id: str, session: Session = Depends(_app_reader)): + pay = _module() + return _call(pay.settlement, _tenant(session), settlement_id, st=session.runtime) + + +@router.get("/items") +def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None), + cursor: str | None = Query(None), limit: int = Query(200, ge=1), + session: Session = Depends(_app_reader)): + pay = _module() + listed = _call(pay.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit, + st=session.runtime) + return {**listed, "statuses": list(pay.STATUSES), "canEdit": _may_edit(session, _app(session, app))} + + +@router.patch("/items/{item_id}") +def update_item(item_id: str, status: str = Body(..., embed=True), note: str | None = Body(None, embed=True), + session: Session = Depends(_app_writer)): + pay = _module() + # History records the account name, never a display name. + actor = {**session.user, "username": session.uname} + return _call(pay.set_status, _tenant(session), actor, item_id, status, note=note, st=session.runtime, + missing="payouts_item_not_found", missing_message="That item does not exist.") + + +# --------------------------------------------------------------------------- screen (S79-T27) + +#: The module's refusal for a workspace with no stored settlements; the screen reads it as empty. +EMPTY_SOURCE = "No Amazon settlements are stored" +SCREEN_PAGE = 50 +LIST_FIELDS = ("id", "title", "currency", "start", "end", "depositDate", "amazonTotal", "linesTotal", "gap", + "reconciled") + + +def _empty(exc): + detail = getattr(exc, "detail", None) + error = detail.get("error") if isinstance(detail, dict) else None + return (isinstance(error, dict) and error.get("code") == "payouts_no_source" + and str(error.get("message") or "").startswith(EMPTY_SOURCE)) + + +def _offset(cursor): + try: + offset = int(cursor or 0) + except (TypeError, ValueError): + offset = -1 + if offset < 0: + raise err(400, "payouts_invalid", "The cursor is not valid. Start again from the first page.") + return offset + + +def _last_deposits(rows, as_of): + """The newest deposit on or before `as_of` and its change against the one before in its currency.""" + paid = sorted((r for r in rows if r.get("depositDate") and r["depositDate"] <= as_of + and r.get("amazonTotal") is not None), key=lambda r: (r["depositDate"], r["id"]), reverse=True) + if not paid: + return None + last = paid[0] + prev = next((r for r in paid[1:] if r["currency"] == last["currency"]), None) + return {"settlementId": last["id"], "date": last["depositDate"], "amount": last["amazonTotal"], + "currency": last["currency"], "previousAmount": None if prev is None else prev["amazonTotal"], + "previousDate": None if prev is None else prev["depositDate"], + "change": None if prev is None else round(last["amazonTotal"] - prev["amazonTotal"], 2)} + + +@router.get("/screen") +def read_screen(app: str = Query(..., min_length=1), currency: str | None = Query(None), + view: str = Query("all", pattern="^(all|unreconciled)$"), cursor: str | None = Query(None), + limit: int = Query(SCREEN_PAGE, ge=1), session: Session = Depends(_app_reader)): + """The App screen in one read: the KPI strip, tab counts and one page of settlements, newest first. No cap.""" + pay = _module() + offset = _offset(cursor) + tenant, st = _tenant(session), session.runtime + can_edit = _may_edit(session, _app(session, app)) + base = {"canEdit": can_edit, "statuses": list(pay.STATUSES), "view": view, "currency": currency or None} + try: + over = _call(pay.overview, tenant, st=st) + except HTTPException as exc: + if not _empty(exc): + raise + return {**base, "empty": True, "currencies": [], "counts": {"all": 0, "unreconciled": 0}, + "settlements": [], "cursor": None, "kpis": {}} + listed = _call(pay.settlements, tenant, st=st, currency=currency or None, limit=None)["settlements"] + counts = {"all": len(listed), "unreconciled": sum(1 for r in listed if not r["reconciled"])} + rows = listed if view == "all" else [r for r in listed if not r["reconciled"]] + page = rows[offset:offset + limit] + opened = over.get("openSettlements") or [] + open_now = next((o for o in opened if currency and o.get("currency") == currency.strip().upper()), + opened[0] if opened else None) + return {**base, "empty": not over.get("settlementCount"), "asOf": over.get("asOf"), + "currencies": over.get("currencies") or [], "economicsCurrency": over.get("economicsCurrency"), + "counts": counts, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page], + "cursor": str(offset + limit) if offset + limit < len(rows) else None, + "kpis": {"nextDeposit": over.get("nextDeposit"), "openSettlement": open_now, + "lastDeposit": _last_deposits(listed, over.get("asOf") or ""), + "unreconciled": counts["unreconciled"], "settlements": counts["all"]}} + + +@router.get("/screen/settlements/{settlement_id}") +def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1), + session: Session = Depends(_app_reader)): + """One settlement for the drawer: its waterfall, the tie line and, when unreconciled, its owned item.""" + pay = _module() + tenant, st = _tenant(session), session.runtime + can_edit = _may_edit(session, _app(session, app)) + detail = _call(pay.settlement, tenant, settlement_id, st=st) + item = None + if not detail.get("reconciled"): + held = _call(pay.items, tenant, session.user, st=st, limit=None)["items"] + item = next((i for i in held if str(i.get("id")) == str(settlement_id)), None) + return {"canEdit": can_edit, "statuses": list(pay.STATUSES), "settlement": detail, "item": item} + + +@router.get("/settings") +def read_settings(session: Session = Depends(_app_reader)): + pay = _module() + return _call(pay.settings, _tenant(session), st=session.runtime) + + +@router.patch("/settings") +def write_settings(defaultOwner: str | None = Body(None, embed=True), session: Session = Depends(_app_writer)): + pay = _module() + return _call(pay.set_settings, _tenant(session), default_owner=defaultOwner, st=session.runtime) diff --git a/api/routes_query.py b/api/routes_query.py index a6eacc3446ee24a25ac60c87457bc3b90755f645..49fd366a241d6294b1a19ef145e723b00df43cce 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -1324,7 +1324,9 @@ def _request_model(messages, tools, requested, session, chosen=None, *, before_a turn_audit.requested(turn.get("handle"), active["name"], active["model"]) try: response = provider_stream.post(request, provider=active, post=requests.post) + prov.llm_breaker_record(active["name"], response.status_code) except Exception: + prov.llm_breaker_record(active["name"], 0) if priced: after_attempt(ticket, None) raise diff --git a/api/routes_ranking_diagnostic.py b/api/routes_ranking_diagnostic.py new file mode 100644 index 0000000000000000000000000000000000000000..361372d9671e1fb0edc7dfb53f7ceec4630aae54 --- /dev/null +++ b/api/routes_ranking_diagnostic.py @@ -0,0 +1,158 @@ +"""Ranking diagnostic App routes (C3): drop items, one item's cause card, status, ASIN picker, trend, settings, validate.""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1/app-kinds/ranking_diagnostic", tags=["ranking_diagnostic"]) +SOURCE = "ut_amzn_pricing" +TEMPLATE_KIND = "ranking_diagnostic" + + +def _module(): + from modules import ranking_diagnostic + return ranking_diagnostic + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need read access to the Amazon Sales and Traffic database.""" + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "ranking_diagnostic_forbidden", "Your account cannot read Amazon sales and traffic.") + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _user(session): + return {**session.user, "username": session.uname} + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.") from exc + except ValueError as exc: + raise err(400, "ranking_diagnostic_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, "ranking_diagnostic_not_found", "That item or ASIN is not in the history.") from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "ranking_diagnostic_unavailable", + "The ranking diagnostic could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Ranking diagnostic App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "ranking_diagnostic_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "ranking_diagnostic_app_not_found", "That Ranking diagnostic App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session: + """C3: a read names the App, and the caller must be able to read that App.""" + _app(session, app) + return session + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session: + """C3: a write names the App, and only an editor of that App may change it.""" + if not _may_edit(session, _app(session, app)): + raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.") + return session + + +@router.get("/screen") +def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None), + week: str | None = Query(None), cursor: str | None = Query(None), + session: Session = Depends(_app_reader)): + """The week's drop list (the newest week by default), with counts, problems and settings.""" + rd = _module() + record = _app(session, app) + st = session.runtime + held = _call(rd.weeks, _tenant(session), st=st) + listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, + week=week or (held[-1] if held else None), st=st) + return {"canEdit": _may_edit(session, record), "week": listed["week"], "weeks": listed["weeks"], + "counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"], + "problems": listed["problems"], "statuses": list(rd.STATUSES), + "settings": _call(rd.settings, _tenant(session), st=st)} + + +@router.get("/items/{item_id}") +def item_detail(item_id: str, session: Session = Depends(_app_reader)): + rd = _module() + return _call(rd.item_detail, _tenant(session), session.uname, item_id, st=session.runtime) + + +@router.patch("/items/{item_id}") +def update_item(item_id: str, status: str = Body(...), note: str | None = Body(None), + owner: str | None = Body(None), due: str | None = Body(None), + session: Session = Depends(_app_writer)): + rd = _module() + return _call(rd.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner, + due=due, st=session.runtime) + + +@router.get("/asins") +def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None), + session: Session = Depends(_app_reader)): + """The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card.""" + rd = _module() + page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, st=session.runtime) + labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)} + for row in page["asins"]: + flagged = bool(row.get("cause") and page["week"]) + row["causeLabel"] = labels.get(row["cause"]) if flagged else None + row["itemId"] = rd.item_id(page["week"], row["asin"]) if flagged else None + return page + + +@router.get("/trend/{asin}") +def asin_trend(asin: str, session: Session = Depends(_app_reader)): + rd = _module() + return _call(rd.trend, _tenant(session), asin, st=session.runtime) + + +@router.get("/settings") +def read_settings(session: Session = Depends(_app_reader)): + rd = _module() + return _call(rd.settings, _tenant(session), st=session.runtime) + + +@router.put("/settings") +def write_settings(body: dict = Body(...), session: Session = Depends(_app_writer)): + rd = _module() + body = dict(body or {}) + owner = body.pop("defaultOwner", None) + unknown = sorted(k for k in body if k not in rd.DEFAULTS) + if unknown: + raise err(400, "ranking_diagnostic_invalid", "Unknown setting: " + ", ".join(unknown) + ".") + return _call(rd.set_settings, _tenant(session), _user(session), default_owner=owner, st=session.runtime, **body) + + +@router.get("/validate") +def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None), + session: Session = Depends(_app_reader)): + rd = _module() + return _call(rd.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime) diff --git a/api/routes_reimbursements.py b/api/routes_reimbursements.py new file mode 100644 index 0000000000000000000000000000000000000000..dd84d7f2805d71a2520528b783b5d6f5de7eafc8 --- /dev/null +++ b/api/routes_reimbursements.py @@ -0,0 +1,118 @@ +"""Reimbursements App routes (C3): claim items, one item with its case text, status, paid rows, validate.""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1/app-kinds/reimbursements", tags=["reimbursements"]) +SOURCE = "ut_amzn_reimbursements" +TEMPLATE_KIND = "reimbursements" + + +def _module(): + from modules import reimbursements + return reimbursements + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need read access to the Amazon reimbursements database.""" + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "reimbursements_forbidden", "Your account cannot read Amazon reimbursements.") + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "reimbursements_read_only", "Your account can view reimbursements but not change them.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _user(session): + return {**session.user, "username": session.uname} + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "reimbursements_read_only", "You can view this App but not change it.") from exc + except ValueError as exc: + raise err(400, "reimbursements_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, "reimbursements_item_not_found", "That claim does not exist.") from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "reimbursements_unavailable", "Reimbursements could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Reimbursements App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "reimbursements_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "reimbursements_app_not_found", "That Reimbursements App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session: + """C4: a read names the App, and the caller must be able to read that App.""" + _app(session, app) + return session + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session: + """C4: a write names the App, and only an editor of that App may change it.""" + if not _may_edit(session, _app(session, app)): + raise err(403, "reimbursements_read_only", "You can view this App but not change it.") + return session + + +@router.get("/screen") +def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None), + cursor: str | None = Query(None), session: Session = Depends(_app_reader)): + rb = _module() + record = _app(session, app) + listed = _call(rb.items, _tenant(session), session.uname, status=status, cursor=cursor, st=session.runtime) + return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"], + "cursor": listed["cursor"], "problems": listed["problems"], "statuses": list(rb.STATUSES)} + + +@router.get("/items/{item_id}") +def item_detail(item_id: str, session: Session = Depends(_app_reader)): + rb = _module() + return _call(rb.item_detail, _tenant(session), session.uname, item_id, st=session.runtime) + + +@router.patch("/items/{item_id}") +def update_item(item_id: str, status: str = Body(...), caseId: str | None = Body(None), + note: str | None = Body(None), session: Session = Depends(_app_writer)): + rb = _module() + return _call(rb.set_status, _tenant(session), _user(session), item_id, status, note=note, case_id=caseId, + st=session.runtime) + + +@router.get("/paid") +def paid_rows(cursor: str | None = Query(None), session: Session = Depends(_app_reader)): + rb = _module() + return _call(rb.paid, _tenant(session), session.uname, cursor=cursor, st=session.runtime) + + +@router.get("/validate") +def validate(start: str | None = Query(None), end: str | None = Query(None), + session: Session = Depends(_app_reader)): + rb = _module() + return _call(rb.validate, _tenant(session), start=start, end=end, st=session.runtime) diff --git a/api/routes_restock_planner.py b/api/routes_restock_planner.py new file mode 100644 index 0000000000000000000000000000000000000000..02415e8c9f3b49e8b62d6118d3657493607c5219 --- /dev/null +++ b/api/routes_restock_planner.py @@ -0,0 +1,156 @@ +"""Restock planner App routes (C3): items, one SKU, settings, refresh and the rule 8 tie.""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +TEMPLATE_KIND = "restock_planner" +router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["restock-planner"]) +SOURCE = "ut_amzn_inventory_planning" + + +def _module(): + from modules import restock_planner + return restock_planner + + +def _reader(session: Session = Depends(require_session)) -> Session: + """Reads need read access to the inventory planning database.""" + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "restock_planner_forbidden", "Your account cannot read Amazon inventory planning.") + return session + + +def _writer(session: Session = Depends(_reader)) -> Session: + """Writes also need a role that may edit.""" + if str(session.user.get("role") or "user").strip().lower() == "viewer": + raise err(403, "restock_planner_read_only", "You can view this App but not change it.") + return session + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "restock_planner_read_only", "You can view this App but not change it.") from exc + except ValueError as exc: + raise err(400, "restock_planner_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, "restock_planner_item_not_found", "That restock item does not exist.") from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "restock_planner_unavailable", "The restock plan could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Restock planner App record this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "restock_planner_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "restock_planner_app_not_found", "That Restock planner App does not exist.") + return app + + +def _may_edit(session, app): + import application_registry + role = str(session.user.get("role") or "user").strip().lower() + return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session: + """A read names the App, and the caller must be able to read that App.""" + _app(session, app) + return session + + +def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session: + """A write names the App, and only an editor of that App may change it.""" + if not _may_edit(session, _app(session, app)): + raise err(403, "restock_planner_read_only", "You can view this App but not change it.") + return session + + +def owners(session): + """People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name.""" + rp = _module() + try: + users = session.runtime.get("users") + except Exception: # noqa: BLE001 + users = None + want = _tenant(session).strip().lower() + names = set() + for name, rec in (users.items() if isinstance(users, dict) else ()): + if (not isinstance(rec, dict) or not rec.get("active", True) or rp._is_service(name, rec) + or str(rec.get("role") or "").strip().lower() == "viewer" + or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)): + continue + names.add(str(rec.get("name") or name).strip()) + return sorted((n for n in names if n), key=str.lower) + + +def thresholds(session): + """The two band thresholds in days of supply, read from the App settings (R49).""" + conf = _call(_module().settings, _tenant(session), st=session.runtime) + return {"sendToFbaDays": conf["sendToFbaDays"], "overstockDays": conf["overstockDays"]} + + +@router.get("/items") +def list_items(app: str = Query(..., min_length=1), band: str | None = Query(None), + status: str | None = Query(None), cursor: str | None = Query(None), + limit: int = Query(200, ge=1), q: str | None = Query(None), + session: Session = Depends(_app_reader)): + rp = _module() + listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, + band=band, q=q, st=session.runtime) + return {**listed, "owners": owners(session), "thresholds": thresholds(session), + "canEdit": _may_edit(session, _app(session, app))} + + +@router.get("/items/{item_id}") +def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): + rp = _module() + detail = _call(rp.item_detail, _tenant(session), item_id, st=session.runtime) + return {**detail, "thresholds": thresholds(session), "canEdit": _may_edit(session, _app(session, app))} + + +@router.patch("/items/{item_id}") +def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None), + due: str | None = Body(None), note: str | None = Body(None), + session: Session = Depends(_app_writer)): + rp = _module() + return _call(rp.update_item, _tenant(session), item_id, owner=owner, status=status, due=due, note=note, + user=session.uname or "unknown", st=session.runtime) + + +@router.get("/settings") +def read_settings(session: Session = Depends(_app_reader)): + rp = _module() + return _call(rp.settings, _tenant(session), st=session.runtime) + + +@router.patch("/settings") +def write_settings(sendToFbaDays: float | None = Body(None), overstockDays: float | None = Body(None), + dueDays: int | None = Body(None), defaultOwner: str | None = Body(None), + session: Session = Depends(_app_writer)): + rp = _module() + return _call(rp.set_settings, _tenant(session), send_to_fba_days=sendToFbaDays, overstock_days=overstockDays, + due_days=dueDays, default_owner=defaultOwner, st=session.runtime) + + +@router.post("/refresh") +def refresh(session: Session = Depends(_app_writer)): + rp = _module() + return _call(rp.refresh, _tenant(session), st=session.runtime) + + +@router.get("/validate") +def validate(session: Session = Depends(_app_reader)): + rp = _module() + return _call(rp.validate, _tenant(session), st=session.runtime) diff --git a/api/routes_returns_repeat.py b/api/routes_returns_repeat.py new file mode 100644 index 0000000000000000000000000000000000000000..9c82c4f39bb3bef67b5b8c298d3fc1a264e7707e --- /dev/null +++ b/api/routes_returns_repeat.py @@ -0,0 +1,159 @@ +"""Returns and repeat App routes (C3): summary, SKU segments, SKU detail, owned items and settings. + +Every endpoint names the App in its path. A caller who cannot read that App gets 404. Item writes +need a non-viewer account that may read FBA returns. Settings writes need an editor of the App. +""" +from fastapi import APIRouter, Body, Depends, Path, Query + +from deps import Session, err, require_session + +TEMPLATE_KIND = "returns_repeat" +router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["returns-repeat"]) +SOURCE = "ut_amzn_fba_customer_returns" + + +def _module(): + from modules import returns_repeat + return returns_repeat + + +def _tenant(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except PermissionError as exc: + raise err(403, "returns_repeat_read_only", str(exc)) from exc + except ValueError as exc: + raise err(400, "returns_repeat_invalid", str(exc)) from exc + except KeyError as exc: + raise err(404, "returns_repeat_not_found", "That SKU or item does not exist.") from exc + except Exception as exc: # noqa: BLE001 + raise err(503, "returns_repeat_unavailable", "Returns and repeat could not be read. Try again shortly.") from exc + + +def _app(session, app_id): + """The Returns and repeat App this request names, readable by the caller (404 otherwise).""" + import application_registry + try: + app = application_registry.load(session.runtime)["applications"].get(str(app_id or "")) + except Exception as exc: # noqa: BLE001 + raise err(503, "returns_repeat_unavailable", "The App could not be read. Try again shortly.") from exc + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True + or not application_registry.may_read(app, session.uname, is_admin=session.admin)): + raise err(404, "returns_repeat_app_not_found", "That Returns and repeat App does not exist.") + return app + + +def _viewer(session): + return str(session.user.get("role") or "user").strip().lower() == "viewer" + + +def _may_edit_app(session, app): + import application_registry + return not _viewer(session) and application_registry.may_edit(app, session.uname, is_admin=session.admin) + + +def _can_edit_items(session): + return _module().may_edit(session.runtime, session.user) + + +def _app_reader(app_id: str = Path(..., min_length=1), session: Session = Depends(require_session)) -> Session: + """A read names the App, and the caller must read that App and the FBA returns database.""" + _app(session, app_id) + from core import perm_scope + if not perm_scope.may_read(session.user, SOURCE, st=session.runtime): + raise err(403, "returns_repeat_forbidden", "Your account cannot read Amazon FBA returns.") + return session + + +def _item_writer(session: Session = Depends(_app_reader)) -> Session: + """An item write also refuses a viewer.""" + if _viewer(session) or not _can_edit_items(session): + raise err(403, "returns_repeat_read_only", "You can view this App but not change its items.") + return session + + +def _settings_writer(app_id: str = Path(..., min_length=1), session: Session = Depends(_app_reader)) -> Session: + """A settings write needs an editor of the App.""" + if not _may_edit_app(session, _app(session, app_id)): + raise err(403, "returns_repeat_read_only", "You can view this App but not change its settings.") + return session + + +@router.get("/{app_id}/summary") +def read_summary(start: str | None = Query(None), end: str | None = Query(None), + session: Session = Depends(_app_reader)): + rr = _module() + out = _call(rr.summary, _tenant(session), start=start, end=end, st=session.runtime) + return {**out, "canEdit": _can_edit_items(session) and not _viewer(session)} + + +@router.get("/{app_id}/skus") +def list_skus(segment: str = Query("highest"), start: str | None = Query(None), end: str | None = Query(None), + cursor: str | None = Query(None), limit: int = Query(200, ge=1), q: str | None = Query(None), + session: Session = Depends(_app_reader)): + rr = _module() + return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor, + limit=limit, q=q, st=session.runtime) + + +@router.get("/{app_id}/sku") +def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(None), + end: str | None = Query(None), session: Session = Depends(_app_reader)): + rr = _module() + return _call(rr.detail, _tenant(session), key, start=start, end=end, st=session.runtime) + + +def owners(session): + """People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name.""" + rr = _module() + try: + users = session.runtime.get("users") + except Exception: # noqa: BLE001 + users = None + want = _tenant(session).strip().lower() + names = set() + for name, rec in (users.items() if isinstance(users, dict) else ()): + if (not isinstance(rec, dict) or not rec.get("active", True) or rr._is_service(name, rec) + or str(rec.get("role") or "").strip().lower() == "viewer" + or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)): + continue + names.add(str(rec.get("name") or name).strip()) + return sorted((n for n in names if n), key=str.lower) + + +@router.get("/{app_id}/items") +def list_items(status: str | None = Query(None), cursor: str | None = Query(None), + limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)): + rr = _module() + listed = _call(rr.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit, + st=session.runtime) + return {**listed, "statuses": list(rr.STATUSES), "owners": owners(session), + "canEdit": _can_edit_items(session) and not _viewer(session)} + + +@router.patch("/{app_id}/items/{item_id}") +def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None), + due: str | None = Body(None), note: str | None = Body(None), + session: Session = Depends(_item_writer)): + rr = _module() + return _call(rr.update_item, _tenant(session), session.user, item_id, owner=owner, status=status, + due=due, note=note, st=session.runtime) + + +@router.get("/{app_id}/settings") +def read_settings(app_id: str, session: Session = Depends(_app_reader)): + rr = _module() + out = _call(rr.settings, _tenant(session), st=session.runtime) + return {**out, "canEdit": _may_edit_app(session, _app(session, app_id))} + + +@router.put("/{app_id}/settings") +def write_settings(risingRatio: float | None = Body(None), risingMinPct: float | None = Body(None), + defaultOwner: str | None = Body(None), session: Session = Depends(_settings_writer)): + rr = _module() + return _call(rr.set_settings, _tenant(session), rising_ratio=risingRatio, rising_min_pct=risingMinPct, + default_owner=defaultOwner, st=session.runtime) diff --git a/api/routes_shares.py b/api/routes_shares.py index 0d823d8c60193f260f0bc8cc847cd589462e623b..66d8c7bbd81f9940b4ed808d69da3d1aaea3ca05 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1090,6 +1090,97 @@ def my_shares(session: Session = Depends(require_session)): return shares.shared_with(session.uname, st=session.runtime) +def _name_resolver(): + """`username -> display name` from ONE registry read; falls back to the username.""" + try: + registry = users.registry() or {} + except Exception: # noqa: BLE001 + registry = {} + + def name(username): + who = str(username or "").strip() + row = registry.get(who) if isinstance(registry, dict) else None + return (str(row.get("name") or "").strip() if isinstance(row, dict) else "") or who + return name + + +def _audience_mark(owner, audience, everyone, viewer): + """The mark an owner-plus-audience object gives `viewer`, or None. Same rule as `shares.mark_from`.""" + owner = str(owner or "").strip() + named = [u for u in dict.fromkeys(str(u or "").strip() for u in audience or ()) if u] + if owner and owner == viewer: + others = [u for u in named if u != viewer] + if not others and not everyone: + return None + return {"direction": "by_me", "count": len(others), "everyone": bool(everyone)} + if everyone or viewer in named: + return {"direction": "with_me", "by": owner} + return None + + +def _app_marks(session): + """App marks from the App registry's own record: `createdBy` plus its `share`.""" + try: + import application_registry as _apps + rows = (_apps.load(session.runtime).get("applications") or {}).values() + except Exception: # noqa: BLE001 + return {} + out = {} + for app in rows: + if not isinstance(app, dict) or app.get("retired") is True or not app.get("id"): + continue + share = app.get("share") if isinstance(app.get("share"), dict) else {} + mark = _audience_mark(app.get("createdBy"), share.get("users"), + share.get("mode") == "tenant", session.uname) + if mark: + out[f"app:{app['id']}"] = mark + return out + + +def _agent_marks(session): + """Agent marks from each Agent contract's `sharing`, read once.""" + try: + import automation_engine as _engine + contracts = _engine._agent_contracts(session.runtime) + except Exception: # noqa: BLE001 + return {} + out = {} + for aid, rec in (contracts or {}).items(): + if not isinstance(rec, dict): + continue + sharing = rec.get("sharing") if isinstance(rec.get("sharing"), dict) else {} + audience = sharing.get("sharedWith") if sharing.get("visibility") == "shared" else () + mark = _audience_mark(rec.get("createdBy"), audience, False, session.uname) + if mark: + out[f"agent:{aid}"] = mark + return out + + +@router.get("/share/marks") +def get_share_marks(session: Session = Depends(require_session)): + """`{marks: {":": mark}}`: every shared icon a module home draws, in ONE request. + + A mark exists only for the owner (`by_me`, with `count`) or a grantee (`with_me`, with `by`), + so no viewer learns of a share they are not part of. View keys are bare view ids, because the + grant registry stores them bare. An unreadable grant bucket answers 503 and the client draws + no icon. App and Agent marks drop out alone when their own record cannot be read. + """ + held = shares.share_marks(session.uname, st=session.runtime) + if held is None: + raise err(503, "marks_unavailable", "Sharing could not be read. Reload to try again.") + name = _name_resolver() + marks = {} + for kind, rows in held.items(): + for oid, mark in rows.items(): + marks[f"{kind}:{oid}"] = mark + marks.update(_app_marks(session)) + marks.update(_agent_marks(session)) + for mark in marks.values(): + if mark["direction"] == "with_me": + mark["by"] = name(mark.get("by")) if mark.get("by") else "" + return {"marks": marks} + + @router.get("/share/diagnostics/{reference}") def get_view_share_diagnostic(reference: str, session: Session = Depends(require_session)): """Read one support reference from this tenant's ring, after administrator admission.""" @@ -1240,6 +1331,18 @@ def _people(tenant): return sorted(out, key=lambda p: p["name"].lower()) +def _stranger(session, kind, oid): + """True when the caller holds no role and cannot reach the object: `get_share`'s 404 test.""" + admin = bool(session.admin) and kind != "cohort" + if kind == "field": + table_key, field_key = shares.split_field_oid(oid) + role = _field_permissions.field_role_sources( + table_key, field_key, session.uname, is_admin=admin, st=session.runtime)["direct"] + else: + role = shares.role_for(kind, oid, session.uname, is_admin=admin, st=session.runtime) + return role is None and not _can_see_object(session, kind, oid) + + @router.put("/share/{kind}/{oid}") def put_share(kind: str, oid: str, body: dict = Body(default=None), session: Session = Depends(require_session)): @@ -1277,6 +1380,9 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N # both answerable (*"every editable field has an owner"*), so the `field` arm prints # them. ⛔ THE CODE WORD `not_owner` IS UNCHANGED — clients match on it — and the # OTHER THREE KINDS KEEP THIS SENTENCE BYTE FOR BYTE, because their gates assert it. + # D-476: a caller with no relationship gets GET's 404, so this door confirms no id. + if _stranger(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") if kind == "field": table_key, field_key = shares.split_field_oid(oid) raise err(403, "not_owner", _field_refusal(session, table_key, field_key)) diff --git a/api/routes_sheet.py b/api/routes_sheet.py index f173fb848cc694ad95c4ae7dd26d267325763137..51ed98de050d6b2c975953bcc3773a164b42ed3f 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -3774,6 +3774,26 @@ REFRESH_KEEP_SECONDS = 3600 _REFRESH_FAILED = {"code": "sheet_unresolved", "message": "This workbook could not be refreshed."} _REFRESH_TIMEOUT = {"code": "sheet_refresh_timeout", "message": "This refresh took too long. Try again in a few minutes."} +#: Seconds the supervisor waits past a run's own deadline before it fails the row for the desk. +REFRESH_GRACE_SECONDS = 5.0 + + +def _refresh_budget_env(): + """`AIOS_SHEET_REFRESH_BUDGET_SECONDS`, else 300. Held below half of REFRESH_STALE_SECONDS.""" + try: + value = float(os.environ.get("AIOS_SHEET_REFRESH_BUDGET_SECONDS") or 300) + except ValueError: + value = 300.0 + return min(value if value > 0 else 300.0, REFRESH_STALE_SECONDS / 2) + + +#: The total budget of one desk run, resolver included. It stays below REFRESH_STALE_SECONDS, so +#: a requeue of another process's row never meets an executor that is still inside its budget. +REFRESH_BUDGET_SECONDS = _refresh_budget_env() +#: This process. A claimed row names its runner, so a reader can tell whose desk holds it. +_REFRESH_RUNNER = uuid.uuid4().hex +#: Claim token -> {"tenant", "store", "id", "deadline"} for every run this process executes now. +_REFRESH_LIVE = {} def _refresh_desk_queues(session): @@ -3857,36 +3877,78 @@ def _refresh_desk_enqueue(store, tenant, workbook, reader, args, now=None, path= return made["row"] +def _refresh_desk_overdue(row, now): + """"fail", "requeue" or None for one row. A row this process still executes is never requeued.""" + if row.get("state") != "running": + return None + live = _REFRESH_LIVE.get(row.get("claim")) if row.get("claim") else None + if live is not None: + return "fail" if now >= float(live["deadline"]) + REFRESH_GRACE_SECONDS else None + if now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS: + return "requeue" if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS else "fail" + return None + + +def _refresh_desk_settle(rows, now, by): + """Requeue or fail every overdue row in place. Returns [(request id, new state)].""" + changed = [] + for row in rows.values(): + verdict = _refresh_desk_overdue(row, now) + if verdict == "requeue": + # A dropped claim token stops the old executor's late finish from landing on a new run. + row.update(state="queued", claim=None, requeuedBy=by) + elif verdict == "fail": + row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now), + error=dict(_REFRESH_TIMEOUT), finishedBy=by) + if verdict: + changed.append((row["id"], row["state"])) + return changed + + +def _refresh_desk_reap(store, now=None, by="supervisor"): + """The stale step on its own: settle every overdue running row. An idle call writes nothing.""" + now = _desk_time.time() if now is None else now + rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now) + if not any(_refresh_desk_overdue(row, now) for row in rows.values()): + return [] + changed = [] + + def apply(current): + rows = _refresh_desk_rows(current, now) + changed[:] = _refresh_desk_settle(rows, now, by) + return {"requests": rows} + + store.update(REFRESH_REQUESTS_KEY, apply, flush="sync") + return changed + + def _refresh_desk_claim(store, now=None): - """Mark the oldest queued request running and return it. None while one is already running.""" + """Settle overdue rows, then mark the oldest queued request running and return it. + + None while one is already running. The claimed row carries a fresh `claim` token and this + process's `runner` id. Only that token can finish it. + """ now = _desk_time.time() if now is None else now claimed = {} - def stale(row): - return row.get("state") == "running" and now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS - # Read first: an idle poll must never write the store. rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now).values() - busy = any(row.get("state") == "running" and not stale(row) for row in rows) - if not any(stale(row) for row in rows) and (busy or not any(row.get("state") == "queued" for row in rows)): + overdue = {row.get("id") for row in rows if _refresh_desk_overdue(row, now)} + busy = any(row.get("state") == "running" and row.get("id") not in overdue for row in rows) + if not overdue and (busy or not any(row.get("state") == "queued" for row in rows)): return None def apply(current): rows = _refresh_desk_rows(current, now) - for row in rows.values(): - if stale(row): - if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS: - row["state"] = "queued" - else: - row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now), - error=dict(_REFRESH_TIMEOUT)) + _refresh_desk_settle(rows, now, "claim") if not any(row.get("state") == "running" for row in rows.values()): queued = sorted((row for row in rows.values() if row.get("state") == "queued"), key=_refresh_desk_order) if queued: row = queued[0] row.update(state="running", startedTs=now, startedAt=_desk_iso(now), - attempts=int(row.get("attempts") or 0) + 1) + attempts=int(row.get("attempts") or 0) + 1, + claim=uuid.uuid4().hex, runner=_REFRESH_RUNNER) claimed["row"] = dict(row) return {"requests": rows} @@ -3894,19 +3956,38 @@ def _refresh_desk_claim(store, now=None): return claimed.get("row") -def _refresh_desk_finish(store, request_id, error=None, now=None): - """Mark one running request done, or failed with a public `{code, message}`.""" +def _refresh_desk_owns(row, claim): + """True when `claim` is the row's current token. A caller without a token skips the check.""" + return claim is None or row.get("claim") == claim + + +def _refresh_desk_finish(store, request_id, error=None, now=None, claim=None): + """Mark one running request done, or failed with a public `{code, message}`. + + Returns False and writes nothing when the row is no longer running under `claim`. + """ now = _desk_time.time() if now is None else now + done = {"ok": False} + + def mine(rows): + row = rows.get(request_id) + return row if row is not None and row.get("state") == "running" and _refresh_desk_owns(row, claim) else None + + if mine(_refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now)) is None: + return False def apply(current): rows = _refresh_desk_rows(current, now) - row = rows.get(request_id) - if row is not None and row.get("state") == "running": + row = mine(rows) + if row is not None: row.update(state="failed" if error else "done", finishedTs=now, - finishedAt=_desk_iso(now), error=dict(error) if error else None) + finishedAt=_desk_iso(now), error=dict(error) if error else None, + finishedBy="desk") + done["ok"] = True return {"requests": rows} store.update(REFRESH_REQUESTS_KEY, apply, flush="sync") + return done["ok"] def _refresh_desk_entry(store, workbook_id, reader, request_id): @@ -6478,13 +6559,16 @@ def view_rows(workbook_id: str, body: object = Body(default=None), _DRILL_MIRRORS = OrderedDict() +# The registry guard: held only for lookups and bookkeeping, never across a build or a yield. _DRILL_MIRROR_LOCK = threading.RLock() _DRILL_MIRROR_MAX = 6 +# One lease per cache key: a mirror cursor holds connection-scoped temp tables and serves one thread. +_DRILL_LEASES = {} @contextlib.contextmanager def _drill_mirror_lock(): - """Include shared-cursor contention in the caller's absolute request budget.""" + """Include registry-guard contention in the caller's absolute request budget.""" from core import request_timing timeout = request_timing.remaining_budget() acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None @@ -6501,6 +6585,38 @@ def _drill_mirror_lock(): _DRILL_MIRROR_LOCK.release() +@contextlib.contextmanager +def _drill_lease(key): + """Serialize one cache key's cursor. Other keys never wait, and a waiter spends its budget.""" + from core import request_timing + if key is None: + # An uncacheable drill owns a private cursor, so it shares nothing to serialize. + yield {"entry": None} + return + with _drill_mirror_lock(): + lease = _DRILL_LEASES.get(key) + if lease is None: + lease = _DRILL_LEASES[key] = {"cond": threading.Condition(threading.Lock()), + "busy": False, "refs": 0, "entry": None} + lease["refs"] += 1 + try: + with lease["cond"]: + while lease["busy"]: + lease["cond"].wait(request_timing.remaining_budget()) + lease["busy"] = True + try: + yield lease + finally: + with lease["cond"]: + lease["busy"] = False + lease["cond"].notify_all() + finally: + with _DRILL_MIRROR_LOCK: + lease["refs"] -= 1 + if not lease["refs"] and _DRILL_LEASES.get(key) is lease: + _DRILL_LEASES.pop(key) + + def _drill_tokens(session, databases): """Unknown freshness disables reuse. No workspace documents are retained here.""" from core import sheet_binding @@ -6519,7 +6635,7 @@ def _drill_tokens(session, databases): @contextlib.contextmanager def _drill_mirror(session, database, window, needed, databases): - """A small reader-scoped LRU of query mirrors, with serialized cursor use and eviction.""" + """A small reader-scoped LRU of query mirrors: one lease per key, short registry guard.""" from core import sheet_binding, perm_scope from core.request_timing import raise_if_resource_error import datetime @@ -6552,8 +6668,10 @@ def _drill_mirror(session, database, window, needed, databases): "_sheet_drill_entries") absent = object() previous = {name: getattr(session, name, absent) for name in attrs} - with _drill_mirror_lock(): - hit = _DRILL_MIRRORS.get(key) if token else None + with _drill_lease(key if token else None) as lease: + with _drill_mirror_lock(): + hit = _DRILL_MIRRORS.get(key) if token else None + lease["entry"] = hit session._sheet_drill_isolated = True session._sheet_drill_cursor = hit["owned"] if hit else None session._sheet_drill_entries = hit["mirrors"] if hit else {} @@ -6565,21 +6683,30 @@ def _drill_mirror(session, database, window, needed, databases): want = set(needed) | (hit["needed"] if hit else set()) if hit and want <= hit["needed"]: made = hit["made"] - _DRILL_MIRRORS.move_to_end(key) + with _DRILL_MIRROR_LOCK: + if _DRILL_MIRRORS.get(key) is hit: + _DRILL_MIRRORS.move_to_end(key) else: - _DRILL_MIRRORS.pop(key, None) + with _DRILL_MIRROR_LOCK: + if hit is not None and _DRILL_MIRRORS.get(key) is hit: + _DRILL_MIRRORS.pop(key) + # The build runs under this key's lease only, so other keys build at once. made = _mirror(session)(database, metric_window=window, needed=want) owned = getattr(session, "_sheet_drill_cursor", None) hit = {"made": made, "needed": want, "owned": owned, "mirrors": session._sheet_drill_entries} # A build racing a source change is usable for this request, never reusable. if token and _drill_tokens(session, databases) == sources: - _DRILL_MIRRORS[key] = hit - _DRILL_MIRRORS.move_to_end(key) - while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX: - _, evicted = _DRILL_MIRRORS.popitem(last=False) - if evicted["owned"]: - evicted["owned"][0].close() + with _DRILL_MIRROR_LOCK: + _DRILL_MIRRORS[key] = hit + _DRILL_MIRRORS.move_to_end(key) + lease["entry"] = hit + while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX: + gone, evicted = _DRILL_MIRRORS.popitem(last=False) + # A leased cursor closes when its holder leaves, never under it. + held = _DRILL_LEASES.get(gone) + if evicted["owned"] and not (held and held["entry"] is evicted): + evicted["owned"][0].close() else: token = None # Related leaves may revisit the primary registry. Keep its table intact and widen @@ -6589,8 +6716,16 @@ def _drill_mirror(session, database, window, needed, databases): yield made, token finally: owned = getattr(session, "_sheet_drill_cursor", None) - if owned and not any(item["owned"] is owned for item in _DRILL_MIRRORS.values()): - owned[0].close() + entry = lease["entry"] + with _DRILL_MIRROR_LOCK: + lease["entry"] = None + kept = [item["owned"] for item in _DRILL_MIRRORS.values()] + # The last lease on an uncached or evicted cursor closes it. + spent = [owned, entry["owned"] if entry else None] + for index, cursor in enumerate(spent): + if (cursor and not any(seen is cursor for seen in spent[:index]) + and not any(item is cursor for item in kept)): + cursor[0].close() for name, value in previous.items(): if value is absent: session.__dict__.pop(name, None) diff --git a/api/routes_tables.py b/api/routes_tables.py index 7847b6f542fd8b2b48c538e0ca60220fa0e06fd3..f1032605434890491cdfc7dfd2816dd29c0d2074 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -3767,8 +3767,7 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` # asks whether THIS write landed, and this write goes to the definition. - stored = dict(((_ut().get_one(table_key, st=session.runtime) or {}).get("rows") or {}) - .get(str(pid)) or {}) + stored = dict(_ut().get_record(table_key, pid, st=session.runtime) or {}) accepted = {k: stored.get(k) for k in updates if k in stored} def _took(k): diff --git a/api/routes_templates.py b/api/routes_templates.py index df2d7c99539ad36f21bc347f41047bd3557ed708..79e58eb751f6dbed0bda40e772bbecb9ebf747f3 100644 --- a/api/routes_templates.py +++ b/api/routes_templates.py @@ -24,6 +24,7 @@ columns listed. The alternative — writing the views anyway — is the `_seed_w verbatim: `clean_filter_tree` DROPS a leaf on an unknown column, and a view whose only condition was dropped shows EVERY row under a name that promises a shortlist. """ +import app_templates import dash_norm import uuid @@ -38,6 +39,10 @@ _TEMPLATE_CATEGORIES = { "sheet": "Sheet", "data": "Table", "app": "App", "agents": "Automation", "saved": "Saved", "route": "Operations", "catalog": "Sales", "competitor_watch": "Sales", } +# C1 connector keys for a shipped Sheet descriptor's `source` (F25: Amazon is `amazon_sp` there). +_SHEET_CONNECTORS = {"odoo": "odoo", "amazon_sp": "amazon"} +# C1 fields each App row carries so the client can pick its screen and marks. +_APP_WIRE_KEYS = ("view", "mark", "connector", "process", "purpose") _TEMPLATE_PREVIEW_LABELS = {**_TEMPLATE_CATEGORIES, "route": "Routing", "catalog": "Catalog", "competitor_watch": "Competitor watch"} @@ -395,16 +400,21 @@ def _own_template_apps(session): return {} +def _app_address(app_id, source): + """The hash an App opens at. An App that reads no database names none.""" + from urllib.parse import quote + database = "database=" + quote(str(source), safe="") + "&" if source else "" + return "#/app?" + database + "application=" + quote(str(app_id), safe="") + + def _existing_app(held): """{id, name, address} of an owned App, where address is the hash the picker navigates to.""" - from urllib.parse import quote if not isinstance(held, dict) or not held.get("id"): return None if held.get("deepLink"): address = "#" + str(held["deepLink"]) else: - address = ("#/app?database=" + quote(str(held.get("sourceDatabase") or ""), safe="") - + "&application=" + quote(str(held["id"]), safe="")) + address = _app_address(held["id"], held.get("sourceDatabase")) return {"id": held["id"], "name": dash_norm.plain(str(held.get("name") or "")), "address": address} @@ -485,11 +495,19 @@ def _module_templates(session): database_states = _database_states(session, definitions, databases) requirement = lambda database, connector="": _requirement( database, definitions, have, database_states, connector) - sheet = [_catalog_row(row["key"], row["title"], row["answers"], - [requirement(row["database_hint"], row["source"])]) - for row in sheet_templates.TEMPLATES] - sheet.extend(_catalog_row(row["key"], row["name"], row["description"], - [requirement(db) for db in row["databases"]], module="saved") + sheet = [] + for row in sheet_templates.TEMPLATES: + # QI20: a shipped Sheet template is bundled with its connector and offered only with it. + connector = _SHEET_CONNECTORS.get(row["source"], row["source"]) + if app_templates.entry_type(connector) not in have: + continue + sheet.append({**_catalog_row(row["key"], row["title"], row["answers"], + [requirement(row["database_hint"], row["source"])]), + "connector": connector}) + # R34: a user-saved template keeps its databases and names no connector. + sheet.extend({**_catalog_row(row["key"], row["name"], row["description"], + [requirement(db) for db in row["databases"]], module="saved"), + "databases": list(row["databases"])} for row in saved) data = [] for row, database in data_templates: @@ -497,8 +515,11 @@ def _module_templates(session): connector = "odoo" if row["source"].startswith("odoo_") else row["source"] data.append(_catalog_row(row["key"], row["label"], row["desc"], [requirement(database, connector)], module="data")) - # These App contracts map fields in the existing wizard and require no fixed connector. - app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [], module="app") + # These App contracts map fields in the existing wizard. A connector-gated kind is listed + # only when the registry's offer predicate holds, so it needs no requirement row here. + # S79-T05: an App card reads its C1 purpose. R33 removed the database and field steps. + app = [{**_catalog_row(row["kind"], row["label"], row.get("purpose") or row["label"], [], module="app"), + **{key: row.get(key) for key in _APP_WIRE_KEYS}} for row in application_templates(session=session)["templates"]] # Owned copies are optional links. They never replace the template's create action. owned = _own_template_apps(session) @@ -585,6 +606,10 @@ def apply_sheet_template(key: str, body: dict = Body(default=None), created_by=session.uname) # Owners read their persisted values. Use the same permission-scoped # refresh as Sheet so the first open already contains computed cells. + import routes_sheet + if routes_sheet._refresh_desk_queues(session): + routes_sheet.refresh_sheet_door(book["id"], body={}, session=session) + return {"workbook": book} refreshed = refresh_sheet(book["id"], session=session)["workbook"] failures = [] for tab in refreshed["sheets"]: @@ -616,13 +641,14 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None session: Session = Depends(require_session)): from routes_nav import create_nav_application from urllib.parse import quote + import application_registry body = body if isinstance(body, dict) else {} if module == "agents": # A direct call to create_automation does not run its FastAPI dependency. session.require("automation") - if module == "app" and key == "competitor_watch": - pass # S78-T17: gated by application_registry.template_contracts(session=...). + if module == "app" and key in application_registry.GATED_TEMPLATE_KINDS: + pass # C1: gated by application_registry.template_contracts(session=...). elif module == "app" and key not in {"route", "catalog"}: raise err(404, "unknown_template", "That template does not exist.") section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None) @@ -647,7 +673,36 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None return {"address": "#/automation?agent=" + quote(result["automation"]["id"], safe=""), **result} # The existing wizard still owns its POST-then-PATCH share sequence. API clients may # dispatch creation here; the existing App writer retains schema and permission walls. - result = create_nav_application({**body, "templateKind": key}, session=session) + # S79-T05: the App directory's one-click Add sends no body, so the C1 name and source fill in. + defaults = {"name": template["name"]} + if app_templates.source(key): + defaults["sourceDatabase"] = app_templates.source(key) + result = create_nav_application({**defaults, **body, "templateKind": key}, session=session) + else: + result = _create_sourceless_app({**defaults, **body, "templateKind": key}, session) app = result["application"] - return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""), - **result} + return {"address": _app_address(app["id"], app.get("sourceDatabase")), **result} + + +def _create_sourceless_app(payload, session): + """S79-T05: create an App whose C1 kind reads no database, behind the nav writer's walls. + + `create_nav_application` requires a sourceDatabase, so this door calls the registry itself. + """ + import application_registry + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.") + try: + application = application_registry.create( + session.runtime, session.tenant, session.uname, payload, + field_contracts={}, allowed_databases=set()) + except PermissionError as exc: + raise err(403, "forbidden", str(exc)) + except application_registry.ApplicationRuleError as exc: + raise err(400, exc.code, str(exc)) + except application_registry.ApplicationError as exc: + raise err(400, "bad_application", str(exc)) + except Exception: + raise err(503, "store_unavailable", "the Application was not saved.") + return {"application": application} diff --git a/api/sheet_warm.py b/api/sheet_warm.py index d653e6fcda7a90debf784f7380167277aa778ddb..de81a2ca5790b67b2155882c6dd093f35ec19dd1 100644 --- a/api/sheet_warm.py +++ b/api/sheet_warm.py @@ -19,7 +19,7 @@ from datetime import datetime, time as dtime, timedelta, timezone from time import perf_counter import routes_sheet -from core import external_access, perms, refresh_proof, shares, sheets, users +from core import external_access, perms, refresh_proof, request_timing, shares, sheets, users from deps import Session #: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader. @@ -335,7 +335,12 @@ def _on_exit(): # ── the refresh desk: person refreshes the web role queued, run here one at a time ────────────── #: Seconds between two reads of each tenant's request document. DESK_POLL_SECONDS = 5.0 -_DESK = {"thread": None, "seen": {}} +#: Seconds between two supervisor checks of the desk and the runs it holds. +SUPERVISOR_POLL_SECONDS = 5.0 +#: Seconds between two supervisor reads of every tenant, for rows another process left running. +SUPERVISOR_SWEEP_SECONDS = 60.0 +#: generation: the desk loop that may claim. beat: monotonic time the desk last showed progress. +_DESK = {"thread": None, "seen": {}, "generation": 0, "beat": None, "supervisor": None, "swept": None} def _desk_tenants(): @@ -499,37 +504,65 @@ def _desk_result_log(row, tenant, session, before, result, *, admitted=None): pass -def desk_run_one(rt, tenant): - """Claim and run the oldest queued refresh of one tenant. Returns True when one ran.""" +def desk_run_one(rt, tenant, generation=None): + """Claim and run the oldest queued refresh of one tenant. Returns True when one ran. + + The run holds REFRESH_BUDGET_SECONDS in total. A desk of a replaced generation claims nothing. + """ + if generation is not None and generation != _DESK["generation"]: + return False + _DESK["beat"] = time.monotonic() row = routes_sheet._refresh_desk_claim(rt) if row is None: return False started = perf_counter() + budget = float(routes_sheet.REFRESH_BUDGET_SECONDS) + deadline = time.monotonic() + budget + claim = row.get("claim") + if claim: + # The supervisor reads this to fail a run that overran its deadline, never to requeue it. + routes_sheet._REFRESH_LIVE[claim] = {"tenant": tenant, "store": rt, "id": row["id"], + "deadline": time.time() + budget} error = None book, session, result, admitted = None, None, None, None try: - book = sheets.get_workbook(rt, row["workbook"]) - rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt) - session = reader_session(rt, tenant, row.get("reader"), book, rec) - if session is None: - error = {"code": "workbook_missing", "message": "That workbook no longer exists."} - else: - admitted = _desk_input_identity(book) - result = routes_sheet._refresh_desk_run(session, row) - except Exception as exc: # noqa: BLE001 - error = routes_sheet._refresh_desk_error(exc) - _desk_result_log(row, tenant, session, book, result, admitted=admitted) - routes_sheet._refresh_desk_finish(rt, row["id"], error) + try: + with request_timing.budget_scope(deadline): + book = sheets.get_workbook(rt, row["workbook"]) + rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt) + session = reader_session(rt, tenant, row.get("reader"), book, rec) + if session is None: + error = {"code": "workbook_missing", "message": "That workbook no longer exists."} + else: + admitted = _desk_input_identity(book) + result = routes_sheet._refresh_desk_run(session, row) + except Exception as exc: # noqa: BLE001 + timed_out = (isinstance(exc, request_timing.RequestDeadlineExceeded) + or time.monotonic() >= deadline) + error = dict(routes_sheet._REFRESH_TIMEOUT) if timed_out else routes_sheet._refresh_desk_error(exc) + _desk_result_log(row, tenant, session, book, result, admitted=admitted) + owned = routes_sheet._refresh_desk_finish(rt, row["id"], error, claim=claim) + finally: + if claim: + routes_sheet._REFRESH_LIVE.pop(claim, None) print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} " f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}" - + (f" code={error['code']}" if error else ""), flush=True) + + (f" code={error['code']}" if error else "") + + (" late=1" if owned is False else ""), flush=True) return True -def desk_drain(tenant): - """Run every queued refresh of one tenant, oldest first. Returns how many ran.""" +def _desk_runtime(tenant): from harness import runtime - rt = runtime.get_runtime(tenant) + return runtime.get_runtime(tenant) + + +def desk_drain(tenant, generation=None): + """Run every queued refresh of one tenant, oldest first. Returns how many ran.""" + if generation is not None and generation != _DESK["generation"]: + return 0 + _DESK["beat"] = time.monotonic() + rt = _desk_runtime(tenant) if _desk_skip(rt, tenant): return 0 # The revision is taken BEFORE the reads, so a request queued between them still moves it. @@ -538,7 +571,7 @@ def desk_drain(tenant): except Exception: # noqa: BLE001 before = None ran = 0 - while not _STOP.is_set() and desk_run_one(rt, tenant): + while not _STOP.is_set() and desk_run_one(rt, tenant, generation): ran += 1 if not ran and before is not None and not routes_sheet._refresh_desk_pending(rt): _DESK["seen"][tenant] = before @@ -547,32 +580,99 @@ def desk_drain(tenant): return ran -def _desk_loop(): - while not _STOP.is_set(): +def _desk_loop(generation=None): + """One desk generation. It exits once the supervisor starts a newer one.""" + generation = _DESK["generation"] if generation is None else generation + while not _STOP.is_set() and _DESK["generation"] == generation: + _DESK["beat"] = time.monotonic() try: tenants = _desk_tenants() except Exception as exc: # noqa: BLE001 print(f"[sheet-desk] tenant list failed {type(exc).__name__}: {exc}", flush=True) tenants = [] for tenant in tenants: - if _STOP.is_set(): + if _STOP.is_set() or _DESK["generation"] != generation: return try: - desk_drain(tenant) + desk_drain(tenant, generation) except Exception as exc: # noqa: BLE001 print(f"[sheet-desk] tenant={tenant} failed {type(exc).__name__}: {exc}", flush=True) _STOP.wait(DESK_POLL_SECONDS) +def _desk_stuck(now): + """True when the desk thread died, or showed no progress for longer than one whole run.""" + thread = _DESK["thread"] + if thread is None: + return False + if not thread.is_alive(): + return True + limit = (float(routes_sheet.REFRESH_BUDGET_SECONDS) + routes_sheet.REFRESH_GRACE_SECONDS + + DESK_POLL_SECONDS) + return _DESK["beat"] is not None and now - _DESK["beat"] > limit + + +def _desk_replace(reason): + """Start a desk of the next generation. The stuck one claims nothing once it returns.""" + import process_role + with _LOCK: + _DESK["generation"] += 1 + generation = _DESK["generation"] + _DESK["beat"] = time.monotonic() + _DESK["seen"].clear() + _DESK["thread"] = process_role.start("sheet-refresh-desk", "heavy", _desk_loop, args=(generation,)) + print(f"[sheet-desk] supervisor replaced the desk reason={reason} generation={generation}", flush=True) + return _DESK["thread"] + + +def supervise_once(): + """Fail runs past their deadline, settle rows another process left, and replace a stuck desk.""" + wall, mono = time.time(), time.monotonic() + stores = {} + for entry in list(routes_sheet._REFRESH_LIVE.values()): + if wall >= float(entry["deadline"]) + routes_sheet.REFRESH_GRACE_SECONDS: + stores[entry["tenant"]] = entry["store"] + # A read of every tenant at a slower pace finds a row whose runner process died. + if _DESK["swept"] is None or mono - _DESK["swept"] >= SUPERVISOR_SWEEP_SECONDS: + _DESK["swept"] = mono + try: + for tenant in _desk_tenants(): + if tenant not in stores: + stores[tenant] = _desk_runtime(tenant) + except Exception as exc: # noqa: BLE001 + print(f"[sheet-desk] supervisor tenant list failed {type(exc).__name__}: {exc}", flush=True) + for tenant, rt in stores.items(): + try: + for request_id, state in routes_sheet._refresh_desk_reap(rt, by="supervisor"): + print(f"[sheet-desk] supervisor tenant={tenant} request={request_id} state={state}" + f" code={routes_sheet._REFRESH_TIMEOUT['code']}", flush=True) + except Exception as exc: # noqa: BLE001 + print(f"[sheet-desk] supervisor tenant={tenant} failed {type(exc).__name__}: {exc}", flush=True) + if _desk_stuck(time.monotonic()): + _desk_replace("dead" if not _DESK["thread"].is_alive() else "no_progress") + + +def _supervise_loop(): + while not _STOP.is_set(): + try: + supervise_once() + except Exception as exc: # noqa: BLE001 + print(f"[sheet-desk] supervisor failed {type(exc).__name__}: {exc}", flush=True) + _STOP.wait(SUPERVISOR_POLL_SECONDS) + + def start_desk(): - """Start the one "sheet-refresh-desk" thread where heavy jobs run. Returns it, or None.""" + """Start the "sheet-refresh-desk" thread and its supervisor where heavy jobs run. Returns the desk.""" + import process_role with _LOCK: thread = _DESK["thread"] - if thread is not None and thread.is_alive(): - return thread - import process_role - thread = process_role.start("sheet-refresh-desk", "heavy", _desk_loop) - _DESK["thread"] = thread + if thread is None or not thread.is_alive(): + _DESK["beat"] = time.monotonic() + thread = process_role.start("sheet-refresh-desk", "heavy", _desk_loop, args=(_DESK["generation"],)) + _DESK["thread"] = thread + supervisor = _DESK["supervisor"] + if thread is not None and (supervisor is None or not supervisor.is_alive()): + _DESK["supervisor"] = process_role.start("sheet-refresh-supervisor", "heavy", _supervise_loop) return thread diff --git a/api/sync_worker.py b/api/sync_worker.py index 04cc4a37e3cc4519c0bf5ef741490c7b727e5394..bf0ae45dea935ed80e9c629e2dffcce1cbfe92b8 100644 --- a/api/sync_worker.py +++ b/api/sync_worker.py @@ -2,6 +2,7 @@ Run as `python sync_worker.py` from aios-web/api. The web process spawns and supervises it. It keeps its own DuckDB files, so it never opens the web process's royal.duckdb. +Its heavy jobs wait until it holds the store's sync writer lease, so two Spaces never both sync. """ import os import sys @@ -22,8 +23,12 @@ def configure(env=None): worker_db = worker_dir / web_db.name env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir) env["AIOS_DUCKDB_PATH"] = str(worker_db) + # The launcher (request_class.py --env) sizes these to the box. Unset: the 2 vCPU values. env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB" env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1" + # The worker holds its own file read-write. Mirror versions are for the web workers only. + env["AIOS_MIRROR_VERSIONS"] = "0" + env["AIOS_UVICORN_WORKERS"] = "1" return worker_db @@ -58,6 +63,9 @@ def main(): process_role.start_worker_services() # Importing the app starts every role-allowed thread at module level. Startup hooks need a call. import main as app_main # noqa: F401 + if getattr(app_main, "_SEED_THREAD", None) is None: + # No boot restore was scheduled, so no boot hand-off will come. Say so, or web readiness waits. + process_role.note_boot_restore("skipped") import routes_connectors routes_connectors._start_database_jobs() print(f"[sync-worker] running: {', '.join(sorted(process_role.started_jobs()))}", flush=True) diff --git a/api/test_odoo_rank_refresh.py b/api/test_odoo_rank_refresh.py index 7ffc44b731ca56c06aa69f38091061e669ee7a9a..3c9f282c8761a9a32d62aa1ac009326dd84dcbc4 100644 --- a/api/test_odoo_rank_refresh.py +++ b/api/test_odoo_rank_refresh.py @@ -198,7 +198,7 @@ class RankRefreshTests(unittest.TestCase): # every pass would ask for a zero wait. clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60, strftime=lambda *_: 'synthetic-time', gmtime=lambda: ()) - ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None, + ns = dict(_restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None, _configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF, _resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False, tenant='synthetic', tenantSource='test', tenantSeconds=60, @@ -520,7 +520,7 @@ class RankRefreshTests(unittest.TestCase): clock = SimpleNamespace(sleep=sleep, time=lambda: now, strftime=lambda *_: 'synthetic-time', gmtime=lambda: ()) - ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None, + ns = dict(_restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None, _configured_mirror_entities=lambda *_: None, freeze_notice=SimpleNamespace(enabled=lambda: frozen), _resolve_resync_cadence=lambda: dict( @@ -795,7 +795,7 @@ class RankRefreshTests(unittest.TestCase): store = self.SlotStore(tenant, {rel.CONFIG_KEY: dict(config or {})}) clock = SimpleNamespace(sleep=wait, time=lambda: now[0], strftime=lambda *_: 'synthetic-time', gmtime=lambda: ()) - ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None, + ns = dict(_restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None, _configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF, _resolve_resync_cadence=lambda: dict( resolvedSeconds=86400, manual=False, tenant=tenant, tenantSource='test', diff --git a/api/tickets.py b/api/tickets.py index 58496c8972e4f9d11d22e471f4b2bf6e5a253d79..27a55102e4c55d62111fdd6f8c84560c2d9c927a 100644 --- a/api/tickets.py +++ b/api/tickets.py @@ -345,7 +345,7 @@ def _application_source_readable(rt, principal, app): if apps._is_finance(app): key = apps._finance_permission(app) return bool(key and apps._account_wall(rt, principal)(key)) - return apps._database_free_code(app) or any( + return apps._database_free(app) or any( perm_scope.may_read(principal, database, st=rt) for database in app.get("compatibleDatabases") or ()) diff --git a/api/tool_groups/__init__.py b/api/tool_groups/__init__.py index 17c9d2322378751fe879462dc7732fea572ca4c3..181379ed743cbc47ba6086c4af612e7b57970b28 100644 --- a/api/tool_groups/__init__.py +++ b/api/tool_groups/__init__.py @@ -1,6 +1,7 @@ """Discover MCP groups and adapt the previous tool names.""" import copy import importlib +import json import pkgutil from core import external_tools, sheets @@ -113,6 +114,67 @@ def groups(): return found +def _kind(rule): + """Name the JSON type a schema rule accepts, in plain words.""" + if not isinstance(rule, dict): + return "value" + if isinstance(rule.get("enum"), list) and rule["enum"]: + return "one of " + ", ".join(str(choice) for choice in rule["enum"]) + kind = rule.get("type") + if isinstance(kind, list): + return " or ".join(str(item) for item in kind) + if isinstance(kind, str): + return kind + branches = rule.get("oneOf") or rule.get("anyOf") or [] + kinds = list(dict.fromkeys(_kind(branch) for branch in branches)) + return " or ".join(kinds) or "value" + + +def _example(key, rule): + """Return a placeholder value of the type a schema rule accepts.""" + if not isinstance(rule, dict): + return "<" + key + ">" + if isinstance(rule.get("enum"), list) and len(rule["enum"]) == 1: + return rule["enum"][0] + kind = rule.get("type") + if kind == "array": + return [_example(key, rule.get("items"))] + if kind == "object": + return {} + if kind in ("integer", "number"): + return max(1, rule.get("minimum", 1)) + if kind == "boolean": + return True + return "<" + key + ">" + + +def call_shape(group, action, schema, sent=None, example=None): + """Name the current address, the argument shape and any key the caller got wrong.""" + schema = schema if isinstance(schema, dict) else {} + properties = schema.get("properties", {}) + required = [key for key in schema.get("required", ()) if key in properties] + shown = example if isinstance(example, dict) else {key: _example(key, properties[key]) for key in required} + sentences = [] + if isinstance(sent, dict): + stray = sorted(str(key) for key in sent if key not in properties) + missing = [key for key in required if key not in sent] + if stray and schema.get("additionalProperties") is False: + sentences.append(", ".join(stray) + (" is not an argument of " if len(stray) == 1 + else " are not arguments of ") + action + ".") + if missing: + sentences.append(", ".join(missing) + (" is required." if len(missing) == 1 else " are required.")) + sentences.append("Call " + group + " with " + json.dumps( + {"action": action, "arguments": shown}, separators=(", ", ": ")) + ".") + optional = [key for key in properties if key not in required and key not in shown + and key not in {"action", "arguments"}] + if optional: + sentences.append("Optional arguments: " + ", ".join( + key + " (" + _kind(properties[key]) + ")" for key in optional) + ".") + elif not properties or set(properties) <= {"action", "arguments"}: + sentences.append(action + " takes no arguments.") + return " ".join(sentences) + + def resolve(name, arguments): """Validate the selected action before a preview or handler can run.""" group = groups().get(canonical_name(name)) if isinstance(name, str) else None @@ -126,15 +188,27 @@ def resolve(name, arguments): named + "Tool arguments must match the published input schema. Set action to one of the " + group["name"] + " actions: " + ", ".join(group["actions"]) + ". Put that action's inputs inside " "arguments, and intent, about and purpose beside action and arguments.")) - external_tools.validate_arguments(arguments, { - "type": "object", "properties": { - "action": {"type": "string", "enum": list(group["actions"])}, - "arguments": {"type": "object"}, - }, "required": ["action"], "additionalProperties": False, - }) - entry = group["actions"][arguments["action"]] - args = arguments.get("arguments", {}) - external_tools.validate_arguments(args, entry["schema"]) + entry = group["actions"][arguments["action"]] if isinstance(arguments, dict) else None + try: + external_tools.validate_arguments(arguments, { + "type": "object", "properties": { + "action": {"type": "string", "enum": list(group["actions"])}, + "arguments": {"type": "object"}, + }, "required": ["action"], "additionalProperties": False, + }) + args = arguments.get("arguments", {}) + external_tools.validate_arguments(args, entry["schema"]) + except sheets.SheetRefusal as exc: + if exc.code != "tool_arguments_invalid" or entry is None: + raise + # S79-T11: a schema refusal names the current tool, action and argument shape. + stray = sorted(str(key) for key in arguments if key not in {"action", "arguments"}) + beside = (", ".join(stray) + (" goes" if len(stray) == 1 else " go") + + " inside arguments, not beside action. ") if stray else "" + inner = arguments.get("arguments") + raise sheets.SheetRefusal(exc.code, exc.message + " " + beside + call_shape( + group["name"], arguments["action"], entry["schema"], + inner if isinstance(inner, dict) else {}, entry.get("example"))) from exc legacy = entry.get("legacy_name") if legacy and external_tools.is_write(legacy, args) == entry["read"]: raise sheets.SheetRefusal( diff --git a/api/tool_groups/apps.py b/api/tool_groups/apps.py index 55e673c37efb99fa5b458fbaf614c21df793262e..31002bca628f3953e5be5e07f8f3b37ee9d147f7 100644 --- a/api/tool_groups/apps.py +++ b/api/tool_groups/apps.py @@ -2,7 +2,12 @@ import base64 import copy import hashlib +import importlib +import importlib.util +import inspect import json +import re +import sys from core import external_tools, perm_scope, sheets import application_registry as applications @@ -60,6 +65,87 @@ def _catalog_link(app): return app +#: R35: a template App answers over tools with these keys only. Its definition stays sealed. +TEMPLATE_APP_KEYS = ("id", "name", "templateKind", "purpose", "status", "tools") +#: Tools a template App can use today when the C1 template catalogue is absent. +_FALLBACK_TOOLS = {"catalog": ("catalog_build", "catalog_pdf")} +TEMPLATE_SEALED_CODE = "template_app_sealed" + + +def is_template_app(app): + """A Loopable template App: its templateKind is set and it is not a code App.""" + return isinstance(app, dict) and bool(app.get("templateKind")) and app.get("type") != "code" + + +def _template_entry(kind): + """The C1 `app_templates.TEMPLATES` entry for a kind, or None while C1 is absent.""" + try: + import app_templates + except ImportError: + return None + rows = getattr(app_templates, "TEMPLATES", None) + if isinstance(rows, dict): + entry = rows.get(kind) + if isinstance(entry, dict): + return entry + rows = rows.values() + for entry in rows or (): + if isinstance(entry, dict) and entry.get("kind") == kind: + return entry + return None + + +def _template_purpose(kind, entry): + purpose = entry.get("purpose") if entry else None + if not isinstance(purpose, str) or not purpose.strip(): + contract = applications._template_contract(kind) or {} + purpose = str(contract.get("label") or kind or "").replace("_", " ") + purpose = purpose[:1].upper() + purpose[1:].lower() + return " ".join(purpose.split()) + + +def template_tools(kind): + """The tool names a template App answers to, from C1 or the actions it can use today.""" + entry = _template_entry(kind) + tools = entry.get("tools") if entry else None + if isinstance(tools, (list, tuple)): + return [str(tool) for tool in tools] + return list(_FALLBACK_TOOLS.get(kind, ())) + + +def template_public(app): + """R35: name, one line purpose, status and tool list. No definition, source binding or code.""" + kind = app.get("templateKind") + entry = _template_entry(kind) + return {"id": app.get("id"), "name": app.get("name"), "templateKind": kind, + "purpose": _template_purpose(kind, entry), "status": "active", "tools": template_tools(kind)} + + +def template_sealed_message(app): + """The fix sentence for a definition write on a template App.""" + tools = ", ".join("apps." + tool if "." not in tool else tool + for tool in template_tools(app.get("templateKind"))) + use = ("Use its own tools: " + tools + ". ") if tools else "" + return ("This App is a Loopable template. Its definition cannot be read or changed over tools. " + use + + "An Administrator can rename or share it in Loopable.") + + +def template_sealed(app): + """The named refusal for a definition write on a template App.""" + from deps import err + return err(403, TEMPLATE_SEALED_CODE, template_sealed_message(app)) + + +def _list_row(app): + if not is_template_app(app): + return _catalog_link(app) + row = template_public(app) + if app.get("updatedAt"): + # files.list reads the change time of each build. + row["updatedAt"] = app["updatedAt"] + return row + + def list_apps(ctx, args): session = _admit(ctx) from core import external_access @@ -72,7 +158,7 @@ def list_apps(ctx, args): if external_access.database_allowed(scope, database)} try: allowed, _ = _access(ctx, sorted(databases)) - return {"applications": [_catalog_link(app) for app in applications.list_apps( + return {"applications": [_list_row(app) for app in applications.list_apps( session.runtime, session.uname, session.admin, allowed_databases=allowed)]} except (applications.ApplicationError, PermissionError) as exc: _refuse(exc) @@ -129,6 +215,8 @@ def _read(ctx, app_id): def get(ctx, args): try: app, allowed = _read(ctx, args["app_id"]) + if is_template_app(app): + return {"application": template_public(app)} result = applications._public(app, ctx.session.uname, ctx.session.admin, allowed_databases=allowed) if app["type"] == "code": result["config"]["display"]["html"] = app["config"]["display"]["html"] @@ -137,6 +225,211 @@ def get(ctx, args): _refuse(exc) +#: C8 refusal codes. A caller is refused by name, never shown an empty list. +APP_UNAVAILABLE = "app_unavailable" +APP_ITEMS_UNAVAILABLE = "app_items_unavailable" +APP_ITEMS_FORBIDDEN = "app_items_forbidden" +APP_ITEM_STATUS_INVALID = "app_item_status_invalid" +APP_ITEM_UNKNOWN = "app_item_unknown" +_KIND = re.compile(r"[a-z][a-z0-9_]{0,63}") + + +def _refusal(status, code, message, details=None): + from deps import err + return err(status, code, message, details) + + +def _item_app(ctx, app_id, *, edit): + """C8: the App the caller may read (items) or edit (set_status). One refusal for absent and unshared.""" + try: + app, _ = _read(ctx, app_id) + except (applications.ApplicationError, PermissionError): + raise _refusal(404, APP_UNAVAILABLE, "No App with this id is shared with this account. " + "Call apps.list for the Apps this account can open.") from None + session = ctx.session + if edit and not applications.may_edit(app, session.uname, session.admin): + raise _refusal(403, APP_ITEMS_FORBIDDEN, "This account can read this App but cannot change its items. " + "Ask its owner or an Administrator to change them.") + return app + + +def _item_module(app): + """C8 dispatch: templateKind to the C1 `module`, else `modules.`.""" + kind = app.get("templateKind") if is_template_app(app) else None + entry = _template_entry(kind) if kind else None + name = entry.get("module") if entry else None + if not isinstance(name, str) or not name.strip(): + name = "modules." + str(kind or "") + module = None + if kind and _KIND.fullmatch(str(kind)) and re.fullmatch(r"modules\.[a-z][a-z0-9_]{0,63}", name): + try: + module = importlib.import_module(name) + except Exception: + module = None + if module is None or not callable(getattr(module, "items", None)) or not callable( + getattr(module, "set_status", None)): + raise _refusal(409, APP_ITEMS_UNAVAILABLE, "This App has no items to list or change over tools. " + "Open it in Loopable, or ask an Administrator to update it.") + return module + + +def _kind_modules(): + """Each template kind with the module name C8 dispatches it to: the C1 row, else `modules.`.""" + try: + import app_templates + rows = getattr(app_templates, "TEMPLATES", None) or {} + except ImportError: + rows = {} + entries = rows.values() if isinstance(rows, dict) else rows + named = {entry["kind"]: entry.get("module") for entry in entries + if isinstance(entry, dict) and isinstance(entry.get("kind"), str)} + for row in applications.template_contracts(): + named.setdefault(row["kind"], None) + return {kind: name if isinstance(name, str) and name.strip() else "modules." + kind + for kind, name in named.items() if _KIND.fullmatch(kind)} + + +def items_supported(): + """Capability report: some template kind resolves to a module with items and set_status.""" + for name in _kind_modules().values(): + if not re.fullmatch(r"modules\.[a-z][a-z0-9_]{0,63}", name): + continue + try: + # An absent module is skipped without an import attempt. A module already loaded is read as is. + if name not in sys.modules and importlib.util.find_spec(name) is None: + continue + module = importlib.import_module(name) + except Exception: + continue + if callable(getattr(module, "items", None)) and callable(getattr(module, "set_status", None)): + return True + return False + + +def _accepts_store(fn): + """True when a C4 function takes the `st=` store override, so a preview writes privately.""" + try: + params = inspect.signature(fn).parameters + except (TypeError, ValueError): + return False + # Only a named `st`: a **kwargs catch-all may ignore it and write the real store during a preview. + return "st" in params + + +def _tenant_key(session): + return str(getattr(session.runtime, "key", "") or session.tenant) + + +def _statuses(module): + values = getattr(module, "STATUSES", None) + return [str(value) for value in values] if isinstance(values, (list, tuple, set, frozenset)) else None + + +def _status_refusal(module, status): + allowed = _statuses(module) + tail = (" Use one of: " + ", ".join(allowed) + ".") if allowed else " Use a status this App lists on its items." + return _refusal(400, APP_ITEM_STATUS_INVALID, "This App does not allow the status " + json.dumps(str(status)) + + "." + tail, {"allowed": allowed} if allowed else None) + + +def _item_refusal(module, exc, status=None): + if isinstance(exc, PermissionError): + return _refusal(403, APP_ITEMS_FORBIDDEN, "This account cannot work this App's items. " + "Ask its owner or an Administrator for access.") + if isinstance(exc, KeyError): + return _refusal(404, APP_ITEM_UNKNOWN, "Choose item ids that apps.items returned for this App.") + return _status_refusal(module, status) + + +def items(ctx, args): + """C8 apps.items: the App's own items with status, owner and reason, paged by the module's cursor.""" + _admit(ctx) + app = _item_app(ctx, args["application"], edit=False) + module = _item_module(app) + session = ctx.session + kwargs = {key: args[key] for key in ("status", "cursor") if args.get(key) is not None} + if _accepts_store(module.items): + kwargs["st"] = session.runtime + try: + page = module.items(_tenant_key(session), session.uname, **kwargs) + except (PermissionError, ValueError) as exc: + raise _item_refusal(module, exc, args.get("status")) from None + page = page if isinstance(page, dict) else {} + # Rule 1: the module pages by cursor. Nothing here caps or trims its page. + return {"app_id": app["id"], "items": list(page.get("items") or []), "cursor": page.get("cursor")} + + +def _current_items(module, session, ids): + """Each requested id with its current status, read from the module's pages.""" + found, cursor, seen = {}, None, set() + while True: + kwargs = {"cursor": cursor} if cursor is not None else {} + if _accepts_store(module.items): + kwargs["st"] = session.runtime + page = module.items(_tenant_key(session), session.uname, **kwargs) or {} + for item in page.get("items") or (): + if isinstance(item, dict) and str(item.get("id")) in ids: + found[str(item["id"])] = item + cursor = page.get("cursor") + if not cursor or cursor in seen or len(found) == len(ids): + return found + seen.add(cursor) + + +def set_status(ctx, args): + """C8 apps.set_status: one preview for every id, applied only through confirm_change (R14 batch approve).""" + from routes_external import _ConfirmationRuntime, _PreviewRuntime + session = _admit(ctx, write=True) + app = _item_app(ctx, args["application"], edit=True) + module = _item_module(app) + ids = list(dict.fromkeys(str(value) for value in args["item_ids"])) + status, note = args["status"], args.get("note") + allowed = _statuses(module) + if allowed is not None and status not in allowed: + raise _status_refusal(module, status) + runtime = session.runtime + previewing = isinstance(runtime, _PreviewRuntime) + if not previewing and not isinstance(runtime, _ConfirmationRuntime): + raise sheets.SheetRefusal("confirmation_not_write", "Preview this change, then apply it with confirm_change.") + tenant = _tenant_key(session) + if _accepts_store(module.set_status): + # The module writes into the preview store, so the diff and the stale check see its own document. + changed = [] + for item_id in ids: + try: + changed.append(module.set_status(tenant, session.uname, item_id, status, note=note, st=runtime)) + except (PermissionError, KeyError, ValueError) as exc: + raise _item_refusal(module, exc, status) from None + return {"app_id": app["id"], "item_ids": ids, "status": status, "items": changed} + # The module writes its own store: a preview names the change and never calls set_status. + try: + current = _current_items(module, session, set(ids)) + except (PermissionError, ValueError) as exc: + raise _item_refusal(module, exc, status) from None + if len(current) != len(ids): + raise _refusal(404, APP_ITEM_UNKNOWN, "Choose item ids that apps.items returned for this App.", + {"unknown": [item_id for item_id in ids if item_id not in current][:100]}) + plan = {item_id: {"status": status, "from": current[item_id].get("status")} for item_id in ids} + if previewing: + runtime.update(_PLAN_KEY_PREFIX + str(app["id"]), lambda doc: {**(doc or {}), **plan}) + return {"app_id": app["id"], "item_ids": ids, "status": status} + if getattr(runtime, "preview_only", False) or getattr(runtime, "activity_preview", False): + # A staged confirm (the Assistant's YES hold) must not reach a store it cannot stage. + raise _refusal(409, APP_ITEMS_UNAVAILABLE, "This App applies item changes only through confirm_change. " + "Confirm the preview from the AI that made it.") + changed = [] + for item_id in ids: + try: + changed.append(module.set_status(tenant, session.uname, item_id, status, note=note)) + except (PermissionError, KeyError, ValueError) as exc: + raise _item_refusal(module, exc, status) from None + return {"app_id": app["id"], "item_ids": ids, "status": status, "items": changed} + + +#: A preview-only document naming a planned change for a module that writes its own store. +_PLAN_KEY_PREFIX = "app_item_changes/" + + def apply_template(ctx, args): session = _admit(ctx, write=True) try: @@ -145,7 +438,8 @@ def apply_template(ctx, args): raise applications.ApplicationError("Choose a template from apps.templates.") _template_inputs(payload) share = payload.pop("share", None) - source = payload.get("sourceDatabase") + import app_templates + source = payload.get("sourceDatabase") or app_templates.source(payload["templateKind"]) databases = [source] if source else [] allowed, records = _access(ctx, databases, strict=True) payload.setdefault("navPlacement", {"section": "app", "order": 0}) @@ -167,6 +461,9 @@ def apply_template(ctx, args): def _template_inputs(payload): """Name the missing template input and where it goes before the registry answers generically.""" contract = next(row for row in applications.template_contracts() if row["kind"] == payload["templateKind"]) + import app_templates + if payload["templateKind"] in app_templates.BY_KIND: + return required = [role["key"] for role in contract["fieldRoles"] if role["required"]] example = json.dumps({role: "" for role in required}) if not isinstance(payload.get("sourceDatabase"), str) or not payload["sourceDatabase"].strip(): @@ -202,6 +499,8 @@ def save_code(ctx, args): app_id = args.get("app_id") if app_id: current, allowed = _read(ctx, app_id) + if is_template_app(current): + raise template_sealed(current) if current["type"] != "code": raise applications.ApplicationError("Choose a code App to update.") proposed = payload.get("compatibleDatabases", current.get("compatibleDatabases") or []) @@ -248,9 +547,28 @@ def preview_documents(documents): htmlBytes=len(html.encode("utf-8")), htmlSha256=hashlib.sha256(html.encode("utf-8")).hexdigest(), firstLines="\n".join(html.split("\n")[:5])[:500]) + import catalog_store + pair = projected.get(catalog_store.STORE_KEY) + for side in ("before", "after") if pair else (): + doc = pair[side] if isinstance(pair[side], dict) else {} + # R35: a catalog preview names its template App, never its field map or source binding. + # Each digest keeps the confirmation bound to the definition the preview saw. + snapshots = doc.get("appSnapshots") if isinstance(doc.get("appSnapshots"), dict) else {} + for app_id, snapshot in snapshots.items(): + if is_template_app(snapshot): + snapshots[app_id] = {**{key: snapshot[key] for key in ("id", "templateKind") if key in snapshot}, + "definitionSha256": _digest(snapshot)} + catalogs = doc.get("catalogs") if isinstance(doc.get("catalogs"), dict) else {} + for record in catalogs.values(): + if isinstance(record, dict) and "fields" in record: + record["fieldsSha256"] = _digest(record.pop("fields")) return projected +def _digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode("utf-8")).hexdigest() + + def pending_arguments(arguments, *, encode): """Encode the pending code body for JSONB and restore it before confirmation.""" result = copy.deepcopy(arguments) @@ -299,7 +617,7 @@ def loopy_confirm(session, args): """Apply an agreed Assistant App preview through confirm_change.""" from routes_external import _dispatch_mcp, _read_confirmation_token payload = _read_confirmation_token(args.get("confirmation_token"), session=session) - if payload.get("tool") != "apps" or payload["arguments"].get("action") not in ("apply_template", "save_code", "catalog_build"): + if payload.get("tool") != "apps" or payload["arguments"].get("action") not in ("apply_template", "save_code", "catalog_build", "set_status"): raise sheets.SheetRefusal("confirmation_not_write", "Choose an App preview to confirm.") reply = _dispatch_mcp({"jsonrpc": "2.0", "id": "loopy-apps-confirm", "method": "tools/call", "params": {"name": "confirm_change", "arguments": args}}, session) @@ -352,10 +670,23 @@ GROUP = { "compatibleDatabases": {"type": "array", "items": _TEXT}, "share": _SHARE, }, ("name", "html"), read=False, description=BRIEF), "get": _action(get, {"app_id": _TEXT}, ("app_id",)), + "items": _action(items, {"application": _TEXT, "status": _TEXT, + "cursor": {"type": ["string", "integer"]}}, ("application",), + description="List an App's items with status, owner and reason. Pass cursor for the next page."), + "set_status": _action(set_status, { + "application": _TEXT, "item_ids": {"type": "array", "items": _TEXT, "minItems": 1}, + "status": _TEXT, "note": {"type": "string", "maxLength": 1000}, + }, ("application", "item_ids", "status"), read=False, + description="Change the status of one or more App items. Status approved on several ids approves them " + "in one confirm_change."), "guide": _action(guide, {"section": _TEXT}), }, } +# C8 reports items and set_status as supported once some template kind has a C4 module. +for _name in ("items", "set_status"): + GROUP["actions"][_name]["available"] = items_supported + # The Assistant callback returns a preview. Only confirm_change applies its product write. TOOLS = [{"name": "apps", "schema": {**LOOPY_SCHEMA, "description": GROUP["guide"]}, "fn": loopy, "effect": "read"}, diff --git a/api/tool_groups/data.py b/api/tool_groups/data.py index 0e56cc7bea6914771890182f4366da3d4411a8f5..42cdbdbca797b7ad5b2670d8c3cb43a1d272c449 100644 --- a/api/tool_groups/data.py +++ b/api/tool_groups/data.py @@ -13,6 +13,92 @@ READ_ARGUMENT_HELP = ( ) +#: The one skill.md section every query refusal names. +QUERY_SYNTAX_SECTION = "Query syntax" +QUERY_FILTER_EXAMPLE = {"conj": "and", "nodes": [{"colId": "", "op": "eq", "value": ""}]} +QUERY_SYNTAX_FIX = ( + "Filters use {conj, nodes: [{colId, op, value}]}, for example " + + json.dumps(QUERY_FILTER_EXAMPLE, separators=(", ", ": ")) + ". " + "groupBy is a list of 1 to 4 field keys, or {field, unit: month}, with at least one aggregate and no sorts. " + "Only a connected database groups. Any other database answers query_groups_unavailable. " + "See " + QUERY_SYNTAX_SECTION + " in skill.md." +) +#: Query refusal codes that carry QUERY_SYNTAX_FIX. +QUERY_SYNTAX_CODES = frozenset({ + "query_arguments_invalid", "query_aggregates_invalid", "query_aggregate_type", + "query_groups_invalid", "query_groups_unavailable", "assistant_bad_filter", +}) +#: S79-T11: a schema-valid filter the read path refuses names the field and operator rules. +BAD_FILTER_FIX = ( + "Use field keys and operators that describe_database returns. " + "isAnyOf and isNoneOf take values, an array. between takes value and value2." +) +#: S79-T11: an unknown database key names the call that lists keys, then the current query shape. +UNKNOWN_DATABASE_FIX = ( + 'No readable database has that key. Call table with {"action": "list_databases"} and use a database key it ' + 'returns. Then call table with {"action": "query_database", "arguments": {"database": "", ' + '"limit": 100}}.' +) + + +def bad_filter_message(message): + """Rewrite the read path's filter refusal as one plain sentence plus the field and operator rules.""" + text = " ".join(str(message or "").split()).rstrip(".") + for prefix, subject in (("assistant filters ", "Filters "), ("assistant filter ", "The filter ")): + if text.startswith(prefix): + text = subject + text[len(prefix):] + text = (text[:1].upper() + text[1:] + ".") if text else "The filter cannot be read." + return text if BAD_FILTER_FIX in text else text + " " + BAD_FILTER_FIX + + +def with_query_syntax(message): + """Append the filter shape, the groupBy rule and the skill.md pointer once.""" + text = str(message or "").rstrip() + return text if QUERY_SYNTAX_FIX in text else (text + " " + QUERY_SYNTAX_FIX).strip() + + +#: The measured full product pull and its limit, cause and fix (rule 1). Never a cap. +FULL_PULL_NOTE = ( + "Product data: a full pull reads about 6,000 rows in 12 pages, and each page after the first takes about 2 seconds. " + "The first page after a server restart can take about 3 minutes, because the product list is built from Odoo " + "on first use. If that call times out, repeat it. The next call reads the built list and nothing is capped." +) + + +def query_syntax_section(): + """Return the skill.md section that every query refusal names.""" + example = {"action": "query_database", "arguments": { + "database": "", "fields": [""], + "filters": QUERY_FILTER_EXAMPLE, "limit": 500}} + lines = [ + "## " + QUERY_SYNTAX_SECTION, "", + "Every query refusal names this section. Call table with action query_database:", "", + "```json", json.dumps(example, indent=2), "```", "", + "- filters: {conj, nodes: [{colId, op, value}]}. conj is and or or. A node is a condition " + "{colId, op, value}, or a group {conj, children: [...]}. isAnyOf and isNoneOf take values, an array. " + "between takes value and value2. A link condition is {link, where}. Filters allow 3 levels, " + "100 nodes and 50 conditions per group.", + "- A flat object such as {\"status\": \"open\"}, or a condition with field or operator keys, is refused " + "with tool_arguments_invalid. Use colId and op.", + "- op: " + ", ".join(sorted(external_tools.FILTER_OPS)) + ". describe_database names the operators " + "each field accepts.", + "- sorts: [{field, direction}], direction asc or desc, at most 8.", + "- aggregates: [{fn, field}], fn count, count_distinct, sum, avg, min or max, at most 16. " + "count may omit field. sum and avg need a numeric field (query_aggregate_type otherwise).", + "- groupBy: 1 to 4 field keys, or {field, unit: month} for a date field. A grouped query needs at least " + "one aggregate and no sorts (query_groups_invalid otherwise). Only a connected database, such as an " + "Odoo sales lines mirror, groups. Any other database answers 409 query_groups_unavailable: read its " + "records by page, or ask for aggregates without groupBy.", + "- Paging: limit is 1 to 500, default 100. Send page.next_cursor as cursor with the same arguments " + "until page.has_more is false. Only fields, filters, sorts, limit, cursor, aggregates and groupBy are " + "accepted (query_arguments_invalid otherwise).", + "- Full pulls: no row cap applies to a connected source. Read every page at limit 500.", + ] + if FULL_PULL_NOTE: + lines.append("- " + FULL_PULL_NOTE) + return lines + [""] + + GROUPING_EXAMPLE = { "action": "query_database", "arguments": { "database": "ut_odoo_order_lines", "fields": [], @@ -36,6 +122,7 @@ def query_guide(rows): }}, indent=2), "```", "## Filter operators", "", "Accepted operator names: " + ", ".join(sorted(external_tools.FILTER_OPS)) + ".", + QUERY_SYNTAX_FIX, "describe_database returns operators and operands for each readable field. " "Use eq for equality, gte and lte for inclusive date bounds, and isAnyOf with values for membership. " "Date comparisons use value in YYYY-MM-DD format. Membership uses a values array. " diff --git a/api/tool_groups/sheet.py b/api/tool_groups/sheet.py index f4fc6d05ea0684647bd861dfcdef257c17c878fc..f7073bf52dd7a7e8c1ae9adc216e4b0679f30b0c 100644 --- a/api/tool_groups/sheet.py +++ b/api/tool_groups/sheet.py @@ -8,6 +8,10 @@ from . import legacy_action WORKBOOK_RESOURCE_URI = "ui://loopable/workbook.html" WORKBOOK_RESOURCE_MIME = "text/html;profile=mcp-app" +#: The next call after workbook_missing (S79-T11): list readable workbooks, then read one by its id. +WORKBOOK_MISSING_FIX = ( + 'Call sheet with {"action": "workbook.list", "arguments": {"limit": 100}} and follow cursor until it is null. ' + 'Then call sheet with {"action": "workbook.get", "arguments": {"workbook_id": ""}} using a returned id.') def _readable_workbooks(ctx): @@ -82,7 +86,7 @@ def workbook_get(ctx, args): {"candidates": matches}) if not matches: names = get_close_matches(args["name"], sorted({row["name"] for row in rows}), n=5) - raise err(404, "workbook_missing", "No readable workbook has that exact name.", + raise err(404, "workbook_missing", "No readable workbook has that exact name. " + WORKBOOK_MISSING_FIX, {"suggestions": names}) book = external_tools._book(ctx, {"workbook_id": matches[0]["id"]}) projected = ctx.project_book(book) @@ -182,6 +186,7 @@ for name, handler, properties in ( "additionalProperties": False}, "available": lambda operation=name: _object_available(operation), } +GROUP["actions"]["workbook.get"]["example"] = {"workbook_id": ""} GROUP["actions"]["workbook.view"] = { "read": True, "destructive": False, "confirm": True, "grants": ("sheets",), "handler": _workbook_view, "_meta": {"ui": {"resourceUri": WORKBOOK_RESOURCE_URI}}, diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index 376ebc6fc12c839973aba85ed0ae7ac99bc099a3..0001c086c36895c5eb487dc367d9eeabd64d9461 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -197,18 +197,25 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + "description": "Receivable not yet due on Odoo's Aged Receivable basis: due today or later, no grace." }, { "key": "ar_overdue", - "label": "AR overdue $", + "label": "Overdue", "type": "currency", "source": "odoo", "default": false, - "description": "Invoiced money past due, on the same basis as the Collections page." + "description": "Odoo Aged Receivable overdue: open receivable lines past their due date, no grace, New Customer Hold due at once, credits and unapplied payments netted per customer. Ties to Odoo to the cent." }, { "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", + "key": "ar_overdue_past_grace", + "label": "Overdue past grace", + "type": "currency", + "source": "odoo", + "description": "Invoice balances more than 5 days past due (the 5-day grace collections uses). Not Odoo's overdue." + }, + { "key": "ar_outstanding", "label": "AR outstanding $", "type": "currency", @@ -230,7 +237,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + "description": "Overdue between 1 and 30 days. The four aging buckets sum to Overdue." }, { "key": "ar_aged_31_60", @@ -238,7 +245,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + "description": "Overdue between 31 and 60 days. The four aging buckets sum to Overdue." }, { "key": "ar_aged_61_90", @@ -246,7 +253,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + "description": "Overdue between 61 and 90 days. The four aging buckets sum to Overdue." }, { "key": "ar_aged_90_plus", @@ -254,7 +261,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + "description": "Overdue by more than 90 days. The four aging buckets sum to Overdue." }, { "key": "days_to_pay", @@ -418,7 +425,8 @@ "custom": true, "default": true, "description": "Supplier or manufacturer for this product.", - "shared": true + "shared": true, + "walled": true }, { "key": "origin_country", diff --git a/platform/core/connector_spec.py b/platform/core/connector_spec.py index 512953342f88167ef014fde97c8e330693f65899..22bc350ef0ef178328f7fa0c25d9440b42ab7d25 100644 --- a/platform/core/connector_spec.py +++ b/platform/core/connector_spec.py @@ -3469,10 +3469,103 @@ AMAZON_GROUPS = { } +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# AMAZON ADS: S79-T12 (contract C6). Sponsored Products, Brands and Display on Reporting v1. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +#: The connector key C1 names. It is its own registration, not rows inside `"amazon"`: the Amazon +#: specs are held to the SP-API schema artifact (`amazon_schema.TABLE_SPECS`) by verify_amazon and +#: verify_connectors, and an Ads report is not an SP-API report. The mirror is still the Amazon one. +AMAZON_ADS_CONNECTOR = "amazon_ads" +_ADS = "ut_amzn_ads_" +AMAZON_ADS_AD_TYPES = ("sp", "sb", "sd") +AMAZON_ADS_REPORTS = ("campaigns", "targeting", "advertised_products", "search_terms") +_ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"} +_ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting", + "advertised_products": "advertised products", "search_terms": "search terms"} +#: `native` is the Ads report type id. Unknown: the exact v1 spelling; the first credentialed +#: report settles it (PRD P17), and each id is one literal here. +_ADS_NATIVE = { + "sp": {"campaigns": "spCampaigns", "targeting": "spTargeting", + "advertised_products": "spAdvertisedProduct", "search_terms": "spSearchTerm"}, + "sb": {"campaigns": "sbCampaigns", "targeting": "sbTargeting", + "advertised_products": "sbAds", "search_terms": "sbSearchTerm"}, + "sd": {"campaigns": "sdCampaigns", "targeting": "sdTargeting", + "advertised_products": "sdAdvertisedProduct", "search_terms": "sdSearchTerm"}, +} +_ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting", + "advertised_products": "asin", "search_terms": "search_term"} +_ADS_IDENTITY = { + "campaigns": ("date", "campaign_id"), + "targeting": ("date", "campaign_id", "ad_group_id", "target_id"), + "advertised_products": ("date", "campaign_id", "ad_group_id", "ad_id", "asin"), + "search_terms": ("date", "campaign_id", "ad_group_id", "target_id", "search_term"), +} + +#: One spec per ad type and report. No links: every table is a daily grain, so a campaign id +#: names many campaign rows and a many_to_one link would be false. +AMAZON_ADS_TABLES = tuple( + ConnectorTableSpec( + key=f"{_ADS}{ad_type}_{report}", + native=_ADS_NATIVE[ad_type][report], + source=f"amzn_ads_{ad_type}_{report}", + label=f"{_ADS_PRODUCT[ad_type]} {_ADS_REPORT_LABEL[report]}", + pinned=_ADS_PINNED[report], + identity=_ADS_IDENTITY[report], + links=(), + ) + for ad_type in AMAZON_ADS_AD_TYPES for report in AMAZON_ADS_REPORTS) + +#: Every Ads table sits in the Amazon "Advertising" rail group. Kept apart from `AMAZON_GROUPS`, +#: which verify_connectors holds equal to the `"amazon"` registration. +AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES} + +#: Days the first sync reaches back, per table: the longest window Amazon documents per ad type +#: (SP 95, SB 60, SD 65, search terms 60). Unknown: v1 retention (primer F12). The first +#: credentialed sync measures the oldest day Amazon returns. +AMAZON_ADS_BACKFILL_DAYS = { + spec.key: (60 if spec.key.endswith("_search_terms") + else {"sp": 95, "sb": 60, "sd": 65}[spec.key[len(_ADS):len(_ADS) + 2]]) + for spec in AMAZON_ADS_TABLES} + +#: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen. +AMAZON_ADS_THROUGHPUT = ( + "Amazon Reporting v1 runs one report at a time per advertiser and about 100 reports a day. " + "Sponsored Products, Brands and Display need 11 report types, and one report can take up to " + "3 hours, so one day of refresh may not finish in a day. Loopable runs the reports one at a " + "time and keeps every day it already has. If the refresh falls behind, ask Amazon Ads support " + "to raise the report limit for this advertiser.") + +#: Standing rule 1: the report v1 does not offer, with cause and fix. On screen. +AMAZON_ADS_NOT_OFFERED = { + _ADS + "sd_search_terms": ( + "Amazon Reporting v1 offers no search term report for Sponsored Display, because Display " + "ads target audiences and products, not shopper searches. This table stays empty. To see " + "what a Display campaign matched, open Sponsored Display targeting."), +} + + +def amazon_ads_description(key): + """The Data page description of one Ads table, or None off Ads. Never raises.""" + try: + spec = SPECS.get(AMAZON_ADS_CONNECTOR, {}).get(str(key or "")) + except Exception: # noqa: BLE001 + return None + if spec is None: + return None + unoffered = AMAZON_ADS_NOT_OFFERED.get(spec.key) + if unoffered: + return f"{unoffered} {AMAZON_ADS_THROUGHPUT}" + days = AMAZON_ADS_BACKFILL_DAYS[spec.key] + return (f"{spec.label} by day, from Amazon Ads Reporting v1. History is kept from the day you " + f"connect, and the first sync reaches back {days} days, the longest Amazon keeps. " + f"{AMAZON_ADS_THROUGHPUT}") + + def table_group(key): - """The rail group of one Amazon table, or None off Amazon. Never raises.""" + """The rail group of one Amazon or Amazon Ads table, or None off Amazon. Never raises.""" try: - return AMAZON_GROUPS.get(str(key or "")) + key = str(key or "") + return AMAZON_GROUPS.get(key) or AMAZON_ADS_GROUPS.get(key) except Exception: # noqa: BLE001 return None @@ -3480,9 +3573,13 @@ def table_group(key): register("amazon", AMAZON_TABLES, mirror=ConnectorMirrorSpec(tuple(s.source for s in AMAZON_TABLES if not s.native.startswith("derived:")) - + ("amzn_apify_rank_observations",), state_prefix="amzn_")) + + ("amzn_apify_rank_observations",) + # The Ads tables live in this mirror, so it saves and restores them. + + tuple(s.source for s in AMAZON_ADS_TABLES), state_prefix="amzn_")) register("meta", META_TABLES, mirror=ConnectorMirrorSpec(tuple(s.source for s in META_TABLES), state_entities=("meta",))) +# No mirror of its own: the Ads tables are declared in the Amazon mirror above (contract C6). +register(AMAZON_ADS_CONNECTOR, AMAZON_ADS_TABLES) __all__ = ["CARDINALITIES", "ConnectorTableSpec", "SPECS", "CONNECTOR_SPEC_KEYS", "AMAZON_TABLES", "AMAZON_FIELDS", "AMAZON_LABELS", "AMAZON_DEFAULT_COLUMNS", "META_TABLES", @@ -3492,3 +3589,6 @@ __all__ = ["CARDINALITIES", "ConnectorTableSpec", "SPECS", "CONNECTOR_SPEC_KEYS" "register_scoped_row_decoder", "decode_scoped_rows", "edges_for", "system_link_fields"] __all__ += ["AMAZON_GROUP_ORDER", "AMAZON_GROUPS", "table_group"] +__all__ += ["AMAZON_ADS_CONNECTOR", "AMAZON_ADS_AD_TYPES", "AMAZON_ADS_REPORTS", "AMAZON_ADS_TABLES", + "AMAZON_ADS_GROUPS", "AMAZON_ADS_BACKFILL_DAYS", "AMAZON_ADS_THROUGHPUT", + "AMAZON_ADS_NOT_OFFERED", "amazon_ads_description"] diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py index 82148e48076c9a0f79f829269a4e01297e72600c..26f392c14b02207c72f81ca01a6b32749bcbd972 100644 --- a/platform/core/external_tools.py +++ b/platform/core/external_tools.py @@ -248,7 +248,10 @@ _APPLICATION = { "type": {"enum": ["chart", "route", "map", "code"]}, "compatibleDatabases": {"type": "array", "items": _STRING}, "sourceDatabase": {"type": "string", "description": "Source for a template Application only."}, - "templateKind": {"enum": ["route", "catalog", "competitor_watch"]}, + # Mirrors aios-web/api/app_templates.TEMPLATES kinds; api_app_templates holds the parity. + "templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic", + "amazon_operator", "restock_planner", "reimbursements", "aged_stock", + "opportunity_finder", "returns_repeat", "payouts", "account_health"]}, "fieldMap": {"type": "object", "additionalProperties": _STRING}, "navPlacement": _object({"section": {"const": "app"}, "order": {"type": "integer", "minimum": 0, "maximum": 999}}, diff --git a/platform/core/field_permissions.py b/platform/core/field_permissions.py index e4dc1e38e8fc21c8cd3de607fd64516a823a7b73..4569cd1806e273f4bbc9d84079a8ffa13ba1f747 100644 --- a/platform/core/field_permissions.py +++ b/platform/core/field_permissions.py @@ -716,6 +716,44 @@ def _grant_mark(): return "granted" +#: S79-T68 / R64 (D-487): a contract column that the wall governs only where a grant names it. +#: It is not the wall's mark: `granted` with no record hides the column from every account. +FIELD_WALL_MARK = "walled" + + +def walled_field_keys(fields): + """The keys of `fields` that carry `FIELD_WALL_MARK: true`.""" + return {str(f["key"]) for f in (fields or ()) + if isinstance(f, dict) and f.get("key") and f.get(FIELD_WALL_MARK) is True} + + +def _names_somebody(record): + """True if a grant record has an owner or at least one entry (the arming rule).""" + record = record if isinstance(record, dict) else {} + return bool(record.get("owner") or record.get("entries")) + + +def arm_walls(fields, grant_topic, st=None): + """A new field list where each walled field whose grant record names somebody carries the wall's mark. + + Every other field comes back unchanged and the input list is never mutated. + """ + fields = list(fields or ()) + walled = walled_field_keys(fields) + if not walled: + return fields + mark = _grant_mark() + out = [] + for f in fields: + key = str(f.get("key") or "") if isinstance(f, dict) else "" + # A lenient read that answers empty in a cold-cache window leaves the column visible (R64). + if key in walled and f.get(mark) is not True and _names_somebody( + _grant_record(key, grant_topic=grant_topic, st=st)): + f = dict(f, **{mark: True}) + out.append(f) + return out + + def field_stratum(field): """⭐⭐ W42-T47 / R15 / C10 — THE **FALLBACK** DERIVATION OF WHICH STRATUM A DEFINITION CAME FROM, for a caller that cannot lend the fact. See `FIELD_DEFINITION_STRATA`. @@ -984,7 +1022,8 @@ def field_class(field, viewer, grants=None, *, table_key=None, grant_topic=None, values_shared = _shared_value_keys(table_key, st=st) values = "shared" if key and key in {str(k) for k in values_shared} else "personal" - if _governed(field): + # R64: an armed wall is governed, so the badge agrees with `arm_walls` on the same record. + if _governed(field) or (key in walled_field_keys([field]) and _names_somebody(record)): audience = _AUDIENCE_FROM_EDIT.get( permissions_from_grants(entries).get("edit"), "private") elif not anchored_personal and (values == "shared" or origin == "preset"): diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 944618258378e0474f2e7acf09bcc5944fff19a6..3c6e3563d4fbab82a7eaad229000adcccc7ddc8f 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -607,7 +607,7 @@ def _docs_key(ctx): # ------------------------------------------------------------------ the workspace family (X1 amd.) -def _grant_marker(ctx, ws): +def _grant_marker(ctx, ws, uname=None): """`has_grants(view_id) -> bool` for this tenant, or None when nothing here is granted at all. The grant registry is the only View-audience model. Direct grants and grants inherited from @@ -619,7 +619,7 @@ def _grant_marker(ctx, ws): # bucket is tenant-scoped and must be read out of the tenant whose workspace this is. Written # as `_tops(ctx).st` rather than `_store_of(ctx)` so the two legs cannot drift on tenancy. st = getattr(_tops(ctx), 'st', None) - view_oids = shares.granted_oids('view', st=st) + view_oids = shares.granted_oids('view', st=st, viewer=uname) folder_oids = shares.granted_oids('folder', st=st) if not view_oids and not folder_oids: return None # nothing in this tenant is granted: no marks, no closure @@ -774,7 +774,7 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do # `granted` is already stamped. Mark owner sources separately so their display metadata # stays symmetric with recipient projections without creating a second copy or audience. try: - _has_grants = _grant_marker(ctx, ws) if ws.get('views') else None + _has_grants = _grant_marker(ctx, ws, uname) if ws.get('views') else None except Exception as _me: # A share mark is DECORATION. It must never be the thing that takes down a workspace # read, so it fails the way its two siblings above do: log and carry on unmarked. diff --git a/platform/core/keychain.py b/platform/core/keychain.py index f01b4254acaa8047ebc075e2d49d4ac3c39d0bce..d233d35a7a328a9eaeaee6717c388a6d32b56cdd 100644 --- a/platform/core/keychain.py +++ b/platform/core/keychain.py @@ -54,7 +54,7 @@ AMAZON_ROTATION_DEADLINE_FIELD = 'client_secret_rotation_deadline' #: ⚠ R8 keeps the two doors separate on purpose: this is the TENANT's own token (GTM Lab pastes #: theirs), while R5's "Connect Facebook" is the delegated-OAuth door Nango owns. Neither replaces #: the other, and the schema work is blocked on neither. -ENTRY_TYPES = {'odoo', 'generic', 'stripe', 'shopify', 'meta_ads', 'amazon_sp', 'slack'} +ENTRY_TYPES = {'odoo', 'generic', 'stripe', 'shopify', 'meta_ads', 'amazon_sp', 'amazon_ads', 'slack'} #: which stored field feeds the masked preview, per type (first present wins) #: ⚠ `access_token` JOINED IN WAVE 31, AND ITS ABSENCE WAS A REAL DEFECT FOUND BY BUILDING THE #: SEAM RATHER THAN BY READING IT. Meta's credential is conventionally an `access_token`, and with @@ -191,7 +191,7 @@ def _bucket(rt): #: type -> the human word the default label is built from. `generic` has no product name behind #: it, so it says what it is rather than inventing one. -_KIND_LABELS = {'odoo': 'Odoo', 'meta_ads': 'Meta Ads', 'amazon_sp': 'Amazon', +_KIND_LABELS = {'odoo': 'Odoo', 'meta_ads': 'Meta Ads', 'amazon_sp': 'Amazon', 'amazon_ads': 'Amazon Ads', 'stripe': 'Stripe', 'shopify': 'Shopify', 'generic': 'Connection', 'slack': 'Slack'} @@ -308,9 +308,13 @@ def add_entry(rt, label, etype, fields, username, *, scope=None, write_scope=Non continue if k and isinstance(v, (str, int, float, bool)): clean[k] = str(v)[:MAX_FIELD_LEN] - if etype == 'amazon_sp': - from harness import amazon_client - missing = [key for key in amazon_client.CREDENTIAL_FIELDS + if etype in ('amazon_sp', 'amazon_ads'): + # S79-T13: Amazon Ads stores its four fields whole, like Amazon SP stores its five. + if etype == 'amazon_sp': + from harness import amazon_client as roster + else: + from harness import amazon_ads_client as roster + missing = [key for key in roster.CREDENTIAL_FIELDS if not isinstance(fields.get(key), str) or not clean.get(key, '').strip()] if missing: raise ValueError('Missing ' + ', '.join(missing) + '. Enter every required field.') @@ -553,6 +557,11 @@ def amazon_creds(rt, connection_id=None): return _creds_for(rt, 'amazon_sp', connection_id) +def amazon_ads_creds(rt, connection_id=None): + """S79-T13: the designated `amazon_ads` entry's four fields, else None. No environment fallback.""" + return _creds_for(rt, 'amazon_ads', connection_id) + + #: ⭐⭐ WAVE 32 (R4 / contract C1) — THE SCOPE SIDE-BUCKET, READ HERE, WRITTEN NOWHERE IN THIS FILE. #: #: `{entry_id: {"scope": "personal", "owner": ""}}`. A `business` entry writes NO row, so @@ -579,7 +588,7 @@ SCOPE_KEY = 'keychain_scopes' #: "their own Amazon": they would become the seller account the entire workspace's Amazon databases #: are built from. The list was written to prevent exactly this and a new tenant-wide connector has #: to join it in the same change that adds the connector, or the guard silently does not cover it. -TENANT_WIDE_TYPES = ('odoo', 'meta_ads', 'amazon_sp') +TENANT_WIDE_TYPES = ('odoo', 'meta_ads', 'amazon_sp', 'amazon_ads') def _personal_ids(rt): @@ -876,20 +885,30 @@ def entry_verified(rt, entry_id): str(entry['enc']).encode('utf-8')).hexdigest()) +#: S79-T13: the fixed messages per Amazon type. Provider text never crosses this boundary. +_AMAZON_TEST_MESSAGES = { + 'amazon_sp': ('Amazon confirmed marketplace access.', + 'Amazon did not confirm marketplace access. Check the five fields and test again.'), + 'amazon_ads': ('Amazon Ads confirmed profile access.', + 'Amazon Ads did not confirm profile access. Check the four fields and connect again.'), +} + + def _test_amazon_entry(rt, entry_id, entry, timeout): """Only an explicit test calls the registered probe; storage and listing stay offline. Bind the result to ciphertext so a replacement cannot inherit a previous credential's proof. Neither provider messages nor exception text cross this boundary: both can echo input. """ - result = {'ok': False, - 'message': 'Amazon did not confirm marketplace access. Check the five fields and test again.'} + etype = entry.get('type') if entry.get('type') in _AMAZON_TEST_MESSAGES else 'amazon_sp' + passed, failed = _AMAZON_TEST_MESSAGES[etype] + result = {'ok': False, 'message': failed} try: fields = read_fields(rt, entry_id) or {} - prober = _PROBES.get('amazon_sp') + prober = _PROBES.get(etype) got = prober(dict(fields), timeout) if prober is not None else None if isinstance(got, dict) and got.get('ok') is True: - result = {'ok': True, 'message': 'Amazon confirmed marketplace access.'} + result = {'ok': True, 'message': passed} except Exception: # noqa: BLE001 pass @@ -918,7 +937,7 @@ def test_entry(rt, entry_id, timeout=8): e = (_bucket(rt).get('entries') or {}).get(str(entry_id)) if not isinstance(e, dict): return {'ok': False, 'message': 'no such entry'} - if e.get('type') == 'amazon_sp': + if e.get('type') in _AMAZON_TEST_MESSAGES: return _test_amazon_entry(rt, entry_id, dict(e), timeout) fields = read_fields(rt, entry_id) or {} # ⭐ W31 — A REGISTERED PROBER ANSWERS FOR ITS OWN TYPE, if the connector layer filled the diff --git a/platform/core/odoo.py b/platform/core/odoo.py index 3847c97cf8c589f55d0c1175808f608b40a74f88..57760824539939824bdbfd938e9a2351a97cb5df 100644 --- a/platform/core/odoo.py +++ b/platform/core/odoo.py @@ -176,6 +176,10 @@ def _bounded_proxy(uri): class OdooClient(_BaseOdooClient): def _connect(self, _retried=False): + with _breaker_call(self): + return self._connect_once(_retried) + + def _connect_once(self, _retried=False): try: with _bounded_proxy(f'{self.url}/xmlrpc/2/common') as common: self.uid = common.authenticate(self.db, self.user, self.api_key, {}) @@ -189,11 +193,17 @@ class OdooClient(_BaseOdooClient): self.models = _bounded_proxy(f'{self.url}/xmlrpc/2/object') def execute_kw(self, model, method, args, kwargs=None): - if _timing.current_budget() is None: - return super().execute_kw(model, method, args, kwargs) self._guard(method) + with _breaker_call(self) as call: + if _timing.current_budget() is None: + return super().execute_kw(model, method, args, kwargs) + return self._budget_execute(call, model, method, args, kwargs) + + def _budget_execute(self, call, model, method, args, kwargs): if not self._xlock.acquire(timeout=_rpc_remaining()): _timing.remaining_budget() + if call is not None: + call.local = True # Local lock contention says nothing about the upstream. raise OdooRequestTimeout("Odoo client lock timed out") try: _timing.remaining_budget() @@ -374,6 +384,100 @@ def _checkin(c): _pool_q.put(c) +# ---- circuit breaker, one per Odoo URL --------------------------------------------------------- +# After N consecutive transport failures the URL opens for a cooldown and every call refuses at +# once with no network I/O. After the cooldown exactly one caller runs as the half-open trial. +# ⛔ Only transport failures count. A Fault means Odoo answered, and our own request budget +# (RequestDeadlineExceeded) or a write refusal (ReadOnlyError) says nothing about the upstream. +_monotonic = time.monotonic # Gates patch this clock so no leg sleeps. +_BREAKERS = {} # url -> {"failures": int, "opened_at": float | None, "trial": bool} +_BREAKER_LOCK = threading.Lock() +_breaker_local = threading.local() +_BREAKER_FAILURES = (OSError, http.client.HTTPException) + + +class OdooBreakerOpen(OdooPoolTimeout): + """Odoo failed too often in a row, so reads refuse at once until the cooldown ends.""" + + +def _breaker_setting(name, default, minimum): + try: + value = float(os.environ.get(name, str(default))) + return max(minimum, value) if math.isfinite(value) else float(default) + except ValueError: + return float(default) + + +def _breaker_admit(key): + """Admit one call: True for the half-open trial, False when closed. Raise while open.""" + with _BREAKER_LOCK: + state = _BREAKERS.get(key) + if state is None or state["opened_at"] is None: + return False + cooldown = _breaker_setting("AIOS_ODOO_BREAKER_COOLDOWN_S", 30, 0) + if state["trial"] or _monotonic() - state["opened_at"] < cooldown: + message = "Odoo is not answering right now. Reads pause briefly before the next try." + raise OdooBreakerOpen(message) from OdooPoolTimeout(message) + state["trial"] = True + return True + + +def _breaker_settle(key, trial, outcome): + """Record one admitted call: True closes, False counts a failure, None changes nothing.""" + with _BREAKER_LOCK: + if outcome is True: + _BREAKERS.pop(key, None) + return + state = _BREAKERS.get(key) + if outcome is None: + if trial and state is not None: + state["trial"] = False + return + if state is None: + state = _BREAKERS[key] = {"failures": 0, "opened_at": None, "trial": False} + state["failures"] += 1 + limit = _breaker_setting("AIOS_ODOO_BREAKER_FAILURES", 5, 1) + if trial or state["opened_at"] is not None or state["failures"] >= limit: + state["opened_at"] = _monotonic() + state["trial"] = False + + +class _BreakerCall: + local = False + + +@contextmanager +def _breaker_call(client): + """Guard one outermost wire call of `client`. A nested call (reconnect, retry) passes through.""" + if getattr(_breaker_local, "depth", 0): + yield None + return + key = str(getattr(client, "url", "") or "") + trial = _breaker_admit(key) + call, outcome = _BreakerCall(), None + _breaker_local.depth = 1 + try: + yield call + outcome = True + except xmlrpc.client.Fault: + outcome = True + raise + except _timing.RequestDeadlineExceeded: + raise + except _BREAKER_FAILURES: + outcome = None if call.local else False + raise + finally: + _breaker_local.depth = 0 + _breaker_settle(key, trial, outcome) + + +def reset_breakers(): + """Close every Odoo breaker. For gates.""" + with _BREAKER_LOCK: + _BREAKERS.clear() + + def get_odoo(): """The active read-only client, in priority order: 1. a NON-default tenant binding (tenant_scope) — its own client, ALWAYS. The parallel() diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index ffee0a7382c4cd914e73c60d7a2019223c3f4d17..c5e7b9be1cfb843d5fe85092481c9e8c3d059513 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -316,6 +316,11 @@ def field_grant_hidden(user, table_key, fields, st=None): """ if perms.is_admin(user): return set() + try: + import core.field_permissions as field_permissions + fields = field_permissions.arm_walls(fields, table_key, st=st) + except ImportError: + pass marked = granted_field_keys(fields) if not marked: return set() diff --git a/platform/core/shares.py b/platform/core/shares.py index e1b8e84954826eb6f9fb96f71a2f47a2b6a8da56..538f33bd09d9b78b97053ac48f78208d43350d4d 100644 --- a/platform/core/shares.py +++ b/platform/core/shares.py @@ -878,9 +878,11 @@ def shared_with(user, kind=None, st=None): return out if kind is None else {_check_kind(kind): out[_check_kind(kind)]} -def granted_oids(kind, st=None): +def granted_oids(kind, st=None, viewer=None): """Every object id of one `kind` that carries AT LEAST ONE grant entry — in ONE bucket read. + With `viewer`, a grant on a `PINNED_VIEW_IDS` id counts only on the owner's copy (D-472). + ⭐ W40-T01 / OWNER INSTRUCTION 2 ("a shared View must show the shared icon in EVERY account, not only the recipient's"). `shared_with` answers *what was granted TO me* and deliberately EXCLUDES what the caller owns — so it structurally cannot answer the other question a share @@ -911,8 +913,79 @@ def granted_oids(kind, st=None): rows = list(by_kind.items()) except Exception: return set() + who = str(viewer or '').strip().lower() return {str(oid) for oid, rec in rows - if isinstance(rec, dict) and _clean_entries(rec.get('entries'))} + if isinstance(rec, dict) and _clean_entries(rec.get('entries')) + and (not who or not _pinned_elsewhere(kind, oid, rec, who))} + + +#: View ids every account holds its OWN copy of. `tpl_overview` is `aios_grid.IG_OVERVIEW_ID`, +#: spelled here because core never imports up. A grant on one marks its owner's copy alone (D-472). +PINNED_VIEW_IDS = frozenset({'all-customers', 'tpl_overview'}) + +#: The kinds a tile can wear a share mark for. `field` and `cohort` have no tile. +MARK_KINDS = ('view', 'folder', 'database', 'workbook') + + +def _pinned_elsewhere(kind, oid, rec, viewer): + """True when this grant sits on a pinned per-user view id that is NOT the viewer's copy.""" + if kind != 'view' or str(oid) not in PINNED_VIEW_IDS: + return False + owner = str((rec or {}).get('owner') or '').strip().lower() + return owner != viewer + + +def mark_from(kind, oid, rec, viewer): + """The share mark one grant record gives `viewer`, or None. PURE. + + `{'direction': 'by_me', 'count': N, 'everyone': bool}` for the owner of a shared object, + `{'direction': 'with_me', 'by': owner}` for a grantee. Nobody else gets a mark, so a mark never + tells a viewer about a share they are not part of. + """ + viewer = str(viewer or '').strip().lower() + if not viewer or not isinstance(rec, dict): + return None + entries = _clean_entries(rec.get('entries')) + if not entries: + return None + owner = str(rec.get('owner') or '').strip().lower() + if owner == viewer: + others = [e['user'] for e in entries if e['user'] != viewer] + if not others: + return None + return {'direction': 'by_me', 'count': sum(1 for u in others if u != EVERYONE), + 'everyone': EVERYONE in others} + # A pinned id names the viewer's OWN copy on their tile, never the owner's (D-472). + if kind == 'view' and str(oid) in PINNED_VIEW_IDS: + return None + if any(e['user'] in (viewer, EVERYONE) for e in entries): + return {'direction': 'with_me', 'by': owner} + return None + + +def share_marks(viewer, st=None): + """`{kind: {oid: mark}}` for every `MARK_KINDS` object shared with or by `viewer`, ONE read. + + None when the grant bucket cannot be read: the caller then shows no mark, never a guessed one. + """ + try: + data = _st(st).get(SHARES_KEY) or {} + except Exception: + return None + if not isinstance(data, dict): + return None + out = {} + for kind in MARK_KINDS: + rows = data.get(kind) or {} + if not isinstance(rows, dict): + continue + hits = {} + for oid, rec in rows.items(): + mark = mark_from(kind, oid, rec, viewer) + if mark: + hits[str(oid)] = mark + out[kind] = hits + return out def drop_objects(pairs, st=None): diff --git a/platform/core/sheet_binding.py b/platform/core/sheet_binding.py index 511b84471bf6465726e67aee02f3e84ecebe08f4..69747418c9ff00f0f00c3e3f8bb9f8aaf4c29235 100644 --- a/platform/core/sheet_binding.py +++ b/platform/core/sheet_binding.py @@ -58,6 +58,19 @@ def database_change_token(st, database): key(connector_sync.JOBS_KEY), database, database, key(connector_sync.STATE_KEY))).fetchone() token, definition = row[0], row[1] or {} + if store_pg._ut_rows(key(user_tables.STORE_KEY), handle.tenant_slug): + from core import user_table_rows + from psycopg import sql + definition = user_table_rows.definition_view(user_table_rows.read_definition( + key(user_tables.STORE_KEY), handle.tenant_slug, database)) or {} + with store_pg._pool().connection() as con: + row_token = con.execute(sql.SQL( + "SELECT (SELECT value::text FROM {} WHERE table_key=%s), " + "count(*), max(rev), max(updated_at)::text FROM {} WHERE table_key=%s" + ).format(store_pg._table("user_table_defs", handle.tenant_slug), + store_pg._table("user_table_records", handle.tenant_slug)), + (database, database)).fetchone() + token = fingerprint([token, row_token]) else: definition = (st.get(user_tables.STORE_KEY, fresh=True) or {}).get(database) or {} jobs = st.get(connector_sync.JOBS_KEY, fresh=True) or {} diff --git a/platform/core/sheet_templates.py b/platform/core/sheet_templates.py index 5c90a44816e4e764e9c720fd2fb6b752c83d1738..1b8701a3a9a3a127dbbeae1d43fd2bbf83cf96b5 100644 --- a/platform/core/sheet_templates.py +++ b/platform/core/sheet_templates.py @@ -145,10 +145,655 @@ TEMPLATES = [ ] +#: Registry grids a template binding may name beside its `ut_*` hint, with their connector. +REGISTRY_GRIDS = {"customer_data": "odoo"} + +# Collection report (R26, R38): the business unit is the customer's Odoo sales team +# (`customer_data.sales_team` = res.partner.team_id), typed once in Summary row 7. +_CR_TEAM = {"B": "$B$7", "C": "$C$7"} +_CR_AR = ( # row, label, customer_data field + (8, "Accounts receivable", "ar_outstanding"), (9, "Not yet due", "ar_open"), + (10, "1 to 30 days", "ar_aged_1_30"), (11, "31 to 60 days", "ar_aged_31_60"), + (12, "61 to 90 days", "ar_aged_61_90"), (13, "Over 90 days", "ar_aged_90_plus"), + (14, "Overdue", "ar_overdue")) +_CR_COD_WORDS = ("immediate", "cod", "cash") # collections_exec._is_cod +# `agent` is absent from the placed-view field set on staging (view_field_unknown), so it is not listed. +_CR_LIST_FIELDS = ["customer", "payment_terms", "ar_overdue", "ar_aged_1_30", "ar_aged_31_60", + "ar_aged_61_90", "ar_aged_90_plus", "ar_outstanding", "days_since"] + + +def _ref(field, op, ref): + return {"colId": field, "op": op, "valueRef": {"ref": ref}} + + +def _leaf(field, op, value=""): + return {"colId": field, "op": op, "value": value} + + +def _collection_report(): + """The biweekly Executive Collection report: Summary plus one overdue list per unit.""" + cells = { + "A1": _literal("Collection report"), + "A2": _literal("As of"), "B2": _formula("=TODAY()"), + "A3": _literal("Period from"), "B3": _formula('=DATEADD(B2,-13,"days")'), + "A4": _literal("Twelve months from"), "B4": _formula('=DATEADD(B2,-364,"days")'), + "A6": _literal("Measure"), "B6": _literal("FFS"), "C6": _literal("RI"), "D6": _literal("Total"), + "A7": _literal("Sales team"), "B7": _literal("Fisch"), "C7": _literal("Royal"), + **{f"A{row}": _literal(label) for row, label, _field in _CR_AR}, + "A15": _literal("Overdue %"), "A16": _literal("Over 30 days"), + "A17": _literal("Sales, last twelve months"), "A18": _literal("DSO (days)"), + "A19": _literal("New sales"), "A20": _literal("New collection"), + "A21": _literal("New sales against collection"), + "A22": _literal("Idle 3 to 6 months"), "A23": _literal("Idle 6 to 12 months"), + "A24": _literal("Idle over 12 months"), + "A25": _literal("COD customers"), "A26": _literal("Customers on terms"), + } + bindings, formats = [], [] + + def bind(addr, database, fn, field, *nodes): + entry = {"sheet": 0, "addr": addr, "database": database, "fn": fn, + "scope": {"filter": list(nodes)}} + if field: + entry["field"] = field + bindings.append(entry) + + for col, team in _CR_TEAM.items(): + in_team = _ref("sales_team", "eq", team) + debtor = _leaf("ar_outstanding", "gt", "1") + for row, _label, field in _CR_AR: + bind(f"{col}{row}", "customer_data", "sum", field, in_team) + cells[f"{col}15"] = _formula(f"=IF({col}8=0,0,{col}14/{col}8)") + cells[f"{col}16"] = _formula(f"={col}11+{col}12+{col}13") + related_team = _ref("related:customer_link:sales_team", "eq", team) + # Posted invoices and credit notes, signed with tax: collections_exec._ltm_invoiced. + bind(f"{col}17", "ut_odoo_invoices", "sum", "amount_total_signed", + _ref("invoice_date", "gte", "$B$4"), _ref("invoice_date", "lte", "$B$2"), related_team) + cells[f"{col}18"] = _formula(f"=IF({col}17=0,0,{col}8/({col}17/365))") + # No wholesale_scope leaf: the orders summary default applies (D-7537). + bind(f"{col}19", "ut_odoo_orders", "sum", "amount_untaxed", + _ref("order_date", "gte", "$B$3"), _ref("order_date", "lte", "$B$2"), related_team) + for row, low, high in ((22, "90", "180"), (23, "180", "360")): + bind(f"{col}{row}", "customer_data", "count", None, in_team, debtor, + _leaf("days_since", "gt", low), _leaf("days_since", "lte", high)) + bind(f"{col}24", "customer_data", "count", None, in_team, debtor, + {"conj": "or", "children": [_leaf("days_since", "gt", "360"), + _leaf("days_since", "isEmpty")]}) + bind(f"{col}25", "customer_data", "count", None, in_team, debtor, + {"conj": "or", "children": [_leaf("payment_terms", "contains", w) for w in _CR_COD_WORDS]}) + bind(f"{col}26", "customer_data", "count", None, in_team, debtor, + *[_leaf("payment_terms", "doesNotContain", w) for w in _CR_COD_WORDS]) + for row in (8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 22, 23, 24, 25, 26): + cells[f"D{row}"] = _formula(f"=B{row}+C{row}") + cells["D15"] = _formula("=IF(D8=0,0,D14/D8)") + cells["D18"] = _formula("=IF(D17=0,0,D8/(D17/365))") + # Payments carry no edge to the customer's team, so collection is a company total. + bind("D20", "ut_odoo_payments", "sum", "amount_company_currency_signed", + _ref("date", "gte", "$B$3"), _ref("date", "lte", "$B$2"), + _leaf("state", "eq", "posted"), _leaf("partner_type", "eq", "customer")) + cells["D21"] = _formula("=IF(D20=0,0,D19/D20)") + for col in "BCD": + formats.extend({"sheet": 0, "addr": f"{col}{row}", "kind": kind} for row, kind in ( + *((r, "currency") for r in (8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 20)), + (15, "percent"), (18, "number"), (21, "number"), + *((r, "number") for r in (22, 23, 24, 25, 26)))) + formats.extend({"sheet": 0, "addr": f"B{row}", "kind": "date"} for row in (2, 3, 4)) + for addr, cell in cells.items(): + if addr[0] == "A" or addr[1:] == "6": + cell["style"] = {"bold": True} + + tabs = [{"name": "Summary", "cells": cells}] + views = [] + for index, (unit, col) in enumerate((("FFS", "B"), ("RI", "C")), start=1): + tabs.append({"name": f"Overdue ({unit})", "cells": { + "A1": {**_literal(f"Overdue customers, {unit}"), "style": {"bold": True}}, + "A2": _literal("Sales team"), "B2": _formula(f"=Summary!{col}7"), + "A3": _literal("As of"), "B3": _formula("=Summary!B2"), + }}) + formats.append({"sheet": index, "addr": "B3", "kind": "date"}) + # The filter reads the typed Summary cell: a formula input reads empty before its first refresh. + views.append({"sheet": index, "addr": "A5", "database": "customer_data", "viewType": "grid", + "fields": list(_CR_LIST_FIELDS), + "filter": {"conj": "and", "nodes": [_ref("sales_team", "eq", f"Summary!${col}$7"), + _leaf("ar_overdue", "gt", "0")]}, + "sort": [{"field": "ar_overdue", "dir": "desc"}], + "limit": COLLECTION_LIST_LIMIT}) + return { + "key": "odoo_collection_report", "title": "Collection report", "source": "odoo", + "database_hint": "ut_odoo_invoices", + "answers": "How much does each business unit owe, how late, and who is overdue?", + "semantic_fields": {"invoice_date": ("invoice_date",), "amount": ("amount_total_signed",), + "customer": ("customer_link",)}, + "sheets": tabs, "bindings": bindings, "formats": formats, "views": views, + } + + +#: The placed-view maximum. The view still reports its true matching total. +COLLECTION_LIST_LIMIT = sheets.VIEW_BINDING_LIMIT_MAX + +TEMPLATES.append(_collection_report()) + + +# ── 3-statement model (S79-T57, R39, R55) ───────────────────────────────────────────────────── +# Monthly P&L, balance sheet and cash flow actuals over the Odoo GL, no projections or drivers. +# The business unit split reads each GL line's `analytic_distribution`. Rows are the tenant's chart +# of accounts, so the TEMPLATES entry is a stub that `apply_template` expands at apply time. +# `modules.three_statement` holds the read-only Odoo oracle and `validate()`. +THREE_STATEMENT = { + "key": "odoo_three_statement", "title": "Three statement model", "source": "odoo", + "database_hint": "ut_odoo_gl_lines", + "answers": "What were monthly profit, balance sheet and cash flow, by business unit?", + "semantic_fields": {"date": ("date",), "amount": ("balance",)}, + "sheets": [], "bindings": [], "formats": [], "expand": "three_statement", +} +TEMPLATES.append(THREE_STATEMENT) + +TS_KEY = "odoo_three_statement" +TS_GL = "ut_odoo_gl_lines" +TS_ACCOUNTS = "ut_odoo_accounts" + +TS_PL_TAB, TS_BS_TAB, TS_CF_TAB = "Profit and loss", "Balance sheet", "Cash flow" + +# (label, account types, sign that turns the Odoo balance into the statement figure) +TS_PL_GROUPS = ( + ("Revenue", ("income",), -1), + ("Cost of revenue", ("expense_direct_cost",), 1), + ("Operating expenses", ("expense",), 1), + ("Depreciation", ("expense_depreciation",), 1), + ("Other income", ("income_other",), -1), +) +TS_ASSET_GROUPS = ( + ("Bank and cash", ("asset_cash",)), ("Receivables", ("asset_receivable",)), + ("Current assets", ("asset_current",)), ("Prepayments", ("asset_prepayments",)), + ("Fixed assets", ("asset_fixed",)), ("Non-current assets", ("asset_non_current",)), +) +TS_LIABILITY_GROUPS = ( + ("Credit cards", ("liability_credit_card",)), ("Payables", ("liability_payable",)), + ("Current liabilities", ("liability_current",)), + ("Non-current liabilities", ("liability_non_current",)), +) +TS_EQUITY_GROUPS = (("Equity", ("equity",)), ("Unallocated earnings", ("equity_unaffected",))) +# Any type not named above lands here, so no chart account is ever dropped. +TS_OTHER_GROUP = ("Off balance and other", ("off_balance",)) +TS_PL_TYPES = tuple(t for _l, types, _s in TS_PL_GROUPS for t in types) +TS_BS_TYPES = tuple(t for _l, types in TS_ASSET_GROUPS + TS_LIABILITY_GROUPS + TS_EQUITY_GROUPS + (TS_OTHER_GROUP,) + for t in types) +# Cash is the balance sheet's bank and cash accounts. Odoo's own cash flow report uses its bank +# journals' accounts instead, a set an account type cannot name. +TS_CASH_GROUPS = ("Bank and cash",) + +# R55: business units are analytic accounts. The typed name in row 5 is the filter operand. +TS_UNITS = (("FFS", "Fisch"), ("RI", "Royal"), ("GD", "Giftware Deals")) +# A line booked 100% to one unit. A line split across units carries no 100 share and falls in Other. +TS_FULL_SHARE = '"percentage": 100' + +TS_MONTHS = 12 +TS_BLOCK = TS_MONTHS + 1 # twelve months and a year total +TS_FIRST = 3 # column C + + +def _ts_col(index): + """Column letters for a 1-based column index.""" + out = "" + while index: + index, rem = divmod(index - 1, 26) + out = chr(65 + rem) + out + return out + + +def _ts_lit(value, **style): + cell = {"kind": "literal", "text": str(value), "value": value} + if style: + cell["style"] = style + return cell + + +def _ts_fx(formula, **style): + cell = {"kind": "formula", "formula": formula} + if style: + cell["style"] = style + return cell + + +def _ts_leaf(field, op, value): + return {"colId": field, "op": op, "value": value} + + +def _ts_ref(field, op, ref): + return {"colId": field, "op": op, "valueRef": {"ref": ref}} + + +TS_POSTED = _ts_leaf("parent_state", "eq", "posted") + + +def ts_default_year(today=None): + """The last full calendar year.""" + return (today or date.today()).year - 1 + + +def ts_month_bounds(year): + """Twelve (first day, last day) ISO pairs for one calendar year.""" + out = [] + for month in range(1, 13): + first = date(year, month, 1) + last = (date(year + month // 12, month % 12 + 1, 1) - timedelta(days=1)) + out.append((first.isoformat(), last.isoformat())) + return out + + +def ts_group_of(account_type): + """The statement group label for an Odoo account type.""" + for label, types, _sign in TS_PL_GROUPS: + if account_type in types: + return label + for label, types in TS_ASSET_GROUPS + TS_LIABILITY_GROUPS + TS_EQUITY_GROUPS: + if account_type in types: + return label + return TS_OTHER_GROUP[0] + + +def ts_read_chart(st): + """The caller's chart of accounts as `[{code, name, account_type}]`, read only from its own Odoo. + + An unbound thread reads tenant #0 through the environment, so the read runs inside the + runtime's own Odoo binding. The runtime is duck-typed: core never imports harness. + """ + from core import odoo as O + source = st.odoo_source() if hasattr(st, "odoo_source") else None + if source is None: + raise TemplateRefusal("template_chart_unreadable", detail="This workspace has no Odoo connection.") + with source._bound(): + rows = O.search_read("account.account", [], ["code", "name", "account_type"]) + return ts_clean_chart(rows) + + +def ts_clean_chart(rows): + """Coded accounts only, one per code, sorted by code.""" + seen, out = set(), [] + for row in rows or (): + code = str(row.get("code") or "").strip() + if not code or code in seen: + continue + seen.add(code) + out.append({"code": code, "name": str(row.get("name") or "").strip() or code, + "account_type": str(row.get("account_type") or "")}) + return sorted(out, key=lambda r: r["code"]) + + +class _TsTab: + def __init__(self, index, name): + self.index, self.name = index, name + self.cells, self.rules = {}, [] + + def put(self, addr, cell): + self.cells[addr] = cell + + +class TsBuilder: + pl_net_row = 16 # the P&L "Net income" summary row + + def __init__(self, chart, year): + self.chart, self.year = chart, year + self.bounds = ts_month_bounds(year) + self.bindings, self.formats = [], [] + self.pl, self.bs, self.cf = _TsTab(0, TS_PL_TAB), _TsTab(1, TS_BS_TAB), _TsTab(2, TS_CF_TAB) + + # ── helpers ──────────────────────────────────────────────────────────────────────────── + def bind(self, tab, addr, database, fn, field, *nodes): + entry = {"sheet": tab.index, "addr": addr, "database": database, "fn": fn, + "scope": {"filter": list(nodes)}} + if field: + entry["field"] = field + self.bindings.append(entry) + + def fmt(self, tab, addr, kind): + self.formats.append({"sheet": tab.index, "addr": addr, "kind": kind}) + + def red_when_nonzero(self, tab, first, last): + tab.rules.append({"range": f"{first}:{last}", "op": "neq", "value": 0, + "style": {"fill": "--lp-red-tint", "color": "--lp-red-deep", "bold": True}}) + + def accounts(self, types): + return [row for row in self.chart if row["account_type"] in types] + + # ── profit and loss ──────────────────────────────────────────────────────────────────── + def pl_block_cols(self, block): + start = TS_FIRST + block * TS_BLOCK + return [_ts_col(start + m) for m in range(TS_MONTHS)], _ts_col(start + TS_MONTHS) + + def build_pl(self): + tab = self.pl + months_total, year_total = self.pl_block_cols(0) + before = _ts_col(TS_FIRST + 5 * TS_BLOCK) + self.pl_before = before + tab.put("A1", _ts_lit("Profit and loss", bold=True)) + tab.put("A2", _ts_lit("Account rows show debit minus credit. Summary rows show income as positive.")) + tab.put("A4", _ts_lit("Block", bold=True)) + tab.put("A5", _ts_lit("Analytic account", bold=True)) + tab.put("A6", _ts_lit("From", bold=True)) + tab.put("A7", _ts_lit("To", bold=True)) + tab.put("B8", _ts_lit("Account", bold=True)) + tab.put("A8", _ts_lit("Code", bold=True)) + blocks = [("Total", None)] + [(unit, name) for unit, name in TS_UNITS] + [("Other", None)] + for b, (title, analytic) in enumerate(blocks): + months, year_col = self.pl_block_cols(b) + tab.put(f"{months[0]}4", _ts_lit(title, bold=True)) + if analytic: + tab.put(f"{months[0]}5", _ts_lit(analytic)) + for m, (first, last) in enumerate(self.bounds): + if b == 0: + tab.put(f"{months[m]}6", _ts_lit(first)) + tab.put(f"{months[m]}7", _ts_lit(last)) + else: + tab.put(f"{months[m]}6", _ts_fx(f"={months_total[m]}6")) + tab.put(f"{months[m]}7", _ts_fx(f"={months_total[m]}7")) + for row in (6, 7): + self.fmt(tab, f"{months[m]}{row}", "date") + tab.put(f"{year_col}6", _ts_lit("Year", bold=True)) + tab.put(f"{before}4", _ts_lit("Earlier", bold=True)) + tab.put(f"{before}6", _ts_lit("Before", bold=True)) + tab.put(f"{before}7", _ts_fx(f"={months_total[0]}6")) + self.fmt(tab, f"{before}7", "date") + + value_cols = [c for b in range(5) for c in (lambda ms, y: ms + [y])(*self.pl_block_cols(b))] + value_cols.append(before) + summary = {"Revenue": 10, "Cost of revenue": 11, "Gross profit": 12, "Operating expenses": 13, + "Depreciation": 14, "Other income": 15, "Net income": 16} + for label, row in summary.items(): + tab.put(f"A{row}", _ts_lit(label, bold=True)) + tab.put("A9", _ts_lit("Summary", bold=True)) + + row = 21 + subtotal_rows = {} + listed = 0 + for label, types, _sign in TS_PL_GROUPS: + tab.put(f"A{row}", _ts_lit(label, bold=True)) + row += 1 + first_account = row + for account in self.accounts(types): + listed += 1 + tab.put(f"A{row}", _ts_lit(account["code"])) + tab.put(f"B{row}", _ts_lit(account["name"])) + self.pl_account_row(tab, row, account["code"], before) + row += 1 + last_account = row - 1 + tab.put(f"B{row}", _ts_lit("Subtotal", bold=True)) + for c in value_cols: + tab.put(f"{c}{row}", _ts_fx(f"=SUM({c}{first_account}:{c}{last_account})" + if last_account >= first_account else "=0", bold=True)) + self.fmt(tab, f"{c}{row}", "currency") + subtotal_rows[label] = row + row += 2 + for c in value_cols: + s = {label: f"{c}{r}" for label, r in subtotal_rows.items()} + tab.put(f"{c}10", _ts_fx(f"=-{s['Revenue']}")) + tab.put(f"{c}11", _ts_fx(f"={s['Cost of revenue']}")) + tab.put(f"{c}12", _ts_fx(f"={c}10-{c}11")) + tab.put(f"{c}13", _ts_fx(f"={s['Operating expenses']}")) + tab.put(f"{c}14", _ts_fx(f"={s['Depreciation']}")) + tab.put(f"{c}15", _ts_fx(f"=-{s['Other income']}")) + tab.put(f"{c}16", _ts_fx(f"={c}12-{c}13-{c}14+{c}15", bold=True)) + for r in range(10, 17): + self.fmt(tab, f"{c}{r}", "currency") + # Check: every chart account of a P&L type has a row. + tab.put("A18", _ts_lit("Chart accounts", bold=True)) + self.bind(tab, "C18", TS_ACCOUNTS, "count", None, _ts_types_any(TS_PL_TYPES)) + self.fmt(tab, "C18", "number") + tab.put("A19", _ts_lit("Chart accounts not listed", bold=True)) + tab.put("C19", _ts_fx(f"=C18-{listed}", bold=True)) + self.fmt(tab, "C19", "number") + self.red_when_nonzero(tab, "C19", "C19") + self.pl_listed = listed + self.pl_rows = row + + def pl_account_row(self, tab, row, code, before): + months_total, year_total = self.pl_block_cols(0) + account = _ts_leaf("account_code", "eq", code) + for b in range(5): + months, year_col = self.pl_block_cols(b) + for m in range(TS_MONTHS): + addr = f"{months[m]}{row}" + dates = (_ts_ref("date", "gte", f"${months_total[m]}$6"), + _ts_ref("date", "lte", f"${months_total[m]}$7")) + if b == 0: + self.bind(tab, addr, TS_GL, "sum", "balance", TS_POSTED, account, *dates) + elif b <= len(TS_UNITS): + unit_col = months[0] + self.bind(tab, addr, TS_GL, "sum", "balance", TS_POSTED, account, *dates, + _ts_ref("analytic_distribution", "contains", f"${unit_col}$5"), + _ts_leaf("analytic_distribution", "contains", TS_FULL_SHARE)) + else: + units = [self.pl_block_cols(u)[0][m] for u in range(1, len(TS_UNITS) + 1)] + tab.put(addr, _ts_fx(f"={months_total[m]}{row}-" + "-".join(f"{u}{row}" for u in units))) + self.fmt(tab, addr, "currency") + tab.put(f"{year_col}{row}", _ts_fx(f"=SUM({months[0]}{row}:{months[-1]}{row})")) + self.fmt(tab, f"{year_col}{row}", "currency") + self.bind(tab, f"{before}{row}", TS_GL, "sum", "balance", TS_POSTED, account, + _ts_ref("date", "lt", f"${months_total[0]}$6")) + self.fmt(tab, f"{before}{row}", "currency") + + # ── balance sheet ────────────────────────────────────────────────────────────────────── + def bs_cols(self): + """Opening column C, then one column per month end.""" + return [_ts_col(TS_FIRST + i) for i in range(TS_MONTHS + 1)] + + def build_bs(self): + tab = self.bs + cols = self.bs_cols() + opening = (date(self.year, 1, 1) - timedelta(days=1)).isoformat() + tab.put("A1", _ts_lit("Balance sheet", bold=True)) + tab.put("A2", _ts_lit("Account rows show debit minus credit. Summary rows show balances as positive.")) + tab.put("A6", _ts_lit("As of", bold=True)) + tab.put("A5", _ts_lit("Column", bold=True)) + tab.put("C5", _ts_lit("Opening", bold=True)) + for i, c in enumerate(cols): + tab.put(f"{c}6", _ts_lit(opening if i == 0 else self.bounds[i - 1][1])) + self.fmt(tab, f"{c}6", "date") + tab.put("A8", _ts_lit("Code", bold=True)) + tab.put("B8", _ts_lit("Account", bold=True)) + + row = 30 + sub = {} + listed = 0 + sections = (("Assets", TS_ASSET_GROUPS), ("Liabilities", TS_LIABILITY_GROUPS), + ("Equity", TS_EQUITY_GROUPS), ("Off balance", (TS_OTHER_GROUP,))) + known = set(TS_BS_TYPES) | set(TS_PL_TYPES) + for title, groups in sections: + tab.put(f"A{row}", _ts_lit(title, bold=True)) + row += 1 + for label, types in groups: + accounts = self.accounts(types) + if label == TS_OTHER_GROUP[0]: + accounts += [a for a in self.chart if a["account_type"] not in known] + tab.put(f"A{row}", _ts_lit(label, bold=True)) + row += 1 + first_account = row + for account in accounts: + listed += 1 + tab.put(f"A{row}", _ts_lit(account["code"])) + tab.put(f"B{row}", _ts_lit(account["name"])) + for c in cols: + self.bind(tab, f"{c}{row}", TS_GL, "sum", "balance", TS_POSTED, + _ts_leaf("account_code", "eq", account["code"]), + _ts_ref("date", "lte", f"${c}$6")) + self.fmt(tab, f"{c}{row}", "currency") + row += 1 + last_account = row - 1 + tab.put(f"B{row}", _ts_lit("Subtotal", bold=True)) + for c in cols: + tab.put(f"{c}{row}", _ts_fx(f"=SUM({c}{first_account}:{c}{last_account})" + if last_account >= first_account else "=0", bold=True)) + self.fmt(tab, f"{c}{row}", "currency") + sub[label] = row + row += 2 + self.bs_sub = sub + self.bs_rows = row + + labels = {9: "Total assets", 10: "Total liabilities", 11: "Equity accounts", + 12: "Earnings to date", 13: "Total equity", 14: "Liabilities and equity", + 16: "Balance check", 17: "Sum of posted lines", 18: "Out of balance in Odoo", + 19: "Accounts check", 21: "Chart accounts", 22: "Chart accounts not listed"} + tab.put("A15", _ts_lit("Checks", bold=True)) + for r, label in labels.items(): + tab.put(f"A{r}", _ts_lit(label, bold=True)) + months_total, _year = self.pl_block_cols(0) + net = self.pl_net_row + for i, c in enumerate(cols): + ref = lambda labels_: "+".join(f"{c}{sub[l]}" for l, _t in labels_) + tab.put(f"{c}9", _ts_fx(f"={ref(TS_ASSET_GROUPS)}", bold=True)) + tab.put(f"{c}10", _ts_fx(f"=-({ref(TS_LIABILITY_GROUPS)})", bold=True)) + tab.put(f"{c}11", _ts_fx(f"=-({ref(TS_EQUITY_GROUPS)})")) + if i == 0: + tab.put(f"{c}12", _ts_fx(f"='{TS_PL_TAB}'!{self.pl_before}{net}")) + else: + tab.put(f"{c}12", _ts_fx(f"={cols[i - 1]}12+'{TS_PL_TAB}'!{months_total[i - 1]}{net}")) + tab.put(f"{c}13", _ts_fx(f"={c}11+{c}12", bold=True)) + tab.put(f"{c}14", _ts_fx(f"={c}10+{c}13", bold=True)) + # Assets less liabilities and equity. Odoo's own unbalanced entries show here. + tab.put(f"{c}16", _ts_fx(f"=ROUND({c}9-{c}14,2)", bold=True)) + # Every posted line: zero in a balanced ledger. Rounded below, so float dust never paints red. + self.bind(tab, f"{c}17", TS_GL, "sum", "balance", TS_POSTED, _ts_ref("date", "lte", f"${c}$6")) + tab.put(f"{c}18", _ts_fx(f"=ROUND({c}17,2)", bold=True)) + # Zero unless a posted balance sits on an account with no row. + tab.put(f"{c}19", _ts_fx(f"=ROUND({c}16-{c}17+{c}{sub[TS_OTHER_GROUP[0]]},2)", bold=True)) + for r in (9, 10, 11, 12, 13, 14, 16, 17, 18, 19): + self.fmt(tab, f"{c}{r}", "currency") + # Every account that is not a P&L type, unknown types included. + self.bind(tab, "C21", TS_ACCOUNTS, "count", None, + *[_ts_leaf("account_type", "neq", t) for t in TS_PL_TYPES]) + self.fmt(tab, "C21", "number") + tab.put("C22", _ts_fx(f"=C21-{listed}", bold=True)) + self.fmt(tab, "C22", "number") + last = cols[-1] + for r in (16, 18, 19): + self.red_when_nonzero(tab, f"C{r}", f"{last}{r}") + self.red_when_nonzero(tab, "C22", "C22") + self.bs_listed = listed + + # ── cash flow (indirect, from balance sheet movements) ───────────────────────────────── + CF_ROWS = ( + ("Net income", "net"), ("Receivables", ("Receivables",)), + ("Current assets", ("Current assets",)), ("Prepayments", ("Prepayments",)), + ("Payables", ("Payables",)), ("Credit cards", ("Credit cards",)), + ("Current liabilities", ("Current liabilities",)), + ("Cash from operating activities", "subtotal"), + ("Fixed assets", ("Fixed assets",)), ("Non-current assets", ("Non-current assets",)), + ("Cash from investing activities", "subtotal"), + ("Non-current liabilities", ("Non-current liabilities",)), + ("Equity accounts", ("Equity", "Unallocated earnings")), + ("Cash from financing activities", "subtotal"), + ("Off balance and other", (TS_OTHER_GROUP[0],)), + ("Net change in cash", "change"), ("Cash at start", "start"), ("Cash at end", "end"), + ("Cash check", "check"), + ) + + def build_cf(self): + tab = self.cf + bs = self.bs_cols() + months_total, _year = self.pl_block_cols(0) + sub = self.bs_sub + year_col = _ts_col(TS_FIRST + TS_MONTHS + 1) + tab.put("A1", _ts_lit("Cash flow", bold=True)) + tab.put("A2", _ts_lit("Indirect method from balance sheet movements. Cash is the bank and cash accounts.")) + tab.put("A6", _ts_lit("Month ending", bold=True)) + tab.put(f"{year_col}6", _ts_lit("Year", bold=True)) + at = {label: 8 + i for i, (label, _how) in enumerate(self.CF_ROWS)} + subtotals = [at[label] for label, how in self.CF_ROWS if how == "subtotal"] + cash = lambda c: "+".join(f"'{TS_BS_TAB}'!{c}{sub[g]}" for g in TS_CASH_GROUPS) # noqa: E731 + for label, how in self.CF_ROWS: + tab.put(f"A{at[label]}", _ts_lit(label, bold=not isinstance(how, tuple))) + for i in range(1, TS_MONTHS + 1): + c, prev = bs[i], bs[i - 1] + tab.put(f"{c}6", _ts_fx(f"='{TS_BS_TAB}'!{c}6")) + self.fmt(tab, f"{c}6", "date") + section_start = at["Net income"] + for label, how in self.CF_ROWS: + r = at[label] + if how == "net": + formula = f"='{TS_PL_TAB}'!{months_total[i - 1]}{self.pl_net_row}" + elif isinstance(how, tuple): + now = "+".join(f"'{TS_BS_TAB}'!{c}{sub[g]}" for g in how) + was = "+".join(f"'{TS_BS_TAB}'!{prev}{sub[g]}" for g in how) + formula = f"=-(({now})-({was}))" + elif how == "subtotal": + formula = f"=SUM({c}{section_start}:{c}{r - 1})" + section_start = r + 1 + elif how == "change": + formula = "=" + "+".join(f"{c}{x}" for x in subtotals) + f"+{c}{at['Off balance and other']}" + elif how == "start": + formula = f"={cash(prev)}" + elif how == "end": + formula = f"={cash(c)}" + else: + formula = (f"=ROUND({c}{at['Net change in cash']}-({c}{at['Cash at end']}" + f"-{c}{at['Cash at start']}),2)") + tab.put(f"{c}{r}", _ts_fx(formula, bold=not isinstance(how, tuple))) + self.fmt(tab, f"{c}{r}", "currency") + first, last = bs[1], bs[-1] + for label, how in self.CF_ROWS: + r = at[label] + if how == "start": + formula = f"={first}{r}" + elif how == "end": + formula = f"={last}{r}" + elif how == "check": + y = year_col + formula = (f"=ROUND({y}{at['Net change in cash']}-({y}{at['Cash at end']}" + f"-{y}{at['Cash at start']}),2)") + else: + formula = f"=SUM({first}{r}:{last}{r})" + tab.put(f"{year_col}{r}", _ts_fx(formula, bold=True)) + self.fmt(tab, f"{year_col}{r}", "currency") + self.red_when_nonzero(tab, f"{first}{at['Cash check']}", f"{year_col}{at['Cash check']}") + + def descriptor(self): + self.build_pl() + self.build_bs() + self.build_cf() + tabs = [{"name": t.name, "cells": t.cells, "rules": t.rules} for t in (self.pl, self.bs, self.cf)] + stub = {key: value for key, value in THREE_STATEMENT.items() if key != "expand"} + return {**stub, "sheets": tabs, "bindings": self.bindings, "formats": self.formats, + "year": self.year, "accounts": {"pl": self.pl_listed, "bs": self.bs_listed}} + + +def _ts_types_any(types): + """An OR group of `account_type eq` leaves.""" + return {"conj": "or", "children": [_ts_leaf("account_type", "eq", t) for t in types]} + + + + +def three_statement_descriptor(chart, year=None): + """The full 3-statement template for one chart of accounts and one calendar year.""" + return TsBuilder(ts_clean_chart(chart), year or ts_default_year()).descriptor() + + +def expanded(descriptor, st): + """A stub descriptor built out for the runtime `st`; any other descriptor unchanged.""" + if descriptor.get("expand") != "three_statement": + return descriptor + try: + chart = ts_read_chart(st) + except TemplateRefusal: + raise + except Exception as exc: # noqa: BLE001 + raise TemplateRefusal("template_chart_unreadable", + detail="The chart of accounts could not be read. Try again after the next sync.") from exc + if not chart: + raise TemplateRefusal("template_chart_unreadable", detail="The chart of accounts is empty.") + return three_statement_descriptor(chart) + + def template(key): return next((row for row in TEMPLATES if row["key"] == key), None) +def template_databases(descriptor): + """Every Cell database a built-in descriptor binds, the hint included.""" + named = {entry["database"] for entry in descriptor["bindings"] if "database" in entry} + named.update(entry["database"] for entry in descriptor.get("views", ())) + return named | {descriptor["database_hint"]} + + def templates_for_sources(sources): allowed = set(sources) return [row for row in TEMPLATES if row["source"] in allowed] @@ -302,19 +947,33 @@ def build_cells(template, resolved, database_key): for entry in template["bindings"]: cells, addr = target(entry) - if set(entry) - {"sheet", "addr", "fn", "field", "metric_key", "scope", "period"}: + if set(entry) - {"sheet", "addr", "fn", "field", "metric_key", "scope", "period", "database"}: raise TemplateRefusal("template_invalid", detail="The template binding has an unknown setting.") - aggregate = ({"metric_key": entry["metric_key"]} if "metric_key" in entry else - {"fn": entry["fn"], "field": resolved[entry["field"]]}) + # A binding naming its own database uses that database's field keys as written. + own = "database" in entry + # verify_templates.py mutates this exact expression; keep its first clause verbatim. + aggregate = ({"metric_key": entry["metric_key"]} if "metric_key" in entry + else {"fn": entry["fn"], **({"field": entry["field"]} if "field" in entry else {})} if own + else {"fn": entry["fn"], "field": resolved[entry["field"]]}) if "metric_key" in entry and (not entry["metric_key"] or "field" in entry or "fn" in entry or not entry.get("period")): raise TemplateRefusal("template_invalid", detail="A template metric needs a period and no field aggregate.") - binding = {"database": database_key, + scope = entry.get("scope", {"filter": []}) + binding = {"database": entry["database"] if own else database_key, "aggregate": aggregate, - "scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)} + "scope": deepcopy(scope) if own else _resolved_scope(scope, resolved)} if "period" in entry: binding["period"] = deepcopy(entry["period"]) cells[addr] = {**cells.get(addr, {}), "kind": "bound", "binding": clean_binding(binding)} + for entry in template.get("views", ()): + # A placed view: Sheet reads its rows live through POST /sheets/{id}/view-rows. + cells, addr = target(entry) + raw = {key: deepcopy(value) for key, value in entry.items() if key not in ("sheet", "addr")} + try: + binding = sheets.clean_view_binding({**raw, "mode": "view"}) + except sheets.SheetRefusal as exc: + raise TemplateRefusal("template_invalid", detail=exc.message) from exc + cells[addr] = {"kind": "bound", "binding": binding} for entry in template["formats"]: cells, addr = target(entry) cell = cells.setdefault(addr, {"kind": "empty"}) @@ -328,6 +987,12 @@ def build_cells(template, resolved, database_key): return tabs +def _tab_rules(descriptor, index): + """A descriptor tab's conditional format rules, when it has any.""" + rules = descriptor["sheets"][index].get("rules") + return {"rules": deepcopy(rules)} if rules else {} + + def apply_template(st, key, database_key, *, name="", created_by=""): descriptor = template(key) if descriptor is None: @@ -337,12 +1002,22 @@ def apply_template(st, key, database_key, *, name="", created_by=""): raise TemplateRefusal("unknown_database", detail="That database does not exist.") if source_of_database(database_key) != descriptor["source"]: raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.") + descriptor = expanded(descriptor, st) field_keys = [field["key"] for field in definition.get("fields", [])] resolved = resolve_fields(descriptor, field_keys) + defs = user_tables.all_defs(st) + for named in template_databases(descriptor) - {database_key}: + source = REGISTRY_GRIDS.get(named) or source_of_database(named) + if named not in REGISTRY_GRIDS and named not in defs: + raise TemplateRefusal("unknown_database", detail=f"This template needs the database {named}.") + if source != descriptor["source"]: + raise TemplateRefusal("template_source_mismatch", + detail="That database uses a different connector from this template.") prepared = build_cells(descriptor, resolved, database_key) - # Validate tab names, extents and every cell before the first store write. + # Validate tab names, extents, rules and every cell before the first store write. checked = sheets.migrate_workbook({"sheets": [ - {"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]}) + {"name": tab_name, "cells": cells, **_tab_rules(descriptor, index)} + for index, tab_name, cells in prepared]}) book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by, on_duplicate="suffix") try: diff --git a/platform/core/store.py b/platform/core/store.py index 64a0ed20b47d8b4eeb74a629389628dabf516963..967ff7f8cdea425a9eac24c963ed3bede09b216e 100644 --- a/platform/core/store.py +++ b/platform/core/store.py @@ -1274,8 +1274,37 @@ def flush(name=None, timeout=30.0): return _d().flush(name=name, timeout=timeout) -def update(name, fn, flush='sync'): - return _d().update(name, fn, flush=flush) +def update(name, fn, flush='sync', expect_rev=None, audit_row=None): + """`update` on the active backend. `expect_rev` and `audit_row` forward to a handle that names them. + + A handle that does not (the HF store) keeps both promises here: an advisory revision check, + and the audit rows appended to the `__audit__` document ring inside `fn` (D-635). + """ + target = _d() + if expect_rev is None and audit_row is None: + return target.update(name, fn, flush=flush) + from core import view_audit + if view_audit.guards(target) and view_audit.home(target) == view_audit.CONTROL_AUDIT: + return target.update(name, fn, flush=flush, expect_rev=expect_rev, audit_row=audit_row) + if expect_rev is not None: + try: + current = int((target.revision(name) or {}).get('rev') or 0) + except Exception as exc: # noqa: BLE001 + raise view_audit.RevisionUnverifiable( + 'the revision could not be read, so the change was not saved. ' + 'Reload and try again.') from exc + if current != int(expect_rev): + raise view_audit.ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) + if audit_row is not None: + inner, rows = fn, audit_row + + def fn(data): + result = inner(data) + for entry in (rows if isinstance(rows, list) else [rows]): + if entry: + view_audit.append_to_document(result if isinstance(result, dict) else data, entry) + return result + return target.update(name, fn, flush=flush) def get_projection(name, drop=()): diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index d67d3871bb429c8f63d357634ccdecce27ab9f4a..7eeadeeae182683da3ff766e091333e4b3f97d2a 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -48,7 +48,7 @@ import tempfile import threading import time from collections import OrderedDict -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from functools import lru_cache from pathlib import Path from urllib.parse import urlparse @@ -56,6 +56,7 @@ from urllib.parse import urlparse import core.data_binding as data_binding from core import freeze_notice from core import request_timing as _request_timing +from core import user_table_rows as _utr from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER _URL_ENV = 'DATABASE_URL' @@ -378,6 +379,28 @@ def _holder_pool_class(base, timeout_error): f"{int(limits['idle'] * 1000)}ms")) yield con + @contextmanager + def autocommit(self, timeout=None): + """A pooled connection in autocommit mode: the caller sends BEGIN, settings and COMMIT.""" + with super().connection(timeout=timeout) as con: + _request_timing.remaining_budget() + con.autocommit = True + try: + yield con + finally: + _end_autocommit(con) + + def read_rows(self, query, params=None, timeout=None): + """Every row of one read in one round trip: BEGIN, settings, the statement, COMMIT.""" + with self.autocommit(timeout=timeout) as con: + # One Sync at pipeline exit; an error raises there with its own psycopg type. + with con.pipeline(): + con.execute('BEGIN') + con.execute(_SETTINGS_SQL, _settings_params()) + cur = con.execute(query, params) + con.execute('COMMIT') + return cur.fetchall() + def _log_exhausted(self): now = time.monotonic() with self._holders_lock: @@ -392,6 +415,39 @@ def _holder_pool_class(base, timeout_error): return HolderPool +#: Transaction-local limits every pooled transaction sets (`set_config(..., true)` survives PgBouncer). +_SETTINGS_SQL = ("SELECT set_config('statement_timeout', %s, true), " + "set_config('idle_in_transaction_session_timeout', %s, true)") + + +def _settings_params(): + limits = _pg_limits() + return f"{int(limits['statement'] * 1000)}ms", f"{int(limits['idle'] * 1000)}ms" + + +def _end_autocommit(con): + """Roll back a transaction an autocommit borrower left open, then hand the pool its default.""" + if con.closed: + return + try: + from psycopg.pq import TransactionStatus + if con.info.transaction_status != TransactionStatus.IDLE: + con.execute('ROLLBACK') + con.autocommit = False + except Exception: # noqa: BLE001 + con.close() # the pool discards a closed connection + + +def _read(pool, query, params=None, *, one=False): + """One read statement: one round trip on this module's pool, a plain pooled read on a stub.""" + if getattr(pool, '_aios_pipelines', False) is True: + rows = pool.read_rows(query, params) + return (rows[0] if rows else None) if one else rows + with pool.connection() as con: + cur = con.execute(query, params) + return cur.fetchone() if one else cur.fetchall() + + def _lock_timeout_s(): """Seconds `update` waits for a row lock: `AIOS_STORE_LOCK_TIMEOUT_S`, default 10, floor 1.""" try: @@ -510,6 +566,7 @@ def _pool(): _POOL['pool'] = pool_cls(url, min_size=0, max_size=8, open=True, timeout=limits['acquire'], max_idle=30.0, max_waiting=limits['waiting'], kwargs=connection_kwargs, connection_class=_budget_connection_class(Connection)) + _POOL['pool']._aios_pipelines = True # a real libpq pool: `_read` may pipeline _POOL['url'] = url _POOL['sslrootcert'] = sslrootcert return _POOL['pool'] @@ -607,6 +664,9 @@ def _bump_write_generation(name, tenant_slug): for content_key in list(_WHOLE_TEXT): if content_key[1:] == key: _forget_whole(content_key) + with _KV_LOCK: + for cache_key in [k for k in _KV if k[1:] == key]: + _kv_forget(cache_key) def _shared_read(name, drops, tenant_slug, fetch_text, *, pool=None): @@ -682,6 +742,29 @@ def _parse_read(name, operation, text): _request_timing.record_store(operation, str(name), elapsed, count=0) +def _ut_rows(name, tenant_slug): + """True when `name` is a user-table document this tenant stores as rows (S79-T47).""" + return str(name).endswith('user_tables') and _utr.rows_mode(name, tenant_slug) + + +def _ut_marker(name, tenant_slug, value): + """True when a document-mode read found the row-mode marker; caches row mode for the next call.""" + if str(name).endswith('user_tables') and _utr.is_marker(value): + _utr.remember_mode(name, tenant_slug) + return True + return False + + +def _rows_projection(name, drop, tenant_slug): + """`get_projection` in row mode: definition rows when `rows` is dropped, else the whole read.""" + import core.store as _hf + dropped = frozenset(str(d) for d in drop) + data = (_utr.read_definitions(name, tenant_slug, dropped) if 'rows' in dropped + else _utr.read_document(name, tenant_slug)) + return {key: _hf._Projected({k: v for k, v in value.items() if k not in dropped}, dropped) + if isinstance(value, dict) else value for key, value in data.items()} + + def _request_key(name, shape, tenant_slug, pool=None): """Address a read shape by pool, tenant and local write generation.""" pool = _pool() if pool is None else pool @@ -736,7 +819,20 @@ def _validated_content(name, drops, tenant_slug): def get(name, fresh=False, tenant_slug=None): - """Parse an independent document from this request's backing read or a new validated read.""" + """Parse an independent document from this request's backing read or a new validated read. + + Row mode: a `fresh` read assembles the whole document now. Otherwise the answer is a detached + `RowsDocument` that reads one table per first access and the rest in one statement. + """ + if _ut_rows(name, tenant_slug): + if fresh: + return _utr.read_document(name, tenant_slug, fresh=True) + return _utr.lazy_document(name, tenant_slug) + if not fresh: + hit, text = _kv_get(name, tenant_slug) + if hit: + return _parse_read(name, 'get', text) + def read(): found, text = (False, None) if fresh or str(name).endswith('user_tables') else ( _validated_content(name, None, tenant_slug)) @@ -744,7 +840,10 @@ def get(name, fresh=False, tenant_slug=None): # Whole row documents never become request metadata snapshots. text = read() if str(name).endswith('user_tables') and not fresh else _request_text( name, 'whole', tenant_slug, read, fresh=fresh) - return _parse_read(name, 'get', text) + data = _parse_read(name, 'get', text) + if _ut_marker(name, tenant_slug, data): + return _utr.read_document(name, tenant_slug, fresh=True) + return data @_request_timing.store_operation("get") @@ -767,14 +866,13 @@ def _get_text(name, fresh=False, tenant_slug=None): cached = _WHOLE_TEXT.get(content_key) if not fresh else None if cached is not None: _WHOLE_TEXT.move_to_end(content_key) - with pool.connection() as con: - row = con.execute( - _q('SELECT kv.rev, kv.updated_at, CASE ' - 'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ' - 'ELSE kv.value::text END FROM {tbl} AS kv WHERE kv.key = %s', - 'store_kv', tenant_slug), - (cached[0] if cached else None, cached[1] if cached else None, - str(name))).fetchone() + started = time.monotonic() + row = _read(pool, _q('SELECT kv.rev, kv.updated_at, CASE ' + 'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ' + 'ELSE kv.value::text END FROM {tbl} AS kv WHERE kv.key = %s', + 'store_kv', tenant_slug), + (cached[0] if cached else None, cached[1] if cached else None, + str(name)), one=True) text = row[2] if row else None _request_timing.record_store("get", str(name), bytes_read=_request_timing.text_bytes(text), count=0) if row and text is None: @@ -783,6 +881,8 @@ def _get_text(name, fresh=False, tenant_slug=None): text = cached[2] _remember_whole(content_key, generation, row[0] if row else None, row[1] if row else None, text) + _kv_fill(content_key, generation, started, row[0] if row else None, + row[1] if row else None, text) return text, row[0] if row else None, row[1] if row else None text, rev, stamp = fetch() if fresh else _shared_read(name, None, tenant_slug, fetch, pool=pool) @@ -824,6 +924,219 @@ def _remember_whole(key, generation, revision, stamp, text): _forget_whole(next(iter(_WHOLE_TEXT))) +# -- the small-value process cache (S79-T49 pass 4) ------------------------------------------- +#: Bytes one cached value may hold; None reads `AIOS_KV_CACHE_VALUE_KB` (default 256). +KV_CACHE_VALUE_BYTES = None +#: Bytes the cache may hold in all; None reads `AIOS_KV_CACHE_MB` (default 16). 0 turns it off. +KV_CACHE_BYTES = None +#: Seconds a request trusts its validation before its next cached read validates again. +KV_CACHE_VALID_S = 2.0 +_KV = OrderedDict() # (pool, slug, name) -> _KvEntry, least recent first +_KV_LOCK = threading.Lock() +_KV_BYTES = [0] + + +class _KvEntry: + """One value's text and the `(rev, updated_at)` it was read at; `checked` is a monotonic time.""" + __slots__ = ('rev', 'stamp', 'text', 'size', 'generation', 'checked') + + def __init__(self, rev, stamp, text, size, generation, checked): + self.rev, self.stamp, self.text = rev, stamp, text + self.size, self.generation, self.checked = size, generation, checked + + +def _kv_limits(): + """`(bytes per value, bytes in all)` of the small-value cache.""" + value, total = KV_CACHE_VALUE_BYTES, KV_CACHE_BYTES + if value is None: + value = int(_bound_env('AIOS_KV_CACHE_VALUE_KB', 256, 0) * 1024) + if total is None: + total = int(_bound_env('AIOS_KV_CACHE_MB', 16, 0) * 1048576) + return value, total + + +def _kv_forget(key): + """Remove one entry while holding `_KV_LOCK`.""" + entry = _KV.pop(key, None) + if entry is not None: + _KV_BYTES[0] -= entry.size + + +def _kv_scope(): + """`(accumulator, pool)` when this read may use the cache: a request before its response starts. + + A streamed body (an event stream polls the store) and any read outside a request read directly. + """ + acc = _request_timing.CURRENT.get() + if (acc is None or acc.closed or acc.cancelled or getattr(acc, 'read_memo_ended', True) + or _kv_limits()[1] <= 0): + return None + pool = _pool() + if getattr(pool, '_aios_pipelines', False) is not True: + return None + return acc, pool + + +@_request_timing.store_operation('kv_check') +def _kv_check(label, names, pool, tenant_slug): + """`[(key, rev, updated_at)]` of `names` in one statement.""" + return _read(pool, _q('SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])', + 'store_kv', tenant_slug), (list(names),)) + + +def _kv_validate(acc, pool, tenant_slug, marker=None): + """`(since, {marker: (token, generation)})`: this request's one check of a tenant's entries. + + One statement reads `(rev, updated_at)` of every cached key and the `user_tables` marker. An + entry that still matches is good for the rest of the request; any other entry is dropped. + """ + slug = _effective_slug(tenant_slug) + now = time.monotonic() + with acc.lock: + checks = getattr(acc, 'kv_checks', None) + if checks is None: + checks = acc.kv_checks = {} + held = checks.get((pool, slug)) + if (held is not None and now - held[0] <= KV_CACHE_VALID_S + and (marker is None or str(marker) in held[1])): + return held + with _KV_LOCK: + names = [k[2] for k in _KV if k[0] is pool and k[1] == slug] + if not names and marker is None: + held = (now, {}) + else: + markers = {_utr.DOCUMENT_KEY} | ({str(marker)} if marker is not None else set()) + with _READ_FLIGHT_LOCK: + generations = {m: _WRITE_GENERATION.get((slug, m), 0) for m in markers} + started = time.monotonic() + live = {str(k): (rev, stamp) for k, rev, stamp in + _kv_check('kv_check', sorted(set(names) | markers), pool, tenant_slug)} + with _KV_LOCK: + for name in names: + entry = _KV.get((pool, slug, name)) + if entry is None: + continue + if live.get(name, (None, None)) == (entry.rev, entry.stamp): + entry.checked = max(entry.checked, started) + else: + _kv_forget((pool, slug, name)) + tokens = {} + for m in markers: + rev, stamp = live.get(m, (None, None)) + ok = rev and hasattr(stamp, 'timestamp') + tokens[m] = ((int(rev), stamp.timestamp()) if ok else None, generations[m]) + held = (started, tokens) + with acc.lock: + checks[(pool, slug)] = held + return held + + +#: The text of an entry a `revision` read made: it answers `revision`, never `get`. +_NO_TEXT = object() + + +def _kv_entry(name, tenant_slug): + """This request's checked entry for `name`, or None.""" + if str(name).endswith('user_tables'): + return None + scope = _kv_scope() + if scope is None: + return None + acc, pool = scope + since = _kv_validate(acc, pool, tenant_slug)[0] + doc = (_effective_slug(tenant_slug), str(name)) + with _READ_FLIGHT_LOCK: + generation = _WRITE_GENERATION.get(doc, 0) + with _KV_LOCK: + entry = _KV.get((pool,) + doc) + if entry is None or entry.generation != generation or entry.checked < since: + return None + _KV.move_to_end((pool,) + doc) + return entry + + +def _kv_get(name, tenant_slug): + """`(True, text)` when this request may serve `name` from the cache, else `(False, None)`.""" + entry = _kv_entry(name, tenant_slug) + if entry is None or entry.text is _NO_TEXT: + return False, None + return True, entry.text + + +def _kv_revision(name, tenant_slug): + """`revision`'s answer from this request's checked entry or held marker, or None.""" + if str(name).endswith('user_tables'): + if _kv_scope() is None or not _ut_rows(name, tenant_slug): + return None + token = _utr.held_marker(name, tenant_slug) + if token is None: + checked = _kv_marker(name, tenant_slug) + with _READ_FLIGHT_LOCK: + generation = _WRITE_GENERATION.get((_effective_slug(tenant_slug), str(name)), 0) + token = checked[0] if checked is not None and checked[1] == generation else None + if token is None: + return None + return {'rev': int(token[0]), 'updated_at': token[1], 'token': f'pg:{int(token[0])}'} + entry = _kv_entry(name, tenant_slug) + return None if entry is None else _revision_value(entry.rev, entry.stamp) + + +def _kv_note(key, generation, started, rev, stamp): + """Record what a `revision` read saw: confirm a matching entry, else keep the revision alone.""" + pool = key[0] + if (_kv_limits()[1] <= 0 or getattr(pool, '_aios_pipelines', False) is not True + or key[2].endswith('user_tables')): + return + with _READ_FLIGHT_LOCK: + if _WRITE_GENERATION.get(key[1:], 0) != generation: + return + with _KV_LOCK: + entry = _KV.get(key) + if entry is not None and (entry.rev, entry.stamp) == (rev, stamp): + entry.checked = max(entry.checked, started) + return + _kv_forget(key) + size = len(key[2]) + 64 + _KV[key] = _KvEntry(rev, stamp, _NO_TEXT, size, generation, started) + _KV_BYTES[0] += size + while _KV_BYTES[0] > _kv_limits()[1] and _KV: + _kv_forget(next(iter(_KV))) + + +def _kv_marker(name, tenant_slug): + """`(token, generation)` of a row-mode marker from this request's validation, or None.""" + scope = _kv_scope() + if scope is None: + return None + acc, pool = scope + return _kv_validate(acc, pool, tenant_slug, marker=str(name))[1].get(str(name)) + + +def _kv_fill(key, generation, started, rev, stamp, text): + """Keep one plain read when it is small and no write in this process moved its key since.""" + pool = key[0] + value_limit, total = _kv_limits() + if (total <= 0 or getattr(pool, '_aios_pipelines', False) is not True + or key[2].endswith('user_tables') or (rev is None) != (text is None) + or (text is not None and (not isinstance(text, str) or len(text) > value_limit))): + return + size = len(text.encode('utf-8')) if text is not None else 0 + if size > value_limit: + return + size += len(key[2]) + 64 + with _READ_FLIGHT_LOCK: + if _WRITE_GENERATION.get(key[1:], 0) != generation: + return + with _KV_LOCK: + _kv_forget(key) + if size > total: + return + _KV[key] = _KvEntry(rev, stamp, text, size, generation, started) + _KV_BYTES[0] += size + while _KV_BYTES[0] > total and _KV: + _kv_forget(next(iter(_KV))) + + _PROJECTION_TEXT = OrderedDict() _PROJECTION_TEXT_LOCK = threading.Lock() _PROJECTION_TEXT_BYTES = 0 @@ -852,6 +1165,8 @@ def _remember_projection(key, revision, stamp, text): def get_projection(name, drop=(), tenant_slug=None): """Parse independent projected definitions from the current request's backing read.""" import core.store as _hf + if _ut_rows(name, tenant_slug): + return _rows_projection(name, drop, tenant_slug) drops = tuple(sorted(set(str(d) for d in drop))) can_memo = not str(name).endswith('user_tables') or 'rows' in drops def read(): @@ -860,6 +1175,8 @@ def get_projection(name, drop=(), tenant_slug=None): text = (read() if not can_memo else _request_text(name, ('projection', drops), tenant_slug, read)) data = _parse_read(name, 'get_projection', text) + if _ut_marker(name, tenant_slug, data): + return _rows_projection(name, drop, tenant_slug) dropped = frozenset(drops) if not isinstance(text, (str, bytes, bytearray)): return _hf._project(data, dropped) @@ -1044,8 +1361,16 @@ def _list_paths(prefix='', *, tenant_slug=None): def _delete_document(name, *, tenant_slug=None): check_write(tenant_slug, f'delete document {name!r}') + rows = _ut_rows(name, tenant_slug) + if rows: + _utr.settle_lazy(name, tenant_slug) with _pool().connection() as con: + if rows: + # Every table goes with the document; the empty row tables stay for a later migration. + _utr.clear_rows(con, tenant_slug) con.execute(_q('DELETE FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), (str(name),)) + if rows: + _utr.remember_mode(name, tenant_slug, value=False) _bump_write_generation(name, tenant_slug) return True @@ -1072,6 +1397,7 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): if not keys: return {} drops = tuple(str(d) for d in drop) + rows_keys = {key for key in keys if _ut_rows(key, tenant_slug)} pool = _pool() request_keys = {key: _request_key(key, 'revision', tenant_slug, pool) for key in keys} checkpoint = _request_timing.memo_checkpoint() @@ -1099,6 +1425,9 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): bytes_read=_request_timing.text_bytes(value), count=0) found[str(key)] = _parsed(value) _seed_read_token(request_keys[str(key)], checkpoint, rev, stamp) + if str(key) in rows_keys or _ut_marker(key, tenant_slug, found[str(key)]): + rows_keys.add(str(key)) # the marker row carries the revision, never the content + continue if not str(key).endswith('user_tables') or 'rows' in drops: shape_key = (*request_keys[str(key)][:3], ('projection', tuple(sorted(set(drops)))), request_keys[str(key)][-1]) @@ -1114,7 +1443,7 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): request_keys[key][-1]), None, checkpoint) dropped = frozenset(drops) return { - key: _hf._project( + key: _rows_projection(key, drops, tenant_slug) if key in rows_keys else _hf._project( dict(found[key]) if isinstance(found.get(key), dict) else (found[key] if key in found else {}), dropped) @@ -1134,7 +1463,11 @@ def _get_entry(name, entry, tenant_slug=None): `PgStore` handle, never a store operation, so callers keep their whole-read fallback. A stored `ut_*` table read this way crosses the wire as that table alone, not the whole `user_tables` document (~60-100 MB on staging, ~3 s per read, holding a pool connection). + In row mode it reads one definition row; `rows` and the row maps load records on first touch. """ + if _ut_rows(name, tenant_slug): + value = _utr.lazy_table(name, tenant_slug, str(entry)) + return dict(value) if isinstance(value, dict) else None with _pool().connection() as con: row = con.execute( _q('SELECT value -> %s FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), @@ -1152,6 +1485,9 @@ def _get_json_paths(name, paths, tenant_slug=None): raise ValueError("A JSON path read accepts at most 4096 paths per batch") if not paths: return [] + if _ut_rows(name, tenant_slug): + return [_utr.value_at(name, tenant_slug, path)[1] if path else + _utr.read_document(name, tenant_slug) for path in paths] with _pool().connection() as con: row = con.execute(_q( "SELECT ARRAY(SELECT (kv.value #> ARRAY(SELECT jsonb_array_elements_text(p.path)))::text " @@ -1167,19 +1503,25 @@ def _iter_json_items(name, path=(), *, batch_size=1024, tenant_slug=None): size = int(batch_size) if not 1 <= size <= 4096: raise ValueError('A JSON object batch accepts between 1 and 4096 entries') + if _ut_rows(name, tenant_slug): + yield from _rows_json_items(name, path, size, tenant_slug) + return with _pool().connection() as con: with con.cursor(name='registry_' + uuid.uuid4().hex) as cur: + started = time.perf_counter() cur.execute(_q( - 'SELECT jsonb_typeof(kv.value #> %s::text[]), e.key, e.value::text FROM {tbl} AS kv ' - 'LEFT JOIN LATERAL jsonb_each(CASE ' - "WHEN jsonb_typeof(kv.value #> %s::text[]) = 'object' " - "THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) AS e ON TRUE WHERE kv.key = %s", - 'store_kv', tenant_slug), (list(map(str, path)), list(map(str, path)), - list(map(str, path)), str(name))) + 'WITH d AS MATERIALIZED (SELECT kv.value #> %s::text[] AS sub FROM {tbl} AS kv WHERE kv.key = %s) ' + 'SELECT jsonb_typeof(d.sub), e.key, e.value::text FROM d ' + "LEFT JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(d.sub) = 'object' " + "THEN d.sub ELSE '{{}}'::jsonb END) AS e ON TRUE", + 'store_kv', tenant_slug), (list(map(str, path)), str(name))) + _request_timing.record_store('json_items', str(name), milliseconds=(time.perf_counter() - started) * 1000) found = False while True: _request_timing.remaining_budget() + started = time.perf_counter() rows = cur.fetchmany(size) + elapsed_ms = (time.perf_counter() - started) * 1000 if not rows: if not found: raise RuntimeError('canonical_source_unavailable: the stored document disappeared') @@ -1188,32 +1530,70 @@ def _iter_json_items(name, path=(), *, batch_size=1024, tenant_slug=None): if any(kind not in (None, 'null', 'object') for kind, _key, _value in rows): raise RuntimeError('canonical_source_invalid: the stored rows are not an object') _request_timing.record_store('json_items', str(name), - bytes_read=sum(len(value.encode('utf-8')) for _kind, _key, value in rows if value is not None)) + bytes_read=sum(len(value.encode('utf-8')) for _kind, _key, value in rows if value is not None), + milliseconds=elapsed_ms) yield [(key, json.loads(value)) for _kind, key, value in rows if key is not None] rows.clear() +def _rows_json_items(name, path, size, tenant_slug): + """`_iter_json_items` in row mode: a table's records stream, any other object loads its table.""" + path = [str(p) for p in path] + if len(path) == 2 and path[1] == 'rows': + yield from _utr.iter_records(name, tenant_slug, path[0], batch_size=size) + return + found, value = _utr.value_at(name, tenant_slug, path) + if found and value is not None and not isinstance(value, dict): + raise RuntimeError('canonical_source_invalid: the stored rows are not an object') + items = list((value or {}).items()) if found else [] + for start in range(0, len(items), size): + _request_timing.remaining_budget() + yield items[start:start + size] + + +def _rows_json_value(name, path, tenant_slug): + """The value at `path` of a row-mode document, or None when the path is absent.""" + path = [str(p) for p in path] + if not path: + return _utr.read_document(name, tenant_slug) + return _utr.value_at(name, tenant_slug, path)[1] + + def _iter_json_values(name, path=(), *, drop=(), batch_size=1024, tenant_slug=None): """Stream an array with selected object operands removed before transport.""" import uuid size = int(batch_size) if not 1 <= size <= 4096: raise ValueError('A JSON array batch accepts between 1 and 4096 values') + if _ut_rows(name, tenant_slug): + values = _rows_json_value(name, path, tenant_slug) + values = values if isinstance(values, list) else [] + dropped = set(map(str, drop)) + for start in range(0, len(values), size): + _request_timing.remaining_budget() + yield [(str(start + i), {k: v for k, v in item.items() if k not in dropped} + if isinstance(item, dict) else item) + for i, item in enumerate(values[start:start + size])] + return with _pool().connection() as con: with con.cursor(name='canonical_' + uuid.uuid4().hex) as cur: + started = time.perf_counter() cur.execute(_q("SELECT e.n-1, (CASE WHEN jsonb_typeof(e.v)='object' " "THEN e.v - %s::text[] ELSE e.v END)::text FROM {tbl} kv " "CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' " "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) " 'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug), (list(drop), list(map(str, path)), list(map(str, path)), str(name))) + _request_timing.record_store('json_values', str(name), milliseconds=(time.perf_counter() - started) * 1000) while True: _request_timing.remaining_budget() + started = time.perf_counter() rows = cur.fetchmany(size) + elapsed_ms = (time.perf_counter() - started) * 1000 if not rows: return _request_timing.record_store('json_values', str(name), - bytes_read=sum(len(value.encode('utf-8')) for _, value in rows)) + bytes_read=sum(len(value.encode('utf-8')) for _, value in rows), milliseconds=elapsed_ms) yield [(str(index), json.loads(value)) for index, value in rows] rows.clear() @@ -1253,15 +1633,19 @@ def _iter_view_members(name, path, revision, *, batch_size=2048, tenant_slug=Non raise ValueError('A view membership batch accepts between 1 and 4096 values') with _pool().connection() as con: with con.cursor(name='view_members_' + uuid.uuid4().hex) as cur: + started = time.perf_counter() cur.execute(_q("SELECT kv.rev, kv.updated_at, e.n, e.v::text FROM {tbl} kv " "LEFT JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' " "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) ON TRUE " 'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug), (list(map(str, path)), list(map(str, path)), str(name))) + _request_timing.record_store('view_members', str(name), milliseconds=(time.perf_counter() - started) * 1000) found = False while True: _request_timing.remaining_budget() + started = time.perf_counter() rows = cur.fetchmany(size) + elapsed_ms = (time.perf_counter() - started) * 1000 if not rows: if not found: raise RuntimeError('canonical_source_unavailable: the saved-view document disappeared') @@ -1270,23 +1654,31 @@ def _iter_view_members(name, path, revision, *, batch_size=2048, tenant_slug=Non if any(tuple(row[:2]) != tuple(revision or ()) for row in rows): raise RuntimeError('canonical_source_unavailable: the saved view changed during this read') _request_timing.record_store('view_members', str(name), - bytes_read=sum(len(value.encode('utf-8')) for _, _, _, value in rows if value is not None)) + bytes_read=sum(len(value.encode('utf-8')) for _, _, _, value in rows if value is not None), + milliseconds=elapsed_ms) yield [(str(index - 1), json.loads(value)) for _, _, index, value in rows if index is not None] rows.clear() +@_request_timing.store_operation("json_object") def _get_json_object(name, path=(), *, drop=(), omit=(), tenant_slug=None): """Read nested object definitions while removing their population operands in SQL.""" + if _ut_rows(name, tenant_slug): + dropped, omitted = set(map(str, drop)), set(map(str, omit)) + value = (_utr.read_definitions(name, tenant_slug, dropped) if not path and 'rows' in dropped + else _rows_json_value(name, path, tenant_slug)) + return {k: ({kk: vv for kk, vv in v.items() if kk not in dropped} if isinstance(v, dict) else v) + for k, v in (value.items() if isinstance(value, dict) else ()) if k not in omitted} with _pool().connection() as con: row = con.execute(_q("SELECT coalesce(jsonb_object_agg(e.k, CASE WHEN jsonb_typeof(e.v)='object' " "THEN e.v - %s::text[] ELSE e.v END), '{{}}'::jsonb)::text FROM {tbl} kv " "CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='object' " "THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) e(k,v) WHERE kv.key=%s AND NOT e.k=ANY(%s::text[])", 'store_kv', tenant_slug), (list(drop), list(map(str, path)), list(map(str, path)), str(name), list(omit))).fetchone() - _request_timing.record_store('json_object', str(name), bytes_read=_request_timing.text_bytes(row[0]), count=1) return json.loads(row[0]) +@_request_timing.store_operation("json_members") def _get_json_members(name, path, ids, *, id_key=None, source_limit=None, tenant_slug=None): """Read complete selected membership without transferring an entire stored array.""" wanted = list(dict.fromkeys(str(int(pid)) for pid in ids)) @@ -1294,6 +1686,8 @@ def _get_json_members(name, path, ids, *, id_key=None, source_limit=None, tenant raise ValueError('A membership read accepts at most 4096 identities') if not wanted: return [] + if _ut_rows(name, tenant_slug): + return _rows_json_members(_rows_json_value(name, path, tenant_slug), wanted, id_key, source_limit) with _pool().connection() as con: rows = con.execute(_q("WITH members AS (SELECT e.v, e.n, CASE WHEN %s::text IS NULL " "THEN e.v ELSE e.v -> %s END ident FROM {tbl} kv " @@ -1307,12 +1701,54 @@ def _get_json_members(name, path, ids, *, id_key=None, source_limit=None, tenant 'SELECT v::text FROM ranked WHERE r=1 ORDER BY n', 'store_kv', tenant_slug), (id_key, id_key, list(map(str, path)), list(map(str, path)), str(name), source_limit, source_limit, wanted)).fetchall() - _request_timing.record_store('json_members', str(name), bytes_read=sum(len(row[0]) for row in rows), count=1) return [json.loads(row[0]) for row in rows] -@_request_timing.store_operation("definition_counts") +def _rows_json_members(values, wanted, id_key, source_limit): + """`_get_json_members`' SQL selection, applied in Python to a row-mode array.""" + from decimal import Decimal, InvalidOperation + import re as _re + + def ident_of(value): + ident = value if id_key is None else (value.get(id_key) if isinstance(value, dict) else None) + if ident is True: + return '1' + if ident is False: + return '0' + if isinstance(ident, (int, float)): + try: + text = str(int(Decimal(repr(ident)).to_integral_value(rounding='ROUND_DOWN'))) + except (InvalidOperation, ValueError, OverflowError): + return None + elif isinstance(ident, str): + text = ident.strip(' ') + elif ident is None: + return None + else: + text = json.dumps(ident).strip(' ') + return str(int(text)) if _re.fullmatch(r'[+-]?[0-9]+', text) else None + + values = values if isinstance(values, list) else [] + if source_limit is not None: + values = values[:max(0, int(source_limit))] + seen, out, want = set(), [], set(wanted) + for value in values: + ident = ident_of(value) + if ident in want and ident not in seen: + seen.add(ident) + out.append(value) + return out + + def _definition_counts_text(name, tenant_slug=None): + """Count stored row keys without returning row values; row mode counts record rows.""" + if _ut_rows(name, tenant_slug): + return _utr.counts_text(name, tenant_slug) + return _definition_counts_document(name, tenant_slug) + + +@_request_timing.store_operation("definition_counts") +def _definition_counts_document(name, tenant_slug=None): """Count stored row keys in Postgres without returning row values.""" with _pool().connection() as con: row = con.execute(_q( @@ -1349,6 +1785,9 @@ def _get_many(names, tenant_slug=None): found[str(key)] = _parsed(text) _request_timing.record_store('get_many', str(key), bytes_read=_request_timing.text_bytes(text), count=0) _seed_read_token(request_keys[str(key)], checkpoint, rev, stamp) + if _ut_marker(key, tenant_slug, found[str(key)]): + found[str(key)] = _utr.read_document(str(key), tenant_slug) + continue if not str(key).endswith('user_tables'): shape_key = (*request_keys[str(key)][:3], 'whole', request_keys[str(key)][-1]) _request_timing.memo_seed(shape_key, text, checkpoint) @@ -1373,10 +1812,8 @@ def exists(name, tenant_slug=None): distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an unreachable server raises instead of answering. """ - with _pool().connection() as con: - row = con.execute( - _q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), - (str(name),)).fetchone() + row = _read(_pool(), _q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), + (str(name),), one=True) return row is not None @@ -1459,6 +1896,13 @@ def put(name, data, tenant_slug=None): # pool stubbed to raise, "refused" and "reached the database" are two distinguishable answers, # which is what makes the negative control in `verify_store_pg.py` bite without a server. check_write(tenant_slug, f'put {name!r}') + if _ut_rows(name, tenant_slug): + _utr.settle_lazy(name, tenant_slug) + try: + _update_rows(name, lambda _doc: data, tenant_slug, None, None, table_mode=True) + finally: + _bump_write_generation(name, tenant_slug) + return data try: with _pool().connection() as con: payload = _payload(name, data, con, tenant_slug) @@ -1516,7 +1960,14 @@ def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row= """ check_write(tenant_slug, f'update {name!r}') # D-353 — see `put`, and before `_pool()` try: - return _update_locked(name, fn, tenant_slug, expect_rev, audit_row) + if _ut_rows(name, tenant_slug): + _utr.settle_lazy(name, tenant_slug) + return _update_rows(name, fn, tenant_slug, expect_rev, audit_row) + try: + return _update_locked(name, fn, tenant_slug, expect_rev, audit_row) + except _RowsLayoutFound: + # Another process migrated this document after the mode check; the lock saw the marker. + return _update_rows(name, fn, tenant_slug, expect_rev, audit_row) finally: _bump_write_generation(name, tenant_slug) # D-2640: no read joins an older flight @@ -1529,6 +1980,7 @@ def _update_locked(name, fn, tenant_slug, expect_rev, audit_row): except LockNotAvailable as exc: if not getattr(exc, '_aios_lock_diagnosed', False): _log_lock_blockers(getattr(exc, '_aios_waiter_pid', None)) + exc.aios_holder = _lock_holder(('store_kv', str(name), None), tenant_slug, name) raise @@ -1551,16 +2003,29 @@ def _log_lock_blockers(waiter_pid): def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row): - """Run the callback and write under the existing row lock and transaction.""" + """Run the callback and write under the row lock, in one transaction. + + On this module's pool the transaction costs three round trips: BEGIN with the limits and the + backend pid, the locking read, then the write, the audit rows and COMMIT. + """ from psycopg.errors import LockNotAvailable wait_s = _lock_timeout_s() - with _pool().connection() as con: - with con.transaction(): + pool = _pool() + piped = getattr(pool, '_aios_pipelines', False) is True + with (pool.autocommit() if piped else pool.connection()) as con: + with (nullcontext() if piped else con.transaction()): # Transaction-local (PgBouncer transaction mode): a writer queued behind a held row # lock gives its pooled connection back after `wait_s` instead of holding it forever. - con.execute("SELECT set_config('lock_timeout', %s, true)", - (f'{int(wait_s * 1000)}ms',)) - pid_row = con.execute('SELECT pg_backend_pid()').fetchone() + if piped: + with con.pipeline(): + con.execute('BEGIN') + head = con.execute(_SETTINGS_SQL + ", set_config('lock_timeout', %s, true), " + 'pg_backend_pid()', _settings_params() + (f'{int(wait_s * 1000)}ms',)) + pid_row = head.fetchone()[3:] + else: + con.execute("SELECT set_config('lock_timeout', %s, true)", + (f'{int(wait_s * 1000)}ms',)) + pid_row = con.execute('SELECT pg_backend_pid()').fetchone() waiter_pid = pid_row[0] if pid_row else None with _request_timing.lock_wait_probe(waiter_pid, wait_s, _log_lock_blockers) as probe: try: @@ -1582,38 +2047,230 @@ def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row): from core.view_audit import ViewRevisionStale raise ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) data = _parsed(row[0] if row else None) + if _ut_marker(name, tenant_slug, data): + raise _RowsLayoutFound(name) result = fn(data) data = result if result is not None else data payload = _payload(name, data, con, tenant_slug) - con.execute( - _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' - 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' - 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug), - (str(name), payload)) + # The write, the audit rows and COMMIT go in one round trip on this module's pool. + with (con.pipeline() if piped else nullcontext()): + con.execute( + _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' + 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' + 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug), + (str(name), payload)) + # ⚠ A LIST, OR ONE DICT, AND THE LIST IS NOT A CONVENIENCE. Moving views between + # folders is ONE workspace write that changes several views' placements, so the + # trail for it is several rows or it is wrong; a single-row keyword would have + # logged one of them and silently dropped the rest. + # ⚠ READ AFTER `fn` DELIBERATELY: the caller hands in a container its own closure + # fills, because the row's `before`/`after` are only knowable once the mutation has + # been applied to the locked document. + for entry in (audit_row if isinstance(audit_row, list) else [audit_row]): + if not entry: + continue + con.execute( + 'INSERT INTO control.audit ' + '(tenant_slug, username, action, target, detail) ' + 'VALUES (%s, %s, %s, %s, %s::jsonb)', + (_effective_slug(tenant_slug), + str(entry.get('actor') or '') or None, + 'view.' + str(entry.get('action') or ''), + f"{entry.get('table_key') or ''}/{entry.get('view_id') or ''}", + json.dumps(entry))) + if piped: + con.execute('COMMIT') _request_timing.record_store("update", str(name), bytes_written=_request_timing.text_bytes(payload), count=0) - # ⚠ A LIST, OR ONE DICT, AND THE LIST IS NOT A CONVENIENCE. Moving views between - # folders is ONE workspace write that changes several views' placements, so the - # trail for it is several rows or it is wrong; a single-row keyword would have - # logged one of them and silently dropped the rest. - # ⚠ READ AFTER `fn` DELIBERATELY: the caller hands in a container its own closure - # fills, because the row's `before`/`after` are only knowable once the mutation has - # been applied to the locked document. - for entry in (audit_row if isinstance(audit_row, list) else [audit_row]): - if not entry: - continue - con.execute( - 'INSERT INTO control.audit ' - '(tenant_slug, username, action, target, detail) ' - 'VALUES (%s, %s, %s, %s, %s::jsonb)', - (_effective_slug(tenant_slug), - str(entry.get('actor') or '') or None, - 'view.' + str(entry.get('action') or ''), - f"{entry.get('table_key') or ''}/{entry.get('view_id') or ''}", - json.dumps(entry))) return data +class _RowsLayoutFound(Exception): + """The locked `user_tables` value is the row-mode marker: roll back and write rows instead.""" + + +def _update_rows(name, fn, tenant_slug, expect_rev, audit_row, *, table_mode=False): + """`update` in row mode: record mode first, then table mode for each table that needs it. + + Record mode locks one record row per `rows[rid]` access under a shared definition lock, so + writers on different records of one table do not wait on each other. A change record mode + cannot write (a definition change, a full-table access) reruns `fn` once with those tables + in table mode, then once with every table in table mode. A lock or SQL error inside a lazy + load is re-raised after `fn` returns, even when `fn` swallowed it. + """ + from psycopg.errors import DeadlockDetected, LockNotAvailable + if expect_rev is not None: + current = _utr.marker_rev(None, tenant_slug, name) + if current != int(expect_rev): + from core.view_audit import ViewRevisionStale + raise ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) + modes, plan, deadlocks, escalations = (True if table_mode else set()), {}, 0, 0 + while True: + try: + doc, result = _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan) + break + except _utr.Escalate as esc: + escalations += 1 + plan = esc.plan + modes = True if (esc.all or escalations >= 2 or modes is True) else modes | set(esc.tables) + except DeadlockDetected: + # Two writers crossed; Postgres aborted one. Rerun `fn` on fresh state. + deadlocks += 1 + if deadlocks >= 3: + raise + except LockNotAvailable as exc: + exc.aios_holder = _lock_holder(getattr(exc, '_aios_lock_target', None), tenant_slug, name) + print(f'[store-pg] lock wait over {_lock_timeout_s():g}s on ' + f'{_effective_slug(tenant_slug)}/{name} (rows); holder: ' + f'{exc.aios_holder["label"]}', flush=True) + raise + _utr.carry_memo(name, tenant_slug, *getattr(doc, 'committed', (None, None))) + if result is not None and result is not doc: + return result + doc.detach(lambda key: _utr.read_table_split(tenant_slug, key, name), + bulk=lambda: _utr.read_document(name, tenant_slug), + records=_utr.DetachedRecords(tenant_slug, name)) + return doc + + +def _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan): + """One row-mode write attempt. Raises `Escalate` (rolled back) when table mode is needed.""" + from psycopg.errors import LockNotAvailable + from core import user_tables as _ut + wait_s = _lock_timeout_s() + + def budget(key, entry): + if isinstance(entry, dict): + report = _ut.enforce_byte_budget({key: entry}) + if report['evicted']: + print('[store_pg] user_tables byte budget: stored no rows for ' + + ', '.join(f'{k} ({v / 1048576:.1f} MB)' + for k, v in sorted(report['evicted'].items()))) + + with _pool().autocommit() as con: + # BEGIN, the limits, lock_timeout, the backend pid and the table keys: one round trip. + with con.pipeline(): + con.execute('BEGIN') + head = con.execute(_SETTINGS_SQL + ", set_config('lock_timeout', %s, true), pg_backend_pid()", + _settings_params() + (f'{int(wait_s * 1000)}ms',)) + listed = _utr.queue_table_keys(con, tenant_slug) + pid_row = head.fetchone()[3:] + access = _utr.RecordAccess(con, tenant_slug) + doc = _utr.RowsDocument(_utr.ordered_keys(r[0] for r in listed.fetchall()), access.table, + access=access, table_mode=modes) + try: + if plan: + doc.lock_records(plan) + try: + result = fn(doc) + except BaseException: + if doc.error is not None: + raise doc.error + raise + if doc.error is not None: + raise doc.error + changes = None if doc.needs_table_mode() else doc.changes(result, prepare=budget) + if changes is None: + raise _utr.Escalate(doc) + written = _utr.new_written() + finish = None + try: + # The writes, the marker bump, the audit rows and COMMIT go in one round trip. + # With `expect_rev`, COMMIT waits for the bump's answer. + with con.pipeline(): + n, finish = _utr.queue_changes(con, tenant_slug, changes, written) + bump = _utr.queue_bump_marker(con, tenant_slug, name, expect_rev) + for entry in (audit_row if isinstance(audit_row, list) else [audit_row]): + if not entry: + continue + con.execute( + 'INSERT INTO control.audit ' + '(tenant_slug, username, action, target, detail) ' + 'VALUES (%s, %s, %s, %s, %s::jsonb)', + (_effective_slug(tenant_slug), + str(entry.get('actor') or '') or None, + 'view.' + str(entry.get('action') or ''), + f"{entry.get('table_key') or ''}/{entry.get('view_id') or ''}", + json.dumps(entry))) + if expect_rev is None: + con.execute('COMMIT') + except LockNotAvailable: + # A record write that waited has no result; past every write, the wait was the marker. + writes_done = finish is not None and all(c.pgresult is not None for c in finish.cursors) + access.target = ('store_kv', str(name), None) if writes_done else None + raise + new_rev = _utr.marker_bumped(bump, expect_rev, written) + if not new_rev: + from core.view_audit import ViewRevisionStale + raise ViewRevisionStale(rev_current=_utr.marker_rev(con, tenant_slug, name), + rev_sent=int(expect_rev)) + if expect_rev is not None: + con.execute('COMMIT') + finish() + doc.rows_written = n + access.target = None + doc.committed = (written, new_rev) + except LockNotAvailable as exc: + exc._aios_waiter_pid = pid_row[0] if pid_row else None + exc._aios_lock_target = access.target + raise + _request_timing.record_store('update', str(name), count=0, + bytes_written=sum(len(v or '') + len(m or '') + for _k, _r, v, m in changes['records'])) + if doc.whole_read: + _request_timing.record_store('ut_whole', str(name), count=1) + return doc, result + + +def _lock_holder(target, tenant_slug, name='user_tables'): + """`{'pid', 'label', 'lock'}` for the transaction that holds the row `target` names. + + Read after the waiter rolled back: the locked row's `xmax` is the holder's transaction id (or a + multixact of share holders). The label is the holder's application, state and age, never a + statement. `pid` is None when the holder already finished or cannot be read. + """ + from psycopg import sql as _sql + kind, key, rid = target if target else ('store_kv', str(name), None) + holder = {'pid': None, 'label': 'unknown holder', 'lock': f'{kind} {key}'} + if kind == _utr.RECORDS and rid is None: + return holder + select = ('SELECT a.pid, a.application_name, a.backend_type, a.state, ' + 'extract(epoch FROM now() - a.xact_start) FROM pg_stat_activity AS a ') + try: + with _pool().connection(timeout=1.0) as con: + con.execute("SELECT set_config('statement_timeout', '1000ms', true)") + if kind == 'advisory': + found = con.execute( + select + 'JOIN pg_locks AS l ON l.pid = a.pid, ' + '(SELECT hashtextextended(%s, 0) AS h) AS k WHERE l.locktype = \'advisory\' ' + 'AND l.granted AND l.objsubid = 1 AND l.classid::bigint = ((k.h >> 32) & 4294967295) ' + 'AND l.objid::bigint = (k.h & 4294967295) ORDER BY a.xact_start LIMIT 1', + (_utr.table_lock_name(tenant_slug, key),)).fetchone() + else: + where = 'key = %s' if kind == 'store_kv' else ( + 'table_key = %s' if kind == _utr.DEFS else 'table_key = %s AND row_id = %s') + params = (key,) if kind in ('store_kv', _utr.DEFS) else (key, rid) + row = con.execute(_sql.SQL('SELECT xmax::text FROM {tbl} WHERE ' + where).format( + tbl=_table(kind, tenant_slug)), params).fetchone() + xmax = row[0] if row else None + if not xmax or xmax == '0': + return holder + found = con.execute(select + 'WHERE a.backend_xid::text = %s', (xmax,)).fetchone() + if found is None: + found = con.execute(select + 'JOIN pg_get_multixact_members(%s::xid) AS m ' + 'ON a.backend_xid = m.xid ORDER BY a.xact_start LIMIT 1', + (xmax,)).fetchone() + except Exception as exc: # noqa: BLE001 + holder['label'] = f'unknown holder ({type(exc).__name__})' + return holder + if found is not None: + pid, app, btype, state, age = found + holder['pid'] = int(pid) + holder['label'] = (f"{app or btype or 'backend'} pid {int(pid)}, {state or 'unknown state'}, " + f"{float(age or 0):.1f} s in transaction") + return holder + + def upload_bytes(path_in_repo, data, message=None, tenant_slug=None): check_write(tenant_slug, f'write bytes {path_in_repo!r}') # D-353 — before `_pool()` with _pool().connection() as con: @@ -1651,6 +2308,9 @@ def delete_path(path_in_repo, tenant_slug=None): def revision(name, tenant_slug=None): """Return an independent revision object, reusing only this request's backing validation.""" + held = _kv_revision(name, tenant_slug) + if held is not None: + return held return _parsed(_request_text(name, 'revision', tenant_slug, lambda: json.dumps(_revision(name, tenant_slug)))) @@ -1679,9 +2339,8 @@ def _revision_many(names, tenant_slug=None): @_request_timing.store_operation('revisions', many=True) def _read_revisions(names, tenant_slug=None): - with _pool().connection() as con: - rows = con.execute(_q('SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])', - 'store_kv', tenant_slug), (list(names),)).fetchall() + rows = _read(_pool(), _q('SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])', + 'store_kv', tenant_slug), (list(names),)) found = {} for name, raw_rev, stamp in rows: found[name] = _revision_value(raw_rev, stamp) @@ -1707,12 +2366,16 @@ def _revision(name, tenant_slug=None): An absent key is `rev 0`, not an error: a bucket nobody has written yet has changed zero times, which is a fact a client can hold a baseline against. """ - with _pool().connection() as con: - row = con.execute( - _q('SELECT revision.rev, revision.updated_at, octet_length(revision.rev::text) + ' - 'octet_length(revision.updated_at::text) ' - 'FROM (SELECT rev, updated_at FROM {tbl} WHERE key = %s) AS revision', 'store_kv', tenant_slug), - (str(name),)).fetchone() + pool = _pool() + doc = (_effective_slug(tenant_slug), str(name)) + with _READ_FLIGHT_LOCK: + generation = _WRITE_GENERATION.get(doc, 0) + started = time.monotonic() + row = _read(pool, _q('SELECT revision.rev, revision.updated_at, octet_length(revision.rev::text) + ' + 'octet_length(revision.updated_at::text) ' + 'FROM (SELECT rev, updated_at FROM {tbl} WHERE key = %s) AS revision', + 'store_kv', tenant_slug), (str(name),), one=True) + _kv_note((pool,) + doc, generation, started, row[0] if row else None, row[1] if row else None) _request_timing.record_store("revision", str(name), bytes_read=int(row[2] or 0) if row and len(row) > 2 else 0, count=0) rev = int(row[0]) if row and row[0] is not None else 0 @@ -1872,6 +2535,31 @@ class PgStore: """Bound single-entry read; None means absent, and the caller falls back to `get`.""" return _get_entry(name, entry, tenant_slug=self.tenant_slug) + def get_definition(self, name, table_key): + """One definition without rows, read from its own row (`ut_def`). None means absent. + + Returns NotImplemented in document mode: the caller keeps its projected read, which one + request shares across tables, instead of one whole-document detoast per table. + """ + if not _ut_rows(name, self.tenant_slug): + return NotImplemented + import core.store as _hf + key = str(table_key) + text = _request_text(name, ('definition', key), self.tenant_slug, + lambda: _utr.read_definition(name, self.tenant_slug, key)) + view = _utr.definition_view(text) + return None if view is None else _hf._Projected(view, ('rows',) + _utr.ROW_MAPS) + + def get_records(self, name, table_key, row_ids): + """One table entry holding only `row_ids` and their row-map entries (`ut_record`). + + Returns NotImplemented in document mode: the caller reads the table entry instead. + """ + if not _ut_rows(name, self.tenant_slug): + return NotImplemented + return _utr.read_record_table(name, self.tenant_slug, str(table_key), + [str(r) for r in row_ids]) + def _read_strict(self, name): """Interface parity with the HF Store. There, `get` is lenient (swallows a transient failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead diff --git a/platform/core/test_accounts.py b/platform/core/test_accounts.py new file mode 100644 index 0000000000000000000000000000000000000000..883092499cb042bbd162ff4e3470053c30b70826 --- /dev/null +++ b/platform/core/test_accounts.py @@ -0,0 +1,398 @@ +"""Test accounts: an exact copy of one member's permissions that takes no seat and sends nothing. + +S79-T63 (QI34, rulings R28 and R42). The copy carries the source's role, business units, modules, +agent, permission block (row filters and field walls stay locks, rule 3) and every share entry +that names the source. It never carries a password, MFA, email, WhatsApp, avatar or epoch. +Creation is all or nothing: a failure removes every part already written. +""" +import copy +import datetime as _dt +import re + +import core.shares as shares +import core.store as store +import core.users as users + +PREFIX = 'test-' +#: `routes_admin._UNAME_RE` admits 2 to 32 characters; a generated name stays inside it. +_NAME_MAX = 32 +_RETRIES = 3 + +#: The record fields a copy takes verbatim from its source. +COPIED_FIELDS = ('role', 'bus', 'modules', 'agent', 'perms', 'perms_v') + + +class CopyRefused(ValueError): + """The request names a source or target this door does not copy or remove.""" + + def __init__(self, code, message, status=400): + self.code, self.status = code, status + super().__init__(message) + + +class CopyFailed(RuntimeError): + """A copy or removal stopped part way. `leftovers` names what cleanup could not remove.""" + + def __init__(self, cause, leftovers=()): + self.cause, self.leftovers = cause, list(leftovers) + super().__init__(cause) + + +def is_test(record): + """Whether this stored record is a copied test account.""" + return users.is_test_account(record) + + +def _tenant_of(record): + return str((record or {}).get('tenant') or 'royal-imports').strip().lower() + + +def paid_seats(registry, tenant): + """Active accounts in `tenant` that take a paid seat. A test account takes none (R42).""" + want = str(tenant or '').strip().lower() + return sum(1 for record in (registry or {}).values() + if isinstance(record, dict) and record.get('active', True) + and _tenant_of(record) == want and not is_test(record)) + + +def test_username(source, registry): + """`test-` cut to fit the username rule, with a numeric suffix on a collision.""" + base = re.sub(r'[^a-z0-9._-]+', '-', str(source or '').strip().lower()).strip('.-_') + stem = (PREFIX + base)[:_NAME_MAX].rstrip('.-_') + taken = set(registry or {}) + if stem not in taken: + return stem + for n in range(2, 1000): + suffix = f'-{n}' + candidate = stem[:_NAME_MAX - len(suffix)].rstrip('.-_') + suffix + if candidate not in taken: + return candidate + raise CopyRefused('no_free_username', 'No free name is left for another test account.', 409) + + +class _Doc: + """A read-only store over one already-read shares document.""" + + def __init__(self, doc): + self._doc = doc if isinstance(doc, dict) else {} + + def get(self, name, *args, **kwargs): + return self._doc if name == shares.SHARES_KEY else None + + +class _Batch: + """Stage `shares.set_grants` writes in memory, then replay them all in ONE store update. + + Every staged closure keeps its `expect` check, so the replay refuses if any record changed. + """ + + def __init__(self, st, doc): + self._st, self._doc, self._fns = st, copy.deepcopy(doc or {}), [] + + def get(self, name, *args, **kwargs): + if name != shares.SHARES_KEY: + raise RuntimeError(f'a staged grant write cannot read {name!r}') + return self._doc + + def update(self, name, fn, flush='sync'): + if name != shares.SHARES_KEY: + raise RuntimeError(f'a staged grant write cannot write {name!r}') + self._doc = fn(self._doc) + self._fns.append(fn) + return self._doc + + def commit(self): + if not self._fns: + return + fns = list(self._fns) + + def replay(data): + data = data if isinstance(data, dict) else {} + for fn in fns: + data = fn(data) + return data + + self._st.update(shares.SHARES_KEY, replay, flush='sync') + + +def _read_shares(st): + """The tenant's shares document, read strictly: an unreadable bucket stops the copy.""" + doc = st.get(shares.SHARES_KEY, fresh=True) + if doc is None: + return {} + if not isinstance(doc, dict): + raise RuntimeError('the sharing records could not be read') + return doc + + +def grants_naming(username, doc): + """`[{kind, oid, role}]` for every share entry that names `username`, from one document.""" + username = str(username or '').strip().lower() + found = [] + view = _Doc(doc) + for kind in shares.KINDS: + for oid, record in sorted(shares.grants_bulk(kind, st=view).items()): + for entry in record['entries']: + if entry['user'] == username: + found.append({'kind': kind, 'oid': oid, 'role': entry['role']}) + return found + + +def _check_source(source, registry, tenant): + uname = str(source or '').strip().lower() + record = (registry or {}).get(uname) + if not uname or not isinstance(record, dict) or _tenant_of(record) != tenant: + raise CopyRefused('no_such_user', f'No account named {uname or "(blank)"}.', 404) + if is_test(record): + raise CopyRefused('source_is_test', 'A test account cannot be copied. Copy a member.') + if str(record.get('role') or 'user').strip().lower() != 'user': + raise CopyRefused('source_is_admin', + 'An administrator cannot be copied. Copy a member account.') + if not record.get('active', True): + raise CopyRefused('source_inactive', 'This account is turned off. Turn it on first.') + return uname, record + + +def plan(source, registry, *, tenant, st): + """What a copy of `source` would carry, read now: the access fields and every share entry.""" + tenant = str(tenant or '').strip().lower() + uname, record = _check_source(source, registry, tenant) + access = {key: copy.deepcopy(record[key]) for key in COPIED_FIELDS if key in record} + return {'source': uname, + 'sourceName': str(record.get('name') or uname), + 'username': test_username(uname, registry), + 'name': f'Test copy of {record.get("name") or uname}', + 'access': access, + 'grants': grants_naming(uname, _read_shares(st))} + + +def _password(username, name): + for _ in range(8): + candidate = users.generate_password() + if not users.password_failures(candidate, username=username, name=name): + return candidate + raise RuntimeError('could not generate a password for the test account') + + +def copy_grants(username, grants, *, st, granter): + """Add `username` beside each listed share entry, same role, in one store update.""" + batch = _Batch(st, _read_shares(st)) + for grant in grants: + current = shares.grants(grant['kind'], grant['oid'], st=batch) + if any(entry['user'] == username for entry in current['entries']): + continue + entries = current['entries'] + [{'user': username, 'role': grant['role']}] + shares.set_grants(grant['kind'], grant['oid'], entries, st=batch, + granter=granter, expect=current['entries']) + batch.commit() + + +def remove_grants(username, *, st, granter): + """Remove every share entry naming `username`, retrying if a record changes meanwhile.""" + for attempt in range(_RETRIES): + doc = _read_shares(st) + batch = _Batch(st, doc) + for grant in grants_naming(username, doc): + current = shares.grants(grant['kind'], grant['oid'], st=batch) + entries = [entry for entry in current['entries'] if entry['user'] != username] + shares.set_grants(grant['kind'], grant['oid'], entries, st=batch, + granter=granter, expect=current['entries']) + try: + batch.commit() + return + except shares.GrantsChanged: + if attempt == _RETRIES - 1: + raise + + +def _fresh_registry(): + registry = store.get('users', fresh=True) + if not isinstance(registry, dict): + raise RuntimeError('the account registry could not be read') + return registry + + +def _confirm(username, plan_, marker, st): + record = _fresh_registry().get(username) or {} + if record.get(users.TEST_ACCOUNT_KEY) != marker: + raise RuntimeError('the account was not saved') + if any(record.get(key) != plan_['access'].get(key) for key in plan_['access']): + raise RuntimeError('the copied permissions were not saved') + held = {(g['kind'], g['oid'], g['role']) for g in grants_naming(username, _read_shares(st))} + missing = [g for g in plan_['grants'] if (g['kind'], g['oid'], g['role']) not in held] + if missing: + raise RuntimeError(f'{len(missing)} shared items were not saved') + + +def _cleanup(username, tenant, marker, st, granter, steps): + """Undo every part of a copy. Returns the names of the parts that could not be removed.""" + leftovers = [] + for name, _do, undo in reversed(list(steps)): + try: + undo(username) + except Exception: # noqa: BLE001 + leftovers.append(name) + try: + remove_grants(username, st=st, granter=granter) + except Exception: # noqa: BLE001 + leftovers.append('shared items') + try: + users.delete_test_account(username, tenant=tenant, marker=marker) + left = _fresh_registry().get(username) + if is_test(left) and left.get(users.TEST_ACCOUNT_KEY) == marker: + leftovers.append('sign in') + except Exception: # noqa: BLE001 + leftovers.append('sign in') + return leftovers + + +def _cause(exc): + text = str(exc).strip() or type(exc).__name__ + return text[:1].upper() + text[1:] + + +def create(source, *, admin, tenant, st, steps=(), now=None): + """Copy `source` into a new test account. Returns `{username, password, plan}`. + + `steps` are `(name, do(username, source), undo(username))` for grant stores outside core. + Any failure removes what was written and raises `CopyFailed`. + """ + tenant = str(tenant or '').strip().lower() + admin = str(admin or '').strip().lower() + for attempt in range(_RETRIES): + plan_ = plan(source, _fresh_registry(), tenant=tenant, st=st) + username = plan_['username'] + marker = {'copied_from': plan_['source'], 'created_by': admin, + 'created_at': (now or _dt.datetime.now(_dt.timezone.utc)).isoformat( + timespec='seconds')} + password = _password(username, plan_['name']) + try: + users.create_test_account(username, password, name=plan_['name'], + access=plan_['access'], marker=marker, tenant=tenant) + except users.TestAccountExists: + if attempt == _RETRIES - 1: + raise CopyFailed('Another account took the test account name. Try again.') + continue + except Exception as exc: # noqa: BLE001 + # The write may have landed before it reported failure: clean up as for any step. + leftovers = _cleanup(username, tenant, marker, st, admin, ()) + raise CopyFailed(_cause(exc), leftovers) from exc + try: + copy_grants(username, plan_['grants'], st=st, granter=admin) + for _name, do, _undo in steps: + do(username, plan_['source']) + _confirm(username, plan_, marker, st) + except Exception as exc: # noqa: BLE001 + leftovers = _cleanup(username, tenant, marker, st, admin, steps) + raise CopyFailed(_cause(exc), leftovers) from exc + return {'username': username, 'password': password, 'plan': plan_} + raise CopyFailed('The test account could not be created. Try again.') + + +def remove(username, *, tenant, st, granter, steps=()): + """Delete a test account and every grant naming it. Refuses a real account.""" + tenant = str(tenant or '').strip().lower() + uname = str(username or '').strip().lower() + record = _fresh_registry().get(uname) + if not isinstance(record, dict) or _tenant_of(record) != tenant: + raise CopyRefused('no_such_user', f'No account named {uname or "(blank)"}.', 404) + if not is_test(record): + raise CopyRefused('not_a_test_account', + 'This is a real account. Only a test account can be removed here.') + leftovers = _cleanup(uname, tenant, record.get(users.TEST_ACCOUNT_KEY), st, granter, steps) + if leftovers: + raise CopyFailed('Some parts are still saved', leftovers) + + +# ------------------------------------------------------------------ preview +def _label(key, defs): + import core.perms as perms + import core.registry as registry + definition = (defs or {}).get(key) or {} + # The editor's surface rows (Assistant, Agents, AI apps (MCP)) are named by perms, not the registry. + surfaces = {m['key']: m['label'] for m in perms.governable_surface_modules()} + return str(definition.get('label') or (registry.BY_KEY.get(key) or {}).get('label') + or surfaces.get(key) or key) + + +def _plural(n, one, many): + return f'{n} {one if n == 1 else many}' + + +def preview(plan_, *, st=None, limits=None): + """The contract body for GET .../test-account/preview, built from `plan`. + + `st` is a definitions store (`user_tables.lend_defs`) for the database lock checks. + `limits` is `perm_scope.scope_limits` for the source; it names each row limit, business units included. + """ + from core import user_tables + access = plan_['access'] + try: + defs = user_tables.all_defs(st=st, schema_only=True) or {} + except Exception: # noqa: BLE001 + defs = {} + block = access.get('perms') if isinstance(access.get('perms'), dict) else {} + locks, reached = [], set() + said = {} + for limit in limits or (): + text = str(limit.get('text') or '').strip().rstrip('.') + if limit.get('database') and text: + said[str(limit['database'])] = text + for key in sorted(said): + locks.append({'kind': 'row_scope', 'label': f'{_label(key, defs)}: {said[key]}'}) + for key, entry in sorted(block.items()): + if not isinstance(entry, dict) or not entry.get('access', True): + continue + reached.add(key) + if entry.get('filter') and key not in said: + locks.append({'kind': 'row_scope', 'label': f'Rows are limited in {_label(key, defs)}'}) + hidden = [f for f in entry.get('hiddenFields') or () if f] + if hidden: + locks.append({'kind': 'field_wall', + 'label': f'{_plural(len(hidden), "field is", "fields are")} hidden in ' + f'{_label(key, defs)}'}) + if not access.get('perms_v') and not said: + bus = access.get('bus', 'all') + if bus != 'all' and isinstance(bus, list): + names = ', '.join(users.BU_LABELS.get(b, str(b)) for b in bus) + locks.append({'kind': 'row_scope', 'label': f'Rows are limited to {names}'}) + if access.get('agent'): + locks.append({'kind': 'row_scope', 'label': "Rows are limited to one salesperson's book"}) + for grant in plan_['grants']: + if grant['kind'] == 'cohort': + scope, _cid = shares.split_cohort_oid(grant['oid']) + where = _label(scope, defs) if scope else 'a database' + locks.append({'kind': 'locked_view', 'label': f'A locked view in {where}'}) + if grant['kind'] == 'database': + reached.add(grant['oid']) + for key in sorted(k for k in reached if str(k).startswith(user_tables.KEY_PREFIX)): + definition = defs.get(key) + if not isinstance(definition, dict): + continue + if not user_tables.records_mutable(key, st=st): + locks.append({'kind': 'locked_database', + 'label': f'{_label(key, defs)} is locked for record changes'}) + preset = [f for f in definition.get('fields') or () + if isinstance(f, dict) and isinstance(f.get('automation'), dict) + and f['automation'].get('preset') is True + and not user_tables.preset_editable(f)] + if preset: + locks.append({'kind': 'preset_field', + 'label': f'{_plural(len(preset), "pre-set field", "pre-set fields")} ' + f'in {_label(key, defs)}'}) + grants = [] + for grant in plan_['grants']: + row = dict(grant) + if grant['kind'] == 'database': + row['label'] = _label(grant['oid'], defs) + grants.append(row) + database_access = [{'key': key, 'label': _label(key, defs), + 'access': bool(entry.get('access', True))} + for key, entry in sorted(block.items()) if isinstance(entry, dict)] + return {'source': {'username': plan_['source'], 'name': plan_['sourceName']}, + 'testUsername': plan_['username'], + 'copy': {'role': access.get('role', 'user'), 'bus': access.get('bus', 'all'), + 'buLabels': users.allowed_bus_labels(access), + 'modules': access.get('modules', 'all'), 'agent': access.get('agent'), + 'grants': grants, 'databaseAccess': database_access, 'locks': locks}, + 'counts': {'grants': len(grants), 'locks': len(locks)}} diff --git a/platform/core/user_table_rows.py b/platform/core/user_table_rows.py new file mode 100644 index 0000000000000000000000000000000000000000..9081fc4a982bb653675e82de6d32cfec56378589 --- /dev/null +++ b/platform/core/user_table_rows.py @@ -0,0 +1,2635 @@ +"""core/user_table_rows.py: the `user_tables` document stored as one row per table and per record. + +Layout (S79-T47, lane Q design `STORE-SPLIT-DESIGN.md`): `user_table_defs` holds each table entry +minus `rows` and minus every `ROW_MAPS` object. `user_table_records` holds `rows[row_id]` in +`value` and each row map's entry for that id in `meta`. `store_kv['user_tables']` becomes a small +marker whose `rev` and `updated_at` bump on every row-mode write, so revision tokens keep working. + +`core.store_pg` routes every operation on a migrated name here. This module imports `store_pg` +lazily inside functions, because `store_pg` imports `user_tables` at load. +""" +import hashlib +import json +import os +import threading +import time +import weakref +from collections import OrderedDict +from datetime import datetime, timezone + +from core import request_timing as _request_timing + +DOCUMENT_KEY = 'user_tables' +DEFS = 'user_table_defs' +RECORDS = 'user_table_records' +#: Definition keys whose value is `{row id: entry}`. Each entry lives in `meta` of that record row. +ROW_MAPS = ('_rowAudit', '_rowCreationOrder', 'ticketSourceRows', '_activity_outbox') +LAYOUT_KEY = '__layout__' +MAPS_KEY = '__rowMaps__' +NO_ROWS_KEY = '__noRows__' +INTERNAL_KEYS = (MAPS_KEY, NO_ROWS_KEY) +VERSION = 1 + + +class _Absent: + """The record value of a row that keeps only row-map entries (SQL NULL, not JSON null).""" + + def __repr__(self): + return '' + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + +ABSENT = _Absent() + + +class _Pending: + """Fills dict storage while tables are unloaded, so C code calls the overridden `items()`.""" + + def __repr__(self): + return '' + + +_PENDING = _Pending() +#: The storage key of the placeholder a partly loaded `_LazyMap` keeps. No row id equals it. +_HOLE = _Pending() +#: Row maps whose key may stay listed while empty. Activity code pops an empty outbox key. +STICKY_MAPS = ('_activity_outbox',) +#: Detached per-table loads before the next miss reads the whole document in one statement. +BULK_AFTER = 3 + + +# -------------------------------------------------------------------------------------------- +# Pure layout: split, order, assemble. No SQL, so the gate drives these in memory. +# -------------------------------------------------------------------------------------------- +def _sort_key(key): + raw = str(key).encode('utf-8') + return (len(raw), raw) + + +def ordered_keys(keys): + """Keys in jsonb's stored order: byte length first, then bytes.""" + return sorted(keys, key=_sort_key) + + +def _json_key(key): + """The object key `json.dumps` writes for a Python dict key.""" + if isinstance(key, str): + return key + if key is True: + return 'true' + if key is False: + return 'false' + if key is None: + return 'null' + if isinstance(key, float): + return json.dumps(key) + return str(key) + + +def is_marker(value): + """True for the small `store_kv` value a migrated tenant holds instead of the document.""" + return isinstance(value, dict) and value.get(LAYOUT_KEY) == 'rows' + + +def marker(): + return {LAYOUT_KEY: 'rows', 'version': VERSION, + 'migratedAt': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')} + + +def split_table(entry): + """`(definition, {row_id: [value, meta]})` for one table entry; `ABSENT` marks a meta-only row.""" + if not isinstance(entry, dict): + return entry, {} + definition, records, maps = {}, {}, [] + for key, value in entry.items(): + key = _json_key(key) + if key == 'rows' and isinstance(value, dict): + continue + if key in ROW_MAPS and isinstance(value, dict): + maps.append(key) + continue + definition[key] = value + if 'rows' not in entry: + definition[NO_ROWS_KEY] = True + if maps: + definition[MAPS_KEY] = ordered_keys(maps) + rows = entry.get('rows') + if isinstance(rows, dict): + for rid, value in rows.items(): + records[_json_key(rid)] = [value, None] + for map_key in maps: + for rid, item in entry[map_key].items(): + record = records.setdefault(_json_key(rid), [ABSENT, None]) + if record[1] is None: + record[1] = {} + record[1][map_key] = item + return definition, records + + +def assemble_table(definition, records): + """One table entry from its definition and `(row_id, value, meta)` records, in jsonb key order.""" + if not isinstance(definition, dict): + return definition + entry = {k: v for k, v in definition.items() if k not in INTERNAL_KEYS} + maps = [m for m in (definition.get(MAPS_KEY) or ()) if m in ROW_MAPS] + rows, mapped = {}, {m: {} for m in maps} + for rid, value, meta in sorted(records, key=lambda r: _sort_key(r[0])): + if value is not ABSENT: + rows[rid] = value + if meta: + for map_key in maps: + if map_key in meta: + mapped[map_key][rid] = meta[map_key] + if not definition.get(NO_ROWS_KEY) and 'rows' not in entry: + entry['rows'] = rows + entry.update(mapped) + return {k: entry[k] for k in ordered_keys(entry)} + + +def split_document(doc): + """`{table_key: (definition, records)}` for a whole document.""" + return {_json_key(k): split_table(v) for k, v in (doc or {}).items()} + + +def assemble_document(tables): + """The whole document from `{table_key: (definition, records)}`, in jsonb key order.""" + return {k: assemble_table(tables[k][0], [(rid, rec[0], rec[1]) for rid, rec in tables[k][1].items()]) + for k in ordered_keys(tables)} + + +def _definition_view(definition, drop, mapped): + """A definition with `drop` and internal keys removed and wanted row maps added back.""" + if not isinstance(definition, dict): + return definition + view = {k: v for k, v in definition.items() if k not in INTERNAL_KEYS and k not in drop} + for map_key in definition.get(MAPS_KEY) or (): + if map_key not in drop and map_key in ROW_MAPS: + view[map_key] = mapped.get(map_key, {}) + return {k: view[k] for k in ordered_keys(view)} + + +def _dump(value): + return json.dumps(value) + + +def _record_json(record): + value, meta = record + return (None if value is ABSENT else _dump(value), _dump(meta) if meta else None) + + +def _canon(value): + """Order-free JSON text: a definition read back from jsonb compares equal to the same content.""" + return json.dumps(value, sort_keys=True) + + +# -------------------------------------------------------------------------------------------- +# RECORD MODE (S79-T48): one table's `rows` and row maps load and lock one record at a time. +# -------------------------------------------------------------------------------------------- +class Escalate(Exception): + """A record-mode attempt needs table mode for `tables` (or every table): roll back and rerun.""" + + def __init__(self, doc): + super().__init__('record mode escalates to table mode') + self.tables = dict(doc.escalated) + self.all = doc.escalate_all + self.plan = doc.plan() + + +class _LazyMap(dict): + """`rows` or one row map of a record-mode table. + + `[rid]`, `.get`, `in`, assignment, `del`, `pop` and `setdefault` load and lock one record row. + Iteration, `len`, `keys`, `items`, `values`, `==` and copies load every record and mark the + table for table mode. Truth asks one EXISTS query when no loaded record answers it. + """ + lazy_records = True + + def __init__(self, state, kind): + super().__init__() + self._state = state + self._kind = kind + dict.__setitem__(self, _HOLE, _PENDING) + + def _one(self, rid): + if not isinstance(rid, str) and getattr(self._state, 'exact_keys', False): + return rid # a read-only map answers a non-text key as a dict does + rid = _json_key(rid) + self._state.load((rid,)) + return rid + + def _all(self, why): + self._state.load_all(f'{self._kind} {why}') + + def _plain(self): + return {k: v for k, v in dict.items(self) if k is not _HOLE} + + # -- one record ---------------------------------------------------------------------------- + def __getitem__(self, rid): + return dict.__getitem__(self, self._one(rid)) + + def get(self, rid, default=None): + return dict.get(self, self._one(rid), default) + + def __contains__(self, rid): + return dict.__contains__(self, self._one(rid)) + + def __setitem__(self, rid, value): + dict.__setitem__(self, self._one(rid), value) + + def __delitem__(self, rid): + dict.__delitem__(self, self._one(rid)) + + def pop(self, rid, *default): + rid = self._one(rid) + if dict.__contains__(self, rid): + return dict.pop(self, rid) + if default: + return default[0] + raise KeyError(rid) + + def setdefault(self, rid, default=None): + rid = self._one(rid) + if not dict.__contains__(self, rid): + dict.__setitem__(self, rid, default) + return dict.__getitem__(self, rid) + + def update(self, other=(), **kw): + pairs = list(other.items() if hasattr(other, 'items') else other) + list(kw.items()) + self._state.load([_json_key(k) for k, _v in pairs]) + for key, value in pairs: + dict.__setitem__(self, _json_key(key), value) + + def __ior__(self, other): + self.update(other) + return self + + def __bool__(self): + if any(k is not _HOLE for k in dict.keys(self)): + return True + return False if self._state.full else self._state.any(self._kind) + + # -- every record (escalates) -------------------------------------------------------------- + def __iter__(self): + self._all('iterated') + return iter(list(dict.keys(self))) + + def __reversed__(self): + self._all('iterated') + return iter(list(reversed(dict.keys(self)))) + + def __len__(self): + self._all('counted') + return dict.__len__(self) + + def keys(self): + self._all('listed') + return dict.keys(self) + + def values(self): + self._all('listed') + return dict.values(self) + + def items(self): + self._all('listed') + return dict.items(self) + + def popitem(self): + self._all('changed whole') + return dict.popitem(self) + + def clear(self): + self._all('changed whole') + dict.clear(self) + + def copy(self): + self._all('copied') + return self._plain() + + def __copy__(self): + return self.copy() + + def __deepcopy__(self, memo): + import copy + return copy.deepcopy(self.copy(), memo) + + def __eq__(self, other): + self._all('compared') + return self._plain() == other + + def __ne__(self, other): + return not self == other + + __hash__ = None + + def __or__(self, other): + out = self.copy() + out.update(other) + return out + + def __reduce__(self): + return (dict, (self.copy(),)) + + def __repr__(self): + loaded = sum(1 for k in dict.keys(self) if k is not _HOLE) + return f'' + + +class _TableState: + """One record-mode table inside a `RowsDocument`: its definition, views and loaded originals.""" + + def __init__(self, doc, key, definition): + self.doc, self.key, self.definition = doc, key, definition + self.def_canon = _canon(definition) + self.maps = [m for m in (definition.get(MAPS_KEY) or ()) if m in ROW_MAPS] + self.views = {} + if not definition.get(NO_ROWS_KEY) and 'rows' not in definition: + self.views['rows'] = _LazyMap(self, 'rows') + for map_key in self.maps: + self.views[map_key] = _LazyMap(self, map_key) + entry = {k: v for k, v in definition.items() if k not in INTERNAL_KEYS} + entry.update(self.views) + self.entry = {k: entry[k] for k in ordered_keys(entry)} + self.orig = {} # rid -> (value json, meta json), or None: no row + self.full = False + + def _fill(self, rid, record): + value, meta = record + rows = self.views.get('rows') + if rows is not None and value is not ABSENT and not dict.__contains__(rows, rid): + dict.__setitem__(rows, rid, value) + for map_key in self.maps: + if meta and map_key in meta and not dict.__contains__(self.views[map_key], rid): + dict.__setitem__(self.views[map_key], rid, meta[map_key]) + + def load(self, rids): + missing = [r for r in dict.fromkeys(rids) if r not in self.orig] + if not missing: + return + if self.full: + self.orig.update(dict.fromkeys(missing)) + return + found = self.doc._read_records(self.key, missing) + for rid in missing: + record = found.get(rid) + self.orig[rid] = None if record is None else _record_json(record) + if record is not None: + self._fill(rid, record) + + def load_all(self, why): + if self.full: + return + self.doc._escalate(self.key, why) + everything = self.doc._read_records(self.key, None) + for rid, record in everything.items(): + if rid not in self.orig: + self.orig[rid] = _record_json(record) + self._fill(rid, record) + self.full = True + for view in self.views.values(): + items = [(k, v) for k, v in dict.items(view) if k is not _HOLE] + dict.clear(view) + dict.update(view, sorted(items, key=lambda kv: _sort_key(kv[0]))) + + def any(self, kind): + return self.doc._any_record(self.key, kind, list(self.orig)) + + def changes(self, entry, out): + """Append this table's record writes to `out`; return a reason when it needs table mode.""" + if entry is not self.entry: + return 'table replaced' + for kind, view in self.views.items(): + if kind not in entry: + if kind in STICKY_MAPS and not view: + continue + return f'{kind} removed' + if entry[kind] is not view: + return f'{kind} replaced' + definition = {} + for key, value in entry.items(): + key = _json_key(key) + if key in self.views and value is self.views[key]: + continue + if key in ROW_MAPS and isinstance(value, dict): + return f'{key} added' + definition[key] = value + for key in INTERNAL_KEYS: + if key in self.definition: + definition[key] = self.definition[key] + if _canon(definition) != self.def_canon: + return 'definition changed' + rows = self.views.get('rows') + for rid in ordered_keys(self.orig): + value = dict.get(rows, rid, ABSENT) if rows is not None else ABSENT + meta = {m: dict.__getitem__(self.views[m], rid) for m in self.maps + if dict.__contains__(self.views[m], rid)} + pair = _record_json([value, meta or None]) + if pair != (self.orig[rid] or (None, None)): + out['records'].append((self.key, rid) + pair) + return None + + +# -------------------------------------------------------------------------------------------- +# The compat document a row-mode `update` callback receives (TABLE MODE). +# -------------------------------------------------------------------------------------------- +class RowsDocument(dict): + """A `user_tables` dict whose tables load on first access, diffed into row writes afterwards. + + `table_keys` lists every table in jsonb order. `loader(key)` returns `(definition, records)` or + None, and in a store transaction it locks that table's definition row. `items()`, `values()`, + `copy()`, `==` and `deepcopy` load every table: that is the named whole read (`whole_read`). + + With `access` (a `RecordAccess`), a table not in `table_mode` loads in RECORD MODE: its + definition row `FOR SHARE`, and `rows` plus each row map as `_LazyMap`s. A change record mode + cannot write sets `escalated` and is still served; the store reruns the callback. + """ + lazy_tables = True + + def __init__(self, table_keys, loader, *, access=None, table_mode=()): + super().__init__() + self._keys = dict.fromkeys(str(k) for k in table_keys) + self._listed = frozenset(self._keys) + self._loader = loader + self._orig = {} + self._deleted = set() + self.whole_read = False + self.loads = [] + self._access = access + self._table_mode = table_mode if table_mode is True else set(table_mode) + self._states = {} + self.escalated = {} + self.escalate_all = None + self.error = None + self.rows_written = 0 + self._bulk = None + self._single_loads = 0 + self._detached = False + self._reader = None + self._iterated = False + self._source = None + self._defs = None + self._fill() + + # -- record mode --------------------------------------------------------------------------- + def _record_mode(self, key): + return self._live() and key not in self._table_mode + + def _live(self): + """True while a record-mode store transaction owns this document.""" + return self._access is not None and not self._detached and self._table_mode is not True + + def _fail(self, exc): + if self.error is None: + self.error = exc + + def _escalate(self, key, why): + if self._live(): + self.escalated.setdefault(key, why) + + def _read_records(self, key, rids): + try: + return (self._reader if self._detached else self._access).records(key, rids) + except Exception as exc: + self._fail(exc) + raise + + def _any_record(self, key, kind, exclude): + try: + return (self._reader if self._detached else self._access).any_record(key, kind, exclude) + except Exception as exc: + self._fail(exc) + raise + + def needs_table_mode(self): + return bool(self.escalated) or self.escalate_all is not None + + def plan(self): + """`{table: row ids}` this attempt locked, for the rerun to lock up front in order.""" + out = {key: list(state.orig) for key, state in self._states.items()} + for key in self.loads: + out.setdefault(key, []) + return out + + def lock_records(self, plan): + """Lock `{table_key: row ids}` up front: definitions sorted, then records sorted. + + A `None` id list asks for table mode on that table (a writer that reads every record). + """ + if self._access is None or self.loads: + return + for key in ordered_keys(_json_key(k) for k in plan): + if plan.get(key, ()) is None and self._table_mode is not True: + self._table_mode.add(key) + self._load(key) + for key in ordered_keys(_json_key(k) for k in plan): + state = self._states.get(key) + if state is not None and plan.get(key): + state.load(ordered_keys(_json_key(r) for r in plan[key])) + + # -- storage ------------------------------------------------------------------------------ + def _is_loaded(self, key): + return dict.__contains__(self, key) and dict.__getitem__(self, key) is not _PENDING + + def _fill(self): + """Keep one placeholder in storage while any table is unloaded (see `_Pending`).""" + real = [k for k in dict.keys(self) if dict.__getitem__(self, k) is not _PENDING] + for k in [k for k in dict.keys(self) if dict.__getitem__(self, k) is _PENDING]: + dict.__delitem__(self, k) + if len(real) < len(self._keys): + first = next(k for k in self._keys if not dict.__contains__(self, k)) + dict.__setitem__(self, first, _PENDING) + + def _load(self, key): + if self._is_loaded(key): + return True + if key not in self._keys: + return False + if self._source is not None: + return self._load_read(key) + record = self._record_mode(key) + # `dict(doc)` and `{**doc}` list the keys, then read each in order: one statement instead. + listed_first = self._iterated and key == next(iter(self._keys), None) + if self._detached and self._bulk is not None and (listed_first or self._single_loads >= BULK_AFTER): + self._load_all() + return self._is_loaded(key) + try: + split = self._access.definition(key) if record else self._loader(key) + except Exception as exc: + self._fail(exc) + raise + self.loads.append(key) + if self._detached: + self._single_loads += 1 + if split is None: + self._keys.pop(key, None) + self._fill() + return False + if record and not isinstance(split, dict): + # A scalar entry has no records: keep it as a plain value, diffed in table form. + self._orig.setdefault(key, (_canon(split), {})) + dict.__setitem__(self, key, split) + self._fill() + return True + if record: + state = _TableState(self, key, split) + self._states[key] = state + self._orig.setdefault(key, None) + dict.__setitem__(self, key, state.entry) + self._fill() + return True + definition, records = split + if key not in self._orig: + self._orig[key] = (_canon(definition), + {rid: _record_json(rec) for rid, rec in records.items()}) + dict.__setitem__(self, key, assemble_table( + definition, [(rid, rec[0], rec[1]) for rid, rec in records.items()])) + self._fill() + return True + + def _load_read(self, key): + """A read-only answer's table: its definition now, its records on first touch.""" + listed_first = self._iterated and key == next(iter(self._keys), None) + if self._defs is None and (listed_first or self._single_loads >= BULK_AFTER): + self._defs = self._source.definitions() + if self._defs is not None: + definition = self._defs.pop(key, None) + else: + self._single_loads += 1 + definition = self._source.definition(key) + self.loads.append(key) + if definition is None: + self._keys.pop(key, None) + self._fill() + return False + dict.__setitem__(self, key, read_entry(self._source, key, definition)) + self._fill() + return True + + def _load_all(self): + if self._source is not None: + if any(not self._is_loaded(k) for k in self._keys): + if self._defs is None: + self._defs = self._source.definitions() + for key in list(self._keys): + self._load(key) + entries = [(k, dict.__getitem__(self, k)) for k in self._keys if self._is_loaded(k)] + dict.clear(self) + dict.update(self, entries) + return + if (self._detached and self._bulk is not None) or self._live(): + # One statement for every unloaded table: a detached read, or an escalating serve. + if self._live(): + self.escalate_all = self.escalate_all or 'whole document read' + missing = [k for k in self._keys if not self._is_loaded(k)] + if missing: + try: + whole = self._bulk() if self._detached else self._access.document() + except Exception as exc: + self._fail(exc) + raise + for key in missing: + if key in whole: + dict.__setitem__(self, key, whole[key]) + else: + self._keys.pop(key, None) + else: + for key in list(self._keys): + self._load(key) + self.whole_read = True + entries = [(k, dict.__getitem__(self, k)) for k in self._keys if self._is_loaded(k)] + dict.clear(self) + dict.update(self, entries) + + def detach(self, loader, *, bulk=None, records=None): + """Serve later reads from `loader` without locks and stop recording originals. + + `bulk()` returns the whole document in one read. `records` answers record-mode tables. + """ + self._loader = loader + self._bulk = bulk + self._orig = _Frozen(self._orig) + self._reader = records + self._detached = True + for state in self._states.values(): + state.full = state.full or records is None + + def read_lazily(self, source): + """Serve a read-only answer: definitions through `source`, each table's records on touch.""" + self._source = source + self._loader = None + self._orig = _Frozen(self._orig) + self._detached = True + + def settle(self): + """Before this thread writes: finish the reads a caller holding this document could make.""" + if self._source is not None: + return # a handed-out table settles through its own state (`_ReadTable`) + if any(not self._is_loaded(k) for k in self._keys): + self._load_all() + for state in self._states.values(): + state.load_all('settled') + + # -- the dict surface ---------------------------------------------------------------------- + def __getitem__(self, key): + key = _json_key(key) + if not self._load(key): + raise KeyError(key) + return dict.__getitem__(self, key) + + def get(self, key, default=None): + key = _json_key(key) + return dict.__getitem__(self, key) if self._load(key) else default + + def __contains__(self, key): + return _json_key(key) in self._keys + + def __len__(self): + return len(self._keys) + + def __bool__(self): + return bool(self._keys) + + def __iter__(self): + self._iterated = True + return iter(list(self._keys)) + + def __reversed__(self): + return iter(list(reversed(self._keys))) + + def keys(self): + self._iterated = True + return dict.fromkeys(self._keys).keys() + + def items(self): + self._load_all() + return dict.items(self) + + def values(self): + self._load_all() + return dict.values(self) + + def __setitem__(self, key, value): + key = _json_key(key) + if key in self._listed and key not in self._deleted and key not in self._orig: + self._load(key) # lock and record the original before replacing + state = self._states.get(key) + if self._live() and (state is None or value is not state.entry): + self._escalate(key, 'table replaced or added') + dict.__setitem__(self, key, value) + self._keys[key] = None + self._fill() + + def __delitem__(self, key): + key = _json_key(key) + if key not in self._keys: + raise KeyError(key) + self._escalate(key, 'table deleted') + if dict.__contains__(self, key): + dict.__delitem__(self, key) + self._keys.pop(key) + self._deleted.add(key) + self._fill() + + def pop(self, key, *default): + key = _json_key(key) + if key not in self._keys or not self._load(key): + if default: + return default[0] + raise KeyError(key) + value = dict.__getitem__(self, key) + del self[key] + return value + + def popitem(self): + if not self._keys: + raise KeyError('popitem(): dictionary is empty') + key = list(self._keys)[-1] + return key, self.pop(key) + + def setdefault(self, key, default=None): + key = _json_key(key) + if self._load(key): + return dict.__getitem__(self, key) + self[key] = default + return default + + def update(self, other=(), **kw): + pairs = other.items() if hasattr(other, 'items') else other + for key, value in list(pairs) + list(kw.items()): + self[key] = value + + def clear(self): + for key in list(self._keys): + del self[key] + + def copy(self): + return dict(self.items()) + + def __copy__(self): + return self.copy() + + def __deepcopy__(self, memo): + import copy + return copy.deepcopy(dict(self.items()), memo) + + def __eq__(self, other): + return dict(self.items()) == other + + def __ne__(self, other): + return not self == other + + __hash__ = None + + def __or__(self, other): + out = dict(self.items()) + out.update(other) + return out + + def __ior__(self, other): + self.update(other) + return self + + def __repr__(self): + loaded = sum(1 for k in self._keys if self._is_loaded(k)) + return f'' + + def __reduce__(self): + return (dict, (dict(self.items()),)) + + # -- the diff ------------------------------------------------------------------------------ + def touched(self, result=None): + """`{key: entry}` the callback could have changed: loaded or set tables, or all on replace.""" + if result is not None and result is not self: + self._load_all() + return {_json_key(k): v for k, v in result.items()} + return {k: dict.__getitem__(self, k) for k in self._keys if self._is_loaded(k)} + + def changes(self, result=None, prepare=None): + """The row writes that turn the loaded originals into the callback's result. + + `prepare(key, entry)` runs on each touched entry first (the store's byte budget). Returns + `{'defs': {key: json}, 'drop': [key], 'clear': [key], 'records': [(key, rid, value_json, + meta_json)], 'delete': [(key, rid)]}`. A table whose original was never read is cleared + and rewritten whole. + """ + replaced = result is not None and result is not self + if replaced and self._live(): + self.escalate_all = self.escalate_all or 'callback returned a new document' + if self.needs_table_mode(): + return None + current = self.touched(result) + if prepare is not None: + for key, entry in current.items(): + if key not in self._states: + prepare(key, entry) + gone = (self._listed | set(self._orig)) - set(current) if replaced else ( + self._deleted - set(self._keys)) + out = {'defs': {}, 'drop': sorted(gone), 'clear': [], 'records': [], 'delete': []} + for key in ordered_keys(current): + if key in self._states: + why = self._states[key].changes(current[key], out) + if why is not None: + self._escalate(key, why) + continue + definition, records = split_table(current[key]) + def_json = _dump(definition) + new = {rid: _record_json(rec) for rid, rec in records.items()} + orig = self._orig.get(key) + if orig is None: + out['defs'][key] = def_json + out['clear'].append(key) + out['records'].extend((key, rid, v, m) for rid, (v, m) in new.items()) + continue + if _canon(definition) != orig[0]: + out['defs'][key] = def_json + for rid, pair in new.items(): + if orig[1].get(rid) != pair: + out['records'].append((key, rid, pair[0], pair[1])) + out['delete'].extend((key, rid) for rid in orig[1] if rid not in new) + return None if self.needs_table_mode() else out + + +class _Frozen(dict): + """Originals after commit: later loads record nothing.""" + + def __contains__(self, key): + return True + + +# -------------------------------------------------------------------------------------------- +# READ-ONLY ANSWERS (S79-T49): a row-mode `get` and `get_entry` parse definitions now and load +# each table's records on first touch. Every answer is the caller's own objects: the request +# memo holds only immutable row texts. +# -------------------------------------------------------------------------------------------- +class _ReadTable: + """The record loader behind one table entry's lazy `rows` and row maps. + + The views hold this state and it holds them weakly, so a dropped entry frees it at once. + """ + exact_keys = True + + def __init__(self, source, key, maps): + self.source, self.key, self.maps = source, key, maps + self.views = weakref.WeakValueDictionary() + self.seen = set() + self.full = False + self.pinned = None + + def _fill(self, rid, record): + value, meta = record + rows = self.views.get('rows') + if rows is not None and value is not ABSENT: + dict.__setitem__(rows, rid, value) + for map_key in self.maps: + view = self.views.get(map_key) + if view is not None and meta and map_key in meta: + dict.__setitem__(view, rid, meta[map_key]) + + def load(self, rids): + missing = [r for r in dict.fromkeys(rids) if r not in self.seen] + if not missing or self.full: + return + found = self.source.records(self.key, missing, self.pinned) + self.seen.update(missing) + for rid in missing: + if rid in found: + self._fill(rid, found[rid]) + + def load_all(self, why=None): + if self.full: + return + for rid, record in self.source.records(self.key, None, self.pinned).items(): + if rid not in self.seen: + self.seen.add(rid) + self._fill(rid, record) + self.full = True + for view in list(self.views.values()): + items = [(k, v) for k, v in dict.items(view) if k is not _HOLE] + dict.clear(view) + dict.update(view, sorted(items, key=lambda kv: _sort_key(kv[0]))) + + def any(self, kind): + return self.source.any_record(self.key, kind, self.seen, self.pinned) + + def settle(self): + """Before this thread writes: keep the records as they are now (memo texts, else one read).""" + if self.full or self.pinned is not None: + return + found, raw = self.source.peek_table(self.key) + if found and raw is not None: + self.pinned = raw + else: + self.load_all('settled') + + +def read_entry(source, key, definition): + """A private table entry: `definition` (internal keys removed) plus lazy `rows` and row maps.""" + if not isinstance(definition, dict): + return definition + maps = [m for m in (definition.get(MAPS_KEY) or ()) if m in ROW_MAPS] + entry = {k: v for k, v in definition.items() if k not in INTERNAL_KEYS} + state = _ReadTable(source, key, maps) + views = {} + if not definition.get(NO_ROWS_KEY) and 'rows' not in definition: + views['rows'] = _LazyMap(state, 'rows') + for map_key in maps: + views[map_key] = _LazyMap(state, map_key) + if views: + state.views.update(views) + entry.update(views) + _register_lazy(state, source.name, source.tenant_slug) + return {k: entry[k] for k in ordered_keys(entry)} + + +def _records_of(raw, rids=None): + """`{rid: [value, meta]}` parsed from row texts: every record, or only `rids`.""" + if raw is None: + return {} + if rids is None: + return _records_from(raw[1]) + index = _raw_index(raw) + return _records_from((rid,) + index[rid] for rid in rids if rid in index) + + +def _raw_index(raw): + """`{rid: (value text, meta text)}` of row texts, built once per texts object.""" + index = getattr(raw, 'rid_index', None) # not `index`: tuple has an `index` method + if index is None: + index = {rid: (v, m) for rid, v, m in raw[1]} + try: + raw.rid_index = index + except AttributeError: + pass + return index + + +#: Whole-table record loads one read-only document makes before it reads every other table's +#: records in one statement (`ut_whole`): a caller walking every table pays one read, not 70. +BULK_TABLES = 8 +#: Single definition reads one request makes before it reads every definition once (`ut_defs`). +DEFS_AFTER = 2 + + +class _ReadSource: + """Unlocked reads behind read-only answers, through the request memo.""" + + def __init__(self, name, tenant_slug): + self.name, self.tenant_slug = name, tenant_slug + self.full_reads = set() + self.rest = None + + def peek_table(self, key): + """`(found, row texts)` of one table from this request's memo, without a read.""" + found, _rev, raw = _memo_get(self.name, self.tenant_slug, ('ut_table', str(key))) + return found, raw + + def keys(self): + for shape in (('ut_defs',), ('ut_keys',)): + found, _rev, payload = _memo_get(self.name, self.tenant_slug, shape) + if found: + return [k if shape == ('ut_keys',) else k[0] for k in payload] + rev, keys = _read_keys_rev(self.name, self.tenant_slug) + _memo_put(self.name, self.tenant_slug, ('ut_keys',), rev, keys) + return list(keys) + + def definition(self, key): + """One parsed definition, internal keys kept, or None when the table is absent.""" + key = str(key) + found, raw = self.peek_table(key) + if found: + return None if raw is None else _parse(raw[0]) + found, _rev, defs = _memo_get(self.name, self.tenant_slug, ('ut_defs',)) + if found: + text = dict(defs).get(key) + return None if text is None else _parse(text) + found, _rev, text = _memo_get(self.name, self.tenant_slug, ('ut_def', key)) + if not found: + if _count_def_miss() > DEFS_AFTER: + text = dict(self._definition_texts()).get(key) + return None if text is None else _parse(text) + rev, text = _read_def_rev(self.name, self.tenant_slug, key) + _memo_put(self.name, self.tenant_slug, ('ut_def', key), rev, text) + return None if text is None else _parse(text) + + def _definition_texts(self, traced=True): + found, _rev, defs = _memo_get(self.name, self.tenant_slug, ('ut_defs',)) + if not found: + read = _read_defs_rev if traced else _read_defs_rev.__wrapped__ + rev, defs = read(self.name, self.tenant_slug) + _memo_put(self.name, self.tenant_slug, ('ut_defs',), rev, defs) + return defs + + def definitions(self): + """`{key: parsed definition}` of every table in one read (`ut_defs`), in jsonb order.""" + return {k: _parse(text) for k, text in self._definition_texts()} + + def records(self, key, rids, pinned=None): + """`{rid: [value, meta]}`: every record (`ut_table`, memoised) or the named ones (`ut_record`).""" + raw = pinned + if raw is None: + found, raw = self.peek_table(key) + if not found: + if rids is None: + return _records_of(self._whole_table(key)) + return _read_named_records(self.name, self.tenant_slug, key, list(rids)) + return _records_of(raw, None if rids is None else list(rids)) + + def _whole_table(self, key): + """One table's row texts; past `BULK_TABLES` loads, every other table's in one read.""" + if self.rest is None and len(self.full_reads - {key}) >= BULK_TABLES: + self.rest = _read_rest_records(self.name, self.tenant_slug, sorted(self.full_reads)) + if self.rest is not None and key not in self.full_reads: + self.full_reads.add(key) + return self.rest.pop(key, None) or _Raw((None, [])) + self.full_reads.add(key) + return table_raw(self.name, self.tenant_slug, key) + + def any_record(self, key, kind, exclude, pinned=None): + raw = pinned + if raw is None: + found, raw = self.peek_table(key) + raw = raw if found else None + if raw is None or not isinstance(raw, tuple): + params = (str(key), [str(r) for r in exclude]) + (() if kind == 'rows' else (kind,)) + return bool(_pg()._read(_pg()._pool(), _sql(_any_sql(kind), self.tenant_slug), params, one=True)[0]) + for rid, value, meta in raw[1]: + if rid in exclude: + continue + if kind == 'rows' and value is not None: + return True + if kind != 'rows' and meta and kind in (_parse(meta) or {}): + return True + return False + + +# -------------------------------------------------------------------------------------------- +# SQL. Every function addresses one tenant schema through `core.store_pg`'s pool and guard. +# -------------------------------------------------------------------------------------------- +def _pg(): + from core import store_pg + return store_pg + + +def _sql(template, tenant_slug, name=DOCUMENT_KEY): + """Compose `{defs}`, `{recs}`, `{kv}` (qualified) and `{kv_bare}` for one tenant schema.""" + from psycopg import sql + pg = _pg() + return sql.SQL(template).format(defs=pg._table(DEFS, tenant_slug), + recs=pg._table(RECORDS, tenant_slug), + kv=pg._table('store_kv', tenant_slug), + kv_bare=sql.Identifier('store_kv')) + + +def ensure_schema(con, tenant_slug): + """Create the two row tables in the tenant schema when they are missing.""" + con.execute(_sql('CREATE TABLE IF NOT EXISTS {defs} (' + 'table_key text PRIMARY KEY, value jsonb NOT NULL, ' + 'rev bigint NOT NULL DEFAULT 1, updated_at timestamptz NOT NULL DEFAULT now())', + tenant_slug)) + con.execute(_sql('CREATE TABLE IF NOT EXISTS {recs} (' + 'table_key text NOT NULL, row_id text NOT NULL, value jsonb, meta jsonb, ' + 'rev bigint NOT NULL DEFAULT 1, updated_at timestamptz NOT NULL DEFAULT now(), ' + 'PRIMARY KEY (table_key, row_id))', tenant_slug)) + + +_MODE = {} +_MODE_LOCK = threading.Lock() + + +def _mode_key(pool, name, tenant_slug): + return (id(pool), _pg()._effective_slug(tenant_slug), str(name)) + + +def rows_mode(name, tenant_slug=None): + """True when `name` is stored as rows in this tenant's schema. True is cached per pool.""" + if not str(name).endswith(DOCUMENT_KEY): + return False + pg = _pg() + pool = pg._pool() + # A pool this module did not build is a gate's stub: stubs stay in document mode. + if type(pool) not in tuple(pg._HOLDER_POOL_CLASS.values()): + return False + key = _mode_key(pool, name, tenant_slug) + with _MODE_LOCK: + if _MODE.get(key): + return True + + def probe(): + row = pg._read(pool, _sql( + 'SELECT CASE WHEN pg_column_size(value) < 8192 THEN value ->> %s::text END ' + 'FROM {kv} WHERE key = %s', tenant_slug), (LAYOUT_KEY, str(name)), one=True) + return 'rows' if row and row[0] == 'rows' else 'document' + + answer = pg._request_text(name, 'rows_mode', tenant_slug, probe) + if answer == 'rows': + remember_mode(name, tenant_slug) + return answer == 'rows' + + +def remember_mode(name, tenant_slug=None, value=True): + """Cache (or forget) row mode for this process's pool.""" + key = _mode_key(_pg()._pool(), name, tenant_slug) + with _MODE_LOCK: + if value: + _MODE[key] = True + else: + _MODE.pop(key, None) + + +def _parse(text): + return json.loads(text) if isinstance(text, str) else text + + +def _record_bytes(op, name, size): + _request_timing.record_store(op, str(name), bytes_read=size, count=0) + + +def _table_rows(con, tenant_slug, table_key): + """`(def_text, [(row_id, value_text, meta_text)])` for one table, in one statement snapshot.""" + rows = con.execute(_sql( + 'SELECT 0, NULL, value::text, NULL FROM {defs} WHERE table_key = %s UNION ALL ' + 'SELECT 1, row_id, value::text, meta::text FROM {recs} WHERE table_key = %s', + tenant_slug), (str(table_key), str(table_key))).fetchall() + head = [r for r in rows if r[0] == 0] + if not head: + return None + return head[0][2], [(r[1], r[2], r[3]) for r in rows if r[0] == 1] + + +def _split_from_rows(raw): + def_text, records = raw + return _parse(def_text), {rid: [ABSENT if v is None else _parse(v), _parse(m) if m else None] + for rid, v, m in records} + + +def _raw_bytes(raw): + if raw is None: + return 0 + return len(raw[0] or '') + sum(len(v or '') + len(m or '') for _rid, v, m in raw[1]) + + +def load_locked(con, tenant_slug, table_key): + """Lock one table's definition row FOR UPDATE and read the table (table mode).""" + con.execute(_sql('SELECT 1 FROM {defs} WHERE table_key = %s FOR UPDATE', tenant_slug), + (str(table_key),)) + raw = _table_rows(con, tenant_slug, table_key) + return None if raw is None else _split_from_rows(raw) + + +def table_keys(con, tenant_slug): + rows = con.execute(_sql('SELECT table_key FROM {defs}', tenant_slug)).fetchall() + return ordered_keys(r[0] for r in rows) + + +def queue_table_keys(con, tenant_slug): + """Send the table-keys read; its cursor answers after the caller's pipeline syncs.""" + return con.execute(_sql('SELECT table_key FROM {defs}', tenant_slug)) + + +_RECORDS_SQL = ('SELECT row_id, value::text, meta::text FROM {recs} WHERE table_key = %s ' + 'AND row_id = ANY(%s::text[]) ORDER BY octet_length(row_id), row_id COLLATE "C"') +_ALL_RECORDS_SQL = 'SELECT row_id, value::text, meta::text FROM {recs} WHERE table_key = %s' + + +def _records_from(rows): + return {rid: [ABSENT if v is None else _parse(v), _parse(m) if m else None] for rid, v, m in rows} + + +def _any_sql(kind): + """EXISTS over one table's records outside `exclude`: a stored record, or one row-map entry.""" + test = 'value IS NOT NULL' if kind == 'rows' else 'meta ? %s::text' + return ('SELECT EXISTS (SELECT 1 FROM {recs} WHERE table_key = %s ' + f'AND NOT (row_id = ANY(%s::text[])) AND {test})') + + +def table_lock_name(tenant_slug, table_key): + """The text whose `hashtextextended` keys one table's advisory lock.""" + return f'{_pg()._schema_for(tenant_slug)}.{DEFS}.{table_key}' + + +class RecordAccess: + """Record-mode SQL inside one store write transaction. `target` names the lock being waited on.""" + + def __init__(self, con, tenant_slug): + self.con, self.tenant_slug = con, tenant_slug + self.target = None + + def _run(self, target, template, params): + self.target = target + rows = self.con.execute(_sql(template, self.tenant_slug), params).fetchall() + self.target = None + return rows + + def _table_lock(self, table_key, shared): + """The table's advisory lock: shared for record mode, exclusive for table mode. + + Postgres queues advisory locks in order, so a waiting table-mode writer is not starved by + record writers that keep re-taking a shared row lock on the definition. + """ + call = 'pg_advisory_xact_lock_shared' if shared else 'pg_advisory_xact_lock' + self._run(('advisory', str(table_key), None), f'SELECT {call}(hashtextextended(%s, 0))', + (table_lock_name(self.tenant_slug, table_key),)) + + def definition(self, table_key): + """The definition row `FOR SHARE` (record writers share it), or None when absent.""" + lock = read = None + self.target = ('advisory', str(table_key), None) + try: + # The advisory lock, then the row lock, in one round trip. + with self.con.pipeline(): + lock = self.con.execute(_sql('SELECT pg_advisory_xact_lock_shared(hashtextextended(%s, 0))', + self.tenant_slug), (table_lock_name(self.tenant_slug, table_key),)) + read = self.con.execute(_sql('SELECT value::text FROM {defs} WHERE table_key = %s FOR SHARE', + self.tenant_slug), (str(table_key),)) + except Exception: + if lock is not None and lock.pgresult is not None: + self.target = (DEFS, str(table_key), None) + raise + self.target = None + rows = read.fetchall() + return _parse(rows[0][0]) if rows else None + + def table(self, table_key): + """Table mode: the definition row `FOR UPDATE`, then every record of the table.""" + self._table_lock(table_key, shared=False) + self.target = (DEFS, str(table_key), None) + split = load_locked(self.con, self.tenant_slug, table_key) + self.target = None + return split + + def records(self, table_key, rids): + """`{rid: [value, meta]}` for `rids` locked `FOR UPDATE` in id order; None reads all unlocked.""" + if rids is None: + rows = self._run((RECORDS, str(table_key), None), _ALL_RECORDS_SQL, (str(table_key),)) + else: + rids = [str(r) for r in rids] + rows = self._run((RECORDS, str(table_key), rids[0] if rids else None), + _RECORDS_SQL + ' FOR UPDATE', (str(table_key), rids)) + return _records_from(rows) + + def any_record(self, table_key, kind, exclude): + params = (str(table_key), [str(r) for r in exclude]) + (() if kind == 'rows' else (kind,)) + return bool(self._run(None, _any_sql(kind), params)[0][0]) + + def document(self): + """The whole document in one unlocked statement (a record-mode attempt that escalates).""" + return _assemble_rows(self._run(None, _DOCUMENT_SQL, ())) + + +class DetachedRecords: + """Unlocked record reads for a document whose write transaction committed.""" + + def __init__(self, tenant_slug, name=DOCUMENT_KEY): + self.tenant_slug, self.name = tenant_slug, name + + def records(self, table_key, rids): + params = (str(table_key),) if rids is None else (str(table_key), [str(r) for r in rids]) + rows = _pg()._read(_pg()._pool(), _sql(_ALL_RECORDS_SQL if rids is None else _RECORDS_SQL, + self.tenant_slug), params) + _record_bytes('ut_records', self.name, sum(len(v or '') + len(m or '') for _r, v, m in rows)) + return _records_from(rows) + + def any_record(self, table_key, kind, exclude): + params = (str(table_key), [str(r) for r in exclude]) + (() if kind == 'rows' else (kind,)) + return bool(_pg()._read(_pg()._pool(), _sql(_any_sql(kind), self.tenant_slug), params, one=True)[0]) + + +def marker_rev(con_or_none, tenant_slug, name=DOCUMENT_KEY): + """The marker row's `rev`, read without a lock (0 when absent).""" + def read(con): + row = con.execute(_sql('SELECT rev FROM {kv} WHERE key = %s', tenant_slug), + (str(name),)).fetchone() + return int(row[0]) if row and row[0] is not None else 0 + if con_or_none is not None: + return read(con_or_none) + row = _pg()._read(_pg()._pool(), _sql('SELECT rev FROM {kv} WHERE key = %s', tenant_slug), + (str(name),), one=True) + return int(row[0]) if row and row[0] is not None else 0 + + +# -- detached documents a row-mode `get` returns -------------------------------------------- +_LAZY = {} # id(obj) -> weakref: a RowsDocument is unhashable +_LAZY_LOCK = threading.Lock() + + +def _forget_lazy(obj_id): + with _LAZY_LOCK: + _LAZY.pop(obj_id, None) + + +def _register_lazy(obj, name, tenant_slug): + """Track a lazy answer for `settle_lazy`. The registry holds it weakly.""" + obj._lazy_owner = (threading.get_ident(), _pg()._effective_slug(tenant_slug), str(name)) + obj_id = id(obj) + with _LAZY_LOCK: + _LAZY[obj_id] = weakref.ref(obj, lambda _ref: _forget_lazy(obj_id)) + + +def lazy_document(name, tenant_slug): + """The row-mode document as a read-only `RowsDocument`: nothing whole is ever read. + + Membership, `len` and `keys` read one list of table keys (`ut_keys`). A table's first access + reads its definition (`ut_def`); `items()`, `values()`, `dict()` and `json.dumps` read every + definition in one statement (`ut_defs`). Each entry's `rows` and row maps are lazy maps: a + named id reads one record (`ut_record`), iteration reads that table once (`ut_table`). + """ + source = _ReadSource(name, tenant_slug) + keys = source.keys() + doc = RowsDocument(keys, None) + doc.read_lazily(source) + return doc + + +def lazy_table(name, tenant_slug, table_key): + """One table entry with its definition parsed now and lazy `rows` and row maps, or None.""" + source = _ReadSource(name, tenant_slug) + return read_entry(source, str(table_key), source.definition(str(table_key))) + + +def settle_lazy(name, tenant_slug): + """Before a write, settle every lazy answer this thread holds for this name. + + A handed-out table keeps the records it would have read before the write. Tables of a + read-only document that nobody opened stay unread and answer from the store when opened. + """ + owner = (threading.get_ident(), _pg()._effective_slug(tenant_slug), str(name)) + with _LAZY_LOCK: + held = [d for d in (ref() for ref in list(_LAZY.values())) + if d is not None and getattr(d, '_lazy_owner', None) == owner] + for obj in held: + obj.settle() + _forget_lazy(id(obj)) + + +def _read_table_split_traced(name, tenant_slug, table_key): + """`(definition, records)` of one table for a detached document, through the request memo.""" + raw = table_raw(name, tenant_slug, table_key) + return None if raw is None else _split_from_rows(raw) + + +@_request_timing.store_operation('ut_record') +def _read_record_table_now(name, tenant_slug, table_key, row_ids): + """`read_record_table` read from the store now.""" + rows = _pg()._read(_pg()._pool(), _sql( + 'SELECT 0, NULL, value::text, NULL FROM {defs} WHERE table_key = %s UNION ALL ' + 'SELECT 1, row_id, value::text, meta::text FROM {recs} WHERE table_key = %s ' + 'AND row_id = ANY(%s::text[])', tenant_slug), + (str(table_key), str(table_key), [str(r) for r in row_ids])) + head = [r for r in rows if r[0] == 0] + raw = None if not head else (head[0][2], [(r[1], r[2], r[3]) for r in rows if r[0] == 1]) + _record_bytes('ut_record', name, _raw_bytes(raw)) + if raw is None: + return None + definition, records = _split_from_rows(raw) + return assemble_table(definition, [(rid, rec[0], rec[1]) for rid, rec in records.items()]) + + +def read_record_table(name, tenant_slug, table_key, row_ids): + """One table entry holding only the records `row_ids` and their row-map entries, or None. + + A table this request already read whole answers from the memo texts without a read. + """ + found, raw = _ReadSource(name, tenant_slug).peek_table(table_key) + if not found: + return _read_record_table_now(name, tenant_slug, table_key, row_ids) + if raw is None: + return None + records = _records_of(raw, [str(r) for r in row_ids]) + return assemble_table(_parse(raw[0]), [(rid, rec[0], rec[1]) for rid, rec in records.items()]) + + +@_request_timing.store_operation('ut_def') +def _read_definition_now(name, tenant_slug, table_key): + """`read_definition` read from the store now.""" + row = _pg()._read(_pg()._pool(), _sql('SELECT value::text FROM {defs} WHERE table_key = %s', tenant_slug), + (str(table_key),), one=True) + text = row[0] if row else None + _record_bytes('ut_def', name, len(text or '')) + return text + + +def read_definition(name, tenant_slug, table_key): + """One definition text without `rows` or row maps, or None when absent. + + A definition this request already holds in the memo answers without a read. + """ + key = str(table_key) + source = _ReadSource(name, tenant_slug) + found, raw = source.peek_table(key) + if found: + return None if raw is None else raw[0] + for shape in (('ut_defs',), ('ut_def', key)): + found, _rev, payload = _memo_get(name, tenant_slug, shape) + if found: + return dict(payload).get(key) if shape == ('ut_defs',) else payload + return _read_definition_now(name, tenant_slug, table_key) + + +def definition_view(text, drop=('rows',) + ROW_MAPS): + """The parsed definition `read_definition` returned, internal keys removed.""" + definition = _parse(text) + return _definition_view(definition, frozenset(drop), {}) if text is not None else None + + +def read_definitions(name, tenant_slug, drop): + """Every definition with `drop` removed; each row map not in `drop` is rebuilt from meta. + + The definition texts and each table's row-map rows (`ut_maps`) come from the memo. Any miss + reads them, counted as one `ut_defs` operation. A row map of a table whose rows the memo + holds is rebuilt from those texts. + """ + drop = frozenset(str(d) for d in drop) + found, _rev, texts = _memo_get(name, tenant_slug, ('ut_defs',)) + if found: + answer = _definitions(name, tenant_slug, drop, texts, memo_only=True) + if answer is not None: + return answer + return _read_definitions_now(name, tenant_slug, drop) + + +@_request_timing.store_operation('ut_defs') +def _read_definitions_now(name, tenant_slug, drop): + texts = _ReadSource(name, tenant_slug)._definition_texts(traced=False) + return _definitions(name, tenant_slug, drop, texts, memo_only=False) + + +def _scan_row_maps(name, tenant_slug, keys, wanted): + """`(marker rev, {table_key: [(rid, meta_text)]})` of the records holding a wanted row map.""" + rows = _pg()._read(_pg()._pool(), _sql( + 'SELECT 0, NULL, NULL, (SELECT k.rev::text FROM {kv} AS k WHERE k.key = %s) UNION ALL ' + 'SELECT 1, table_key, row_id, meta::text FROM {recs} ' + 'WHERE table_key = ANY(%s::text[]) AND meta ?| %s::text[]', tenant_slug), + (str(name), list(keys), list(wanted))) + _record_bytes('ut_defs', name, sum(len(r[3] or '') for r in rows if r[0] == 1)) + head = [r[3] for r in rows if r[0] == 0] + rev = int(head[0]) if head and head[0] is not None else None + found = {} + for _kind, key, rid, meta in (r for r in rows if r[0] == 1): + found.setdefault(key, []).append((rid, meta)) + return rev, found + + +def _definitions(name, tenant_slug, drop, texts, *, memo_only): + """`read_definitions` from `texts`; None when `memo_only` and a row map is not in the memo.""" + wanted = tuple(m for m in ROW_MAPS if m not in drop) + source = _ReadSource(name, tenant_slug) + parsed = {key: _parse(text) for key, text in texts} + mapped, scan = {}, [] + for key, definition in parsed.items(): + if not wanted or not isinstance(definition, dict): + continue + if not set(definition.get(MAPS_KEY) or ()) & set(wanted): + continue + found, raw = source.peek_table(key) + if found and raw is not None: + for rid, _value, meta in raw[1]: + if meta: + _collect_maps(mapped, key, rid, _parse(meta), wanted) + continue + found, _rev, held = _memo_get(name, tenant_slug, ('ut_maps', key, wanted)) + if not found: + scan.append(key) + continue + for rid, meta in held: + _collect_maps(mapped, key, rid, _parse(meta) or {}, wanted) + if scan: + if memo_only: + return None + rev, read = _scan_row_maps(name, tenant_slug, scan, wanted) + for key in scan: + held = tuple(read.get(key, ())) + _memo_put(name, tenant_slug, ('ut_maps', key, wanted), rev, held) + for rid, meta in held: + _collect_maps(mapped, key, rid, _parse(meta) or {}, wanted) + out = {} + for key, definition in parsed.items(): + maps = {m: {rid: v for rid, v in sorted(items, key=lambda i: _sort_key(i[0]))} + for m, items in mapped.get(key, {}).items()} + out[key] = _definition_view(definition, drop, maps) + return {k: out[k] for k in ordered_keys(out)} + + +def _collect_maps(mapped, key, rid, meta, wanted): + for map_key in wanted: + if map_key in meta: + mapped.setdefault(key, {}).setdefault(map_key, []).append((rid, meta[map_key])) + + +@_request_timing.store_operation('ut_table') +def _read_table_raw(name, tenant_slug, table_key): + """`(def_text, [(row_id, value_text, meta_text)])` of one table read now, or None when absent. + + The answer's `rev` is the marker `rev` read in the same statement snapshot. + """ + rows = _pg()._read(_pg()._pool(), _sql( + 'SELECT 0, NULL, d.value::text, (SELECT k.rev::text FROM {kv} AS k WHERE k.key = %s) ' + 'FROM {defs} AS d WHERE d.table_key = %s UNION ALL ' + 'SELECT 1, row_id, value::text, meta::text FROM {recs} WHERE table_key = %s', + tenant_slug), (str(name), str(table_key), str(table_key))) + head = [r for r in rows if r[0] == 0] + raw = None + if head: + raw = _Raw((head[0][2], [(r[1], r[2], r[3]) for r in rows if r[0] == 1])) + raw.rev = int(head[0][3]) if head[0][3] is not None else None + _record_bytes('ut_table', name, _raw_bytes(raw)) + return raw + + +class _Raw(tuple): + """One table's row texts `(def_text, [(rid, value_text, meta_text)])`, with the marker `rev`.""" + rev = None + + +@_request_timing.store_operation('ut_def') +def _read_def_rev(name, tenant_slug, table_key): + """`(marker rev, definition text or None)` in one statement.""" + row = _pg()._read(_pg()._pool(), _sql('SELECT d.value::text, k.rev FROM {kv} AS k LEFT JOIN {defs} AS d ' + 'ON d.table_key = %s WHERE k.key = %s', tenant_slug), + (str(table_key), str(name)), one=True) + text = row[0] if row else None + _record_bytes('ut_def', name, len(text or '')) + return (int(row[1]) if row and row[1] is not None else None), text + + +@_request_timing.store_operation('ut_defs') +def _read_defs_rev(name, tenant_slug): + """`(marker rev, [(table_key, definition text)])` of every table, in jsonb key order.""" + rows = _pg()._read(_pg()._pool(), _sql('SELECT d.table_key, d.value::text, (SELECT k.rev FROM {kv} AS k ' + 'WHERE k.key = %s) FROM {defs} AS d', tenant_slug), (str(name),)) + _record_bytes('ut_defs', name, sum(len(r[1] or '') for r in rows)) + rev = int(rows[0][2]) if rows and rows[0][2] is not None else None + return rev, sorted(((r[0], r[1]) for r in rows), key=lambda kv: _sort_key(kv[0])) + + +def _read_keys_rev(name, tenant_slug): + """`(marker rev, [table_key])` in jsonb key order (`ut_keys`).""" + rows = _pg()._read(_pg()._pool(), _sql('SELECT d.table_key, (SELECT k.rev FROM {kv} AS k WHERE k.key = %s) ' + 'FROM {defs} AS d', tenant_slug), (str(name),)) + keys = ordered_keys(r[0] for r in rows) + _record_bytes('ut_keys', name, sum(len(k) for k in keys)) + _request_timing.record_store('ut_keys', str(name), count=1) + return (int(rows[0][1]) if rows and rows[0][1] is not None else None), keys + + +@_request_timing.store_operation('ut_whole') +def _read_rest_records(name, tenant_slug, exclude): + """`{table_key: row texts}` of every record outside the tables `exclude`, in one statement.""" + rows = _pg()._read(_pg()._pool(), _sql('SELECT table_key, row_id, value::text, meta::text FROM {recs} ' + 'WHERE NOT (table_key = ANY(%s::text[]))', tenant_slug), + ([str(k) for k in exclude],)) + _record_bytes('ut_whole', name, sum(len(v or '') + len(m or '') for _k, _r, v, m in rows)) + out = {} + for key, rid, value, meta in rows: + out.setdefault(key, _Raw((None, [])))[1].append((rid, value, meta)) + return out + + +def _count_def_miss(): + """This request's single definition reads so far, this one included (0 outside a request).""" + acc = _request_timing.CURRENT.get() + if acc is None: + return 0 + with acc.lock: + acc.ut_def_misses = getattr(acc, 'ut_def_misses', 0) + 1 + return acc.ut_def_misses + + +@_request_timing.store_operation('ut_record') +def _read_named_records(name, tenant_slug, table_key, row_ids): + """`{rid: [value, meta]}` of the named record rows of one table, unlocked.""" + rows = _pg()._read(_pg()._pool(), _sql(_RECORDS_SQL, tenant_slug), + (str(table_key), [str(r) for r in row_ids])) + _record_bytes('ut_record', name, sum(len(v or '') + len(m or '') for _r, v, m in rows)) + return _records_from(rows) + + +def _assemble_raw(raw): + if raw is None: + return None + definition, records = _split_from_rows(raw) + return assemble_table(definition, [(rid, rec[0], rec[1]) for rid, rec in records.items()]) + + +#: Text bytes one request's table memo may hold. Past it, reads are not memoised. +TABLE_MEMO_BYTES = 32 * 1024 * 1024 + + +def _table_memo(): + """`(accumulator, {key: (rev, payload, bytes)})` of the current request, or `(None, None)`. + + Not `request_timing`'s read memo: that one ends 2 s into a request, and a cold grid request + reads its table eight times over 4 s. Keys are `store_pg._request_key` tuples, so they carry + the write generation. Shapes: `('ut_table', key)` row texts, `('ut_def', key)` one definition + text, `('ut_defs',)` every `(key, text)`, `('ut_keys',)` the table keys, `('ut_maps', key, + wanted)` the `(rid, meta)` texts of one table's records that hold a wanted row map. + """ + acc = _request_timing.CURRENT.get() + if acc is None or acc.closed or acc.cancelled: + return None, None + with acc.lock: + memo = getattr(acc, 'ut_table_memo', None) + if memo is None: + memo = acc.ut_table_memo = {} + return acc, memo + + +def _payload_bytes(shape, payload): + if payload is None: + return 0 + if shape[0] == 'ut_table': + return _raw_bytes(payload) + if shape[0] == 'ut_def': + return len(payload) + if shape[0] == 'ut_defs': + return sum(len(k) + len(t or '') for k, t in payload) + if shape[0] == 'ut_maps': + return sum(len(rid) + len(meta or '') for rid, meta in payload) + return sum(len(k) for k in payload) + + +def _memo_get(name, tenant_slug, shape): + """`(found, rev, payload)` of one memo entry at the current write generation. + + A request memo miss asks the process cache, which answers at this request's marker token. + """ + acc, memo = _table_memo() + if memo is None: + return False, None, None + key = _pg()._request_key(name, shape, tenant_slug) + with acc.lock: + hit = memo.get(key) + if hit is not None: + return True, hit[0], hit[1] + shared = _shared_get(acc, name, tenant_slug, shape) + if shared is None: + return False, None, None + rev, payload = shared + size = _payload_bytes(shape, payload) + with acc.lock: + if sum(v[2] for v in memo.values()) + size <= TABLE_MEMO_BYTES: + memo.setdefault(key, (rev, payload, size)) + return True, rev, payload + + +def _memo_put(name, tenant_slug, shape, rev, payload): + acc, memo = _table_memo() + if memo is None: + return + key = _pg()._request_key(name, shape, tenant_slug) + size = _payload_bytes(shape, payload) + with acc.lock: + if sum(v[2] for v in memo.values()) + size <= TABLE_MEMO_BYTES: + memo[key] = (rev, payload, size) + _shared_put(acc, name, tenant_slug, key, rev, payload, size) + + +# -- the process cache (S79-T49 pass 3) ------------------------------------------------------- +#: Megabytes of text the process cache holds; None reads `AIOS_UT_CACHE_MB` (default 32). +SHARED_CACHE_MB = None +_SHARED = OrderedDict() # (pool, slug, name, shape) -> (payload, bytes), least recent first +_SHARED_TOKEN = {} # (pool, slug, name) -> the marker (rev, updated_at) its entries hold +_SHARED_LOCK = threading.Lock() +_SHARED_BYTES = [0] + + +def _shared_limit(): + mb = SHARED_CACHE_MB + if mb is None: + try: + mb = float(os.environ.get('AIOS_UT_CACHE_MB') or 32) + except ValueError: + mb = 32.0 + return max(0, int(mb * 1024 * 1024)) if mb == mb else 0 + + +def _shared_drop(prefix): + """Forget every entry of one tenant document. Call with `_SHARED_LOCK` held.""" + for key in [k for k in _SHARED if k[:3] == prefix]: + _SHARED_BYTES[0] -= _SHARED.pop(key)[1] + _SHARED_TOKEN.pop(prefix, None) + + +def _shared_adopt(prefix, token): + """Move one tenant document to `token`, dropping entries of another token. Lock held. + + Returns False when the cache already holds a newer `rev`: the caller's view is the stale one. + """ + current = _SHARED_TOKEN.get(prefix) + if current == token: + return True + if current is not None and current[0] > token[0]: + return False + _shared_drop(prefix) + _SHARED_TOKEN[prefix] = token + return True + + +def _shared_store(key, payload, size, limit): + """Insert one entry and evict the least recently used past `limit` bytes. Lock held.""" + old = _SHARED.pop(key, None) + if old is not None: + _SHARED_BYTES[0] -= old[1] + _SHARED[key] = (payload, size) + _SHARED_BYTES[0] += size + while _SHARED_BYTES[0] > limit and _SHARED: + _SHARED_BYTES[0] -= _SHARED.popitem(last=False)[1][1] + + +def _request_token(acc, name, tenant_slug, *, read=True): + """`(prefix, (rev, updated_at) or None)`: the marker this request validated at this generation. + + The first call in a request takes the marker from `store_pg._kv_marker` (else one revision + read); a write in this process + moves the generation, so the next call reads it again unless `carry_memo` moved the token. + """ + head = _pg()._request_key(name, ('token',), tenant_slug) + prefix, generation = head[:3], head[-1] + with acc.lock: + tokens = getattr(acc, 'ut_tokens', None) + if tokens is None: + tokens = acc.ut_tokens = {} + held = tokens.get(prefix) + if held is not None and held[1] == generation: + return prefix, held[0] + if not read: + return prefix, None + # The store's one validation statement per request carries the marker, so this costs no read. + checked = _pg()._kv_marker(name, tenant_slug) + if checked is not None and checked[1] == generation: + token = checked[0] + else: + revision = _pg().revision(name, tenant_slug) + stamp = revision.get('updated_at') + token = (int(revision['rev']), stamp) if stamp is not None and revision.get('rev') else None + with acc.lock: + tokens[prefix] = (token, generation) + return prefix, token + + +def held_marker(name, tenant_slug): + """The marker `(rev, updated_at)` this request holds at the current write generation, or None.""" + acc = _request_timing.CURRENT.get() + if acc is None: + return None + return _request_token(acc, name, tenant_slug, read=False)[1] + + +def _shared_get(acc, name, tenant_slug, shape): + """`(rev, payload)` from the process cache at this request's token, or None.""" + if _shared_limit() <= 0: + return None + prefix, token = _request_token(acc, name, tenant_slug) + if token is None: + return None + with _SHARED_LOCK: + if not _shared_adopt(prefix, token): + return None + hit = _SHARED.get(prefix + (shape,)) + if hit is None: + return None + _SHARED.move_to_end(prefix + (shape,)) + return token[0], hit[0] + + +def _shared_put(acc, name, tenant_slug, key, rev, payload, size): + """Keep one read in the process cache when it was read at this request's validated token.""" + limit = _shared_limit() + if rev is None or size > limit: + return + prefix, shape = key[:3], key[3] + _prefix, token = _request_token(acc, name, tenant_slug, read=False) + if token is None or token[0] != int(rev): + return + with _SHARED_LOCK: + if _shared_adopt(prefix, token): + _shared_store(prefix + (shape,), payload, size, limit) + + +def _shared_carry(prefix, written, by_table, rev): + """Move the process cache past this process's write, or drop it when it cannot follow.""" + stamp = written.get('stamp') if written is not None else None + with _SHARED_LOCK: + current = _SHARED_TOKEN.get(prefix) + if current is None: + return + if stamp is None or not _carry_ok(current[0], rev): + _shared_drop(prefix) + return + for key in [k for k in _SHARED if k[:3] == prefix]: + payload, size = _SHARED.pop(key) + _SHARED_BYTES[0] -= size + new = _carry(key[3], payload, written, by_table) + if new is _DROP: + continue + if isinstance(new, _Raw): + new.rev = rev + new_size = _payload_bytes(key[3], new) + _SHARED[key] = (new, new_size) + _SHARED_BYTES[0] += new_size + _SHARED_TOKEN[prefix] = (rev, stamp) + limit = _shared_limit() + while _SHARED_BYTES[0] > limit and _SHARED: + _SHARED_BYTES[0] -= _SHARED.popitem(last=False)[1][1] + + +def table_raw(name, tenant_slug, table_key): + """One table's row texts (`_read_table_raw`), read once per request. + + The memo key carries `store_pg`'s write generation, so after a write in this process + (`_bump_write_generation`) the next read goes to the store, unless `carry_memo` moved the + entry forward exactly. The texts are immutable: every caller parses its own objects. + """ + shape = ('ut_table', str(table_key)) + found, _rev, raw = _memo_get(name, tenant_slug, shape) + if not found: + raw = _read_table_raw(name, tenant_slug, table_key) + _memo_put(name, tenant_slug, shape, getattr(raw, 'rev', None), raw) + return raw + + +def new_written(): + """The collector `apply_changes` fills with what a write committed, for `carry_memo`.""" + return {'defs': {}, 'records': {}, 'dropped': set(), 'cleared': set()} + + +_DROP = object() + + +def _carry(shape, payload, written, by_table): + """One memo payload after the write `written` describes, or `_DROP` when it cannot follow.""" + kind = shape[0] + defs, dropped, cleared = written['defs'], written['dropped'], written['cleared'] + if kind == 'ut_keys' or kind == 'ut_defs': + if kind == 'ut_keys': + current = {k: None for k in payload} + else: + current = dict(payload) + for key in dropped: + current.pop(key, None) + for key, text in defs.items(): + current[key] = text + ordered = ordered_keys(current) + return ordered if kind == 'ut_keys' else [(k, current[k]) for k in ordered] + key = shape[1] + if key in dropped: + return None if kind != 'ut_maps' else _DROP + if kind == 'ut_def': + return defs.get(key, payload) + if kind == 'ut_maps': + return _carry_maps(key, shape[2], payload, by_table.get(key, {}), key in cleared) + changed = by_table.get(key, {}) + if key in cleared: + if key not in defs: + return _DROP + base = {} + elif payload is None: + return _DROP if (key in defs or changed) else None + else: + base = dict(_raw_index(payload)) + for rid, pair in changed.items(): + if pair is None: + base.pop(rid, None) + else: + base[rid] = pair + raw = _Raw((defs.get(key, payload[0] if payload is not None else None), + [(rid, v, m) for rid, (v, m) in base.items()])) + return raw + + +def _carry_maps(key, wanted, payload, changed, cleared): + """One table's row-map rows after a write: each written record's meta replaces its row.""" + base = {} if cleared else dict(payload) + for rid, pair in changed.items(): + meta = _parse(pair[1]) if pair is not None and pair[1] is not None else None + if isinstance(meta, dict) and any(m in meta for m in wanted): + base[rid] = pair[1] + else: + base.pop(rid, None) + return tuple(base.items()) + + +def _carry_ok(memo_rev, rev): + """True when the write that set marker `rev` is the only one since the memo read `memo_rev`.""" + return type(rev) is int and memo_rev is not None and int(memo_rev) + 1 == rev + + +def carry_memo(name, tenant_slug, written, rev): + """After this process's write: move the memos to the next write generation. + + Each entry becomes the state the write committed: the definition and record texts Postgres + returned, applied to the memo texts. It moves only when the memo was read at marker `rev - 1`, + so no other write landed between that read and this one. The process cache moves the same way + and this request's token moves with it. Returns the request entries carried. + Call it after the commit and before `store_pg._bump_write_generation`. + """ + pg = _pg() + head = pg._request_key(name, ('carry',), tenant_slug) + prefix, generation = head[:3], head[-1] + by_table = {} + if written is not None: + for (key, rid), pair in written['records'].items(): + by_table.setdefault(key, {})[rid] = pair + _shared_carry(prefix, written, by_table, rev) + acc, memo = _table_memo() + if memo is None: + return 0 + stamp = written.get('stamp') if written is not None else None + carried = 0 + with acc.lock: + tokens = getattr(acc, 'ut_tokens', None) + if tokens is None: + tokens = acc.ut_tokens = {} + held = tokens.pop(prefix, None) + if (held is not None and held[0] is not None and held[1] == generation and stamp is not None + and _carry_ok(held[0][0], rev)): + tokens[prefix] = ((rev, stamp), generation + 1) + for key in [k for k in memo if k[:3] == prefix]: + mem_rev, payload, _size = memo.pop(key) + if written is None or key[-1] != generation or not _carry_ok(mem_rev, rev): + continue + new = _carry(key[3], payload, written, by_table) + if new is _DROP: + continue + if isinstance(new, _Raw): + new.rev = rev + memo[key[:-1] + (generation + 1,)] = (rev, new, _payload_bytes(key[3], new)) + carried += 1 + return carried + + +def read_table(name, tenant_slug, table_key): + """One assembled table entry (definition, records and row maps), or None when absent.""" + return _assemble_raw(table_raw(name, tenant_slug, table_key)) + + +def read_table_split(tenant_slug, table_key, name=DOCUMENT_KEY): + """`(definition, records)` for one table without a lock, or None (a detached document's loader).""" + with _pg()._pool().connection() as con: + raw = _table_rows(con, tenant_slug, table_key) + return None if raw is None else _split_from_rows(raw) + + +@_request_timing.store_operation('ut_records') +def read_records(name, tenant_slug, table_key, row_ids=None): + """`(rows_kind, {row_id: value})` for one table; `rows_kind` is None when the table is absent. + + `row_ids=None` reads every record. `rows_kind` is 'object', or the JSON type of a `rows` value + that the definition kept because it was not an object. + """ + with _pg()._pool().connection() as con: + head = con.execute(_sql( + "SELECT jsonb_typeof(value), CASE WHEN value ? 'rows' THEN " + "coalesce(jsonb_typeof(value -> 'rows'), 'null') END, value ? %s::text " + 'FROM {defs} WHERE table_key = %s', tenant_slug), (NO_ROWS_KEY, str(table_key))).fetchone() + if head is None or head[0] != 'object' or head[2] or head[1] is not None: + _record_bytes('ut_records', name, 0) + return (None if head is None or head[0] != 'object' or head[2] else head[1]), {} + if row_ids is None: + rows = con.execute(_sql('SELECT row_id, value::text FROM {recs} ' + 'WHERE table_key = %s AND value IS NOT NULL', tenant_slug), + (str(table_key),)).fetchall() + else: + rows = con.execute(_sql('SELECT row_id, value::text FROM {recs} WHERE table_key = %s ' + 'AND row_id = ANY(%s::text[]) AND value IS NOT NULL', tenant_slug), + (str(table_key), [str(r) for r in row_ids])).fetchall() + _record_bytes('ut_records', name, sum(len(r[1]) for r in rows)) + return 'object', {rid: _parse(v) for rid, v in sorted(rows, key=lambda r: _sort_key(r[0]))} + + +@_request_timing.store_operation('ut_records') +def iter_records(name, tenant_slug, table_key, batch_size=1024): + """Stream one table's records as `[(row_id, value)]` batches through a server cursor.""" + import uuid + kind, _none = read_records.__wrapped__(name, tenant_slug, table_key, row_ids=()) + if kind != 'object': + if kind not in (None, 'null'): + raise RuntimeError('canonical_source_invalid: the stored rows are not an object') + return + with _pg()._pool().connection() as con: + with con.cursor(name='ut_records_' + uuid.uuid4().hex) as cur: + cur.execute(_sql('SELECT row_id, value::text FROM {recs} WHERE table_key = %s ' + 'AND value IS NOT NULL ORDER BY octet_length(row_id), row_id COLLATE "C"', + tenant_slug), (str(table_key),)) + while True: + _request_timing.remaining_budget() + rows = cur.fetchmany(int(batch_size)) + if not rows: + return + _record_bytes('ut_records', name, sum(len(v) for _rid, v in rows)) + yield [(rid, _parse(v)) for rid, v in rows] + + +@_request_timing.store_operation('ut_records') +def read_row_map(name, tenant_slug, table_key, map_key): + """One row map of one table rebuilt from meta, or None when the table does not keep it.""" + with _pg()._pool().connection() as con: + head = con.execute(_sql('SELECT (value -> %s::text) ? %s::text FROM {defs} WHERE table_key = %s', + tenant_slug), (MAPS_KEY, str(map_key), str(table_key))).fetchone() + if not head or not head[0]: + return None + rows = con.execute(_sql('SELECT row_id, (meta -> %s::text)::text FROM {recs} ' + 'WHERE table_key = %s AND meta ? %s::text', tenant_slug), + (str(map_key), str(table_key), str(map_key))).fetchall() + _record_bytes('ut_records', name, sum(len(v) for _rid, v in rows)) + return {rid: _parse(v) for rid, v in sorted(rows, key=lambda r: _sort_key(r[0]))} + + +@_request_timing.store_operation('ut_counts') +def counts_text(name, tenant_slug): + """`{table_key: stored record count}` as JSON text, without reading a record value.""" + rows = _pg()._read(_pg()._pool(), _sql( + "SELECT d.table_key, CASE WHEN jsonb_typeof(d.value) <> 'object' THEN 0 " + 'WHEN d.value ? %s::text THEN 0 ' + "WHEN d.value ? 'rows' THEN CASE WHEN d.value -> 'rows' = 'null'::jsonb THEN 0 END " + 'ELSE (SELECT count(*) FROM {recs} AS r WHERE r.table_key = d.table_key ' + 'AND r.value IS NOT NULL) END FROM {defs} AS d', tenant_slug), (NO_ROWS_KEY,)) + counts = {key: (int(n) if n is not None else None) for key, n in rows} + text = json.dumps({k: counts[k] for k in ordered_keys(counts)}) + _record_bytes('ut_counts', name, len(text)) + return text + + +_DOCUMENT_SQL = ('SELECT 0, table_key, NULL, value::text, NULL FROM {defs} UNION ALL ' + 'SELECT 1, table_key, row_id, value::text, meta::text FROM {recs}') + + +def _document_rows(pool, tenant_slug): + with pool.connection() as con: + return tuple(con.execute(_sql(_DOCUMENT_SQL, tenant_slug)).fetchall()) + + +def _assemble_rows(rows): + tables = {} + for kind, key, rid, value, meta in rows: + if kind == 0: + tables.setdefault(key, [None, {}])[0] = _parse(value) + for kind, key, rid, value, meta in rows: + if kind == 1 and key in tables: + tables[key][1][rid] = [ABSENT if value is None else _parse(value), + _parse(meta) if meta else None] + return assemble_document({k: (v[0], v[1]) for k, v in tables.items()}) + + +@_request_timing.store_operation('ut_whole') +def read_document(name, tenant_slug, fresh=False): + """The assembled whole document: correct, slow and named. Concurrent reads share one SELECT.""" + pg = _pg() + pool = pg._pool() + rows = (_document_rows(pool, tenant_slug) if fresh else + pg._shared_read(name, ('ut_whole',), tenant_slug, + lambda: _document_rows(pool, tenant_slug), pool=pool)) + _record_bytes('ut_whole', name, sum(len(r[3] or '') + len(r[4] or '') for r in rows)) + return _assemble_rows(rows) + + +def value_at(name, tenant_slug, path): + """`(found, value)` at a JSON path, reading only the table, records or row map it names.""" + path = [str(p) for p in path] + if not path: + return True, read_document(name, tenant_slug) + key = path[0] + if len(path) == 1: + entry = read_table(name, tenant_slug, key) + return entry is not None, entry + if path[1] == 'rows': + kind, rows = read_records(name, tenant_slug, key, None if len(path) == 2 else [path[2]]) + if kind is None: + return False, None + if kind != 'object': + entry = read_table(name, tenant_slug, key) or {} + return _walk(entry, path[1:]) + if len(path) == 2: + return True, rows + return _walk(rows, path[2:]) + if path[1] in ROW_MAPS: + mapped = read_row_map(name, tenant_slug, key, path[1]) + if mapped is not None: + return _walk(mapped, path[2:]) + text = read_definition(name, tenant_slug, key) + if text is None: + return False, None + return _walk(definition_view(text, drop=('rows',) + ROW_MAPS), path[1:]) + + +def _walk(value, path): + for part in path: + if isinstance(value, dict) and part in value: + value = value[part] + elif isinstance(value, list) and part.lstrip('-').isdigit() and -len(value) <= int(part) < len(value): + value = value[int(part)] + else: + return False, None + return True, value + + +# -------------------------------------------------------------------------------------------- +# Writes (inside the caller's transaction). +# -------------------------------------------------------------------------------------------- +def apply_changes(con, tenant_slug, changes, written=None): + """Write one `RowsDocument.changes()` result. Returns the number of rows written or deleted. + + With `written` (from `new_written`), it also collects the texts Postgres stored, for + `carry_memo`: each written definition and record as `value::text`, each deleted record as None. + """ + n, finish = queue_changes(con, tenant_slug, changes, written) + finish() + return n + + +_UPSERT_RECORDS = ('INSERT INTO {recs} AS r (table_key, row_id, value, meta) ' + 'SELECT u.k, u.r, u.v::jsonb, u.m::jsonb FROM unnest(%s::text[], %s::text[], ' + '%s::text[], %s::text[]) WITH ORDINALITY AS u(k, r, v, m, i) ORDER BY u.i ' + 'ON CONFLICT (table_key, row_id) DO UPDATE SET value = EXCLUDED.value, ' + 'meta = EXCLUDED.meta, rev = r.rev + 1, updated_at = now()') + + +def queue_changes(con, tenant_slug, changes, written=None): + """Send the statements of `apply_changes` without reading a result: `(rows, finish)`. + + Inside a psycopg pipeline every statement goes in the same round trip. `finish()` reads the + returned texts into `written` after the pipeline syncs. `cursors` on `finish` lists every + statement in order, so a caller can tell which one failed. + """ + n, cursors, reads = 0, [], [] + back = ' RETURNING value::text' if written is not None else '' + + def run(template, params): + cur = con.execute(_sql(template, tenant_slug), params) + cursors.append(cur) + return cur + for key in changes['drop']: + run('DELETE FROM {recs} WHERE table_key = %s', (key,)) + run('DELETE FROM {defs} WHERE table_key = %s', (key,)) + n += 1 + if written is not None: + written['dropped'].add(key) + for key in changes['clear']: + run('DELETE FROM {recs} WHERE table_key = %s', (key,)) + if written is not None: + written['cleared'].add(key) + for key, value in changes['defs'].items(): + cur = run('INSERT INTO {defs} AS d (table_key, value) VALUES (%s, %s::jsonb) ' + 'ON CONFLICT (table_key) DO UPDATE SET value = EXCLUDED.value, ' + 'rev = d.rev + 1, updated_at = now()' + back, (key, value)) + n += 1 + if written is not None: + reads.append(('def', key, cur)) + by_table = {} + for key, rid in changes['delete']: + by_table.setdefault(key, []).append(rid) + for key, rids in by_table.items(): + run('DELETE FROM {recs} WHERE table_key = %s AND row_id = ANY(%s::text[])', (key, rids)) + n += len(rids) + if written is not None: + written['records'].update(((key, rid), None) for rid in rids) + upserts = [rec for rec in changes['records'] if rec[2] is not None or rec[3] is not None] + empties = [(k, rid) for k, rid, v, m in changes['records'] if v is None and m is None] + if upserts: + # One statement for every record: `executemany` costs a round trip of its own in a pipeline. + cur = run(_UPSERT_RECORDS + (' RETURNING table_key, row_id, value::text, meta::text' + if written is not None else ''), + tuple([rec[i] for rec in upserts] for i in range(4))) + if written is not None: + reads.append(('records', None, cur)) + n += len(upserts) + for key, rid in empties: + run('DELETE FROM {recs} WHERE table_key = %s AND row_id = %s', (key, rid)) + if written is not None: + written['records'][(key, rid)] = None + + def finish(): + for kind, key, cur in reads: + if kind == 'def': + written['defs'][key] = cur.fetchone()[0] + else: + for table_key, rid, value, meta in cur.fetchall(): + written['records'][(table_key, rid)] = (value, meta) + finish.cursors = cursors + return n, finish + + +def clear_rows(con, tenant_slug): + """Delete every definition and record row of this tenant (the document was deleted).""" + con.execute(_sql('DELETE FROM {recs}', tenant_slug)) + con.execute(_sql('DELETE FROM {defs}', tenant_slug)) + + +def bump_marker(con, tenant_slug, name=DOCUMENT_KEY, expect_rev=None): + """Advance the marker's `rev` and `updated_at` so every revision token sees the write. + + With `expect_rev`, the bump applies only while the marker still holds that `rev`, in the same + statement that locks it. Returns the new `rev` (always truthy), or False for a stale caller. + """ + return marker_bumped(queue_bump_marker(con, tenant_slug, name, expect_rev), expect_rev) + + +def queue_bump_marker(con, tenant_slug, name=DOCUMENT_KEY, expect_rev=None): + """Send `bump_marker`'s statement; the cursor answers `(rev, updated_at)` or no row.""" + if expect_rev is None: + return con.execute(_sql('UPDATE {kv} SET rev = {kv_bare}.rev + 1, updated_at = now() ' + 'WHERE key = %s RETURNING rev, updated_at', tenant_slug), (str(name),)) + return con.execute(_sql('UPDATE {kv} SET rev = {kv_bare}.rev + 1, updated_at = now() ' + 'WHERE key = %s AND rev = %s RETURNING rev, updated_at', tenant_slug), + (str(name), int(expect_rev))) + + +def marker_bumped(cur, expect_rev=None, written=None): + """`bump_marker`'s answer from its cursor; stores the marker stamp in `written`.""" + row = cur.fetchone() + if row is None: + return True if expect_rev is None else False + if written is not None: + written['stamp'] = row[1].timestamp() if hasattr(row[1], 'timestamp') else None + return int(row[0]) + + +# -------------------------------------------------------------------------------------------- +# Migration (S79-T47 basic, S79-T49 hardened). An ops action: it runs outside any request. +# Every byte stays on the server: digests, copies and the rebuild are SQL, never a client round trip. +# -------------------------------------------------------------------------------------------- +#: Statement timeout for one migrate, verify or rollback statement. Staging's 23.5 MB document +#: migrated in 7.0 s end to end; 57 MB scales to about 17 s, so 300 s keeps a wide margin. +STATEMENT_TIMEOUT_S = 300 + + +class MigrationMismatch(RuntimeError): + """The rows do not reproduce the document (or the reverse); the transaction rolled back.""" + + def __init__(self, action, report): + super().__init__(f'{action} rolled back: {len(report["mismatches"])} table(s) differ, ' + f'first: {", ".join(report["mismatches"][:3])}') + self.report = report + + +def _settings(con, statement_timeout_s, read_only=False): + """Transaction-local settings. A read-only transaction must say so before any other statement.""" + if read_only: + con.execute('SET TRANSACTION READ ONLY') + con.execute("SELECT set_config('statement_timeout', %s, true)", + (f'{int(statement_timeout_s * 1000)}ms',)) + con.execute("SET LOCAL work_mem = '64MB'") + + +def _assembled_select(): + """`(sql, params)` of `SELECT table_key, entry`: every table entry rebuilt from its rows in SQL.""" + n = len(ROW_MAPS) + agg = ''.join(', jsonb_object_agg(row_id, meta -> %s::text) FILTER (WHERE meta ? %s::text) AS m' + + str(i) for i in range(n)) + add = ''.join(" || CASE WHEN (d.value -> %s::text) ? %s::text THEN jsonb_build_object(%s::text, " + 'coalesce(a.m' + str(i) + ", '{{}}'::jsonb)) ELSE '{{}}'::jsonb END" for i in range(n)) + text = ("SELECT d.table_key, CASE WHEN jsonb_typeof(d.value) <> 'object' THEN d.value ELSE " + "(d.value - %s::text - %s::text) || CASE WHEN d.value ? 'rows' OR d.value ? %s::text " + "THEN '{{}}'::jsonb ELSE jsonb_build_object('rows', coalesce(a.rows, '{{}}'::jsonb)) END" + + add + ' END AS entry FROM {defs} AS d LEFT JOIN (SELECT table_key, ' + 'jsonb_object_agg(row_id, value) FILTER (WHERE value IS NOT NULL) AS rows' + agg + + ' FROM {recs} GROUP BY table_key) AS a USING (table_key)') + params = ([MAPS_KEY, NO_ROWS_KEY, NO_ROWS_KEY] + [p for m in ROW_MAPS for p in (MAPS_KEY, m, m)] + + [p for m in ROW_MAPS for p in (m, m)]) + return text, params + + +_DOCUMENT_ENTRIES = ('SELECT e.key, e.value FROM {kv} AS kv CROSS JOIN LATERAL jsonb_each(kv.value) AS e ' + 'WHERE kv.key = %s') + + +def _digest_sql(source): + """Per-table `(key, record count, md5, bytes)` over `source`, a `SELECT key, entry` statement. + + MATERIALIZED: an inlined `source` is rebuilt once per reference, which tripled the time. + """ + return ('WITH t(k, e) AS MATERIALIZED (' + source + '), ' + 'x AS MATERIALIZED (SELECT k, e, e::text AS s FROM t) ' + "SELECT k, CASE WHEN jsonb_typeof(e -> 'rows') = 'object' THEN " + "(SELECT count(*) FROM jsonb_object_keys(e -> 'rows')) ELSE 0 END, " + 'md5(s), octet_length(s) FROM x') + + +def _digest(rows): + return {key: (int(count), md5, int(size)) for key, count, md5, size in rows} + + +def document_digest(con, tenant_slug, name=DOCUMENT_KEY): + """`{table: (records, md5, bytes)}` of the stored document. jsonb text is canonical.""" + return _digest(con.execute(_sql(_digest_sql(_DOCUMENT_ENTRIES), tenant_slug), (str(name),)).fetchall()) + + +def rows_digest(con, tenant_slug): + """`{table: (records, md5, bytes)}` of every table rebuilt from its rows.""" + text, params = _assembled_select() + return _digest(con.execute(_sql(_digest_sql(text), tenant_slug), params).fetchall()) + + +def digest_md5(digest): + """One md5 over every table's `key:records:md5`, in jsonb key order.""" + text = '\n'.join(f'{k}:{digest[k][0]}:{digest[k][1]}' for k in ordered_keys(digest)) + return hashlib.md5(text.encode('utf-8')).hexdigest() + + +def compare_digests(expected, got): + """A per-table report of `got` against `expected`. `ok` only when every table matches.""" + per_table, mismatches = [], [] + for key in ordered_keys(set(expected) | set(got)): + want, have = expected.get(key), got.get(key) + ok = want is not None and have is not None and want[:2] == have[:2] + if not ok: + mismatches.append(key if want and have else f'{key} ({"missing" if want else "extra"})') + shown = have or want + per_table.append({'table': key, 'records': shown[0], 'bytes': shown[2], + 'md5': have[1] if have else None, + 'expected_md5': want[1] if want else None, 'ok': ok}) + return {'ok': not mismatches, 'tables': len(got), 'records': sum(v[0] for v in got.values()), + 'bytes': sum(v[2] for v in got.values()), 'md5': digest_md5(got), + 'per_table': per_table, 'mismatches': mismatches} + + +def _table_exists(con, tenant_slug, table): + qualified = f'"{_pg()._schema_for(tenant_slug)}".{table}' + return con.execute('SELECT to_regclass(%s) IS NOT NULL', (qualified,)).fetchone()[0] + + +def fingerprint(tenant_slug, name=DOCUMENT_KEY): + """What a run that changed nothing leaves equal: the marker, and every row's id, rev and content.""" + with _pg()._pool().connection() as con: + _settings(con, STATEMENT_TIMEOUT_S, read_only=True) + kv = con.execute(_sql('SELECT rev, md5(value::text), updated_at::text FROM {kv} WHERE key = %s', + tenant_slug), (str(name),)).fetchone() + out = {'marker': list(kv) if kv else None, 'defs': None, 'records': None} + if _table_exists(con, tenant_slug, DEFS): + out['defs'] = list(con.execute(_sql( + 'SELECT count(*), coalesce(sum(rev), 0)::bigint, max(updated_at)::text, ' + "md5(coalesce(string_agg(table_key || chr(31) || rev || chr(31) || md5(value::text), " + "chr(30) ORDER BY table_key COLLATE \"C\"), '')) FROM {defs}", tenant_slug)).fetchone()) + if _table_exists(con, tenant_slug, RECORDS): + out['records'] = list(con.execute(_sql( + 'SELECT count(*), coalesce(sum(rev), 0)::bigint, max(updated_at)::text, ' + "md5(coalesce(string_agg(table_key || chr(31) || row_id || chr(31) || rev || chr(31) " + "|| md5(coalesce(value::text, '~')) || md5(coalesce(meta::text, '~')), chr(30) " + "ORDER BY table_key COLLATE \"C\", row_id COLLATE \"C\"), '')) FROM {recs}", + tenant_slug)).fetchone()) + return out + + +def _head(con, tenant_slug, name, lock): + """`(layout, json type, rev)` of the stored value, without detoasting a big document.""" + return con.execute(_sql( + 'SELECT CASE WHEN pg_column_size(value) < 8192 THEN value ->> %s::text END, ' + 'jsonb_typeof(value), rev FROM {kv} WHERE key = %s' + (' FOR UPDATE' if lock else ''), + tenant_slug), (LAYOUT_KEY, str(name))).fetchone() + + +def migrate(tenant_slug, name=DOCUMENT_KEY, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False): + """Copy the document into rows in SQL, prove each table's count and checksum, write the marker. + + One transaction holds the document row FOR UPDATE throughout and compares a per-table digest + of the document with the same digest of the rows. A mismatch raises `MigrationMismatch` and + nothing commits. A tenant already in row mode changes nothing. `dry_run` reports tables, + records and bytes from a read-only transaction. + """ + pg = _pg() + if str(name) != DOCUMENT_KEY: + raise ValueError('only the user_tables document migrates to rows') + if dry_run: + return _dry_run(tenant_slug, name, statement_timeout_s) + pg.check_write(tenant_slug, f'migrate {name!r} to rows') + started = time.monotonic() + with pg._pool().connection() as con: + _settings(con, statement_timeout_s) + head = _head(con, tenant_slug, name, lock=True) + if head and head[0] == 'rows': + return {'changed': False, 'reason': 'already rows'} + if head and head[1] != 'object': + raise ValueError(f'{name!r} is not a JSON object; it cannot migrate') + source = document_digest(con, tenant_slug, name) if head else {} + ensure_schema(con, tenant_slug) + con.execute(_sql('DELETE FROM {recs}', tenant_slug)) + con.execute(_sql('DELETE FROM {defs}', tenant_slug)) + maps = list(ROW_MAPS) + tables = 0 + record_rows = 0 + if head: + tables = con.execute(_sql( + 'INSERT INTO {defs} (table_key, value) ' + 'SELECT e.key, CASE WHEN jsonb_typeof(e.value) <> \'object\' THEN e.value ELSE ' + "(e.value - CASE WHEN jsonb_typeof(e.value -> 'rows') = 'object' " + "THEN ARRAY['rows'] ELSE ARRAY[]::text[] END " + "- ARRAY(SELECT m FROM unnest(%s::text[]) AS m WHERE jsonb_typeof(e.value -> m) = 'object')) " + '|| CASE WHEN EXISTS (SELECT 1 FROM unnest(%s::text[]) AS m ' + "WHERE jsonb_typeof(e.value -> m) = 'object') THEN jsonb_build_object(%s::text, " + 'to_jsonb(ARRAY(SELECT m FROM unnest(%s::text[]) AS m ' + "WHERE jsonb_typeof(e.value -> m) = 'object' ORDER BY octet_length(m), m COLLATE \"C\"))) " + "ELSE '{{}}'::jsonb END " + "|| CASE WHEN e.value ? 'rows' THEN '{{}}'::jsonb " + 'ELSE jsonb_build_object(%s::text, true) END END ' + 'FROM {kv} AS kv CROSS JOIN LATERAL jsonb_each(kv.value) AS e WHERE kv.key = %s', + tenant_slug), (maps, maps, MAPS_KEY, maps, NO_ROWS_KEY, str(name))).rowcount + record_rows = con.execute(_sql( + 'WITH t AS MATERIALIZED (SELECT e.key AS tk, e.value AS v FROM {kv} AS kv ' + 'CROSS JOIN LATERAL jsonb_each(kv.value) AS e ' + "WHERE kv.key = %s AND jsonb_typeof(e.value) = 'object'), " + 'r AS (SELECT t.tk, x.key AS rid, x.value AS val FROM t CROSS JOIN LATERAL ' + "jsonb_each(CASE WHEN jsonb_typeof(t.v -> 'rows') = 'object' THEN t.v -> 'rows' " + "ELSE '{{}}'::jsonb END) AS x), " + 'm AS (SELECT t.tk, y.key AS rid, jsonb_object_agg(mk, y.value) AS meta FROM t ' + 'CROSS JOIN LATERAL unnest(%s::text[]) AS mk CROSS JOIN LATERAL ' + "jsonb_each(CASE WHEN jsonb_typeof(t.v -> mk) = 'object' THEN t.v -> mk " + "ELSE '{{}}'::jsonb END) AS y GROUP BY t.tk, y.key) " + 'INSERT INTO {recs} (table_key, row_id, value, meta) ' + 'SELECT coalesce(r.tk, m.tk), coalesce(r.rid, m.rid), r.val, m.meta ' + 'FROM r FULL JOIN m ON r.tk = m.tk AND r.rid = m.rid', tenant_slug), + (str(name), maps)).rowcount + report = compare_digests(source, rows_digest(con, tenant_slug)) + if not report['ok']: + raise MigrationMismatch('migrate', report) + stamp = marker() + stamp['source'] = {'tables': report['tables'], 'records': report['records'], + 'bytes': report['bytes'], 'md5': digest_md5(source), + 'rev': int(head[2]) + 1 if head else 1} + rev = con.execute(_sql( + 'INSERT INTO {kv} (key, value) VALUES (%s, %s::jsonb) ON CONFLICT (key) DO UPDATE ' + 'SET value = EXCLUDED.value, rev = {kv_bare}.rev + 1, updated_at = now() RETURNING rev', + tenant_slug), (str(name), json.dumps(stamp))).fetchone()[0] + if int(rev) != stamp['source']['rev']: + con.execute(_sql("UPDATE {kv} SET value = jsonb_set(value, '{{source,rev}}', to_jsonb(%s::bigint)) " + 'WHERE key = %s', tenant_slug), (int(rev), str(name))) + pg._bump_write_generation(name, tenant_slug) + remember_mode(name, tenant_slug) + return {'changed': True, 'ok': True, 'tables': tables, 'records': report['records'], + 'record_rows': record_rows, 'bytes': report['bytes'], 'md5': report['md5'], + 'seconds': round(time.monotonic() - started, 3), 'per_table': report['per_table']} + + +def _dry_run(tenant_slug, name, statement_timeout_s): + """What `migrate` would copy: per-table records, bytes and md5, from a read-only transaction.""" + started = time.monotonic() + with _pg()._pool().connection() as con: + _settings(con, statement_timeout_s, read_only=True) + head = _head(con, tenant_slug, name, lock=False) + sizes = None + if head and head[0] == 'rows': + mode, digest = 'rows', rows_digest(con, tenant_slug) + elif head: + mode, digest = 'document', document_digest(con, tenant_slug, name) + sizes = con.execute(_sql('SELECT octet_length(value::text), pg_column_size(value) FROM {kv} ' + 'WHERE key = %s', tenant_slug), (str(name),)).fetchone() + else: + mode, digest = 'absent', {} + report = compare_digests(digest, digest) + return {'changed': False, 'dry_run': True, 'mode': mode, + 'reason': 'already rows' if mode == 'rows' else None, 'tables': report['tables'], + 'records': report['records'], 'bytes': report['bytes'], 'md5': report['md5'], + 'document_bytes': sizes[0] if sizes else None, 'stored_bytes': sizes[1] if sizes else None, + 'seconds': round(time.monotonic() - started, 3), 'per_table': report['per_table']} + + +def verify(tenant_slug, name=DOCUMENT_KEY, statement_timeout_s=STATEMENT_TIMEOUT_S): + """Per-table count and checksum of the stored state, read-only. + + In row mode the rows are checked against the digest the migration stamped in the marker. + After a row-mode write the checksums move on, so `ok` then rests on the structural checks. + """ + started = time.monotonic() + with _pg()._pool().connection() as con: + _settings(con, statement_timeout_s, read_only=True) + row = con.execute(_sql('SELECT CASE WHEN pg_column_size(value) < 8192 THEN value::text END, rev ' + 'FROM {kv} WHERE key = %s', tenant_slug), (str(name),)).fetchone() + stamp = _parse(row[0]) if row and row[0] else None + rows = is_marker(stamp) + digest = (rows_digest(con, tenant_slug) if rows else + document_digest(con, tenant_slug, name) if row else {}) + if rows: + orphans, empty = con.execute(_sql( + 'SELECT count(*) FILTER (WHERE NOT EXISTS (SELECT 1 FROM {defs} AS d ' + 'WHERE d.table_key = r.table_key)), count(*) FILTER (WHERE r.value IS NULL ' + 'AND r.meta IS NULL) FROM {recs} AS r', tenant_slug)).fetchone() + report = compare_digests(digest, digest) + out = {'mode': 'rows' if rows else ('document' if row else 'absent'), 'tables': report['tables'], + 'records': report['records'], 'bytes': report['bytes'], 'md5': report['md5'], 'ok': True, + 'seconds': round(time.monotonic() - started, 3), 'per_table': report['per_table']} + if rows: + source = stamp.get('source') or {} + writes = int(row[1]) - int(source['rev']) if source.get('rev') is not None else None + matches = (source.get('md5') == report['md5']) if source.get('md5') else None + out.update(source=source, writes_since_migration=writes, matches_source=matches, + orphan_records=int(orphans), empty_rows=int(empty)) + out['ok'] = orphans == 0 and empty == 0 and not (matches is False and writes == 0) + return out + + +def table_digest(entry): + """`(record count, md5 of the sorted JSON)` for one table entry.""" + rows = entry.get('rows') if isinstance(entry, dict) else None + text = json.dumps(entry, sort_keys=True, separators=(',', ':')) + return (len(rows) if isinstance(rows, dict) else 0), hashlib.md5(text.encode('utf-8')).hexdigest() + + +def verify_migration(tenant_slug, original, name=DOCUMENT_KEY): + """Compare the rows, assembled in Python, against a pre-migration document the caller holds.""" + assembled = read_document.__wrapped__(name, tenant_slug, fresh=True) + mismatches = [] + if list(assembled) != list(original or {}): + mismatches.append('table keys or their order differ') + for key, entry in (original or {}).items(): + got = assembled.get(key) + if table_digest(got) != table_digest(entry): + mismatches.append(f'{key}: count or checksum differs') + elif isinstance(entry, dict) and list(got) != list(entry): + mismatches.append(f'{key}: key order differs') + return {'ok': not mismatches and assembled == original, 'tables': len(assembled), + 'records': sum(table_digest(v)[0] for v in assembled.values()), + 'mismatches': mismatches[:20]} + + +def rollback(tenant_slug, name=DOCUMENT_KEY, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False): + """Rebuild the document from rows in SQL, prove it per table, restore it, drop the row tables. + + An ops escape hatch. Another process that cached row mode must restart after it. `dry_run` + reports what would be restored (the read-only `verify`). + """ + if dry_run: + return dict(verify(tenant_slug, name, statement_timeout_s), changed=False, dry_run=True) + pg = _pg() + pg.check_write(tenant_slug, f'roll {name!r} back to one document') + started = time.monotonic() + with pg._pool().connection() as con: + _settings(con, statement_timeout_s) + head = _head(con, tenant_slug, name, lock=True) + if not head or head[0] != 'rows': + return {'changed': False, 'reason': 'not in row mode'} + before = rows_digest(con, tenant_slug) + text, params = _assembled_select() + con.execute(_sql('UPDATE {kv} SET value = (SELECT coalesce(jsonb_object_agg(t.k, t.e), ' + "'{{}}'::jsonb) FROM (" + text + ') AS t(k, e)), rev = {kv_bare}.rev + 1, ' + 'updated_at = now() WHERE key = %s', tenant_slug), params + [str(name)]) + report = compare_digests(before, document_digest(con, tenant_slug, name)) + if not report['ok']: + raise MigrationMismatch('rollback', report) + con.execute(_sql('DROP TABLE {recs}', tenant_slug)) + con.execute(_sql('DROP TABLE {defs}', tenant_slug)) + pg._bump_write_generation(name, tenant_slug) + remember_mode(name, tenant_slug, value=False) + return {'changed': True, 'ok': True, 'tables': report['tables'], 'records': report['records'], + 'bytes': report['bytes'], 'md5': report['md5'], + 'seconds': round(time.monotonic() - started, 3), 'per_table': report['per_table']} + + +# -------------------------------------------------------------------------------------------- +# The operator CLI, run from `platform/` at release, after the combine: +# python -m core.user_table_rows migrate|verify|rollback [--dry-run] [--json] +# -------------------------------------------------------------------------------------------- +def _print_result(action, slug, result): + head = {k: v for k, v in result.items() if k not in ('per_table', 'verify')} + print(f'{action} {slug}: ' + ' '.join(f'{k}={v}' for k, v in head.items())) + if result.get('verify'): + check = {k: v for k, v in result['verify'].items() if k != 'per_table'} + print(' verify: ' + ' '.join(f'{k}={v}' for k, v in check.items())) + rows = result.get('per_table') or [] + if rows: + print(f' {"table":<44} {"records":>8} {"bytes":>10} {"md5":<32} ok') + for item in rows: + print(f' {item["table"]:<44} {item["records"]:>8} {item["bytes"]:>10} ' + f'{item["md5"] or "-":<32} {"ok" if item["ok"] else "MISMATCH"}') + + +def main(argv=None): + import argparse + ap = argparse.ArgumentParser(prog='python -m core.user_table_rows', + description="Move a tenant's user_tables document to rows, check it, " + 'or move it back. Run from platform/.') + ap.add_argument('action', choices=('migrate', 'verify', 'rollback')) + ap.add_argument('slug') + ap.add_argument('--dry-run', action='store_true', help='report only; write nothing') + ap.add_argument('--statement-timeout', type=float, default=STATEMENT_TIMEOUT_S, + help=f'seconds per statement (default {STATEMENT_TIMEOUT_S})') + ap.add_argument('--json', action='store_true', help='print the result as JSON') + args = ap.parse_args(argv) + timeout = args.statement_timeout + try: + if args.action == 'migrate': + before = None if args.dry_run else fingerprint(args.slug) + result = migrate(args.slug, statement_timeout_s=timeout, dry_run=args.dry_run) + if before is not None and not result['changed']: + result['unchanged'] = fingerprint(args.slug) == before + result['fingerprint'] = before + result['verify'] = verify(args.slug, statement_timeout_s=timeout) + result['ok'] = result['unchanged'] and result['verify']['ok'] + elif args.action == 'verify': + result = verify(args.slug, statement_timeout_s=timeout) + else: + result = rollback(args.slug, statement_timeout_s=timeout, dry_run=args.dry_run) + except MigrationMismatch as exc: + result = dict(exc.report, changed=False, ok=False, error=str(exc)) + if args.json: + print(json.dumps(result, default=str)) + else: + _print_result(args.action, args.slug, result) + return 0 if result.get('ok', True) else 1 + + +if __name__ == '__main__': + import sys + from core import user_table_rows as _module # one module copy: store_pg imports it by name + sys.exit(_module.main()) diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index a7b50713b10cee34a9dec1fea13241343994f5b9..884a844bf1e522d66e62949036da39ff262f592f 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -281,7 +281,8 @@ def is_connected(table_key, st=None): # exactly `rows`, raises loudly (or falls back to the whole read) rather than answering empty, # and a lent handle already serves whichever document the pass is holding. A caller that lends # the WHOLE document gets MORE than asked for, which is the one direction this may be wrong in. - table = all_defs(st).get(key) or {} + # S79-T47 (D-739): a row-mode store answers from this table's own definition row. + table = get_definition(key, st) or {} if not table: return False return _connected_definition(key, table) @@ -379,7 +380,7 @@ def materialises(table_key, st=None, defn=None): if key in _READ_THROUGH: return False if defn is None: - defn = get(key, st) or {} + defn = get_definition(key, st, fallback=get) or {} return not bool((defn or {}).get('readThrough')) @@ -511,7 +512,7 @@ def oversize_of(table_key, st=None, defn=None): """The eviction stamp on this database, or None. Reads the rows-free projection.""" if defn is None: try: - defn = all_defs(st).get(str(table_key or '').strip()) or {} + defn = get_definition(str(table_key or '').strip(), st) or {} except Exception: # noqa: BLE001 defn = {} stamp = (defn or {}).get(OVERSIZE_FIELD) @@ -641,7 +642,7 @@ def connection_of_table(table_key, rt=None, st=None): def connector_database_identity(table_key, rt=None, st=None, definition=None): """Read a connected database's provider identity without importing its API implementation.""" if definition is None: - definition = all_defs(st=st if st is not None else rt).get(str(table_key)) or {} + definition = get_definition(str(table_key), st if st is not None else rt) or {} if not definition: return None declared = definition.get('connector') or {} @@ -893,7 +894,9 @@ def _activity_definition(table=None): def _with_activity(tables): - tables = dict(tables or {}) + # A row-mode document stays lazy: copying it with dict() reads every table. + if not getattr(tables, 'lazy_tables', False): + tables = dict(tables or {}) activity = _activity_definition(tables.get('ut_activity')) if activity is not None: tables['ut_activity'] = activity @@ -909,7 +912,7 @@ def records_mutable(table_key, st=None): mode; engine writers use their direct coalesced path and are intentionally not routed through this human-door predicate. """ - table = get_one(table_key, st) or {} + table = get_definition(table_key, st, fallback=get_one) or {} if not table: return False if str(table_key) in _LOCKED_RECORD_KEYS: @@ -1284,18 +1287,32 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, def public_result(result): if (isinstance(result, dict) and isinstance(result.get('ut_activity'), dict) and '_activity_outbox' in result['ut_activity']): - result = {**result, 'ut_activity': { - key: value for key, value in result['ut_activity'].items() - if key != '_activity_outbox'}} + activity = {key: value for key, value in result['ut_activity'].items() + if key != '_activity_outbox'} + if getattr(result, 'lazy_tables', False): + result['ut_activity'] = activity # caller-owned; a copy would read every table + else: + result = {**result, 'ut_activity': activity} return result def mutate(current): + lock = getattr(current, 'lock_records', None) + if callable(lock): + # Row mode: lock this write's records (or the table) first, in one sorted order. + plan = {str(table_key): None if row_ids is None else list(row_ids)} + if append is not None: + plan.setdefault('ut_activity', []) + lock(plan) prior_table = (current or {}).get(str(table_key)) or {} - selected = prior_table.get('rows') or {} + selected = prior_table.get('rows') + if selected is None: + selected = {} if row_ids is not None: selected = {str(r): selected[str(r)] for r in row_ids if str(r) in selected} before_rows = deepcopy(selected) - queued = deepcopy(((current or {}).get('ut_activity') or {}).get('_activity_outbox') or {}) + outbox = ((current or {}).get('ut_activity') or {}).get('_activity_outbox') + # A record-mode outbox writes per entry; a replaced document escalates to table mode. + queued = {} if getattr(outbox, 'lazy_records', False) else deepcopy(outbox or {}) document = original_mutate(current) if document is None: document = current @@ -1684,23 +1701,29 @@ def _refuse_audit_values(fields, values): def stamp_audit_rows(table, before_rows, *, by=None, row_ids=None, restore_ids=()): - """Stamp changed records and project server-owned audit facts into their field cells.""" - rows = table.get('rows') or {} + """Stamp changed records and project server-owned audit facts into their field cells. + + With `row_ids`, only those records are read, so a record-mode table stays in record mode. + """ + rows = table.get('rows') + if rows is None: + rows = {} audit_fields = {f['key']: f['type'] for f in table.get('fields') or [] if f.get('type') in AUDIT_FIELD_TYPES} selected = set(rows) if row_ids is None else {str(r) for r in row_ids} restores = {str(r) for r in restore_ids} - metadata = table.get('_rowAudit') or {} + stored = table.get('_rowAudit') + metadata = stored if stored is not None else {} actor = str(by or '') stamp = None def values(row): return {key: value for key, value in row.items() if key not in audit_fields} - for rid in before_rows.keys() - rows.keys(): + for rid in [r for r in before_rows if r not in rows]: if row_ids is None or rid in selected: metadata.setdefault(rid, {}) - for rid in selected & rows.keys(): + for rid in sorted(r for r in selected if r in rows): before = before_rows.get(rid) if before is not None and values(before) == values(rows[rid]): continue @@ -1711,10 +1734,12 @@ def stamp_audit_rows(table, before_rows, *, by=None, row_ids=None, restore_ids=( facts = {'created_by': actor} facts.update(last_modified_time=stamp, last_modified_by=actor) metadata[rid] = facts - if metadata: + if metadata is not stored and metadata: # Deleted records retain trusted facts for C-UNDO. New inserts reset reused identities. table['_rowAudit'] = metadata - projected = rows if row_ids is None or not selected else selected & rows.keys() + if not audit_fields: + return + projected = rows if row_ids is None or not selected else [r for r in selected if r in rows] for rid in projected: facts = metadata.get(rid) or {} for key, kind in audit_fields.items(): @@ -2031,6 +2056,39 @@ def _entry_reader(st): return None +def _definition_reader(st): + """The bound store's one-row definition read, under `_entry_reader`'s ownership rule, or None.""" + if _entry_reader(st) is None: + return None + runtime = st._runtime if type(st).__name__ == '_AssemblyStore' else st + backend = runtime._store() if callable(getattr(runtime, '_store', None)) else runtime + reader = getattr(backend, 'get_definition', None) + if not callable(reader): + return None + name = (runtime.store_key(STORE_KEY) if callable(getattr(runtime, 'store_key', None)) + else STORE_KEY) + return lambda key: reader(name, key) + + +def get_definition(table_key, st=None, *, fallback=None): + """One table's definition without rows: one store row on a row-mode store (S79-T47). + + The answer raises `KeyError` for `rows` and the row maps, as `all_defs` does for `rows`. Any + other reader keeps today's path: `fallback(table_key, st)` when given, else `all_defs`. + """ + key = str(table_key or '').strip() + reader = _definition_reader(st) + if reader is not None: + one = reader(key) + if one is not NotImplemented: + if key == 'ut_activity': + return _activity_definition(one) + return one if isinstance(one, dict) else None + if fallback is not None: + return fallback(key, st) + return all_defs(st).get(key) + + def get_one(table_key, st=None): """`get`, reading ONLY this table's entry when the bound store can (HOTFIX 2026-09-28). @@ -2052,6 +2110,42 @@ def get_one(table_key, st=None): return get(table_key, st=st) +def _records_reader(st): + """The bound store's one-table, named-records read, under `_entry_reader`'s ownership rule.""" + if _entry_reader(st) is None: + return None + runtime = st._runtime if type(st).__name__ == '_AssemblyStore' else st + backend = runtime._store() if callable(getattr(runtime, '_store', None)) else runtime + reader = getattr(backend, 'get_records', None) + if not callable(reader): + return None + name = (runtime.store_key(STORE_KEY) if callable(getattr(runtime, 'store_key', None)) + else STORE_KEY) + return lambda key, ids: reader(name, key, ids) + + +def get_records(table_key, row_ids, st=None): + """`get_one` holding only the records `row_ids` and their row-map entries (S79-T48). + + A row-mode store reads one definition row and the named record rows. Any other reader answers + with `get_one`, the whole table: a superset, so membership and field checks read the same. + """ + key = str(table_key or '').strip() + reader = _records_reader(st) + if reader is not None: + one = reader(key, [str(r) for r in row_ids]) + if one is not NotImplemented: + if key == 'ut_activity': + return _activity_definition(one) + return one if isinstance(one, dict) else None + return get_one(key, st) + + +def get_record(table_key, row_id, st=None): + """`get_records` for one record: the cell doors' pre-check reads one row, not the table.""" + return get_records(table_key, (row_id,), st) + + # --- DURABLE MAP ZONES ------------------------------------------------------- # Zones intentionally do not live in a view config: a view is a person's presentation of rows, # while a named geographic boundary is tenant data used by several views and routes. Keeping a @@ -2258,7 +2352,10 @@ def delete_zone(table_key, zone_id, st=None): def is_user_table(table_key, st=None): """THE permission predicate for row writes. Cheap and total: a key that is not in this store is not a user table, so it cannot accept invented rows.""" - return str(table_key or '').startswith(KEY_PREFIX) and get_one(table_key, st) is not None + if not str(table_key or '').startswith(KEY_PREFIX): + return False + # A row-mode store answers from the definition row; any other reader keeps `get_one`. + return get_definition(table_key, st, fallback=lambda key, s: get_one(key, s)) is not None def is_managed_system_table(table_key, st=None, definition=None): @@ -5129,7 +5226,7 @@ def patch_cells(table_key, row_id, values, st=None, *, by=None, via=None): """Write shared definition-row cells for trusted server-side writers.""" if str(table_key) == 'ut_activity' or not is_user_table(table_key, st): return False - rows = (get_one(table_key, st) or {}).get('rows') or {} + rows = (get_record(table_key, row_id, st) or {}).get('rows') or {} if str(row_id) not in rows: return False @@ -5172,7 +5269,7 @@ def patch_many_cells(table_key, updates, st=None, *, by=None, via=None): """ if str(table_key) == 'ut_activity' or not is_user_table(table_key, st): return 0 - have = set((get_one(table_key, st) or {}).get('rows') or {}) + have = set((get_records(table_key, list(updates or {}), st) or {}).get('rows') or {}) clean = {str(r): {str(k): str(v) for k, v in (cells or {}).items()} for r, cells in (updates or {}).items() if str(r) in have and isinstance(cells, dict) and cells} @@ -5208,7 +5305,7 @@ def patch_link_cell(table_key, row_id, fkey, value, st=None, *, by=None, via=Non table_key, row_id, fkey = str(table_key), str(row_id), str(fkey) if not is_user_table(table_key, st) or not records_mutable(table_key, st): return None - table = get_one(table_key, st) or {} + table = get_record(table_key, row_id, st) or {} if row_id not in (table.get('rows') or {}): return None field = next((f for f in (table.get('fields') or []) if f.get('key') == fkey), None) @@ -5216,9 +5313,13 @@ def patch_link_cell(table_key, row_id, fkey, value, st=None, *, by=None, via=Non if (field or {}).get('type') != 'link' or not isinstance(bag, dict) \ or bag.get('on') or bag.get('inverse'): return None - target = get(str(bag.get('table') or ''), st) or {} - valid = set((target.get('rows') or {}).keys()) raw_ids = value if isinstance(value, (list, tuple, set)) else str(value or '').split(',') + wanted = list(dict.fromkeys(str(r).strip() for r in raw_ids if str(r).strip()))[:LINK_MAX_IDS + 1] + target_key = str(bag.get('table') or '') + # Read only the picked target records (a whole-document `get` before S79-T48). + target = ((get_records(target_key, wanted, st) if target_key.startswith(KEY_PREFIX) + else get(target_key, st)) or {}) + valid = set((target.get('rows') or {}).keys()) _validate_link_selection_values(table.get('fields') or [], {fkey: value}, {str(bag.get('table') or ''): target}) picked, seen = [], set() @@ -5274,7 +5375,7 @@ def patch_profile_cell(table_key, row_id, fkey, value, st=None, *, by=None, via= """ if str(table_key) == 'ut_activity' or not is_user_table(table_key, st): return None - defn = get_one(table_key, st) or {} + defn = get_record(table_key, row_id, st) or {} rows = defn.get('rows') or {} if str(row_id) not in rows: return None diff --git a/platform/core/users.py b/platform/core/users.py index 2e1575543eebf24cfe8008ed774c2762e6a1c2c9..f9461fb78babd9dc5e2f3bd416f308043317e31a 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -1048,6 +1048,8 @@ def set_whatsapp(username, value): u = reg.get(username) if not isinstance(u, dict): raise ValueError('This account is not available.') + # R42: a test account has no outbound channel, so no writer may add one. + refuse_test_account_change(u, whatsapp=value) if value is None: u.pop('whatsapp', None) else: @@ -1070,6 +1072,111 @@ def set_whatsapp_opt_out(username, on): return store.update('users', _set) +# ------------------------------------------------------------------ test accounts (S79-T63, R42) +#: The record key that marks a copied test account: `{copied_from, created_by, created_at}`. +TEST_ACCOUNT_KEY = 'test_account' + + +class TestAccountRefused(ValueError): + """A writer refused to give a test account an outbound channel or the admin role.""" + + def __init__(self, code, message): + self.code = code + super().__init__(message) + + +class TestAccountExists(ValueError): + """The chosen test username is already a key in the registry.""" + + +def is_test_account(record): + """Whether this stored record is a copied test account.""" + return isinstance(record, dict) and isinstance(record.get(TEST_ACCOUNT_KEY), dict) + + +def refuse_test_account_change(record, *, email=None, whatsapp=None, role=None): + """Raise `TestAccountRefused` if a change would give a test account email, WhatsApp or admin.""" + if not is_test_account(record): + return + if email: + raise TestAccountRefused('test_account_no_email', + 'A test account cannot have an email address.') + if whatsapp: + raise TestAccountRefused('test_account_no_whatsapp', + 'A test account cannot connect WhatsApp.') + if role is not None and str(role).strip().lower() != 'user': + raise TestAccountRefused('test_account_not_admin', + 'A test account cannot be an administrator.') + + +def create_test_account(username, pw, *, name, access, marker, tenant): + """Write one test account in one update: `access` fields verbatim, no email, no WhatsApp. + + Raises `TestAccountExists` when the key is taken by any record other than this exact write. + """ + import copy + freeze_notice.check_write() + username = (username or '').strip().lower() + failures = password_failures(pw, username=username, name=name) + if failures: + raise PasswordPolicyError(failures) + marker = dict(marker) + salt = secrets.token_hex(16) + digest = _hash(pw, salt) + + def _add(reg): + prior = reg.get(username) + if prior is not None: + # A replayed closure finds its own landed write; anything else is a real collision. + if isinstance(prior, dict) and prior.get(TEST_ACCOUNT_KEY) == marker: + return reg + raise TestAccountExists(username) + rec = {'salt': salt, 'hash': digest, 'name': name, 'role': 'user', + 'password_changed_at': _now_iso(), + 'bus': copy.deepcopy(access.get('bus', 'all')), 'active': True, + 'modules': copy.deepcopy(access.get('modules', 'all')), + 'agent': access.get('agent') or None, 'email': None, + 'tenant': str(tenant or 'royal-imports').strip().lower(), + 'username_original': username, TEST_ACCOUNT_KEY: dict(marker)} + # The permission block and its marker travel together or not at all (see `_record`). + for key in ('perms', 'perms_v'): + if key in access: + rec[key] = copy.deepcopy(access[key]) + reg[username] = rec + return reg + + store.update('users', _add) + + +def delete_test_account(username, *, tenant, marker=None): + """Delete one test account record. Returns deleted, missing or not_test; never a real account. + + `marker`, when given, must equal the stored marker, so a rollback never removes a copy + another administrator made under the same name. + """ + freeze_notice.check_write() + username = (username or '').strip().lower() + tenant = str(tenant or '').strip().lower() + outcome = ['missing'] + + def _del(reg): + outcome[0] = 'missing' + rec = reg.get(username) + if (not isinstance(rec, dict) + or str(rec.get('tenant') or 'royal-imports').strip().lower() != tenant): + return reg + if not is_test_account(rec) or (marker is not None + and rec.get(TEST_ACCOUNT_KEY) != dict(marker)): + outcome[0] = 'not_test' + return reg + reg.pop(username, None) + outcome[0] = 'deleted' + return reg + + store.update('users', _del) + return outcome[0] + + def database_access_state(record): """Copy the permission state used to admit a narrow database change.""" import copy @@ -1160,6 +1267,7 @@ def set_access(username, role=None, bus=None, modules=None, agent=None, email=No def _set(reg): u = reg.get(username) if u: + refuse_test_account_change(u, email=email, role=role) if name is not None: u['name'] = name if role is not None: @@ -1195,8 +1303,9 @@ def update_access_with_active_admin_guard(username, role=None, bus=None, modules deliberately separate from the older narrow setters: those are also used for independent maintenance changes, while this is the public account-access writer's complete transition. - Returns updated, last_admin, or missing. A named result lets the HTTP adapter preserve its - public error vocabulary without treating a refused change as a failed store write. + Returns updated, last_admin, missing, or a `TestAccountRefused.code`. A named result lets the + HTTP adapter preserve its public error vocabulary without treating a refused change as a + failed store write. """ username = (username or '').strip().lower() outcome = ['missing'] @@ -1214,6 +1323,11 @@ def update_access_with_active_admin_guard(username, role=None, bus=None, modules if not isinstance(current, dict): outcome[0] = 'missing' return reg + try: + refuse_test_account_change(current, email=email, role=role) + except TestAccountRefused as exc: + outcome[0] = exc.code + return reg # Build the whole candidate before touching reg. On a refused transition the original # record remains byte-for-byte intact, including its epoch. @@ -1289,14 +1403,17 @@ def assignable_people(tenant=None): changes — and a future caller that forgets gets an empty picker somebody notices instead of a leak nobody does. """ - return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)}) + # A test account (S79-T63) is never offered: nobody assigns work to a copy. + return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant) + if not is_test_account(u)}) def assignable_identities(tenant=None): - """Canonical username/display-name pairs for durable permission pickers.""" + """Canonical username/display-name pairs for durable permission pickers, test accounts excluded.""" return [{"username": str(username).strip().lower(), "name": str(record.get("name") or username)} - for username, record in sorted(_tenant_accounts(tenant), key=lambda item: item[0])] + for username, record in sorted(_tenant_accounts(tenant), key=lambda item: item[0]) + if not is_test_account(record)] def set_avatar(username, data_url): diff --git a/platform/core/view_templates.py b/platform/core/view_templates.py index d816f14f54068b77f9047f793a5d74e4512c780f..33bb16b80f00e4c65fc839d20665c9ccaccc64fd 100644 --- a/platform/core/view_templates.py +++ b/platform/core/view_templates.py @@ -120,8 +120,8 @@ TEMPLATES = [ _view( 'tpl_collections_focus_overdue', 'Past due', 'Customers with an overdue balance, largest first. The four aging columns sum ' - 'to AR overdue; a balance inside the grace period counts as open, not overdue.', - ['customer', 'agent', 'ar_overdue', 'ar_aged_1_30', 'ar_aged_31_60', + 'to Overdue; Overdue past grace is the collections number with its 5-day grace.', + ['customer', 'agent', 'ar_overdue', 'ar_overdue_past_grace', 'ar_aged_1_30', 'ar_aged_31_60', 'ar_aged_61_90', 'ar_aged_90_plus', 'ar_open', 'days_to_pay', 'payment_terms', 'last_order'], sorts=[{'colId': 'ar_overdue', 'dir': 'desc'}], @@ -130,7 +130,7 @@ TEMPLATES = [ _view( 'tpl_collections_focus_worst', 'Over 90 days', 'The part of the book that is no longer a payment-terms conversation.', - ['customer', 'agent', 'ar_aged_90_plus', 'ar_overdue', 'ar_exposure', + ['customer', 'agent', 'ar_aged_90_plus', 'ar_overdue', 'ar_overdue_past_grace', 'ar_exposure', 'days_to_pay', 'last_order'], sorts=[{'colId': 'ar_aged_90_plus', 'dir': 'desc'}], filters=[{'colId': 'ar_aged_90_plus', 'op': 'gt', 'value': '0'}], diff --git a/platform/harness/amazon_ads_client.py b/platform/harness/amazon_ads_client.py new file mode 100644 index 0000000000000000000000000000000000000000..0322b7b36da1cc7669fa3c2b68f23093c1692ce5 --- /dev/null +++ b/platform/harness/amazon_ads_client.py @@ -0,0 +1,540 @@ +"""amazon_ads_client.py: the Amazon Ads API transport (S79-T12, contract C6). + +⛔ THE CLIENT READS REPORTS AND SENDS THREE NAMED WRITES, NOTHING ELSE. The Ads API has no +read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the +refusal lives here, in the request builder, before any credential read, token exchange or socket: + + reports create, status and row pages on Reporting v1 (R8), the only reads + named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in + `bid`, `budget`, `negative_keyword` (QI3), each operation one path constant + +Every other call raises `AdsWriteRefused`. A named write also needs an approval id that the +Operator App stored as approved for this very item (`APPROVAL_SOURCE`, read lazily, because a +harness module never imports `modules` at load). A missing approval source refuses. + +`AMAZON_ADS_FIXTURE=1` routes every call to the recording fixture below. No live Ads call is +made until Royal's Direct Advertiser credential arrives (R7, R17). + +NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is +stored with the credential, so the client needs no profile listing call. +""" +import importlib +import json +import os +import re +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from contextlib import contextmanager +from datetime import date, timedelta + +KEY = "amazon_ads" +LABEL = "Amazon Ads" +ENTRY_TYPE = "amazon_ads" +FIXTURE_ENV = "AMAZON_ADS_FIXTURE" + +HOSTS = {"na": "https://advertising-api.amazon.com", + "eu": "https://advertising-api-eu.amazon.com", + "fe": "https://advertising-api-fe.amazon.com"} +DEFAULT_REGION = "na" +LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token" +CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id") + +#: Reporting v1 limits (primer F11, F12): one report in flight per advertiser, about 100 a day, +#: up to 3 hours per report, at most 31 days per request. +REPORT_CONCURRENCY = 1 +REPORTS_PER_DAY = 100 +REPORT_MAX_WAIT_SECONDS = 3 * 3600 +MAX_DAYS_PER_REQUEST = 31 +REPORT_POLL_SECONDS = 30.0 +_TIMEOUT = 60 + +#: The only non-write calls: Reporting v1 create, status and row pages. Unknown: the exact v1 +#: paths; the first credentialed call settles them, and each is one literal here. +REPORT_OPERATIONS = { + "createReport": ("POST", "reporting/v1/reports"), + "getReport": ("GET", "reporting/v1/reports/{reportId}"), + "getReportRows": ("GET", "reporting/v1/reports/{reportId}/rows"), +} + +#: ⛔ C6: THE NAMED WRITES, AS DATA. `kind -> operation -> rule`. `endpoints` names the one path +#: per ad type; an ad type absent there refuses. `item` is the body template: a "{field}" value +#: is read from the validated payload, any other value is a constant. `each` repeats the item per +#: id in that payload list. The body carries only template keys, never a caller's other fields. +NAMED_WRITES = { + "bid": { + "set_bid": { + "endpoints": {"sp": ("PUT", "sp/targets"), "sb": ("PUT", "sb/targets"), + "sd": ("PUT", "sd/targets")}, + "required": ("campaign_id", "ad_group_id", "target_id", "bid"), + "collection": "targets", + "item": {"targetId": "{target_id}", "adGroupId": "{ad_group_id}", + "campaignId": "{campaign_id}", "bid": "{bid}"}, + }, + # QI3 "pause ads on a money loser": an ad state change, carried under the bid kind. + "pause_ads": { + "endpoints": {"sp": ("PUT", "sp/productAds"), "sb": ("PUT", "sb/v4/ads"), + "sd": ("PUT", "sd/productAds")}, + "required": ("campaign_id", "ad_group_id", "ad_ids", "state"), + "collection": "ads", "each": "ad_ids", + "item": {"adId": "{each}", "adGroupId": "{ad_group_id}", + "campaignId": "{campaign_id}", "state": "PAUSED"}, + }, + }, + "budget": { + "set_daily_budget": { + "endpoints": {"sp": ("PUT", "sp/campaigns"), "sb": ("PUT", "sb/v4/campaigns"), + "sd": ("PUT", "sd/campaigns")}, + "required": ("campaign_id", "daily_budget"), + "collection": "campaigns", + "item": {"campaignId": "{campaign_id}", + "budget": {"budget": "{daily_budget}", "budgetType": "DAILY"}}, + }, + }, + "negative_keyword": { + # Sponsored Display takes no negative keywords, so it has no endpoint and refuses. + "add_negative_keyword": { + "endpoints": {"sp": ("POST", "sp/negativeKeywords"), + "sb": ("POST", "sb/negativeKeywords")}, + "required": ("campaign_id", "ad_group_id", "keyword_text", "match_type"), + "collection": "negativeKeywords", + "item": {"campaignId": "{campaign_id}", "adGroupId": "{ad_group_id}", + "keywordText": "{keyword_text}", "matchType": "{match_type}", + "state": "ENABLED"}, + }, + }, +} + +#: Payload field rules for the named writes. A tuple is the allowed values. +FIELD_RULES = { + "campaign_id": "id", "ad_group_id": "id", "target_id": "id", "ad_ids": "ids", + "bid": "money", "daily_budget": "money", "keyword_text": "text", + "match_type": ("negativeExact", "negativePhrase"), "state": ("paused",), +} + +#: C6: `fn(tenant_key, approval_id) -> record or None`, imported lazily. The record must be +#: approved and name the payload's `item_id`. +APPROVAL_SOURCE = "modules.amazon_operator:approval" + + +class AdsError(RuntimeError): + """An Ads API failure, safe to show: no token or secret ever reaches it.""" + + +class AdsWriteRefused(AdsError): + """A call this connector does not make. Raised before any network I/O.""" + + def __init__(self, cause): + self.cause = str(cause) + super().__init__(f"Refused: {self.cause}. This connector reads reports and sends only " + f"the three named writes: bid, budget and negative keyword.") + + +class AdsReportUnavailable(AdsError): + """Amazon does not offer this report to this advertiser.""" + + +class AdsReportPending(AdsError): + """A report did not finish inside the wait limit; the next sync asks again.""" + + +def fixture_mode(): + """True when `AMAZON_ADS_FIXTURE=1` routes every call to the recording fixture.""" + return os.environ.get(FIXTURE_ENV) == "1" + + +def _template(path): + return tuple(path.strip("/").split("/")) + + +def _matches(template, path): + parts = str(path or "").strip("/").split("/") + if len(parts) != len(template): + return False + return all((bool(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part))) + if seg.startswith("{") and seg.endswith("}") else part == seg + for part, seg in zip(parts, template)) + + +def report_operation(method, path): + """The Reporting v1 operation name for one call, or None.""" + for name, (verb, tmpl) in REPORT_OPERATIONS.items(): + if str(method).upper() == verb and _matches(_template(tmpl), path): + return name + return None + + +class _Prepared: + """A request the builder accepted. Only `build_request` and `_build_named` make one.""" + __slots__ = ("method", "path", "body", "query", "operation", "named", "_seal") + + def __init__(self, method, path, body, query, operation, named): + self.method, self.path, self.body, self.query = method, path, body, query + self.operation, self.named = operation, named + self._seal = _SEAL + + +_SEAL = object() + + +def build_request(method, path, *, body=None, query=None): + """Accept a Reporting v1 call, or raise `AdsWriteRefused` before any I/O.""" + operation = report_operation(method, path) + if operation is None: + raise AdsWriteRefused(f"{str(method).upper()} {_safe_path(path)} is not a report call") + return _Prepared(str(method).upper(), str(path).strip("/"), body, dict(query or {}), + operation, None) + + +def _safe_path(path): + return re.sub(r"[^A-Za-z0-9/_{}.-]", "", str(path or ""))[:120] or "an empty path" + + +def _valid(field, value): + rule = FIELD_RULES.get(field) + if rule == "id": + return isinstance(value, (str, int)) and not isinstance(value, bool) \ + and bool(re.fullmatch(r"[0-9]{1,24}", str(value))) + if rule == "ids": + return isinstance(value, (list, tuple)) and bool(value) and all(_valid("campaign_id", v) for v in value) + if rule == "money": + try: + number = float(value) + except (TypeError, ValueError): + return False + return not isinstance(value, bool) and number > 0 and number == number and number != float("inf") + if rule == "text": + return isinstance(value, str) and 0 < len(value.strip()) <= 80 + if isinstance(rule, tuple): + return value in rule + return False + + +def _render(value, payload, each=None): + if isinstance(value, dict): + return {k: _render(v, payload, each) for k, v in value.items()} + if isinstance(value, str) and value.startswith("{") and value.endswith("}"): + field = value[1:-1] + raw = each if field == "each" else payload[field] + if FIELD_RULES.get(field) == "money": + return round(float(raw), 2) + return str(raw) if field == "each" or FIELD_RULES.get(field) == "id" else raw + return value + + +def _build_named(kind, payload): + """Accept one of C6's named writes, or raise `AdsWriteRefused` before any I/O.""" + if kind not in NAMED_WRITES: + raise AdsWriteRefused(f"write kind {str(kind)[:40]!r} is not a named write") + if not isinstance(payload, dict): + raise AdsWriteRefused("the write payload is not an object") + operation = str(payload.get("operation") or "") + rule = NAMED_WRITES[kind].get(operation) + if rule is None: + raise AdsWriteRefused(f"operation {operation[:40]!r} is not named under {kind}") + ad_type = str(payload.get("ad_type") or "") + endpoint = rule["endpoints"].get(ad_type) + if endpoint is None: + raise AdsWriteRefused(f"{operation} is not named for ad type {ad_type[:8]!r}") + for field in rule["required"]: + if not _valid(field, payload.get(field)): + raise AdsWriteRefused(f"{operation} needs a valid {field}") + each = rule.get("each") + items = ([_render(rule["item"], payload, v) for v in payload[each]] if each + else [_render(rule["item"], payload)]) + method, path = endpoint + return _Prepared(method, path, {rule["collection"]: items}, {}, operation, + {"kind": kind, "operation": operation, "ad_type": ad_type}) + + +def _approved(tenant_key, approval_id, kind, payload): + """Refuse unless the Operator App stored this approval for this item and this payload.""" + if not isinstance(approval_id, str) or not approval_id.strip(): + raise AdsWriteRefused("a named write needs an approval id") + item_id = payload.get("item_id") + if not isinstance(item_id, str) or not item_id: + raise AdsWriteRefused("a named write needs the item id it was approved for") + module_name, _, name = APPROVAL_SOURCE.partition(":") + try: + reader = getattr(importlib.import_module(module_name), name) + except Exception: # noqa: BLE001 + raise AdsWriteRefused("the approval source is not installed") from None + try: + record = reader(tenant_key, approval_id) + except Exception: # noqa: BLE001 + raise AdsWriteRefused("the approval could not be read") from None + if not isinstance(record, dict) or record.get("decision") != "approved": + raise AdsWriteRefused("the approval id is not an approved decision") + if item_id not in (record.get("item_ids") or ()): + raise AdsWriteRefused("the approval does not name this item") + writes = record.get("writes") + if writes is not None: + stored = [w for w in writes if isinstance(w, dict) and w.get("item_id") == item_id] + if not stored or stored[0].get("kind") != kind or stored[0].get("payload") != payload: + raise AdsWriteRefused("the payload is not the one that was approved") + return record + + +def send_named_write(tenant_key, kind, payload, *, approval_id, rt=None): + """C6: send one approved bid, budget or negative keyword write. -> the sent request and reply.""" + prepared = _build_named(kind, payload) + key = str(tenant_key or "").strip().lower() + if not key: + raise AdsWriteRefused("a named write needs a tenant") + _approved(key, approval_id, kind, payload) + reply = _send(prepared, key, rt=rt, profile_id=payload.get("profile_id")) + return {"sent": True, "kind": kind, "operation": prepared.operation, + "ad_type": prepared.named["ad_type"], "method": prepared.method, + "path": prepared.path, "approval_id": approval_id, "item_id": payload["item_id"], + "response": reply} + + +def request(tenant_key, method, path, *, body=None, query=None, rt=None, profile_id=None): + """A report call by method and path. Anything else raises `AdsWriteRefused` before any I/O.""" + return _send(build_request(method, path, body=body, query=query), + str(tenant_key or "").strip().lower(), rt=rt, profile_id=profile_id) + + +def _admitted(prepared): + """Re-check at the transport door, so no hand-made object reaches a socket.""" + if not isinstance(prepared, _Prepared) or prepared._seal is not _SEAL: + return False + if prepared.named is None: + return report_operation(prepared.method, prepared.path) == prepared.operation + rule = NAMED_WRITES.get(prepared.named["kind"], {}).get(prepared.named["operation"]) + return bool(rule) and rule["endpoints"].get(prepared.named["ad_type"]) == (prepared.method, prepared.path) + + +def _send(prepared, tenant_key, *, rt=None, profile_id=None): + if not _admitted(prepared): + raise AdsWriteRefused("the request did not come from the request builder") + if fixture_mode(): + return FIXTURE.answer(prepared, tenant_key, profile_id) + return _live(prepared, tenant_key, rt=rt, profile_id=profile_id) + + +# --------------------------------------------------------------------------- live transport + +def credential(tenant_key, rt=None): + """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source.""" + try: + import core.keychain as kc + fields = kc._creds_for(rt, ENTRY_TYPE, None) + except Exception: # noqa: BLE001 + return None + if not isinstance(fields, dict) or not all(fields.get(f) for f in CREDENTIAL_FIELDS): + return None + return fields + + +def _token(fields): + data = urllib.parse.urlencode({"grant_type": "refresh_token", + "refresh_token": fields["refresh_token"], + "client_id": fields["client_id"], + "client_secret": fields["client_secret"]}).encode() + req = urllib.request.Request(LWA_TOKEN_URL, data=data, method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}) + try: + with urllib.request.urlopen(req, timeout=_TIMEOUT) as r: + return json.loads(r.read().decode("utf-8", "replace"))["access_token"] + except Exception as exc: # noqa: BLE001 + raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None + + +def _live(prepared, tenant_key, *, rt=None, profile_id=None): + fields = credential(tenant_key, rt) + if fields is None: + raise AdsError("No Amazon Ads credential is stored for this tenant. " + "Connect Amazon Ads under Settings, Connectors.") + host = HOSTS.get(str(fields.get("region") or DEFAULT_REGION).lower(), HOSTS[DEFAULT_REGION]) + url = f"{host}/{prepared.path}" + if prepared.query: + url += "?" + urllib.parse.urlencode(prepared.query) + headers = {"Authorization": f"Bearer {_token(fields)}", + "Amazon-Advertising-API-ClientId": fields["client_id"], + "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]), + "Content-Type": "application/json", "Accept": "application/json"} + data = json.dumps(prepared.body).encode() if prepared.body is not None else None + req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers) + try: + with urllib.request.urlopen(req, timeout=_TIMEOUT) as r: + return json.loads(r.read().decode("utf-8", "replace") or "{}") + except urllib.error.HTTPError as exc: + try: + detail = json.loads(exc.read().decode("utf-8", "replace") or "{}") + message = str(detail.get("details") or detail.get("message") or "")[:200] + except Exception: # noqa: BLE001 + message = "" + if exc.code == 400 and re.search(r"not (?:supported|offered|available)", message, re.I): + raise AdsReportUnavailable(f"Amazon does not offer this report: {message}") from None + raise AdsError(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}: {message}") from None + except Exception as exc: # noqa: BLE001 + raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None + + +# --------------------------------------------------------------------------- reports + +_REPORT_LOCKS = {} +_REPORT_LOCKS_GUARD = threading.Lock() + + +def _report_lock(tenant_key, profile_id): + with _REPORT_LOCKS_GUARD: + return _REPORT_LOCKS.setdefault((tenant_key, str(profile_id or "")), threading.Lock()) + + +def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, sleep=time.sleep, + now=time.monotonic): + """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).""" + from harness import amazon_ads_schema as schema + spec = schema.TABLE_SPECS[schema.short_name(table)] + if (end - start).days + 1 > MAX_DAYS_PER_REQUEST: + raise ValueError("a report window is at most 31 days") + body = {"reportTypeId": spec["report_type"], "adProduct": spec["ad_type"].upper(), + "startDate": start.isoformat(), "endDate": end.isoformat(), + "timeUnit": "DAILY", "columns": list(schema.vendor_columns(table)), "format": "JSON"} + # One report in flight per advertiser (F11): the lock serialises this process's reports. + with _report_lock(tenant_key, profile_id): + created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body, + rt=rt, profile_id=profile_id) + report_id = str(created.get("reportId") or "") + if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id): + raise AdsError("Amazon Ads returned no report id") + started = now() + status_path = REPORT_OPERATIONS["getReport"][1].replace("{reportId}", report_id) + while True: + status = request(tenant_key, "GET", status_path, rt=rt, profile_id=profile_id) + state = str(status.get("status") or "").upper() + if state == "COMPLETED": + break + if state in ("FAILED", "FAILURE", "CANCELLED"): + reason = str(status.get("failureReason") or state)[:200] + if re.search(r"not (?:supported|offered|available)", reason, re.I): + raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}") + raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}") + if now() - started > REPORT_MAX_WAIT_SECONDS: + raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours") + sleep(0 if fixture_mode() else REPORT_POLL_SECONDS) + rows, token = [], None + rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id) + while True: + page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None, + rt=rt, profile_id=profile_id) + rows.extend(page.get("rows") or []) + token = page.get("nextToken") + if not token: + return report_id, rows + + +# --------------------------------------------------------------------------- recording fixture + +class _Fixture: + """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to. Records each call; answers reports + with deterministic rows, two report polls, and paged rows.""" + + PAGE_SIZE = 25 + + def __init__(self): + self.reset() + + def reset(self): + self.calls, self.reports = [], {} + self.in_flight = self.max_in_flight = 0 + self.drop_days, self.unsupported = set(), set() + self.responder = None + + def answer(self, prepared, tenant_key, profile_id): + self.calls.append({"method": prepared.method, "path": prepared.path, + "operation": prepared.operation, "body": prepared.body, + "query": dict(prepared.query), "tenant": tenant_key, + "named": dict(prepared.named) if prepared.named else None}) + if self.responder is not None: + return self.responder(prepared) + if prepared.named is not None: + items = next(iter(prepared.body.values())) + return {"success": [{"index": i} for i in range(len(items))], "error": []} + if prepared.operation == "createReport": + report_id = f"fx{len(self.reports) + 1:05d}" + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + self.reports[report_id] = {"body": prepared.body, "polls": 0, "rows": None} + return {"reportId": report_id, "status": "PENDING"} + report_id = prepared.path.split("/")[3] + report = self.reports[report_id] + if prepared.operation == "getReport": + report["polls"] += 1 + if report["body"]["reportTypeId"] in self.unsupported: + self.in_flight -= 1 + return {"status": "FAILED", "failureReason": "Report type is not supported for this advertiser"} + if report["polls"] < 2: + return {"status": "PROCESSING"} + return {"status": "COMPLETED"} + if report["rows"] is None: + report["rows"] = fixture_rows(report["body"], drop_days=self.drop_days) + start = int(prepared.query.get("nextToken") or 0) + page = report["rows"][start:start + self.PAGE_SIZE] + end = start + len(page) + if end >= len(report["rows"]): + self.in_flight -= 1 + return {"rows": page} + return {"rows": page, "nextToken": str(end)} + + +def fixture_rows(body, *, drop_days=()): + """Deterministic Ads report rows for one request body, every day in its window.""" + report_type, columns = body["reportTypeId"], body["columns"] + start, end = date.fromisoformat(body["startDate"]), date.fromisoformat(body["endDate"]) + ad = report_type[:2] + out, day = [], start + while day <= end: + if day.isoformat() not in drop_days: + for c in (1, 2): + for g in ((0,) if report_type.endswith("Campaigns") else (1, 2)): + n = day.toordinal() % 7 + c + g + row = { + "date": day.isoformat(), "campaignId": f"{ad.encode().hex()}10{c}", + "campaignName": f"{ad.upper()} campaign {c}", "campaignStatus": "ENABLED", + "campaignBudgetAmount": 25.0 * c, "campaignBudgetCurrencyCode": "USD", + "adGroupId": f"20{c}{g}", "adGroupName": f"Ad group {c}.{g}", + "keywordId": f"30{c}{g}", "targetingId": f"30{c}{g}", + "targeting": f"term {c}{g}", "targetingExpression": f"asin is B00FX{c}{g}", + "matchType": "exact", "keywordBid": 0.5 + g / 10, + "adId": f"40{c}{g}", "advertisedSku": f"SKU-{c}{g}", + "advertisedAsin": f"B00FX0{c}{g}0", "promotedSku": f"SKU-{c}{g}", + "promotedAsin": f"B00FX0{c}{g}0", "searchTerm": f"search {c}{g}", + "impressions": 100 * n, "clicks": 3 * n, "cost": round(1.25 * n, 2), + "sales": round(9.5 * n, 2), "sales14d": round(9.5 * n, 2), + "purchases": n, "purchases14d": n, "unitsSold": n, "unitsSoldClicks14d": n, + } + out.append({k: row.get(k) for k in columns}) + day += timedelta(days=1) + return out + + +FIXTURE = _Fixture() + + +@contextmanager +def recording_fixture(): + """Route every call to a fresh recording fixture for the block. -> the fixture.""" + previous = os.environ.get(FIXTURE_ENV) + os.environ[FIXTURE_ENV] = "1" + FIXTURE.reset() + try: + yield FIXTURE + finally: + FIXTURE.reset() + if previous is None: + os.environ.pop(FIXTURE_ENV, None) + else: + os.environ[FIXTURE_ENV] = previous + + +__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "NAMED_WRITES", + "FIELD_RULES", "APPROVAL_SOURCE", "REPORT_CONCURRENCY", "REPORTS_PER_DAY", + "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsReportUnavailable", + "AdsReportPending", "fixture_mode", "build_request", "report_operation", + "send_named_write", "request", "credential", "run_report", "FIXTURE", + "recording_fixture", "fixture_rows"] diff --git a/platform/harness/amazon_ads_schema.py b/platform/harness/amazon_ads_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..d14a6af979ecad2182702c8f3972ce78edb9fe3f --- /dev/null +++ b/platform/harness/amazon_ads_schema.py @@ -0,0 +1,178 @@ +"""amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6). + +One entry per ad type and report: the report type, the columns we store, the Ads column each one +is read from, the row identity and the backfill window. The table keys, labels and backfill days +come from `core.connector_spec`, so the two files cannot disagree on a name. + +Unknown: the exact Reporting v1 column spellings. The vendor names below are the Ads reporting +vocabulary the primer records (F11, F12). The first credentialed report settles them (PRD P17), +and a rename is one literal in `_VENDOR`. +""" +import hashlib +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if str(_HERE.parent) not in sys.path: + sys.path.insert(0, str(_HERE.parent)) + +from core import connector_spec as _cs # noqa: E402 + +AD_TYPES = _cs.AMAZON_ADS_AD_TYPES +REPORTS = _cs.AMAZON_ADS_REPORTS +DATABASE_PREFIX = "ut_amzn_" +MIRROR_PREFIX = "amzn_ads_" +#: Amazon accepts at most 31 days per report request. +MAX_DAYS_PER_REQUEST = 31 + +#: The C6 columns every Ads table carries, in this order. +C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date", + "impressions", "clicks", "cost", "sales", "orders") + +_INT = {"impressions", "clicks", "orders", "units"} +_DBL = {"cost", "sales", "bid", "campaign_budget_amount"} + +#: Column labels, at most 40 characters, sentence case, no dash. +LABELS = { + "campaign_id": "Campaign ID", "ad_group_id": "Ad group ID", "target_id": "Target ID", + "advertised_sku": "Advertised SKU", "asin": "ASIN", "date": "Date", + "impressions": "Impressions", "clicks": "Clicks", "cost": "Spend", "sales": "Sales", + "orders": "Orders", "units": "Units sold", "campaign_name": "Campaign", + "campaign_status": "Campaign status", "campaign_budget_amount": "Daily budget", + "ad_group_name": "Ad group", "ad_id": "Ad ID", "targeting": "Targeting", + "match_type": "Match type", "bid": "Bid", "search_term": "Search term", + "profile_id": "Profile ID", "currency": "Currency", +} + +#: Per report, the columns beyond C6, in display order. +_EXTRA = { + "campaigns": ("campaign_name", "campaign_status", "campaign_budget_amount", "units"), + "targeting": ("campaign_name", "ad_group_name", "targeting", "match_type", "bid", "units"), + "advertised_products": ("campaign_name", "ad_group_name", "ad_id", "units"), + "search_terms": ("campaign_name", "ad_group_name", "search_term", "targeting", "match_type", + "units"), +} +_TAIL = ("profile_id", "currency") + +#: Our column -> the Ads column, shared by all three ad types unless `_VENDOR_BY_TYPE` says not. +_VENDOR = { + "campaign_id": "campaignId", "campaign_name": "campaignName", + "campaign_status": "campaignStatus", "campaign_budget_amount": "campaignBudgetAmount", + "ad_group_id": "adGroupId", "ad_group_name": "adGroupName", "target_id": "keywordId", + "targeting": "targeting", "match_type": "matchType", "bid": "keywordBid", + "ad_id": "adId", "advertised_sku": "advertisedSku", "asin": "advertisedAsin", + "search_term": "searchTerm", "date": "date", "impressions": "impressions", + "clicks": "clicks", "cost": "cost", "sales": "sales", "orders": "purchases", + "units": "unitsSold", "currency": "campaignBudgetCurrencyCode", +} +_VENDOR_BY_TYPE = { + "sp": {"sales": "sales14d", "orders": "purchases14d", "units": "unitsSoldClicks14d"}, + "sd": {"target_id": "targetingId", "targeting": "targetingExpression", + "advertised_sku": "promotedSku", "asin": "promotedAsin"}, +} + + +def _short(ad_type, report): + return f"{ad_type}_{report}" + + +def _columns(report): + out = list(C6_COLUMNS) + for name in _EXTRA[report] + _TAIL: + if name not in out: + out.append(name) + return tuple(out) + + +def coltype(name): + """The DuckDB type of one Ads column. Ids and dates are text, so a 17 digit id stays exact.""" + return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR" + + +def vendor_name(ad_type, column): + """The Ads column one of our columns is read from, or None when the vendor has none.""" + if column == "profile_id": + return None + return _VENDOR_BY_TYPE.get(ad_type, {}).get(column) or _VENDOR.get(column) + + +TABLE_SPECS = {} +for _spec in _cs.AMAZON_ADS_TABLES: + _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1) + TABLE_SPECS[_short(_ad_type, _report)] = { + "key": _spec.key, "source": _spec.source, "label": _spec.label, + "ad_type": _ad_type, "report": _report, "report_type": _spec.native, + "columns": _columns(_report), "row_identity": tuple(_spec.identity), + "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key], + "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED, + "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key), + } +del _spec, _ad_type, _report + + +def short_name(table): + """`sp_campaigns`, `ads_sp_campaigns`, `amzn_ads_sp_campaigns` or the `ut_` key -> `sp_campaigns`.""" + name = str(table or "") + for prefix in (DATABASE_PREFIX + "ads_", MIRROR_PREFIX, "ads_"): + if name.startswith(prefix): + name = name[len(prefix):] + break + if name not in TABLE_SPECS: + raise KeyError(f"unknown Amazon Ads table {table!r}") + return name + + +def columns_for(table): + """Our stored columns for one Ads table, in order.""" + return TABLE_SPECS[short_name(table)]["columns"] + + +def vendor_columns(table): + """The Ads columns one report asks for, in our column order, each once.""" + spec = TABLE_SPECS[short_name(table)] + names = (vendor_name(spec["ad_type"], c) for c in spec["columns"]) + return tuple(dict.fromkeys(n for n in names if n)) + + +def _cell(name, value): + if value is None or value == "": + return None + kind = coltype(name) + try: + if kind == "BIGINT": + return int(float(value)) + if kind == "DOUBLE": + return float(value) + except (TypeError, ValueError): + return None + return str(value) + + +def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None): + """One Ads report row -> one stored row, with its stable `_row_key` and report origin.""" + spec = TABLE_SPECS[short_name(table)] + out = {} + for column in spec["columns"]: + source = vendor_name(spec["ad_type"], column) + value = vendor_row.get(source) if source else None + if column == "profile_id": + value = profile_id + out[column] = _cell(column, value) + out["_row_key"] = row_key(table, out) + out["_report_id"] = None if report_id is None else str(report_id) + return out + + +def row_key(table, row): + """The stable identity of one stored row: a re-sync of the same day updates, never appends.""" + spec = TABLE_SPECS[short_name(table)] + raw = "\x1f".join(str(row.get(c) if row.get(c) is not None else "") for c in spec["row_identity"]) + return hashlib.sha1(f"{spec['source']}\x1f{raw}".encode("utf-8")).hexdigest() + + +#: Columns the mirror adds to every Ads table, beside `columns_for`. +ORIGIN_COLUMNS = ("_row_key", "_report_id") + +__all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "MAX_DAYS_PER_REQUEST", + "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for", + "vendor_columns", "row_from_vendor", "row_key"] diff --git a/platform/harness/amazon_ads_store.py b/platform/harness/amazon_ads_store.py new file mode 100644 index 0000000000000000000000000000000000000000..44da574b37d012d85f471c8822ce37612c0dc8a0 --- /dev/null +++ b/platform/harness/amazon_ads_store.py @@ -0,0 +1,343 @@ +"""amazon_ads_store.py: the Amazon Ads loader into the Amazon tenant mirror (S79-T12, contract C6). + + python platform/harness/amazon_ads_store.py --sync --tenant royal-imports + python platform/harness/amazon_ads_store.py --status --tenant royal-imports + +One table per ad type and report, `amzn_ads__`, spawned as `ut_amzn_ads_*`. + +⛔ HISTORY IS KEPT FROM THE CONNECT DAY. The first sync stamps the connect day once and reaches +back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_BACKFILL_DAYS`). +A later sync re-reads the last `RESTATEMENT_DAYS`, because Amazon restates attributed sales, and +upserts by row identity. No sync deletes a day an earlier sync stored, so a day Amazon has aged +out stays here. + +⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one +report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as +`amzn_limit_ads_` with the reason a person reads, and stated on the table description. + +Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_
` is the newest landed day, +`amzn_sync_floor_ads_
` the oldest, `amzn_ads_
` the table state. +""" +import argparse +import importlib +import json +import sys +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if str(_HERE.parent) not in sys.path: + sys.path.insert(0, str(_HERE.parent)) + +from core import connector_spec # noqa: E402 +from harness import amazon_ads_client as client # noqa: E402 +from harness import amazon_ads_schema as schema # noqa: E402 +from harness import amazon_store # noqa: E402 +from harness import datastore # noqa: E402 + +#: C6: called as `fn(tenant_key, synced_tables)` after a sync lands rows. Imported lazily; a +#: failure is one log line, never a failed sync. +ADS_AFTER_SYNC = ("modules.amazon_operator:on_ads_sync",) + +TABLES = tuple(schema.TABLE_SPECS) +#: Amazon restates attributed sales for about 14 days, so each sync re-reads them. +RESTATEMENT_DAYS = 14 +CONNECT_DAY_KEY = amazon_store.entity_key("ads_connect_day") +BUDGET_KEY = amazon_store.entity_key("ads_report_budget") +INSERT_BATCH_ROWS = 500 + + +def table_name(table): + """`sp_campaigns` -> `amzn_ads_sp_campaigns`, the mirror table. One place, so a rename lands whole.""" + return schema.TABLE_SPECS[schema.short_name(table)]["source"] + + +def _marker_table(table): + """The `
` the Amazon mirror markers name: `ads_sp_campaigns`.""" + return table_name(table)[len(amazon_store.PREFIX):] + + +def limit_key(table): + return amazon_store.entity_key("limit_" + _marker_table(table)) + + +def table_description(table, con=None): + """The description a person reads, plus any limit the last sync recorded. Never raises.""" + try: + key = schema.TABLE_SPECS[schema.short_name(table)]["key"] + except KeyError: + return None + text = connector_spec.amazon_ads_description(key) or "" + if con is None: + return text + try: + state = amazon_store._read_state(con, limit_key(table)) + reason = str(json.loads(state["detail"]).get("reason") or "") if state and state.get("detail") else "" + except Exception: # noqa: BLE001 + reason = "" + return f"{text} {reason}" if reason and reason not in text else text + + +def _utc_today(now=None): + now = now or datetime.now(timezone.utc) + return now.date() if isinstance(now, datetime) else now + + +def _connect_day(con, today): + """The connect day, stamped once on the first sync and never moved.""" + state = amazon_store._read_state(con, CONNECT_DAY_KEY) + if state and state.get("detail"): + try: + return date.fromisoformat(str(state["detail"])[:10]) + except ValueError: + pass + amazon_store._record_state(con, CONNECT_DAY_KEY, "done", detail=today.isoformat()) + return today + + +def _day_marker(con, key): + state = amazon_store._read_state(con, key) + try: + return date.fromisoformat(str(state["detail"])[:10]) if state and state.get("detail") else None + except ValueError: + return None + + +def windows(con, table, connect_day, today): + """The report windows one sync asks for, oldest first, each at most 31 days.""" + spec = schema.TABLE_SPECS[schema.short_name(table)] + last = today - timedelta(days=1) + target_floor = max(connect_day, today) - timedelta(days=spec["backfill_days"]) + mark_name = _marker_table(table) + newest = _day_marker(con, amazon_store.entity_key(amazon_store.WATERMARK_PREFIX + mark_name)) + oldest = _day_marker(con, amazon_store.entity_key(amazon_store.BACKFILL_FLOOR_PREFIX + mark_name)) + spans = [] + if newest is None or oldest is None: + spans.append((target_floor, last)) + else: + if oldest > target_floor: + spans.append((target_floor, oldest - timedelta(days=1))) + spans.append((max(target_floor, min(newest, last) - timedelta(days=RESTATEMENT_DAYS - 1)), last)) + out = [] + for start, end in spans: + cursor = start + while cursor <= end: + stop = min(cursor + timedelta(days=client.MAX_DAYS_PER_REQUEST - 1), end) + out.append((cursor, stop)) + cursor = stop + timedelta(days=1) + return out + + +def _ensure_table(con, table): + name = table_name(table) + cols = schema.columns_for(table) + schema.ORIGIN_COLUMNS + have = {r[0] for r in con.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [name]).fetchall()} + if not have: + con.execute(f'CREATE TABLE IF NOT EXISTS "{name}" (' + + ", ".join(f'"{c}" {schema.coltype(c) if not c.startswith("_") else "VARCHAR"}' + for c in cols) + ")") + return + for c in cols: + if c not in have: + con.execute(f'ALTER TABLE "{name}" ADD COLUMN "{c}" ' + f'{schema.coltype(c) if not c.startswith("_") else "VARCHAR"}') + + +def _upsert(con, table, rows): + """Delete-then-insert by `_row_key` only. ⛔ Never by window: a day absent from a re-read stays.""" + if not rows: + return 0 + name = table_name(table) + cols = list(schema.columns_for(table) + schema.ORIGIN_COLUMNS) + by_key = {r["_row_key"]: r for r in rows} + keys = list(by_key) + for i in range(0, len(keys), INSERT_BATCH_ROWS): + part = keys[i:i + INSERT_BATCH_ROWS] + con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part) + con.executemany( + f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) ' + f'VALUES ({", ".join("?" for _ in cols)})', + [[by_key[k].get(c) for c in cols] for k in part]) + return len(by_key) + + +def _budget(con, today): + state = amazon_store._read_state(con, BUDGET_KEY) + try: + detail = json.loads(state["detail"]) if state and state.get("detail") else {} + except (ValueError, TypeError): + detail = {} + return int(detail.get("used") or 0) if detail.get("day") == today.isoformat() else 0 + + +def _spend_budget(con, today, used): + amazon_store._record_state(con, BUDGET_KEY, "done", rows=used, + detail=json.dumps({"day": today.isoformat(), "used": used})) + + +def _record_limit(con, table, reason, **extra): + amazon_store._record_state(con, limit_key(table), "done", + detail=json.dumps({"reason": reason, **extra})) + + +def _clear_limit(con, table): + con.execute("DELETE FROM _sync_state WHERE entity = ?", [limit_key(table)]) + + +def _count(con, table): + return con.execute(f'SELECT count(*) FROM "{table_name(table)}"').fetchone()[0] + + +def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print, + profile_id=None): + """Pull every offered Ads report into the tenant's Amazon mirror. -> a report dict. + + When this call opens the connection and has a runtime, it saves the tables that landed rows + through `amazon_store.persist_mirror`. A caller that lends `con` persists itself. + """ + key = amazon_store._tenant_key(tenant_key) + report = {"tenant": key, "tables": {}, "problems": [], "limits": {}} + fields = None if client.fixture_mode() else client.credential(key, rt) + if not client.fixture_mode() and fields is None: + report["problems"].append("No Amazon Ads credential is stored for this tenant, so nothing " + "was pulled. Connect Amazon Ads under Settings, Connectors.") + return report + profile_id = profile_id or (fields or {}).get("profile_id") + own = con is None + if own: + con, _path = amazon_store._connect_tenant_mirror(key, rt) + try: + datastore._ensure_sync_state(con) + today = _utc_today(now) + report["connectDay"] = _connect_day(con, today).isoformat() + connect_day = date.fromisoformat(report["connectDay"]) + used = _budget(con, today) + synced = [] + for table in (tables or TABLES): + table = schema.short_name(table) + spec = schema.TABLE_SPECS[table] + _ensure_table(con, table) + if not spec["offered"]: + _record_limit(con, table, spec["not_offered_reason"], state="not_offered") + report["tables"][table] = {"state": "not_offered", "pulled": 0, "in_mirror": _count(con, table)} + report["limits"][table] = spec["not_offered_reason"] + continue + pulled, state, mark = 0, "done", _marker_table(table) + for start, end in windows(con, table, connect_day, today): + if used >= client.REPORTS_PER_DAY: + state = "deferred" + reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} " + f"reports are used, so days from {start.isoformat()} load on the next sync.") + _record_limit(con, table, reason, state="deferred") + report["limits"][table] = reason + report["problems"].append(f"{spec['label']}: waiting for tomorrow's report limit") + break + used += 1 + _spend_budget(con, today, used) + try: + report_id, rows = client.run_report(key, table, start, end, rt=rt, profile_id=profile_id) + except client.AdsReportUnavailable as exc: + state = "not_offered" + reason = (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser " + f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, " + f"contact Amazon Ads support with the report type {spec['report_type']}.") + _record_limit(con, table, reason, state="not_offered") + report["limits"][table] = reason + break + except client.AdsError as exc: + state = "failed" + report["problems"].append(f"{spec['label']}: {str(exc)[:200]}") + break + stored = [schema.row_from_vendor(table, r, profile_id=profile_id, report_id=report_id) + for r in rows] + pulled += _upsert(con, table, stored) + # The receipts move only after a window lands. + newest = _day_marker(con, amazon_store.entity_key(amazon_store.WATERMARK_PREFIX + mark)) + oldest = _day_marker(con, amazon_store.entity_key(amazon_store.BACKFILL_FLOOR_PREFIX + mark)) + if newest is None or end > newest: + amazon_store._record_state(con, amazon_store.entity_key(amazon_store.WATERMARK_PREFIX + mark), + "done", _count(con, table), end.isoformat()) + if oldest is None or start < oldest: + amazon_store._record_state(con, amazon_store.entity_key(amazon_store.BACKFILL_FLOOR_PREFIX + mark), + "done", detail=start.isoformat()) + if state == "done": + _clear_limit(con, table) + total = _count(con, table) + amazon_store._record_state(con, amazon_store.entity_key(mark), + "done" if state == "done" else "partial", total, state) + report["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": total} + if pulled: + synced.append(table_name(table)) + log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}") + if own and rt is not None and synced: + _persist(key, rt, con, synced, report, log) + finally: + if own: + con.close() + _after_sync(key, synced, log) + return report + + +def _persist(tenant_key, rt, con, synced, report, log): + """Save the Ads tables that landed rows, with their markers. A failure is a problem, never a raise.""" + try: + amazon_store.persist_mirror(tenant_key, rt, con, tuple(synced), report=report, log=log) + except Exception as exc: # noqa: BLE001 + report["problems"].append(f"The Amazon Ads rows landed here but could not be saved to the " + f"store ({type(exc).__name__}). The next sync saves them again.") + log(report["problems"][-1]) + + +def _after_sync(tenant_key, synced, log): + """C6: each callback is imported lazily; any failure is one log line and changes nothing.""" + if not synced: + return + for target in ADS_AFTER_SYNC: + try: + module_name, _, name = target.partition(":") + getattr(importlib.import_module(module_name), name)(tenant_key, tuple(synced)) + except Exception as exc: # noqa: BLE001 + log(f"Amazon Ads after sync callback {target} failed ({type(exc).__name__}); " + f"the sync result is unchanged.") + + +def status(tenant_key="royal-imports", rt=None): + """Rows per Ads table in the mirror now, None when never synced. Never fetches.""" + con, _path = amazon_store._connect_tenant_mirror(tenant_key, rt) + out = {} + try: + for table in TABLES: + try: + out[table_name(table)] = _count(con, table) + except Exception: # noqa: BLE001 + out[table_name(table)] = None + finally: + con.close() + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description="Pull Amazon Ads reports into the tenant's mirror.") + ap.add_argument("--sync", action="store_true") + ap.add_argument("--status", action="store_true") + ap.add_argument("--tenant", default="royal-imports") + a = ap.parse_args(argv) + if a.status: + for name, rows in status(a.tenant).items(): + print(f" {name:<36} {'(never synced)' if rows is None else rows}") + return 0 + if not a.sync: + ap.print_help() + return 2 + rep = sync(a.tenant) + for problem in rep["problems"]: + print(" PROBLEM:", problem) + return 1 if rep["problems"] else 0 + + +__all__ = ["ADS_AFTER_SYNC", "TABLES", "RESTATEMENT_DAYS", "table_name", "limit_key", + "table_description", "windows", "sync", "status"] + +if __name__ == "__main__": + sys.exit(main()) diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index 931acc668b5023ae1afb8101aa1bb3428eb672bc..33951c64e57647c8ac798b1530c98ed83146a0a3 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -88,9 +88,12 @@ LEGACY_SNAPSHOT_TABLES = ("listings", "inventory") #: Head -> lines: both tables read the same document; a second report buys no new rows. SHARED_REPORT_SIBLINGS = {"orders": "order_items"} -#: Contract C2: called as `fn(tenant, synced_tables)` after a pass publishes. Each import is lazy and a -#: failure is one log line, never a failed sync. -AFTER_SYNC = ("modules.amazon_profit:on_amazon_sync", "modules.competitor_watch:on_amazon_sync") +#: Contracts C2 and C5: called as `fn(tenant, synced_tables)` after a pass publishes. Each import is +#: lazy and a failure (a missing App module too) is one log line, never a failed sync. +AFTER_SYNC = ("modules.amazon_profit:on_amazon_sync", "modules.competitor_watch:on_amazon_sync", + "modules.ranking_diagnostic:on_amazon_sync", "modules.restock_planner:on_amazon_sync", + "modules.account_health:on_amazon_sync", "modules.aged_stock:on_amazon_sync", + "modules.returns_repeat:on_amazon_sync", "modules.opportunity_finder:on_amazon_sync") #: R18: `fn(tenant) -> iterable of str`. Older brand analytics weeks keep these search terms only. #: A missing module is an empty set; a reader that raises prunes nothing that pass. @@ -540,6 +543,46 @@ class _DocumentProbe: flush=True) +#: D-2682: unquote a document whose header cells are all wrapped in `"`. Off only in a negative control. +UNQUOTE_QUOTED_DOCUMENTS = True + + +def _quote_closed(cell): + """True when `cell`, which opens with `"`, also ends its quoted value (an odd closing run).""" + run = len(cell[1:]) - len(cell[1:].rstrip('"')) + return run % 2 == 1 + + +def _quoted_header(header): + """True when every non-empty header cell is wrapped in one pair of `"` (the ledger detail).""" + cells = [cell for cell in header if cell != ""] + return bool(cells) and all(cell.startswith('"') and len(cell) >= 2 and _quote_closed(cell) + for cell in cells) + + +def _unquote_cells(raw): + """One quoted document's split line -> its values: wrapping quotes off, `""` -> `"`. + + A tab inside a quoted value split it in two; the pieces join again up to the closing quote. + A value never closed on its line keeps its original pieces unchanged. + """ + out, open_cells = [], None + for cell in raw: + if open_cells is not None: + open_cells.append(cell) + if _quote_closed("\t".join(open_cells)): + out.append("\t".join(open_cells)[1:-1].replace('""', '"')) + open_cells = None + continue + if cell.startswith('"') and len(cell) >= 2 and _quote_closed(cell): + out.append(cell[1:-1].replace('""', '"')) + elif cell.startswith('"'): + open_cells = [cell] + else: + out.append(cell) + return out + (open_cells or []) + + def _flat_rows(lines, document_id, report_type): """`(columns, row generator)` over one flat file's lines, or `(None, ())` with no header line.""" reader = csv.reader(lines, delimiter="\t", quoting=csv.QUOTE_NONE) @@ -547,6 +590,9 @@ def _flat_rows(lines, document_id, report_type): header = next(reader) except StopIteration: return None, () + quoted = UNQUOTE_QUOTED_DOCUMENTS and _quoted_header(header) + if quoted: + header = _unquote_cells(header) cols = _dedupe([_snake(h) for h in header]) width = len(cols) # The read-only connectors_amazon report probe measures headers without capture metadata. @@ -558,6 +604,8 @@ def _flat_rows(lines, document_id, report_type): for row_number, raw in enumerate(reader, 1): if not raw or (len(raw) == 1 and not str(raw[0]).strip()): continue + if quoted: + raw = _unquote_cells(raw) vals = list(raw[:width]) + [None] * max(0, width - len(raw)) row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals) if not buyer_column(c)} @@ -2954,6 +3002,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): def _after_restore(self, con, result, log): _heal_snapshots(con, result, log) + _heal_quoted_values(con, result, log) refresh_sku_keywords(con) def _restored_state_rows(self, rows, tables, refused): @@ -3112,6 +3161,90 @@ def _heal_snapshots(con, result, log): log(message) +_QUOTE_IDENTITY = ("msku", "fnsku", "asin", "sku") + + +def _app_date_sql(col): + """SQL twin of `app_date` over one VARCHAR column: US date or stamp to ISO, else unchanged.""" + q = _quoted_identifier(col) + parts = (f"regexp_extract({q}, '^\\s*(\\d{{1,2}}/\\d{{1,2}}/\\d{{4}}[ T]\\d{{1,2}}:\\d{{2}}" + f"(?::\\d{{2}})?)\\s*(.*?)\\s*$', ['stamp', 'zone'])") + stamp = f"replace(({parts}).stamp, 'T', ' ')" + ts = (f"coalesce(try_strptime({stamp}, '%m/%d/%Y %H:%M:%S'), " + f"try_strptime({stamp}, '%m/%d/%Y %H:%M'))") + tail = f"({parts}).zone" + named = " ".join(f"WHEN '{name}' THEN '{offset}'" for name, offset in _ZONES.items()) + zone = (f"CASE upper({tail}) WHEN '' THEN '' {named} ELSE CASE " + f"WHEN regexp_full_match({tail}, '[+-]\\d{{2}}:\\d{{2}}') THEN {tail} " + f"WHEN regexp_full_match({tail}, '[+-]\\d{{4}}') " + f"THEN substr({tail}, 1, 3) || ':' || substr({tail}, 4) END END") + return (f"CASE WHEN regexp_full_match(trim({q}), '\\d{{1,2}}/\\d{{1,2}}/\\d{{4}}') " + f"AND try_strptime(trim({q}), '%m/%d/%Y') IS NOT NULL " + f"THEN strftime(try_strptime(trim({q}), '%m/%d/%Y'), '%Y-%m-%d') " + f"WHEN {q} IS NOT NULL AND {ts} IS NOT NULL AND ({zone}) IS NOT NULL " + f"THEN strftime({ts}, '%Y-%m-%dT%H:%M:%S') || ({zone}) ELSE {q} END") + + +def _unquote_mirror_values(con, log=None): + """D-2682 heal: a mirror table saved with quoted values gets them stripped, in SQL. + + -> {mirror: rows}. A base table qualifies when every sampled identity value is wrapped in + `"`. Every VARCHAR value wrapped in `"` loses one pair, `""` becomes `"`, and the date-named + columns of an App table become ISO. Idempotent: a healed table no longer qualifies. + """ + healed = {} + for mirror, kind in _amazon_objects(con): + if kind != "BASE TABLE" or not mirror.startswith(PREFIX): + continue + types = dict(con.execute( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_schema='main' AND table_name=?", [mirror]).fetchall()) + text = [c for c, t in types.items() if str(t).upper() == "VARCHAR"] + identity = [c for c in _QUOTE_IDENTITY if c in text] + if not identity: + continue + sampled = quoted = 0 + for col in identity: + q = _quoted_identifier(col) + n, w = con.execute( + f"SELECT count(*), count(*) FILTER (WHERE v LIKE '\"%\"') FROM (SELECT {q} AS v " + f"FROM {_quoted_identifier(mirror)} WHERE {q} IS NOT NULL AND {q} <> '' LIMIT 200)").fetchone() + sampled, quoted = sampled + n, quoted + w + if not sampled or quoted != sampled: + continue + sets = [] + dated = mirror[len(PREFIX):] in APP_DATE_TABLES + for col in text: + q = _quoted_identifier(col) + plain = (f"CASE WHEN {q} LIKE '\"%\"' THEN replace(substr({q}, 2, length({q}) - 2), " + f"'\"\"', '\"') ELSE {q} END") + if dated and is_app_date_column(col): + plain = _app_date_sql(col).replace(q, f"({plain})") + sets.append(f"{q} = {plain}") + con.execute("BEGIN TRANSACTION") + try: + con.execute(f"UPDATE {_quoted_identifier(mirror)} SET {', '.join(sets)}") + con.execute("COMMIT") + except Exception: + con.execute("ROLLBACK") + raise + healed[mirror] = con.execute(f"SELECT count(*) FROM {_quoted_identifier(mirror)}").fetchone()[0] + if log: + log(f"Amazon quoted values {mirror}: {healed[mirror]:,} rows unquoted") + return healed + + +def _heal_quoted_values(con, result, log): + """Unquote after a restore; healed tables are dirty so the next checkpoint saves them clean.""" + try: + result["dirty"].update(_unquote_mirror_values(con, log)) + except Exception as exc: # noqa: BLE001 + message = (f"Amazon quoted value heal failed ({type(exc).__name__}); the next resync of each " + "quoted table writes clean values.") + result["problems"].append(message) + log(message) + + def _purge_json(rt, name): """A saved mirror JSON document for a purge: the parsed dict, or {} when absent or unreadable.""" try: @@ -4082,6 +4215,58 @@ def _ledger_date(value): raise ValueError("Amazon inventory summary omitted a daily data date") +#: F36: flat-file App tables whose date-named text columns are written as ISO. +APP_DATE_TABLES = ("inventory_ledger_detail", "inventory_planning", "restock_recommendations", + "storage_fees", "long_term_storage_fees", "fba_customer_returns", "reimbursements") +_US_DATE = re.compile(r"(\d{1,2})/(\d{1,2})/(\d{4})") +_US_STAMP = re.compile(r"(\d{1,2})/(\d{1,2})/(\d{4})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?" + r"(?:\s*([A-Za-z]{1,5}|[+-]\d{2}:?\d{2}))?") +_ZONES = {"Z": "+00:00", "UTC": "+00:00", "GMT": "+00:00", "PST": "-08:00", "PDT": "-07:00", + "MST": "-07:00", "MDT": "-06:00", "CST": "-06:00", "CDT": "-05:00", "EST": "-05:00", + "EDT": "-04:00"} + + +def is_app_date_column(name): + """True for `date`, `*_date`, `date_and_time` and `*_date_and_time`.""" + name = str(name or "") + return name == "date" or name.endswith("_date") or name == "date_and_time" \ + or name.endswith("_date_and_time") + + +def app_date(value): + """`MM/DD/YYYY` -> `YYYY-MM-DD` and a US stamp -> ISO. Any other shape returns unchanged.""" + if not isinstance(value, str): + return value + text = value.strip() + try: + if (m := _US_DATE.fullmatch(text)): + return datetime(int(m[3]), int(m[1]), int(m[2])).date().isoformat() + if (m := _US_STAMP.fullmatch(text)): + zone = "" if m[7] is None else _ZONES.get(m[7].upper(), m[7] if m[7][0] in "+-" else None) + if zone is None: + return value + if zone and ":" not in zone: + zone = zone[:3] + ":" + zone[3:] + stamp = datetime(int(m[3]), int(m[1]), int(m[2]), int(m[4]), int(m[5]), int(m[6] or 0)) + return stamp.isoformat() + zone + except ValueError: + return value + return value + + +def normalise_app_dates(table, rows): + """`rows` with every date-named value of an App table in ISO; other tables pass through.""" + if table not in APP_DATE_TABLES: + return rows + + def fixed(row): + for key, value in row.items(): + if is_app_date_column(key) and isinstance(value, str): + row[key] = app_date(value) + return row + return rows.map(fixed) if isinstance(rows, _DocumentRows) else [fixed(row) for row in rows] + + def _snapshot_identity(table, row): """Mirror identity only; legacy grid IDs and AWD's replacement keys stay unchanged.""" sku = row.get("seller_sku") if table == "listings" else row.get("msku") or row.get("fnsku") @@ -4560,6 +4745,7 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log): else [row for part in parts for row in part]) else: columns, rows = parse_flat_file(text, **parser_args) + rows = normalise_app_dates(table, rows) if table in LEGACY_SNAPSHOT_TABLES: return _replace_legacy_snapshot(con, table, columns, rows, meta, document_row_count(text)) if spec.get("snapshot"): @@ -4824,7 +5010,7 @@ def _synced_tables(report): def _after_sync(tenant_key, report, log): - """Contract C2: each callback is imported lazily; any failure is one log line and changes nothing.""" + """Contracts C2 and C5: each callback is imported lazily; any failure is one log line and changes nothing.""" synced = tuple(_synced_tables(report)) if not synced: return diff --git a/platform/harness/analyst.py b/platform/harness/analyst.py index e3af6226e89e4fedf6ce7a332fd667d66bec74cd..938b14cc1fc02c778201e0975d059aa10190f6e8 100644 --- a/platform/harness/analyst.py +++ b/platform/harness/analyst.py @@ -172,7 +172,8 @@ def _get_recipe(arguments): def _tool_schemas(): """Keep complete argument contracts; skill bodies and catalogs load on demand.""" descriptions = { - "list_topics": "List all available topics, metrics, dims and grain. Start here.", + "list_topics": "List topics with metrics, dims and grain. The system prompt's TOPIC " + "INDEX already holds this list.", "describe_topic": "Full topic schema, scope, definitions, dims and business context.", "get_field_values": "Resolve real ids/names BEFORE filtering typed names.", "run_semantic_query": "Read governed data using registered measures and dims.", @@ -212,22 +213,53 @@ def _tool_schemas(): "required": ["skill"]}}}] +def _topic_index(topics): + """One line per queryable topic with its metrics and dims; grid-only topics on one line.""" + lines = [f"{t['topic']} ({t.get('label') or t['topic']}): {', '.join(t['metrics'])}; " + f"dims {', '.join(t.get('dims') or []) or 'none'}" + for t in topics if t.get("metrics")] + grids = [t["topic"] for t in topics if not t.get("metrics")] + if grids: + lines.append("grids with no metrics (no numbers to query): " + ", ".join(grids)) + return "\n".join(lines) + + +def _ops_index(): + """Every analytics op signature as written in the analytics skill, on one line.""" + import re + + ops = (_skill_catalog().get("analytics") or {}).get("ops") or {} + text = " ".join(map(str, ops.values())) if isinstance(ops, dict) else str(ops) + sigs = dict.fromkeys(f"{name}{{{params}}}" + for name, params in re.findall(r"\b([a-z_0-9]+)\{([^}]*)\}", text)) + return " ".join(sigs) or "(read get_recipe(skill=analytics, section=ops))" + + def system_prompt(): """The small model's standing instructions: the platform rules + the skill recipes. Stable content (cache-friendly); per-question content goes in the user turn.""" topics = T.list_topics() - topic_summary = ", ".join(t["topic"] for t in topics) + topic_summary = _topic_index(topics) return f"""AIOS Analyst: wholesale data questions via governed TOOLS. Non-negotiable rules: 1. NEVER write SQL or invent fields/joins/numbers. Use registered schema keys only; recipes - and tools version together. Start unknown asks with list_topics/describe_topic; describe - the relevant topic before selecting metrics/dims from this topic index. -2. Resolve typed customer/product names with get_field_values BEFORE filtering. + and tools version together. The TOPIC INDEX below names every metric and dim: pick the + topic from it, describe_topic it once for its scope rules, then query. Wholesale sales, + margin, units and distinct customers come from sales_lines; order counts from sales_orders. + Invoice-line basis, commission agent, allocated or unallocated sales: invoice_lines. +2. Resolve every typed name or place (customer, product, agent, salesperson, city, state) with + get_field_values BEFORE filtering, even when the spelling looks exact. Every filter value + of any dim comes from get_field_values or an earlier result, never typed from the ask. A + metric named for a subset (agent_allocated_sales, unallocated_line_sales) already applies + that subset: add no filter for it. Once a name is resolved, EVERY query of the ask filters + on its id: a customer list for an agent is filters {{"agent": []}}, not the whole book. Business units: Fisch=5, Royal=6; never mix them in one series unless explicitly comparing. 3. Every presented number comes from a tool result with drillable result_id. Read tool errors and correct calls; never fabricate numbers, including rounded approximate aggregates. 4. Artifact actions (save_view, compose_dashboard, schedule_report, create_alert) need the user's explicit confirmation first. Follow every recipe guard, including updates/deletes. 5. Lead plainly with the finding; state scope and the result's 'window', never guess it. + The final answer names the finding in words with its numbers (for a ranking: the top + names and values), even when a chart or table is also shown. Never only point to a chart. No dates means ALL recorded history, no hidden default. Time-bound asks require date_from/date_to. 6. Single-number asks: give the window/scope TOTAL first, then breakdowns. Consolidated asks: query WITHOUT group_by or sum the exact returned rows; never give only monthly/per-BU components. @@ -239,9 +271,44 @@ def system_prompt(): ONCE; state what is missing in one honest sentence and offer the nearest answerable ask. YoY/decline/growth, rankings (top_n/sort/head), shares, running totals and distributions are answerable via transform_result. Refusing without report_gap OR reporting supported asks fails. + Share or concentration of the top N customers is ANSWERABLE (concentration transform). + Balances at a point in time (stock on hand, units in stock now, inventory level) have NO + metric: a movement topic such as stock_moves sums flows inside a window and is never a + balance. For such an ask call report_gap once, then offer the movement figure as the + nearest ask. Never present a sum of movements as what is on hand. + A window is never a gap: every dated topic takes date_from/date_to, with no date dim. 9. Charts ONLY via make_chart/make_table; say "the chart below". NEVER emit markdown images, data: URIs, chart links, HTML or vega. Derived analytics (top-N/YoY/shares/running totals/ moving averages/ranks/bins) ONLY via transform_result, never mental arithmetic. +10. Write plain text. Never use an em dash or an en dash; write "to" for a range. + +QUERY ARGUMENTS (run_semantic_query; a wrong argument costs a turn): +- Window: date_from and date_to as YYYY-MM-DD, never inside filters. +- Business unit: team_id=5 (Fisch) or team_id=6 (Royal) only when the ask names one; omit it + otherwise. Never filters.team. +- Monthly or weekly series: grain="month" or "week" with NO group_by returns one row per + period. group_by takes only the topic's dims; there is no date dim to group by. +- filters: {{"": [values]}} with the exact ids get_field_values returned (agent, partner, + product). For a city or state pass EVERY matching spelling it returned in one list + (case and trailing-space variants are separate values). +- sort: "-" is largest first (top, biggest, most); "" is smallest first. +- A total: no group_by. Rows: a result shows at most 100 preview rows, and row_count is the + true count. Never count preview rows. Transforms and charts read every row. +- To count rows that meet a condition (customers in a tier, above a value), apply + transform_result {{"op":"filter_rows","col":,"cmp":"==","value":}} (cmp: ==, !=, + >, >=, <, <=, contains) and report the row_count of THAT new result. abc_classify{{of}} + adds abc_class A/B/C. add_total appends a total row: never count rows after it. +- A map needs no coordinates: group_by the customer dim, then make_chart kind="map" with + x=the customer dim and y=the measure. The platform places each customer. +- Same window last year, YoY, decline or growth per customer: ONE query of the current window + with group_by, then transform_result [{{"op":"yoy"}}, {{"op":"sort","by":"revenue_delta", + "direction":"asc"}}, {{"op":"head","n":10}}]. Never query the prior year yourself: two + results cannot be joined by hand. +- Share held by the top 1, 5 or 10 customers: query grouped by customer, then + transform_result [{{"op":"concentration","of":"revenue"}}] ALONE on that full result (no + sort, head or add_total before it). Report its top1/top5/top10_share_pct as given. +- TRANSFORM OPS, exact names and params (no others exist): {_ops_index()} +- Never repeat a call with the same arguments: read its error, change the call, or answer. CHART PICKER: trend=line (few periods=bar); seasonal/two-dim intensity=heatmap; YoY by period=yoy transform then yoy_bars; ranking=top_n then sorted/labeled ranked_bar; @@ -252,7 +319,9 @@ or bin transform then bar; stages=funnel; actual/target=bullet (y=actual,y2=targ level/rate=combo (y=bars left,y2=line right); small multiples=facet on line/bar/area/scatter; exact figures/many columns/lists=make_table with totals; customer geography=map. Ties: simpler. -AVAILABLE TOPIC INDEX (full metrics/dims/scope: list_topics/describe_topic): +TOPIC INDEX (topic (label): metrics; dims. describe_topic gives definitions and scope; +an ask that no metric here measures, such as stock on hand, is a gap: call report_gap once, +never a prose-only refusal): {topic_summary} SKILL RECIPE INDEX (get_recipe(skill,recipe) loads the exact plan, context and guard; @@ -274,7 +343,327 @@ tools; use query/transform rows for numbers, never infer totals from a render re _DEAD_STATUS = {401, 402, 403} # payment/auth: skip the provider immediately, no retry _RETRY_STATUS = {429, 500, 502, 503} # rate/server: retry with backoff before failing over _MAX_ANSWER_TOKENS = 1500 # answers are short; also keeps low-credit providers usable +#: Reasoning models spend completion tokens on thought: 1500 cut glm-4.7-flash off mid-answer. +_MAX_TOKENS_BY_PROVIDER = {"openrouter": 6000} _COOLDOWN = {} # provider name -> unix ts until which we skip it (in-proc) +#: Live turns left at which the model is told to stop reading and answer. +_BUDGET_WARN_TURNS = 3 + + +#: Re-asks per ask for an empty reply or a tool call written as text. +_MAX_REPLY_RETRIES = 2 +#: An identical successful read may run twice: once, then once to restore its receipt. +_MAX_SAME_READS = 2 +_TOOL_MARKUP = ("", "", "<|tool", "_delta\"," + "\"direction\":\"asc\"}}, {{\"op\":\"head\",\"n\":10}}]. yoy adds _ly, " + "_delta and _yoy_pct per row.") + + +def _note_field_values(name, arguments, res, field_values): + """Remember the text values get_field_values returned for each (topic, dim).""" + if name != "get_field_values" or not res.get("ok") or not isinstance(res.get("data"), list): + return + args = _parsed(arguments) + ids = [row.get("id") for row in res["data"] if isinstance(row, dict)] + key = (args.get("topic"), args.get("dim")) + # Ids are kept as text too: an id the model echoes as "9959" is still a resolved value. + field_values.setdefault(key, set()).update(str(v) for v in ids if v is not None) + + +def _expand_variants(arguments, field_values): + """Add every returned spelling of a text filter value: 'BROOKLYN' also takes 'Brooklyn '.""" + args = _parsed(arguments) + filters = args.get("filters") + if not isinstance(filters, dict): + return arguments + fixed = dict(filters) + for dim, value in filters.items(): + known = field_values.get((args.get("topic"), dim)) + values = value if isinstance(value, list) else [value] + if not known or not all(isinstance(v, str) for v in values): + continue + want = {v.strip().casefold() for v in values} + extra = sorted(k for k in known if k.strip().casefold() in want and k not in values) + if extra: + fixed[dim] = list(values) + extra + if fixed == filters: + return arguments + return json.dumps({**args, "filters": fixed}, ensure_ascii=False) + + +_YOY_REASK_NOTE = ("\nCHECK: you queried the same grouping for two windows one year apart and " + "have not run yoy. Two results are never compared by hand. Call " + "transform_result on result_id {result_id} (the LATER window) with " + "[{{\"op\":\"yoy\"}}, {{\"op\":\"sort\",\"by\":\"_delta\",\"direction\":\"asc\"}}, " + "{{\"op\":\"head\",\"n\":10}}], then answer from that result.") +_GAP_REASK_NOTE = ("\nCHECK: your draft says the data is not available, but report_gap was never " + "called. If no registered metric answers the ask, call report_gap now, once, " + "then answer. If a listed metric or transform does answer it, call that tool.") +_UNAVAILABLE = (r"\b(not available|unavailable|not tracked|isn't tracked|is not tracked|" + r"no (metric|topic|data|dim|dimension|field) (for|that|to|on|which)|" + r"(cannot|can't|can not|unable to) (answer|provide|determine|report|compute|" + r"calculate|give|tell)|does not (track|hold|record|provide|contain|have a)|" + r"doesn't (track|hold|record|provide|contain|have a)|not (in|part of) the " + r"(governed|available|registered))") + + +_DATE_GROUP_ERROR = ("There is no date dim to group by. A window is date_from and date_to; a " + "monthly or weekly series is grain=\"month\" or \"week\" with no group_by.") +_WINDOW_GAP_ERROR = ("A window is never a gap: every dated topic takes date_from and date_to, and " + "grain gives months or weeks. Run the query with them instead.") +_TYPED_FILTER_ERROR = ("The filter value for '{dim}' was typed, not resolved. Call get_field_values " + "for that dim (with search) first, then filter on the exact values it returns.") +_WORDS_REASK_NOTE = ("\nCHECK: your draft does not state the finding. Write the answer in words " + "with the exact names and numbers from the tool results (for a ranking, the " + "top names and values). A chart or table alone is not an answer, and " + "placeholders such as [Customer 1] are never allowed.") +_DATE_WORDS = ("date", "month", "period", "year", "week", "day", "time") + + +def _date_group(arguments, topic_dims): + """True when a query groups by a date-like name that is not a dim of its topic.""" + args = _parsed(arguments) + group = args.get("group_by") or [] + group = group if isinstance(group, list) else [group] + dims = topic_dims.get(args.get("topic")) or set() + return any(str(g) not in dims and any(w in str(g).lower() for w in _DATE_WORDS) + for g in group) + + +def _window_gap(arguments): + """True when a gap report claims a missing date dimension or window.""" + missing = str(_parsed(arguments).get("missing") or "").lower() + return any(w in missing for w in ("date dim", "date dimension", "no date", "date field", + "time dimension", "date filter", "filtered by 20", + "filter by year", "by date")) + + +def _typed_filter(arguments, field_values, entries): + """The dim whose text filter value no earlier tool result in this ask returned, else None.""" + args = _parsed(arguments) + filters = args.get("filters") + if not isinstance(filters, dict): + return None + seen = None + for dim, value in filters.items(): + values = value if isinstance(value, list) else [value] + texts = [v for v in values if isinstance(v, str)] + if not texts: + continue + known = field_values.get((args.get("topic"), dim)) or set() + if all(v in known for v in texts): + continue + if seen is None: + seen = " ".join(e["content"] for e in entries if e["response"].get("ok")) + if not all(json.dumps(v, ensure_ascii=False) in seen for v in texts): + return dim + return None + + +def _answer_lacks_finding(content): + """True when a reply carries no figure, or fills a template with placeholders.""" + import re + + text = content or "" + return (not re.search(r"\d", text) + or bool(re.search(r"\[(customer|amount|name|value|revenue)[^\]]*\]", text, re.I))) + + +def _says_unavailable(content): + """True when a reply says the data cannot be had: the case the gap protocol covers.""" + import re + + return bool(re.search(_UNAVAILABLE, content or "", re.IGNORECASE)) + + +def _has_op(arguments, op): + """True when a transform_result chain contains `op`.""" + ops = _parsed(arguments).get("transforms") or [] + return any(isinstance(t, dict) and t.get("op") == op for t in ops) + + +def _parsed(arguments): + """Tool arguments as a dict, or {} when they do not parse.""" + try: + args = json.loads(arguments) if isinstance(arguments, str) else arguments + except (TypeError, ValueError): + return {} + return args if isinstance(args, dict) else {} + + +def _note_result(name, arguments, res, cut_results, queries): + """Record which results are cut, and each query's window, for the guards in ask().""" + data = res.get("data") if res.get("ok") else None + rid = data.get("result_id") if isinstance(data, dict) else None + if not rid: + return + args = _parsed(arguments) + if name == "run_semantic_query": + limit = args.get("limit") + if isinstance(limit, int) and isinstance(data.get("row_count"), int) \ + and data["row_count"] >= limit: + cut_results.add(rid) + rest = {k: v for k, v in args.items() if k not in ("date_from", "date_to", "limit", "sort")} + queries.append((json.dumps(rest, sort_keys=True, default=str), + args.get("date_from"), args.get("date_to"), rid)) + elif name == "transform_result": + ops = [str(t.get("op")) for t in args.get("transforms") or [] if isinstance(t, dict)] + if args.get("result_id") in cut_results or any(op in _CUT_OPS for op in ops): + cut_results.add(rid) + + +def _whole_on_cut(arguments, cut_results): + """True when a whole-result share op runs on a result that a limit or cut made partial.""" + args = _parsed(arguments) + ops = [str(t.get("op")) for t in args.get("transforms") or [] if isinstance(t, dict)] + return args.get("result_id") in cut_results and bool(ops) and ops[0] in _WHOLE_OPS + + +def _prior_year_of(arguments, queries, years=-1): + """The result_id of an earlier query that this one repeats exactly `years` years off, else None. + + years=-1 finds an earlier query one year later than this one; years=1 one year earlier. + """ + args = _parsed(arguments) + lo, hi = args.get("date_from"), args.get("date_to") + if not (isinstance(lo, str) and isinstance(hi, str) and args.get("group_by")): + return None + rest = json.dumps({k: v for k, v in args.items() + if k not in ("date_from", "date_to", "limit", "sort")}, + sort_keys=True, default=str) + + def shift(day, years): + return f"{int(day[:4]) + years:04d}{day[4:]}" if len(day) >= 4 and day[:4].isdigit() else None + + for sig, qlo, qhi, rid in queries: + if sig == rest and isinstance(qlo, str) and isinstance(qhi, str) and \ + ((shift(qlo, years), shift(qhi, years)) == (lo, hi)): + return rid + return None + + +def _cut_before_whole(arguments): + """True when a transform chain cuts rows away before a whole-result share op.""" + try: + args = json.loads(arguments) if isinstance(arguments, str) else arguments + ops = [str(t.get("op")) for t in args.get("transforms") or [] if isinstance(t, dict)] + except (TypeError, ValueError, AttributeError): + return False + cut = next((i for i, op in enumerate(ops) if op in _CUT_OPS), None) + return cut is not None and any(op in _WHOLE_OPS for op in ops[cut + 1:]) + + +def _coerce_args(name, arguments, tools): + """Repair array parameters the model sent as a string, such as "[opex]" or "opex".""" + try: + args = json.loads(arguments) if isinstance(arguments, str) else arguments + except (TypeError, ValueError): + return arguments + schema = next((t["function"].get("parameters") or {} for t in tools + if t.get("function", {}).get("name") == name), {}) + props = schema.get("properties") or {} + if not isinstance(args, dict): + return arguments + fixed = dict(args) + for key, value in args.items(): + if (props.get(key) or {}).get("type") != "array" or not isinstance(value, str): + continue + text = value.strip() + try: + parsed = json.loads(text) + except ValueError: + parsed = None + if isinstance(parsed, list): + fixed[key] = parsed + else: + items = [x.strip().strip("'\"") for x in text.strip("[]").split(",")] + fixed[key] = [x for x in items if x] + if fixed == args: + return arguments + return json.dumps(fixed, ensure_ascii=False) +_EMPTY_REPLY_NOTE = ("\nYour last reply was empty or wrote a tool call as text. Write the final " + "answer in plain words from the tool results above, or make a real tool call.") +_REPEAT_ERROR = ("repeat: this exact call already ran in this ask and its result is above. " + "Change the arguments, or write the final answer from the results you have.") + + +def _budget_note(left): + """The working-context note that tells the model how many turns it has left.""" + if left <= 1: + return ("\nBUDGET: this is your LAST turn and tools are off. Write the final answer now " + "from the tool results above, with the exact numbers and names they returned.") + return (f"\nBUDGET: {left} turns left including the final answer. Call only a tool the ask " + "still needs (its query, transform or requested chart), then answer. Skip re-reads, " + "schema lookups and tables nobody asked for.") def _live_chat(messages, tools, model, tool_choice="auto"): @@ -305,7 +694,10 @@ def _live_chat(messages, tools, model, tool_choice="auto"): sent_model = model or os.environ.get("ANALYST_MODEL") or p["model"] body = {"model": sent_model, "messages": messages, "tools": tools, "tool_choice": tool_choice, "temperature": 0.1, - "max_tokens": _MAX_ANSWER_TOKENS} + "max_tokens": _MAX_TOKENS_BY_PROVIDER.get(p["name"], _MAX_ANSWER_TOKENS)} + if tools is None: # an answer-only turn sends no tool list + body.pop("tools") + body.pop("tool_choice") if p["name"] == "openrouter": body["usage"] = {"include": True} # OpenRouter returns usage.cost in USD try: @@ -408,6 +800,12 @@ def may_ask(user): and not perms.scope_agent(user)) +_PREVIEW_NOTE = ("rows holds only the first {shown} of {total} rows. Never count, rank or total " + "from it. Count rows with transform_result filter_rows and read its row_count; " + "find the largest or smallest with sort{{by,direction}} then head{{n}}; get " + "shares and totals from a transform or a query with no group_by.") + + def _tool_content(name, response, tool_call_id, metadata, artifact_index=None): """Lossless JSON and receipts for retained renders. @@ -419,6 +817,13 @@ def _tool_content(name, response, tool_call_id, metadata, artifact_index=None): def serialize(value): return json.dumps(value, default=str, ensure_ascii=False, separators=(",", ":")) + data = response.get("data") if isinstance(response, dict) else None + if (name in ("run_semantic_query", "transform_result") and isinstance(data, dict) + and isinstance(data.get("rows"), list) + and isinstance(data.get("row_count"), int) and data["row_count"] > len(data["rows"])): + # A preview read as the whole result gave wrong counts and totals: label it in place. + response = {**response, "data": {**data, "preview": _PREVIEW_NOTE.format( + shown=len(data["rows"]), total=data["row_count"])}} content = serialize(response) if artifact_index is not None and name in ("make_chart", "make_table"): data = response["data"] @@ -534,7 +939,13 @@ class _AskContext: if isinstance(data, dict): # Scope/warnings and result identity survive; receipts never assert totals. receipt.update({k: v for k, v in data.items() - if k in ("result_id", "window", "warning", "note")}) + if k in ("result_id", "row_count", "window", "warning", "note")}) + rows = data.get("rows") + # A small result keeps its exact rows, so a later turn never loses an earlier figure. + if (isinstance(rows, list) and len(rows) <= _RECEIPT_ROWS + and data.get("row_count", len(rows)) == len(rows) + and len(self._json(rows)) <= _RECEIPT_ROW_BYTES): + receipt["rows"] = rows return self._json(receipt) def working_messages(self): @@ -610,6 +1021,13 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=Non # The live ladder's first provider sets the ceiling. An injected chat_fn has no provider. provider = (available_providers() or [{}])[0].get("name") if chat is _live_chat else None context = _AskContext(messages, tools, request_budget(provider)) + calls, failed_calls, retries = {}, set(), 0 + cut_results, queries, field_values = set(), [], {} + yoy_pending, yoy_done, reasked = set(), False, set() + try: + topic_dims = {t["topic"]: set(t.get("dims") or []) for t in T.list_topics()} + except Exception: # noqa: BLE001 - guard only + topic_dims = {} for i in range(max_iters): try: @@ -620,7 +1038,16 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=Non "artifacts": artifacts, "tool_trace": trace, "iterations": i, "exhausted": True, "context_exhausted": True, "usage": usage_total, **context.retained()} - msg, usage = chat(working, tools, model) + left = max_iters - i + final_turn = chat is _live_chat and left == 1 and bool(trace) + if chat is _live_chat and left <= _BUDGET_WARN_TURNS and trace: + # Working copy only: the stored system message stays byte-stable. + working[0] = dict(working[0], content=working[0]["content"] + _budget_note(left)) + if final_turn: + # The last turn answers from the results already read, so a long ask ends in an answer. + msg, usage = _live_chat(working, tools, model, tool_choice="none") + else: + msg, usage = chat(working, tools, model) _log_call({"provider": "chat_fn", "model": model, **(usage or {})}) for k in usage_total: usage_total[k] += usage.get(k, 0) or 0 @@ -635,6 +1062,36 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=Non for k in usage_total: usage_total[k] += usage.get(k, 0) or 0 tool_calls = msg.get("tool_calls") or [] + while (chat is _live_chat and retries < _MAX_REPLY_RETRIES + and not tool_calls and _unusable_reply(msg.get("content"))): + # An empty reply, or a tool call written as text, is a broken turn: ask again. + retries += 1 + nudge = dict(working[0], content=working[0]["content"] + _EMPTY_REPLY_NOTE) + msg, usage = _live_chat([nudge] + working[1:], None if final_turn else tools, model) + _log_call(usage) + for k in usage_total: + usage_total[k] += usage.get(k, 0) or 0 + tool_calls = msg.get("tool_calls") or [] + if chat is _live_chat and not tool_calls and not final_turn: + # A final answer that skipped a required step gets one corrected re-ask. + note = None + if yoy_pending and not yoy_done and "yoy" not in reasked: + reasked.add("yoy") + note = _YOY_REASK_NOTE.format(result_id=sorted(yoy_pending)[-1]) + elif ("gap" not in reasked and _says_unavailable(msg.get("content")) + and not any(t["tool"] == "report_gap" for t in trace)): + reasked.add("gap") + note = _GAP_REASK_NOTE + elif artifacts and "words" not in reasked and _answer_lacks_finding(msg.get("content")): + reasked.add("words") + note = _WORDS_REASK_NOTE + if note: + nudge = dict(working[0], content=working[0]["content"] + note) + msg, usage = _live_chat([nudge] + working[1:], tools, model) + _log_call(usage) + for k in usage_total: + usage_total[k] += usage.get(k, 0) or 0 + tool_calls = msg.get("tool_calls") or [] messages.append({"role": "assistant", "content": msg.get("content") or "", **({"tool_calls": tool_calls} if tool_calls else {})}) if not tool_calls: @@ -664,7 +1121,52 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=Non for tc in tool_calls: name = tc["function"]["name"] args = tc["function"].get("arguments") or "{}" - res = context.dispatch(name, args) + args = _coerce_args(name, args, tools) + if name == "run_semantic_query": + args = _expand_variants(args, field_values) + key = _AskContext._key(name, args) + seen = calls.get(key, 0) + if key in failed_calls or (seen and name not in _AskContext._READS) \ + or seen >= _MAX_SAME_READS: + # A repeat cannot change the outcome: say so instead of running it again. + res = {"ok": False, "error": _REPEAT_ERROR} + elif name == "run_semantic_query" and _bu_in_filters(args): + res = {"ok": False, "error": _BU_FILTER_ERROR} + elif name == "run_semantic_query" and _date_in_filters(args): + res = {"ok": False, "error": _DATE_FILTER_ERROR} + elif name == "run_semantic_query" and _date_group(args, topic_dims): + res = {"ok": False, "error": _DATE_GROUP_ERROR} + elif name == "report_gap" and _window_gap(args): + res = {"ok": False, "error": _WINDOW_GAP_ERROR} + elif name == "run_semantic_query" and (dim := _typed_filter(args, field_values, + context.entries)): + res = {"ok": False, "error": _TYPED_FILTER_ERROR.format(dim=dim)} + elif name == "transform_result" and _cut_before_whole(args): + res = {"ok": False, "error": _CUT_BEFORE_WHOLE_ERROR} + elif name == "transform_result" and _whole_on_cut(args, cut_results): + res = {"ok": False, "error": _WHOLE_ON_CUT_ERROR} + elif name == "run_semantic_query" and (prior := _prior_year_of(args, queries)): + res = {"ok": False, "error": _YOY_BY_HAND_ERROR.format(result_id=prior)} + yoy_pending.add(prior) + else: + earlier = (_prior_year_of(args, queries, years=1) + if name == "run_semantic_query" else None) + res = context.dispatch(name, args) + if earlier and res.get("ok") and isinstance(res.get("data"), dict) \ + and res["data"].get("result_id"): + # The prior year ran first: point the model at yoy on THIS result. + rid = res["data"]["result_id"] + res = {**res, "data": {**res["data"], + "yoy_note": _YOY_BY_HAND_ERROR.format(result_id=rid)}} + yoy_pending.add(rid) + if name == "transform_result" and res.get("ok") and _has_op(args, "yoy"): + yoy_pending.clear() + yoy_done = True + _note_result(name, args, res, cut_results, queries) + _note_field_values(name, args, res, field_values) + calls[key] = seen + 1 + if not res.get("ok"): + failed_calls.add(key) trace.append({"tool": name, "args": args if isinstance(args, str) else json.dumps(args), "ok": res["ok"], "error": res.get("error")}) artifact_index = None diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index 5bb2edb955f2af4c88505746303ab421b1605375..b7cdac90046ec72803ec913b564900c8e2bf0cad 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -26,8 +26,8 @@ import re import sys import time import tempfile -from collections import OrderedDict -from contextlib import nullcontext +from collections import OrderedDict, deque +from contextlib import contextmanager, nullcontext from pathlib import Path import duckdb @@ -203,17 +203,13 @@ def use_path(path): `harness.runtime` hands out the PATH rather than switching the global underneath a request. """ global DB_PATH - with _INSTANCE_LOCK: - con = _INSTANCE.get("con") - if con is not None: - try: - con.close() - except Exception: - pass - _INSTANCE["con"] = None + # Before the drain: an idle cursor from the old store must refuse, never rebind to the new one. + _STORE_EPOCH[0] += 1 + with _swap_instance(): _RO_TLS.__dict__.pop("cur", None) _READY["ok"] = False _GENERATION[0] += 1 # every other thread's cached cursor is now stale + _VERSION.update(stamp=None, file=None, version=None, checked=0.0) DB_PATH = Path(path) _invalidate_counts() return DB_PATH @@ -373,9 +369,9 @@ ENTITIES = { # count-of-lines measure. "fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit", "balance", "display_type", "move_type", "parent_state", "price_subtotal", - "product_id", "analytic_distribution", "write_date"], + "product_id", "analytic_distribution", "write_date", "amount_residual", "date_maturity", "reconciled"], "backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal", - "product_id", "analytic_distribution"], + "product_id", "analytic_distribution", "amount_residual", "date_maturity", "reconciled"], "backfill_before_delta": True, # the BU bridge: account.move carries NO business unit (every invoice sits on team 1 — # see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable @@ -775,6 +771,22 @@ def _mirror_measurement(started): def restore_odoo_mirror(tenant_key, rt, log=print): """Restore only this tenant's saved Odoo tables and preserve a refusal until restart.""" + if (versioned_mode() and getattr(_SYNC_TLS, "candidate", None) is None + and Path(rt.datastore_path).resolve() == DB_PATH.resolve()): + # Readers hold the serving version read-only, so the restore writes the next version. + try: + with versioned_write(tenant_key) as handle: + result = _restore_odoo_mirror(tenant_key, rt, log) + handle["publish"] = result.get("state") != "refused" + except MirrorVersionRefused as exc: + # The restore itself worked. Its version failed verify, so the old version serves. + log(str(exc)) + result = {**result, "problems": [*result.get("problems", []), str(exc)]} + return result + return _restore_odoo_mirror(tenant_key, rt, log) + + +def _restore_odoo_mirror(tenant_key, rt, log): started = time.monotonic() result = _odoo_mirror_store().restore_mirror(tenant_key, rt, log=log) status = ODOO_MIRROR_STATUS.setdefault(str(Path(rt.datastore_path).resolve()), {}) @@ -913,14 +925,529 @@ def _instance(): never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole life, so a SEPARATE process — e.g. mcp_server.py — cannot open the same store file concurrently; in production only the app runs, and that was already true during any sync pass.)""" + if versioned_mode(): + _follow_version() con = _INSTANCE["con"] if con is not None: return con + opened = False with _INSTANCE_LOCK: + _wait_for_swap_locked() if _INSTANCE["con"] is None: DB_PATH.parent.mkdir(parents=True, exist_ok=True) - _INSTANCE["con"] = apply_limits(duckdb.connect(str(DB_PATH))) - return _INSTANCE["con"] + if versioned_mode(): + stamp = _pointer_stamp() + physical, version = serving_file() + _INSTANCE["con"] = apply_limits(duckdb.connect(str(physical), read_only=True)) + _VERSION.update(stamp=stamp, file=str(physical), version=version, + checked=time.monotonic()) + opened = True + else: + _INSTANCE["con"] = apply_limits(duckdb.connect(str(DB_PATH))) + _own(_INSTANCE["con"]) + con = _INSTANCE["con"] + if opened: + _publish_serving() + return con + + +# ----------------------------------------------------------------------------- read drain (F47) +# A swap never closes an instance under a cursor that is mid-read. Each `_TimedCursor` taken off a +# shared instance registers with that instance's `_Owner`. A swap marks the owner not live, waits +# (bounded) until none of its cursors is busy, then closes it. An idle cursor whose instance was +# swapped out rebinds to the serving instance on its next statement, within the same store only. +#: The longest a swap waits for open reads before it closes their instance anyway. +DRAIN_SECONDS = 30.0 +#: A result nobody fetched from for this long counts as abandoned, not mid-read. +READ_IDLE_SECONDS = 2.0 +_READS = _thr.Condition() +_OWNERS = {} # id(instance) -> _Owner, for every shared instance this module opened +#: Bumped by `use_path` only. A cursor opened under another epoch never rebinds (tenant switch). +_STORE_EPOCH = [0] +_SWAP = {"active": False, "thread": None} +_SWAP_COND = _thr.Condition(_INSTANCE_LOCK) +_SWAP_LOCK = _thr.RLock() + + +class _Owner: + """One shared DuckDB instance and the cursors reading it.""" + + __slots__ = ("live", "cursors", "__weakref__") + + def __init__(self): + import weakref + self.live = True + self.cursors = weakref.WeakSet() + + +def _own(con): + owner = _Owner() + _OWNERS[id(con)] = owner + return owner + + +def _drain_seconds(): + try: + return max(0.0, float(os.environ.get("AIOS_MIRROR_DRAIN_SECONDS") or DRAIN_SECONDS)) + except ValueError: + return DRAIN_SECONDS + + +def open_reads(con=None): + """How many cursors on `con` (default: the serving instance) are mid-read right now.""" + owner = _OWNERS.get(id(con if con is not None else _INSTANCE.get("con"))) + if owner is None: + return 0 + now = time.monotonic() + with _READS: + return sum(1 for cur in list(owner.cursors) if cur._busy(now)) + + +def _drain(owner, seconds, ignore_thread=None): + """Wait until no cursor on `owner` is mid-read, at most `seconds`. True when it drained. + + `ignore_thread` skips a pending result left on the swapping thread: it cannot be mid-read. + """ + if owner is None: + return True + deadline = time.monotonic() + max(0.0, seconds) + with _READS: + while True: + now = time.monotonic() + if not any(cur._busy(now, ignore_thread) for cur in list(owner.cursors)): + return True + if now >= deadline: + return False + _READS.wait(min(deadline - now, 0.05)) + + +def _close_instance(con): + owner = _OWNERS.pop(id(con), None) + if owner is not None: + owner.live = False + try: + con.close() + except Exception: # noqa: BLE001 + pass + + +def _wait_for_swap_locked(): + """Hold a new reader until a running swap installs the next file. Call under _INSTANCE_LOCK.""" + deadline = time.monotonic() + _drain_seconds() + 5.0 + while _SWAP["active"] and _SWAP["thread"] != _thr.get_ident(): + left = deadline - time.monotonic() + if left <= 0: + break + _SWAP_COND.wait(left) + + +@contextmanager +def _swap_instance(): + """Detach the serving instance, drain its open reads, then run the body under _INSTANCE_LOCK + after closing it. New readers wait for the body (they would reopen the old file otherwise: + DuckDB hands back the cached instance for a path that is still open).""" + with _SWAP_LOCK: + with _INSTANCE_LOCK: + old = _INSTANCE.get("con") + owner = _OWNERS.get(id(old)) if old is not None else None + _INSTANCE["con"] = None + _SWAP.update(active=True, thread=_thr.get_ident()) + if owner is not None: + owner.live = False + try: + if not _drain(owner, _drain_seconds(), ignore_thread=_thr.get_ident()): + print(f"[datastore] mirror swap closed {open_reads(old)} read(s) still open after " + f"{_drain_seconds():.0f} s", flush=True) + with _INSTANCE_LOCK: + if old is not None: + _close_instance(old) + yield + finally: + with _INSTANCE_LOCK: + _SWAP.update(active=False, thread=None) + _SWAP_COND.notify_all() + + +def _open_cursor(): + """(raw cursor, owner) on the serving instance. Retries once if a swap closed it meanwhile.""" + for attempt in range(2): + con = _instance() + owner = _OWNERS.get(id(con)) + try: + return con.cursor(), owner + except Exception: # noqa: BLE001 + if attempt or owner is None or owner.live: + raise + raise RuntimeError("unreachable") + + +# ----------------------------------------------------------------------------- mirror versions +# Several uvicorn workers serve one mirror: DuckDB allows one read-write process OR many read-only +# processes per file, never both. In versioned mode every web process opens the serving version +# read-only, and a write (hand-off install, boot restore) copies it to the next version, writes +# there, checkpoints, and moves the pointer `.current`. The logical DB_PATH never changes, +# so every path-equality guard keeps holding. Readers follow the pointer on their next read. +VERSIONS_ENV = "AIOS_MIRROR_VERSIONS" +#: How often a reader stats the pointer. A stat is microseconds, the throttle keeps hot loops flat. +VERSION_CHECK_SECONDS = 0.25 +#: Versions kept on disk: the serving one and the one before it. +VERSIONS_KEPT = 2 +_VERSION = {"stamp": None, "file": None, "version": None, "checked": 0.0} +_VERSION_WRITE_LOCK = _thr.RLock() +#: Instances dropped by a version switch: (version, connection). Their open cursors finish on the +#: old file. `_reap_retired` closes each once its cursors are done, or past the drain bound. +_RETIRED = deque() +_RETIRED_AT = {} # id(retired instance) -> when it was retired +#: The last version verify: {version, ok, problems, at}. A refused version never serves. +VERSION_VERIFY = {"last": None} + + +class MirrorVersionRefused(RuntimeError): + """A new mirror version failed its verify, so the previous version keeps serving.""" + + +def versioned_mode(): + """True for a web process launched with AIOS_MIRROR_VERSIONS=1 (several uvicorn workers). + + Web only: its sole mirror writes are the Odoo boot restore and hand-off installs, both versioned. + """ + # A literal name, so the portability gate resolves the read (VERSIONS_ENV names the same var). + return os.environ.get("AIOS_MIRROR_VERSIONS") == "1" and os.environ.get("AIOS_PROCESS_ROLE") == "web" + + +def _pointer_path(path=None): + base = Path(path or DB_PATH) + return base.with_name(base.name + ".current") + + +def _version_file(version, path=None): + base = Path(path or DB_PATH) + return base.with_name(f"{base.stem}.v{int(version)}{base.suffix}") + + +def _pointer_stamp(path=None): + try: + st = os.stat(_pointer_path(path)) + except OSError: + return None + return (st.st_ino, st.st_mtime_ns, st.st_size) + + +def serving_file(path=None): + """(physical file, version) the pointer names. The logical path and 0 when no version exists.""" + base = Path(path or DB_PATH) + try: + with open(_pointer_path(base), encoding="utf-8") as handle: + pointer = json.load(handle) + candidate = base.with_name(str(pointer["file"])) + if candidate.name == Path(str(pointer["file"])).name and candidate.exists(): + return candidate, int(pointer.get("version") or 0) + except (OSError, ValueError, KeyError, TypeError): + pass + return base, 0 + + +def serving_version(): + """{version, file} this process reads, for readiness and the primary's state file.""" + return {"version": _VERSION["version"], "file": _VERSION["file"]} + + +def _follow_version(force=False): + """Drop the instance when the pointer moved, so the next read opens the new version.""" + now = time.monotonic() + if not force and now - _VERSION["checked"] < VERSION_CHECK_SECONDS: + return False + _VERSION["checked"] = now + if _RETIRED: + _reap_retired() + stamp = _pointer_stamp() + if stamp == _VERSION["stamp"] or _INSTANCE["con"] is None: + return False + with _INSTANCE_LOCK: + old = _INSTANCE["con"] + if old is None or stamp == _VERSION["stamp"]: + return False + # Dropped, not closed: an in-flight cursor finishes on the old version. + version = _VERSION["version"] + owner = _OWNERS.get(id(old)) + if owner is not None: + owner.live = False # an idle cursor rebinds to the new version on its next statement + _RETIRED.append((version, old)) + _INSTANCE["con"] = None + _VERSION["stamp"] = stamp + _RO_TLS.__dict__.pop("cur", None) + _READY["ok"] = False + _GENERATION[0] += 1 # every thread's cached cursor reopens on the new version + _invalidate_counts() + _RETIRED_AT[id(old)] = time.monotonic() + _reap_retired() + return True + + +def _reap_retired(): + """Close each retired instance whose cursors finished, or that passed the drain bound. + + Readers call this through `_follow_version`, so no thread of its own is needed. + """ + now = time.monotonic() + closed = False + for version, con in list(_RETIRED): + if open_reads(con) and now - _RETIRED_AT.get(id(con), now) < _drain_seconds(): + continue + with _INSTANCE_LOCK: + try: + _RETIRED.remove((version, con)) + except ValueError: + continue + _RETIRED_AT.pop(id(con), None) + _close_instance(con) + closed = True + if closed: + try: + _publish_serving() + _sweep_versions() + except OSError as exc: + print(f"[datastore] mirror version sweep failed ({type(exc).__name__})", flush=True) + + +def _serving_dir(path=None): + base = Path(path or DB_PATH) + return base.with_name(base.name + ".serving") + + +def _held_versions(): + held = {version for version, _con in list(_RETIRED) if version is not None} + if _INSTANCE.get("con") is not None and _VERSION["version"] is not None: + held.add(int(_VERSION["version"])) + return sorted(held) + + +def _publish_serving(path=None): + """Write this worker's open versions to `.serving/.json` for the sweep.""" + if not versioned_mode(): + return + folder = _serving_dir(path) + folder.mkdir(parents=True, exist_ok=True) + target = folder / f"{os.getpid()}.json" + tmp = folder / f"{os.getpid()}.json.tmp" + with open(tmp, "w", encoding="utf-8", newline="") as handle: + json.dump({"pid": os.getpid(), "versions": _held_versions(), "at": time.time()}, handle) + os.replace(tmp, target) + + +def _pid_alive(pid): + if pid == os.getpid(): + return True + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except (PermissionError, OSError): + return True + return True + + +def workers_holding(path=None): + """{version: [pid, ...]} every live worker still reads. Dead workers' markers are removed.""" + held = {} + folder = _serving_dir(path) + if not folder.is_dir(): + return held + for marker in folder.glob("*.json"): + try: + data = json.loads(marker.read_text(encoding="utf-8")) + pid = int(data["pid"]) + except (OSError, ValueError, KeyError, TypeError): + continue + if not _pid_alive(pid): + marker.unlink(missing_ok=True) + continue + for version in data.get("versions") or (): + held.setdefault(int(version), []).append(pid) + return held + + +def _sweep_versions(path=None): + """Unlink versions older than the kept window that no live worker still reads.""" + _physical, serving = serving_file(path) + held = workers_holding(path) + removed = [] + for old, files in _existing_versions(path).items(): + if old <= serving - VERSIONS_KEPT and old not in held: + for item in files: + item.unlink(missing_ok=True) + removed.append(old) + return sorted(removed) + + +@contextmanager +def _file_lock(path): + try: + import fcntl + except ImportError: # not Linux: one process only + yield + return + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + yield + finally: + try: + fcntl.flock(fd, fcntl.LOCK_UN) + finally: + os.close(fd) + + +def _existing_versions(path=None): + base = Path(path or DB_PATH) + rx = re.compile(re.escape(base.stem) + r"\.v(\d+)" + re.escape(base.suffix) + r"(\.wal)?$") + found = {} + if base.parent.is_dir(): + for item in base.parent.iterdir(): + match = rx.fullmatch(item.name) + if match: + found.setdefault(int(match.group(1)), []).append(item) + return found + + +def _publish_version(version, new): + pointer = _pointer_path() + tmp = pointer.with_name(pointer.name + f".{os.getpid()}.tmp") + with open(tmp, "w", encoding="utf-8", newline="") as handle: + json.dump({"version": int(version), "file": new.name, "at": time.time(), "pid": os.getpid()}, handle) + os.replace(tmp, pointer) + if not DB_PATH.exists(): + # Readiness and seed checks ask for the logical file. It never receives a write. + try: + os.link(new, DB_PATH) + except OSError: + import shutil + shutil.copyfile(new, DB_PATH) + _publish_serving() + # An old version goes only once no live worker reads it (each worker writes its marker). + _sweep_versions() + + +def _version_facts(con): + """{tables, phases, publications} of one mirror file, the baseline a new version must keep.""" + tables = {row[0] for row in con.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main'").fetchall()} + phases = (dict(con.execute("SELECT entity, phase FROM _sync_state").fetchall()) + if "_sync_state" in tables else {}) + publications = ({row[0] for row in con.execute("SELECT source FROM _mirror_publications").fetchall()} + if "_mirror_publications" in tables else set()) + return {"tables": tables, "phases": phases, "publications": publications} + + +def verify_version(path, baseline=None, expect_tables=()): + """Problems that stop a new version serving. An empty list means it may serve. + + It must open read-only, read `_sync_state`, keep each required entity that was live live and + readable, keep each publication, and hold every table in `expect_tables`. + """ + try: + con = duckdb.connect(str(path), read_only=True) + except Exception as exc: # noqa: BLE001 + return [f"does not open read-only ({type(exc).__name__}: {str(exc)[:120]})"] + problems = [] + try: + try: + facts = _version_facts(con) + except Exception as exc: # noqa: BLE001 + return [f"metadata unreadable ({type(exc).__name__}: {str(exc)[:120]})"] + if "_sync_state" not in facts["tables"]: + problems.append("_sync_state is missing") + base = baseline or {} + needed = set(expect_tables) + for key, spec in ENTITIES.items(): + if spec.get("optional") or (base.get("phases") or {}).get(key) != "live": + continue + if facts["phases"].get(key) != "live": + problems.append(f"{key} phase went from live to {facts['phases'].get(key)}") + if key in (base.get("tables") or ()): + needed.add(key) + for table in sorted(needed): + if table not in facts["tables"]: + problems.append(f"table {table} is missing") + continue + try: + con.execute(f'SELECT 1 FROM "{table}" LIMIT 1').fetchall() + except Exception as exc: # noqa: BLE001 + problems.append(f"table {table} is unreadable ({type(exc).__name__})") + for source in sorted(base.get("publications") or ()): + if source not in facts["publications"]: + problems.append(f"publication {source} is missing") + finally: + con.close() + return problems + + +@contextmanager +def versioned_write(tenant_key="royal-imports"): + """Write the next mirror version, then publish it. Yields {publish, version, path} or None. + + Outside versioned mode it yields None and the caller writes in place. Inside, every + `connect()` and `_mirror_connection` on this thread lands in the new version. Set + `handle["publish"] = False` to discard an unchanged version. Before the pointer moves the new + version passes `verify_version` (add tables to `handle["expect_tables"]`); a refused version + is discarded, the old one keeps serving, and `MirrorVersionRefused` is raised. + """ + if not versioned_mode(): + yield None + return + import shutil + if getattr(_SYNC_TLS, "candidate", None) is not None: + raise RuntimeError("A source publication is already active on this thread") + published = refused = None + with _VERSION_WRITE_LOCK, _file_lock(_pointer_path().with_name(_pointer_path().name + ".lock")): + source, version = serving_file() + number = max([version, *_existing_versions().keys()]) + 1 + new = _version_file(number) + for stale in (new, new.with_name(new.name + ".wal")): + stale.unlink(missing_ok=True) + if source.exists(): + tmp = new.with_name(new.name + f".{os.getpid()}.tmp") + shutil.copyfile(source, tmp) + os.replace(tmp, new) + wal = source.with_name(source.name + ".wal") + if wal.exists(): + shutil.copyfile(wal, new.with_name(new.name + ".wal")) + con = apply_limits(duckdb.connect(str(new))) + handle = {"publish": True, "version": number, "path": new, "expect_tables": []} + ok = False + baseline = None + try: + _ensure_sync_state(con) + baseline = _version_facts(con) + _SYNC_TLS.candidate = {"tenant": tenant_key, "connector": "mirror-version", + "path": DB_PATH.resolve(), "con": con} + yield handle + ok = True + finally: + _SYNC_TLS.candidate = None + try: + if ok and handle["publish"]: + con.execute("CHECKPOINT") + finally: + con.close() + if ok and handle["publish"]: + refused = verify_version(new, baseline, handle.get("expect_tables") or ()) + VERSION_VERIFY["last"] = {"version": number, "ok": not refused, + "problems": list(refused), "at": time.time()} + if ok and handle["publish"] and not refused: + _publish_version(number, new) + published = number + else: + for item in (new, new.with_name(new.name + ".wal")): + item.unlink(missing_ok=True) + if refused: + message = f"mirror version {number} refused, version {version} keeps serving: " + "; ".join(refused) + print(f"[datastore] {message}", flush=True) + raise MirrorVersionRefused(message) + if published is not None: + note_mirror_install() + _follow_version(force=True) class _TimedCursor: @@ -929,16 +1456,86 @@ class _TimedCursor: _FETCH = {"fetchone", "fetchall", "fetchmany", "fetchdf", "fetch_df", "df", "fetchnumpy", "fetch_arrow_table", "fetch_record_batch", "arrow", "pl", "fetch_df_chunk"} - def __init__(self, cursor): + #: A fetch that returns the whole remaining result: the read is over after it. + _TERMINAL = {"fetchall", "fetchdf", "fetch_df", "df", "fetchnumpy", "fetch_arrow_table", "arrow", "pl"} + + def __init__(self, cursor, owner=None, factory=None): self._cursor = cursor self._statement = "" self._statement_ms = 0.0 self._request = None self._in_transaction = False + # Drain state (F47): calls in progress, an unfetched result, and who read it last. + self._owner = owner + self._factory = factory + self._epoch = _STORE_EPOCH[0] + self._calls = 0 + self._pending = False + self._last = 0.0 + self._thread = None + self._read_request = None + if owner is not None: + with _READS: + owner.cursors.add(self) + + def _busy(self, now, ignore_thread=None): + """True while a call runs, or a result waits to be fetched by a live request.""" + if self._calls: + return True + if not self._pending or (ignore_thread is not None and self._thread == ignore_thread): + return False + if getattr(self._read_request, "closed", False): + return False # the request that read it has finished + return now - self._last < READ_IDLE_SECONDS + + def _rebind(self): + """Move an idle cursor off a swapped-out instance onto the serving one (same store only).""" + if self._epoch != _STORE_EPOCH[0]: + raise RuntimeError("The data store moved to another file. Retry the request.") + if self._in_transaction: + raise RuntimeError("The data store was swapped during a transaction. Retry the request.") + raw, owner = self._factory() + with _READS: + if self._owner is not None: + self._owner.cursors.discard(self) + self._cursor, self._owner = raw, owner + if owner is not None: + owner.cursors.add(self) + + def _settle(self, name, result, failed): + with _READS: + self._calls -= 1 + self._last = time.monotonic() + if failed or name in self._TERMINAL: + self._pending = False + elif name == "fetchone": + self._pending = result is not None + elif name in {"fetchmany", "fetch_df_chunk"}: + try: + self._pending = len(result) > 0 + except TypeError: + self._pending = False + _READS.notify_all() def _call(self, name, *args, **kwargs): acc = _request_timing.CURRENT.get() execute = name in {"execute", "executemany"} + owner = self._owner + if execute and owner is not None and not owner.live and self._factory is not None: + self._rebind() + with _READS: + self._calls += 1 + self._thread = _thr.get_ident() + self._read_request = acc + result, failed = None, True + try: + result = self._run(name, acc, execute, args, kwargs) + failed = False + return result + finally: + self._settle(name, result, failed) + + def _run(self, name, acc, execute, args, kwargs): writes = False transaction = None if execute: @@ -969,6 +1566,10 @@ class _TimedCursor: result = getattr(self._cursor, name)(*args, **kwargs) if transaction is not None: self._in_transaction = transaction + if execute: + with _READS: + # A row result now waits to be fetched. A write or transaction leaves none. + self._pending = not writes and transaction is None return self if result is self._cursor else result finally: if writes: @@ -988,7 +1589,15 @@ class _TimedCursor: return self._call("executemany", *args, **kwargs) def cursor(self, *args, **kwargs): - return _TimedCursor(self._cursor.cursor(*args, **kwargs)) + return _TimedCursor(self._cursor.cursor(*args, **kwargs), self._owner, self._factory) + + def close(self): + with _READS: + self._pending = False + if self._owner is not None: + self._owner.cursors.discard(self) + _READS.notify_all() + return self._cursor.close() def __getattr__(self, name): if name in self._FETCH: @@ -1000,6 +1609,9 @@ class _TimedCursor: return self def __exit__(self, *args): + with _READS: + self._pending = False + _READS.notify_all() return self._cursor.__exit__(*args) def __iter__(self): @@ -1014,7 +1626,7 @@ class _TimedCursor: def ro_cursor(): """A read cursor on the shared instance — the contention-free way to read the store.""" - return _TimedCursor(_instance().cursor()) + return _TimedCursor(*_open_cursor(), factory=_open_cursor) def ready(): @@ -1110,6 +1722,8 @@ def ro_con(): discarded here, which is the only place every thread is guaranteed to pass through.""" if _odoo_restore_refused(): raise RuntimeError("The saved Odoo mirror could not be restored. Repair store access and restart.") + if versioned_mode(): + _follow_version() cur = getattr(_RO_TLS, "cur", None) if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]: try: @@ -1119,7 +1733,7 @@ def ro_con(): cur, _RO_TLS.cur = None, None # the store moved underneath this thread if cur is not None: try: - cur.execute("SELECT 1") # cheap liveness probe + cur.execute("SELECT 1").fetchall() # cheap liveness probe, fetched so it is not a pending read return cur except Exception as exc: _request_timing.raise_if_resource_error(exc) @@ -1796,18 +2410,11 @@ def _install_seed(tmp): (Linux keeps the unlinked inode), so the whole process read an EMPTY mirror, 0 of 8 Odoo phases, while the seed on disk had all 8 `live`. Staging's Odoo cadence is manual, so nothing ever refilled it: every Odoo grid, measure column and Sheet cell refused "first sync". - Under `_INSTANCE_LOCK` (a plain Lock, so `use_path` is inlined, not called) no reader can - reopen the old file between the close and the swap; the close checkpoints its WAL, and any - leftover WAL is removed so it cannot be replayed onto the seed. + `_swap_instance` drains open reads first (F47), then closes and swaps under `_INSTANCE_LOCK`, + so no reader can reopen the old file between the close and the swap; the close checkpoints its + WAL, and any leftover WAL is removed so it cannot be replayed onto the seed. """ - with _INSTANCE_LOCK: - con = _INSTANCE.get("con") - if con is not None: - try: - con.close() - except Exception: # noqa: BLE001 - pass - _INSTANCE["con"] = None + with _swap_instance(): wal = DB_PATH.with_name(DB_PATH.name + ".wal") if wal.exists(): wal.unlink() @@ -1896,11 +2503,18 @@ def connect(): cursor so callers' con.close() frees the cursor without closing the process-wide instance; one writer cursor is active at a time (the sync thread is sequential), readers use others.""" cur = _candidate_connection() + read_only = cur is None and versioned_mode() + owner = factory = None if cur is None: - cur = _instance().cursor() - if _request_timing.CURRENT.get() is not None and not isinstance(cur, _TimedCursor): - cur = _TimedCursor(cur) - _ensure_sync_state(cur) + cur, owner = _open_cursor() + factory = _open_cursor + # In versioned mode a retired instance closes once drained, so every reader cursor registers. + if ((_request_timing.CURRENT.get() is not None or (owner is not None and read_only)) + and not isinstance(cur, _TimedCursor)): + cur = _TimedCursor(cur, owner, factory) + if not read_only: + # A read-only version takes no DDL. A write there raises, loudly, by design. + _ensure_sync_state(cur) return cur @@ -1994,7 +2608,7 @@ def _flatten_records(recs, fields, m2m=(), *, named_m2m=None): #: is dropped and re-synced. Correcting the list fixes every FRESH mirror, which is the Space's #: case (it has never synced this entity) and was not mine (I had to ALTER my scratch copy). NUMERIC_FIELDS = ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed", - "amount_company_currency_signed", + "amount_company_currency_signed", "amount_residual", "price_subtotal", "margin", "purchase_price", "product_uom_qty", "standard_price", "debit", "credit", "balance", "amount", # W37-T13 — the stock-movement quantity. See the note above. diff --git a/platform/harness/field_audit.py b/platform/harness/field_audit.py index 9d5da20fad04f261fd93fcc8a65d07e5f832b2e3..6a1b00986e6360d5942cb7f969fb713b2d54797e 100644 --- a/platform/harness/field_audit.py +++ b/platform/harness/field_audit.py @@ -540,6 +540,22 @@ def _parquet_fill(cur, keys, scratch): def _store_fill(cur, table): """Per-key non-empty counts over the rows held inside the user_tables document.""" + cur.execute(f"SELECT value ->> '__layout__' FROM {_schema()}.store_kv WHERE key='user_tables'") + layout = cur.fetchone() + if layout and layout[0] == 'rows': + cur.execute( + f"SELECT kv.key, count(*) FILTER (WHERE kv.value IS NOT NULL AND trim(kv.value) <> ''), " + f"min(trim(kv.value)) FILTER (WHERE trim(kv.value) <> ''), " + f"max(trim(kv.value)) FILTER (WHERE trim(kv.value) <> '') " + f"FROM {_schema()}.user_table_records r, jsonb_each_text(r.value) kv " + f"WHERE r.table_key=%s AND r.value IS NOT NULL GROUP BY kv.key", (table,)) + filled, spans = {}, {} + for key, n, lo, hi in cur.fetchall(): + filled[key] = n + _merge_span(spans, key, lo, hi) + cur.execute(f"SELECT count(*) FROM {_schema()}.user_table_records " + "WHERE table_key=%s AND value IS NOT NULL", (table,)) + return filled, dict(filled), spans, cur.fetchone()[0] cur.execute( f"SELECT kv.key, count(*) FILTER (WHERE kv.value IS NOT NULL AND trim(kv.value) <> ''), " f"min(trim(kv.value)) FILTER (WHERE trim(kv.value) <> ''), " @@ -570,10 +586,16 @@ def audit_amazon(log=print): cur.execute(f"SELECT bytes FROM {_schema()}.store_blobs WHERE path = 'amazon_mirror/manifest.json'") row = cur.fetchone() manifest = json.loads(bytes(row[0])) if row else {} - cur.execute( - f"SELECT t.key, jsonb_path_query_array(t.value -> 'fields', '$[*].key') " - f"FROM {_schema()}.store_kv s, jsonb_each(s.value) t " - f"WHERE s.key = 'user_tables' AND t.key LIKE 'ut\\_amzn\\_%%'") + cur.execute(f"SELECT value ->> '__layout__' FROM {_schema()}.store_kv WHERE key='user_tables'") + layout = cur.fetchone() + if layout and layout[0] == 'rows': + cur.execute(f"SELECT table_key, jsonb_path_query_array(value -> 'fields', '$[*].key') " + f"FROM {_schema()}.user_table_defs WHERE table_key LIKE 'ut\\_amzn\\_%%'") + else: + cur.execute( + f"SELECT t.key, jsonb_path_query_array(t.value -> 'fields', '$[*].key') " + f"FROM {_schema()}.store_kv s, jsonb_each(s.value) t " + f"WHERE s.key = 'user_tables' AND t.key LIKE 'ut\\_amzn\\_%%'") defs = {k: list(v or []) for k, v in cur.fetchall()} mtables = manifest.get("tables") or {} with tempfile.TemporaryDirectory(prefix="field_audit_") as scratch: diff --git a/platform/harness/pg/schema.sql b/platform/harness/pg/schema.sql index a3d43ff7c1e47550044778c4e865f1fc7e5c6a59..59bd6e13e172943a48f0a9b1b34b47d96b174f17 100644 --- a/platform/harness/pg/schema.sql +++ b/platform/harness/pg/schema.sql @@ -184,6 +184,18 @@ BEGIN rev bigint NOT NULL DEFAULT 1 )$f$, v_schema); + EXECUTE format($f$ + CREATE TABLE IF NOT EXISTS %I.user_table_defs ( + table_key text PRIMARY KEY, value jsonb NOT NULL, + rev bigint NOT NULL DEFAULT 1, updated_at timestamptz NOT NULL DEFAULT now() + )$f$, v_schema); + EXECUTE format($f$ + CREATE TABLE IF NOT EXISTS %I.user_table_records ( + table_key text NOT NULL, row_id text NOT NULL, value jsonb, meta jsonb, + rev bigint NOT NULL DEFAULT 1, updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (table_key, row_id) + )$f$, v_schema); + -- store_blobs — `upload_bytes`/`download_bytes`/`delete_path`. Customer documents live here -- rather than base64'd into a JSON value, for the reason `core.store.upload_bytes` gives: -- the workspace blob is READ ON EVERY RENDER, so a few MB of base64 inside it would be paid diff --git a/platform/model/metrics/receivables.yml b/platform/model/metrics/receivables.yml index 629e9cec4e3946892843debdd038a32036bdf526..ad21c5a9400b22cf1da57188d78d73fbfd7a8928 100644 --- a/platform/model/metrics/receivables.yml +++ b/platform/model/metrics/receivables.yml @@ -41,14 +41,17 @@ metrics: method: ar_outstanding note: "Store-path outstanding must equal the live Odoo aggregate over the same domain, to the cent." + # R37 / C10: this metric names the Overdue definition. The validated number is the customer + # field `ar_overdue` (ar.aged_receivable, tied by ar.overdue_tie). This document-grain store sum + # has no validate contract because the line mirror lacks amount_residual. - key: ar_overdue - label: AR overdue + label: Overdue agg: sum field: amount_residual_signed store_filter_sql: "CAST(m.invoice_date_due AS DATE) < CURRENT_DATE AND m.payment_state != 'paid'" format: usd - description: "Unpaid balance on documents past their due date as of today." - ai_context: "Overdue is judged against today's date at query time; pairs with the Collections worklist for who-to-call." + description: "Overdue, defined as Odoo Aged Receivable overdue: open receivable past its due date with no grace, New Customer Hold due at once, credits and unapplied payments netted per customer." + ai_context: "For the total that ties to Odoo, sum the customer database field ar_overdue (label Overdue). This document sum leaves out unapplied payments. The 5-day grace number is the field ar_overdue_past_grace (label Overdue past grace)." - key: open_invoices label: Open invoices diff --git a/platform/model/topics/odoo_customers.yml b/platform/model/topics/odoo_customers.yml index a4653ac79165a4672162cdfe86197789b2f6ed26..833c4003679845f7d0ce0bb9d3bd36104693e469 100644 --- a/platform/model/topics/odoo_customers.yml +++ b/platform/model/topics/odoo_customers.yml @@ -7,10 +7,10 @@ # trained on a schema the product does not have. key: odoo_customers semantic: - approved: [customer, agent, dba, salesperson, city, state, country, customer_since, last_order, days_since_invoice, pricelist, payment_terms, source_notes, company_name, commercial_partner, customer_rank, doing_business_as, invoice_age_limit, reminders, is_company, max_invoice_age, online_payment, parent_company, sales_warning, supplier_rank, sales_team, address_type, timezone, hubspot_company_id] + approved: [customer, ar_overdue, ar_overdue_past_grace, agent, dba, salesperson, city, state, country, customer_since, last_order, days_since_invoice, pricelist, payment_terms, source_notes, company_name, commercial_partner, customer_rank, doing_business_as, invoice_age_limit, reminders, is_company, max_invoice_age, online_payment, parent_company, sales_warning, supplier_rank, sales_team, address_type, timezone, hubspot_company_id] date_field: "" date_col_note: "sales_lines.o.date_order" - scope: "One row per customer in the customer book who ordered in the last 24 months; this registry has no date dimension, and dated wholesale revenue uses sales_lines joined by order_partner." + scope: "One row per customer in the customer book: every customer-rank partner, every partner with a confirmed order or a posted customer document, and every debtor on Odoo's Aged Receivable. This registry has no date dimension, and dated wholesale revenue uses sales_lines joined by order_partner." measure: {key: revenue, source: sales_lines, validate: order_level_revenue} sensitive: [street, street2, zip, lat, lon, notes, source_notes, hubspot_company_id] purposes: @@ -30,7 +30,7 @@ grid: customer_data subject: "odoo:res.partner" grain: "One row per customer in the scoped book. Identity is the Odoo res.partner id, also used as the row pid." scope: - population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months" + population: "the customer book `modules/customer_data.pool()` builds: customer-rank partners, confirmed-order partners, posted customer-document partners and every Aged Receivable debtor" identity: "The Odoo res.partner id is exposed as partner_id and used as the row pid." merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved" no_date: "this registry owns no date column of its own, so it stores nothing dated. It does serve WINDOWED sales measures: the `measures:` binding below draws them from sales_lines, which owns the date column, joined on that topic's `order_partner` dim. For anything else time-windowed about a customer, or at line grain, ask sales_lines or customer_invoices" @@ -218,17 +218,22 @@ fields: label: "AR current $" type: currency kind: data - means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + means: "Receivable not yet due on Odoo's Aged Receivable basis: due today or later, no grace." - key: ar_overdue - label: "AR overdue $" + label: "Overdue" type: currency kind: data - means: "Invoiced money past due, on the same basis as the Collections page." + means: "Odoo Aged Receivable overdue: open receivable lines past their due date, no grace, New Customer Hold due at once, credits and unapplied payments netted per customer. Ties to Odoo to the cent." + - key: ar_overdue_past_grace + label: "Overdue past grace" + type: currency + kind: data + means: "Invoice balances more than 5 days past due (the 5-day grace collections uses). Not Odoo's overdue." - key: ar_outstanding label: "AR outstanding $" type: currency kind: data - means: "Total invoiced money owed right now: AR current $ plus AR overdue $." + means: "Total receivable owed right now on Odoo's Aged Receivable basis: AR current $ plus Overdue." - key: ar_exposure label: "Credit exposure $" type: currency @@ -238,22 +243,22 @@ fields: label: "1-30 days $" type: currency kind: data - means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + means: "Overdue between 1 and 30 days. The four aging buckets sum to Overdue." - key: ar_aged_31_60 label: "31-60 days $" type: currency kind: data - means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + means: "Overdue between 31 and 60 days. The four aging buckets sum to Overdue." - key: ar_aged_61_90 label: "61-90 days $" type: currency kind: data - means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + means: "Overdue between 61 and 90 days. The four aging buckets sum to Overdue." - key: ar_aged_90_plus label: "90+ days $" type: currency kind: data - means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + means: "Overdue by more than 90 days. The four aging buckets sum to Overdue." - key: days_to_pay label: "Days to pay" type: int diff --git a/platform/model/topics/odoo_products.yml b/platform/model/topics/odoo_products.yml index 2b289bfcd754454109f9751499387bed8e6e332a..f6a289fdcb29c3a9af73e2daf3428ac944bbda09 100644 --- a/platform/model/topics/odoo_products.yml +++ b/platform/model/topics/odoo_products.yml @@ -509,7 +509,7 @@ fields: label: "Occassion" type: multiselect kind: data - means: "A user-entered column, typed in the grid and not read from Odoo: the occasions the product suits, such as a holiday. A product can carry more than one. Blank means nobody set it." + means: "A user-entered column, typed in the grid and not read from Odoo: the occasions and seasons the product suits. Seasonal lines such as Christmas live here, so filter this field to find a seasonal line. No separate season or category field exists for them. A product can carry more than one. Blank means nobody set it." - key: collection label: "Collection" type: select diff --git a/platform/modules/account_health.py b/platform/modules/account_health.py new file mode 100644 index 0000000000000000000000000000000000000000..180e659548f57d82fe08cadcca3d3630e69673a9 --- /dev/null +++ b/platform/modules/account_health.py @@ -0,0 +1,733 @@ +"""Account health App: the rating, each metric against Amazon's target, a daily history and owned alerts. + +Store documents (tenant runtime store, App owned, never the mirror and never Amazon): + account_health_settings {alert_distance_pct, default_owner} + account_health_alerts {seq, items: {id: alert}} + account_health_history {metrics: {metric id: {label, unit, points: {day: point}}}} + account_health_state {last_sync, last_error} + +Amazon returns only the current snapshot (`amzn_account_health`), so this module keeps the history. +Metric status: Good, Near target (within the alert distance of the target), At risk (past the target), +No target (Amazon sets none). Alert statuses: Open and Investigating are open, Resolved and Dismissed +are closed, and a metric holds one open alert at most. Removing the App deletes the four documents +and leaves every connector table (rule 13). + +Limit (rule 1): each document is one JSON value, and the Postgres store refuses a document over +`core.store_pg.DOCUMENT_CEILING_BYTES` (128 MB). The history grows about 1.5 KB a day for one +marketplace, so it reaches that size after about 200 years. Nothing here caps rows or alerts. +""" +import contextlib +import datetime as dt +from pathlib import Path + +APP_LABEL = "Account health" +SOURCE_KIND = "account_health" +#: Alert (item) statuses. Lane B validates a requested status against this tuple. +STATUSES = ("Open", "Investigating", "Resolved", "Dismissed") +OPEN_STATUSES = ("Open", "Investigating") +#: The metric chip. +METRIC_STATUSES = ("Good", "Near target", "At risk", "No target", "No data", "No orders") +ALERTING = ("Near target", "At risk") +DECISIONS = {"approved": "Investigating", "dismissed": "Dismissed"} +DEFAULTS = {"alert_distance_pct": 20.0} +DUE_DAYS = 7 +EMPTY = "No account health data" +INBOUND_DAYS = 30 + +KEY_SETTINGS = "account_health_settings" +KEY_ALERTS = "account_health_alerts" +KEY_HISTORY = "account_health_history" +KEY_STATE = "account_health_state" +STORE_KEYS = (KEY_SETTINGS, KEY_ALERTS, KEY_HISTORY, KEY_STATE) + +AH_TABLE = "amzn_account_health" +INBOUND_TABLE = "amzn_inbound_noncompliance" +SOURCE = "ut_amzn_account_health" + +#: Rate metrics: (column prefix, label, denominator column). Value `_rate`, target +#: `_target_value`. A denominator of 0 means no orders in the window: Amazon stores rate 0 then. +RATE_METRICS = ( + ("order_defect_rate_afn", "FBA order defect rate", "order_defect_rate_afn_order_count"), + ("order_defect_rate_mfn", "Seller fulfilled order defect rate", "order_defect_rate_mfn_order_count"), + ("late_shipment_rate", "Late shipment rate", "late_shipment_rate_order_count"), + ("pre_fulfillment_cancellation_rate", "Cancellation rate", "pre_fulfillment_cancellation_rate_order_count"), + ("valid_tracking_rate", "Valid tracking rate", "valid_tracking_rate_shipment_count"), + ("on_time_delivery_rate", "On time delivery rate", "on_time_delivery_rate_shipment_count_with_valid_tracking"), + ("unit_on_time_delivery_rate", "Unit on time delivery rate", "unit_on_time_delivery_rate_total_unit_count"), + ("invoice_defect_rate", "Invoice defect rate", "invoice_defect_rate_order_count"), +) +#: Policy count metrics: (column prefix, label). Value `_defects_count`. +COUNT_METRICS = ( + ("listing_policy_violations", "Listing policy violations"), + ("product_authenticity_customer_complaints", "Authenticity complaints"), + ("product_condition_customer_complaints", "Condition complaints"), + ("product_safety_customer_complaints", "Safety complaints"), + ("food_and_product_safety_issues", "Food and product safety issues"), + ("received_intellectual_property_complaints", "Intellectual property complaints"), + ("suspected_intellectual_property_violations", "Suspected intellectual property violations"), + ("restricted_product_policy_violations", "Restricted product violations"), + ("customer_product_reviews_policy_violations", "Review policy violations"), + ("other_policy_violations", "Other policy violations"), + ("document_requests", "Document requests"), +) +WARNINGS = ("policy_violation_warnings", "Policy warnings", "policy_violation_warnings_warnings_count") +INBOUND = ("inbound_noncompliance", "Inbound noncompliance issues, last 30 days") + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at): + return at.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + value = st.get(key) + return value if isinstance(value, dict) else {} + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _num(value): + if value is None or isinstance(value, bool): + return None + try: + return float(str(value).strip().replace(",", "")) + except (TypeError, ValueError): + return None + + +def _snapshot_rows(cur): + """(snapshot date, {marketplace: {column: value}}) for the newest snapshot; rows of one marketplace merge.""" + cols = _columns(cur, AH_TABLE) if cur is not None else set() + if not cols: + return None, {} + date_col = "_snapshot_date" in cols + newest = None + if date_col: + newest = cur.execute(f'SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {AH_TABLE}').fetchone()[0] + where = 'WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date_col and newest is not None else "" + order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else "" + res = cur.execute(f"SELECT * FROM {AH_TABLE} {where}{order}", [newest] if where else []) + names = [d[0] for d in res.description] + out = {} + for row in res.fetchall(): + rec = dict(zip(names, row)) + merged = out.setdefault(str(rec.get("marketplace_id") or ""), {}) + for key, value in rec.items(): + if merged.get(key) is None and value is not None: + merged[key] = value + return (str(newest) if newest is not None else None), out + + +def _inbound_count(cur, snapshot_date): + """Inbound noncompliance rows reported in the 30 days up to the snapshot date, or None without the table.""" + cols = _columns(cur, INBOUND_TABLE) if cur is not None else set() + if not cols: + return None + col = next((c for c in ("issue_reported_date", "_period_start") if c in cols), None) + if col is None: + return int(cur.execute(f"SELECT COUNT(*) FROM {INBOUND_TABLE}").fetchone()[0]) + try: + end = dt.date.fromisoformat(str(snapshot_date)[:10]) + except (TypeError, ValueError): + end = _now().date() + start = end - dt.timedelta(days=INBOUND_DAYS) + count = 0 + for (value,) in cur.execute(f'SELECT CAST("{col}" AS VARCHAR) FROM {INBOUND_TABLE}').fetchall(): + try: + day = dt.date.fromisoformat(str(value)[:10]) + except (TypeError, ValueError): + continue + if start < day <= end: + count += 1 + return count + + +# --------------------------------------------------------------------------- metric status + +def _plain(value): + """An Amazon enum such as AT_RISK as sentence case words: At risk.""" + text = " ".join(str(value or "").replace("_", " ").split()).lower() + return text[:1].upper() + text[1:] if text else None + + +def _condition_text(condition, target, unit): + if target is None: + return None + cond = str(condition or "").upper() + word = ("At least" if "EQUAL" in cond else "Above") if "GREATER" in cond else \ + ("At most" if "EQUAL" in cond or target == 0 else "Below") + return f"{word} {_display(target, unit)}" + + +def _display(value, unit): + if value is None: + return None + if unit == "percent": + return f"{value:.2f}%" + return f"{int(value)}" if float(value).is_integer() else f"{value:g}" + + +def metric_status(value, target, condition, unit, distance_pct=DEFAULTS["alert_distance_pct"]): + """Good, Near target, At risk, No target or No data for one value against its target.""" + if value is None: + return "No data" + if target is None: + return "No target" + cond = str(condition or "").upper() + higher = "GREATER" in cond + inclusive = "EQUAL" in cond + v, t = value, target + if higher: + # A higher is better rate measures its distance on the defect side, or 20% flags every healthy value. + full = 100.0 if unit == "percent" else max(value, target) + v, t = full - value, full - target + past = v > t if (inclusive or t == 0) else v >= t + if past: + return "At risk" + if t > 0 and v >= t * (1 - float(distance_pct) / 100.0): + return "Near target" + return "Good" + + +def _rate_scale(value, target): + # Amazon documents rate and target as fractions (0.04 is 4%). A value above 1 already reads in percent. + known = [abs(x) for x in (value, target) if x is not None] + return 100.0 if not known or max(known) <= 1 else 1.0 + + +def _metric_rows(snapshot_date, by_market, inbound, distance): + rows = [] + for market in sorted(by_market): + rec = by_market[market] + suffix = f"@{market}" if market else "" + for prefix, label, denominator in RATE_METRICS: + if f"{prefix}_rate" not in rec and f"{prefix}_target_value" not in rec: + continue + raw, raw_target = _num(rec.get(f"{prefix}_rate")), _num(rec.get(f"{prefix}_target_value")) + scale = _rate_scale(raw, raw_target) + value = None if raw is None else round(raw * scale, 6) + target = None if raw_target is None else round(raw_target * scale, 6) + orders = _num(rec.get(denominator)) + rows.append(_row(prefix + suffix, prefix, market, label, "percent", + None if orders == 0 else value, target, + rec.get(f"{prefix}_target_condition"), rec.get(f"{prefix}_status"), distance, + rec, no_orders=orders == 0)) + rows[-1].update({"raw": raw, "scale": scale, "orders": orders}) + for prefix, label in COUNT_METRICS: + if f"{prefix}_defects_count" not in rec and f"{prefix}_target_value" not in rec: + continue + rows.append(_row(prefix + suffix, prefix, market, label, "count", + _num(rec.get(f"{prefix}_defects_count")), _num(rec.get(f"{prefix}_target_value")), + rec.get(f"{prefix}_target_condition"), rec.get(f"{prefix}_status"), distance, rec)) + key, label, col = WARNINGS + if col in rec: + rows.append(_row(key + suffix, key, market, label, "count", _num(rec.get(col)), None, None, None, + distance, rec)) + if inbound is not None: + rows.append(_row(INBOUND[0], INBOUND[0], "", INBOUND[1], "count", float(inbound), None, None, None, distance)) + return rows + + +def _day(value): + return None if value is None else str(value)[:10] or None + + +def _row(mid, key, market, label, unit, value, target, condition, amazon_status, distance, rec=None, + no_orders=False): + status = "No orders" if no_orders else metric_status(value, target, condition, unit, distance) + window = f"{key}_reporting_date_range_reporting_date_" + return {"id": mid, "key": key, "marketplaceId": market or None, "label": label, "unit": unit, + "value": value, "display": "No orders" if no_orders else _display(value, unit), "target": target, + "targetDisplay": _display(target, unit), "condition": _condition_text(condition, target, unit), + "targetCondition": condition, "status": status, "amazonStatus": _plain(amazon_status), + "windowFrom": _day((rec or {}).get(window + "from")), "windowTo": _day((rec or {}).get(window + "to"))} + + +def summary(tenant_key, *, st=None): + """The rating, rating status, snapshot date and one row per metric with value, target and status.""" + st = _rt(tenant_key, st) + distance = settings(tenant_key, st=st)["alertDistancePct"] + with _reader(tenant_key, st) as cur: + snapshot_date, by_market = _snapshot_rows(cur) + inbound = _inbound_count(cur, snapshot_date) if by_market else None + if not by_market: + return {"state": "empty", "message": EMPTY, "rating": None, "ratingStatus": None, "snapshotDate": None, + "marketplaceId": None, "marketplaces": [], "metrics": [], "alertDistancePct": distance, + "statuses": list(METRIC_STATUSES)} + markets = [{"marketplaceId": m or None, "rating": _num(r.get("account_health_rating_ahr_score")), + "ratingStatus": _plain(r.get("account_health_rating_ahr_status"))} + for m, r in sorted(by_market.items())] + first = next((m for m in markets if m["rating"] is not None), markets[0]) + return {"state": "ready", "message": None, "rating": first["rating"], "ratingStatus": first["ratingStatus"], + "snapshotDate": snapshot_date, "marketplaceId": first["marketplaceId"], "marketplaces": markets, + "metrics": _metric_rows(snapshot_date, by_market, inbound, distance), "alertDistancePct": distance, + "statuses": list(METRIC_STATUSES)} + + +# --------------------------------------------------------------------------- settings and owner + +#: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit). +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """R31: the tenant's earliest active person Administrator by display name, else `admin`.""" + try: + users = st.get("users") + except Exception: # noqa: BLE001 + users = None + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in (users or {}).items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def settings(tenant_key, *, st=None): + """R49: the alert distance (percent of target, default 20) and the default owner, defaults filled in.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + out = {"alertDistancePct": float(doc.get("alert_distance_pct", DEFAULTS["alert_distance_pct"])), + "alertDistanceDefault": DEFAULTS["alert_distance_pct"]} + out["defaultOwnerSetting"] = doc.get("default_owner") or None + out["defaultOwner"] = out["defaultOwnerSetting"] or owner_account(st, tenant_key) + out["statuses"] = list(STATUSES) + return out + + +def set_settings(tenant_key, *, alert_distance_pct=None, default_owner=None, st=None): + """Set the alert distance (above 0, at most 100 percent) or the default owner ('' resets it).""" + st = _rt(tenant_key, st) + changes = {} + if alert_distance_pct is not None: + try: + number = float(alert_distance_pct) + except (TypeError, ValueError): + raise ValueError("The alert distance must be a number.") from None + if not 0 < number <= 100: + raise ValueError("The alert distance must be above 0 and at most 100 percent.") + changes["alert_distance_pct"] = number + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(changes) + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +# --------------------------------------------------------------------------- daily sync + +def _relevant(synced_tables): + names = [str(t or "") for t in synced_tables or ()] + return not names or any(n.endswith("account_health") or n.endswith("inbound_noncompliance") for n in names) + + +def on_amazon_sync(tenant_key, synced_tables): + """Contract C5 callback: write today's history point per metric and raise alerts. Never raises.""" + try: + sync(tenant_key, synced_tables) + except Exception as exc: # noqa: BLE001 + try: + def _note(doc): + doc = doc if isinstance(doc, dict) else {} + doc["last_error"] = f"The account health update failed ({type(exc).__name__}): {exc}" + return doc + _rt(tenant_key).update(KEY_STATE, _note) + except Exception: # noqa: BLE001 + pass + + +def sync(tenant_key, synced_tables=("account_health",), *, st=None, now=None): + """One history point per metric for the snapshot day (a second sync that day replaces it), then alerts.""" + st = _rt(tenant_key, st) + now = now or _now() + if not _relevant(synced_tables): + return {"state": "skipped"} + view = summary(tenant_key, st=st) + if view["state"] != "ready": + return {"state": "empty"} + day = str(view["snapshotDate"] or "")[:10] + try: + day = dt.date.fromisoformat(day).isoformat() + except ValueError: + day = now.date().isoformat() + # A no orders day keeps its point (value None), so the daily series has no gaps. + points = [(row["id"], row["label"], row["unit"], + {"value": row["value"], "target": row["target"], "status": row["status"]}) + for row in view["metrics"] if row["value"] is not None or row["status"] == "No orders"] + for market in view["marketplaces"]: + if market["rating"] is not None: + mid = "rating" + (f"@{market['marketplaceId']}" if market["marketplaceId"] else "") + points.append((mid, "Account health rating", "score", + {"value": market["rating"], "target": None, "status": market["ratingStatus"]})) + + def _history(doc): + doc = doc if isinstance(doc, dict) else {} + metrics = doc.setdefault("metrics", {}) + for mid, label, unit, point in points: + entry = metrics.setdefault(mid, {"label": label, "unit": unit, "points": {}}) + entry["label"], entry["unit"] = label, unit + entry.setdefault("points", {})[day] = point + return doc + st.update(KEY_HISTORY, _history) + created = raise_alerts(tenant_key, view, st=st, now=now) + + def _state(doc): + doc = doc if isinstance(doc, dict) else {} + doc["last_sync"] = {"at": _stamp(now), "day": day, "points": len(points), "alertsRaised": len(created)} + doc.pop("last_error", None) + return doc + st.update(KEY_STATE, _state) + return {"state": "synced", "day": day, "points": len(points), "created": created} + + +def _reason(row): + text = f"{row['label']} is {row['display']}." + if row.get("condition"): + text += f" Amazon's target is {row['condition'][0].lower() + row['condition'][1:]}." + return text + + +def _title(row): + return f"{row['label']} is {'at risk' if row['status'] == 'At risk' else 'near target'}" + + +def raise_alerts(tenant_key, view=None, *, st=None, now=None): + """Raise one owned alert per metric that is near or past its target, never a second while one is open.""" + st = _rt(tenant_key, st) + now = now or _now() + view = view if view is not None else summary(tenant_key, st=st) + hits = [row for row in view.get("metrics") or () if row["status"] in ALERTING] + if not hits: + return [] + owner = settings(tenant_key, st=st)["defaultOwner"] + created = [] + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.setdefault("seq", 0) + items = doc.setdefault("items", {}) + open_by_metric = {a.get("metricId"): a for a in items.values() if a.get("status") in OPEN_STATUSES} + for row in hits: + held = open_by_metric.get(row["id"]) + if held is not None: + # The open alert follows the metric each sync; a level change is also an event in its history. + if held.get("level") != row["status"]: + held.setdefault("history", []).append({"at": _stamp(now), "by": "system", "level": row["status"]}) + if (held.get("level"), held.get("value"), held.get("target")) != (row["status"], row["value"], row["target"]): + held["updatedAt"], held["updatedBy"] = _stamp(now), "system" + held["level"], held["title"], held["reason"] = row["status"], _title(row), _reason(row) + held["value"], held["target"], held["snapshotDate"] = row["value"], row["target"], view.get("snapshotDate") + continue + doc["seq"] += 1 + aid = f"h{doc['seq']}" + alert = {"id": aid, "metricId": row["id"], "metric": row["key"], "marketplaceId": row["marketplaceId"], + "label": row["label"], "title": _title(row), "level": row["status"], "reason": _reason(row), + "value": row["value"], "target": row["target"], "unit": row["unit"], + "snapshotDate": view.get("snapshotDate"), "owner": owner, "status": "Open", + "due": (now.date() + dt.timedelta(days=DUE_DAYS)).isoformat(), "note": None, + "createdAt": _stamp(now), "updatedAt": _stamp(now), "updatedBy": "system", + "history": [{"at": _stamp(now), "by": "system", "event": "created"}]} + items[aid] = alert + open_by_metric[row["id"]] = alert + created.append(aid) + return doc + st.update(KEY_ALERTS, _apply) + return created + + +def history(tenant_key, metric, *, cursor=None, limit=400, st=None): + """One metric's daily points, newest first, paged by a date cursor. No cap: every day is reachable.""" + st = _rt(tenant_key, st) + metrics = _doc(st, KEY_HISTORY).get("metrics") or {} + mid = str(metric or "") + if mid not in metrics: + mid = next((k for k in sorted(metrics) if k.split("@", 1)[0] == mid), mid) + entry = metrics.get(mid) + if entry is None: + raise KeyError(metric) + days = sorted((entry.get("points") or {}), reverse=True) + if cursor: + days = [d for d in days if d < str(cursor)] + page = days if limit is None else days[:_limit(limit)] + more = len(page) < len(days) + return {"metric": mid, "label": entry.get("label"), "unit": entry.get("unit"), + "points": [{"day": d, **entry["points"][d]} for d in page], + "cursor": page[-1] if more and page else None} + + +# --------------------------------------------------------------------------- items (C4) + +def _limit(limit): + try: + n = int(limit) + except (TypeError, ValueError): + raise ValueError("The page size must be a whole number.") from None + if n < 1: + raise ValueError("The page size must be at least 1.") + return n + + +def _seq(item_id): + try: + return int(str(item_id)[1:]) + except ValueError: + return 0 + + +def _editor(st, user): + """The editing person's name; PermissionError for a viewer, an inactive account or an unknown name.""" + if isinstance(user, dict): + name = str(user.get("username") or user.get("name") or "").strip() + rec = user + else: + name = str(user or "").strip() + try: + users = st.get("users") + except Exception: # noqa: BLE001 + users = None + users = users if isinstance(users, dict) else {} + rec = users.get(name) + if rec is None: + rec = next((r for n, r in users.items() if str(n).lower() == name.lower()), None) + if not isinstance(rec, dict): + raise PermissionError("That account cannot change account health alerts.") + if rec.get("active", True) is False: + raise PermissionError("That account is inactive.") + if str(rec.get("role") or "user").strip().lower() == "viewer": + raise PermissionError("A viewer can read account health alerts but not change them.") + return name or "unknown" + + +def _public(alert): + return {"id": alert["id"], "title": alert.get("title"), "status": alert.get("status"), + "owner": alert.get("owner"), "due": alert.get("due"), "reason": alert.get("reason"), + "metric": alert.get("metric"), "metricId": alert.get("metricId"), "label": alert.get("label"), + "level": alert.get("level"), "marketplaceId": alert.get("marketplaceId"), + "value": alert.get("value"), "target": alert.get("target"), "unit": alert.get("unit"), + "note": alert.get("note"), "createdAt": alert.get("createdAt"), "updatedAt": alert.get("updatedAt")} + + +def items(tenant_key, user=None, *, status=None, cursor=None, limit=200, st=None): + """C4: every alert, newest first, paged by cursor. No cap: the cursor reaches every row.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + st = _rt(tenant_key, st) + rows = [a for a in (_doc(st, KEY_ALERTS).get("items") or {}).values() + if status is None or a.get("status") == status] + rows.sort(key=lambda a: _seq(a.get("id")), reverse=True) + if cursor: + rows = [a for a in rows if _seq(a.get("id")) < _seq(cursor)] + page = rows if limit is None else rows[:_limit(limit)] + more = len(page) < len(rows) + return {"items": [_public(a) for a in page], "cursor": page[-1]["id"] if more and page else None, + "statuses": list(STATUSES)} + + +def update_item(tenant_key, user, item_id, *, owner=None, status=None, due=None, note=None, st=None, + _by=None): + """Reassign, move status, move the due date or note one alert. Returns the alert.""" + st = _rt(tenant_key, st) + who = _by or _editor(st, user) + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + if due is not None: + try: + due = dt.date.fromisoformat(str(due)).isoformat() + except ValueError: + raise ValueError("Due must be a date such as 2026-10-30.") from None + if owner is not None and not str(owner).strip(): + raise ValueError("Owner cannot be empty.") + out = {} + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + alert = (doc.get("items") or {}).get(str(item_id)) + if not alert: + return doc + if status in OPEN_STATUSES and alert.get("status") not in OPEN_STATUSES and any( + a.get("metricId") == alert.get("metricId") and a.get("status") in OPEN_STATUSES + for a in doc["items"].values()): + out["refused"] = True + return doc + at = _stamp(_now()) + event = {"at": at, "by": who} + if owner is not None: + alert["owner"] = event["owner"] = str(owner).strip() + if status is not None: + alert["status"] = event["status"] = status + if due is not None: + alert["due"] = event["due"] = due + if note is not None: + alert["note"] = event["note"] = str(note) + alert["updatedAt"], alert["updatedBy"] = at, who + alert.setdefault("history", []).append(event) + out["alert"] = dict(alert) + return doc + st.update(KEY_ALERTS, _apply) + if out.get("refused"): + raise ValueError("This metric already has an open alert.") + if "alert" not in out: + raise KeyError(item_id) + return _public(out["alert"]) + + +def set_status(tenant_key, user, item_id, status, *, note=None, st=None): + """C4: move one alert's status. PermissionError without an edit grant, ValueError on an unknown status.""" + st = _rt(tenant_key, st) + who = _editor(st, user) + if status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + return update_item(tenant_key, user, item_id, status=status, note=note, st=st, _by=who) + + +def proposals(tenant_key, *, st=None): + """C7: each alert still Open, once, as a proposal for the Operator inbox. It never writes to Amazon.""" + st = _rt(tenant_key, st) + rows = [a for a in (_doc(st, KEY_ALERTS).get("items") or {}).values() if a.get("status") == "Open"] + rows.sort(key=lambda a: _seq(a.get("id"))) + return [{"source_kind": SOURCE_KIND, "item_id": a["id"], "sku": None, + "action": f"Investigate {str(a.get('label') or 'the metric')[:1].lower()}" + f"{str(a.get('label') or 'the metric')[1:]}", + "reason": a.get("reason")} for a in rows] + + +def on_decision(tenant_key, item_id, decision, reason=None, *, user=None, st=None): + """C7: an approved proposal moves to Investigating, a dismissed one to Dismissed.""" + key = str(decision or "").strip().lower() + if key not in DECISIONS: + raise ValueError("A decision is approved or dismissed.") + who = (user.get("username") if isinstance(user, dict) else user) or "operator" + return update_item(tenant_key, user, item_id, status=DECISIONS[key], note=reason, st=st, _by=str(who)) + + +# --------------------------------------------------------------------------- validate and removal + +def validate(tenant_key, *, st=None, **period): + """Tie the rating and each rate in `summary` to the raw `ut_amzn_account_health` snapshot.""" + st = _rt(tenant_key, st) + view = summary(tenant_key, st=st) + with _reader(tenant_key, st) as cur: + independent, date = {}, None + cols = _columns(cur, AH_TABLE) if cur is not None else set() + if cols: + date = cur.execute(f'SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {AH_TABLE}').fetchone()[0] \ + if "_snapshot_date" in cols else None + wanted = [("rating", "account_health_rating_ahr_score", 1.0)] + \ + [(p, f"{p}_rate", None) for p, _l, _d in RATE_METRICS] + for key, col, _s in wanted: + if col not in cols: + continue + where = ' WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date is not None else "" + q = (f'SELECT CAST("marketplace_id" AS VARCHAR), MAX(TRY_CAST("{col}" AS DOUBLE)) ' + f'FROM {AH_TABLE}{where} GROUP BY 1') + for market, value in cur.execute(q, [date] if date is not None else []).fetchall(): + independent[(key, str(market or ""))] = value + app = {} + for m in view["marketplaces"]: + app[("rating", m["marketplaceId"] or "")] = m["rating"] + for row in view["metrics"]: + if row["unit"] == "percent": + # The shown percent ties back through its scale; a no orders row ties its raw rate (Amazon's 0). + raw = row["value"] / row.get("scale", 100.0) if row["value"] is not None else row.get("raw") + app[(row["key"], row["marketplaceId"] or "")] = raw + detail, app_total, ind_total = [], 0.0, 0.0 + for key in sorted(set(app) | set(independent)): + a, b = app.get(key), independent.get(key) + app_total += a or 0.0 + ind_total += b or 0.0 + tied = (a is None and b is None) or (a is not None and b is not None and abs(a - b) <= 1e-7) + detail.append({"metric": key[0], "marketplaceId": key[1] or None, "app": a, "independent": b, "ok": tied}) + ok = bool(detail) and all(d["ok"] for d in detail) and str(view.get("snapshotDate")) == str(date) + return {"ok": ok, "app_total": round(app_total, 9), "independent_total": round(ind_total, 9), + "detail": detail, "snapshotDate": date, "message": None if detail else EMPTY} + + +def remove_app_data(tenant_key, *, st=None): + """Rule 13: delete every document this App owns. No connector table is touched.""" + st = _rt(tenant_key, st) + removed = [] + for key in STORE_KEYS: + deleter = getattr(st, "delete_document", None) + if callable(deleter): + deleter(key) + else: + st.put(key, {}) + removed.append(key) + return {"removed": removed} + + +#: The delete hook name differs by lane (D, E, G call `remove`; F calls `remove_app_data`): both reach this one. +remove = remove_app_data diff --git a/platform/modules/aged_stock.py b/platform/modules/aged_stock.py new file mode 100644 index 0000000000000000000000000000000000000000..b9c01b935ef279ef5207c57d86cd202db109f83f --- /dev/null +++ b/platform/modules/aged_stock.py @@ -0,0 +1,1095 @@ +"""Aged stock App: Amazon storage and long-term fees per SKU, the next surcharge tier, and one action per SKU. + +Store documents (tenant runtime store, App owned, never the mirror and never Amazon): + aged_stock_settings {ads_from_days, price_down_from_days, remove_from_days, fee_horizon_days, + due_days, default_owner} + aged_stock_items {state, message, refreshedAt, notices, primaryCurrency, items: {id: item}, + last_error} + aged_stock_history {months: ["2026-10", ...]} (the index of month documents) + aged_stock_history_ {removals: {key: {snapshot: point}}, stranded: {key: {day: point}}} + +Sources, read only from the tenant mirror: `amzn_storage_fees`, `amzn_long_term_storage_fees`, +`amzn_recommended_removals`, `amzn_stranded_inventory`, `amzn_inventory_planning` (and +`amzn_fba_inventory` for the FNSKU to SKU map only), plus the user table `ut_amazon_sku_profit` for +the unit margin. Removal and stranded reports keep one snapshot, so their history is this App's own. +Deleting this module and its documents (`remove_app_data`) leaves every connector table (rule 13). + +Money is kept per currency and never added across currencies. + +Limit (rule 1): each document is one JSON value, and the Postgres store refuses a document over +`core.store_pg.DOCUMENT_CEILING_BYTES` (128 MB). Nothing here caps SKUs, months, items or history. +""" +import base64 +import contextlib +import datetime as dt +import hashlib +import json +import re +from pathlib import Path + +APP_LABEL = "Aged stock" +SOURCE_KIND = "aged_stock" +STATUSES = ("Open", "Accepted", "In progress", "Done", "Dismissed", "Watching") +REMOVE = "Remove" +PRICE_DOWN = "Price down" +ADS = "Push with ads" +ACTIONS = (REMOVE, PRICE_DOWN, ADS) +#: An item in one of these statuses reopens when its SKU gains a different recommended action. +REOPEN_FROM = ("Watching", "Done", "Dismissed") +DECISIONS = {"approved": "Accepted", "dismissed": "Dismissed"} +DEFAULTS = {"ads_from_days": 91, "price_down_from_days": 181, "remove_from_days": 271, + "fee_horizon_days": 90, "due_days": 7} +NOT_READY = "The aged stock view starts after the next Amazon storage fee sync." +LIMIT = ("Each aged stock document is limited to 128 MB by the store. About 40,000 SKUs with two years " + "of monthly fees fit in the items document, and history splits by month. Split the items " + "document by action when it nears that size.") +UNKNOWN_CURRENCY = "UNKNOWN" + +KEY_SETTINGS = "aged_stock_settings" +KEY_ITEMS = "aged_stock_items" +KEY_HISTORY = "aged_stock_history" +HISTORY_PREFIX = "aged_stock_history_" + +STORAGE = "amzn_storage_fees" +LONG_TERM = "amzn_long_term_storage_fees" +REMOVALS = "amzn_recommended_removals" +STRANDED = "amzn_stranded_inventory" +PLANNING = "amzn_inventory_planning" +FBA_INVENTORY = "amzn_fba_inventory" +MARGIN_TABLE = "ut_amazon_sku_profit" +SOURCES = (STORAGE, LONG_TERM, REMOVALS, STRANDED, PLANNING) +SOURCE_NAMES = {STORAGE: "Amazon storage fees", LONG_TERM: "Amazon long term storage fees", + REMOVALS: "Amazon recommended removals", STRANDED: "Amazon stranded inventory", + PLANNING: "Amazon inventory planning"} +SYNC_TRIGGERS = tuple(t[len("amzn_"):] for t in SOURCES) + +#: (column stem, label, youngest age in days) for the five age bands read from inventory planning. +BANDS = (("0_to_90", "0 to 90 days", 0), ("91_to_180", "91 to 180 days", 91), + ("181_to_270", "181 to 270 days", 181), ("271_to_365", "271 to 365 days", 271), + ("365_plus", "over 365 days", 366)) +#: The next surcharge tiers in age order. `365_plus` is the coarse tier read only when 366 and up are blank. +TIERS = ("181_210", "211_240", "241_270", "271_300", "301_330", "331_365", "365_plus", "366_455", "456_plus") +FINE_FROM_366 = ("366_455", "456_plus") + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at): + return at.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + value = st.get(key) + return value if isinstance(value, dict) else {} + + +def item_id_for(key): + """A stable item id from the SKU (or the FNSKU when no SKU is known), so a refresh never duplicates.""" + return "as_" + hashlib.sha1(str(key).encode("utf-8")).hexdigest()[:10] + + +def _num(value): + try: + return None if value is None or str(value).strip() == "" else float(value) + except (TypeError, ValueError): + return None + + +def _text(value): + text = None if value is None else str(value).strip() + return text or None + + +def _fmt(value): + number = float(value) + return str(int(round(number))) if abs(number - round(number)) < 0.005 else f"{number:.2f}" + + +def _money(value, currency): + if currency == "USD": + return f"${float(value):,.2f}" + return f"{currency} {float(value):,.2f}" + + +def _tier_label(tier): + if tier.endswith("_plus"): + return f"over {int(tier.split('_')[0]) - (0 if tier == '365_plus' else 1)} days" + low, high = tier.split("_") + return f"{low} to {high} days" + + +def _iso_date(value): + """An ISO date from `MM/DD/YYYY` text or an ISO timestamp, else None.""" + text = _text(value) + if not text: + return None + found = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", text) + try: + if found: + return dt.date(int(found.group(3)), int(found.group(1)), int(found.group(2))).isoformat() + return dt.date.fromisoformat(text[:10]).isoformat() + except ValueError: + return None + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + if cur is None: + return set() + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _newest(cols, table): + """A WHERE clause that keeps the newest `_snapshot_date` only, or every row without one.""" + if "_snapshot_date" not in cols: + return "TRUE" + return (f'CAST("_snapshot_date" AS VARCHAR) = ' + f'(SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {table})') + + +def _col(cols, name): + """A column as trimmed text, NULL when blank or absent.""" + return f'NULLIF(TRIM(CAST("{name}" AS VARCHAR)), \'\')' if name in cols else "NULL" + + +def _dbl(cols, name): + return f'TRY_CAST("{name}" AS DOUBLE)' if name in cols else "NULL" + + +def _first_of(cols, names, width): + """The first non-blank of several text columns, cut to `width` characters.""" + parts = [f'NULLIF(LEFT(TRIM(CAST("{n}" AS VARCHAR)), {width}), \'\')' for n in names if n in cols] + if not parts: + return "NULL" + return parts[0] if len(parts) == 1 else f"COALESCE({', '.join(parts)})" + + +def _currency_sql(cols): + return f"COALESCE({_col(cols, 'currency')}, '{UNKNOWN_CURRENCY}')" + + +def storage_month_sql(cols): + """The month of a storage fee row: `month`, else `month_of_charge`, as `YYYY-MM`.""" + return _first_of(cols, ("month", "month_of_charge"), 7) + + +def long_term_month_sql(cols): + """The month of a long-term charge: the first 7 characters of `snapshot_date`, else `_period_start`.""" + return _first_of(cols, ("snapshot_date", "_period_start"), 7) + + +def _fnsku_map(cur): + """{fnsku: sku}, first match in order: planning, long-term fees, removals, stranded, FBA inventory.""" + out = {} + for table, sku_col, fn_col in ((PLANNING, "sku", "fnsku"), (LONG_TERM, "sku", "fnsku"), + (REMOVALS, "sku", "fnsku"), (STRANDED, "sku", "fnsku"), + (FBA_INVENTORY, "seller_sku", "fn_sku")): + cols = _columns(cur, table) + if not {sku_col, fn_col} <= cols: + continue + rows = cur.execute(f"SELECT {_col(cols, fn_col)}, MIN({_col(cols, sku_col)}) FROM {table} " + f"WHERE {_col(cols, fn_col)} IS NOT NULL AND {_col(cols, sku_col)} IS NOT NULL " + f"GROUP BY 1").fetchall() + for fnsku, sku in rows: + out.setdefault(fnsku, sku) + return out + + +def _storage(cur): + """[(month, fnsku, asin, title, currency, fee, rows)], or None when the table is absent.""" + cols = _columns(cur, STORAGE) + if "estimated_monthly_storage_fee" not in cols: + return None + sql = (f"SELECT {storage_month_sql(cols)}, {_col(cols, 'fnsku')}, {_col(cols, 'asin')}, " + f"MAX({_col(cols, 'product_name')}), {_currency_sql(cols)}, " + f"SUM({_dbl(cols, 'estimated_monthly_storage_fee')}), COUNT(*) FROM {STORAGE} GROUP BY 1, 2, 3, 5") + return cur.execute(sql).fetchall() + + +def _long_term(cur): + """[(month, sku, fnsku, asin, title, currency, tier, amount, qty, rows)], or None when absent.""" + cols = _columns(cur, LONG_TERM) + if "amount_charged" not in cols: + return None + sql = (f"SELECT {long_term_month_sql(cols)}, {_col(cols, 'sku')}, {_col(cols, 'fnsku')}, " + f"{_col(cols, 'asin')}, MAX({_col(cols, 'product_name')}), {_currency_sql(cols)}, " + f"{_col(cols, 'surcharge_age_tier')}, SUM({_dbl(cols, 'amount_charged')}), " + f"SUM({_dbl(cols, 'qty_charged')}), COUNT(*) FROM {LONG_TERM} GROUP BY 1, 2, 3, 4, 6, 7") + return cur.execute(sql).fetchall() + + +def _removals(cur): + """[(snapshot, sku, fnsku, asin, title, sellable, removal, unsellable)] over every snapshot, or None.""" + cols = _columns(cur, REMOVALS) + if not cols & {"sku", "fnsku"}: + return None + sql = (f"SELECT {_first_of(cols, ('snapshot_date', '_snapshot_date'), 10)}, {_col(cols, 'sku')}, " + f"{_col(cols, 'fnsku')}, {_col(cols, 'asin')}, MAX({_col(cols, 'product_name')}), " + f"SUM({_dbl(cols, 'sellable_quantity')}), SUM({_dbl(cols, 'sellable_removal_quantity')}), " + f"SUM({_dbl(cols, 'unsellable_quantity')}) FROM {REMOVALS} GROUP BY 1, 2, 3, 4") + return cur.execute(sql).fetchall() + + +STRANDED_COLS = ("sku", "fnsku", "asin", "product_name", "stranded_reason", "primary_action", "fulfillable_qty", + "unfulfillable_qty", "reserved_quantity", "date_stranded", "date_to_take_auto_removal", + "your_price") + + +def _stranded(cur): + """[row dict] from the newest stranded snapshot, or None when the table is absent.""" + cols = _columns(cur, STRANDED) + if not cols & {"sku", "fnsku"}: + return None + sql = (f"SELECT {', '.join(_col(cols, c) for c in STRANDED_COLS)} FROM {STRANDED} " + f"WHERE {_newest(cols, STRANDED)}") + return [dict(zip(STRANDED_COLS, r)) for r in cur.execute(sql).fetchall()] + + +PLANNING_COLS = (("sku", "fnsku", "asin", "product_name", "currency", "recommended_action") + + tuple(f"inv_age_{b}_days" for b, _l, _d in BANDS) + + ("inv_age_366_to_455_days", "inv_age_456_plus_days") + + tuple(f"{p}_{t}_days" for t in TIERS for p in ("quantity_to_be_charged_ais", "estimated_ais")) + + ("estimated_storage_cost_next_month", "inventory_supply_at_fba", "available", + "recommended_removal_quantity", "recommended_sales_price", "your_price")) + + +def _planning(cur): + """{sku: planning values} from the newest snapshot, summed per SKU; None when the table is absent.""" + cols = _columns(cur, PLANNING) + if "sku" not in cols: + return None + sql = (f"SELECT {', '.join(_col(cols, c) for c in PLANNING_COLS)} FROM {PLANNING} " + f"WHERE {_newest(cols, PLANNING)} AND {_col(cols, 'sku')} IS NOT NULL") + out = {} + for raw in cur.execute(sql).fetchall(): + r = dict(zip(PLANNING_COLS, raw)) + p = out.setdefault(r["sku"], {"bands": {b: None for b, _l, _d in BANDS}, "tiers": {}, + "nums": {}, "text": {}}) + for key in ("fnsku", "asin", "product_name", "currency", "recommended_action"): + p["text"].setdefault(key, None) + p["text"][key] = p["text"][key] or r[key] + for stem, _label, _days in BANDS: + value = _num(r[f"inv_age_{stem}_days"]) + if stem == "365_plus" and value is None: + parts = [_num(r["inv_age_366_to_455_days"]), _num(r["inv_age_456_plus_days"])] + value = None if all(v is None for v in parts) else sum(v or 0 for v in parts) + if value is not None: + p["bands"][stem] = (p["bands"][stem] or 0) + value + fine_blank = all(_num(r[f"{pre}_{t}_days"]) is None for t in FINE_FROM_366 + for pre in ("quantity_to_be_charged_ais", "estimated_ais")) + for tier in TIERS: + if tier == "365_plus" and not fine_blank: + continue + qty, est = _num(r[f"quantity_to_be_charged_ais_{tier}_days"]), _num(r[f"estimated_ais_{tier}_days"]) + if qty is None and est is None: + continue + got = p["tiers"].setdefault(tier, [None, None]) + got[0] = None if qty is None and got[0] is None else (got[0] or 0) + (qty or 0) + got[1] = None if est is None and got[1] is None else (got[1] or 0) + (est or 0) + for key in ("estimated_storage_cost_next_month", "inventory_supply_at_fba", "available", + "recommended_removal_quantity"): + value = _num(r[key]) + if value is not None: + p["nums"][key] = (p["nums"].get(key) or 0) + value + for key in ("recommended_sales_price", "your_price"): + if p["nums"].get(key) is None: + p["nums"][key] = _num(r[key]) + return out + + +def _margins(st): + """{sku: unit margin} from the user table, data only (no App code import). Unknown margins are absent.""" + from core import user_tables + try: + table = user_tables.get_one(MARGIN_TABLE, st) or {} + except Exception: # noqa: BLE001 + table = {} + rows = table.get("rows") or {} + sums = {} + for row in (rows.values() if isinstance(rows, dict) else rows): + if not isinstance(row, dict): + continue + sku = str(row.get("sku") or "").strip() + profit, units = _num(row.get("net_profit")), _num(row.get("units_sold")) + if sku and profit is not None and units is not None: + got = sums.setdefault(sku, [0.0, 0.0]) + got[0] += profit + got[1] += units + return {sku: profit / units for sku, (profit, units) in sums.items() if units > 0} + + +# --------------------------------------------------------------------------- settings and owner + +#: Accounts that sign in for a process, never a person who can own an item. +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """P22: the tenant's earliest active person Administrator, else `admin`, else its first Administrator.""" + try: + users = st.get("users") + except Exception: # noqa: BLE001 + users = None + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in (users or {}).items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def settings(tenant_key, *, st=None): + """The age thresholds, the fee horizon, the due window and the default owner, with defaults filled in.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + owner_setting = doc.get("default_owner") or None + return {"adsFromDays": int(doc.get("ads_from_days", DEFAULTS["ads_from_days"])), + "priceDownFromDays": int(doc.get("price_down_from_days", DEFAULTS["price_down_from_days"])), + "removeFromDays": int(doc.get("remove_from_days", DEFAULTS["remove_from_days"])), + "feeHorizonDays": int(doc.get("fee_horizon_days", DEFAULTS["fee_horizon_days"])), + "dueDays": int(doc.get("due_days", DEFAULTS["due_days"])), + "defaultOwnerSetting": owner_setting, + "defaultOwner": owner_setting or owner_account(st, tenant_key), + "statuses": list(STATUSES), "actions": list(ACTIONS), "limit": LIMIT} + + +def _whole(value, low, high, message): + number = _num(value) + if number is None or number != int(number) or not low <= number <= high: + raise ValueError(message) + return int(number) + + +def set_settings(tenant_key, *, ads_from_days=None, price_down_from_days=None, remove_from_days=None, + fee_horizon_days=None, due_days=None, default_owner=None, st=None): + """Set the age thresholds in days, the fee horizon, the due window, or the default owner ('' resets it).""" + st = _rt(tenant_key, st) + now = settings(tenant_key, st=st) + changes = {} + for name, value in (("ads_from_days", ads_from_days), ("price_down_from_days", price_down_from_days), + ("remove_from_days", remove_from_days)): + if value is not None: + changes[name] = _whole(value, 1, 3650, "An age threshold must be a whole number of days from 1 to 3650.") + ads = changes.get("ads_from_days", now["adsFromDays"]) + price = changes.get("price_down_from_days", now["priceDownFromDays"]) + remove = changes.get("remove_from_days", now["removeFromDays"]) + if not ads < price < remove: + raise ValueError("The thresholds must rise: push with ads, then price down, then remove.") + if fee_horizon_days is not None: + changes["fee_horizon_days"] = _whole(fee_horizon_days, 1, 365, + "The fee horizon must be a whole number of days from 1 to 365.") + if due_days is not None: + changes["due_days"] = _whole(due_days, 1, 365, "Days to act must be a whole number from 1 to 365.") + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(changes) + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +# --------------------------------------------------------------------------- the row builder + +def _tier_rank(tier): + found = re.match(r"\d+", str(tier or "")) + return (int(found.group(0)) if found else -1, str(tier or "")) + + +def _recommend(row, conf): + """(recommended action or None, reason, fee per unit or None) for one SKU row under the settings.""" + units = row["units"] + if units is None: + return None, "Amazon inventory planning has no units for this SKU, so no action.", None + if units <= 0: + return None, "No units at Amazon, so no action.", None + oldest = None + for stem, label, days in BANDS: + if (row["age_bands"].get(stem) or 0) > 0: + oldest = (label, days) + head = f"{_fmt(units)} units at Amazon" + age_action = None + if oldest is None: + head += ", with no inventory age reported." + else: + head += f", the oldest {oldest[0]} old." + if oldest[1] >= conf["removeFromDays"]: + age_action = REMOVE + elif oldest[1] >= conf["priceDownFromDays"]: + age_action = PRICE_DOWN + elif oldest[1] >= conf["adsFromDays"]: + age_action = ADS + margin, cur = row["unit_margin"], row["currency"] + per_unit = None + if margin is not None: + stake = row["at_stake"].get(cur) or 0 + per_unit = conf["feeHorizonDays"] / 30.0 * ((row["storage_next_month"] or 0) + stake) / units + if per_unit is not None and per_unit > 0 and per_unit > margin and age_action != REMOVE: + return REMOVE, (f"{head} {conf['feeHorizonDays']} days of storage fees cost {_money(per_unit, cur)} " + f"per unit, over the unit margin of {_money(margin, cur)}."), per_unit + rule = {REMOVE: f"Remove at {conf['removeFromDays']} days or older.", + PRICE_DOWN: f"Lower the price at {conf['priceDownFromDays']} days or older.", + ADS: f"Push with ads at {conf['adsFromDays']} days or older."} + if age_action is None: + return None, f"{head} No action under {conf['adsFromDays']} days.", per_unit + return age_action, f"{head} {rule[age_action]}", per_unit + + +def build_rows(cur, margins, conf): + """(rows, notices, primary currency): one row per SKU in any source, or (None, notices, None) when none synced.""" + planning, storage, long_term = _planning(cur), _storage(cur), _long_term(cur) + removals, stranded = _removals(cur), _stranded(cur) + got = {PLANNING: planning, STORAGE: storage, LONG_TERM: long_term, REMOVALS: removals, STRANDED: stranded} + notices = [f"{SOURCE_NAMES[t]} has not synced yet." for t in SOURCES if got[t] is None] + if all(v is None for v in got.values()): + return None, notices, None + fmap = _fnsku_map(cur) + rows = {} + counts = {} + + def row_for(sku, fnsku, asin=None, title=None): + sku = sku or fmap.get(fnsku) + key = sku or fnsku or asin or "Unidentified" + row = rows.get(key) + if row is None: + row = rows[key] = {"key": key, "sku": sku, "fnsku": fnsku, "asin": asin, "title": title, + "_fees": {}, "_lt": {}, "_cur": {}} + row["fnsku"] = row["fnsku"] or fnsku + row["asin"] = row["asin"] or asin + row["title"] = row["title"] or title + return row + + for sku, p in (planning or {}).items(): + t = p["text"] + row = row_for(sku, t["fnsku"], t["asin"], t["product_name"]) + row["_plan"] = p + for month, fnsku, asin, title, cur, fee, n in storage or (): + row = row_for(None, fnsku, asin, title) + point = row["_fees"].setdefault((month or "", cur), {"storage": 0.0, "long_term": 0.0, "tier": None, + "units": None}) + point["storage"] += fee or 0.0 + row["_cur"][cur] = row["_cur"].get(cur, 0) + n + counts[cur] = counts.get(cur, 0) + n + for month, sku, fnsku, asin, title, cur, tier, amount, qty, n in long_term or (): + row = row_for(sku, fnsku, asin, title) + point = row["_fees"].setdefault((month or "", cur), {"storage": 0.0, "long_term": 0.0, "tier": None, + "units": None}) + point["long_term"] += amount or 0.0 + if qty is not None: + point["units"] = (point["units"] or 0) + qty + if tier and _tier_rank(tier) > _tier_rank(point["tier"]): + point["tier"] = tier + row["_cur"][cur] = row["_cur"].get(cur, 0) + n + counts[cur] = counts.get(cur, 0) + n + snapshots = {} + for snap, sku, fnsku, asin, title, sellable, removal, unsellable in removals or (): + row = row_for(sku, fnsku, asin, title) + point = snapshots.setdefault(row["key"], {}).setdefault( + snap or "", {"sellable_quantity": 0.0, "sellable_removal_quantity": 0.0, "unsellable_quantity": 0.0}) + point["sellable_quantity"] += sellable or 0 + point["sellable_removal_quantity"] += removal or 0 + point["unsellable_quantity"] += unsellable or 0 + for r in stranded or (): + row = row_for(r["sku"], r["fnsku"], r["asin"], r["product_name"]) + s = row.setdefault("stranded", {"reasons": [], "primary_actions": [], "fulfillable": 0.0, + "unfulfillable": 0.0, "reserved": 0.0, "units": 0.0, + "date_stranded": None, "auto_removal_date": None}) + for key, src in (("reasons", "stranded_reason"), ("primary_actions", "primary_action")): + if r[src] and r[src] not in s[key]: + s[key].append(r[src]) + for key, src in (("fulfillable", "fulfillable_qty"), ("unfulfillable", "unfulfillable_qty"), + ("reserved", "reserved_quantity")): + s[key] += _num(r[src]) or 0 + s["units"] = s["fulfillable"] + s["unfulfillable"] + for key, src in (("date_stranded", "date_stranded"), ("auto_removal_date", "date_to_take_auto_removal")): + day = _iso_date(r[src]) + if day and (s[key] is None or day < s[key]): + s[key] = day + primary = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] if counts else None + + out = [] + for key in sorted(rows): + row = rows.pop(key) + p = row.pop("_plan", None) + fees, seen = row.pop("_fees"), row.pop("_cur") + plan_cur = (p or {}).get("text", {}).get("currency") + cur = plan_cur or (sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] if seen else None) \ + or primary or UNKNOWN_CURRENCY + history = [{"month": m, "currency": c, **v} for (m, c), v in sorted(fees.items())] + totals = {} + for point in history: + t = totals.setdefault(point["currency"], {"storage": 0.0, "long_term": 0.0, "total": 0.0}) + t["storage"] += point["storage"] + t["long_term"] += point["long_term"] + t["total"] += point["storage"] + point["long_term"] + charged = [pt for pt in history if pt["long_term"] or pt["tier"]] + nums = (p or {}).get("nums", {}) + bands = dict((p or {}).get("bands") or {b: None for b, _l, _d in BANDS}) + units = nums.get("inventory_supply_at_fba") + if units is None: + units = nums.get("available") + if units is None and p is not None and any(v is not None for v in bands.values()): + units = sum(v or 0 for v in bands.values()) + next_tier = None + tiers = [(t, q, e) for t, (q, e) in ((p or {}).get("tiers") or {}).items() if q and q > 0] + if tiers: + tiers.sort(key=lambda x: TIERS.index(x[0])) + estimates = [e for _t, _q, e in tiers if e is not None] + next_tier = {"tier": _tier_label(tiers[-1][0]), "units": sum(q for _t, q, _e in tiers), + "at_stake": sum(estimates) if estimates else None, "currency": cur, + "last_charged_tier": charged[-1]["tier"] if charged else None} + snaps = snapshots.get(key) or {} + newest = max(snaps) if snaps else None + row.update({ + "title": row["title"] or row["sku"] or row["fnsku"] or key, "currency": cur, "units": units, + "age_bands": bands, "fee_history": history, "fees": totals, "next_tier": next_tier, + "at_stake": ({cur: next_tier["at_stake"]} if next_tier and next_tier["at_stake"] is not None else {}), + "storage_next_month": nums.get("estimated_storage_cost_next_month"), + "unit_margin": margins.get(row["sku"]) if row["sku"] else None, + "amazon_action": (p or {}).get("text", {}).get("recommended_action"), + "recommended_removal_quantity": nums.get("recommended_removal_quantity"), + "recommended_sales_price": nums.get("recommended_sales_price"), + "your_price": nums.get("your_price"), + "removal": {"snapshot": newest or None, **snaps[newest]} if newest is not None else None, + "stranded": row.get("stranded"), + "_removal_points": snaps}) + row["recommended_action"], row["reason"], row["fee_per_unit"] = _recommend(row, conf) + if row["stranded"]: + s = row["stranded"] + row["reason"] += f" {_fmt(s['units'])} units are stranded" + row["reason"] += (f": {', '.join(s['reasons'])}." if s["reasons"] else ".") + out.append(row) + return out, notices, primary + + +# --------------------------------------------------------------------------- refresh and history + +def _month_key(month): + return HISTORY_PREFIX + month + + +def refresh(tenant_key, *, st=None, now=None): + """Compute every SKU row from the mirror, write removal and stranded history, and merge items.""" + st = _rt(tenant_key, st) + now = now or _now() + conf = settings(tenant_key, st=st) + margins = _margins(st) + with _reader(tenant_key, st) as cur: + if cur is None: + rows, notices, primary = None, ["The tenant mirror does not exist yet."], None + else: + rows, notices, primary = build_rows(cur, margins, conf) + if rows is None: + message = notices[0] if notices else NOT_READY + + def _waiting(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(state="not_ready", message=message, notices=notices, checkedAt=_stamp(now)) + doc.pop("last_error", None) + return doc + st.update(KEY_ITEMS, _waiting) + return {"state": "not_ready", "message": message, "notices": notices} + + day = now.date().isoformat() + by_month = {} + for row in rows: + for snap, point in row.pop("_removal_points").items(): + if snap: + by_month.setdefault(snap[:7], {"removals": {}, "stranded": {}})["removals"].setdefault( + row["key"], {})[snap] = point + by_month.setdefault(day[:7], {"removals": {}, "stranded": {}})["stranded"][row["key"]] = { + day: {"units": (row["stranded"] or {}).get("units", 0.0), "at_stake": row["at_stake"]}} + for month, adds in by_month.items(): + def _history(doc, adds=adds): + doc = doc if isinstance(doc, dict) else {} + for kind in ("removals", "stranded"): + bucket = doc.setdefault(kind, {}) + for key, points in adds[kind].items(): + bucket.setdefault(key, {}).update(points) + return doc + st.update(_month_key(month), _history) + + def _index(doc): + doc = doc if isinstance(doc, dict) else {} + doc["months"] = sorted(set(doc.get("months") or ()) | set(by_month)) + return doc + st.update(KEY_HISTORY, _index) + + owner = conf["defaultOwner"] + due = (now.date() + dt.timedelta(days=conf["dueDays"])).isoformat() + counts = {"created": 0, "reopened": 0} + + def _items(doc): + doc = doc if isinstance(doc, dict) else {} + items = doc.setdefault("items", {}) + seen = set() + for row in rows: + iid = item_id_for(row["key"]) + seen.add(iid) + old = items.get(iid) + rec = row["recommended_action"] + if old is None: + item = {"id": iid, "owner": owner, "status": "Open" if rec else "Watching", + "due": due if rec else None, "action": rec, "createdAt": _stamp(now), + "history": [{"at": _stamp(now), "by": "system", "event": "created", + "recommended_action": rec}]} + counts["created"] += 1 + else: + item = old + if rec and rec != old.get("recommended_action") and old.get("status") in REOPEN_FROM: + item["status"], item["due"] = "Open", due + item.setdefault("history", []).append( + {"at": _stamp(now), "by": "system", "event": "reopened", "recommended_action": rec, + "status": "Open", "due": due}) + counts["reopened"] += 1 + # The action follows the recommendation until a person chooses one. + if not item.get("actionChosenBy"): + item["action"] = rec + item.update(row) + item["inReport"] = True + item["refreshedAt"] = _stamp(now) + items[iid] = item + for iid, item in items.items(): + if iid not in seen: + item["inReport"] = False + doc.update(state="ready", message=None, notices=notices, primaryCurrency=primary, + refreshedAt=_stamp(now)) + doc.pop("last_error", None) + return doc + st.update(KEY_ITEMS, _items) + return {"state": "ready", "skus": len(rows), "notices": notices, **counts} + + +def _short(name): + name = str(name or "").strip().lower() + for prefix in ("ut_", "amzn_"): + if name.startswith(prefix): + name = name[len(prefix):] + return name + + +def on_amazon_sync(tenant_key, synced_tables): + """Contract C5 callback: refresh when a fee, removal, stranded or planning report synced. Never raises.""" + try: + names = {_short(t) for t in (synced_tables or ())} + except Exception: # noqa: BLE001 + return + if not names & set(SYNC_TRIGGERS): + return + try: + refresh(tenant_key) + except Exception as exc: # noqa: BLE001 + try: + def _note(doc): + doc = doc if isinstance(doc, dict) else {} + doc["last_error"] = f"The aged stock refresh failed ({type(exc).__name__}): {exc}" + return doc + _rt(tenant_key).update(KEY_ITEMS, _note) + except Exception: # noqa: BLE001 + pass + + +def history(tenant_key, key, *, st=None): + """{removals, stranded}: every stored point for one SKU key, oldest first.""" + st = _rt(tenant_key, st) + out = {"removals": [], "stranded": []} + for month in _doc(st, KEY_HISTORY).get("months") or (): + doc = _doc(st, _month_key(month)) + for kind, field in (("removals", "snapshot"), ("stranded", "date")): + for at, point in ((doc.get(kind) or {}).get(key) or {}).items(): + out[kind].append({field: at, **point}) + out["removals"].sort(key=lambda p: p["snapshot"]) + out["stranded"].sort(key=lambda p: p["date"]) + return out + + +# --------------------------------------------------------------------------- summary + +def summary(tenant_key, *, st=None): + """Month trend and totals per currency, the next tier list by money at stake, and counts.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_ITEMS) + ready = doc.get("state") == "ready" + live = [i for i in (doc.get("items") or {}).values() if i.get("inReport", True)] + trend, totals = {}, {} + for item in live: + for point in item.get("fee_history") or (): + month = trend.setdefault(point["currency"], {}).setdefault( + point["month"], {"storage": 0.0, "long_term": 0.0, "total": 0.0}) + for key in ("storage", "long_term"): + month[key] += point[key] + month["total"] += point["storage"] + point["long_term"] + for cur, amount in (item.get("at_stake") or {}).items(): + totals.setdefault(cur, {"storage": 0.0, "long_term": 0.0, "total": 0.0, "at_stake": 0.0}) + totals[cur]["at_stake"] += amount + for cur, months in trend.items(): + t = totals.setdefault(cur, {"storage": 0.0, "long_term": 0.0, "total": 0.0, "at_stake": 0.0}) + for m in months.values(): + for key in ("storage", "long_term", "total"): + t[key] += m[key] + next_tier = sorted(({"id": i["id"], "sku": i.get("sku"), "fnsku": i.get("fnsku"), "title": i.get("title"), + **i["next_tier"]} for i in live if i.get("next_tier")), + key=lambda r: (r["at_stake"] is None, -(r["at_stake"] or 0), str(r.get("sku") or ""))) + by_status = {s: 0 for s in STATUSES} + by_action = {a: 0 for a in ACTIONS} + for item in live: + if item.get("status") in by_status: + by_status[item["status"]] += 1 + if item.get("action") in by_action: + by_action[item["action"]] += 1 + return {"ready": ready, "notReady": None if ready else (doc.get("message") or NOT_READY), + "empty": not live, "notices": list(doc.get("notices") or ()), + "primaryCurrency": doc.get("primaryCurrency"), + "trend": {cur: [{"month": m, **v} for m, v in sorted(months.items())] for cur, months in trend.items()}, + "totals": totals, "nextTier": next_tier, "statusCounts": by_status, "actionCounts": by_action, + "refreshedAt": doc.get("refreshedAt"), "lastError": doc.get("last_error"), "limit": LIMIT} + + +# --------------------------------------------------------------------------- C4 item protocol + +def _order(item): + action = item.get("action") + rank = ACTIONS.index(action) if action in ACTIONS else len(ACTIONS) + stake = max((item.get("at_stake") or {}).values(), default=0) or 0 + return (not item.get("inReport", True), rank, -stake, str(item.get("key") or "")) + + +def _cursor_offset(cursor): + if not cursor: + return 0 + try: + return int(json.loads(base64.urlsafe_b64decode(str(cursor).encode()).decode())["o"]) + except Exception: # noqa: BLE001 + raise ValueError("The page cursor is not valid. Start from the first page.") from None + + +def _cursor(offset): + return base64.urlsafe_b64encode(json.dumps({"o": offset}).encode()).decode() + + +def _public(item): + out = {k: v for k, v in item.items() if k != "history"} + out.setdefault("title", item.get("sku") or item.get("key")) + return out + + +def _matches(item, needle): + """True when the lowercased needle is a substring of the item's sku, fnsku, title or asin.""" + return any(needle in str(item.get(k) or "").lower() for k in ("sku", "fnsku", "title", "asin")) + + +def items(tenant_key, user=None, *, status=None, action=None, cursor=None, limit=200, q=None, st=None): + """C4: every item matching q, filtered by status and action, paged by an opaque cursor with no total cap.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + if action is not None and action not in ACTIONS: + raise ValueError("Action must be one of: " + ", ".join(ACTIONS) + ".") + try: + limit = int(limit) + except (TypeError, ValueError): + raise ValueError("The page size must be a whole number.") from None + if limit < 1: + raise ValueError("The page size must be at least 1.") + st = _rt(tenant_key, st) + doc = _doc(st, KEY_ITEMS) + every = list((doc.get("items") or {}).values()) + needle = str(q or "").strip().lower() + if needle: + every = [i for i in every if _matches(i, needle)] + rows = sorted((i for i in every if (status is None or i.get("status") == status) + and (action is None or i.get("action") == action)), key=_order) + start = _cursor_offset(cursor) + page = rows[start:start + limit] + ready = doc.get("state") == "ready" + return {"items": [_public(i) for i in page], + "cursor": _cursor(start + limit) if start + limit < len(rows) else None, + "ready": ready, "message": None if ready else (doc.get("message") or NOT_READY), + "notices": list(doc.get("notices") or ()), "refreshedAt": doc.get("refreshedAt"), + "lastError": doc.get("last_error"), "statuses": list(STATUSES), "actions": list(ACTIONS)} + + +def item_detail(tenant_key, item_id, *, st=None): + """One SKU with its fee history, age bands, removal and stranded units, events and stored history.""" + st = _rt(tenant_key, st) + item = (_doc(st, KEY_ITEMS).get("items") or {}).get(item_id) + if not item: + raise KeyError(item_id) + past = history(tenant_key, item.get("key"), st=st) + return {**_public(item), "events": list(item.get("history") or ()), + "removal_history": past["removals"], "stranded_history": past["stranded"]} + + +def _uname(user): + if isinstance(user, dict): + return str(user.get("username") or user.get("name") or "") + return str(user or "") + + +def _may_change(st, user): + """A known viewer or inactive account may not change an item (the App grant is the caller's check).""" + try: + users = st.get("users") or {} + except Exception: # noqa: BLE001 + users = {} + rec = users.get(_uname(user)) if isinstance(users, dict) else None + if isinstance(rec, dict) and (str(rec.get("role") or "").lower() == "viewer" or rec.get("active") is False): + return False + return True + + +def update_item(tenant_key, item_id, *, owner=None, status=None, due=None, action=None, note=None, + user="system", st=None): + """Reassign, move status, move the due date or choose the action of one item. Returns the item.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + if action is not None and action not in ACTIONS: + raise ValueError("Action must be one of: " + ", ".join(ACTIONS) + ".") + if due is not None and due != "": + try: + due = dt.date.fromisoformat(str(due)).isoformat() + except ValueError: + raise ValueError("Due must be a date such as 2026-10-30.") from None + if owner is not None and not str(owner).strip(): + raise ValueError("Owner cannot be empty.") + st = _rt(tenant_key, st) + if not _may_change(st, user): + raise PermissionError("Your account can view this App but not change it.") + name = _uname(user) or "unknown" + out = {} + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + item = (doc.get("items") or {}).get(item_id) + if not item: + return doc + at = _stamp(_now()) + event = {"at": at, "by": name} + if owner is not None: + item["owner"] = event["owner"] = str(owner).strip() + if status is not None: + item["status"] = event["status"] = status + if due is not None: + item["due"] = event["due"] = due or None + if action is not None: + item["action"] = event["action"] = action + item["actionChosenBy"] = name + if note: + event["note"] = str(note) + item["updatedAt"], item["updatedBy"] = at, name + item.setdefault("history", []).append(event) + out["item"] = _public(item) + return doc + st.update(KEY_ITEMS, _apply) + if "item" not in out: + raise KeyError(item_id) + return out["item"] + + +def set_status(tenant_key, user, item_id, status, *, note=None, st=None): + """C4: move one item's status with an optional note.""" + return update_item(tenant_key, item_id, status=status, note=note, user=user, st=st) + + +def _proposal_text(item): + action = item.get("action") + if action == REMOVE: + removal = item.get("removal") or {} + for qty in (removal.get("sellable_removal_quantity"), item.get("recommended_removal_quantity"), + item.get("units")): + if qty is not None and qty > 0: + return f"Remove {_fmt(qty)} units" + return "Remove the units" + if action == PRICE_DOWN: + price = item.get("recommended_sales_price") + if price is not None and price > 0: + return f"Lower the price to {_money(price, item.get('currency') or UNKNOWN_CURRENCY)}" + return "Lower the price" + return "Push with ads" + + +def proposals(tenant_key, *, st=None): + """C7: one proposal per Open item with an action, each listed once.""" + st = _rt(tenant_key, st) + out, seen = [], set() + for item in sorted((_doc(st, KEY_ITEMS).get("items") or {}).values(), key=_order): + if (item.get("status") != "Open" or item.get("action") not in ACTIONS + or item.get("inReport") is False or item["id"] in seen): + continue + seen.add(item["id"]) + out.append({"source_kind": SOURCE_KIND, "item_id": item["id"], "sku": item.get("sku") or item.get("fnsku"), + "action": _proposal_text(item), "reason": item.get("reason")}) + return out + + +def on_decision(tenant_key, item_id, decision, reason, *, st=None): + """C7: `approved` sets Accepted, `dismissed` sets Dismissed, with the reason as a history note.""" + if decision not in DECISIONS: + raise ValueError("Decision must be approved or dismissed.") + return update_item(tenant_key, item_id, status=DECISIONS[decision], note=reason or None, + user="operator", st=st) + + +# --------------------------------------------------------------------------- rule 8 and rule 13 + +_MONTH = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$") + + +def _period(start, end): + for value in (start, end): + if value is not None and not _MONTH.match(str(value)): + raise ValueError("A period month must read like 2026-09.") + if start and end and start > end: + raise ValueError("The period start must not be after its end.") + return start or None, end or None + + +def _in_period(month, start, end): + if start is None and end is None: + return True + return bool(month) and (start is None or month >= start) and (end is None or month <= end) + + +def validate(tenant_key, *, start=None, end=None, st=None, **period): + """Rule 8: storage plus long-term fees over the App's SKU rows tie, per currency, to the fee tables' own sums.""" + start, end = _period(start, end) + st = _rt(tenant_key, st) + conf = settings(tenant_key, st=st) + with _reader(tenant_key, st) as cur: + s_cols, l_cols = _columns(cur, STORAGE), _columns(cur, LONG_TERM) + has_s, has_l = "estimated_monthly_storage_fee" in s_cols, "amount_charged" in l_cols + if cur is None or not (has_s or has_l): + return {"ok": False, "app_total": None, "independent_total": None, "currency": None, + "detail": {"summary": f"{SOURCE_NAMES[STORAGE]} has not synced yet.", "currencies": {}}} + rows, _notices, _primary = build_rows(cur, _margins(st), conf) + independent, n_rows = {}, {} + for has, cols, table, fee, month_sql in ( + (has_s, s_cols, STORAGE, "estimated_monthly_storage_fee", storage_month_sql), + (has_l, l_cols, LONG_TERM, "amount_charged", long_term_month_sql)): + if not has: + continue + where, args = "TRUE", [] + if start is not None: + where += f" AND {month_sql(cols)} >= ?" + args.append(start) + if end is not None: + where += f" AND {month_sql(cols)} <= ?" + args.append(end) + for currency, total, n in cur.execute( + f'SELECT {_currency_sql(cols)}, SUM(TRY_CAST("{fee}" AS DOUBLE)), COUNT(*) FROM {table} ' + f'WHERE {where} GROUP BY 1', args).fetchall(): + independent[currency] = independent.get(currency, 0.0) + (total or 0.0) + n_rows[currency] = n_rows.get(currency, 0) + n + app = {} + for row in rows or (): + for point in row.get("fee_history") or (): + if _in_period(point["month"], start, end): + app[point["currency"]] = app.get(point["currency"], 0.0) + point["storage"] + point["long_term"] + currencies = sorted(set(app) | set(independent)) + detail = {cur: {"app_total": round(app.get(cur, 0.0), 2), "independent_total": round(independent.get(cur, 0.0), 2), + "ok": abs(app.get(cur, 0.0) - independent.get(cur, 0.0)) < 0.005} for cur in currencies} + primary = sorted(n_rows.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] if n_rows else None + ok = all(d["ok"] for d in detail.values()) + span = (f"{start or 'the first month'} to {end or 'the last month'}" if start or end else "every month") + tied = ", ".join(f"{c} {d['app_total']:,.2f} against {d['independent_total']:,.2f}" for c, d in detail.items()) + return {"ok": ok, "app_total": detail[primary]["app_total"] if primary else 0.0, + "independent_total": detail[primary]["independent_total"] if primary else 0.0, + "currency": primary, + "detail": {"summary": f"Storage plus long term fees for {span}: {tied or 'no fees'}.", + "currencies": detail, "start": start, "end": end}} + + +def app_keys(st): + """Every store document this App owns.""" + months = _doc(st, KEY_HISTORY).get("months") or () + return [KEY_SETTINGS, KEY_ITEMS, KEY_HISTORY, *(_month_key(m) for m in months)] + + +def remove_app_data(tenant_key, *, st=None): + """Rule 13: delete this App's own documents. No mirror table and no other document is touched.""" + st = _rt(tenant_key, st) + deleted, emptied = [], [] + for key in app_keys(st): + try: + present = st.get(key) not in (None, {}, []) + except Exception: # noqa: BLE001 + present = True + if not present: + continue + remover = getattr(st, "delete_document", None) + if callable(remover): + try: + remover(key) + deleted.append(key) + continue + except Exception: # noqa: BLE001 + pass + # A store without a delete door keeps an empty document in place of the App's data. + st.update(key, lambda _doc: {}) + emptied.append(key) + return {"deleted": deleted, "emptied": emptied} diff --git a/platform/modules/amazon_operator.py b/platform/modules/amazon_operator.py new file mode 100644 index 0000000000000000000000000000000000000000..e99cb28407fda6b2a0f85fdc8e39bf3a07997237 --- /dev/null +++ b/platform/modules/amazon_operator.py @@ -0,0 +1,1342 @@ +"""Amazon operator App: daily ad proposals on SP, SB and SD, gated by profit after COGS and stock cover. + +The agent proposes and a person approves a batch before any write (R14). `set_status(..., "approved")` +is the one approve path: it stores one approval record, then sends one named Ads write per proposal +through `_ads_client()`. Rules per SKU, every threshold an App setting (R48, R49): + stop cover under `stop_cover_days`: pause the ad group's ads for the SKU. A stopped SKU gets no raise. + pause linked SKU with profit after COGS under `pause_below_profit`: pause the same way. + raise cover over `raise_cover_days` and a linked SKU at or above the pause line: bid up `bid_step_pct`. + budget a campaign whose every evaluated SKU got a raise: daily budget up `budget_step_pct`. + negative keyword: a search term with clicks, spend and no orders over the lookback. +An unlinked SKU (blank COGS) gets no profit-gated proposal and a held note that says why. + +Inputs: COGS from the `ut_amazon_sku_profit` table (`cogs_per_unit`, linked SKUs only), cover from +the mirror `amzn_inventory_planning` (`days_of_supply`, latest snapshot per SKU), ad rows from the +Ads tables named by `_ads_table`. This module imports no other App at import time, so deleting any App +leaves it whole, and deleting this App removes only the documents in `KEYS` (rule 13). + +C7 inbox: `pull_app_proposals` imports each module in `PROPOSAL_SOURCES` lazily, reads its C4 +`proposals()` into `amazon_operator_inbox`, and a decision on an inbox item calls that module's +`on_decision`. An App proposal never reaches the Ads client: it is a decision handed back, not a write. + +Store documents (tenant runtime store, App owned, never the mirror and never a `ut_amzn_*` table): + amazon_operator_settings thresholds and the default owner + amazon_operator_days {latest, days: {day: {built_at, data_date, items, held, spend, ...}}} + amazon_operator_baseline the 60 day ad profit after COGS baseline and the R13 time baseline + amazon_operator_decisions {decisions: {id: {id, decision, by, at, item_ids, note, writes}}} + amazon_operator_outcomes {items: {item id: {status, cause, decision_id, at, kind}}} + amazon_operator_inbox {items: {inbox id: {source_kind, source_module, item_id, sku, action, reason, app, status}}} + +Limit (rule 1): each document is one JSON value, and the Postgres store refuses a document over +`core.store_pg.DOCUMENT_CEILING_BYTES` (128 MB). At about 1 KB a proposal, the days document reaches +it near 120,000 proposals. Nothing here caps rows, proposals or pages. The fix when it nears: +split `amazon_operator_days` by month. +""" +import base64 +import contextlib +import copy +import datetime as dt +import hashlib +import json +import logging +import math +import uuid +from pathlib import Path + +SOURCE_KIND = "amazon_operator" +AD_TYPES = ("sp", "sb", "sd") +#: validate() sums every Ads type on its own list, so a build that skips one cannot tie. +ALL_AD_TYPES = ("sp", "sb", "sd") +AD_LABELS = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"} +REPORTS = ("campaigns", "targeting", "advertised_products", "search_terms") + +KEY_SETTINGS = "amazon_operator_settings" +KEY_DAYS = "amazon_operator_days" +KEY_BASELINE = "amazon_operator_baseline" +KEY_DECISIONS = "amazon_operator_decisions" +KEY_OUTCOMES = "amazon_operator_outcomes" +KEY_INBOX = "amazon_operator_inbox" +#: Every store document this App owns. remove() deletes these and nothing else. +KEYS = (KEY_SETTINGS, KEY_DAYS, KEY_BASELINE, KEY_DECISIONS, KEY_OUTCOMES, KEY_INBOX) + +#: C7: the App modules whose C4 proposals() feed the inbox, imported lazily by name, in this order. +PROPOSAL_SOURCES = ("modules.restock_planner", "modules.reimbursements", "modules.payouts", + "modules.account_health", "modules.returns_repeat", "modules.aged_stock", + "modules.opportunity_finder") +#: The App's label when its module carries no APP_LABEL. +APP_LABELS = {"restock_planner": "Restock planner", "reimbursements": "Reimbursements", "payouts": "Payouts and cash", + "account_health": "Account health", "returns_repeat": "Returns and repeat", + "aged_stock": "Aged stock", "opportunity_finder": "Opportunity finder"} +#: Inbox statuses. `stale` is an item its App no longer lists; it cannot be approved. +INBOX_OPEN = ("proposed",) +#: The `kind` of an inbox item, and the `source` of a decision record on App items. +APP_KIND = "app" +STALE = "This item is no longer open in its App." +APPROVED_NOTE = "Approved in the Operator inbox." + +PROFIT_TABLE = "ut_amazon_sku_profit" +PLANNING_TABLE = "amzn_inventory_planning" + +DEFAULTS = {"stop_cover_days": 14, "raise_cover_days": 90, "pause_below_profit": 0.0, + "bid_step_pct": 10.0, "budget_step_pct": 10.0, "lookback_days": 14, + "negative_min_clicks": 10, "negative_min_cost": 5.0} +_INTS = ("stop_cover_days", "raise_cover_days", "lookback_days", "negative_min_clicks") +STATUSES = ("proposed", "approved", "sent", "refused", "dismissed") +OPEN = ("proposed",) +#: The statuses a person sets. `sent` and `refused` are outcomes that only the write records. +DECISIONS = ("approved", "dismissed") +#: Fields a decision or an outcome writes on a proposal. A same day rebuild carries every one over. +DECIDED_FIELDS = ("status", "owner", "due", "decision_id", "decided_by", "decided_at", "note", "cause", + "outcome_at", "result", "sent_payload") +NOT_INSTALLED = "The Amazon Ads connector is not installed." +NOT_FOUND = "That proposal does not exist." +READ_ONLY = "Your account can view these proposals but not approve or dismiss them." +DUE_DAYS = 1 +BASELINE_DAYS = 60 +TIME_BASELINE = {"low": 10, "high": 20, "unit": "hours per week", "source": "R13, R52", + "basis": "Jake's stated ads time before Loopable. No time log exists."} +BAR = {"ads_hours_per_week_under": 3, "profit_after_cogs": "level or up", "days": 60, "source": "R15, R52"} +BASELINE_BASIS = ("Ad sales minus ad cost minus ad orders times COGS per unit, for linked SKUs only. " + "Ad orders stand in for units. Unlinked SKUs are left out and counted.") +NO_COGS = "No COGS yet. Link this SKU in Amazon SKU profit." +NO_SKU = "No SKU on this ad. Its ASIN matches no SKU in Amazon SKU profit or inventory planning." +NO_PROFIT = "No profit after COGS yet. Refresh Amazon SKU profit." +NO_COVER = "No cover yet. Inventory planning has no days of supply for this SKU." +NO_CLICKS = "No clicks in the lookback, so there is no bid to raise." +LIMIT = ("Each Amazon operator document is limited to 128 MB by the store. About 120,000 proposals " + "fit in one. Split the days document by month when it nears that size.") + +_log = logging.getLogger(__name__) + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at): + return at.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _num(value): + """A finite float, or None for a blank or unreadable cell.""" + if value is None or (isinstance(value, str) and not value.strip()): + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def _money(value): + return f"{value:.2f}" + + +def _days(value): + return str(int(value)) if float(value).is_integer() else f"{value:.1f}" + + +def _earns(net): + """'makes 5.00' or 'loses 12.50': a negative profit is never printed after 'makes'.""" + return f"loses {_money(-net)}" if net < 0 else f"makes {_money(net)}" + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + value = st.get(key) + return value if isinstance(value, dict) else {} + + +# --------------------------------------------------------------------------- mirror and Ads reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _ads_table(ad_type, report): + """The mirror table for one Ads report. Every Ads read resolves its table name here.""" + plain = f"amzn_ads_{ad_type}_{report}" + try: + from harness import amazon_ads_store + except Exception: # noqa: BLE001 + return plain + namer = getattr(amazon_ads_store, "table_name", None) + if callable(namer): + try: + name = str(namer(f"{ad_type}_{report}")) + except Exception: # noqa: BLE001 + return plain + if name.startswith("amzn_ads_"): + return name + return plain + + +def _columns(cur, table): + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _text(cols, *names): + for name in names: + if name in cols: + return f'CAST("{name}" AS VARCHAR)' + return "NULL" + + +def _double(cols, name): + return f'COALESCE(TRY_CAST("{name}" AS DOUBLE), 0)' if name in cols else "0" + + +_DATE = 'CAST(TRY_CAST("date" AS DATE) AS VARCHAR)' + + +def _data_date(cur, ad_types): + """The latest `date` across the advertised products tables, as YYYY-MM-DD, or None.""" + best = None + for ad_type in ad_types: + table = _ads_table(ad_type, "advertised_products") + if "date" not in _columns(cur, table): + continue + row = cur.execute(f"SELECT MAX({_DATE}) FROM {table}").fetchone() + if row and row[0] and (best is None or row[0] > best): + best = str(row[0]) + return best + + +def _product_rows(cur, ad_type, start, end): + """Advertised products rows dated start to end inclusive, as dicts.""" + table = _ads_table(ad_type, "advertised_products") + cols = _columns(cur, table) + if not {"date", "cost", "campaign_id"} <= cols: + return [] + rows = cur.execute( + f"SELECT {_DATE}, {_text(cols, 'campaign_id')}, {_text(cols, 'campaign_name', 'campaign')}, " + f"{_text(cols, 'ad_group_id')}, {_text(cols, 'target_id')}, {_text(cols, 'ad_id')}, " + f"{_text(cols, 'advertised_sku')}, {_text(cols, 'asin', 'advertised_asin')}, " + f"{_double(cols, 'clicks')}, {_double(cols, 'cost')}, {_double(cols, 'sales')}, " + f"{_double(cols, 'orders')} FROM {table} WHERE {_DATE} BETWEEN ? AND ?", [start, end]).fetchall() + keys = ("date", "campaign_id", "campaign", "ad_group_id", "target_id", "ad_id", "sku", "asin", + "clicks", "cost", "sales", "orders") + out = [] + for row in rows: + rec = dict(zip(keys, row)) + for k in ("campaign_id", "campaign", "ad_group_id", "target_id", "ad_id", "sku", "asin"): + rec[k] = str(rec[k]).strip() if rec[k] is not None else "" + rec["ad_type"] = ad_type + out.append(rec) + return out + + +def _search_rows(cur, ad_type, start, end): + table = _ads_table(ad_type, "search_terms") + cols = _columns(cur, table) + term = _text(cols, "search_term", "query", "keyword") + if not {"date", "campaign_id"} <= cols or term == "NULL": + return [] + rows = cur.execute( + f"SELECT {_text(cols, 'campaign_id')}, {_text(cols, 'campaign_name', 'campaign')}, " + f"{_text(cols, 'ad_group_id')}, {term}, SUM({_double(cols, 'clicks')}), SUM({_double(cols, 'cost')}), " + f"SUM({_double(cols, 'orders')}) FROM {table} WHERE {_DATE} BETWEEN ? AND ? GROUP BY ALL", + [start, end]).fetchall() + return [{"ad_type": ad_type, "campaign_id": str(c or "").strip(), "campaign": str(n or "").strip(), + "ad_group_id": str(g or "").strip(), "term": " ".join(str(t or "").split()).lower(), + "clicks": float(k or 0), "cost": float(s or 0), "orders": float(o or 0)} + for c, n, g, t, k, s, o in rows if str(t or "").strip()] + + +def _cover(cur): + """{sku: (days_of_supply, asin)} from each SKU's latest planning snapshot. The lowest row wins a tie.""" + cols = _columns(cur, PLANNING_TABLE) if cur is not None else set() + if not {"sku", "days_of_supply"} <= cols: + return {} + snap = _text(cols, "_snapshot_date") + rows = cur.execute(f'SELECT CAST("sku" AS VARCHAR), {_text(cols, "asin")}, {snap}, ' + f'TRY_CAST("days_of_supply" AS DOUBLE) FROM {PLANNING_TABLE}').fetchall() + latest = {} + for sku, asin, when, days in rows: + sku = str(sku or "").strip() + if not sku: + continue + when = str(when or "") + have = latest.get(sku) + if have is None or when > have[0]: + latest[sku] = [when, days, str(asin or "").strip()] + elif when == have[0] and days is not None and (have[1] is None or days < have[1]): + have[1] = days + return {sku: (_num(days), asin) for sku, (_when, days, asin) in latest.items()} + + +def _profit(st): + """{sku: {"linked", "cogs_per_unit", "net_profit", "asin"}} from the Amazon SKU profit table.""" + import core.user_tables as ut + rows = (ut.get_one(PROFIT_TABLE, st) or {}).get("rows") or {} + out = {} + for row in rows.values(): + if not isinstance(row, dict): + continue + sku = str(row.get("sku") or "").strip() + if not sku: + continue + cogs = _num(row.get("cogs_per_unit")) + out[sku] = {"linked": cogs is not None, "cogs_per_unit": cogs, + "net_profit": _num(row.get("net_profit")) if cogs is not None else None, + "asin": str(row.get("asin") or "").strip()} + return out + + +def _sku_for(rec, by_asin): + return rec["sku"] or by_asin.get(rec["asin"], "") + + +# --------------------------------------------------------------------------- settings and owner + +#: Accounts that sign in for a process, never a person who can own a proposal. +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """The tenant's earliest active person Administrator, else its `admin` account, else its first Administrator.""" + try: + users = st.get("users") + except Exception: # noqa: BLE001 + users = None + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in (users or {}).items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def _clean(name, value): + """One setting checked and typed, or ValueError with a plain message.""" + if isinstance(value, bool): + raise ValueError("A threshold must be a number.") + try: + number = float(value) + except (TypeError, ValueError): + raise ValueError("A threshold must be a number.") from None + if not math.isfinite(number): + raise ValueError("A threshold must be a number.") + if name in _INTS: + if not number.is_integer(): + raise ValueError("Days and clicks must be whole numbers.") + number = int(number) + if number < (1 if name in ("lookback_days", "negative_min_clicks") else 0): + raise ValueError("Days and clicks must be 0 or more, and the lookback at least 1 day.") + return number + if name in ("bid_step_pct", "budget_step_pct") and not 0 < number <= 100: + raise ValueError("A step must be above 0 and at most 100 percent.") + if name == "negative_min_cost" and number < 0: + raise ValueError("The negative keyword spend must be 0 or more.") + return number + + +def settings(tenant_key, *, st=None): + """Thresholds and the default owner, with defaults filled in.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + out = {} + for name, default in DEFAULTS.items(): + try: + out[name] = _clean(name, doc.get(name, default)) + except ValueError: + out[name] = default + out["defaults"] = dict(DEFAULTS) + out["defaultOwnerSetting"] = doc.get("default_owner") or None + out["defaultOwner"] = out["defaultOwnerSetting"] or owner_account(st, tenant_key) + out["limit"] = LIMIT + return out + + +def set_settings(tenant_key, *, st=None, default_owner=None, **changes): + """Set any threshold in DEFAULTS, or the default owner ('' resets to the owner account).""" + st = _rt(tenant_key, st) + unknown = sorted(set(changes) - set(DEFAULTS)) + if unknown: + raise ValueError("Unknown setting: " + ", ".join(unknown) + ".") + clean = {k: _clean(k, v) for k, v in changes.items() if v is not None} + current = settings(tenant_key, st=st) + stop = clean.get("stop_cover_days", current["stop_cover_days"]) + raise_ = clean.get("raise_cover_days", current["raise_cover_days"]) + if raise_ <= stop: + raise ValueError("The raise cover days must be above the stop cover days.") + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(clean) + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +# --------------------------------------------------------------------------- the daily build + +def _pid(day, kind, action, ad_type, campaign_id, ad_group_id, target_id, subject): + """Stable id: a rebuild on the same day yields the same id, so it never duplicates.""" + raw = "|".join(str(p or "") for p in (day, kind, action, ad_type, campaign_id, ad_group_id, + target_id, subject)) + return "p" + hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16] + + +def _units(rows, by_asin, data_date): + """Group rows by (ad type, campaign, ad group, SKU); rows with no SKU go to the held list.""" + units, held = {}, {} + for rec in rows: + sku = _sku_for(rec, by_asin) + key = (rec["ad_type"], rec["campaign_id"], rec["ad_group_id"], sku) + if not sku: + held.setdefault(key, {"ad_type": rec["ad_type"], "campaign_id": rec["campaign_id"], + "campaign": rec["campaign"] or rec["campaign_id"], + "ad_group_id": rec["ad_group_id"], "sku": "", "asin": rec["asin"], + "reason": NO_SKU}) + continue + unit = units.setdefault(key, {"ad_type": rec["ad_type"], "campaign_id": rec["campaign_id"], + "campaign": "", "ad_group_id": rec["ad_group_id"], "sku": sku, + "asin": rec["asin"], "targets": set(), "ad_ids": set(), + "clicks": 0.0, "cost": 0.0, "sales": 0.0, "orders": 0.0, + "spend": 0.0}) + unit["campaign"] = unit["campaign"] or rec["campaign"] + unit["asin"] = unit["asin"] or rec["asin"] + if rec["target_id"]: + unit["targets"].add(rec["target_id"]) + if rec["ad_id"]: + unit["ad_ids"].add(rec["ad_id"]) + for k in ("clicks", "cost", "sales", "orders"): + unit[k] += rec[k] + if rec["date"] == data_date: + unit["spend"] += rec["cost"] + for unit in units.values(): + unit["campaign"] = unit["campaign"] or unit["campaign_id"] + return units, list(held.values()) + + +def _proposal(day, data_date, unit, kind, action, change, payload, reason, gate, owner, due, *, + target_id="", subject=None, spend=None): + pid = _pid(day, kind, action, unit["ad_type"], unit["campaign_id"], unit.get("ad_group_id", ""), + target_id, subject if subject is not None else unit.get("sku", "")) + return {"id": pid, "day": day, "data_date": data_date, "ad_type": unit["ad_type"], + "campaign_id": unit["campaign_id"], "campaign": unit["campaign"], + "ad_group_id": unit.get("ad_group_id", ""), "target_id": target_id, + "sku": unit.get("sku", ""), "asin": unit.get("asin", ""), "kind": kind, "action": action, + "change": change, "payload": payload, "reason": reason, "gate": gate, + "spend": round(float(unit.get("spend", 0.0) if spend is None else spend), 2), + "status": "proposed", "owner": owner, "due": due} + + +def _evaluate(day, data_date, conf, units, profit, cover, owner, due): + """(proposals, held) for the grouped units. Pure: no store and no mirror.""" + stop_at, raise_at = conf["stop_cover_days"], conf["raise_cover_days"] + pause_at, step = conf["pause_below_profit"], conf["bid_step_pct"] + proposals, held, raised = [], [], {} + for key in sorted(units): + unit = units[key] + sku = unit["sku"] + days = (cover.get(sku) or (None, ""))[0] + info = profit.get(sku) or {"linked": False, "net_profit": None} + camp = (unit["ad_type"], unit["campaign_id"]) + raised.setdefault(camp, []) + pause_payload = {"operation": "pause_ads", "ad_type": unit["ad_type"], + "campaign_id": unit["campaign_id"], "ad_group_id": unit["ad_group_id"], + "sku": sku, "asin": unit["asin"], "ad_ids": sorted(unit["ad_ids"]), + "state": "paused"} + if days is not None and days < stop_at: + proposals.append(_proposal( + day, data_date, unit, "bid", "stop", f"Pause ads for {sku}", pause_payload, + f"Stock covers {_days(days)} days. Stop ads so spend does not outrun stock.", + f"Cover {_days(days)} days is under the {_days(stop_at)} day stop line.", owner, due)) + raised[camp].append(False) + continue + if not info["linked"]: + held.append({**_held(unit), "reason": NO_COGS}) + raised[camp].append(False) + continue + net = info["net_profit"] + if net is not None and net < pause_at: + proposals.append(_proposal( + day, data_date, unit, "bid", "pause", f"Pause ads for {sku}", pause_payload, + f"This SKU {_earns(net)} after COGS over the profit period.", + f"Profit after COGS {_money(net)} is under the {_money(pause_at)} pause line.", owner, due)) + raised[camp].append(False) + continue + if net is None: + held.append({**_held(unit), "reason": NO_PROFIT}) + raised[camp].append(False) + continue + if days is None: + held.append({**_held(unit), "reason": NO_COVER}) + raised[camp].append(False) + continue + if days > raise_at: + if unit["clicks"] <= 0: + held.append({**_held(unit), "reason": NO_CLICKS}) + raised[camp].append(False) + continue + bid = unit["cost"] / unit["clicks"] + new = round(bid * (1 + step / 100.0), 2) + target = sorted(unit["targets"])[0] if len(unit["targets"]) == 1 else "" + proposals.append(_proposal( + day, data_date, unit, "bid", "raise", f"Bid {_money(bid)} to {_money(new)}", + {"operation": "set_bid", "ad_type": unit["ad_type"], "campaign_id": unit["campaign_id"], + "ad_group_id": unit["ad_group_id"], "target_id": target, "sku": sku, + "asin": unit["asin"], "previous_bid": round(bid, 2), "bid": new}, + f"Stock covers {_days(days)} days and the SKU {_earns(net)} after COGS. " + f"Raise the bid to sell through.", + f"Cover {_days(days)} days is over the {_days(raise_at)} day raise line. Profit after COGS " + f"{_money(net)} is at or above {_money(pause_at)}.", owner, due, target_id=target)) + raised[camp].append(True) + continue + raised[camp].append(False) + return proposals, held, raised + + +def _held(unit): + return {"ad_type": unit["ad_type"], "campaign_id": unit["campaign_id"], "campaign": unit["campaign"], + "ad_group_id": unit["ad_group_id"], "sku": unit["sku"], "asin": unit["asin"]} + + +def _raise_gate_copy(count): + """The budget gate line, with the count and the noun in agreement.""" + return "The 1 SKU passed the raise gate." if count == 1 else f"All {count} SKUs passed the raise gate." + + +def _budgets(day, data_date, conf, units, raised, owner, due): + step, lookback = conf["budget_step_pct"], conf["lookback_days"] + out = [] + for (ad_type, campaign_id), flags in sorted(raised.items()): + if not flags or not all(flags): + continue + members = [u for u in units.values() if (u["ad_type"], u["campaign_id"]) == (ad_type, campaign_id)] + cost = sum(u["cost"] for u in members) + if cost <= 0: + continue + daily = cost / lookback + new = round(daily * (1 + step / 100.0), 2) + camp = {"ad_type": ad_type, "campaign_id": campaign_id, + "campaign": members[0]["campaign"], "ad_group_id": "", "sku": "", "asin": ""} + out.append(_proposal( + day, data_date, camp, "budget", "raise_budget", f"Daily budget to {_money(new)}", + {"operation": "set_daily_budget", "ad_type": ad_type, "campaign_id": campaign_id, + "average_daily_cost": round(daily, 2), "daily_budget": new}, + f"Every SKU in this campaign got a bid raise. Average daily spend is {_money(daily)} " + f"over {lookback} days.", + _raise_gate_copy(len(flags)), owner, due, + subject=campaign_id, spend=sum(u["spend"] for u in members))) + return out + + +def _negatives(day, data_date, conf, terms, owner, due): + min_clicks, min_cost, lookback = conf["negative_min_clicks"], conf["negative_min_cost"], conf["lookback_days"] + out = [] + for row in sorted(terms, key=lambda r: (r["ad_type"], r["campaign_id"], r["ad_group_id"], r["term"])): + if row["clicks"] < min_clicks or row["orders"] > 0 or row["cost"] < min_cost: + continue + unit = {"ad_type": row["ad_type"], "campaign_id": row["campaign_id"], + "campaign": row["campaign"] or row["campaign_id"], "ad_group_id": row["ad_group_id"], + "sku": "", "asin": ""} + out.append(_proposal( + day, data_date, unit, "negative_keyword", "add_negative", + f'Add "{row["term"]}" as a negative exact keyword', + {"operation": "add_negative_keyword", "ad_type": row["ad_type"], "campaign_id": row["campaign_id"], + "ad_group_id": row["ad_group_id"], "keyword_text": row["term"], "match_type": "negativeExact"}, + f'"{row["term"]}" took {_days(row["clicks"])} clicks and {_money(row["cost"])} spend in ' + f"{lookback} days with no orders.", + f"Clicks {_days(row['clicks'])} at or above {min_clicks}, spend {_money(row['cost'])} at or " + f"above {_money(min_cost)}, orders 0.", owner, due, subject=row["term"], spend=0.0)) + return out + + +def build(tenant_key, *, st=None, now=None): + """Build today's proposals from the Ads tables, cover and COGS, and store them. Returns a summary.""" + st = _rt(tenant_key, st) + now = now or _now() + day = now.date().isoformat() + conf = settings(tenant_key, st=st) + owner = conf["defaultOwner"] + due = (now.date() + dt.timedelta(days=DUE_DAYS)).isoformat() + profit = _profit(st) + with _reader(tenant_key, st) as cur: + if cur is None: + return {"state": "no_mirror", "day": day} + data_date = _data_date(cur, AD_TYPES) + if data_date is None: + return {"state": "no_ads_data", "day": day} + end = dt.date.fromisoformat(data_date) + start = (end - dt.timedelta(days=conf["lookback_days"] - 1)).isoformat() + cover = _cover(cur) + rows, terms = [], [] + for ad_type in AD_TYPES: + rows += _product_rows(cur, ad_type, start, data_date) + terms += _search_rows(cur, ad_type, start, data_date) + baseline = _doc(st, KEY_BASELINE) + base_rows = None + if baseline.get("state") != "computed": + first = baseline.get("first_data_date") or data_date + b_end = dt.date.fromisoformat(first) - dt.timedelta(days=1) + b_start = b_end - dt.timedelta(days=BASELINE_DAYS - 1) + base_rows = (first, b_start.isoformat(), b_end.isoformat(), + [r for t in AD_TYPES for r in _product_rows(cur, t, b_start.isoformat(), + b_end.isoformat())]) + by_asin = {} + for sku, (_days_, asin) in cover.items(): + if asin: + by_asin.setdefault(asin, sku) + for sku, info in profit.items(): + if info["asin"]: + by_asin[info["asin"]] = sku + units, held = _units(rows, by_asin, data_date) + proposals, more_held, raised = _evaluate(day, data_date, conf, units, profit, cover, owner, due) + held += more_held + proposals += _budgets(day, data_date, conf, units, raised, owner, due) + proposals += _negatives(day, data_date, conf, terms, owner, due) + spend = {t: 0.0 for t in AD_TYPES} + for rec in rows: + if rec["date"] == data_date: + spend[rec["ad_type"]] += rec["cost"] + spend = {t: round(v, 2) for t, v in spend.items()} + + def _days_doc(doc): + doc = doc if isinstance(doc, dict) else {} + days = doc.setdefault("days", {}) + before = (days.get(day) or {}).get("items") or {} + items = {} + for prop in proposals: + old = before.get(prop["id"]) + if old: + prop = {**prop, **{k: old[k] for k in DECIDED_FIELDS if k in old}} + items[prop["id"]] = prop + # A decided proposal whose rule no longer fires stays whole: its decision and outcome are history. + for pid, old in before.items(): + if pid not in items and isinstance(old, dict) and old.get("status") not in OPEN: + items[pid] = old + days[day] = {"built_at": _stamp(now), "data_date": data_date, "lookback_start": start, + "items": items, "held": held, "spend": spend, + "evaluated_rows": len(rows), "ad_types": list(AD_TYPES)} + doc["latest"] = max(days) + doc.pop("last_error", None) + return doc + st.update(KEY_DAYS, _days_doc) + if base_rows is not None: + _store_baseline(st, profit, by_asin, *base_rows, now=now) + return {"state": "built", "day": day, "data_date": data_date, "proposals": len(proposals), + "held": len(held), "spend": spend} + + +def compute_baseline(rows, profit, by_asin): + """Ad profit after COGS over rows for linked SKUs; unlinked SKUs are left out and counted.""" + linked, unlinked, unresolved = set(), set(), 0 + sales = cost = cogs = 0.0 + for rec in rows: + sku = _sku_for(rec, by_asin) + info = profit.get(sku) if sku else None + if not sku: + unresolved += 1 + continue + if not info or not info["linked"]: + unlinked.add(sku) + continue + linked.add(sku) + sales += rec["sales"] + cost += rec["cost"] + cogs += rec["orders"] * info["cogs_per_unit"] + return {"ad_profit_after_cogs": round(sales - cost - cogs, 2), "ad_sales": round(sales, 2), + "ad_cost": round(cost, 2), "cogs": round(cogs, 2), "linked_skus": len(linked), + "unlinked_skus": len(unlinked), "unlinked": sorted(unlinked), "rows_without_sku": unresolved, + "days_with_data": len({r["date"] for r in rows})} + + +def _store_baseline(st, profit, by_asin, first, start, end, rows, *, now): + figures = compute_baseline(rows, profit, by_asin) + ready = figures["days_with_data"] > 0 + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + if doc.get("state") == "computed": + return doc + doc.update(figures if ready else {}) + doc.update({"state": "computed" if ready else "waiting", "first_data_date": first, + "window_start": start, "window_end": end, "days": BASELINE_DAYS, + "basis": BASELINE_BASIS, "time_baseline_hours_per_week": dict(TIME_BASELINE), + "bar": dict(BAR)}) + if ready: + doc["computed_at"] = _stamp(now) + return doc + st.update(KEY_BASELINE, _apply) + + +def baseline(tenant_key, *, st=None): + """The stored 60 day baseline, or a waiting record before the first build.""" + doc = _doc(_rt(tenant_key, st), KEY_BASELINE) + return doc or {"state": "waiting", "time_baseline_hours_per_week": dict(TIME_BASELINE), "bar": dict(BAR)} + + +def on_ads_sync(tenant_key, synced_tables): + """Contract C6 callback: build today's proposals, then pull the App inbox. Logs one line on failure and never raises.""" + try: + out = build(tenant_key) + except Exception as exc: # noqa: BLE001 + _log.warning("amazon_operator: the proposal build for %s failed (%s): %s", + tenant_key, type(exc).__name__, exc) + return None + _safe_pull(tenant_key) + return out + + +# --------------------------------------------------------------------------- the C4 item protocol + +def _all_proposals(st): + days = _doc(st, KEY_DAYS).get("days") or {} + out = [] + for day in days.values(): + out += list(((day or {}).get("items") or {}).values()) + return out + + +def day_record(tenant_key, day=None, *, st=None): + """One built day (the latest when day is None), or None.""" + doc = _doc(_rt(tenant_key, st), KEY_DAYS) + day = day or doc.get("latest") + return ((doc.get("days") or {}).get(day) or None) if day else None + + +def _item(prop): + label = AD_LABELS.get(prop.get("ad_type"), str(prop.get("ad_type") or "").upper()) + return {"id": prop["id"], "title": f"{label} {prop.get('campaign') or prop.get('campaign_id')}: " + f"{prop.get('change')}", + "status": prop.get("status"), "owner": prop.get("owner"), "due": prop.get("due"), + "reason": prop.get("reason"), "sku": prop.get("sku") or None, "asin": prop.get("asin") or None, + "kind": prop.get("kind"), "action": prop.get("action"), "change": prop.get("change"), + "gate": prop.get("gate"), "ad_type": prop.get("ad_type"), "campaign": prop.get("campaign"), + "campaign_id": prop.get("campaign_id"), "day": prop.get("day"), + "data_date": prop.get("data_date"), "spend": prop.get("spend"), + "decision_id": prop.get("decision_id"), "decided_by": prop.get("decided_by"), + "decided_at": prop.get("decided_at"), "note": prop.get("note"), "cause": prop.get("cause"), + "outcome_at": prop.get("outcome_at")} + + +def _encode(key): + return base64.urlsafe_b64encode(json.dumps(list(key)).encode("utf-8")).decode("ascii") + + +def _decode(cursor): + try: + day, pid = json.loads(base64.urlsafe_b64decode(str(cursor).encode("ascii")).decode("utf-8")) + return str(day), str(pid) + except Exception: # noqa: BLE001 + raise ValueError("The cursor could not be read.") from None + + +def items(tenant_key, user=None, *, status=None, cursor=None, limit=200, st=None): + """C4: every proposal and decided App item, newest day first, paged by an opaque cursor. limit=None returns every row.""" + st = _rt(tenant_key, st) + rows = [p for p in _all_proposals(st) + _decided_app_rows(st) if status is None or p.get("status") == status] + # Newest day first, then id ascending: the sort key is (inverted day, id). + rows.sort(key=lambda p: (_inv(p.get("day") or ""), p["id"])) + if cursor: + day, pid = _decode(cursor) + after = (_inv(day), pid) + rows = [p for p in rows if (_inv(p.get("day") or ""), p["id"]) > after] + if limit is not None: + limit = int(limit) + if limit < 1: + raise ValueError("The page size must be at least 1.") + page = rows if limit is None else rows[:limit] + more = limit is not None and len(rows) > limit + return {"items": [_app_item(p) if p.get("kind") == APP_KIND else _item(p) for p in page], + "cursor": _encode((page[-1].get("day") or "", page[-1]["id"])) if more else None} + + +def _inv(day): + """A string key that sorts days newest first.""" + return "".join(chr(0x10FFFF - ord(c)) for c in day) + + +def proposals(tenant_key, *, st=None): + """C4: open proposals for the governed inbox.""" + st = _rt(tenant_key, st) + rows = [p for p in _all_proposals(st) if p.get("status") in OPEN] + rows.sort(key=lambda p: (_inv(p.get("day") or ""), p["id"])) + return [{"source_kind": SOURCE_KIND, "item_id": p["id"], "sku": p.get("sku") or None, + "action": p.get("action"), "reason": p.get("reason")} for p in rows] + + +def today(tenant_key, *, st=None, now=None): + """The latest built day as the App shows it: every proposal, held notes, settings and the App inbox.""" + st = _rt(tenant_key, st) + _safe_pull(tenant_key, st=st, now=now) + rec = day_record(tenant_key, st=st) or {} + conf = settings(tenant_key, st=st) + rows = sorted(((rec.get("items") or {}).values()), + key=lambda p: (p.get("status") not in OPEN, p.get("ad_type") or "", p.get("campaign") or "", + p["id"])) + return {"day": rec.get("day") or _doc(st, KEY_DAYS).get("latest"), "dataDate": rec.get("data_date"), + "builtAt": rec.get("built_at"), "items": [_item(p) for p in rows], "held": list(rec.get("held") or []), + "spend": dict(rec.get("spend") or {}), "appItems": app_items_today(st, now=now), + "settings": {k: conf[k] for k in DEFAULTS} | {"defaultOwner": conf["defaultOwner"]}} + + +# --------------------------------------------------------------------------- decisions and named writes + +def _ads_client(): + """The one seam to the Ads client that sends the three named writes, or None when it cannot import.""" + import importlib + try: + client = importlib.import_module("harness.amazon_ads_client") + except Exception: # noqa: BLE001 + return None + return client if callable(getattr(client, "send_named_write", None)) else None + + +def _plain(text): + """One line of plain text with every em and en dash rewritten as a comma.""" + out = " ".join(str(text or "").split()) + for dash in (chr(0x2014), chr(0x2013)): + out = out.replace(f" {dash} ", ", ").replace(dash, ", ") + return out + + +def _cause(exc): + detail = _plain(exc).rstrip(".") + if type(exc).__name__ == "AdsWriteRefused": + return f"Amazon Ads refused this change: {detail}." if detail else "Amazon Ads refused this change." + return f"The change could not be sent: {detail}." if detail else "The change could not be sent." + + +def _json_safe(value): + try: + return json.loads(json.dumps(value, default=str)) + except (TypeError, ValueError): + return str(value) + + +def _editor(st, user): + """The deciding account's name. PermissionError for a viewer, an inactive or an unknown account.""" + name, rec = None, user + if isinstance(user, str): + name = user.strip() + try: + rec = (st.get("users") or {}).get(name) + except Exception: # noqa: BLE001 + rec = None + if not isinstance(rec, dict) or rec.get("active") is False: + raise PermissionError(READ_ONLY) + role = str(rec.get("role") or "user").strip().lower() + if role == "viewer" or rec.get("can_edit") is False or rec.get("canEdit") is False: + raise PermissionError(READ_ONLY) + return str(rec.get("username") or name or rec.get("name") or "unknown") + + +def _ids(item_id): + raw = [item_id] if isinstance(item_id, str) else list(item_id or ()) + out = [] + for pid in raw: + pid = str(pid or "").strip() + if pid and pid not in out: + out.append(pid) + if not out: + raise ValueError("Pick at least one proposal.") + return out + + +def _skip_cause(status, decision): + verb = "sent" if decision == "approved" else "dismissed" + return f"This proposal is already {status}. Nothing was {verb}." + + +def _put_decision(st, record): + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.setdefault("decisions", {})[record["id"]] = record + return doc + st.update(KEY_DECISIONS, _apply) + + +def _claim(st, ids, status, decision_id, who, at, note): + """Move each open proposal in ids to status in one store update. Returns the ids it moved.""" + want, moved = set(ids), [] + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + moved.clear() + for day in (doc.get("days") or {}).values(): + for pid, prop in ((day or {}).get("items") or {}).items(): + if pid in want and isinstance(prop, dict) and prop.get("status") in OPEN: + prop.update(status=status, decision_id=decision_id, decided_by=who, decided_at=at, note=note) + moved.append(pid) + return doc + st.update(KEY_DAYS, _apply) + return [pid for pid in ids if pid in set(moved)] + + +def _record_outcome(st, pid, decision_id, kind, status, cause, result, payload, at): + def _row(doc): + doc = doc if isinstance(doc, dict) else {} + for day in (doc.get("days") or {}).values(): + prop = ((day or {}).get("items") or {}).get(pid) + if isinstance(prop, dict): + prop.update(status=status, cause=cause, outcome_at=at, result=result, sent_payload=payload) + return doc + + def _log(doc): + doc = doc if isinstance(doc, dict) else {} + doc.setdefault("items", {})[pid] = {"status": status, "cause": cause, "decision_id": decision_id, + "kind": kind, "at": at} + return doc + st.update(KEY_DAYS, _row) + st.update(KEY_OUTCOMES, _log) + + +def set_status(tenant_key, user, item_id, status, *, note=None, st=None, now=None): + """C4: decide one proposal or a list. `approved` is one batch: an approval record, then one named write each. + + PermissionError for a viewer, before any store change. ValueError on an unknown status. Only a + `proposed` row moves. A refused write records its cause on its row and the rest of the batch goes on. + Ads proposal ids and App inbox ids may share one call. An inbox id goes back to its App's on_decision. + """ + st = _rt(tenant_key, st) + who = _editor(st, user) + status = str(status or "").strip().lower() + if status not in STATUSES: + raise ValueError(f"Unknown status: {status or 'blank'}. Use approved or dismissed.") + if status not in DECISIONS: + raise ValueError("Only approved or dismissed can be set. Sent and refused come from the Ads write.") + ids = _ids(item_id) + note = (_plain(note) or None) if note is not None else None + if status == "dismissed" and not note: + raise ValueError("Give a reason to dismiss.") + at = _stamp(now or _now()) + inbox = _inbox(st) + app_ids = [pid for pid in ids if pid in inbox] + app_rows, app_decision = (_decide_apps(st, tenant_key, app_ids, inbox, status, note, who, at) + if app_ids else ({}, None)) + ad_ids = [pid for pid in ids if pid not in inbox] + found = {p["id"]: p for p in _all_proposals(st)} + rows = dict(app_rows) + for pid in ad_ids: + prop = found.get(pid) + if prop is None: + rows[pid] = {"id": pid, "status": "not_found", "cause": NOT_FOUND} + elif prop.get("status") not in OPEN: + rows[pid] = {"id": pid, "status": "skipped", "current": prop.get("status"), + "cause": _skip_cause(prop.get("status"), status)} + eligible = [pid for pid in ad_ids if pid not in rows] + decision_id = None + if eligible: + decision_id = ("apv_" if status == "approved" else "dsm_") + uuid.uuid4().hex[:16] + record = {"id": decision_id, "decision": status, "by": who, "at": at, "item_ids": eligible, "note": note} + if status == "approved": + record["writes"] = [{"item_id": pid, "kind": found[pid]["kind"], "payload": _write_payload(found[pid])} + for pid in eligible] + # The record is stored before any row moves, so a write never names an approval that is not there. + _put_decision(st, record) + claimed = _claim(st, eligible, status, decision_id, who, at, note) + if claimed != eligible: + for pid in eligible: + if pid not in claimed: + rows[pid] = {"id": pid, "status": "skipped", "current": "decided", + "cause": _skip_cause("decided", status)} + record["item_ids"] = claimed + if "writes" in record: + record["writes"] = [w for w in record["writes"] if w["item_id"] in claimed] + _put_decision(st, record) + if status == "dismissed": + for pid in claimed: + rows[pid] = {"id": pid, "status": "dismissed"} + else: + client = _ads_client() + for pid in claimed: + rows[pid] = _send(st, tenant_key, client, found[pid], decision_id) + out = [rows[pid] for pid in ids] + count = {s: sum(1 for r in out if r["status"] == s) + for s in ("sent", "refused", "dismissed", "skipped", "approved")} + return {"decision": status, "decision_id": decision_id or app_decision, + "approval_id": decision_id if status == "approved" else None, "rows": out, + **count, "not_found": sum(1 for r in out if r["status"] == "not_found")} + + +def _write_payload(prop): + """What send_named_write gets: the proposal's payload plus its item id, so the client can match the approval.""" + return {**(prop.get("payload") or {}), "item_id": prop["id"]} + + +def _send(st, tenant_key, client, prop, approval_id): + """One named write for one approved proposal, its outcome stored on its row. Never raises.""" + payload = _write_payload(prop) + result = None + if client is None: + status, cause = "refused", NOT_INSTALLED + else: + try: + result = _json_safe(client.send_named_write(tenant_key, prop["kind"], payload, + approval_id=approval_id)) + except Exception as exc: # noqa: BLE001 + status, cause = "refused", _cause(exc) + else: + status, cause = "sent", None + at = _stamp(_now()) + try: + _record_outcome(st, prop["id"], approval_id, prop["kind"], status, cause, result, + payload if status == "sent" else None, at) + except Exception as exc: # noqa: BLE001 + _log.warning("amazon_operator: the outcome of %s could not be stored (%s): %s", + prop["id"], type(exc).__name__, exc) + row = {"id": prop["id"], "status": status, "kind": prop["kind"]} + if cause: + row["cause"] = cause + return row + + +def approval(tenant_key, approval_id, *, st=None): + """The stored approval record for approval_id, or None. The Ads client checks an id here before a write.""" + rec = (_doc(_rt(tenant_key, st), KEY_DECISIONS).get("decisions") or {}).get(str(approval_id or "")) + # An App inbox decision is never an Ads approval. + ok = isinstance(rec, dict) and rec.get("decision") == "approved" and rec.get("source") != APP_KIND + return rec if ok else None + + +def on_decision(tenant_key, item_id, decision, reason, *, user=None, st=None): + """C4 inbox callback. It runs set_status, so it needs the deciding user and refuses without one.""" + return set_status(tenant_key, user, item_id, decision, note=reason, st=st) + + +# --------------------------------------------------------------------------- C7 App inbox + +def _short(name): + return str(name).rsplit(".", 1)[-1] + + +def _inbox_id(source_kind, item_id): + """Stable inbox id: a second pull of the same App item yields the same id, so it never duplicates.""" + return "a" + hashlib.sha1(f"{source_kind}|{item_id}".encode("utf-8")).hexdigest()[:16] + + +def _inbox(st): + items = _doc(st, KEY_INBOX).get("items") + return {k: v for k, v in items.items() if isinstance(v, dict)} if isinstance(items, dict) else {} + + +def _load_source(name): + """(module, False), or (None, True) for a module that is not installed, or (None, False) for a failed import. + + Each failure logs one line. Sources are imported here only, by name, so deleting an App leaves this module whole. + """ + import importlib + try: + return importlib.import_module(name), False + except ModuleNotFoundError as exc: + if exc.name == name: + _log.info("amazon_operator: %s is not installed; its proposals are skipped", name) + return None, True + _log.warning("amazon_operator: %s could not be imported (%s); its proposals are skipped", name, exc) + except Exception as exc: # noqa: BLE001 + _log.warning("amazon_operator: %s could not be imported (%s: %s); its proposals are skipped", + name, type(exc).__name__, exc) + return None, False + + +def _label(mod, name): + label = getattr(mod, "APP_LABEL", None) + if isinstance(label, str) and label.strip(): + return _plain(label) + short = _short(name) + return APP_LABELS.get(short) or short.replace("_", " ").capitalize() + + +def _read_sources(tenant_key): + """{module name: (label, proposals) or None}. None marks a missing module. A failed read is left out.""" + out = {} + for name in PROPOSAL_SOURCES: + mod, missing = _load_source(name) + if mod is None: + if missing: + out[name] = None + continue + reader = getattr(mod, "proposals", None) + if not callable(reader): + _log.warning("amazon_operator: %s has no proposals(); it is skipped", name) + continue + try: + got = list(reader(tenant_key) or ()) + except Exception as exc: # noqa: BLE001 + _log.warning("amazon_operator: %s proposals() failed for %s (%s: %s); it is skipped", + name, tenant_key, type(exc).__name__, exc) + continue + out[name] = (_label(mod, name), got) + return out + + +def _merge_inbox(doc, read, at): + """The inbox after one pull. Pure. Decided items never change; an unlisted open item becomes stale.""" + doc = doc if isinstance(doc, dict) else {} + items = doc.get("items") if isinstance(doc.get("items"), dict) else {} + seen = set() + for name, got in read.items(): + if got is None: + continue + label, rows = got + for raw in rows: + if not isinstance(raw, dict) or not str(raw.get("item_id") or "").strip(): + continue + item_id = str(raw["item_id"]).strip() + kind = str(raw.get("source_kind") or _short(name)).strip() + iid = _inbox_id(kind, item_id) + seen.add(iid) + fields = {"source_kind": kind, "source_module": name, "item_id": item_id, + "sku": _plain(raw.get("sku")) or None, "action": _plain(raw.get("action")), + "reason": _plain(raw.get("reason")), "app": label} + old = items.get(iid) + if not isinstance(old, dict): + items[iid] = {"id": iid, "kind": APP_KIND, **fields, "status": "proposed", "cause": None, + "first_seen": at, "last_seen": at} + continue + if old.get("status") not in INBOX_OPEN + ("stale",): + continue + new = {**old, **fields, "status": "proposed", + "cause": old.get("cause") if old.get("status") in INBOX_OPEN else None} + if new != old: + items[iid] = {**new, "last_seen": at} + for iid, rec in list(items.items()): + name = rec.get("source_module") if isinstance(rec, dict) else None + # A missing module (None) or a read that no longer lists the item. A failed read is not in `read`. + if iid in seen or name not in read or rec.get("status") not in INBOX_OPEN: + continue + items[iid] = {**rec, "status": "stale", "cause": STALE, "last_seen": at} + doc["items"] = items + return doc + + +def pull_app_proposals(tenant_key, *, st=None, now=None): + """C7: read every source App's open proposals into the inbox. Idempotent, and it writes only on a change.""" + st = _rt(tenant_key, st) + read = _read_sources(tenant_key) + at = _stamp(now or _now()) + current = st.get(KEY_INBOX) + current = current if isinstance(current, dict) else {} + merged = _merge_inbox(copy.deepcopy(current), read, at) + changed = merged != current and bool(current or merged.get("items")) + if changed: + st.update(KEY_INBOX, lambda doc: _merge_inbox(doc, read, at)) + inbox = _inbox(st) + return {"changed": changed, "read": sorted(n for n, g in read.items() if g is not None), + "missing": sorted(n for n, g in read.items() if g is None), + "open": sum(1 for r in inbox.values() if r.get("status") in INBOX_OPEN), + "stale": sum(1 for r in inbox.values() if r.get("status") == "stale")} + + +def _safe_pull(tenant_key, *, st=None, now=None): + try: + return pull_app_proposals(tenant_key, st=st, now=now) + except Exception as exc: # noqa: BLE001 + _log.warning("amazon_operator: the App inbox pull for %s failed (%s): %s", + tenant_key, type(exc).__name__, exc) + return None + + +def _app_item(rec): + """One inbox item as the App and History show it.""" + return {"id": rec["id"], "kind": APP_KIND, "title": f"{rec.get('app')}: {rec.get('action')}", + "app": rec.get("app"), "source_kind": rec.get("source_kind"), "source_module": rec.get("source_module"), + "item_id": rec.get("item_id"), "status": rec.get("status"), "owner": rec.get("decided_by"), + "due": None, "reason": rec.get("reason"), "sku": rec.get("sku") or None, "asin": None, + "action": rec.get("action"), "change": rec.get("action"), "cause": rec.get("cause"), + "note": rec.get("note"), "decision_id": rec.get("decision_id"), "decided_by": rec.get("decided_by"), + "decided_at": rec.get("decided_at"), "day": rec.get("day") or str(rec.get("decided_at") or "")[:10], + "first_seen": rec.get("first_seen"), "last_seen": rec.get("last_seen")} + + +def _decided_app_rows(st): + return [{**rec, "kind": APP_KIND, "day": str(rec.get("decided_at") or "")[:10]} + for rec in _inbox(st).values() if rec.get("status") in DECISIONS] + + +def app_items_today(st, *, now=None): + """Open and stale inbox items, plus those decided today. Uncapped (rule 1).""" + day = (now or _now()).date().isoformat() + order = {"proposed": 0, "stale": 1} + rows = [r for r in _inbox(st).values() + if r.get("status") in INBOX_OPEN + ("stale",) or str(r.get("decided_at") or "")[:10] == day] + rows.sort(key=lambda r: (order.get(r.get("status"), 2), r.get("app") or "", r.get("sku") or "", r["id"])) + return [_app_item(r) for r in rows] + + +def _hand_back(tenant_key, rec, decision, reason): + """Call the source App's on_decision. None on success, else the plain cause.""" + name, app = rec.get("source_module") or "", rec.get("app") or "Its App" + if name not in PROPOSAL_SOURCES: + return f"{app} is not an inbox source, so it could not take this decision." + mod, _missing = _load_source(name) + fn = getattr(mod, "on_decision", None) if mod is not None else None + if not callable(fn): + return f"{app} is not installed, so it could not take this decision." + try: + fn(tenant_key, rec["item_id"], decision, reason) + except Exception as exc: # noqa: BLE001 + detail = _plain(exc).rstrip(".") + return f"{app} could not take this decision: {detail}." if detail else f"{app} could not take this decision." + return None + + +def _decide_apps(st, tenant_key, ids, inbox, status, note, who, at): + """Decide inbox items: claim them in one update, then hand each decision back to its App. Never writes outbound.""" + rows, eligible = {}, [] + for iid in ids: + current = inbox[iid].get("status") + if current == "stale" and status == "approved": + rows[iid] = {"id": iid, "status": "skipped", "current": "stale", + "cause": f"{STALE} It cannot be approved."} + elif current not in INBOX_OPEN + ("stale",): + rows[iid] = {"id": iid, "status": "skipped", "current": current, "cause": _skip_cause(current, status)} + else: + eligible.append(iid) + if not eligible: + return rows, None + decision_id = ("apd_" if status == "approved" else "dsm_") + uuid.uuid4().hex[:16] + record = {"id": decision_id, "decision": status, "source": APP_KIND, "by": who, "at": at, + "item_ids": eligible, "note": note} + _put_decision(st, record) + claimed = {} + + def _claim_apps(doc): + doc = doc if isinstance(doc, dict) else {} + claimed.clear() + for iid in eligible: + rec = (doc.get("items") or {}).get(iid) + if isinstance(rec, dict) and rec.get("status") in INBOX_OPEN + ("stale",): + claimed[iid] = rec["status"] + rec.update(status=status, decision_id=decision_id, decided_by=who, decided_at=at, note=note, + cause=None if rec["status"] in INBOX_OPEN else rec.get("cause")) + return doc + st.update(KEY_INBOX, _claim_apps) + done = [] + for iid in eligible: + if iid not in claimed: + rows[iid] = {"id": iid, "status": "skipped", "current": "decided", "cause": _skip_cause("decided", status)} + continue + rec = inbox[iid] + # A stale item is gone from its App, so dismissing it closes it here only. + cause = None if claimed[iid] == "stale" else _hand_back( + tenant_key, rec, status, note or (APPROVED_NOTE if status == "approved" else "")) + if cause: + _reopen(st, iid, cause) + rows[iid] = {"id": iid, "status": "refused", "cause": cause, "app": rec.get("app")} + else: + done.append(iid) + rows[iid] = {"id": iid, "status": status, "app": rec.get("app")} + if done != eligible: + record["item_ids"] = done + _put_decision(st, record) + return rows, decision_id + + +def _reopen(st, iid, cause): + """Put an inbox item back to proposed with the cause its App gave.""" + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + rec = (doc.get("items") or {}).get(iid) + if isinstance(rec, dict): + rec.update(status="proposed", cause=cause, decision_id=None, decided_by=None, decided_at=None, note=None) + return doc + st.update(KEY_INBOX, _apply) + + +def validate(tenant_key, day=None, *, st=None, **_period): + """C4: a built day's evaluated spend ties to SUM(cost) over the three advertised products tables.""" + st = _rt(tenant_key, st) + rec = day_record(tenant_key, day, st=st) + if not rec: + return {"ok": False, "app_total": None, "independent_total": None, + "detail": "No proposals are built for this day."} + data_date = rec.get("data_date") + app_total = round(sum(float(v or 0) for v in (rec.get("spend") or {}).values()), 2) + independent, by_type = 0.0, {} + with _reader(tenant_key, st) as cur: + if cur is None: + return {"ok": False, "app_total": app_total, "independent_total": None, + "detail": "The tenant mirror is not there, so the Ads cost cannot be read."} + for ad_type in ALL_AD_TYPES: + table = _ads_table(ad_type, "advertised_products") + if not {"date", "cost"} <= _columns(cur, table): + continue + row = cur.execute(f'SELECT SUM(TRY_CAST("cost" AS DOUBLE)) FROM {table} ' + f"WHERE TRY_CAST(\"date\" AS DATE) = CAST(? AS DATE)", [data_date]).fetchone() + by_type[ad_type] = round(float(row[0] or 0), 2) + independent += float(row[0] or 0) + independent = round(independent, 2) + ok = abs(app_total - independent) <= 0.01 + detail = (f"Ad spend on {data_date}: proposals read {_money(app_total)}, the Ads tables hold " + f"{_money(independent)}.") + if not ok: + missed = [AD_LABELS[t] for t in ALL_AD_TYPES + if abs(by_type.get(t, 0.0) - float((rec.get("spend") or {}).get(t, 0.0))) > 0.01] + detail += " They do not tie" + (": " + ", ".join(missed) + " differ." if missed else ".") + return {"ok": ok, "app_total": app_total, "independent_total": independent, "detail": detail, + "by_type": by_type} + + +def remove(tenant_key, *, st=None): + """Delete every document in KEYS and nothing else. Connector tables and the mirror stay (rule 13).""" + st = _rt(tenant_key, st) + for key in KEYS: + deleter = getattr(st, "delete_document", None) + if callable(deleter): + deleter(key) + else: + st.update(key, lambda _doc: {}) + return list(KEYS) diff --git a/platform/modules/ar.py b/platform/modules/ar.py index f020dd547a7cafa05d6bf45af8d8f4e278688466..cada1c5710b28bd583503a8371010e55d7a15a09 100644 --- a/platform/modules/ar.py +++ b/platform/modules/ar.py @@ -115,6 +115,120 @@ def _bucket(days): return '90+' +#: The payment term Odoo names for new accounts that must pay before shipping. It ages as due on +#: the invoice date, so no grace period ever delays it. +NCH_TERM_NAME = 'New Customer Hold' + +#: Odoo's Aged Receivable population: posted, unreconciled lines on trade receivable accounts. +#: Credit notes and unapplied payments sit in it as negative residuals and net per partner. +AGED_LINE_DOMAIN = [('account_id.account_type', '=', 'asset_receivable'), + ('account_id.non_trade', '=', False), + ('parent_state', '=', 'posted'), ('reconciled', '=', False)] + +#: The Overdue definition's name, shared by the field, the metric and the validate() check. +OVERDUE_DEFINITION = 'Odoo Aged Receivable overdue' + + +def _aged_domain(t, pids=None): + dom = AGED_LINE_DOMAIN + [('date', '<=', str(t))] + if pids is not None: + dom.append(('partner_id', 'in', sorted(pids))) + return dom + + +def aged_receivable(t=None, *, pids=None, grace_days=0): + """Per-partner Overdue on Odoo's Aged Receivable basis: line due date, no grace, every debtor. + + A line is overdue when its due date is before `t`, as in the report's "At Date" column. + A New Customer Hold line is due on its invoice date. `grace_days` exists only for the negative + control. Residuals are current, so the as-of semantics hold for `t` = today. + """ + today = t or P.today() + dom = _aged_domain(today, pids) + lines = O.search_read('account.move.line', dom, + ['partner_id', 'date', 'date_maturity', 'amount_residual', + 'invoice_date']) + nch = {r['id'] for r in O.search_read( + 'account.move.line', dom + [('move_id.invoice_payment_term_id.name', '=', NCH_TERM_NAME)], + ['id'])} + per, unkeyed, nch_shift, n_lines = {}, 0.0, 0.0, 0 + for ln in lines: + n_lines += 1 + due = str(ln.get('date_maturity') or ln.get('date') or '')[:10] + amount = ln.get('amount_residual') or 0.0 + if ln.get('id') in nch and ln.get('invoice_date'): + held = str(ln['invoice_date'])[:10] + if held < due: + nch_shift += amount + due = held + try: + days = (today - dt.date.fromisoformat(due)).days if due else 0 + except ValueError: + days = 0 + pid = O.m2o_id(ln.get('partner_id')) + if pid is None: + unkeyed += amount + continue + e = per.setdefault(pid, {'pid': pid, 'current': 0.0, 'overdue': 0.0, 'total': 0.0, + **{f'aged_{b}': 0.0 for b in OVERDUE_BUCKETS}}) + e['total'] += amount + if days > grace_days: + e['overdue'] += amount + e[f'aged_{_bucket(days)}'] += amount + else: + e['current'] += amount + return {'date': str(today), 'grace_days': grace_days, 'rows': per, 'unkeyed': unkeyed, + 'nch_lines': len(nch), 'nch_shift': nch_shift, 'lines': n_lines, + 'pids': None if pids is None else sorted(pids)} + + +def overdue_tie(aged, tol=0.01): + """The Overdue tie: the per-partner field total against Odoo's own past-due residual sum. + + The independent side is a server-side read_group over the report's line domain. The report + engine itself answers only over JSON-RPC (its payload carries nulls XML-RPC cannot marshal). + """ + today = dt.date.fromisoformat(aged['date']) + theirs = O.sum_field('account.move.line', + _aged_domain(today, aged['pids']) + + ['|', ('date_maturity', '<', str(today)), + '&', ('date_maturity', '=', False), ('date', '<', str(today))], + 'amount_residual') + ours = sum(r['overdue'] for r in aged['rows'].values()) + gap = ours - theirs + return {'check': f"Overdue ({OVERDUE_DEFINITION}): field total == Odoo receivable lines past " + f"due on {aged['date']}, grace {aged['grace_days']}", + 'a': round(ours, 2), 'b': round(theirs, 2), 'gap': round(gap, 2), + 'ok': abs(gap) <= tol, 'date': aged['date'], + 'detail': {'unkeyed': round(aged['unkeyed'], 2), 'nch_lines': aged['nch_lines'], + 'nch_shift': round(aged['nch_shift'], 2), 'partners': len(aged['rows'])}} + + +def overdue_checks(t=None, aged=None): + """The Overdue rows validate() records: the tie, its grace negative control, the split, the total.""" + today = t or P.today() + aged = aged or aged_receivable(today) + tie = overdue_tie(aged) + nc = overdue_tie(aged_receivable(today, pids=aged['pids'], grace_days=5)) + rows = list(aged['rows'].values()) + overdue = sum(r['overdue'] for r in rows) + split = sum(r[f'aged_{b}'] for r in rows for b in OVERDUE_BUCKETS) + ours_total = sum(r['total'] for r in rows) + aged['unkeyed'] + theirs_total = O.sum_field('account.move.line', _aged_domain(today, aged['pids']), + 'amount_residual') + return [ + tie, + {'check': 'NC: Overdue with the 5-day grace restored FAILS the Odoo tie (ok = the control fired)', + 'a': nc['a'], 'b': nc['b'], 'gap': nc['gap'], 'ok': not nc['ok']}, + {'check': 'Overdue split: sum(1-30, 31-60, 61-90, 90+) == Overdue', + 'a': round(split, 2), 'b': round(overdue, 2), 'gap': round(split - overdue, 2), + 'ok': abs(split - overdue) <= 0.01}, + {'check': 'Overdue basis: current + overdue + unkeyed == Odoo receivable lines total', + 'a': round(ours_total, 2), 'b': round(theirs_total, 2), + 'gap': round(ours_total - theirs_total, 2), 'ok': abs(ours_total - theirs_total) <= 0.01}, + ] + + def _open_docs(t=None): o = O.get_odoo() dom = [('move_type', 'in', ['out_invoice', 'out_refund']), ('state', '=', 'posted'), @@ -154,9 +268,9 @@ def _ltm_invoiced(t=None): def summary(t=None): + # Overdue is `aged_receivable` (C10); an invoice-grain past-due sum here would disagree with it. docs = _open_docs(t) total_ar = sum(d['open'] for d in docs) - overdue = sum(d['open'] for d in docs if d['days_overdue'] > 0) ltm_sales = _ltm_invoiced(t) dso = (total_ar / (ltm_sales / 365.0)) if ltm_sales else None # reconciliation gap vs Odoo partner receivable (company-level) @@ -166,8 +280,6 @@ def summary(t=None): n_customers = len({O.m2o_id(d['partner_id']) for d in docs}) return { 'total_ar_open': total_ar, - 'overdue': overdue, - 'pct_overdue': (overdue / total_ar * 100) if total_ar else 0, 'dso_days': dso, 'ltm_invoiced': ltm_sales, 'open_customers': n_customers, @@ -613,4 +725,4 @@ def validate(t=None, exposure=None, behavior=None): 'a': dtp['violations'], 'b': 0, 'gap': dtp['violations'], 'ok': dtp['violations'] == 0 and dtp['n_customers'] > 0}) - return checks + return checks + overdue_checks(t) diff --git a/platform/modules/collections_exec.py b/platform/modules/collections_exec.py index 64c799cb5e3176cb47e5bae16ff4f62c7ff8de18..b717731641e9af607e25bfbaf836d3bb90540851 100644 --- a/platform/modules/collections_exec.py +++ b/platform/modules/collections_exec.py @@ -346,6 +346,75 @@ def overdue_list(data, team, t=None): return rows +# ---------------------------------------------------------------- Odoo Aged Receivable oracle +#: The Aged Receivable line domain (modules/ar.py AGED_LINE_DOMAIN), restated so this oracle +#: stays independent of the field it checks. +AGED_LINES = [('account_id.account_type', '=', 'asset_receivable'), + ('account_id.non_trade', '=', False), + ('parent_state', '=', 'posted'), ('reconciled', '=', False)] +#: Report columns as due-date day ranges past `t`: (key, newest days, oldest days or None). +AGED_COLUMNS = (('current', None, 0), ('1_30', 1, 30), ('31_60', 31, 60), ('61_90', 61, 90), + ('90_plus', 91, None)) + + +def _due_window(t, newest, oldest): + """Domain for lines due `newest`..`oldest` days before `t`; a line with no due date uses its date.""" + def on(field): + dom = [(field, '!=', False)] if field == 'date_maturity' else [] + if newest is not None: + dom.append((field, '<=', str(t - dt.timedelta(days=newest)))) + if oldest is not None: + dom.append((field, '>=', str(t - dt.timedelta(days=oldest)))) + return dom + due, plain = on('date_maturity'), [('date_maturity', '=', False)] + on('date') + return ['|'] + ['&'] * (len(due) - 1) + due + ['&'] * (len(plain) - 1) + plain + + +def odoo_aging_by_team(t=None, teams=(FFS, RI)): + """Odoo's Aged Receivable per customer sales team, summed server side: {team: {column: $}}. + + The independent side of the Collection report tie. `total` is every open line; `overdue` is + every line past due on `t` (no grace, R37). Read only. + """ + t = t or P.today() + base = AGED_LINES + [('date', '<=', str(t))] + out = {} + for team in teams: + dom = base + [('partner_id.team_id', '=', team)] + row = {'total': O.sum_field('account.move.line', dom, 'amount_residual')} + for key, newest, oldest in AGED_COLUMNS: + row[key] = O.sum_field('account.move.line', dom + _due_window(t, newest, oldest), + 'amount_residual') + row['overdue'] = O.sum_field('account.move.line', dom + _due_window(t, 1, None), 'amount_residual') + out[team] = row + return out + + +def _by_partner(team, t, window): + dom = AGED_LINES + [('date', '<=', str(t)), ('partner_id.team_id', '=', team)] + window + groups = O.read_group('account.move.line', dom, ['amount_residual:sum'], ['partner_id'], lazy=False) + return {O.m2o_id(g['partner_id']): g.get('amount_residual') or 0.0 + for g in groups if g.get('partner_id')} + + +def odoo_overdue_by_partner(team, t=None): + """{partner id: overdue $} for one customer sales team, from one server-side read_group.""" + t = t or P.today() + return _by_partner(team, t, _due_window(t, 1, None)) + + +def odoo_open_by_partner(team, t=None): + """{partner id: open receivable $} for one customer sales team.""" + return _by_partner(team, t or P.today(), []) + + +def odoo_terms(pids): + """{partner id: payment term name} from res.partner, archived partners included.""" + rows = O.search_read('res.partner', [('id', 'in', sorted(pids)), ('active', 'in', [True, False])], + ['property_payment_term_id']) if pids else [] + return {r['id']: O.m2o_name(r.get('property_payment_term_id')) or '' for r in rows} + + def goal_status(key, value): """('goal', 'Achieved'|'Unmet') for a metric vs the report's hardcoded target, or (None, None).""" g = GOALS.get(key) diff --git a/platform/modules/customer_data.py b/platform/modules/customer_data.py index 7d13fbadbb0e76c34bb0a1269a4e8453046ba9b8..fc58c5d7feb38fd263d5448a11cef0624c3c2f53 100644 --- a/platform/modules/customer_data.py +++ b/platform/modules/customer_data.py @@ -116,50 +116,71 @@ def agent_pids(agent_name): AR_BUCKET_FIELDS = {'1-30': 'ar_aged_1_30', '31-60': 'ar_aged_31_60', '61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'} +#: `_ar_attrs` sets this key when the Overdue tie failed or could not run (standing rule 8). +AR_WITHHELD = '_overdue_withheld' +#: The Overdue family. One tie covers all of them, so a failed tie withholds all of them. +AR_OVERDUE_FIELDS = ('ar_open', 'ar_overdue', 'ar_outstanding', *AR_BUCKET_FIELDS.values()) + + +def ar_cell(ar_a, pid, key): + """One AR cell for a pool row: None while a failed Overdue tie withholds its family.""" + if key in AR_OVERDUE_FIELDS and ar_a.get(AR_WITHHELD): + return None + return (ar_a.get(pid) or {}).get(key, 0.0) + + +def _ar_withheld(exc): + """Log a withheld Overdue family loudly: a blank column must say why it is blank.""" + try: + import harness.telemetry as _tel + _tel.error('customer_data:_ar_attrs', exc, fallback='Overdue family withheld') + except Exception: + pass + print(f"[aios] WARNING customer_data._ar_attrs withheld the Overdue family " + f"({type(exc).__name__}: {exc})") + def _ar_attrs(t=None, *, pids=None, st=None): - """{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family. - - Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" — - key kept so saved views keep working); `ar_outstanding` is the total (current + overdue), - which is what "open AR" means everywhere else. - - Composed from `modules/ar.credit_exposure()`, NOT re-derived: it is already reconciled to - Odoo by `ar.validate()`, so the Customer table and the Collections page cannot disagree about - what a customer owes. Read from the UNFILTERED `_all_rows` — `rows` is a display-truncated - top-40, and a column built off that would silently blank everybody else. - - ⚠ `credit_limit` is deliberately NOT a column: 20 of 1,548 customers have one. EXPOSURE is - the number that answers "how much are we out on this customer", and it needs no limit. - - ⚠ `days_to_pay` is NOT COMPUTED here — it is READ from a nightly snapshot. MEASURED - 2026-07-27: `ar.credit_exposure()` costs 9s; `ar.days_to_pay()` costs **301s**, because - settlement date means walking three years of reconciled receivable lines and their - full-reconcile groups. Computing it in `pool()` would add five minutes to every Customer-table - build and every Space container start; computing it LAZILY would just move those five minutes - onto whoever opened the page first. So the expensive half runs on the app's existing - background store-sync thread (`ar.refresh_days_to_pay_snapshot`) and this reads the answer - out of the store for nothing. A missing snapshot yields a blank column, never a slow page. + """{pid: AR family}: Overdue on Odoo's Aged Receivable basis, plus the 5-day grace number. + + R37: `ar_overdue` ("Overdue") is `ar.aged_receivable()`: receivable lines aged by due date + with no grace, New Customer Hold due at once, credits and unapplied payments netted per + partner, every debtor. `ar_open` (not yet due), `ar_outstanding` (total) and the four + `ar_aged_*` buckets are slices of the same lines, so they sum to Odoo's report columns. + The family ships only when `ar.overdue_tie()` holds. Else every cell in it is None. + + `ar_overdue_past_grace` ("Overdue past grace") and `ar_exposure` stay on + `ar.credit_exposure()`, the collections basis with its 5-day grace. + + `days_to_pay` is read from the nightly snapshot, never computed here (about 300 s). """ out = {} + try: + aged = ar.aged_receivable(t, pids=pids) + tie = ar.overdue_tie(aged) + if not tie['ok']: + raise ArithmeticError(f"{tie['check']}: field {tie['a']} vs Odoo {tie['b']}, " + f"gap {tie['gap']}, {tie['detail']}") + for pid, r in aged['rows'].items(): + out[pid] = { + 'ar_open': r['current'], 'ar_overdue': r['overdue'], 'ar_outstanding': r['total'], + **{k: r[f'aged_{b}'] for b, k in AR_BUCKET_FIELDS.items()}, + } + except Exception as e: + out = {AR_WITHHELD: True} + _ar_withheld(e) try: exp = ar.credit_exposure(t) for r in exp.get('_all_rows') or []: - out[r['pid']] = { - 'ar_open': r.get('open', 0.0) or 0.0, - 'ar_overdue': r.get('overdue', 0.0) or 0.0, - # Wave 21 R1 — the TOTAL owed. Sum of the two halves ar.credit_exposure already - # reconciles to Odoo's residual read_group (ar.py validate), so no second oracle. - 'ar_outstanding': (r.get('open', 0.0) or 0.0) + (r.get('overdue', 0.0) or 0.0), - 'ar_exposure': r.get('exposure', 0.0) or 0.0, - # Wave 17 R2 — the aging split, so a saved VIEW can be the collections - # worklist. Keys mirror `ar.OVERDUE_BUCKETS` through AR_BUCKET_FIELDS; the four - # sum EXACTLY to `ar_overdue` (gated in validate()). - **{k: r.get(f'aged_{b}', 0.0) or 0.0 - for b, k in AR_BUCKET_FIELDS.items()}, - } - except Exception: - pass + row = out.setdefault(r['pid'], {}) + row['ar_overdue_past_grace'] = r.get('overdue', 0.0) or 0.0 + row['ar_exposure'] = r.get('exposure', 0.0) or 0.0 + except Exception as e: + try: + import harness.telemetry as _tel + _tel.error('customer_data:_ar_attrs', e, fallback='exposure columns zero') + except Exception: + pass snapshot = (ar.days_to_pay_snapshot() if pids is None and st is None else ar.days_to_pay_snapshot(pids=pids, st=st)) for pid, avg in snapshot.items(): @@ -473,7 +494,25 @@ def validate(team_id=None): 'ours': sum(ws.values()), 'theirs': 0, 'ok': not ws, 'detail': {'per_field': ws, 'fields_checked': text_keys}, }) - return checks + validate_audit_attrs(rows, odoo) + validate_invoice_cadence(rows, odoo) + return (checks + validate_audit_attrs(rows, odoo) + validate_invoice_cadence(rows, odoo) + + validate_overdue(rows)) + + +def validate_overdue(rows, t=None): + """The pool's Overdue column against Odoo's past-due receivable lines, and every debtor present.""" + aged = ar.aged_receivable(t) + tie = ar.overdue_tie(aged) + pool = {int(r['pid']) for r in rows if r.get('pid')} + missing = sorted(set(aged['rows']) - pool) + ours = sum(r.get('ar_overdue') or 0.0 for r in rows) + return [{'check': f"Customer field Overdue: sum over the pool == {ar.OVERDUE_DEFINITION} " + f"on {aged['date']}", + 'column': 'ar_overdue', 'ours': round(ours, 2), 'theirs': tie['b'], + 'gap': round(ours - tie['b'], 2), 'ok': tie['ok'] and abs(ours - tie['b']) <= 0.01, + 'detail': tie['detail']}, + {'check': 'every debtor on the Aged Receivable is a pool row keyed by partner id', + 'column': 'ar_overdue', 'ours': len(missing), 'theirs': 0, 'ok': not missing, + 'detail': {'missing': missing[:20]}}] def _dba_attrs(pids, t): @@ -521,9 +560,11 @@ def _book_records(team_id=None): return yield from O.search_read('res.partner', [('customer_rank', '>', 0), ('active', 'in', [True, False])], ['id'], limit=200000) + # The debtor leg keeps every partner on Odoo's Aged Receivable in the book (R37). for model, domain in [ ('sale.order', [('state', 'in', ('sale', 'done'))]), - ('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))])]: + ('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))]), + ('account.move.line', list(ar.AGED_LINE_DOMAIN))]: for row in O.read_group(model, domain, ['partner_id'], ['partner_id'], lazy=False): pid = O.m2o_id(row.get('partner_id')) if pid is not None: @@ -733,8 +774,8 @@ def _pool_build(agent_name, team_id, limit, fast=False, *, selected=None, needed # a worse failure than one with a blank column. wants = lambda keys: needed is None or bool(set(keys) & set(needed)) ar_a = (_ar_attrs(t, pids=all_pids, st=st) if source is not None else _ar_attrs(t)) \ - if wants(['ar_open', 'ar_overdue', 'ar_outstanding', 'ar_exposure', 'days_to_pay', - *AR_BUCKET_FIELDS.values()]) else {} + if wants(['ar_open', 'ar_overdue', 'ar_overdue_past_grace', 'ar_outstanding', + 'ar_exposure', 'days_to_pay', *AR_BUCKET_FIELDS.values()]) else {} mix_a = _mix_attrs(all_pids, t) if wants(['top_category', 'top_category_pct', 'sku_count', 'top_sku']) else {} sp_a = _salesperson_attrs(all_pids, t) if wants(['salesperson']) else {} # W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache, @@ -808,14 +849,11 @@ def _pool_build(agent_name, team_id, limit, fast=False, *, selected=None, needed # ⭐⭐ W40-T08 (owner I27) — the Google Maps link, replacing the hardcoded "Go" column. 'maps_url': maps_url(a), 'tags': a.get('tags', '(none)'), 'pricelist': a.get('pricelist', '(none)'), - # AR / credit exposure — composed from the already-reconciled ar.py blocks, so the - # Customer table and the Collections page cannot disagree. - 'ar_open': (ar_a.get(pid) or {}).get('ar_open', 0.0), - 'ar_overdue': (ar_a.get(pid) or {}).get('ar_overdue', 0.0), - 'ar_outstanding': (ar_a.get(pid) or {}).get('ar_outstanding', 0.0), - 'ar_exposure': (ar_a.get(pid) or {}).get('ar_exposure', 0.0), + # AR: the Overdue family ties to Odoo's Aged Receivable, or every cell is None. + **{k: ar_cell(ar_a, pid, k) for k in ( + 'ar_open', 'ar_overdue', 'ar_overdue_past_grace', 'ar_outstanding', + 'ar_exposure', *AR_BUCKET_FIELDS.values())}, 'days_to_pay': (ar_a.get(pid) or {}).get('days_to_pay'), - **{k: (ar_a.get(pid) or {}).get(k, 0.0) for k in AR_BUCKET_FIELDS.values()}, # Product mix, LTM, line grain, ALL-channel. 'top_category': (mix_a.get(pid) or {}).get('top_category', '(none)'), 'top_category_pct': (mix_a.get(pid) or {}).get('top_category_pct'), @@ -920,7 +958,8 @@ def _reconcile_ledger(rows, total, agent_name, team_id, *, ledger=None, write=Tr # addressless customer anyway, so the archived rows agree with the live ones. '_created': '', 'created_on': '', 'maps_url': '', 'tags': '(none)', 'pricelist': '(none)', - 'ar_open': 0.0, 'ar_overdue': 0.0, 'ar_outstanding': 0.0, 'ar_exposure': 0.0, + 'ar_open': 0.0, 'ar_overdue': 0.0, 'ar_overdue_past_grace': 0.0, + 'ar_outstanding': 0.0, 'ar_exposure': 0.0, 'days_to_pay': None, **{k: 0.0 for k in AR_BUCKET_FIELDS.values()}, 'top_category': '(none)', 'top_category_pct': None, 'sku_count': 0, diff --git a/platform/modules/opportunity_finder.py b/platform/modules/opportunity_finder.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c5d272bd676228e3a6a0f9936b93c380d560ec --- /dev/null +++ b/platform/modules/opportunity_finder.py @@ -0,0 +1,693 @@ +"""Opportunity finder App: search terms in our departments whose search frequency rank rose week over week. + +Sources (read only, through the tenant Amazon mirror, never written, no SP-API call): + amzn_brand_analytics search terms per department: search_frequency_rank, the top 3 clicked + ASINs (click_share_rank 1 to 3), newest _period_start in full + amzn_search_query_performance our ASINs (SQP reports our ASINs only) + amzn_listings our ASINs (asin1) + +Store documents (tenant runtime store, App owned; removing the App empties every one, rule 13): + opportunity_finder_index {weeks: [period_start, ...], last_sync, last_error} + opportunity_finder_week_ {week, period, capturedAt, departments, terms: {term: rank}} + opportunity_finder_items {items: {id: {term, status, owner, due, note, reason, asin, history}}} + opportunity_finder_settings {departments, min_rise_pct, default_owner} + +The mirror keeps the newest brand analytics week in full and older weeks for watched terms only +(`amazon_store._retain_weekly`). So a week's rank of an unwatched term survives only in this module's +week document, and `on_amazon_sync` writes that document at every sync. +A rank is a position: a lower number is searched more, so a term rises when its number falls. +The department filter and the rank comparison run in DuckDB over every row, with no cap (rule 1). +""" +import contextlib +import datetime as dt +import hashlib +from pathlib import Path + +APP_LABEL = "Opportunity finder" +STATUSES = ("Rising", "Opportunity", "Accepted", "Served", "Dismissed") +DEFAULT_STATUS = "Rising" +DECISIONS = {"approved": "Accepted", "dismissed": "Dismissed"} +DEFAULT_LIMIT = 200 +DUE_DAYS = 7 +DEFAULTS = {"min_rise_pct": 10.0} + +KEY_INDEX = "opportunity_finder_index" +KEY_SETTINGS = "opportunity_finder_settings" +KEY_ITEMS = "opportunity_finder_items" +WEEK_PREFIX = "opportunity_finder_week_" + +BA_TABLE = "amzn_brand_analytics" +SQP_TABLE = "amzn_search_query_performance" +LISTINGS_TABLE = "amzn_listings" +BA_NEED = ("department_name", "search_term", "search_frequency_rank", "_period_start") +#: The synced table key (no prefix) that starts a new history point. +TRIGGER = "brand_analytics" + +NO_MIRROR = "The Amazon mirror is not on this server yet. It appears after the first sync." +NOT_SYNCED = "ut_amzn_brand_analytics has not synced yet, so no search term can be compared." +NO_DEPARTMENTS = ("No brand analytics row shows one of your ASINs yet, so the finder has no department to " + "search. It reads departments from ut_amzn_brand_analytics and your ASINs from " + "ut_amzn_search_query_performance and ut_amzn_listings.") +NOT_READY = "The finder compares two weekly syncs. One week is held so far." +NO_HISTORY = "No weekly history yet. The first week is written at the next Amazon sync." +READ_ONLY = "Your account can view the opportunity finder but not change it." +UNKNOWN_ITEM = "That search term is not in the opportunity finder." +NOT_RISING = "Its search frequency rank did not rise this week." +OLDER_WEEK = ("Only the newest brand analytics week can be tied. The mirror keeps older weeks for " + "Competitor watch's watched terms only.") +LIMIT = ("Each week of history is one store document, limited to 128 MB. About 3,000,000 search terms " + "fit in one week. Split a week by department when a marketplace nears that size.") + +#: Accounts that sign in for a process, never a person who can own an item. +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +class UnknownItem(KeyError, ValueError): + """An item id the finder does not hold. A KeyError for the router, a ValueError for C4.""" + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at=None): + return (at or _now()).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + try: + value = st.get(key) + except Exception: # noqa: BLE001 + value = None + return value if isinstance(value, dict) else {} + + +def _text(value): + return "" if value is None else str(value).strip() + + +def norm(term): + """The stored form of a search term: lower case and trimmed, as the SQL `lower(trim())` reads it.""" + return _text(term).lower() + + +def item_id(term): + """One stable id per search term, the same across weeks.""" + return "o" + hashlib.sha1(norm(term).encode("utf-8")).hexdigest()[:16] + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + if cur is None: + return set() + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _our_asins(cur): + """SQP ASINs plus listing `asin1`: the ASINs the seller owns.""" + ours = set() + if "asin" in _columns(cur, SQP_TABLE): + ours |= {_text(r[0]) for r in cur.execute( + f'SELECT DISTINCT CAST("asin" AS VARCHAR) FROM {SQP_TABLE} WHERE "asin" IS NOT NULL').fetchall()} + if "asin1" in _columns(cur, LISTINGS_TABLE): + ours |= {_text(r[0]) for r in cur.execute( + f'SELECT DISTINCT CAST("asin1" AS VARCHAR) FROM {LISTINGS_TABLE} WHERE "asin1" IS NOT NULL').fetchall()} + return {a for a in ours if a} + + +def _period(cur): + """The newest brand analytics `_period_start` as text, or None.""" + row = cur.execute(f'SELECT MAX(CAST("_period_start" AS VARCHAR)) FROM {BA_TABLE}').fetchone() + return str(row[0]) if row and row[0] is not None else None + + +_TERM = 'lower(trim(CAST("search_term" AS VARCHAR)))' +_DEPT = 'trim(CAST("department_name" AS VARCHAR))' +_IN_PERIOD = 'CAST("_period_start" AS VARCHAR) = ?' +_IN_DEPTS = f"{_DEPT} IN (SELECT unnest(?::VARCHAR[]))" + + +def _derived_departments(cur, ours): + """R49 default: every department where a brand analytics row clicked one of our ASINs, any period.""" + if not ours or "clicked_asin" not in _columns(cur, BA_TABLE): + return [] + rows = cur.execute( + f'SELECT DISTINCT {_DEPT} FROM {BA_TABLE} ' + f'WHERE trim(CAST("clicked_asin" AS VARCHAR)) IN (SELECT unnest(?::VARCHAR[])) ' + f'AND "department_name" IS NOT NULL', [sorted(ours)]).fetchall() + return sorted(_text(r[0]) for r in rows if _text(r[0])) + + +def _source(cur, st): + """What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting.""" + out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": []} + if cur is None: + out["problems"].append(NO_MIRROR) + return out + if not set(BA_NEED) <= _columns(cur, BA_TABLE): + out["problems"].append(NOT_SYNCED) + return out + out["period"] = _period(cur) + if out["period"] is None: + out["problems"].append(NOT_SYNCED) + return out + out["ours"] = _our_asins(cur) + out["derived"] = _derived_departments(cur, out["ours"]) + chosen = _doc(st, KEY_SETTINGS).get("departments") + out["departments"] = sorted({_text(d) for d in chosen if _text(d)}) if chosen else out["derived"] + if not out["departments"]: + out["problems"].append(NO_DEPARTMENTS) + return out + + +def week_ranks(cur, period, departments): + """{term: rank} for every in-department term of one period. One SQL pass, no cap (rule 1).""" + rows = cur.execute( + f'SELECT {_TERM}, MIN(TRY_CAST(trim(CAST("search_frequency_rank" AS VARCHAR)) AS BIGINT)) ' + f'FROM {BA_TABLE} WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND "search_term" IS NOT NULL GROUP BY 1', + [period, list(departments)]).fetchall() + return {str(t): (int(r) if r is not None else None) for t, r in rows} + + +def _rising_rows(cur, period, departments, prior, min_pct): + """Every in-department term whose rank number fell at least `min_pct` percent from `prior`, with its top 3. + + The prior week's ranks join as unnested list parameters, so the comparison runs in SQL.""" + pairs = sorted((t, int(r)) for t, r in prior.items() if r is not None and int(r) > 0) + has_top = {"clicked_asin", "click_share_rank"} <= _columns(cur, BA_TABLE) + cols = _columns(cur, BA_TABLE) + share = 'TRY_CAST(trim(CAST("click_share" AS VARCHAR)) AS DOUBLE)' if "click_share" in cols else "NULL" + conv = 'TRY_CAST(trim(CAST("conversion_share" AS VARCHAR)) AS DOUBLE)' if "conversion_share" in cols else "NULL" + pos = 'TRY_CAST(trim(CAST("click_share_rank" AS VARCHAR)) AS INTEGER)' + top = (f'SELECT {_TERM} AS term, list(struct_pack(asin := trim(CAST("clicked_asin" AS VARCHAR)), ' + f'pos := {pos}, share := {share}, conv := {conv}) ORDER BY {pos}) AS top3 FROM {BA_TABLE} ' + f'WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND {pos} BETWEEN 1 AND 3 AND "clicked_asin" IS NOT NULL ' + f'GROUP BY 1') if has_top else "SELECT NULL::VARCHAR AS term, NULL AS top3 WHERE FALSE" + params_top = [period, list(departments)] if has_top else [] + sql = ( + f'WITH cur AS (SELECT {_TERM} AS term, ' + f'MIN(TRY_CAST(trim(CAST("search_frequency_rank" AS VARCHAR)) AS BIGINT)) AS rank FROM {BA_TABLE} ' + f'WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND "search_term" IS NOT NULL GROUP BY 1), ' + f'prior AS (SELECT unnest(?::VARCHAR[]) AS term, unnest(?::BIGINT[]) AS rank), ' + f'top AS ({top}) ' + f'SELECT cur.term, prior.rank, cur.rank, prior.rank - cur.rank, ' + f'(prior.rank - cur.rank) * 100.0 / prior.rank AS pct, top.top3 ' + f'FROM cur JOIN prior ON prior.term = cur.term LEFT JOIN top ON top.term = cur.term ' + f'WHERE cur.rank IS NOT NULL AND cur.rank < prior.rank ' + f'AND (prior.rank - cur.rank) * 100.0 / prior.rank >= ? ' + f'ORDER BY pct DESC, cur.term') + params = [period, list(departments), [t for t, _r in pairs], [r for _t, r in pairs], *params_top, float(min_pct)] + return cur.execute(sql, params).fetchall() + + +def _top3_list(raw, ours): + out = [] + for entry in raw or (): + if not isinstance(entry, dict) or not entry.get("asin"): + continue + out.append({"asin": entry["asin"], "rank": entry.get("pos"), "clickShare": entry.get("share"), + "conversionShare": entry.get("conv"), "isOurs": entry["asin"] in ours}) + return out + + +def reason_for(prior, current, pct, ours): + """One plain sentence pair for an item and its proposal.""" + tail = ("One of your ASINs is in its top 3." if ours else "None of your ASINs is in its top 3.") + return f"Search frequency rank rose from {prior:,} to {current:,} ({pct:.0f}%). {tail}" + + +# --------------------------------------------------------------------------- settings and owner + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """R31: the tenant's earliest active person Administrator, else its `admin` account.""" + users = _doc(st, "users") + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in users.items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def settings(tenant_key, *, st=None): + """The threshold, the department override and the default owner, with defaults filled in.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + out = {k: float(doc.get(k, v)) for k, v in DEFAULTS.items()} + chosen = doc.get("departments") + out["departments"] = list(chosen) if chosen else None + out["defaultOwnerSetting"] = doc.get("default_owner") or None + out["defaultOwner"] = out["defaultOwnerSetting"] or owner_account(st, tenant_key) + out["statuses"] = list(STATUSES) + out["dueDays"] = DUE_DAYS + out["limit"] = LIMIT + out["defaults"] = {"min_rise_pct": DEFAULTS["min_rise_pct"], + "departments": "Every department where one of your ASINs is clicked."} + return out + + +def _editor(st, user): + """The editing account's name from a username or a user dict. PermissionError for a viewer.""" + name, rec = None, user + if isinstance(user, str): + name = user.strip() + rec = _doc(st, "users").get(name) + if not isinstance(rec, dict) or rec.get("active") is False: + raise PermissionError(READ_ONLY) + role = str(rec.get("role") or "user").strip().lower() + if role == "viewer" or rec.get("can_edit") is False or rec.get("canEdit") is False: + raise PermissionError(READ_ONLY) + return str(rec.get("username") or name or rec.get("name") or "unknown") + + +def set_settings(tenant_key, user=None, *, min_rise_pct=None, departments=None, default_owner=None, st=None): + """Set the rise threshold (0 to below 100), the department list ([] restores the default) or the owner.""" + st = _rt(tenant_key, st) + if user is not None: + _editor(st, user) + changes = {} + if min_rise_pct is not None: + try: + number = float(min_rise_pct) + except (TypeError, ValueError): + raise ValueError("The rise threshold must be a number.") from None + if not 0 <= number < 100: + raise ValueError("The rise threshold must be 0 or more and below 100.") + changes["min_rise_pct"] = number + if departments is not None: + if isinstance(departments, str) or not isinstance(departments, (list, tuple)): + raise ValueError("Departments must be a list of department names.") + changes["departments"] = sorted({_text(d) for d in departments if _text(d)}) or None + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(changes) + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +# --------------------------------------------------------------------------- history + +def _index(st): + return sorted({w for w in _doc(st, KEY_INDEX).get("weeks") or () if isinstance(w, str)}) + + +def weeks(tenant_key, *, st=None): + """Every held week (period start), oldest first.""" + return _index(_rt(tenant_key, st)) + + +def week_point(tenant_key, week, *, st=None): + return _doc(_rt(tenant_key, st), WEEK_PREFIX + week) + + +def record_week(tenant_key, *, st=None, now=None): + """Write the newest brand analytics week's in-department ranks. A same-week sync replaces that week.""" + st = _rt(tenant_key, st) + now = now or _now() + with _reader(tenant_key, st) as cur: + src = _source(cur, st) + if src["period"] is None or not src["departments"]: + return {"state": "nothing", "problems": src["problems"]} + terms = week_ranks(cur, src["period"], src["departments"]) + week = src["period"][:10] + doc = {"week": week, "period": src["period"], "capturedAt": _stamp(now), + "departments": src["departments"], "terms": terms} + st.update(WEEK_PREFIX + week, lambda _old: doc) + + def _add(index): + index = index if isinstance(index, dict) else {} + index["weeks"] = sorted({*(index.get("weeks") or ()), week}) + index["last_sync"] = _stamp(now) + index.pop("last_error", None) + return index + st.update(KEY_INDEX, _add) + return {"state": "recorded", "week": week, "terms": len(terms)} + + +def on_amazon_sync(tenant_key, synced_tables, *, st=None, now=None): + """C5 callback: record this week's history point when brand analytics synced. Never raises.""" + try: + if TRIGGER not in set(synced_tables or ()): + return {"state": "skipped"} + return record_week(tenant_key, st=st, now=now) + except Exception as exc: # noqa: BLE001 + try: + def _note(doc): + doc = doc if isinstance(doc, dict) else {} + doc["last_error"] = f"The weekly history could not be written ({type(exc).__name__}): {exc}" + return doc + _rt(tenant_key, st).update(KEY_INDEX, _note) + except Exception: # noqa: BLE001 + pass + return {"state": "failed", "error": type(exc).__name__} + + +# --------------------------------------------------------------------------- the finder + +def _prior_week(st, period): + """The newest held week before the mirror's newest period, as (week, {term: rank}).""" + before = [w for w in _index(st) if period and w < period[:10]] + if not before: + return None, {} + return before[-1], _doc(st, WEEK_PREFIX + before[-1]).get("terms") or {} + + +def find(tenant_key, *, st=None): + """Every rising in-department term against the prior held week, sorted by rise. No cap (rule 1).""" + st = _rt(tenant_key, st) + min_pct = settings(tenant_key, st=st)["min_rise_pct"] + with _reader(tenant_key, st) as cur: + src = _source(cur, st) + prior_week, prior = _prior_week(st, src["period"]) + rows = [] + if src["period"] and src["departments"] and prior: + rows = _rising_rows(cur, src["period"], src["departments"], prior, min_pct) + problems = list(src["problems"]) + if src["period"] and src["departments"] and not prior_week: + problems.append(NOT_READY if _index(st) else NO_HISTORY) + error = _doc(st, KEY_INDEX).get("last_error") + if error: + problems.append(error) + found = [] + for term, before, now_rank, rise, pct, top in rows: + top3 = _top3_list(top, src["ours"]) + mine = [t["asin"] for t in top3 if t["isOurs"]] + found.append({"id": item_id(term), "term": term, "title": term, "priorRank": int(before), + "rank": int(now_rank), "rise": int(rise), "risePct": round(float(pct), 2), + "ours": bool(mine), "asin": mine[0] if mine else None, "top3": top3, "rising": True, + "reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))}) + return {"week": src["period"][:10] if src["period"] else None, "period": src["period"], + "priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"], + "rising": found, "problems": problems} + + +def _stored(st): + return _doc(st, KEY_ITEMS).get("items") or {} + + +def _row(fields, state): + """One C4 item: the computed fields with the stored status, owner, due and note.""" + return {**fields, "status": state.get("status") or DEFAULT_STATUS, "owner": state.get("owner"), + "due": state.get("due"), "note": state.get("note"), "sku": None} + + +def _all_items(st, found): + stored = _stored(st) + out = [_row(f, stored.get(f["id"]) or {}) for f in found["rising"]] + seen = {r["id"] for r in out} + rest = [] + for iid, state in stored.items(): + if iid in seen or not isinstance(state, dict) or not state.get("status"): + continue + rest.append(_row({"id": iid, "term": state.get("term"), "title": state.get("term"), "priorRank": None, + "rank": None, "rise": None, "risePct": None, "ours": bool(state.get("asin")), + "asin": state.get("asin"), "top3": [], "rising": False, + "reason": " ".join(x for x in (NOT_RISING, state.get("reason")) if x)}, state)) + rest.sort(key=lambda r: _text(r["term"])) + return out + rest + + +def _page(rows, cursor, limit): + try: + start = int(cursor or 0) + size = int(limit) + except (TypeError, ValueError): + raise ValueError("The page cursor is not valid.") from None + if start < 0 or size < 1: + raise ValueError("The page cursor is not valid.") + end = start + size + return rows[start:end], (str(end) if end < len(rows) else None) + + +def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, st=None): + """C4: rising terms and held opportunities, one page. Counts cover every item.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + st = _rt(tenant_key, st) + found = find(tenant_key, st=st) + every = _all_items(st, found) + counts = {s: 0 for s in STATUSES} + for item in every: + counts[item["status"]] = counts.get(item["status"], 0) + 1 + wanted = [i for i in every if status is None or i["status"] == status] + page, nxt = _page(wanted, cursor, limit) + return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"], + "priorWeek": found["priorWeek"], "departments": found["departments"], + "problems": found["problems"], "weeks": _index(st)} + + +def _lookup(st, iid, found=None): + """(computed fields or None, stored state or None, term) for one id. UnknownItem when neither holds it.""" + state = _stored(st).get(iid) + fields = None + if found is not None: + fields = next((f for f in found["rising"] if f["id"] == iid), None) + term = (fields or {}).get("term") or (state or {}).get("term") + if term is None: + for week in reversed(_index(st)): + term = next((t for t in (_doc(st, WEEK_PREFIX + week).get("terms") or {}) if item_id(t) == iid), None) + if term is not None: + break + if fields is None and not (isinstance(state, dict) and state.get("status")) and term is None: + raise UnknownItem(UNKNOWN_ITEM) + return fields, state if isinstance(state, dict) else None, term + + +def set_status(tenant_key, user, iid, status, *, note=None, owner=None, due=None, st=None): + """C4: move one item, optionally its owner and due date. Every change appends a history event.""" + st = _rt(tenant_key, st) + return _move(tenant_key, st, _editor(st, user), iid, status, note=note, owner=owner, due=due) + + +def _move(tenant_key, st, who, iid, status, *, note=None, owner=None, due=None): + if status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + if due is not None: + try: + due = dt.date.fromisoformat(_text(due)[:10]).isoformat() + except ValueError: + raise ValueError("Due must be a date such as 2026-10-30.") from None + fields, state, term = _lookup(st, iid, find(tenant_key, st=st)) + if fields is None and state is None: + raise UnknownItem(UNKNOWN_ITEM) + owner = _text(owner) or None + if status == "Opportunity" and owner is None and not (state or {}).get("owner"): + owner = settings(tenant_key, st=st)["defaultOwner"] + if status == "Opportunity" and due is None and not (state or {}).get("due"): + due = (_now().date() + dt.timedelta(days=DUE_DAYS)).isoformat() + at = _stamp() + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + row = doc.setdefault("items", {}).setdefault(iid, {}) + row["term"] = term + row["status"] = status + if fields is not None: + row["reason"] = fields["reason"] + row["asin"] = fields["asin"] + event = {"at": at, "by": who, "event": "status", "status": status} + if owner is not None: + row["owner"] = event["owner"] = owner + if due is not None: + row["due"] = event["due"] = due + if note is not None: + row["note"] = str(note) + if note: + event["note"] = str(note) + row.setdefault("history", []).append(event) + return doc + st.update(KEY_ITEMS, _apply) + saved = _stored(st).get(iid) or {} + return {"id": iid, **{k: v for k, v in saved.items() if k != "history"}} + + +def proposals(tenant_key, *, st=None): + """C7: every item in status Opportunity, each once.""" + st = _rt(tenant_key, st) + stored = {i: s for i, s in _stored(st).items() if isinstance(s, dict) and s.get("status") == "Opportunity"} + if not stored: + return [] + live = {f["id"]: f for f in find(tenant_key, st=st)["rising"]} + out = [] + for iid in sorted(stored, key=lambda i: _text(stored[i].get("term"))): + state = stored[iid] + term = _text(state.get("term")) + reason = (live.get(iid) or {}).get("reason") or " ".join( + x for x in (NOT_RISING, state.get("reason")) if x) + out.append({"source_kind": "opportunity_finder", "item_id": iid, "sku": None, + "action": f'Serve the rising search term "{term}" with a listing, keywords or ads.', + "reason": reason, "term": term, "owner": state.get("owner"), "due": state.get("due")}) + return out + + +def on_decision(tenant_key, iid, decision, reason, *, st=None): + """C7: approved moves the item to Accepted, dismissed to Dismissed. The reason is kept as a note.""" + if decision not in DECISIONS: + raise ValueError("Decision must be approved or dismissed.") + st = _rt(tenant_key, st) + return _move(tenant_key, st, "Amazon operator", iid, DECISIONS[decision], note=_text(reason) or None) + + +def item_detail(tenant_key, user, iid, *, st=None): + """One item with its history, its weekly rank trend and its newest top 3.""" + st = _rt(tenant_key, st) + found = find(tenant_key, st=st) + fields, state, term = _lookup(st, iid, found) + row = next((r for r in _all_items(st, found) if r["id"] == iid), None) + if row is None: + row = _row({"id": iid, "term": term, "title": term, "rising": False, "reason": NOT_RISING, + "asin": None, "top3": []}, state or {}) + return {"item": row, "history": list((state or {}).get("history") or ()), + **term_detail(tenant_key, iid, st=st, _term=term, _found=found)} + + +def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None): + """The weekly rank trend from history, oldest first, plus the newest week's top 3 clicked ASINs.""" + st = _rt(tenant_key, st) + term = _term + if term is None: + _fields, _state, term = _lookup(st, iid, _found or find(tenant_key, st=st)) + if term is None: + raise UnknownItem(UNKNOWN_ITEM) + trend = [] + for week in _index(st): + terms = _doc(st, WEEK_PREFIX + week).get("terms") or {} + if term in terms: + trend.append({"week": week, "rank": terms[term]}) + top3, period = [], None + with _reader(tenant_key, st) as cur: + src = _source(cur, st) + period = src["period"] + if period and src["departments"] and {"clicked_asin", "click_share_rank"} <= _columns(cur, BA_TABLE): + pos = 'TRY_CAST(trim(CAST("click_share_rank" AS VARCHAR)) AS INTEGER)' + cols = _columns(cur, BA_TABLE) + share = 'TRY_CAST(trim(CAST("click_share" AS VARCHAR)) AS DOUBLE)' if "click_share" in cols else "NULL" + conv = ('TRY_CAST(trim(CAST("conversion_share" AS VARCHAR)) AS DOUBLE)' + if "conversion_share" in cols else "NULL") + rows = cur.execute( + f'SELECT trim(CAST("clicked_asin" AS VARCHAR)), {pos}, {share}, {conv} FROM {BA_TABLE} ' + f'WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND {_TERM} = ? AND {pos} BETWEEN 1 AND 3 ' + f'ORDER BY 2', [period, src["departments"], term]).fetchall() + top3 = _top3_list([{"asin": a, "pos": p, "share": s, "conv": c} for a, p, s, c in rows], src["ours"]) + return {"id": iid, "term": term, "trend": trend, "top3": top3, + "week": period[:10] if period else None} + + +def validate(tenant_key, *, week=None, start=None, end=None, st=None): + """Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments.""" + st = _rt(tenant_key, st) + with _reader(tenant_key, st) as cur: + src = _source(cur, st) + period = src["period"] + if period is None: + return {"ok": False, "app_total": None, "independent_total": None, "week": None, + "detail": " ".join(src["problems"]) or NOT_SYNCED} + newest = period[:10] + asked = _text(week)[:10] or None + outside = ((asked is not None and asked != newest) or (start is not None and newest < _text(start)[:10]) + or (end is not None and newest >= _text(end)[:10])) + if outside: + return {"ok": False, "app_total": None, "independent_total": None, "week": asked or newest, + "detail": OLDER_WEEK} + independent = 0 + if src["departments"]: + independent = int(cur.execute( + f'SELECT COUNT(DISTINCT {_TERM}) FROM {BA_TABLE} WHERE {_IN_PERIOD} AND {_IN_DEPTS}', + [period, src["departments"]]).fetchone()[0]) + doc = _doc(st, WEEK_PREFIX + newest) + if not doc: + return {"ok": False, "app_total": None, "independent_total": independent, "week": newest, + "detail": f"No stored week for {newest}. It is written at the next Amazon sync."} + app_total = len(doc.get("terms") or {}) + same = sorted(doc.get("departments") or ()) == sorted(src["departments"]) + ok = app_total == independent and same and bool(src["departments"]) + detail = (f"{app_total:,} search terms in the week of {newest} tie to the brand analytics week in your " + f"departments." if ok else + f"The week of {newest} holds {app_total:,} search terms, and brand analytics holds {independent:,} " + f"in your departments." + ("" if same else " The department list changed after the sync.")) + return {"ok": ok, "app_total": app_total, "independent_total": independent, "week": newest, + "departments": src["departments"], "detail": detail} + + +def keys(tenant_key, *, st=None): + """Every store key this App owns for the tenant.""" + st = _rt(tenant_key, st) + return [KEY_INDEX, KEY_SETTINGS, KEY_ITEMS, *(WEEK_PREFIX + w for w in _index(st))] + + +def remove(tenant_key, *, st=None): + """Rule 13: empty every App document, history and items included. No mirror table is touched.""" + st = _rt(tenant_key, st) + owned = keys(tenant_key, st=st) + for key in owned: + st.update(key, lambda _doc: {}) + return {"removed": owned} + + +remove_app_data = remove diff --git a/platform/modules/payouts.py b/platform/modules/payouts.py new file mode 100644 index 0000000000000000000000000000000000000000..b642cf6f32fabf43be95e7c465209b38d2e205b4 --- /dev/null +++ b/platform/modules/payouts.py @@ -0,0 +1,880 @@ +"""Payouts App: each Amazon settlement as a waterfall, its tie to economics, the open settlement and owned items. + +Sources (connector tables, read only): `amzn_settlements` (`ut_amzn_settlements`) and `amzn_economics` +(`ut_amzn_economics`). A settlement has one header row (Amazon's `total_amount`, the deposit) and many +line rows (`amount`). The waterfall sums the lines into nine buckets and ties them to the header. + +Store documents (tenant runtime store, App owned, never the mirror and never Amazon): + payouts_settings {default_owner} + payouts_items {items: {settlement_id: item}} + +Rule 13: this module imports no other App. Deleting it and its two documents leaves every +`ut_amzn_*` table intact. Rule 1: nothing caps settlements, lines or items. Each document is one JSON +value under the store's 128 MB ceiling, which holds about 200,000 items. +""" +import atexit +import contextlib +import datetime as dt +import hashlib +import json +import re +import shutil +import statistics +import tempfile +import threading +import time +from pathlib import Path + +KEY_SETTINGS = "payouts_settings" +KEY_ITEMS = "payouts_items" +STORE_KEYS = (KEY_SETTINGS, KEY_ITEMS) +TENANT_ZERO = "royal-imports" + +SETTLEMENTS = "amzn_settlements" +ECONOMICS = "amzn_economics" +SOURCE_KEY = "ut_amzn_settlements" + +STATUSES = ("Open", "Asked Amazon", "Resolved", "Ignore") +DECISIONS = ("approved", "dismissed") +DUE_DAYS = 14 +CENT = 0.005 +TIE_TOLERANCE = 0.02 +PAGE = 200 + +LOAN_REPAYMENT = "Automated Loan Repayment" +WITHHELD_TAX = "ItemWithheldTax" + +#: Waterfall buckets in QI7 order. `other` keeps the lines summing to the deposit. +BUCKETS = (("sales", "Sales"), ("fees", "Fees"), ("refunds", "Refunds"), ("ad_cost", "Ad cost"), + ("reserve", "Reserve"), ("loan_repayment", "Loan repayment"), ("withheld_tax", "Withheld tax"), + ("other", "Other")) +LABELS = dict(BUCKETS) +EXCLUSIONS = ("loan_repayment", "withheld_tax") +#: Other-transaction descriptions that are a fee or a charge. +FEE_DESCRIPTIONS = {"subscription fee", "storage fee", "storagerenewalbilling", "fbainboundtransportationfee", + "removalcomplete", "inbound defect fee"} +FEE_TYPES = {"fbafees", "amazonfees", "servicefee"} +#: Marketplace to currency, used only when the economics report names no currency column. +MARKETPLACE_CURRENCY = {"ATVPDKIKX0DER": "USD", "A2EUQ1WTGCTBG2": "CAD", "A1AM78C64UM0Y8": "MXN"} + +_REQUIRED = ("settlement_id", "amount", "total_amount", "currency", "transaction_type", "amount_type", + "amount_description", "posted_date", "settlement_start_date", "settlement_end_date", "deposit_date") +_ADS_TOTAL = re.compile(r"ads_[a-z0-9_]+_amount") +_POSTED = ("COALESCE(TRY_CAST(left(CAST(posted_date AS VARCHAR), 10) AS DATE), " + "CAST(try_strptime(left(CAST(posted_date AS VARCHAR), 10), '%d.%m.%Y') AS DATE))") +SNAPSHOT_TTL_S = 3600 + + +class SourceUnavailable(RuntimeError): + """The Amazon settlements are not stored for this tenant, or lack a column this module reads.""" + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at): + return at.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _iso(moment): + return _stamp(moment) if moment else None + + +def _money(value): + return round(float(value or 0.0) + 0.0, 2) + + +# --------------------------------------------------------------------------- dates + +def parse_moment(text): + """A UTC datetime from Amazon's ISO or DD.MM.YYYY text, with or without a time; None when unreadable.""" + if isinstance(text, dt.datetime): + return text if text.tzinfo else text.replace(tzinfo=dt.timezone.utc) + if isinstance(text, dt.date): + return dt.datetime(text.year, text.month, text.day, tzinfo=dt.timezone.utc) + s = str(text or "").strip() + if s.upper().endswith(" UTC"): + s = s[:-4].strip() + if not s: + return None + moment = None + try: + moment = dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + except ValueError: + for fmt in ("%d.%m.%Y %H:%M:%S", "%d.%m.%Y %H:%M", "%d.%m.%Y"): + try: + moment = dt.datetime.strptime(s, fmt) + break + except ValueError: + continue + if moment is None: + return None + return moment.replace(tzinfo=dt.timezone.utc) if moment.tzinfo is None else moment.astimezone(dt.timezone.utc) + + +def parse_date(text): + """The calendar date of Amazon's date text (ISO or DD.MM.YYYY), or None.""" + moment = parse_moment(text) + return moment.date() if moment else None + + +def _day(moment): + return f"{moment.day} {moment:%b %Y}" if moment else "an unknown date" + + +def _month_label(month): + try: + return dt.date.fromisoformat(month + "-01").strftime("%B %Y") + except (TypeError, ValueError): + return str(month) + + +# --------------------------------------------------------------------------- store + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + value = st.get(key) + return value if isinstance(value, dict) else {} + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + try: + path = Path(_mirror_path(tenant_key, st)) + except Exception: # noqa: BLE001 + path = None + if path is None or not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +_SNAPSHOT = {} +_SNAPSHOT_LOCK = threading.Lock() + + +def snapshot_cursor(tenant=TENANT_ZERO, tables=(SETTLEMENTS, ECONOMICS)): + """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back.""" + key = (tenant, tuple(tables)) + with _SNAPSHOT_LOCK: + held = _SNAPSHOT.get(key) + if held and time.monotonic() - held[1] < SNAPSHOT_TTL_S: + return held[0].cursor() + import duckdb + from harness import runtime as R + rt = R.get_runtime(tenant) + raw = rt.download_bytes("amazon_mirror/manifest.json") + if raw is None: + raise SourceUnavailable("No Amazon settlements are stored for this workspace yet.") + manifest = json.loads(raw) + if manifest.get("tenant") != tenant: + raise SourceUnavailable("The stored Amazon data names another workspace.") + folder = Path(tempfile.mkdtemp(prefix="payouts-")) + atexit.register(shutil.rmtree, folder, True) + con = duckdb.connect() + for name in tables: + meta = (manifest.get("tables") or {}).get(name) + if not isinstance(meta, dict): + continue + paths = [] + for index, part in enumerate(meta.get("parts") or [meta]): + blob_key = part.get("key") or f"{name}.parquet" + data = rt.download_bytes("amazon_mirror/" + blob_key) + if data is None or (part.get("sha256") and hashlib.sha256(data).hexdigest() != part["sha256"]): + raise SourceUnavailable(f"The stored {name} snapshot failed its checksum.") + path = folder / f"{name}.{index:04d}.parquet" + path.write_bytes(data) + paths.append(str(path)) + if paths: + listed = ", ".join("'" + p.replace("'", "''") + "'" for p in paths) + con.execute(f"CREATE VIEW {name} AS SELECT * FROM read_parquet([{listed}], union_by_name=true)") + if held: + with contextlib.suppress(Exception): + held[0].close() + _SNAPSHOT[key] = (con, time.monotonic()) + return con.cursor() + + +@contextlib.contextmanager +def _source(tenant_key, st=None, con=None): + """`(cursor, source)`: the given cursor, the local mirror when it holds settlements, else the stored snapshot.""" + if con is not None: + yield con, "given" + return + with _reader(tenant_key, st) as cur: + if cur is not None and _columns(cur, SETTLEMENTS): + yield cur, "local mirror" + return + cur = snapshot_cursor(tenant_key) + try: + yield cur, "stored snapshot" + finally: + with contextlib.suppress(Exception): + cur.close() + + +# --------------------------------------------------------------------------- classification + +def bucket(transaction_type, amount_type, description): + """The waterfall bucket key of one settlement line, by the fixed precedence.""" + tt = str(transaction_type or "").strip() + at = str(amount_type or "").strip() + ad = str(description or "").strip() + tl, al, dl = tt.lower(), at.lower(), ad.lower() + if tl == LOAN_REPAYMENT.lower() or al == LOAN_REPAYMENT.lower(): + return "loan_repayment" + if al == WITHHELD_TAX.lower() or "withholding" in dl: + return "withheld_tax" + if "reserve" in dl: + return "reserve" + if al == "cost of advertising" or dl == "cost of advertising": + return "ad_cost" + if tl.startswith("refund"): + return "refunds" + if al in ("itemprice", "promotion"): + return "sales" + if (al in ("itemfees", "itemfee") or tl in FEE_TYPES + or (tl.startswith("other-transaction") and (dl in FEE_DESCRIPTIONS or al in FEE_DESCRIPTIONS))): + return "fees" + return "other" + + +# --------------------------------------------------------------------------- reads + +def _require(cur): + cols = _columns(cur, SETTLEMENTS) + if not cols: + raise SourceUnavailable("No Amazon settlements are stored for this workspace yet.") + missing = [c for c in _REQUIRED if c not in cols] + if missing: + raise SourceUnavailable("Amazon settlements lack the columns " + ", ".join(missing) + ".") + return cols + + +def _load(cur): + """`(headers, groups)`: one header per settlement and line sums by settlement, type and posted month.""" + _require(cur) + headers = {} + for sid, start, end, deposit, total, currency, n in cur.execute( + "SELECT CAST(settlement_id AS VARCHAR), any_value(CAST(settlement_start_date AS VARCHAR)), " + "any_value(CAST(settlement_end_date AS VARCHAR)), any_value(CAST(deposit_date AS VARCHAR)), " + "any_value(TRY_CAST(total_amount AS DOUBLE)), any_value(CAST(currency AS VARCHAR)), count(*) " + f"FROM {SETTLEMENTS} WHERE total_amount IS NOT NULL AND settlement_id IS NOT NULL GROUP BY 1").fetchall(): + headers[sid] = {"start": start, "end": end, "deposit": deposit, "total": total, + "currency": (currency or "").strip(), "headers": int(n)} + groups = cur.execute( + "SELECT CAST(settlement_id AS VARCHAR), COALESCE(CAST(transaction_type AS VARCHAR), ''), " + "COALESCE(CAST(amount_type AS VARCHAR), ''), COALESCE(CAST(amount_description AS VARCHAR), ''), " + f"strftime({_POSTED}, '%Y-%m'), SUM(TRY_CAST(amount AS DOUBLE)), count(*) " + f"FROM {SETTLEMENTS} WHERE amount IS NOT NULL AND settlement_id IS NOT NULL " + "GROUP BY 1, 2, 3, 4, 5").fetchall() + return headers, groups + + +def _economics(cur): + """`(currency, {month: net proceeds}, basis, first day, last day)` from the economics report; empty when absent.""" + cols = _columns(cur, ECONOMICS) + if not cols or "start_date" not in cols: + return None, {}, None, None, None + currency = None + named = (["currency_code"] if "currency_code" in cols else []) + sorted( + c for c in cols if c.endswith("_currency_code") and c != "currency_code") + for col in named: + currency = cur.execute(f'SELECT any_value("{col}") FROM {ECONOMICS} WHERE "{col}" IS NOT NULL').fetchone()[0] + if currency: + break + if not currency and "marketplace_id" in cols: + market = cur.execute(f"SELECT any_value(marketplace_id) FROM {ECONOMICS}").fetchone()[0] + currency = MARKETPLACE_CURRENCY.get(str(market or "")) + if "net_proceeds_total_amount" in cols: + expr, basis = "COALESCE(TRY_CAST(net_proceeds_total_amount AS DOUBLE), 0)", "net proceeds" + else: + ads = " + ".join(f'COALESCE("{c}", 0)' for c in sorted(cols) if _ADS_TOTAL.fullmatch(c) + and "_charge_" not in c) or "0" + sales = "COALESCE(sales_net_product_sales_amount, 0)" if "sales_net_product_sales_amount" in cols else "0" + fees = "COALESCE(fees_total_amount, 0)" if "fees_total_amount" in cols else "0" + expr, basis = f"{sales} - {fees} - ({ads})", "net sales minus fees and ads" + months = {m: float(v or 0.0) for m, v in cur.execute( + f"SELECT left(CAST(start_date AS VARCHAR), 7), SUM({expr}) FROM {ECONOMICS} GROUP BY 1").fetchall() if m} + first, last = cur.execute(f"SELECT min(left(CAST(start_date AS VARCHAR), 10)), " + f"max(left(CAST(start_date AS VARCHAR), 10)) FROM {ECONOMICS}").fetchone() + return (str(currency).strip() if currency else None), months, basis, parse_date(first), parse_date(last) + + +def _waterfall(parts, deposit): + """Bucket rows in QI7 order, each with its parts by absolute amount, then the deposit row.""" + rows = [] + for key, label in BUCKETS: + mine = sorted((p for p in parts if p["bucket"] == key), key=lambda p: -abs(p["amount"])) + rows.append({"key": key, "label": label, "amount": _money(sum(p["amount"] for p in mine)), + "lines": sum(p["lines"] for p in mine), + "parts": [{k: (_money(v) if k == "amount" else v) for k, v in p.items() if k != "bucket"} + for p in mine]}) + rows.append({"key": "deposit", "label": "Deposit", "amount": None if deposit is None else _money(deposit), + "lines": 0, "parts": []}) + return rows + + +def sums_to(rows, deposit): + """True when the bucket rows (deposit row excluded) sum to the deposit to the cent.""" + if deposit is None: + return False + total = sum(r["amount"] for r in rows if r["key"] != "deposit") + return abs(total - float(deposit)) < CENT + + +def _build(headers, groups): + """Every settlement with its waterfall, gap and parsed dates, newest end first.""" + parts, months, undated = {}, {}, {} + for sid, tt, at, ad, month, amount, n in groups: + key = (tt, at, ad) + mine = parts.setdefault(sid, {}) + part = mine.setdefault(key, {"transactionType": tt, "amountType": at, "description": ad, + "amount": 0.0, "lines": 0, "bucket": bucket(tt, at, ad)}) + part["amount"] += float(amount or 0.0) + part["lines"] += int(n) + if month: + per = months.setdefault(sid, {}).setdefault(month, {}) + per[part["bucket"]] = per.get(part["bucket"], 0.0) + float(amount or 0.0) + else: + undated[sid] = undated.get(sid, 0) + int(n) + out = [] + for sid in sorted(set(headers) | set(parts)): + head = headers.get(sid) or {} + mine = list((parts.get(sid) or {}).values()) + lines_total = sum(p["amount"] for p in mine) + total = head.get("total") + start, end, deposit_at = (parse_moment(head.get(k)) for k in ("start", "end", "deposit")) + gap = None if total is None else _money(lines_total - total) + reconciled = total is not None and abs(lines_total - total) < CENT + currency = head.get("currency") or "" + out.append({ + "id": sid, "currency": currency, "start": _iso(start), "end": _iso(end), + "depositDate": _iso(deposit_at), "startText": head.get("start"), "endText": head.get("end"), + "depositText": head.get("deposit"), + "title": f"Settlement {_day(start)} to {_day(end)}" + (f", {currency}" if currency else ""), + "amazonTotal": None if total is None else _money(total), "linesTotal": _money(lines_total), + "gap": gap, "reconciled": reconciled, "headerRows": head.get("headers", 0), + "lineCount": sum(p["lines"] for p in mine), "undatedLines": undated.get(sid, 0), + "waterfall": _waterfall(mine, total), + "_start": start, "_end": end, "_deposit": deposit_at, "_months": months.get(sid) or {}, + }) + far = dt.datetime.min.replace(tzinfo=dt.timezone.utc) + out.sort(key=lambda s: (s["_end"] or far, s["id"]), reverse=True) + return out + + +def _reason(row): + cur = row["currency"] or "" + if row["amazonTotal"] is None: + return f"The lines total {row['linesTotal']:,.2f} {cur} but Amazon's settlement total is missing.".replace(" ", " ") + return (f"The lines total {row['linesTotal']:,.2f} {cur} but Amazon's total is {row['amazonTotal']:,.2f} {cur}, " + f"a gap of {row['gap']:,.2f}.").replace(" ", " ") + + +def _public(row, *, parts=True): + out = {k: v for k, v in row.items() if not k.startswith("_")} + if not row["reconciled"]: + out["reason"] = _reason(row) + if not parts: + out["waterfall"] = [{k: v for k, v in r.items() if k != "parts"} for r in row["waterfall"]] + return out + + +# --------------------------------------------------------------------------- tie to economics + +def _month_totals(rows, currency): + """`{month: {bucket: amount}}` over every settlement in one currency, by posted month.""" + out = {} + for row in rows: + if row["currency"] != currency: + continue + for month, per in row["_months"].items(): + into = out.setdefault(month, {}) + for key, amount in per.items(): + into[key] = into.get(key, 0.0) + amount + return out + + +def _bounds(month): + """The first and last calendar day of a `YYYY-MM` month.""" + first = dt.date.fromisoformat(month + "-01") + return first, (first.replace(day=28) + dt.timedelta(days=4)).replace(day=1) - dt.timedelta(days=1) + + +def _covered(month, first_day, last_day): + """True when economics covers every day of the month (only a whole month ties, F13).""" + lo, hi = _bounds(month) + return bool(first_day and last_day and first_day <= lo and last_day >= hi) + + +def _gap(settled, econ): + gap = settled - econ + pct = gap / abs(econ) if econ else None + return gap, pct, (pct is not None and abs(pct) <= TIE_TOLERANCE) + + +def _reference(totals, months, first_day, last_day, before=None): + """The tie of the newest month economics covers whole that also has settlements, or None.""" + for month in sorted(months, reverse=True): + if (before and month >= before) or month not in totals or not _covered(month, first_day, last_day): + continue + per = totals[month] + settled = sum(v for k, v in per.items() if k not in EXCLUSIONS) + gap, pct, matched = _gap(settled, months[month]) + return {"month": month, "monthLabel": _month_label(month), "settlements": _money(settled), + "economics": _money(months[month]), "gap": _money(gap), + "gapPct": None if pct is None else round(pct * 100, 2), "matched": matched, + "message": (f"{_month_label(month)} matched within {abs(pct) * 100:.1f} percent." if matched + else f"{_month_label(month)} differs by {abs(pct or 0) * 100:.1f} percent.")} + return None + + +def _tie(row, rows, economics): + """The settlement's posted month against economics, with the two exclusions named; final only for a whole month.""" + currency, months, basis, first_day, last_day = economics + if not currency or not months: + return {"available": False, "message": "The economics report is not stored for this workspace yet."} + if row["currency"] != currency: + return {"available": False, "currency": row["currency"], "economicsCurrency": currency, + "message": f"Economics covers {currency} only. This settlement is in {row['currency'] or 'an unknown currency'}."} + month = row["_end"].strftime("%Y-%m") if row["_end"] else None + if not month: + return {"available": False, "message": "This settlement has no readable end date."} + totals = _month_totals(rows, currency) + per = totals.get(month, {}) + including = sum(per.values()) + excluded = {key: per.get(key, 0.0) for key in EXCLUSIONS} + settled = including - sum(excluded.values()) + econ = months.get(month) + own = {key: next((r["amount"] for r in row["waterfall"] if r["key"] == key), 0.0) for key in EXCLUSIONS} + exclusions = [{"key": key, "label": LABELS[key], "monthAmount": _money(excluded[key]), + "settlementAmount": _money(own[key])} for key in EXCLUSIONS] + final = _covered(month, first_day, last_day) + reference = None if final else _reference(totals, months, first_day, last_day, before=month) + if econ is None: + return {"available": False, "month": month, "currency": currency, "exclusions": exclusions, "final": False, + "reference": reference, "message": f"Economics has no rows for {_month_label(month)}."} + gap, pct, matched = _gap(settled, econ) + out = {"available": True, "month": month, "monthLabel": _month_label(month), "currency": currency, + "basis": basis, "settlementsIncluding": _money(including), "settlements": _money(settled), + "economics": _money(econ), "gap": _money(gap), "gapPct": None if pct is None else round(pct * 100, 2), + "matched": matched, "final": True, "tolerancePct": TIE_TOLERANCE * 100, "exclusions": exclusions, + "economicsFrom": first_day.isoformat() if first_day else None, + "economicsThrough": last_day.isoformat() if last_day else None, "reference": None, + "message": (f"{_month_label(month)} settlements without loan repayment and withheld tax are " + f"{settled:,.2f} {currency} against {econ:,.2f} in economics, a gap of {gap:,.2f}" + + (f" ({pct * 100:.1f} percent)." if pct is not None else "."))} + if not final: + lo, _hi = _bounds(month) + if first_day and first_day > lo: + why = (f"Economics starts {_day(first_day)}, so {_month_label(month)} is not complete in economics. " + "This tie is not final.") + else: + why = (f"{_month_label(month)} is not complete in economics yet (through {_day(last_day)}). " + "The tie is final after the month closes.") + out.update(final=False, matched=None, reference=reference, message=why) + return out + + +# --------------------------------------------------------------------------- streams, open settlement, next deposit + +def _median_seconds(values, default): + values = [v for v in values if v is not None and v > 0] + return statistics.median(values) if values else default + + +def streams(rows, now=None): + """Settlement streams by currency and chaining, each with its open settlement; largest 90-day deposits first.""" + now = parse_moment(now) if now is not None else _now() + chains = [] + dated = sorted((r for r in rows if r["_start"] and r["_end"]), key=lambda r: (r["_start"], r["id"])) + for row in dated: + home = next((c for c in chains if c["currency"] == row["currency"] and c["rows"][-1]["_end"] == row["_start"]), + None) + if home is None: + home = {"currency": row["currency"], "rows": []} + chains.append(home) + home["rows"].append(row) + out = [] + since = now - dt.timedelta(days=90) + for chain in chains: + mine = chain["rows"] + length = _median_seconds([(r["_end"] - r["_start"]).total_seconds() for r in mine], 86400 * 14) + lag = _median_seconds([(r["_deposit"] - r["_end"]).total_seconds() for r in mine if r["_deposit"]], 86400 * 2) + newest = max(mine, key=lambda r: r["_end"]) + close = newest["_end"] + dt.timedelta(seconds=length) + expected_deposit = close + dt.timedelta(seconds=lag) + active = (now - newest["_end"]).total_seconds() <= 2 * length + 7 * 86400 + recent = sum(r["amazonTotal"] or 0.0 for r in mine if r["_deposit"] and since <= r["_deposit"] <= now) + out.append({"currency": chain["currency"], "settlements": len(mine), "active": active, + "medianDays": round(length / 86400, 2), "depositLagDays": round(lag / 86400, 2), + "deposits90d": _money(recent), "newestSettlement": newest["id"], + "open": None if not active else { + "start": _iso(newest["_end"]), "expectedClose": _iso(close), + "expectedDeposit": _iso(expected_deposit), "currency": chain["currency"], + "title": f"Open settlement from {_day(newest['_end'])}, {chain['currency']}".rstrip(", ")}, + "_expected": expected_deposit if active else None}) + out.sort(key=lambda s: (-s["deposits90d"], s["currency"], s["newestSettlement"])) + for index, stream in enumerate(out, 1): + stream["id"] = f"{stream['currency'] or 'unknown'}-{index}" + return out + + +def next_deposit(rows, stream_rows, now=None): + """The earliest issued deposit after `now`, else the earliest expected one across open settlements.""" + now = parse_moment(now) if now is not None else _now() + pending = sorted((r for r in rows if r["_deposit"] and r["_deposit"] > now), key=lambda r: (r["_deposit"], r["id"])) + if pending: + row = pending[0] + return {"expected": False, "date": _iso(row["_deposit"]), "amount": row["amazonTotal"], + "currency": row["currency"], "settlementId": row["id"]} + expected = sorted((s for s in stream_rows if s["_expected"]), key=lambda s: s["_expected"]) + if not expected: + return None + ahead = [s for s in expected if s["_expected"] > now] + stream = ahead[0] if ahead else expected[-1] + return {"expected": True, "date": _iso(stream["_expected"]), "amount": None, "currency": stream["currency"], + "stream": stream["id"], "overdue": not ahead} + + +def _strip_streams(stream_rows): + return [{k: v for k, v in s.items() if not k.startswith("_")} for s in stream_rows] + + +# --------------------------------------------------------------------------- public reads + +def _all(tenant_key, st=None, con=None): + with _source(tenant_key, st, con) as (cur, source): + headers, groups = _load(cur) + economics = _economics(cur) + return _build(headers, groups), economics, source + + +def _window(rows, start=None, end=None): + """Keep settlements whose end date is in [start, end]; a `YYYY-MM` bound covers the whole month.""" + lo = hi = None + if start: + lo = parse_date(str(start) + "-01" if len(str(start)) == 7 else start) + if end: + text = str(end) + if len(text) == 7: + first = parse_date(text + "-01") + hi = (first.replace(day=28) + dt.timedelta(days=4)).replace(day=1) - dt.timedelta(days=1) if first else None + else: + hi = parse_date(text) + if (start and lo is None) or (end and hi is None): + raise ValueError("A window bound must be a month such as 2026-09 or a date such as 2026-09-30.") + return [r for r in rows if r["_end"] and (lo is None or r["_end"].date() >= lo) + and (hi is None or r["_end"].date() <= hi)] + + +def _page(rows, cursor, limit): + try: + offset = int(cursor or 0) + except (TypeError, ValueError): + raise ValueError("The cursor is not valid. Start again from the first page.") from None + if offset < 0: + raise ValueError("The cursor is not valid. Start again from the first page.") + if limit is None: + return rows[offset:], None + size = max(1, int(limit)) + chunk = rows[offset:offset + size] + return chunk, (str(offset + size) if offset + size < len(rows) else None) + + +def settlements(tenant_key, *, st=None, con=None, currency=None, start=None, end=None, cursor=None, limit=PAGE): + """Every settlement newest first, with bucket totals and reconciliation, paged by cursor. No cap.""" + rows, _economics_, source = _all(tenant_key, st, con) + rows = _window(rows, start, end) + if currency: + rows = [r for r in rows if r["currency"] == str(currency).strip().upper()] + chunk, nxt = _page(rows, cursor, limit) + return {"settlements": [_public(r, parts=False) for r in chunk], "cursor": nxt, "total": len(rows), + "source": source} + + +def settlement(tenant_key, settlement_id, *, st=None, con=None): + """One settlement: the waterfall with every part, its gap and its tie to economics. KeyError when unknown.""" + rows, economics, source = _all(tenant_key, st, con) + row = next((r for r in rows if r["id"] == str(settlement_id)), None) + if row is None: + raise KeyError(settlement_id) + out = _public(row) + out["tie"] = _tie(row, rows, economics) + out["source"] = source + return out + + +def overview(tenant_key, *, st=None, con=None, now=None): + """Counts, unreconciled settlements, open settlements, the next deposit and the newest settlement's tie.""" + now = parse_moment(now) if now is not None else _now() + rows, economics, source = _all(tenant_key, st, con) + stream_rows = streams(rows, now) + newest = rows[0] if rows else None + return { + "settlementCount": len(rows), "reconciledCount": sum(1 for r in rows if r["reconciled"]), + "unreconciled": [{"id": r["id"], "title": r["title"], "gap": r["gap"], "reason": _reason(r)} + for r in rows if not r["reconciled"]], + "currencies": sorted({r["currency"] for r in rows if r["currency"]}), + "economicsCurrency": economics[0], + "openSettlements": [s["open"] for s in stream_rows if s["open"]], + "nextDeposit": next_deposit(rows, stream_rows, now), "streams": _strip_streams(stream_rows), + "latest": None if newest is None else {**_public(newest, parts=False), "tie": _tie(newest, rows, economics)}, + "source": source, "asOf": _stamp(now), + } + + +# --------------------------------------------------------------------------- settings and owner + +#: Accounts that sign in for a process, never a person who can own an item. +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """The tenant's earliest active person Administrator, else its `admin` account, else its first Administrator.""" + try: + users = st.get("users") + except Exception: # noqa: BLE001 + users = None + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in (users or {}).items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def settings(tenant_key, *, st=None): + """The default owner, with the owner account filled in, and the item statuses.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + setting = doc.get("default_owner") or None + return {"defaultOwnerSetting": setting, "defaultOwner": setting or owner_account(st, tenant_key), + "statuses": list(STATUSES), "dueDays": DUE_DAYS} + + +def set_settings(tenant_key, *, default_owner=None, st=None): + """Set the default owner; '' resets it to the owner account.""" + st = _rt(tenant_key, st) + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +# --------------------------------------------------------------------------- owned items (C4) + +def _refresh_items(tenant_key, st, rows, now): + """Open one item per unreconciled settlement the first time it is seen; returns the items document.""" + doc = _doc(st, KEY_ITEMS) + held = doc.get("items") or {} + states = {r["id"]: r for r in rows} + fresh = [r for r in rows if not r["reconciled"] and r["id"] not in held] + moved = [sid for sid, item in held.items() if sid in states + and bool(item.get("reconciledNow")) != states[sid]["reconciled"]] + if not fresh and not moved: + return doc + owner = settings(tenant_key, st=st)["defaultOwner"] + at = _stamp(now) + + def _apply(current): + current = current if isinstance(current, dict) else {} + items = current.setdefault("items", {}) + for row in fresh: + if row["id"] in items: + continue + items[row["id"]] = { + "id": row["id"], "title": row["title"], "status": "Open", "owner": owner, + "due": (now + dt.timedelta(days=DUE_DAYS)).date().isoformat(), "reason": _reason(row), + "currency": row["currency"], "gap": row["gap"], "linesTotal": row["linesTotal"], + "amazonTotal": row["amazonTotal"], "firstSeen": at, "reconciledNow": False, + "history": [{"at": at, "by": "payouts", "status": "Open"}]} + for sid in moved: + if sid in items: + items[sid]["reconciledNow"] = states[sid]["reconciled"] + return current + st.update(KEY_ITEMS, _apply) + return _doc(st, KEY_ITEMS) + + +def items(tenant_key, user, *, status=None, cursor=None, limit=PAGE, st=None, con=None, now=None): + """Every owned item, due first, paged by cursor. No cap. The caller checks the App grant.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + st = _rt(tenant_key, st) + now = parse_moment(now) if now is not None else _now() + rows, _economics_, _source_ = _all(tenant_key, st, con) + doc = _refresh_items(tenant_key, st, rows, now) + listed = [dict(i) for i in (doc.get("items") or {}).values() if status is None or i.get("status") == status] + listed.sort(key=lambda i: (str(i.get("due") or ""), str(i.get("id")))) + chunk, nxt = _page(listed, cursor, limit) + return {"items": chunk, "cursor": nxt} + + +def _who(user): + """The acting name, or raise PermissionError for no user, a viewer, or a user who may not edit.""" + if not user: + raise PermissionError("Sign in to change a Payouts item.") + if isinstance(user, dict): + if str(user.get("role") or "").strip().lower() == "viewer": + raise PermissionError("Your account can view Payouts but not change it.") + if user.get("canEdit") is False or user.get("can_edit") is False: + raise PermissionError("You can view this App but not change it.") + return str(user.get("username") or user.get("name") or "unknown") + return str(user) + + +def set_status(tenant_key, user, item_id, status, *, note=None, st=None): + """Move one item to a status in STATUSES. Returns the item.""" + who = _who(user) + if status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + st = _rt(tenant_key, st) + out = {} + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + item = (doc.get("items") or {}).get(str(item_id)) + if not item: + return doc + at = _stamp(_now()) + event = {"at": at, "by": who, "status": status} + if note: + event["note"] = str(note).strip() + item["status"] = status + item["updatedAt"], item["updatedBy"] = at, who + item.setdefault("history", []).append(event) + out["item"] = dict(item) + return doc + st.update(KEY_ITEMS, _apply) + if "item" not in out: + raise KeyError(item_id) + return out["item"] + + +def proposals(tenant_key): + """Payouts proposes nothing (`proposes: False`).""" + return [] + + +def on_decision(tenant_key, item_id, decision, reason, *, st=None): + """Record a decision on the item when it exists. Never writes to Amazon or Odoo.""" + if decision not in DECISIONS: + raise ValueError("A decision must be approved or dismissed.") + st = _rt(tenant_key, st) + out = {"recorded": False} + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + item = (doc.get("items") or {}).get(str(item_id)) + if item: + item.setdefault("history", []).append({"at": _stamp(_now()), "by": "decision", "decision": decision, + "reason": str(reason or "")}) + item["decision"] = decision + out["recorded"] = True + return doc + st.update(KEY_ITEMS, _apply) + return out + + +def remove(tenant_key, *, st=None): + """Delete this App's store documents only. Never a mirror table.""" + st = _rt(tenant_key, st) + removed = [] + for key in STORE_KEYS: + if st.get(key) is None: + continue + st.delete_document(key) + removed.append(key) + return {"removed": removed} + + +# --------------------------------------------------------------------------- validate + +def validate(tenant_key=TENANT_ZERO, *, start=None, end=None, con=None, st=None, **_period): + """Each settlement's waterfall deposit, summed from its lines, ties to Amazon's settlement total to the cent.""" + try: + rows, _economics_, source = _all(tenant_key, st, con) + except SourceUnavailable as exc: + return {"ok": False, "app_total": None, "independent_total": None, + "detail": {"error": str(exc), "settlements": [], "unreconciled": []}} + rows = _window(rows, start or _period.get("month"), end or _period.get("month")) + checks = [{"id": r["id"], "currency": r["currency"], "end": r["end"], "linesTotal": r["linesTotal"], + "amazonTotal": r["amazonTotal"], "gap": r["gap"], "ok": r["reconciled"]} for r in rows] + bad = [{"id": c["id"], "gap": c["gap"], "currency": c["currency"]} for c in checks if not c["ok"]] + by_currency = {} + for r in rows: + into = by_currency.setdefault(r["currency"] or "unknown", {"app": 0.0, "amazon": 0.0, "settlements": 0}) + into["app"] += sum(w["amount"] for w in r["waterfall"] if w["key"] != "deposit") + into["amazon"] += r["amazonTotal"] or 0.0 + into["settlements"] += 1 + for into in by_currency.values(): + into["app"], into["amazon"] = _money(into["app"]), _money(into["amazon"]) + app_total = _money(sum(v["app"] for v in by_currency.values())) + independent = _money(sum(v["amazon"] for v in by_currency.values())) + return {"ok": bool(rows) and not bad, "app_total": app_total, "independent_total": independent, + "detail": {"source": source, "count": len(rows), "reconciled": len(rows) - len(bad), + "settlements": checks, "unreconciled": bad, "byCurrency": by_currency}} diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index b361ac5e986687a1522e91781db3799404822bd9..6d38602fbb4f7e535d1d9cb9623de3a6c9fa5a67 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -55,6 +55,7 @@ template and this deliberately mirrors its signature and its row shape. carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two sources are JOINED rather than merged. """ +import contextvars import datetime as _dt import json import math as _math @@ -248,6 +249,35 @@ def image_field_definition(st=None): pass return stored +_ASSET_PREFIX = '/api/v1/assets/' +#: An external door sets its public origin here around one query. Its rows then carry image URLs. +EXTERNAL_IMAGE_BASE = contextvars.ContextVar('product_external_image_base', default=None) +_IMAGE_REF_CHARS = frozenset('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-') + + +def image_path(value, code=None, master_codes=()): + """The asset route path, or an absolute URL, for one product's Image cell. '' means no photo. + + An empty cell falls back to the SKU's master image when `master_codes` holds its upper-cased + code. The path is the existing asset route, so no Odoo column is read or added. + """ + from urllib.parse import quote + ref = value.strip() if isinstance(value, str) else '' + if ref.startswith(('https://', 'http://', _ASSET_PREFIX)): + return ref + if ref.startswith('rec:') and ref[4:] and set(ref[4:]) <= _IMAGE_REF_CHARS: + return _ASSET_PREFIX + 'records/' + ref[4:] + if ref.startswith('ed:') and ref[3:] and set(ref[3:]) <= _IMAGE_REF_CHARS: + return _ASSET_PREFIX + 'editorial/' + ref[3:] + if ref: + return (_ASSET_PREFIX + 'products/' + quote(ref, safe='') + if len(ref) <= 128 and set(ref) <= _IMAGE_REF_CHARS else '') + sku = code.strip() if isinstance(code, str) else '' + if sku and sku.upper() in master_codes: + return _ASSET_PREFIX + 'products/' + quote(sku, safe='') + return '' + + _GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json' _SHARED_KEYS = None @@ -627,9 +657,18 @@ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=( from harness.canonical_source import bounded, BATCH with bounded(cur, 'identity', [], st) as identity: def codes(): - for product in identity.search_read('product.product', [('active', '=', True)], ['id', 'default_code']): - code = str(product['default_code']).strip() if product.get('default_code') else f"pid:{product['id']}" - yield {'code': code, 'pid': sku_pid(code)} + last_id = 0 + while True: + page = cur.execute('SELECT id, default_code FROM product_product ' + 'WHERE active = TRUE AND id > ? ORDER BY id LIMIT ?', + [last_id, BATCH]).fetchall() + if not page: + break + last_id = page[-1][0] + for product_id, default_code in page: + code = str(default_code).strip() if default_code else f"pid:{product_id}" + yield {'code': code, 'pid': sku_pid(code)} + page.clear() raw = identity.snapshot(codes()) collision = cur.execute(f"SELECT json_extract_string(body, '/pid') pid " f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone() diff --git a/platform/modules/ranking_diagnostic.py b/platform/modules/ranking_diagnostic.py new file mode 100644 index 0000000000000000000000000000000000000000..75a27bf6e8cb46b3dc0049425509474b8c1c70ad --- /dev/null +++ b/platform/modules/ranking_diagnostic.py @@ -0,0 +1,895 @@ +"""Ranking diagnostic App: a weekly ASIN history, one cause and one fix per drop, owned items, validated. + +Sources (read only, through the tenant Amazon mirror, never written, no SP-API call): + amzn_pricing Sales and Traffic per child ASIN: sessions, units, Buy Box %. No date (F5). + amzn_catalog_items sales_ranks JSON, newest _snapshot_date (F6) + amzn_brand_analytics search-term click share, newest _period_start (S78-T15 weeks) + amzn_economics ads charge columns (ads__amount), the 7 newest days, per SKU to ASIN + amzn_fba_inventory fulfillable quantity, newest snapshot (out of stock) + amzn_suppressed_listings suppressed listings, newest snapshot + amzn_stranded_inventory stranded inventory, newest snapshot + amzn_inventory_planning your_price and featuredoffer_price, newest snapshot + +Store documents (tenant runtime store, App owned; removing the App empties every one, rule 13): + ranking_diagnostic_index {weeks: [week, ...], last_sync, last_error} + ranking_diagnostic_week_ {week, capturedAt, asins: {asin: point}, sourceTotals, problems} + ranking_diagnostic_items_ {items: {id: item}} + ranking_diagnostic_settings {thresholds..., default_owner} + +A week is the Sunday to Saturday week of the sync (UTC), as brand analytics weeks run. +A re-sync in the same week replaces that week's point and never adds a second one. +Limit (rule 1): one document per week, so no document grows with the history. At about 400 bytes an +ASIN, a week document reaches the store's 128 MB ceiling near 300,000 ASINs. +""" +import contextlib +import datetime as dt +import hashlib +import json +import re +from pathlib import Path + +APP_LABEL = "Ranking diagnostic" +STATUSES = ("Open", "In progress", "Fixed", "Ignored") +DECISIONS = ("approved", "dismissed") +DEFAULT_LIMIT = 200 +DUE_DAYS = 7 + +KEY_INDEX = "ranking_diagnostic_index" +KEY_SETTINGS = "ranking_diagnostic_settings" +WEEK_PREFIX = "ranking_diagnostic_week_" +ITEMS_PREFIX = "ranking_diagnostic_items_" + +PRICING = "amzn_pricing" +CATALOG = "amzn_catalog_items" +BRAND_ANALYTICS = "amzn_brand_analytics" +ECONOMICS = "amzn_economics" +FBA_INVENTORY = "amzn_fba_inventory" +SUPPRESSED = "amzn_suppressed_listings" +STRANDED = "amzn_stranded_inventory" +PLANNING = "amzn_inventory_planning" +SOURCES = (PRICING, CATALOG, BRAND_ANALYTICS, ECONOMICS, FBA_INVENTORY, SUPPRESSED, STRANDED, PLANNING) +#: Synced table names (no prefix) that start a new history point. +TRIGGERS = tuple(t.removeprefix("amzn_") for t in SOURCES) + +SESSIONS = "traffic_by_asin_sessions" +UNITS = "sales_by_asin_units_ordered" +BUY_BOX = "traffic_by_asin_buy_box_percentage" +SALES = "sales_by_asin_ordered_product_sales_amount" + +#: Percent falls, Buy Box in points. Each is an App setting (R49 precedent). +DEFAULTS = {"sessions_pct": 20.0, "units_pct": 20.0, "search_share_pct": 20.0, "rank_pct": 20.0, + "buy_box_points": 10.0, "ad_spend_pct": 20.0} + +#: One cause per ASIN, root causes first. Each carries one fix. +CAUSES = ( + ("out_of_stock", "Out of stock", + "Send a replenishment shipment to Amazon and check the restock report for the quantity."), + ("suppressed_or_stranded", "Suppressed or stranded", + "Fix the listing issue Seller Central names so the offer can be bought again."), + ("priced_above_featured", "Priced above the featured offer", + "Match the featured offer price, or check why your offer is not the featured one."), + ("lost_buy_box", "Lost Buy Box", + "Check the competing offers, your price, shipping speed and stock to win the Buy Box back."), + ("ad_spend_cut", "Ad spend cut", + "Restore the Sponsored Products budget or bids for this ASIN."), + ("search_share_fell", "Search-term share fell", + "Review the top search terms for this ASIN and refresh the title, keywords and ads on them."), + ("traffic_fell", "Traffic fell", + "Check the search rank and ads for this ASIN, and add a coupon or deal to bring shoppers back."), +) +UNEXPLAINED = ("unexplained", "No cause in the Amazon data", + "Open the listing in Seller Central and check its reviews, images and price history.") +CAUSE_ORDER = {key: n for n, (key, _l, _f) in enumerate((*CAUSES, UNEXPLAINED))} + +ABSENT = "Not in this week's Sales and Traffic report." +READ_ONLY = "Your account can view the ranking diagnostic but not change it." +NOT_READY = "The diagnostic starts after two weekly syncs. One week is held so far." +LIMIT = ("Each week of history is one store document, limited to 128 MB. About 300,000 ASINs fit in " + "one week. Split a week by ASIN range when a catalogue nears that size.") + +#: Accounts that sign in for a process, never a person who can own an item. +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + +LABELS = {PRICING: "ut_amzn_pricing", CATALOG: "ut_amzn_catalog_items", + BRAND_ANALYTICS: "ut_amzn_brand_analytics", ECONOMICS: "ut_amzn_economics", + FBA_INVENTORY: "ut_amzn_fba_inventory", SUPPRESSED: "ut_amzn_suppressed_listings", + STRANDED: "ut_amzn_stranded_inventory", PLANNING: "ut_amzn_inventory_planning"} +#: What a missing source stops the App from checking. +MISSING = {PRICING: "sessions, units and Buy Box share", CATALOG: "sales rank", + BRAND_ANALYTICS: "search-term share", ECONOMICS: "ad spend", + FBA_INVENTORY: "out of stock", SUPPRESSED: "suppressed listings", + STRANDED: "stranded inventory", PLANNING: "price against the featured offer"} + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at=None): + return (at or _now()).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def week_of(day): + """The Sunday that starts the week holding `day`, as ISO text.""" + if isinstance(day, dt.datetime): + day = day.date() + return (day - dt.timedelta(days=(day.weekday() + 1) % 7)).isoformat() + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + try: + value = st.get(key) + except Exception: # noqa: BLE001 + value = None + return value if isinstance(value, dict) else {} + + +def _text(value): + return "" if value is None else str(value).strip() + + +def _num(value): + try: + return float(_text(value)) + except ValueError: + return None + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + if cur is None: + return set() + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _newest(cols, column, table): + """A WHERE clause keeping the newest value of `column`, or '' when the table lacks it.""" + if column not in cols: + return "" + return f'WHERE CAST("{column}" AS VARCHAR) = (SELECT MAX(CAST("{column}" AS VARCHAR)) FROM {table})' + + +def _col(cols, name, cast="VARCHAR"): + if name not in cols: + return "NULL" + if cast == "VARCHAR": + return f'TRIM(CAST("{name}" AS VARCHAR))' + return f'TRY_CAST(TRIM(CAST("{name}" AS VARCHAR)) AS {cast})' + + +def _asin_expr(cols): + parts = [f"NULLIF(TRIM(CAST(\"{c}\" AS VARCHAR)), '')" for c in ("child_asin", "parent_asin") if c in cols] + return f"COALESCE({', '.join(parts)})" if parts else None + + +def _pricing(cur): + """{asin: {sessions, units, sales, buyBox, skus}} over every Sales and Traffic row. No cap (rule 1).""" + cols = _columns(cur, PRICING) + asin = _asin_expr(cols) + if asin is None: + return None + rows = cur.execute( + f"SELECT {asin}, {_col(cols, 'sku')}, {_col(cols, SESSIONS, 'DOUBLE')}, {_col(cols, UNITS, 'DOUBLE')}, " + f"{_col(cols, SALES, 'DOUBLE')}, {_col(cols, BUY_BOX, 'DOUBLE')} FROM {PRICING}").fetchall() + out = {} + for a, sku, sessions, units, sales, box in rows: + if not a: + continue + p = out.setdefault(a, {"sessions": 0.0, "units": 0.0, "sales": 0.0, "skus": [], "_box": []}) + p["sessions"] += sessions or 0.0 + p["units"] += units or 0.0 + p["sales"] += sales or 0.0 + if sku and sku not in p["skus"]: + p["skus"].append(sku) + if box is not None: + p["_box"].append((box, sessions or 0.0)) + for p in out.values(): + boxes = p.pop("_box") + weight = sum(w for _b, w in boxes) + p["buyBox"] = (round(sum(b * w for b, w in boxes) / weight, 4) if weight + else (round(max(b for b, _w in boxes), 4) if boxes else None)) + p["skus"].sort() + return out + + +def source_totals(cur): + """P10: one SQL sum of sessions and units over the Sales and Traffic table, apart from `_pricing`.""" + cols = _columns(cur, PRICING) + asin = _asin_expr(cols) + if asin is None or not {SESSIONS, UNITS} <= cols: + return None + sessions, units, count = cur.execute( + f"SELECT COALESCE(SUM(TRY_CAST(\"{SESSIONS}\" AS DOUBLE)), 0), " + f"COALESCE(SUM(TRY_CAST(\"{UNITS}\" AS DOUBLE)), 0), " + f"COUNT(DISTINCT CASE WHEN TRY_CAST(\"{UNITS}\" AS DOUBLE) > 0 THEN {asin} END) FROM {PRICING}").fetchone() + return {"sessions": round(float(sessions), 4), "units": round(float(units), 4), "asinsWithUnits": int(count)} + + +def _rank(raw): + """One sales rank from catalog `sales_ranks` JSON: the display group rank, else the best classification.""" + text = _text(raw) + if not text: + return None + try: + value = json.loads(text) + except ValueError: + value = _num(text) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + entries = value if isinstance(value, list) else [value] if isinstance(value, dict) else [] + for entry in entries: + if not isinstance(entry, dict): + continue + if isinstance(entry.get("rank"), int): + return entry["rank"] + display = [r["rank"] for r in entry.get("displayGroupRanks") or () if isinstance((r or {}).get("rank"), int)] + if display: + return display[0] + ranks = [r["rank"] for r in entry.get("classificationRanks") or () if isinstance((r or {}).get("rank"), int)] + if ranks: + return min(ranks) + return None + + +def _catalog(cur): + cols = _columns(cur, CATALOG) + if not {"asin"} <= cols: + return None + rows = cur.execute(f"SELECT {_col(cols, 'asin')}, {_col(cols, 'sales_ranks')}, " + f"{_col(cols, 'attributes_item_name')} FROM {CATALOG} " + f"{_newest(cols, '_snapshot_date', CATALOG)}").fetchall() + out = {} + for asin, ranks, title in rows: + if asin: + got = out.setdefault(asin, {"salesRank": None, "title": None}) + rank = _rank(ranks) + if rank is not None and (got["salesRank"] is None or rank < got["salesRank"]): + got["salesRank"] = rank + got["title"] = got["title"] or (_text(title) or None) + return out + + +def _search_share(cur): + """({asin: summed click share}, period) from the newest brand analytics week.""" + cols = _columns(cur, BRAND_ANALYTICS) + if not {"clicked_asin", "click_share"} <= cols: + return None, None + where = _newest(cols, "_period_start", BRAND_ANALYTICS) + period = None + if "_period_start" in cols: + period = cur.execute(f'SELECT MAX(CAST("_period_start" AS VARCHAR)) FROM {BRAND_ANALYTICS}').fetchone()[0] + rows = cur.execute(f"SELECT {_col(cols, 'clicked_asin')}, SUM({_col(cols, 'click_share', 'DOUBLE')}) " + f"FROM {BRAND_ANALYTICS} {where} GROUP BY 1").fetchall() + return {a: round(float(s or 0.0), 6) for a, s in rows if a}, (str(period) if period else None) + + +_AD_COLUMN = re.compile(r"^ads_[a-z0-9_]+_amount$") +_AD_CHARGE_DETAIL = re.compile(r"_charge_\d+_") + + +def ad_columns(cols): + """The pivoted ads charge totals (`ads__amount`), never a per-charge detail column.""" + return sorted(c for c in cols if _AD_COLUMN.match(c) and not _AD_CHARGE_DETAIL.search(c)) + + +def _ad_spend(cur, sku_asin): + """({asin: ad spend}, window) over the seven newest days of economics, SKU rows summed to ASIN.""" + cols = _columns(cur, ECONOMICS) + ads = ad_columns(cols) + if "start_date" not in cols: + return None, None + day = 'TRY_CAST(SUBSTR(CAST("start_date" AS VARCHAR), 1, 10) AS DATE)' + newest = cur.execute(f"SELECT MAX({day}) FROM {ECONOMICS}").fetchone()[0] + if newest is None: + return {}, None + first = newest - dt.timedelta(days=6) + total = " + ".join(f'COALESCE(TRY_CAST("{c}" AS DOUBLE), 0)' for c in ads) or "0" + rows = cur.execute(f"SELECT {_col(cols, 'child_asin')}, {_col(cols, 'msku')}, SUM({total}) FROM {ECONOMICS} " + f"WHERE {day} BETWEEN ? AND ? GROUP BY 1, 2", [first, newest]).fetchall() + out = {} + for asin, msku, spend in rows: + asin = asin or sku_asin.get(msku or "") + if asin: + out[asin] = round(out.get(asin, 0.0) + float(spend or 0.0), 4) + return out, {"start": first.isoformat(), "end": newest.isoformat()} + + +def _fulfillable(cur): + cols = _columns(cur, FBA_INVENTORY) + qty = "inventory_details_fulfillable_quantity" + if not {"asin", qty} <= cols: + return None + rows = cur.execute(f"SELECT {_col(cols, 'asin')}, SUM({_col(cols, qty, 'DOUBLE')}) FROM {FBA_INVENTORY} " + f"{_newest(cols, '_snapshot_date', FBA_INVENTORY)} GROUP BY 1").fetchall() + return {a: float(q or 0.0) for a, q in rows if a} + + +def _asin_set(cur, table): + cols = _columns(cur, table) + if "asin" not in cols: + return None + rows = cur.execute(f"SELECT DISTINCT {_col(cols, 'asin')} FROM {table} " + f"{_newest(cols, '_snapshot_date', table)}").fetchall() + return {a for (a,) in rows if a} + + +def _prices(cur): + cols = _columns(cur, PLANNING) + if not {"asin", "your_price", "featuredoffer_price"} <= cols: + return None + rows = cur.execute(f"SELECT {_col(cols, 'asin')}, MIN({_col(cols, 'your_price', 'DOUBLE')}), " + f"MIN({_col(cols, 'featuredoffer_price', 'DOUBLE')}) FROM {PLANNING} " + f"{_newest(cols, '_snapshot_date', PLANNING)} GROUP BY 1").fetchall() + return {a: (p, f) for a, p, f in rows if a} + + +def _missing(table): + return f"{LABELS[table]} has not synced yet, so {MISSING[table]} is not checked." + + +def snapshot(cur, carry=None): + """This sync's point per ASIN in Sales and Traffic, the P10 totals, and each source it could not read. + + `carry` is the week before's points. An ASIN there and absent from Sales and Traffic now gets a zero + point marked absent, so out of stock and suppressed listings still diagnose. An ASIN absent two + weeks running is not carried again.""" + problems = [] + if cur is None: + return {"asins": {}, "sourceTotals": None, + "problems": ["The Amazon mirror is not on this server yet. It appears after the first sync."]} + pricing = _pricing(cur) + if pricing is None: + problems.append(_missing(PRICING)) + pricing, carry = {}, None + absent = {a: (p.get("skus") or []) for a, p in (carry or {}).items() + if a not in pricing and isinstance(p, dict) and not p.get("absent")} + sku_asin = {sku: a for a, skus in absent.items() for sku in skus} + sku_asin.update({sku: a for a, p in pricing.items() for sku in p["skus"]}) + catalog = _catalog(cur) + shares, period = _search_share(cur) + spend, window = _ad_spend(cur, sku_asin) + stock = _fulfillable(cur) + suppressed = _asin_set(cur, SUPPRESSED) + stranded = _asin_set(cur, STRANDED) + prices = _prices(cur) + for table, got in ((CATALOG, catalog), (BRAND_ANALYTICS, shares), (ECONOMICS, spend), + (FBA_INVENTORY, stock), (SUPPRESSED, suppressed), (STRANDED, stranded), (PLANNING, prices)): + if got is None: + problems.append(_missing(table)) + points = {} + for asin in sorted(pricing): + p = pricing[asin] + cat = (catalog or {}).get(asin) or {} + price, featured = (prices or {}).get(asin, (None, None)) + points[asin] = { + "sessions": p["sessions"], "units": p["units"], "sales": round(p["sales"], 2), + "buyBox": p["buyBox"], "skus": p["skus"], "title": cat.get("title"), + "salesRank": cat.get("salesRank"), + "searchShare": None if shares is None else shares.get(asin, 0.0), + "adSpend": None if spend is None else spend.get(asin, 0.0), + "fulfillable": None if stock is None else stock.get(asin), + "suppressed": None if suppressed is None else asin in suppressed, + "stranded": None if stranded is None else asin in stranded, + "price": price, "featuredPrice": featured} + for asin in sorted(absent): + cat = (catalog or {}).get(asin) or {} + price, featured = (prices or {}).get(asin, (None, None)) + points[asin] = { + "sessions": 0.0, "units": 0.0, "sales": 0.0, "buyBox": None, "skus": absent[asin], + "title": cat.get("title"), "salesRank": cat.get("salesRank"), "searchShare": None, "adSpend": None, + "fulfillable": None if stock is None else stock.get(asin), + "suppressed": None if suppressed is None else asin in suppressed, + "stranded": None if stranded is None else asin in stranded, + "price": price, "featuredPrice": featured, "absent": True} + return {"asins": points, "sourceTotals": source_totals(cur), "searchPeriod": period, "adWindow": window, + "problems": problems} + + +# --------------------------------------------------------------------------- settings and owner + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """R31 and P22: the tenant's earliest active person Administrator, else its `admin` account.""" + users = _doc(st, "users") + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in users.items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def settings(tenant_key, *, st=None): + """Thresholds and the default owner, with defaults filled in.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + out = {k: float(doc.get(k, v)) for k, v in DEFAULTS.items()} + out["defaultOwnerSetting"] = doc.get("default_owner") or None + out["defaultOwner"] = out["defaultOwnerSetting"] or owner_account(st, tenant_key) + out["statuses"] = list(STATUSES) + out["dueDays"] = DUE_DAYS + out["limit"] = LIMIT + return out + + +def set_settings(tenant_key, user=None, *, default_owner=None, st=None, **thresholds): + """Set thresholds (above 0, percent below 100, Buy Box points up to 100) or the default owner.""" + st = _rt(tenant_key, st) + if user is not None: + _editor(st, user) + changes = {} + for name, value in thresholds.items(): + if name not in DEFAULTS: + raise ValueError(f"Unknown setting {name}.") + if value is None: + continue + try: + number = float(value) + except (TypeError, ValueError): + raise ValueError("A threshold must be a number.") from None + if not 0 < number < 100 and not (name == "buy_box_points" and number == 100): + raise ValueError("A threshold must be above 0 and below 100.") + changes[name] = number + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(changes) + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +def _editor(st, user): + """The editing account's name from a username or a user dict (C8). PermissionError for a viewer.""" + name, rec = None, user + if isinstance(user, str): + name = user.strip() + rec = _doc(st, "users").get(name) + if not isinstance(rec, dict) or rec.get("active") is False: + raise PermissionError(READ_ONLY) + role = str(rec.get("role") or "user").strip().lower() + if role == "viewer" or rec.get("can_edit") is False or rec.get("canEdit") is False: + raise PermissionError(READ_ONLY) + return str(rec.get("username") or name or rec.get("name") or "unknown") + + +# --------------------------------------------------------------------------- the diagnosis + +def _fall_pct(prev, cur): + """Percent fall from prev to cur, or None when prev is not positive or cur is unknown.""" + if prev is None or prev <= 0 or cur is None: + return None + return (prev - cur) / prev * 100.0 + + +def _fmt(value): + return f"{value:,.2f}".rstrip("0").rstrip(".") if isinstance(value, float) else f"{value:,}" + + +def drops(prev, cur, limits): + """The signals that fell from `prev` to `cur`, each as (signal, sentence).""" + out = [] + for key, name, limit in (("sessions", "Sessions", "sessions_pct"), ("units", "Units", "units_pct")): + pct = _fall_pct(prev.get(key), cur.get(key)) + if pct is not None and pct >= limits[limit]: + out.append((key, f"{name} fell {pct:.0f}% ({_fmt(prev[key])} to {_fmt(cur.get(key) or 0.0)}).")) + b0, b1 = prev.get("buyBox"), cur.get("buyBox") + if b0 is not None and b1 is not None and b0 - b1 >= limits["buy_box_points"]: + out.append(("buyBox", f"Buy Box share fell {b0 - b1:.0f} points ({_fmt(b0)}% to {_fmt(b1)}%).")) + r0, r1 = prev.get("salesRank"), cur.get("salesRank") + if r0 and r1 and r1 > r0 and (r1 - r0) / r0 * 100.0 >= limits["rank_pct"]: + out.append(("salesRank", f"Sales rank worsened {(r1 - r0) / r0 * 100.0:.0f}% (#{r0:,} to #{r1:,}).")) + pct = _fall_pct(prev.get("searchShare"), cur.get("searchShare")) + if pct is not None and pct >= limits["search_share_pct"]: + out.append(("searchShare", f"Search-term share fell {pct:.0f}% ({_fmt(prev['searchShare'])} to " + f"{_fmt(cur.get('searchShare') or 0.0)}).")) + return out + + +def cause(prev, cur, fell, limits): + """(cause key, evidence sentence or None): the first cause in CAUSES order that holds.""" + signals = {s for s, _t in fell} + if cur.get("fulfillable") is not None and cur["fulfillable"] <= 0: + return "out_of_stock", "Fulfillable stock at Amazon is 0." + if cur.get("suppressed") or cur.get("stranded"): + what = " and ".join(w for w, on in (("suppressed", cur.get("suppressed")), + ("stranded", cur.get("stranded"))) if on) + return "suppressed_or_stranded", f"The listing is {what}." + price, featured = cur.get("price"), cur.get("featuredPrice") + if price is not None and featured is not None and price > featured + 0.005: + return "priced_above_featured", f"Your price {price:.2f} is above the featured offer at {featured:.2f}." + if "buyBox" in signals: + return "lost_buy_box", None + pct = _fall_pct(prev.get("adSpend"), cur.get("adSpend")) + if pct is not None and pct >= limits["ad_spend_pct"]: + return "ad_spend_cut", (f"Ad spend fell {pct:.0f}% ({prev['adSpend']:.2f} to " + f"{(cur.get('adSpend') or 0.0):.2f}).") + if "searchShare" in signals: + return "search_share_fell", None + if "sessions" in signals: + return "traffic_fell", None + return UNEXPLAINED[0], None + + +def _cause_row(key): + return next(row for row in (*CAUSES, UNEXPLAINED) if row[0] == key) + + +def item_id(week, asin): + return f"rd{week.replace('-', '')}-" + hashlib.sha1(_text(asin).encode("utf-8")).hexdigest()[:10] + + +def _week_of_id(value): + found = re.fullmatch(r"rd(\d{4})(\d{2})(\d{2})-[0-9a-f]{10}", _text(value)) + return f"{found.group(1)}-{found.group(2)}-{found.group(3)}" if found else None + + +def diagnose(prev_points, cur_points, limits, week): + """One computed item per ASIN that fell against the week before. An ASIN in one week only is skipped.""" + out = {} + for asin in sorted(cur_points): + prev, cur = prev_points.get(asin), cur_points[asin] + if prev is None: + continue + fell = drops(prev, cur, limits) + if not fell: + continue + key, evidence = cause(prev, cur, fell, limits) + _k, label, fix = _cause_row(key) + gone = [ABSENT] if cur.get("absent") else [] + reason = " ".join([*gone, *([evidence] if evidence else []), *(t for _s, t in fell)]) + out[item_id(week, asin)] = { + "week": week, "asin": asin, "sku": (cur.get("skus") or [None])[0], "skus": cur.get("skus") or [], + "title": f"{asin}: {label}", "productTitle": cur.get("title"), + "causeKey": key, "cause": label, "fix": fix, "reason": reason, + "signals": [s for s, _t in fell]} + return out + + +# --------------------------------------------------------------------------- history + +def _index(st): + doc = _doc(st, KEY_INDEX) + return sorted({w for w in doc.get("weeks") or () if isinstance(w, str)}) + + +def weeks(tenant_key, *, st=None): + """Every held week, oldest first.""" + return _index(_rt(tenant_key, st)) + + +def week_point(tenant_key, week, *, st=None): + return _doc(_rt(tenant_key, st), WEEK_PREFIX + week) + + +def _limits(st, tenant_key): + got = settings(tenant_key, st=st) + return {k: got[k] for k in DEFAULTS} + + +def record_week(tenant_key, *, st=None, now=None): + """Write this week's point per ASIN (replacing a same-week point) and recompute that week's items.""" + st = _rt(tenant_key, st) + now = now or _now() + week = week_of(now) + before = [w for w in _index(st) if w < week] + carry = (_doc(st, WEEK_PREFIX + before[-1]).get("asins") or {}) if before else {} + with _reader(tenant_key, st) as cur: + snap = snapshot(cur, carry) + doc = {"week": week, "capturedAt": _stamp(now), **snap} + st.update(WEEK_PREFIX + week, lambda _old: doc) + + def _add(index): + index = index if isinstance(index, dict) else {} + index["weeks"] = sorted({*(index.get("weeks") or ()), week}) + index["last_sync"] = _stamp(now) + index.pop("last_error", None) + return index + st.update(KEY_INDEX, _add) + held = [w for w in _index(st) if w < week] + computed = {} + if held: + prev = _doc(st, WEEK_PREFIX + held[-1]).get("asins") or {} + computed = diagnose(prev, snap["asins"], _limits(st, tenant_key), week) + owner = settings(tenant_key, st=st)["defaultOwner"] + due = (now.date() + dt.timedelta(days=DUE_DAYS)).isoformat() + + def _items(existing): + existing = existing if isinstance(existing, dict) else {} + rows = existing.get("items") or {} + merged = {} + for iid, row in rows.items(): + if iid in computed or row.get("history"): + merged[iid] = row if iid in computed else {**row, "recovered": True} + for iid, fields in computed.items(): + old = merged.get(iid) or {} + merged[iid] = {**fields, "status": old.get("status") or "Open", "owner": old.get("owner") or owner, + "due": old.get("due") or due, "note": old.get("note"), + "history": list(old.get("history") or ()), "createdAt": old.get("createdAt") or _stamp(now)} + return {"week": week, "items": merged} + st.update(ITEMS_PREFIX + week, _items) + return {"week": week, "asins": len(snap["asins"]), "items": len(computed), "problems": snap["problems"]} + + +def on_amazon_sync(tenant_key, synced_tables, *, st=None, now=None): + """C5 callback: record this week's history point when a source table synced. Never raises.""" + try: + if not set(synced_tables or ()) & set(TRIGGERS): + return {"state": "skipped"} + return record_week(tenant_key, st=st, now=now) + except Exception as exc: # noqa: BLE001 + try: + def _note(doc): + doc = doc if isinstance(doc, dict) else {} + doc["last_error"] = f"The weekly history could not be written ({type(exc).__name__}): {exc}" + return doc + _rt(tenant_key, st).update(KEY_INDEX, _note) + except Exception: # noqa: BLE001 + pass + return {"state": "failed", "error": type(exc).__name__} + + +# --------------------------------------------------------------------------- C4 + +def _all_items(st): + out = [] + for week in reversed(_index(st)): + for iid, row in (_doc(st, ITEMS_PREFIX + week).get("items") or {}).items(): + out.append({"id": iid, **{k: v for k, v in row.items() if k != "history"}}) + out.sort(key=lambda i: (-int(i["week"].replace("-", "")), CAUSE_ORDER.get(i.get("causeKey"), 99), i["asin"])) + return out + + +def _page(rows, cursor, limit): + try: + start = int(cursor or 0) + size = int(limit) + except (TypeError, ValueError): + raise ValueError("The page cursor is not valid.") from None + if start < 0 or size < 1: + raise ValueError("The page cursor is not valid.") + end = start + size + return rows[start:end], (str(end) if end < len(rows) else None) + + +def _problems(st): + held = _index(st) + out = list(_doc(st, WEEK_PREFIX + held[-1]).get("problems") or ()) if held else [ + "No weekly history yet. The first point is written at the next Amazon sync."] + if len(held) == 1: + out.append(NOT_READY) + error = _doc(st, KEY_INDEX).get("last_error") + if error: + out.append(error) + return out + + +def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, week=None, asin=None, st=None): + """C4: owned drop items, newest week first, root causes first, one page. Counts cover every item.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + st = _rt(tenant_key, st) + every = [i for i in _all_items(st) if (week is None or i["week"] == week) and (asin is None or i["asin"] == asin)] + counts = {s: 0 for s in STATUSES} + for item in every: + counts[item["status"]] = counts.get(item["status"], 0) + 1 + wanted = [i for i in every if status is None or i["status"] == status] + page, nxt = _page(wanted, cursor, limit) + held = _index(st) + return {"items": page, "cursor": nxt, "counts": counts, "problems": _problems(st), + "week": week or (held[-1] if held else None), "weeks": held} + + +def item_detail(tenant_key, user, iid, *, st=None): + """One item with its history, the two points it compares, and its cause card.""" + st = _rt(tenant_key, st) + week = _week_of_id(iid) + row = (_doc(st, ITEMS_PREFIX + week).get("items") or {}).get(iid) if week else None + if not isinstance(row, dict): + raise KeyError(iid) + held = _index(st) + before = [w for w in held if w < week] + prev = (_doc(st, WEEK_PREFIX + before[-1]).get("asins") or {}).get(row["asin"]) if before else None + cur = (_doc(st, WEEK_PREFIX + week).get("asins") or {}).get(row["asin"]) + return {"item": {"id": iid, **{k: v for k, v in row.items() if k != "history"}}, + "history": list(row.get("history") or ()), "previous": prev, "current": cur, + "previousWeek": before[-1] if before else None, + "card": {"cause": row.get("cause"), "fix": row.get("fix"), "reason": row.get("reason")}} + + +def set_status(tenant_key, user, iid, status, *, note=None, owner=None, due=None, st=None): + """C4: move one item, optionally its owner and due date. Every change appends a history event.""" + st = _rt(tenant_key, st) + who = _editor(st, user) + if status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + if due is not None: + try: + due = dt.date.fromisoformat(_text(due)[:10]).isoformat() + except ValueError: + raise ValueError("Due must be a date such as 2026-10-30.") from None + week = _week_of_id(iid) + if not week or iid not in (_doc(st, ITEMS_PREFIX + week).get("items") or {}): + raise KeyError(iid) + at = _stamp() + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + row = doc.setdefault("items", {}).setdefault(iid, {}) + row["status"] = status + event = {"at": at, "by": who, "event": "status", "status": status} + if owner is not None and _text(owner): + row["owner"] = event["owner"] = _text(owner) + if due is not None: + row["due"] = event["due"] = due + if note is not None: + row["note"] = str(note) + if note: + event["note"] = str(note) + row.setdefault("history", []).append(event) + return doc + st.update(ITEMS_PREFIX + week, _apply) + row = _doc(st, ITEMS_PREFIX + week)["items"][iid] + return {"id": iid, **{k: v for k, v in row.items() if k != "history"}} + + +def proposals(tenant_key, *, st=None): + """C4: this App does not propose (C1 proposes False).""" + return [] + + +def on_decision(tenant_key, iid, decision, reason, *, st=None): + """C4: a no-op that accepts the two decisions. Nothing changes and nothing goes to Amazon.""" + if decision not in DECISIONS: + raise ValueError("Decision must be approved or dismissed.") + return {"id": iid, "decision": decision, "applied": False} + + +def asins(tenant_key, user=None, *, cursor=None, limit=DEFAULT_LIMIT, week=None, st=None): + """The ASIN picker: every ASIN in the week's history (newest by default), most sales first. No cap.""" + st = _rt(tenant_key, st) + held = _index(st) + week = week or (held[-1] if held else None) + points = (_doc(st, WEEK_PREFIX + week).get("asins") or {}) if week else {} + flagged = {i["asin"]: i["causeKey"] for i in _all_items(st) if i["week"] == week} + rows = [{"asin": a, "title": p.get("title"), "skus": p.get("skus") or [], "units": p.get("units"), + "sales": p.get("sales"), "cause": flagged.get(a)} for a, p in points.items()] + rows.sort(key=lambda r: (-(r["sales"] or 0.0), r["asin"])) + page, nxt = _page(rows, cursor, limit) + return {"asins": page, "cursor": nxt, "total": len(rows), "week": week} + + +SIGNALS = ("sessions", "buyBox", "units", "salesRank", "searchShare", "adSpend") + + +def trend(tenant_key, asin, *, st=None): + """Every held week's point for one ASIN, oldest first: one series per signal. No cap.""" + st = _rt(tenant_key, st) + points = [] + for week in _index(st): + point = (_doc(st, WEEK_PREFIX + week).get("asins") or {}).get(asin) + if point is not None: + points.append({"week": week, **{s: point.get(s) for s in SIGNALS}}) + if not points: + raise KeyError(asin) + return {"asin": asin, "points": points, "signals": list(SIGNALS)} + + +def validate(tenant_key, *, week=None, start=None, end=None, st=None): + """Rule 8 and P10: each week's history sessions and units against the Sales and Traffic SQL sum. + + The newest week is read live from the mirror. An older week uses the SQL sum taken at its sync, + because Sales and Traffic rows carry no date (F5) and the next sync replaces them.""" + st = _rt(tenant_key, st) + held = _index(st) + chosen = [w for w in held if (week is None or w == week) and (start is None or w >= str(start)[:10]) + and (end is None or w < str(end)[:10])] + if not chosen: + return {"ok": False, "app_total": None, "independent_total": None, "weeks": [], + "detail": "No weekly history is held for that period, so nothing can be tied."} + live = None + if held and held[-1] in chosen: + with _reader(tenant_key, st) as cur: + live = source_totals(cur) if cur is not None else None + rows, ok = [], True + app = {"sessions": 0.0, "units": 0.0} + ind = {"sessions": 0.0, "units": 0.0} + for w in chosen: + doc = _doc(st, WEEK_PREFIX + w) + points = doc.get("asins") or {} + mine = {"sessions": round(sum(p.get("sessions") or 0.0 for p in points.values()), 4), + "units": round(sum(p.get("units") or 0.0 for p in points.values()), 4), + "asinsWithUnits": sum(1 for p in points.values() if (p.get("units") or 0) > 0)} + theirs = live if w == held[-1] else doc.get("sourceTotals") + tied = bool(theirs) and all(abs(mine[k] - float(theirs.get(k) or 0)) < 0.005 for k in ("sessions", "units")) \ + and mine["asinsWithUnits"] == int(theirs.get("asinsWithUnits") or 0) + ok = ok and tied + for k in ("sessions", "units"): + app[k] += mine[k] + ind[k] += float((theirs or {}).get(k) or 0.0) + rows.append({"week": w, "ok": tied, "app": mine, "independent": theirs, + "basis": "live mirror" if w == held[-1] else "sum taken at sync"}) + bad = [r["week"] for r in rows if not r["ok"]] + detail = (f"{len(rows)} weeks tie sessions and units to the Sales and Traffic totals." if ok else + "These weeks do not tie to the Sales and Traffic totals: " + ", ".join(bad) + ".") + return {"ok": ok, "app_total": {k: round(v, 4) for k, v in app.items()}, + "independent_total": {k: round(v, 4) for k, v in ind.items()}, "weeks": rows, "detail": detail} + + +def keys(tenant_key, *, st=None): + """Every store key this App owns for the tenant.""" + st = _rt(tenant_key, st) + held = _index(st) + return [KEY_INDEX, KEY_SETTINGS, *(WEEK_PREFIX + w for w in held), *(ITEMS_PREFIX + w for w in held)] + + +def remove(tenant_key, *, st=None): + """Rule 13: empty every App document, history and items included. No mirror table is touched.""" + st = _rt(tenant_key, st) + owned = keys(tenant_key, st=st) + for key in owned: + st.update(key, lambda _doc: {}) + return {"removed": owned} diff --git a/platform/modules/reimbursements.py b/platform/modules/reimbursements.py new file mode 100644 index 0000000000000000000000000000000000000000..49685bddff29c42eef015580569aada7fb156006 --- /dev/null +++ b/platform/modules/reimbursements.py @@ -0,0 +1,656 @@ +"""Reimbursement recovery App: units Amazon lost, damaged or refunded without a return, still unpaid. + +Sources (read only, through the tenant Amazon mirror, never written): + amzn_inventory_ledger_detail lost and damaged adjustments, found units + amzn_reimbursements what Amazon paid (reimbursement_id is NOT unique, F30: never dedupe by it) + amzn_fba_customer_returns the evidence that a refunded unit came back + amzn_settlements refunds (assumption: the settlement report is the refund source) + +Assumptions (open to amendment): + - A refund is a settlement line with transaction_type Refund, amount_type ItemPrice and + amount_description Principal, keyed by (order_id, sku). Blank quantity_purchased reads as 1. + - A refund with no returns report row and no reimbursement for the same order and SKU is unreturned. + - The damaged reason codes follow third party guides, not an Amazon reference (see DAMAGED_REASONS). + +Store documents (tenant runtime store, App owned; computed fields come from the mirror on every read): + reimbursements_items {items: {id: {status, owner, caseId, note, history[], dismissed, approvedAt}}} + reimbursements_settings {default_owner} + +Nothing here files a case with Amazon: there is no case API. A person files and records the case id. +Limit (rule 1): every list is uncapped and paged by cursor. The items document holds overrides only. +""" +import contextlib +import datetime as dt +import hashlib +import re +from pathlib import Path + +APP_LABEL = "Reimbursements" +STATUSES = ("To file", "Filed", "Reimbursed", "Denied") +DECISIONS = ("approved", "dismissed") +SOURCE_KIND = "reimbursements" +ACTION = "File a reimbursement case" +DEFAULT_LIMIT = 200 + +KEY_ITEMS = "reimbursements_items" +KEY_SETTINGS = "reimbursements_settings" +KEYS = (KEY_ITEMS, KEY_SETTINGS) + +LEDGER_TABLE = "amzn_inventory_ledger_detail" +REIMBURSEMENTS_TABLE = "amzn_reimbursements" +RETURNS_TABLE = "amzn_fba_customer_returns" +SETTLEMENTS_TABLE = "amzn_settlements" + +# Claim windows from Amazon, read 2026-10-23. +POLICY_READ_DATE = "2026-10-23" +#: FBA inventory reimbursement policy (Seller Central login required). +POLICY_URL = "https://sellercentral.amazon.com/gp/help/external/G200213130" +#: News_Amazon, effective 2024-09-05: lost or damaged in a fulfillment center, 60 days. +WAREHOUSE_WINDOW_URL = ("https://sellercentral.amazon.com/seller-forums/discussions/t/" + "96a05f4a-537a-49d2-a918-703c47716bc0") +#: News_Amazon, effective 2024-10-23: warehouse 60 days, customer return claims from 60 to 120 days. +RETURN_WINDOW_URL = ("https://sellercentral.amazon.com/seller-forums/discussions/t/" + "81c3235d-4c44-47ba-96c5-883cecab3244") +WAREHOUSE_CLAIM_DAYS = 60 +REFUND_CLAIM_OPENS_DAYS = 60 +REFUND_CLAIM_DAYS = 120 + +LOST_REASONS = ("M", "5") +FOUND_REASONS = ("F",) +#: Assumption: damaged in Amazon's network, per third party ledger guides, not an Amazon reference. +DAMAGED_REASONS = ("E", "6", "7", "H") +LOST_REIMBURSEMENT = "Lost_Warehouse" +DAMAGED_REIMBURSEMENT = "Damaged_Warehouse" + +KINDS = {"lost": "Lost in a fulfillment center", "damaged": "Damaged in a fulfillment center", + "refund": "Refunded, not returned"} +D2682 = ("The inventory ledger reads with quoted values (D-2682), so lost and damaged units cannot be " + "counted. They return after the next Amazon sync repairs the ledger.") +READ_ONLY = "Your account can view reimbursements but not change them." + +#: Accounts that sign in for a process, never a person who can own an item. +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _today(): + return _now().date() + + +def _stamp(at=None): + return (at or _now()).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + try: + value = st.get(key) + except Exception: # noqa: BLE001 + value = None + return value if isinstance(value, dict) else {} + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + if cur is None: + return set() + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _rows(cur, table, wanted, where=""): + """Every row as {column: text or None}. A missing column reads None. No cap (rule 1).""" + cols = _columns(cur, table) + if not cols: + return [] + exprs = [f'CAST("{c}" AS VARCHAR)' if c in cols else "NULL" for c in wanted] + got = cur.execute(f'SELECT {", ".join(exprs)} FROM {table} {where}').fetchall() + return [dict(zip(wanted, row)) for row in got] + + +def _text(value): + return "" if value is None else str(value).strip() + + +def _num(value): + try: + return float(_text(value)) + except ValueError: + return None + + +def _date(value): + """A date from ISO text, DD.MM.YYYY (settlements, F32) or MM/DD/YYYY. None when unreadable.""" + raw = _text(value)[:10] + if not raw: + return None + with contextlib.suppress(ValueError): + return dt.date.fromisoformat(raw) + for fmt in ("%d.%m.%Y", "%m/%d/%Y"): + with contextlib.suppress(ValueError): + return dt.datetime.strptime(raw, fmt).date() + return None + + +LEDGER_COLS = ("date", "fnsku", "asin", "msku", "title", "event_type", "reference_id", "quantity", + "fulfillment_center", "disposition", "reason", "country", "date_and_time") +REIMBURSEMENT_COLS = ("approval_date", "reimbursement_id", "case_id", "amazon_order_id", "reason", "sku", + "fnsku", "asin", "product_name", "condition", "currency_unit", "amount_per_unit", + "amount_total", "quantity_reimbursed_cash", "quantity_reimbursed_inventory", + "quantity_reimbursed_total", "original_reimbursement_id", "original_reimbursement_type") +RETURN_COLS = ("return_date", "order_id", "sku", "asin", "fnsku", "product_name", "quantity", + "fulfillment_center_id", "detailed_disposition", "reason", "status", "license_plate_number") +SETTLEMENT_COLS = ("settlement_id", "transaction_type", "order_id", "amazon_order_id", "sku", + "quantity_purchased", "amount_type", "amount_description", "amount", "posted_date", + "posted_date_time") + + +def _ledger_quoted(cur): + """D-2682: True when msku or fnsku values are quoted, or a quantity on an Adjustments row does not cast.""" + cols = _columns(cur, LEDGER_TABLE) + if not cols: + return False + quoted = [f"""CAST("{c}" AS VARCHAR) LIKE '"%"'""" for c in ("msku", "fnsku", "event_type") if c in cols] + if quoted and cur.execute(f"SELECT COUNT(*) FROM {LEDGER_TABLE} WHERE {' OR '.join(quoted)}").fetchone()[0]: + return True + if not {"event_type", "quantity"} <= cols: + return False + bad = cur.execute( + f"""SELECT COUNT(*) FROM {LEDGER_TABLE} WHERE CAST("event_type" AS VARCHAR) = 'Adjustments' """ + f"""AND "quantity" IS NOT NULL """ + f"""AND TRY_CAST(TRIM(CAST("quantity" AS VARCHAR)) AS DOUBLE) IS NULL""").fetchone()[0] + return bool(bad) + + +def _read(tenant_key, st): + """Every source row the App needs, plus the problems met reading them.""" + out = {"ledger": [], "reimbursements": [], "returns": [], "refunds": [], "problems": [], "mirror": True} + with _reader(tenant_key, st) as cur: + if cur is None: + out["mirror"] = False + out["problems"].append("The Amazon mirror is not on this server yet. It appears after the first sync.") + return out + if _ledger_quoted(cur): + out["problems"].append(D2682) + else: + out["ledger"] = _rows(cur, LEDGER_TABLE, LEDGER_COLS, + """WHERE CAST("event_type" AS VARCHAR) = 'Adjustments'""") + out["reimbursements"] = _rows(cur, REIMBURSEMENTS_TABLE, REIMBURSEMENT_COLS) + out["returns"] = _rows(cur, RETURNS_TABLE, RETURN_COLS) + cols = _columns(cur, SETTLEMENTS_TABLE) + if {"transaction_type", "amount_type", "amount_description"} <= cols: + out["refunds"] = _rows(cur, SETTLEMENTS_TABLE, SETTLEMENT_COLS, + """WHERE CAST("transaction_type" AS VARCHAR) = 'Refund' """ + """AND CAST("amount_type" AS VARCHAR) = 'ItemPrice' """ + """AND CAST("amount_description" AS VARCHAR) = 'Principal'""") + for table, label in ((LEDGER_TABLE, "inventory ledger"), (REIMBURSEMENTS_TABLE, "reimbursements report"), + (RETURNS_TABLE, "customer returns report"), (SETTLEMENTS_TABLE, "settlement report")): + if not _columns(cur, table): + out["problems"].append(f"The Amazon {label} has no rows in the mirror yet.") + return out + + +# --------------------------------------------------------------------------- owner + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def _owner_account(st, tenant_key=None): + """The tenant's earliest active person Administrator, else its `admin` account.""" + users = _doc(st, "users") + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in users.items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def _default_owner(st, tenant_key): + return _doc(st, KEY_SETTINGS).get("default_owner") or _owner_account(st, tenant_key) + + +def _editor(st, user): + """The editing account's name. PermissionError for a viewer, an inactive or an unknown account.""" + name, rec = None, user + if isinstance(user, str): + name = user.strip() + rec = _doc(st, "users").get(name) + if not isinstance(rec, dict) or rec.get("active") is False: + raise PermissionError(READ_ONLY) + role = str(rec.get("role") or "user").strip().lower() + if role == "viewer" or rec.get("can_edit") is False or rec.get("canEdit") is False: + raise PermissionError(READ_ONLY) + return str(rec.get("username") or name or rec.get("name") or "unknown") + + +# --------------------------------------------------------------------------- claims + +def _item_id(kind, parts): + raw = "|".join([kind, *(_text(p) for p in parts)]) + return "r" + hashlib.sha1(raw.encode("utf-8")).hexdigest()[:12] + + +def _plural(n, word="unit"): + return f"{n:g} {word}" + ("" if n == 1 else "s") + + +def _per_unit(reimbursements): + """{("fnsku" | "sku", value): (amount_per_unit, currency)} from the newest approval with an amount.""" + newest = {} + for row in reimbursements: + amount = _num(row.get("amount_per_unit")) + if amount is None: + continue + when = _date(row.get("approval_date")) or dt.date.min + for key in (("fnsku", _text(row.get("fnsku"))), ("sku", _text(row.get("sku")))): + if key[1] and (key not in newest or when >= newest[key][0]): + newest[key] = (when, amount, _text(row.get("currency_unit")) or None) + return {k: (v[1], v[2]) for k, v in newest.items()} + + +def _ledger_claims(ledger, reimbursements, filed_cases): + """Lost and damaged events with units left after found and reimbursed units, oldest offset first.""" + found, paid = {}, {} + for row in ledger: + qty = _num(row.get("quantity")) + if _text(row.get("reason")) in FOUND_REASONS and qty is not None and qty > 0: + found[_text(row.get("fnsku"))] = found.get(_text(row.get("fnsku")), 0.0) + qty + for row in reimbursements: + if _text(row.get("case_id")) and _text(row.get("case_id")) in filed_cases: + continue # Paid on a case filed here: the item stays and reads Reimbursed. + kind = {LOST_REIMBURSEMENT: "lost", DAMAGED_REIMBURSEMENT: "damaged"}.get(_text(row.get("reason"))) + qty = _num(row.get("quantity_reimbursed_total")) + if kind and qty: + key = (kind, _text(row.get("fnsku"))) + paid[key] = paid.get(key, 0.0) + qty + events = {} + for row in ledger: + reason, qty = _text(row.get("reason")), _num(row.get("quantity")) + kind = "lost" if reason in LOST_REASONS else "damaged" if reason in DAMAGED_REASONS else None + if kind is None or qty is None or qty >= 0: + continue + events.setdefault((kind, _text(row.get("fnsku"))), []).append(row) + claims, seen = [], {} + for (kind, fnsku), rows in sorted(events.items()): + offset = paid.get((kind, fnsku), 0.0) + (found.get(fnsku, 0.0) if kind == "lost" else 0.0) + rows.sort(key=lambda r: (_date(r.get("date")) or dt.date.min, _text(r.get("date_and_time")), + _text(r.get("reference_id")))) + for row in rows: + units = -_num(row.get("quantity")) + used = min(units, offset) + offset -= used + left = units - used + parts = [row.get("date"), fnsku, row.get("reference_id"), row.get("reason"), + row.get("fulfillment_center"), row.get("quantity")] + base = _item_id(kind, parts) + seen[base] = seen.get(base, 0) + 1 + if left <= 0: + continue + item_id = base if seen[base] == 1 else _item_id(kind, [*parts, seen[base]]) + when = _date(row.get("date")) + claims.append({"id": item_id, "kind": kind, "units": left, "eventDate": when, + "claimBy": when + dt.timedelta(days=WAREHOUSE_CLAIM_DAYS) if when else None, + "sku": _text(row.get("msku")) or None, "asin": _text(row.get("asin")) or None, + "fnsku": fnsku or None, "title": _text(row.get("title")) or None, + "fc": _text(row.get("fulfillment_center")) or None, + "reference": _text(row.get("reference_id")) or None, + "lostUnits": units, "source": row}) + return claims + + +def _refund_claims(refunds, returns, reimbursements, filed_cases, today): + """Refunds at least 60 days old with no return row and no reimbursement for the same order and SKU.""" + returned = {(_text(r.get("order_id")), _text(r.get("sku"))) for r in returns} + paid = {(_text(r.get("amazon_order_id")), _text(r.get("sku"))) for r in reimbursements + if not (_text(r.get("case_id")) and _text(r.get("case_id")) in filed_cases)} + groups = {} + for row in refunds: + order = _text(row.get("order_id")) or _text(row.get("amazon_order_id")) + sku = _text(row.get("sku")) + if order and sku: + groups.setdefault((order, sku), []).append(row) + claims = [] + for (order, sku), rows in sorted(groups.items()): + if (order, sku) in returned or (order, sku) in paid: + continue + dates = [d for d in (_date(r.get("posted_date")) or _date(r.get("posted_date_time")) for r in rows) if d] + when = min(dates) if dates else None + if when is None or (today - when).days < REFUND_CLAIM_OPENS_DAYS: + continue + units = 0.0 + for r in rows: + q = _num(r.get("quantity_purchased")) + units += abs(q) if q else 1.0 + claims.append({"id": _item_id("refund", [order, sku]), "kind": "refund", "units": units, + "eventDate": when, "claimBy": when + dt.timedelta(days=REFUND_CLAIM_DAYS), + "sku": sku, "asin": None, "fnsku": None, "title": None, "fc": None, + "reference": order, "order": order, "source": rows}) + return claims + + +def _reason(claim, value): + units = _plural(claim["units"]) + day = claim["eventDate"].isoformat() if claim["eventDate"] else "an unknown date" + if claim["kind"] == "refund": + text = f"Amazon refunded {units} on order {claim['order']} on {day} and no return or reimbursement followed." + else: + verb = "lost" if claim["kind"] == "lost" else "damaged" + are, them = ("is", "it") if claim["units"] == 1 else ("are", "them") + text = (f"Amazon marked {_plural(claim['lostUnits'])} {verb} on {day}. " + f"{units.capitalize()} {are} not found or reimbursed." if claim["lostUnits"] != claim["units"] + else f"Amazon marked {units} {verb} on {day} and has not found or reimbursed {them}.") + if value is None: + text += " No reimbursement history for this SKU, so the value is blank." + return text + + +def _compute(tenant_key, st): + """(every item, problems, source read). Items carry their store overrides.""" + overrides = _doc(st, KEY_ITEMS).get("items") or {} + filed = {_text(o.get("caseId")) for o in overrides.values() if isinstance(o, dict) and _text(o.get("caseId"))} + src = _read(tenant_key, st) + today = _today() + claims = (_ledger_claims(src["ledger"], src["reimbursements"], filed) + + _refund_claims(src["refunds"], src["returns"], src["reimbursements"], filed, today)) + per_unit = _per_unit(src["reimbursements"]) + paid_cases = {_text(r.get("case_id")) for r in src["reimbursements"] if _text(r.get("case_id"))} + owner = _default_owner(st, tenant_key) + items = [] + for claim in claims: + o = overrides.get(claim["id"]) if isinstance(overrides.get(claim["id"]), dict) else {} + unit = per_unit.get(("fnsku", claim["fnsku"] or "")) or per_unit.get(("sku", claim["sku"] or "")) + value = round(claim["units"] * unit[0], 2) if unit else None + status = o.get("status") if o.get("status") in STATUSES else "To file" + case = _text(o.get("caseId")) or None + if status == "Filed" and case and case in paid_cases: + status = "Reimbursed" + claim_by = claim["claimBy"] + label = claim["sku"] or claim["fnsku"] or claim["reference"] or "Unknown SKU" + items.append({ + "id": claim["id"], "title": f"{_plural(claim['units'])} {KINDS[claim['kind']].lower()}: {label}", + "status": status, "owner": o.get("owner") or owner, + "due": claim_by.isoformat() if claim_by else None, "reason": _reason(claim, value), + "sku": claim["sku"], "asin": claim["asin"], "fnsku": claim["fnsku"], "kind": claim["kind"], + "units": int(claim["units"]) if float(claim["units"]).is_integer() else claim["units"], + "value": value, "currency": unit[1] if unit else None, + "claimBy": claim_by.isoformat() if claim_by else None, + "expired": bool(claim_by and claim_by < today), "caseId": case, + "eventDate": claim["eventDate"].isoformat() if claim["eventDate"] else None, + "dismissed": o.get("dismissed") is True, "_claim": claim}) + items.sort(key=lambda i: (i["claimBy"] or "9999-12-31", i["id"])) + return items, src["problems"], src + + +def _public(item): + return {k: v for k, v in item.items() if not k.startswith("_")} + + +def _page(rows, cursor, limit): + try: + start = int(cursor or 0) + size = int(limit) + except (TypeError, ValueError): + raise ValueError("The page cursor is not valid.") from None + if start < 0 or size < 1: + raise ValueError("The page cursor is not valid.") + end = start + size + return rows[start:end], (str(end) if end < len(rows) else None) + + +# --------------------------------------------------------------------------- C4 + +def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, st=None): + """C4: claim items, soonest claim-by first, one page. Counts cover every item. No cap (rule 1).""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + st = _rt(tenant_key, st) + every, problems, _src = _compute(tenant_key, st) + counts = {s: 0 for s in STATUSES} + for item in every: + counts[item["status"]] += 1 + wanted = [i for i in every if status is None or i["status"] == status] + page, nxt = _page(wanted, cursor, limit) + return {"items": [_public(i) for i in page], "cursor": nxt, "counts": counts, "problems": problems} + + +def _case_text(item, claim): + ids = ", ".join(f"{name} {value}" for name, value in (("FNSKU", item["fnsku"]), ("SKU", item["sku"]), + ("ASIN", item["asin"])) if value) + day = item["eventDate"] or "an unknown date" + units = _plural(item["units"]) + if item["kind"] == "refund": + return (f"Amazon refunded order {claim['order']} for {units} of {ids} on {day}. " + f"The customer returns report shows no return for this order and SKU, and no reimbursement was paid. " + f"Please reimburse {units} under the FBA customer return reimbursement policy.") + verb = "lost" if item["kind"] == "lost" else "damaged" + where = f"at fulfillment center {claim['fc']}" if claim.get("fc") else "in a fulfillment center" + ref = f" The inventory ledger reference ID is {claim['reference']}." if claim.get("reference") else "" + return (f"Amazon marked {units} of {ids} as {verb} {where} on {day}.{ref} " + f"These units were not found or reimbursed. " + f"Please reimburse {units} under the FBA inventory reimbursement policy.") + + +def item_detail(tenant_key, user, item_id, *, st=None): + """One item, its source ledger events (or refund and return rows), and case text ready to copy.""" + st = _rt(tenant_key, st) + every, problems, src = _compute(tenant_key, st) + item = next((i for i in every if i["id"] == item_id), None) + if item is None: + raise KeyError(item_id) + claim = item["_claim"] + if item["kind"] == "refund": + events = list(claim["source"]) + returns = [r for r in src["returns"] + if _text(r.get("order_id")) == claim["order"] and _text(r.get("sku")) == claim["sku"]] + paid = [r for r in src["reimbursements"] + if _text(r.get("amazon_order_id")) == claim["order"] and _text(r.get("sku")) == claim["sku"]] + else: + reasons = (LOST_REASONS + FOUND_REASONS) if item["kind"] == "lost" else DAMAGED_REASONS + events = [r for r in src["ledger"] + if _text(r.get("fnsku")) == (item["fnsku"] or "") and _text(r.get("reason")) in reasons] + returns = [] + want = LOST_REIMBURSEMENT if item["kind"] == "lost" else DAMAGED_REIMBURSEMENT + paid = [r for r in src["reimbursements"] + if _text(r.get("fnsku")) == (item["fnsku"] or "") and _text(r.get("reason")) == want] + override = (_doc(st, KEY_ITEMS).get("items") or {}).get(item_id) or {} + return {"item": _public(item), "events": events, "returns": returns, "reimbursements": paid, + "history": list(override.get("history") or ()), "note": override.get("note"), + "caseText": _case_text(item, claim), "problems": problems} + + +_CASE_RE = re.compile(r"\b(\d{8,13})\b") + + +def set_status(tenant_key, user, item_id, status, *, note=None, case_id=None, st=None): + """C4: move one item. Filed keeps the case id. Every change appends a history event.""" + st = _rt(tenant_key, st) + who = _editor(st, user) + if status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + every, _problems, _src = _compute(tenant_key, st) + if not any(i["id"] == item_id for i in every): + raise KeyError(item_id) + case = _text(case_id) + if not case and note: + found = _CASE_RE.search(str(note)) + case = found.group(1) if found else "" + at = _stamp() + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + row = doc.setdefault("items", {}).setdefault(item_id, {}) + row["status"] = status + if case: + row["caseId"] = case + if note is not None: + row["note"] = str(note) + event = {"at": at, "by": who, "event": "status", "status": status} + if case: + event["caseId"] = case + if note: + event["note"] = str(note) + row.setdefault("history", []).append(event) + return doc + st.update(KEY_ITEMS, _apply) + return next(_public(i) for i in _compute(tenant_key, st)[0] if i["id"] == item_id) + + +def paid(tenant_key, user=None, *, cursor=None, limit=DEFAULT_LIMIT, st=None): + """Every reimbursement row Amazon paid, newest approval first, one page. Rows are never deduped (F30).""" + st = _rt(tenant_key, st) + with _reader(tenant_key, st) as cur: + rows = _rows(cur, REIMBURSEMENTS_TABLE, REIMBURSEMENT_COLS) if cur is not None else [] + rows.sort(key=lambda r: (_date(r.get("approval_date")) or dt.date.min, _text(r.get("approval_date"))), + reverse=True) + page, nxt = _page(rows, cursor, limit) + return {"rows": page, "cursor": nxt, "total": len(rows)} + + +def proposals(tenant_key, *, st=None): + """C7: each To file item once. Expired and dismissed items are never proposed.""" + st = _rt(tenant_key, st) + out, seen = [], set() + for item in _compute(tenant_key, st)[0]: + if item["status"] != "To file" or item["expired"] or item["dismissed"] or item["id"] in seen: + continue + seen.add(item["id"]) + out.append({"source_kind": SOURCE_KIND, "item_id": item["id"], "sku": item["sku"], + "action": ACTION, "reason": item["reason"] + f" Claim by {item['claimBy']}."}) + return out + + +def on_decision(tenant_key, item_id, decision, reason, *, st=None): + """C7: approved records approvedAt, status unchanged. Dismissed stops proposing it. Nothing goes to Amazon.""" + if decision not in DECISIONS: + raise ValueError("Decision must be approved or dismissed.") + st = _rt(tenant_key, st) + if not any(i["id"] == item_id for i in _compute(tenant_key, st)[0]): + raise KeyError(item_id) + at = _stamp() + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + row = doc.setdefault("items", {}).setdefault(item_id, {}) + if decision == "approved": + row["approvedAt"] = at + else: + row["dismissed"] = True + event = {"at": at, "by": "operator", "event": decision} + if reason: + event["reason"] = str(reason) + row.setdefault("history", []).append(event) + return doc + st.update(KEY_ITEMS, _apply) + return {"id": item_id, "decision": decision} + + +def validate(tenant_key, *, start=None, end=None, st=None): + """Rule 8: the paid list's amount total for approvals in [start, end) against one SQL sum.""" + lo = _date(start) if start else None + hi = _date(end) if end else None + if (start and lo is None) or (end and hi is None): + raise ValueError("Start and end must be dates such as 2026-10-01.") + st = _rt(tenant_key, st) + app_total, count, cursor = 0.0, 0, None + while True: + page = paid(tenant_key, cursor=cursor, st=st) + for row in page["rows"]: + when = _date(row.get("approval_date")) + if when is None or (lo and when < lo) or (hi and when >= hi): + continue + app_total += _num(row.get("amount_total")) or 0.0 + count += 1 + cursor = page["cursor"] + if cursor is None: + break + with _reader(tenant_key, st) as cur: + cols = _columns(cur, REIMBURSEMENTS_TABLE) + if not {"amount_total", "approval_date"} <= cols: + return {"ok": False, "app_total": round(app_total, 2), "independent_total": None, + "detail": "The reimbursements report is not in the mirror yet, so nothing can be tied."} + day = 'TRY_CAST(SUBSTR(CAST("approval_date" AS VARCHAR), 1, 10) AS DATE)' + where, args = [f"{day} IS NOT NULL"], [] + if lo: + where.append(f"{day} >= ?") + args.append(lo) + if hi: + where.append(f"{day} < ?") + args.append(hi) + independent = cur.execute( + f'SELECT COALESCE(SUM(TRY_CAST("amount_total" AS DOUBLE)), 0) FROM {REIMBURSEMENTS_TABLE} ' + f'WHERE {" AND ".join(where)}', args).fetchone()[0] + app_total, independent = round(app_total, 2), round(float(independent or 0), 2) + ok = abs(app_total - independent) < 0.005 + period = f"from {lo or 'the first approval'} to before {hi or 'today'}" + detail = (f"{count} paid rows {period} total {app_total:.2f}. The reimbursements report sums to " + f"{independent:.2f}." + ("" if ok else " They do not agree.")) + return {"ok": ok, "app_total": app_total, "independent_total": independent, "detail": detail} + + +def remove(tenant_key, *, st=None): + """Rule 13: empty every App document. No mirror table is touched.""" + st = _rt(tenant_key, st) + for key in KEYS: + st.update(key, lambda _doc: {}) + return {"removed": list(KEYS)} diff --git a/platform/modules/restock_planner.py b/platform/modules/restock_planner.py new file mode 100644 index 0000000000000000000000000000000000000000..2a3bc93ea9bf4b94b590588f90bf7951104cf3de --- /dev/null +++ b/platform/modules/restock_planner.py @@ -0,0 +1,783 @@ +"""Restock planner App: bands every Amazon SKU by days of supply and owns one restock item per SKU. + +Store documents (tenant runtime store, App owned, never the mirror and never Amazon): + restock_planner_settings {send_to_fba_days, overstock_days, due_days, default_owner} + restock_planner_items {state, message, refreshedAt, notices, items: {id: item}, last_error} + restock_planner_history {months: ["2026-10", ...]} (the index of month documents) + restock_planner_history_ {skus: {sku: {date: point}}} + +Sources, read only from the tenant mirror: `amzn_inventory_planning`, `amzn_restock_recommendations`, +the Odoo `stock_quant`, `stock_location`, `purchase_order`, `purchase_order_line` and +`product_product` tables, and the user table `ut_amazon_sku_profit` for the SKU to Odoo link. +The planning report keeps one snapshot, so the daily history is this App's own. +Deleting this module and its documents (`remove_app_data`) leaves every connector table (rule 13). + +Limit (rule 1): each document is one JSON value, and the Postgres store refuses a document over +`core.store_pg.DOCUMENT_CEILING_BYTES` (128 MB). Nothing here caps SKUs, items or history. +""" +import base64 +import contextlib +import datetime as dt +import hashlib +import json +from pathlib import Path + +APP_LABEL = "Restock planner" +SOURCE_KIND = "restock_planner" +STATUSES = ("Open", "Accepted", "Ordered", "Sent to FBA", "Watching", "Done", "Dismissed") +SEND = "Send to FBA" +REORDER = "Reorder from vendor" +OVERSTOCK = "Overstock" +HEALTHY = "Healthy" +BANDS = (SEND, REORDER, OVERSTOCK, HEALTHY) +ACTION_BANDS = (SEND, REORDER, OVERSTOCK) +PROPOSING_BANDS = (SEND, REORDER) +#: An item in one of these statuses reopens when its SKU moves into a different action band. +REOPEN_FROM = ("Watching", "Done", "Dismissed") +DECISIONS = {"approved": "Accepted", "dismissed": "Dismissed"} +DEFAULTS = {"send_to_fba_days": 30.0, "overstock_days": 90.0, "due_days": 7} +NOT_READY = "The restock plan starts after the next Amazon inventory sync." +LIMIT = ("Each restock planner document is limited to 128 MB by the store. About 60,000 SKUs fit in " + "the items document, and history splits by month. Split the items document by band when " + "it nears that size.") + +KEY_SETTINGS = "restock_planner_settings" +KEY_ITEMS = "restock_planner_items" +KEY_HISTORY = "restock_planner_history" +HISTORY_PREFIX = "restock_planner_history_" + +PLANNING = "amzn_inventory_planning" +RESTOCK = "amzn_restock_recommendations" +LINK_TABLE = "ut_amazon_sku_profit" +SOURCE_NAMES = {PLANNING: "Amazon inventory planning", RESTOCK: "Amazon restock recommendations"} +SYNC_TRIGGERS = ("inventory_planning", "restock_recommendations") + +#: The FBA flag exactly as `odoo_relational._stock_location_cols` defines `location_fba`. +FBA_SQL = ("EXISTS (SELECT 1 FROM stock_location fba WHERE fba.name = 'FBA' AND fba.usage = 'view' " + "AND strpos('/' || COALESCE(sl.parent_path, ''), '/' || CAST(fba.id AS VARCHAR) || '/') > 0)") + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at): + return at.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + value = st.get(key) + return value if isinstance(value, dict) else {} + + +def item_id_for(sku): + """A stable item id from the SKU, so a refresh never duplicates an item.""" + return "rp_" + hashlib.sha1(str(sku).encode("utf-8")).hexdigest()[:10] + + +def _num(value): + try: + return None if value is None else float(value) + except (TypeError, ValueError): + return None + + +def _fmt(value): + number = float(value) + return str(int(round(number))) if abs(number - round(number)) < 0.05 else f"{number:.1f}" + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + if cur is None: + return set() + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _newest(cols, table): + """A WHERE clause that keeps the newest `_snapshot_date` only, or every row without one.""" + if "_snapshot_date" not in cols: + return "TRUE" + return (f'CAST("_snapshot_date" AS VARCHAR) = ' + f'(SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {table})') + + +def _expr(cols, name, agg, kind="DOUBLE"): + if name not in cols: + return "NULL" + if kind == "VARCHAR": + return f'{agg}(NULLIF(TRIM(CAST("{name}" AS VARCHAR)), \'\'))' + return f'{agg}(TRY_CAST("{name}" AS {kind}))' + + +def _planning(cur): + """{sku: planning values} from the newest snapshot, summed per SKU; None when the table is absent.""" + cols = _columns(cur, PLANNING) + if "sku" not in cols: + return None + sql = (f'SELECT TRIM(CAST("sku" AS VARCHAR)), {_expr(cols, "asin", "MAX", "VARCHAR")}, ' + f'{_expr(cols, "fnsku", "MAX", "VARCHAR")}, {_expr(cols, "product_name", "MAX", "VARCHAR")}, ' + f'{_expr(cols, "available", "SUM")}, {_expr(cols, "days_of_supply", "MAX")}, ' + f'{_expr(cols, "units_shipped_t30", "SUM")}, {_expr(cols, "inbound_quantity", "SUM")}, ' + f'{_expr(cols, "recommended_ship_in_quantity", "SUM")}, ' + f'{_expr(cols, "recommended_ship_in_date", "MIN", "VARCHAR")}, COUNT(*) ' + f'FROM {PLANNING} WHERE {_newest(cols, PLANNING)} AND "sku" IS NOT NULL ' + f'AND TRIM(CAST("sku" AS VARCHAR)) <> \'\' GROUP BY 1') + out = {} + for sku, asin, fnsku, name, avail, dos, units, inbound, qty, date, n in cur.execute(sql).fetchall(): + # Several rows for one SKU: days of supply comes from the summed quantities, not one row. + if n > 1: + dos = None + out[sku] = {"asin": asin, "fnsku": fnsku, "title": name, "available": avail, "days": dos, + "units": units, "inbound": inbound, "qty": qty, "date": date} + return out + + +def _restock(cur): + """{sku: restock values} from the newest snapshot; None when the table is absent.""" + cols = _columns(cur, RESTOCK) + if "merchant_sku" not in cols: + return None + sql = (f'SELECT TRIM(CAST("merchant_sku" AS VARCHAR)), {_expr(cols, "asin", "MAX", "VARCHAR")}, ' + f'{_expr(cols, "fnsku", "MAX", "VARCHAR")}, {_expr(cols, "product_name", "MAX", "VARCHAR")}, ' + f'{_expr(cols, "available", "SUM")}, ' + f'{_expr(cols, "days_of_supply_at_amazon_fulfillment_network", "MAX")}, ' + f'{_expr(cols, "units_sold_last_30_days", "SUM")}, ' + f'{_expr(cols, "recommended_replenishment_qty", "SUM")}, ' + f'{_expr(cols, "recommended_ship_date", "MIN", "VARCHAR")}, COUNT(*) ' + f'FROM {RESTOCK} WHERE {_newest(cols, RESTOCK)} AND "merchant_sku" IS NOT NULL ' + f'AND TRIM(CAST("merchant_sku" AS VARCHAR)) <> \'\' GROUP BY 1') + out = {} + for sku, asin, fnsku, name, avail, dos, units, qty, date, n in cur.execute(sql).fetchall(): + out[sku] = {"asin": asin, "fnsku": fnsku, "title": name, "available": avail, + "days": None if n > 1 else dos, "units": units, "qty": qty, "date": date} + return out + + +def _codes(cur): + """{default_code: [product id, ...]} from the Odoo product mirror; None when it is absent.""" + if not {"id", "default_code"} <= _columns(cur, "product_product"): + return None + out = {} + for pid, code in cur.execute( + "SELECT TRY_CAST(id AS BIGINT), TRIM(CAST(default_code AS VARCHAR)) FROM product_product " + "WHERE default_code IS NOT NULL").fetchall(): + if code: + out.setdefault(code, []).append(pid) + return out + + +def _odoo_stock(cur): + """({product id: [{warehouse, onHand, fba}]}, fba_known), or (None, False) when quants are absent.""" + quant, loc = _columns(cur, "stock_quant"), _columns(cur, "stock_location") + if not {"product_id", "location_id", "quantity"} <= quant or not {"id", "usage"} <= loc: + return None, False + fba_known = "parent_path" in loc and "name" in loc + fba = FBA_SQL if fba_known else "FALSE" + name = ("COALESCE(sl.complete_name, sl.name, '')" if "complete_name" in loc + else "COALESCE(sl.name, '')" if "name" in loc else "''") + rows = cur.execute( + f"SELECT TRY_CAST(sq.product_id AS BIGINT), TRIM(split_part(CAST({name} AS VARCHAR), '/', 1)), " + f"{fba}, SUM(TRY_CAST(sq.quantity AS DOUBLE)) FROM stock_quant sq " + f"JOIN stock_location sl ON TRY_CAST(sl.id AS BIGINT) = TRY_CAST(sq.location_id AS BIGINT) " + f"WHERE sl.usage = 'internal' GROUP BY 1, 2, 3").fetchall() + out = {} + for pid, warehouse, is_fba, qty in rows: + out.setdefault(pid, []).append({"warehouse": warehouse or "Unnamed", "onHand": float(qty or 0), + "fba": bool(is_fba)}) + return out, fba_known + + +def _open_pos(cur): + """{product id: [{order, vendor, openQty, expected}]}, or None when purchase orders are absent.""" + po, pol = _columns(cur, "purchase_order"), _columns(cur, "purchase_order_line") + if not {"id", "state"} <= po or not {"order_id", "product_id", "product_qty"} <= pol: + return None + received = "COALESCE(TRY_CAST(pol.qty_received AS DOUBLE), 0)" if "qty_received" in pol else "0" + vendor = "po.partner_id_name" if "partner_id_name" in po else "NULL" + order = "po.name" if "name" in po else "CAST(po.id AS VARCHAR)" + planned = "CAST(pol.date_planned AS VARCHAR)" if "date_planned" in pol else "NULL" + rows = cur.execute( + f"SELECT TRY_CAST(pol.product_id AS BIGINT), {order}, {vendor}, " + f"TRY_CAST(pol.product_qty AS DOUBLE) - {received}, {planned} FROM purchase_order_line pol " + f"JOIN purchase_order po ON TRY_CAST(po.id AS BIGINT) = TRY_CAST(pol.order_id AS BIGINT) " + f"WHERE po.state IN ('purchase', 'done') " + f"AND TRY_CAST(pol.product_qty AS DOUBLE) - {received} > 0").fetchall() + out = {} + for pid, name, partner, qty, expected in rows: + out.setdefault(pid, []).append({"order": name, "vendor": partner, "openQty": float(qty), + "expected": str(expected)[:10] if expected else None}) + return out + + +def _links(st): + """{sku: (odoo_product, units_per_sku)} from the user table, data only (no App code import).""" + from core import user_tables + try: + table = user_tables.get_one(LINK_TABLE, st) or {} + except Exception: # noqa: BLE001 + table = {} + out = {} + for row in (table.get("rows") or {}).values(): + if not isinstance(row, dict): + continue + sku = str(row.get("sku") or "").strip() + code = str(row.get("odoo_product") or "").strip() + if sku and code: + units = _num(row.get("units_per_sku")) + out[sku] = (code, units if units and units > 0 else 1.0) + return out + + +# --------------------------------------------------------------------------- settings and owner + +#: Accounts that sign in for a process, never a person who can own an item. +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """P22: the tenant's earliest active person Administrator, else `admin`, else its first Administrator.""" + try: + users = st.get("users") + except Exception: # noqa: BLE001 + users = None + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in (users or {}).items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def settings(tenant_key, *, st=None): + """Thresholds, the due window and the default owner, with defaults filled in.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + owner_setting = doc.get("default_owner") or None + return {"sendToFbaDays": float(doc.get("send_to_fba_days", DEFAULTS["send_to_fba_days"])), + "overstockDays": float(doc.get("overstock_days", DEFAULTS["overstock_days"])), + "dueDays": int(doc.get("due_days", DEFAULTS["due_days"])), + "defaultOwnerSetting": owner_setting, + "defaultOwner": owner_setting or owner_account(st, tenant_key), + "statuses": list(STATUSES), "bands": list(BANDS), "limit": LIMIT} + + +def set_settings(tenant_key, *, send_to_fba_days=None, overstock_days=None, due_days=None, + default_owner=None, st=None): + """Set thresholds in days of supply, the due window, or the default owner ('' resets it).""" + st = _rt(tenant_key, st) + now = settings(tenant_key, st=st) + changes = {} + for name, value in (("send_to_fba_days", send_to_fba_days), ("overstock_days", overstock_days)): + if value is None: + continue + number = _num(value) + if number is None or not 0 < number <= 3650: + raise ValueError("A threshold must be a number of days above 0.") + changes[name] = number + send = changes.get("send_to_fba_days", now["sendToFbaDays"]) + over = changes.get("overstock_days", now["overstockDays"]) + if over <= send: + raise ValueError("The overstock threshold must be above the Send to FBA threshold.") + if due_days is not None: + number = _num(due_days) + if number is None or number != int(number) or not 1 <= number <= 365: + raise ValueError("Days to act must be a whole number from 1 to 365.") + changes["due_days"] = int(number) + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(changes) + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +# --------------------------------------------------------------------------- the row builder + +def _band(row, conf): + """(band, reason) for one SKU row under the settings.""" + send, over = conf["sendToFbaDays"], conf["overstockDays"] + days, avail, units = row["daysOfSupply"], row["fbaAvailable"] or 0, row["units30d"] or 0 + if days is None: + if units <= 0 and avail > 0: + return OVERSTOCK, f"No units sold in 30 days with {_fmt(avail)} units at Amazon." + if units <= 0: + return HEALTHY, "No units sold in 30 days and no units at Amazon." + return HEALTHY, "Amazon reports no days of supply for this SKU." + head = f"{_fmt(days)} days of supply at Amazon" + if days < send: + head += f", under the {_fmt(send)} day threshold." + if row["odooFba"] is not None and row["odooFba"] > 0: + return SEND, f"{head} Odoo holds {_fmt(row['odooFba'])} units in FBA locations." + return REORDER, f"{head} {row['odooNote'] or 'Odoo holds no stock in an FBA location.'}" + if days > over: + return OVERSTOCK, f"{head}, over the {_fmt(over)} day threshold." + return HEALTHY, f"{head}, inside the {_fmt(send)} to {_fmt(over)} day range." + + +def build_rows(cur, links, conf): + """(rows, notices): one row per SKU in either Amazon report, or (None, notices) when neither synced.""" + planning, restock = _planning(cur), _restock(cur) + notices = [f"{SOURCE_NAMES[t]} has not synced yet." for t, got in ((PLANNING, planning), + (RESTOCK, restock)) if got is None] + if planning is None and restock is None: + return None, notices + planning, restock = planning or {}, restock or {} + codes = _codes(cur) + stock, fba_known = _odoo_stock(cur) + pos = _open_pos(cur) + if codes is None or stock is None: + notices.append("Odoo stock has not synced yet.") + elif not fba_known: + notices.append("Odoo locations carry no parent path, so FBA stock is unknown.") + if pos is None: + notices.append("Odoo purchase orders have not synced yet.") + rows = [] + for sku in sorted(set(planning) | set(restock)): + p, r = planning.get(sku) or {}, restock.get(sku) or {} + avail = p.get("available") if p else r.get("available") + units = p.get("units") if p.get("units") is not None else r.get("units") + days = p.get("days") if p.get("days") is not None else r.get("days") + if days is None and units and units > 0: + days = (avail or 0) * 30.0 / units + qty = r.get("qty") if r.get("qty") is not None else p.get("qty") + link, linked_by, units_per = None, None, 1.0 + if sku in links: + link, units_per = links[sku] + linked_by = "table" + elif codes is not None and len(codes.get(sku) or ()) == 1: + link, linked_by = sku, "code" + warehouses, purchase, on_hand, fba, note = [], [], None, None, None + if link is None: + note = "No Odoo product is linked to this SKU, so Odoo stock is unknown." + elif codes is None or stock is None: + note = "Odoo stock has not synced yet, so Odoo stock is unknown." + else: + ids = codes.get(link) or [] + if not ids: + note = f"Odoo has no product with the reference {link}, so Odoo stock is unknown." + else: + merged = {} + for pid in ids: + for q in stock.get(pid) or (): + key = (q["warehouse"], q["fba"]) + merged[key] = merged.get(key, 0.0) + q["onHand"] + purchase += (pos or {}).get(pid) or [] + warehouses = [{"warehouse": w, "fba": f, "onHand": v} for (w, f), v in sorted(merged.items())] + on_hand = sum(merged.values()) + if fba_known: + fba = sum(v for (_w, f), v in merged.items() if f) + else: + note = "Odoo locations carry no parent path, so FBA stock is unknown." + row = {"sku": sku, "asin": p.get("asin") or r.get("asin"), "fnsku": p.get("fnsku") or r.get("fnsku"), + "title": p.get("title") or r.get("title") or sku, + "fbaAvailable": avail, "availableSource": "planning" if p else "restock", + "inbound": p.get("inbound"), "daysOfSupply": None if days is None else round(days, 2), + "units30d": units, "recommendedQty": qty, + "recommendedShipDate": r.get("date") or p.get("date"), + "odooProduct": link, "linkedBy": linked_by, "unitsPerSku": units_per, + "odooOnHand": on_hand, "odooFba": fba, "odooNote": note, + "warehouses": warehouses, "openPurchaseOrders": purchase, + "openPurchaseQty": sum(o["openQty"] for o in purchase) if link and pos is not None else None} + row["band"], row["reason"] = _band(row, conf) + rows.append(row) + return rows, notices + + +# --------------------------------------------------------------------------- refresh and history + +def _month_key(month): + return HISTORY_PREFIX + month + + +def _point(row): + return {"fbaAvailable": row["fbaAvailable"], "daysOfSupply": row["daysOfSupply"], + "units30d": row["units30d"], "inbound": row["inbound"], "band": row["band"]} + + +def refresh(tenant_key, *, st=None, now=None): + """Compute every SKU row from the mirror, write today's history point per SKU, and merge items.""" + st = _rt(tenant_key, st) + now = now or _now() + conf = settings(tenant_key, st=st) + links = _links(st) + with _reader(tenant_key, st) as cur: + if cur is None: + rows, notices = None, ["The tenant mirror does not exist yet."] + else: + rows, notices = build_rows(cur, links, conf) + if rows is None: + message = notices[0] if notices else NOT_READY + + def _waiting(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(state="not_ready", message=message, notices=notices, checkedAt=_stamp(now)) + doc.pop("last_error", None) + return doc + st.update(KEY_ITEMS, _waiting) + return {"state": "not_ready", "message": message, "notices": notices} + + day, month = now.date().isoformat(), now.strftime("%Y-%m") + + def _history(doc): + doc = doc if isinstance(doc, dict) else {} + skus = doc.setdefault("skus", {}) + for row in rows: + skus.setdefault(row["sku"], {})[day] = _point(row) + return doc + st.update(_month_key(month), _history) + + def _index(doc): + doc = doc if isinstance(doc, dict) else {} + doc["months"] = sorted(set(doc.get("months") or ()) | {month}) + return doc + st.update(KEY_HISTORY, _index) + + owner = conf["defaultOwner"] + due = (now.date() + dt.timedelta(days=conf["dueDays"])).isoformat() + counts = {"created": 0, "reopened": 0} + + def _items(doc): + doc = doc if isinstance(doc, dict) else {} + items = doc.setdefault("items", {}) + seen = set() + for row in rows: + iid = item_id_for(row["sku"]) + seen.add(iid) + old = items.get(iid) + action = row["band"] in ACTION_BANDS + if old is None: + item = {"id": iid, "owner": owner, "status": "Open" if action else "Watching", + "due": due if action else None, "createdAt": _stamp(now), + "history": [{"at": _stamp(now), "by": "system", "event": "created", + "band": row["band"]}]} + counts["created"] += 1 + else: + item = old + if action and row["band"] != old.get("band") and old.get("status") in REOPEN_FROM: + item["status"], item["due"] = "Open", due + item.setdefault("history", []).append( + {"at": _stamp(now), "by": "system", "event": "reopened", "band": row["band"], + "status": "Open", "due": due}) + counts["reopened"] += 1 + item.update({k: v for k, v in row.items()}) + item["inReport"] = True + item["refreshedAt"] = _stamp(now) + items[iid] = item + for iid, item in items.items(): + if iid not in seen: + item["inReport"] = False + doc.update(state="ready", message=None, notices=notices, refreshedAt=_stamp(now)) + doc.pop("last_error", None) + return doc + st.update(KEY_ITEMS, _items) + return {"state": "ready", "skus": len(rows), "notices": notices, **counts} + + +def on_amazon_sync(tenant_key, synced_tables): + """Contract C5 callback: refresh when an inventory report synced. Never raises.""" + names = [str(t) for t in (synced_tables or ())] + if not any(trigger in name for name in names for trigger in SYNC_TRIGGERS): + return + try: + refresh(tenant_key) + except Exception as exc: # noqa: BLE001 + try: + def _note(doc): + doc = doc if isinstance(doc, dict) else {} + doc["last_error"] = f"The restock refresh failed ({type(exc).__name__}): {exc}" + return doc + _rt(tenant_key).update(KEY_ITEMS, _note) + except Exception: # noqa: BLE001 + pass + + +def history(tenant_key, sku, *, st=None): + """Every stored daily point for one SKU, oldest first.""" + st = _rt(tenant_key, st) + points = [] + for month in _doc(st, KEY_HISTORY).get("months") or (): + for day, point in ((_doc(st, _month_key(month)).get("skus") or {}).get(sku) or {}).items(): + points.append({"date": day, **point}) + return sorted(points, key=lambda p: p["date"]) + + +# --------------------------------------------------------------------------- C4 item protocol + +def _order(item): + band = item.get("band") + rank = BANDS.index(band) if band in BANDS else len(BANDS) + days = item.get("daysOfSupply") + return (not item.get("inReport", True), rank, days if days is not None else float("inf"), + str(item.get("sku") or "")) + + +def _cursor_offset(cursor): + if not cursor: + return 0 + try: + return int(json.loads(base64.urlsafe_b64decode(str(cursor).encode()).decode())["o"]) + except Exception: # noqa: BLE001 + raise ValueError("The page cursor is not valid. Start from the first page.") from None + + +def _cursor(offset): + return base64.urlsafe_b64encode(json.dumps({"o": offset}).encode()).decode() + + +def _public(item): + out = {k: v for k, v in item.items() if k != "history"} + out.setdefault("title", item.get("sku")) + return out + + +def _matches(item, needle): + """True when the lowercased needle is a substring of the item's sku, title or asin.""" + return any(needle in str(item.get(k) or "").lower() for k in ("sku", "title", "asin")) + + +def items(tenant_key, user=None, *, status=None, cursor=None, limit=200, band=None, q=None, st=None): + """C4: every item matching q, filtered by status and band, paged by an opaque cursor with no total cap.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + if band is not None and band not in BANDS: + raise ValueError("Band must be one of: " + ", ".join(BANDS) + ".") + try: + limit = int(limit) + except (TypeError, ValueError): + raise ValueError("The page size must be a whole number.") from None + if limit < 1: + raise ValueError("The page size must be at least 1.") + st = _rt(tenant_key, st) + doc = _doc(st, KEY_ITEMS) + every = list((doc.get("items") or {}).values()) + needle = str(q or "").strip().lower() + if needle: + every = [i for i in every if _matches(i, needle)] + counts = {b: 0 for b in BANDS} + for item in every: + if item.get("band") in counts: + counts[item["band"]] += 1 + rows = sorted((i for i in every if (status is None or i.get("status") == status) + and (band is None or i.get("band") == band)), key=_order) + start = _cursor_offset(cursor) + page = rows[start:start + limit] + ready = doc.get("state") == "ready" + return {"items": [_public(i) for i in page], + "cursor": _cursor(start + limit) if start + limit < len(rows) else None, + "counts": counts, "ready": ready, + "message": None if ready else (doc.get("message") or NOT_READY), + "notices": list(doc.get("notices") or ()), "refreshedAt": doc.get("refreshedAt"), + "lastError": doc.get("last_error"), "statuses": list(STATUSES), "bands": list(BANDS)} + + +def item_detail(tenant_key, item_id, *, st=None): + """One SKU with its warehouses, open purchase orders, item history and daily points.""" + st = _rt(tenant_key, st) + item = (_doc(st, KEY_ITEMS).get("items") or {}).get(item_id) + if not item: + raise KeyError(item_id) + return {**_public(item), "events": list(item.get("history") or ()), + "points": history(tenant_key, item.get("sku"), st=st)} + + +def _may_change(st, user): + """A known viewer or inactive account may not change an item (the App grant is the caller's check).""" + try: + users = st.get("users") or {} + except Exception: # noqa: BLE001 + users = {} + rec = users.get(str(user or "")) if isinstance(users, dict) else None + if isinstance(rec, dict) and (str(rec.get("role") or "").lower() == "viewer" or rec.get("active") is False): + return False + return True + + +def update_item(tenant_key, item_id, *, owner=None, status=None, due=None, note=None, user="system", st=None): + """Reassign, move status, or move the due date of one item. Returns the item.""" + if status is not None and status not in STATUSES: + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + if due is not None and due != "": + try: + due = dt.date.fromisoformat(str(due)).isoformat() + except ValueError: + raise ValueError("Due must be a date such as 2026-10-30.") from None + if owner is not None and not str(owner).strip(): + raise ValueError("Owner cannot be empty.") + st = _rt(tenant_key, st) + if not _may_change(st, user): + raise PermissionError("Your account can view this App but not change it.") + out = {} + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + item = (doc.get("items") or {}).get(item_id) + if not item: + return doc + at = _stamp(_now()) + event = {"at": at, "by": user} + if owner is not None: + item["owner"] = event["owner"] = str(owner).strip() + if status is not None: + item["status"] = event["status"] = status + if due is not None: + item["due"] = event["due"] = due or None + if note: + event["note"] = str(note) + item["updatedAt"], item["updatedBy"] = at, user + item.setdefault("history", []).append(event) + out["item"] = _public(item) + return doc + st.update(KEY_ITEMS, _apply) + if "item" not in out: + raise KeyError(item_id) + return out["item"] + + +def set_status(tenant_key, user, item_id, status, *, note=None, st=None): + """C4: move one item's status with an optional note.""" + return update_item(tenant_key, item_id, status=status, note=note, user=user, st=st) + + +def proposals(tenant_key, *, st=None): + """C7: one proposal per Open item in Send to FBA or Reorder from vendor, each listed once.""" + st = _rt(tenant_key, st) + out = [] + for item in sorted((_doc(st, KEY_ITEMS).get("items") or {}).values(), key=_order): + if (item.get("status") != "Open" or item.get("band") not in PROPOSING_BANDS + or item.get("inReport") is False): + continue + qty = item.get("recommendedQty") + amount = f"{_fmt(qty)} units" if qty is not None and qty > 0 else "stock" + action = (f"Send {amount} to FBA" if item["band"] == SEND else f"Reorder {amount} from vendor") + out.append({"source_kind": SOURCE_KIND, "item_id": item["id"], "sku": item.get("sku"), + "action": action, "reason": item.get("reason")}) + return out + + +def on_decision(tenant_key, item_id, decision, reason, *, st=None): + """C7: `approved` sets Accepted, `dismissed` sets Dismissed, with the reason as a history note.""" + if decision not in DECISIONS: + raise ValueError("Decision must be approved or dismissed.") + return update_item(tenant_key, item_id, status=DECISIONS[decision], note=reason or None, + user="operator", st=st) + + +# --------------------------------------------------------------------------- rule 8 and rule 13 + +def validate(tenant_key, *, st=None, **period): + """Rule 8: FBA available over the planner's rows ties to the planning table's own total.""" + st = _rt(tenant_key, st) + conf = settings(tenant_key, st=st) + with _reader(tenant_key, st) as cur: + if cur is None or "available" not in _columns(cur, PLANNING): + return {"ok": False, "app_total": None, "independent_total": None, + "detail": f"{SOURCE_NAMES[PLANNING]} has not synced yet."} + rows, _notices = build_rows(cur, _links(st), conf) + cols = _columns(cur, PLANNING) + independent = cur.execute( + f'SELECT SUM(TRY_CAST("available" AS DOUBLE)), COUNT(DISTINCT "sku") FROM {PLANNING} ' + f'WHERE {_newest(cols, PLANNING)}').fetchone() + planned = [r for r in rows or () if r["availableSource"] == "planning"] + app_total = float(sum(r["fbaAvailable"] or 0 for r in planned)) + independent_total = float(independent[0] or 0) + ok = abs(app_total - independent_total) < 1e-6 and len(planned) == int(independent[1] or 0) + return {"ok": ok, "app_total": app_total, "independent_total": independent_total, + "detail": (f"{len(planned)} planner SKUs against {int(independent[1] or 0)} planning SKUs " + f"in the newest snapshot.")} + + +def app_keys(st): + """Every store document this App owns.""" + months = _doc(st, KEY_HISTORY).get("months") or () + return [KEY_SETTINGS, KEY_ITEMS, KEY_HISTORY, *(_month_key(m) for m in months)] + + +def remove_app_data(tenant_key, *, st=None): + """Rule 13: delete this App's own documents. No mirror table and no other document is touched.""" + st = _rt(tenant_key, st) + deleted, emptied = [], [] + for key in app_keys(st): + try: + present = st.get(key) not in (None, {}, []) + except Exception: # noqa: BLE001 + present = True + if not present: + continue + remover = getattr(st, "delete_document", None) + if callable(remover): + try: + remover(key) + deleted.append(key) + continue + except Exception: # noqa: BLE001 + pass + # A store without a delete door keeps an empty document in place of the App's data. + st.update(key, lambda _doc: {}) + emptied.append(key) + return {"deleted": deleted, "emptied": emptied} diff --git a/platform/modules/returns_repeat.py b/platform/modules/returns_repeat.py new file mode 100644 index 0000000000000000000000000000000000000000..4f7966dc33c14ed1ff154d099d0bab8e7b0befe2 --- /dev/null +++ b/platform/modules/returns_repeat.py @@ -0,0 +1,920 @@ +"""Returns and repeat App: return rate per SKU, top return reasons, rising-returns items, repeat buyers. + +Sources (tenant mirror, read-only, never written): `amzn_fba_customer_returns`, `amzn_fba_shipped_sales`, +`amzn_repeat_purchase`, `amzn_market_basket` (Data-page keys `ut_amzn_*`). Nothing here calls Amazon or Odoo. + +Store documents (tenant runtime store, App owned, never a connector table): + returns_repeat_settings {rising_ratio, rising_min_pct, default_owner} + returns_repeat_items {seq, items: {id: item}} + returns_repeat_history {snapshots: {date: {sku: [rate30, rate90, ret30, ship30, ret90, ship90]}}, last_run} + +Deleting this module and its three documents leaves every `amzn_*` table intact (rule 13). + +Limit (rule 1): each document is one JSON value, and the Postgres store refuses a document over +`core.store_pg.DOCUMENT_CEILING_BYTES` (128 MB). The history keeps one snapshot a day at about 60 +bytes a SKU, so 1,000 SKUs reach the ceiling after about 2,000 days. At about 700 bytes an item, the +items document reaches it near 180,000 items. Nothing here caps SKUs, items or rows. The fix when a +document nears the ceiling: split it by year. +""" +import contextlib +import datetime as dt +import logging +from pathlib import Path + +LOG = logging.getLogger(__name__) + +SOURCE_KIND = "returns_repeat" +#: The Data-page key a person must be able to read before the App lets them change an item. +SOURCE = "ut_amzn_fba_customer_returns" +RETURNS_TABLE = "amzn_fba_customer_returns" +SHIPPED_TABLE = "amzn_fba_shipped_sales" +REPEAT_TABLE = "amzn_repeat_purchase" +BASKET_TABLE = "amzn_market_basket" +TABLES = (RETURNS_TABLE, SHIPPED_TABLE, REPEAT_TABLE, BASKET_TABLE) + +STATUSES = ("Open", "Approved", "Investigating", "Listing fixed", "Watching", "Resolved", "Dismissed") +#: A SKU with an item in one of these statuses raises no second item. +HOLDING = ("Open", "Approved", "Investigating", "Watching", "Dismissed") +DECISIONS = {"approved": "Approved", "dismissed": "Dismissed"} +#: P11 defaults (R49): rising when the 30 day rate is at least 1.5 times the 90 day rate and at least 5%. +DEFAULTS = {"rising_ratio": 1.5, "rising_min_pct": 5.0} +DUE_DAYS = 7 +PERIOD_DAYS = 90 +SEGMENTS = {"highest": "Highest return rate", "rising": "Rising returns", "repeat": "Repeat buyers"} +EMPTY = "No returns in this period" +NO_SKU = "No SKU" +ACTION = "Check Voice of the Customer for {target} and fix the listing problem it names." +LIMIT = ("Each Returns and repeat document is limited to 128 MB by the store. About 2,000 daily " + "snapshots of 1,000 SKUs, or 180,000 items, fit in one. Split the document by year when it " + "nears that size.") + +#: Amazon return codes whose plain word reading is wrong. Every other code reads as sentence case. +REASON_LABELS = {"DAMAGED_BY_FC": "Damaged at the fulfillment center", + "UNDELIVERABLE_UNKNOWN": "Undeliverable", + "SWITCHEROO": "Different item returned"} +TREND_DAYS = 7 + + +def reason_label(code): + """The words a person reads for an Amazon return code: never the code, never all caps.""" + text = _clean(code) + if not text: + return "No reason given" + if text.upper() in REASON_LABELS: + return REASON_LABELS[text.upper()] + if text != text.upper(): + return text + words = " ".join(text.replace("_", " ").replace("-", " ").split()).lower() + return words[:1].upper() + words[1:] if words else "No reason given" + + +KEY_SETTINGS = "returns_repeat_settings" +KEY_ITEMS = "returns_repeat_items" +KEY_HISTORY = "returns_repeat_history" +OWN_KEYS = (KEY_SETTINGS, KEY_ITEMS, KEY_HISTORY) + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _stamp(at): + return at.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _rt(tenant_key, st=None): + """The tenant runtime store handle (get and update), or raise when it cannot be resolved.""" + if st is not None: + return st + from harness import amazon_store + rt = amazon_store._runtime_for(tenant_key) + if rt is None: + raise RuntimeError(f"The store for tenant {tenant_key!r} could not be opened.") + return rt + + +def _doc(st, key): + value = st.get(key) + return value if isinstance(value, dict) else {} + + +# --------------------------------------------------------------------------- mirror reads + +def _mirror_path(tenant_key, st): + from harness import amazon_store + rt = st if hasattr(st, "datastore_path") else None + return amazon_store._tenant_mirror_path(tenant_key, rt) + + +@contextlib.contextmanager +def _reader(tenant_key, st): + """A read-only cursor on the tenant mirror, or None when the mirror file does not exist.""" + path = Path(_mirror_path(tenant_key, st)) + if not path.exists(): + yield None + return + from harness import datastore + try: + shared = Path(datastore.DB_PATH).resolve() == path.resolve() + except Exception: # noqa: BLE001 + shared = False + # The process holds one writer on its own mirror: a separate read-only connect would refuse. + if shared: + cur = datastore.ro_cursor() + try: + yield cur + finally: + cur.close() + return + import duckdb + con = duckdb.connect(str(path), read_only=True) + try: + yield con + finally: + con.close() + + +def _columns(cur, table): + try: + return {r[0] for r in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} + except Exception: # noqa: BLE001 + return set() + + +def _text(cols, name): + return f'CAST("{name}" AS VARCHAR)' if name in cols else "NULL" + + +def _num(cols, name): + return f'TRY_CAST("{name}" AS DOUBLE)' if name in cols else "NULL" + + +def _first(cols, *names): + return next((n for n in names if n in cols), None) + + +def _day(value): + """A date from mirror text (F36: Amazon dates may arrive as text in several shapes), or None.""" + if value is None: + return None + if isinstance(value, dt.datetime): + return value.date() + if isinstance(value, dt.date): + return value + text = str(value).strip() + if not text: + return None + try: + return dt.date.fromisoformat(text[:10]) + except ValueError: + pass + for fmt in ("%m/%d/%Y", "%Y/%m/%d", "%d-%b-%Y", "%b %d, %Y"): + try: + return dt.datetime.strptime(text.split(" ")[0] if "/" in text else text, fmt).date() + except ValueError: + continue + return None + + +def _clean(value): + return str(value).strip() if value is not None else "" + + +def _returns(cur): + """Return rows grouped as (sku, asin, day, reason, units, rows). Uncapped.""" + cols = _columns(cur, RETURNS_TABLE) if cur is not None else set() + if not cols & {"sku", "asin"}: + return [] + qty = _num(cols, "quantity") if "quantity" in cols else "1" + rows = cur.execute( + f'SELECT {_text(cols, "sku")}, {_text(cols, "asin")}, {_text(cols, "return_date")}, ' + f'{_text(cols, "reason")}, SUM(COALESCE({qty}, 0)), COUNT(*) FROM {RETURNS_TABLE} ' + f'GROUP BY 1, 2, 3, 4').fetchall() + return [(_clean(s), _clean(a), _day(d), _clean(r), float(u or 0), int(n or 0)) for s, a, d, r, u, n in rows] + + +def _shipped(cur): + """Shipment rows grouped as (sku, asin, day, units). Uncapped.""" + cols = _columns(cur, SHIPPED_TABLE) if cur is not None else set() + if not cols & {"sku", "asin"}: + return [] + qty = _num(cols, "quantity") if "quantity" in cols else "1" + rows = cur.execute( + f'SELECT {_text(cols, "sku")}, {_text(cols, "asin")}, {_text(cols, "shipment_date")}, ' + f'SUM(COALESCE({qty}, 0)) FROM {SHIPPED_TABLE} GROUP BY 1, 2, 3').fetchall() + return [(_clean(s), _clean(a), _day(d), float(u or 0)) for s, a, d, u in rows] + + +def _repeat(cur): + """Repeat purchase rows (Brand Analytics, one per ASIN and month) as dicts.""" + cols = _columns(cur, REPEAT_TABLE) if cur is not None else set() + if "asin" not in cols: + return [] + start = _first(cols, "start_date", "_period_start") + end = _first(cols, "end_date", "_period_end") + rows = cur.execute( + f'SELECT CAST("asin" AS VARCHAR), {_text(cols, start or "")}, {_text(cols, end or "")}, ' + f'{_num(cols, "orders")}, {_num(cols, "unique_customers")}, {_num(cols, "repeat_customers_pct_total")}, ' + f'{_num(cols, "repeat_purchase_revenue_amount")}, {_text(cols, "repeat_purchase_revenue_currency_code")}, ' + f'{_num(cols, "repeat_purchase_revenue_pct_total")} FROM {REPEAT_TABLE}').fetchall() + out = [] + for asin, s, e, orders, customers, pct, revenue, currency, rev_pct in rows: + if not _clean(asin): + continue + s, e = _day(s), _day(e) + out.append({"asin": _clean(asin), "start": s, "end": e or s, "orders": orders, + "customers": customers, "pct": pct, "revenue": revenue, + "currency": _clean(currency) or None, "revenuePct": rev_pct}) + return out + + +def _basket(cur): + """Market basket rows as dicts: which ASINs a buyer bought together with ours.""" + cols = _columns(cur, BASKET_TABLE) if cur is not None else set() + if not {"asin", "purchased_with_asin"} <= cols: + return [] + start = _first(cols, "start_date", "_period_start") + rank = 'TRY_CAST("purchased_with_rank" AS INTEGER)' if "purchased_with_rank" in cols else "NULL" + rows = cur.execute( + f'SELECT CAST("asin" AS VARCHAR), CAST("purchased_with_asin" AS VARCHAR), {rank}, ' + f'{_num(cols, "combination_pct")}, {_text(cols, start or "")} FROM {BASKET_TABLE}').fetchall() + return [{"asin": _clean(a), "with": _clean(w), "rank": r, "pct": p, "start": _day(s)} + for a, w, r, p, s in rows if _clean(a) and _clean(w)] + + +def _load(tenant_key, st): + with _reader(tenant_key, st) as cur: + if cur is None: + return {"mirror": False, "returns": [], "shipped": [], "repeat": [], "basket": []} + return {"mirror": True, "returns": _returns(cur), "shipped": _shipped(cur), + "repeat": _repeat(cur), "basket": _basket(cur)} + + +# --------------------------------------------------------------------------- settings and owner + +#: Accounts that sign in for a process, never a person who can own an item (same list as amazon_profit). +SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership") + + +def _is_service(name, rec): + name = str(name or "").strip().lower() + return (not name or name.startswith("qa") or name in SERVICE_ACCOUNTS + or name.endswith("-admin") or bool(rec.get("platform_admin"))) + + +def owner_account(st, tenant_key=None): + """P22 (SDLC 78 R31): the tenant's earliest active person Administrator by display name. + + A tenant with no person Administrator falls back to its `admin` account, else its first Administrator.""" + try: + users = st.get("users") + except Exception: # noqa: BLE001 + users = None + want = str(tenant_key or "").strip().lower() + admins = [(str(name), rec) for name, rec in (users or {}).items() + if isinstance(rec, dict) and rec.get("active", True) + and str(rec.get("role") or "").lower() == "admin" + and (not want or str(rec.get("tenant") or "royal-imports").strip().lower() == want)] + people = sorted(((n, r) for n, r in admins if not _is_service(n, r)), + key=lambda item: ("@" not in item[0], not item[1].get("password_changed_at"), + str(item[1].get("password_changed_at") or ""), item[0])) + if people: + return str(people[0][1].get("name") or people[0][0]) + names = sorted(n for n, _r in admins) + return "admin" if "admin" in names or not names else names[0] + + +def settings(tenant_key, *, st=None): + """Rising thresholds and the default owner, with defaults filled in.""" + st = _rt(tenant_key, st) + doc = _doc(st, KEY_SETTINGS) + out = {"risingRatio": float(doc.get("rising_ratio", DEFAULTS["rising_ratio"])), + "risingMinPct": float(doc.get("rising_min_pct", DEFAULTS["rising_min_pct"]))} + out["defaultOwnerSetting"] = doc.get("default_owner") or None + out["defaultOwner"] = out["defaultOwnerSetting"] or owner_account(st, tenant_key) + out["statuses"] = list(STATUSES) + out["limit"] = LIMIT + return out + + +def set_settings(tenant_key, *, rising_ratio=None, rising_min_pct=None, default_owner=None, st=None): + """Set the rising ratio (1 to 100), the minimum rate (above 0, below 100 percent) or the default owner.""" + st = _rt(tenant_key, st) + changes = {} + for name, value, low, label in (("rising_ratio", rising_ratio, 1.0, "The rising ratio"), + ("rising_min_pct", rising_min_pct, 0.0, "The minimum return rate")): + if value is None: + continue + try: + number = float(value) + except (TypeError, ValueError): + raise ValueError(f"{label} must be a number.") from None + if not (low <= number < 100 if low else 0 < number < 100): + raise ValueError(f"{label} must be at least 1 and below 100." if low + else f"{label} must be above 0 and below 100 percent.") + changes[name] = number + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + doc.update(changes) + if default_owner is not None: + doc["default_owner"] = str(default_owner).strip() or None + return doc + st.update(KEY_SETTINGS, _apply) + return settings(tenant_key, st=st) + + +# --------------------------------------------------------------------------- the model + +def _period(start=None, end=None, now=None): + """(start, end) dates, inclusive. The default is the 90 days ending today.""" + def parse(value, label): + if value in (None, ""): + return None + if isinstance(value, dt.date): + return value + try: + return dt.date.fromisoformat(str(value)[:10]) + except ValueError: + raise ValueError(f"{label} must be a date such as 2026-10-01.") from None + e = parse(end, "End") or (now or _now()).date() + s = parse(start, "Start") or e - dt.timedelta(days=PERIOD_DAYS - 1) + if s > e: + raise ValueError("Start must be on or before End.") + return s, e + + +def _key(sku, asin): + return sku or asin or NO_SKU + + +def _rate(returned, shipped): + return round(100.0 * returned / shipped, 4) if shipped > 0 else None + + +def _scale(values): + """Unknown: whether Amazon sends shares as fractions or percent. All values at or under 1 read as fractions.""" + nums = [abs(v) for v in values if isinstance(v, (int, float))] + return 100.0 if nums and max(nums) <= 1 else 1.0 + + +def _repeat_by_asin(data, start, end): + """{asin: repeat facts} over the months that overlap the period; a month outside it never stands in.""" + rows = data["repeat"] + used = [r for r in rows if r["start"] and r["start"] <= end and (r["end"] or r["start"]) >= start] + scale = _scale([r["pct"] for r in rows]) + rev_scale = _scale([r["revenuePct"] for r in rows]) + out = {} + for r in used: + agg = out.setdefault(r["asin"], {"orders": 0.0, "customers": 0.0, "weighted": 0.0, "plain": [], + "revenue": 0.0, "hasRevenue": False, "currency": None, + "revenuePct": [], "months": set()}) + agg["orders"] += r["orders"] or 0 + if isinstance(r["pct"], (int, float)): + agg["plain"].append(r["pct"] * scale) + if r["customers"]: + agg["customers"] += r["customers"] + agg["weighted"] += r["customers"] * r["pct"] * scale + if isinstance(r["revenue"], (int, float)): + agg["revenue"] += r["revenue"] + agg["hasRevenue"] = True + if isinstance(r["revenuePct"], (int, float)): + agg["revenuePct"].append(r["revenuePct"] * rev_scale) + agg["currency"] = agg["currency"] or r["currency"] + if r["start"]: + agg["months"].add(r["start"].isoformat()) + facts = {} + for asin, agg in out.items(): + share = (agg["weighted"] / agg["customers"] if agg["customers"] + else (sum(agg["plain"]) / len(agg["plain"]) if agg["plain"] else None)) + facts[asin] = {"asin": asin, "repeatShare": None if share is None else round(share, 4), + "repeatRevenue": round(agg["revenue"], 2) if agg["hasRevenue"] else None, + "repeatRevenueShare": (round(sum(agg["revenuePct"]) / len(agg["revenuePct"]), 4) + if agg["revenuePct"] else None), + "currency": agg["currency"], "orders": agg["orders"], + "uniqueCustomers": agg["customers"] or None, "months": sorted(agg["months"]), + "_weighted": agg["weighted"], "_customers": agg["customers"]} + return facts + + +def _bought_with(data, asin): + """The newest month's frequently-bought-with ASINs for one ASIN, by Amazon's rank.""" + rows = [r for r in data["basket"] if r["asin"] == asin] + if not rows: + return [] + newest = max((r["start"] for r in rows if r["start"]), default=None) + rows = [r for r in rows if r["start"] == newest] + scale = _scale([r["pct"] for r in data["basket"]]) + rows.sort(key=lambda r: (r["rank"] is None, r["rank"] or 0, r["with"])) + return [{"asin": r["with"], "rank": r["rank"], + "combinationPct": None if r["pct"] is None else round(r["pct"] * scale, 4)} for r in rows] + + +def _model(data, start, end, conf): + """{key: per-SKU facts} for every SKU the returns or shipments tables hold, plus the windows.""" + w30 = end - dt.timedelta(days=29) + w90 = end - dt.timedelta(days=89) + skus = {} + + def entry(sku, asin): + k = _key(sku, asin) + e = skus.get(k) + if e is None: + e = skus[k] = {"key": k, "sku": sku or None, "asins": set(), "returned": 0.0, "returnRows": 0, + "shipped": 0.0, "reasons": {}, "r30": 0.0, "s30": 0.0, "r90": 0.0, "s90": 0.0} + if asin: + e["asins"].add(asin) + return e + + for sku, asin, day, reason, units, rows in data["returns"]: + e = entry(sku, asin) + if day is None: + continue + if start <= day <= end: + e["returned"] += units + e["returnRows"] += rows + got = e["reasons"].setdefault(reason or "No reason given", [0.0, 0]) + got[0] += units + got[1] += rows + if w30 <= day <= end: + e["r30"] += units + if w90 <= day <= end: + e["r90"] += units + for sku, asin, day, units in data["shipped"]: + e = entry(sku, asin) + if day is None: + continue + if start <= day <= end: + e["shipped"] += units + if w30 <= day <= end: + e["s30"] += units + if w90 <= day <= end: + e["s90"] += units + ratio, floor = conf["risingRatio"], conf["risingMinPct"] + for e in skus.values(): + e["returnRate"] = _rate(e["returned"], e["shipped"]) + e["rate30"] = _rate(e["r30"], e["s30"]) + e["rate90"] = _rate(e["r90"], e["s90"]) + e["rising"] = (e["rate30"] is not None and e["rate90"] is not None + and e["rate30"] >= ratio * e["rate90"] and e["rate30"] >= floor) + e["asin"] = sorted(e["asins"])[0] if e["asins"] else None + reasons = sorted(e["reasons"].items(), key=lambda kv: (-kv[1][0], -kv[1][1], kv[0])) + e["reasonList"] = [{"reason": r, "label": reason_label(r), "units": u, "count": n} + for r, (u, n) in reasons] + return skus + + +def _row(e, repeat=None, item=None): + rep = (repeat or {}).get(e["asin"]) if e["asin"] else None + return {"key": e["key"], "sku": e["sku"], "asin": e["asin"], "asins": sorted(e["asins"]), + "returnedUnits": e["returned"], "shippedUnits": e["shipped"], "returnRate": e["returnRate"], + "rate30": e["rate30"], "rate90": e["rate90"], "rising": e["rising"], + "topReason": e["reasonList"][0]["reason"] if e["reasonList"] else None, + "topReasonLabel": e["reasonList"][0]["label"] if e["reasonList"] else None, + "repeatShare": rep["repeatShare"] if rep else None, + "repeatRevenue": rep["repeatRevenue"] if rep else None, + "itemId": (item or {}).get("id"), "itemStatus": (item or {}).get("status")} + + +def _items_by_sku(st): + latest = {} + for item in sorted((_doc(st, KEY_ITEMS).get("items") or {}).values(), key=_seq): + latest[item.get("sku")] = item + return latest + + +def _seq(item): + try: + return int(str(item.get("id", "r0"))[1:] or 0) + except ValueError: + return 0 + + +def _page(rows, cursor, limit): + """One page of rows and the cursor of the next page (None on the last). No cap on the total.""" + try: + offset = int(cursor) if cursor not in (None, "") else 0 + except (TypeError, ValueError): + raise ValueError("The cursor is not valid. Start again from the first page.") from None + try: + size = int(limit) if limit is not None else len(rows) or 1 + except (TypeError, ValueError): + raise ValueError("The page size must be a number.") from None + if offset < 0 or size < 1: + raise ValueError("The cursor is not valid. Start again from the first page.") + nxt = offset + size + return rows[offset:nxt], (str(nxt) if nxt < len(rows) else None) + + +# --------------------------------------------------------------------------- read surfaces + +def _share(repeat): + """The customer-weighted repeat purchase share over {asin: facts}, else the plain mean, else None.""" + customers = sum(f["_customers"] for f in repeat.values()) + plain = [f["repeatShare"] for f in repeat.values() if f["repeatShare"] is not None] + return (round(sum(f["_weighted"] for f in repeat.values()) / customers, 4) if customers + else (round(sum(plain) / len(plain), 4) if plain else None)) + + +def _trend(data, start, end): + """Weekly buckets from `start`, the last one cut at `end`: [{start, end, returnRate}].""" + count = (end - start).days // TREND_DAYS + 1 + sums = [[0.0, 0.0] for _ in range(count)] + for _sku, _asin, day, _reason, units, _rows in data["returns"]: + if day is not None and start <= day <= end: + sums[(day - start).days // TREND_DAYS][0] += units + for _sku, _asin, day, units in data["shipped"]: + if day is not None and start <= day <= end: + sums[(day - start).days // TREND_DAYS][1] += units + out = [] + for i, (returned, shipped) in enumerate(sums): + b0 = start + dt.timedelta(days=i * TREND_DAYS) + b1 = min(end, b0 + dt.timedelta(days=TREND_DAYS - 1)) + out.append({"start": b0.isoformat(), "end": b1.isoformat(), "returnRate": _rate(returned, shipped)}) + return out + + +def _previous(data, start, end, conf): + """The KPIs of the period of equal length ending the day before `start`. No month fallback.""" + p1 = start - dt.timedelta(days=1) + p0 = p1 - (end - start) + model = _model(data, p0, p1, conf) + returned = sum(x["returned"] for x in model.values()) + shipped = sum(x["shipped"] for x in model.values()) + return {"start": p0.isoformat(), "end": p1.isoformat(), "returnRate": _rate(returned, shipped), + "repeatShare": _share(_repeat_by_asin(data, p0, p1))} + + +def summary(tenant_key, *, start=None, end=None, st=None, now=None): + """The period's return rate and repeat purchase share, with the empty state and segment counts.""" + st = _rt(tenant_key, st) + s, e = _period(start, end, now) + conf = settings(tenant_key, st=st) + data = _load(tenant_key, st) + model = _model(data, s, e, conf) + repeat = _repeat_by_asin(data, s, e) + returned = sum(x["returned"] for x in model.values()) + shipped = sum(x["shipped"] for x in model.values()) + share = _share(repeat) + revenue = [f["repeatRevenue"] for f in repeat.values() if f["repeatRevenue"] is not None] + months = sorted({m for f in repeat.values() for m in f["months"]}) + open_items = sum(1 for i in (_doc(st, KEY_ITEMS).get("items") or {}).values() if i.get("status") == "Open") + return {"period": {"start": s.isoformat(), "end": e.isoformat()}, + "returnedUnits": returned, "shippedUnits": shipped, "returnRate": _rate(returned, shipped), + "repeatShare": share, "repeatRevenue": round(sum(revenue), 2) if revenue else None, + "repeatMonths": months, "empty": returned == 0, + "emptyMessage": EMPTY if returned == 0 else None, "mirror": data["mirror"], + "openItems": open_items, + "counts": {"highest": len(model), "rising": sum(1 for x in model.values() if x["rising"]), + "repeat": len(repeat)}, + "segments": dict(SEGMENTS), + "previous": _previous(data, s, e, conf), "trend": _trend(data, s, e)} + + +def skus(tenant_key, *, segment="highest", start=None, end=None, cursor=None, limit=200, q=None, st=None, now=None): + """One segment's rows, paged by cursor. `highest` holds every SKU; no SKU is dropped (rule 1). + + `q` keeps the rows whose SKU or ASIN contains it, case blind, before counting and paging.""" + if segment not in SEGMENTS: + raise ValueError("Segment must be one of: " + ", ".join(SEGMENTS) + ".") + st = _rt(tenant_key, st) + s, e = _period(start, end, now) + conf = settings(tenant_key, st=st) + data = _load(tenant_key, st) + model = _model(data, s, e, conf) + repeat = _repeat_by_asin(data, s, e) + items = _items_by_sku(st) + if segment == "repeat": + by_asin = {} + for x in model.values(): + for asin in x["asins"]: + by_asin.setdefault(asin, []).append(x["key"]) + rows = [] + for asin, f in repeat.items(): + rows.append({"key": asin, "asin": asin, "skus": sorted(by_asin.get(asin, [])), + "repeatShare": f["repeatShare"], "repeatRevenue": f["repeatRevenue"], + "currency": f["currency"], "orders": f["orders"], + "uniqueCustomers": f["uniqueCustomers"], + "boughtWith": [b["asin"] for b in _bought_with(data, asin)]}) + rows.sort(key=lambda r: (r["repeatShare"] is None, -(r["repeatShare"] or 0), r["asin"])) + else: + picked = [x for x in model.values() if segment == "highest" or x["rising"]] + if segment == "rising": + picked.sort(key=lambda x: (-(x["rate30"] or 0), x["key"])) + else: + picked.sort(key=lambda x: (x["returnRate"] is None, -(x["returnRate"] or 0), -x["returned"], x["key"])) + rows = [_row(x, repeat, items.get(x["key"])) for x in picked] + needle = " ".join(str(q or "").split()).casefold() + if needle: + rows = [r for r in rows if any(needle in str(v or "").casefold() for v in ( + r.get("key"), r.get("sku"), r.get("asin"), *(r.get("asins") or ()), *(r.get("skus") or ())))] + page, nxt = _page(rows, cursor, limit) + return {"segment": segment, "label": SEGMENTS[segment], "period": {"start": s.isoformat(), "end": e.isoformat()}, + "q": needle or None, "total": len(rows), "rows": page, "cursor": nxt} + + +def detail(tenant_key, key, *, start=None, end=None, st=None, now=None): + """One SKU or ASIN: ranked reasons with counts, repeat share and revenue, frequently-bought-with ASINs.""" + want = _clean(key) + if not want: + raise ValueError("Name a SKU or an ASIN.") + st = _rt(tenant_key, st) + s, e = _period(start, end, now) + conf = settings(tenant_key, st=st) + data = _load(tenant_key, st) + model = _model(data, s, e, conf) + repeat = _repeat_by_asin(data, s, e) + found = model.get(want) or next((x for x in model.values() if want in x["asins"]), None) + asin = want if found is None or want in (found["asins"] or ()) else found["asin"] + if found is None and want not in repeat and not any(b["asin"] == want for b in data["basket"]): + raise KeyError(want) + history = _doc(st, KEY_HISTORY).get("snapshots") or {} + sku_key = found["key"] if found else None + series = [{"date": d, "rate30": v[0], "rate90": v[1]} for d, snap in sorted(history.items()) + for k, v in snap.items() if k == sku_key and isinstance(v, list) and len(v) >= 2] + rep = repeat.get(asin) if asin else None + item = _items_by_sku(st).get(sku_key) if sku_key else None + return {"key": want, "sku": found["sku"] if found else None, "asin": asin, + "period": {"start": s.isoformat(), "end": e.isoformat()}, + "returnedUnits": found["returned"] if found else 0.0, + "shippedUnits": found["shipped"] if found else 0.0, + "returnRate": found["returnRate"] if found else None, + "rate30": found["rate30"] if found else None, "rate90": found["rate90"] if found else None, + "rising": bool(found and found["rising"]), + "reasons": found["reasonList"] if found else [], + "repeat": None if rep is None else {k: v for k, v in rep.items() if not k.startswith("_")}, + "repeatShare": rep["repeatShare"] if rep else None, + "repeatRevenue": rep["repeatRevenue"] if rep else None, + "boughtWith": _bought_with(data, asin) if asin else [], + "history": series, "item": dict(item) if item else None, + "emptyMessage": EMPTY if not found or not found["returned"] else None} + + +# --------------------------------------------------------------------------- the after-sync recompute + +def _table_name(name): + text = str(name or "").strip().lower() + for prefix in ("ut_", "amzn_"): + if text.startswith(prefix): + text = text[len(prefix):] + return "amzn_" + text + + +def on_amazon_sync(tenant_key, synced_tables): + """Contract C5 callback: recompute after any of the four tables synced. Never raises.""" + try: + if not {_table_name(t) for t in synced_tables or ()} & set(TABLES): + return {"state": "skipped"} + return recompute(tenant_key) + except Exception as exc: # noqa: BLE001 + LOG.warning("returns_repeat: the after-sync recompute failed for %s (%s): %s", + tenant_key, type(exc).__name__, exc) + return {"state": "failed"} + + +def recompute(tenant_key, *, st=None, now=None): + """Append today's per-SKU rate snapshot and raise one owned item per newly rising SKU.""" + st = _rt(tenant_key, st) + now = now or _now() + conf = settings(tenant_key, st=st) + data = _load(tenant_key, st) + if not data["mirror"]: + return {"state": "no_mirror", "created": []} + today = now.date() + model = _model(data, today - dt.timedelta(days=PERIOD_DAYS - 1), today, conf) + snapshot = {x["key"]: [x["rate30"], x["rate90"], x["r30"], x["s30"], x["r90"], x["s90"]] + for x in model.values()} + + def _history(doc): + doc = doc if isinstance(doc, dict) else {} + doc.setdefault("snapshots", {})[today.isoformat()] = snapshot + doc["last_run"] = _stamp(now) + return doc + st.update(KEY_HISTORY, _history) + + rising = sorted((x for x in model.values() if x["rising"]), key=lambda x: x["key"]) + owner = conf["defaultOwner"] + created = [] + + def _items(doc): + doc = doc if isinstance(doc, dict) else {} + doc.setdefault("seq", 0) + items = doc.setdefault("items", {}) + latest = {} + for item in sorted(items.values(), key=_seq): + latest[item.get("sku")] = item + for x in rising: + if x["key"] in latest and latest[x["key"]].get("status") in HOLDING: + continue + doc["seq"] += 1 + iid = f"r{doc['seq']}" + times = x["rate30"] / x["rate90"] if x["rate90"] else None + reason = (f"30 day return rate {x['rate30']:.1f}% is {times:.1f} times the 90 day rate " + f"{x['rate90']:.1f}%." if times else f"30 day return rate {x['rate30']:.1f}%.") + item = {"id": iid, "title": f"Return rate rising on {x['key']}", "status": "Open", + "owner": owner, "due": (today + dt.timedelta(days=DUE_DAYS)).isoformat(), + "reason": reason, "sku": x["key"], "asin": x["asin"], + "rate30": x["rate30"], "rate90": x["rate90"], "topReason": + x["reasonList"][0]["reason"] if x["reasonList"] else None, + "createdAt": _stamp(now), "updatedAt": _stamp(now), "updatedBy": "system", + "history": [{"at": _stamp(now), "by": "system", "event": "created"}]} + items[iid] = item + latest[x["key"]] = item + created.append(iid) + return doc + if rising: + st.update(KEY_ITEMS, _items) + return {"state": "computed", "date": today.isoformat(), "skus": len(snapshot), + "rising": [x["key"] for x in rising], "created": created} + + +# --------------------------------------------------------------------------- C4 item protocol + +def _public(item): + return {"id": item.get("id"), "title": item.get("title"), "status": item.get("status"), + "owner": item.get("owner"), "due": item.get("due"), "reason": item.get("reason"), + "sku": item.get("sku"), "asin": item.get("asin"), "rate30": item.get("rate30"), + "rate90": item.get("rate90"), "topReason": item.get("topReason"), + "note": item.get("note"), "updatedAt": item.get("updatedAt"), "updatedBy": item.get("updatedBy")} + + +def _user_record(st, user): + if isinstance(user, dict): + return user + name = str(user or "").strip() + if not name: + return None + try: + rec = (st.get("users") or {}).get(name) + except Exception: # noqa: BLE001 + rec = None + return {**rec, "username": rec.get("username") or name} if isinstance(rec, dict) else None + + +def _may_read_source(st, rec): + try: + from core import perm_scope + return bool(perm_scope.may_read(rec, SOURCE, st=st)) + except Exception: # noqa: BLE001 + return False + + +def may_edit(st, user): + """An edit grant: an account that is not a viewer and may read the FBA returns database.""" + rec = _user_record(st, user) + if not rec or str(rec.get("role") or "user").strip().lower() == "viewer": + return False + return _may_read_source(st, rec) + + +def items(tenant_key, user, *, status=None, cursor=None, limit=200, st=None): + """Every item, newest first, paged by cursor. No cap on the total (rule 1).""" + st = _rt(tenant_key, st) + if user is not None: + rec = _user_record(st, user) + if not rec or not _may_read_source(st, rec): + raise PermissionError("Your account cannot read Amazon FBA returns.") + rows = [i for i in (_doc(st, KEY_ITEMS).get("items") or {}).values() + if status in (None, "") or i.get("status") == status] + rows.sort(key=_seq, reverse=True) + page, nxt = _page(rows, cursor, limit) + return {"items": [_public(i) for i in page], "cursor": nxt} + + +def _status(value): + text = str(value or "").strip().lower() + for name in STATUSES: + if name.lower() == text: + return name + raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") + + +def update_item(tenant_key, user, item_id, *, owner=None, status=None, due=None, note=None, st=None): + """Reassign, move status, move the due date or add a note on one item. Returns the item.""" + st = _rt(tenant_key, st) + if not may_edit(st, user): + raise PermissionError("Your account can view this App but not change its items.") + if status is not None: + status = _status(status) + if due is not None: + try: + due = dt.date.fromisoformat(str(due)).isoformat() + except ValueError: + raise ValueError("Due must be a date such as 2026-10-27.") from None + if owner is not None and not str(owner).strip(): + raise ValueError("Owner cannot be empty.") + rec = _user_record(st, user) or {} + by = str(rec.get("username") or rec.get("name") or user or "unknown") + return _apply_item(st, item_id, by, owner=owner, status=status, due=due, note=note) + + +def _apply_item(st, item_id, by, *, owner=None, status=None, due=None, note=None, event=None): + out = {} + + def _apply(doc): + doc = doc if isinstance(doc, dict) else {} + item = (doc.get("items") or {}).get(str(item_id)) + if not item: + return doc + at = _stamp(_now()) + change = {"at": at, "by": by, **(event or {})} + if owner is not None: + item["owner"] = change["owner"] = str(owner).strip() + if status is not None: + item["status"] = change["status"] = status + if due is not None: + item["due"] = change["due"] = due + if note is not None and str(note).strip(): + item["note"] = change["note"] = str(note).strip() + item["updatedAt"], item["updatedBy"] = at, by + item.setdefault("history", []).append(change) + out["item"] = dict(item) + return doc + st.update(KEY_ITEMS, _apply) + if "item" not in out: + raise KeyError(item_id) + return _public(out["item"]) + + +def set_status(tenant_key, user, item_id, status, *, note=None, st=None): + """C4: move one item to a status. PermissionError without an edit grant, ValueError on an unknown status.""" + st = _rt(tenant_key, st) + if not may_edit(st, user): + raise PermissionError("Your account can view this App but not change its items.") + return update_item(tenant_key, user, item_id, status=_status(status), note=note, st=st) + + +def proposals(tenant_key, *, st=None): + """C7: each open rising-returns item once, with an action in Amazon's own tools (R20).""" + st = _rt(tenant_key, st) + out, seen = [], set() + for item in sorted((_doc(st, KEY_ITEMS).get("items") or {}).values(), key=_seq): + if item.get("status") != "Open" or item.get("id") in seen: + continue + seen.add(item.get("id")) + out.append({"source_kind": SOURCE_KIND, "item_id": item.get("id"), "sku": item.get("sku"), + "action": ACTION.format(target=item.get("asin") or item.get("sku")), + "reason": item.get("reason")}) + return out + + +def on_decision(tenant_key, item_id, decision, reason, *, st=None): + """C7: an operator decision (approved or dismissed) moves the item's status.""" + status = DECISIONS.get(str(decision or "").strip().lower()) + if status is None: + raise ValueError("Decision must be approved or dismissed.") + st = _rt(tenant_key, st) + return _apply_item(st, item_id, "operator", status=status, note=reason, + event={"decision": str(decision).strip().lower()}) + + +# --------------------------------------------------------------------------- validate and removal + +def validate(tenant_key, *, start=None, end=None, st=None, now=None, **_period_kw): + """The App's returned units for the period tie to an independent SUM(quantity) over FBA returns.""" + st = _rt(tenant_key, st) + s, e = _period(start or _period_kw.get("period_start"), end or _period_kw.get("period_end"), now) + conf = settings(tenant_key, st=st) + app_total = sum(x["returned"] for x in _model(_load(tenant_key, st), s, e, conf).values()) + independent = None + with _reader(tenant_key, st) as cur: + cols = _columns(cur, RETURNS_TABLE) if cur is not None else set() + if {"quantity", "return_date"} <= cols: + day = ('COALESCE(TRY_CAST(SUBSTR(CAST("return_date" AS VARCHAR), 1, 10) AS DATE), ' + 'CAST(TRY_STRPTIME(CAST("return_date" AS VARCHAR), \'%m/%d/%Y\') AS DATE))') + got = cur.execute( + f'SELECT COALESCE(SUM(TRY_CAST("quantity" AS DOUBLE)), 0) FROM {RETURNS_TABLE} ' + f'WHERE {day} BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)', + [s.isoformat(), e.isoformat()]).fetchone() + independent = float(got[0] or 0) + elif cur is not None and not cols: + independent = 0.0 + if independent is None: + return {"ok": False, "app_total": app_total, "independent_total": None, + "detail": "The FBA returns table has no quantity or return date column to sum."} + ok = abs(app_total - independent) < 1e-6 + return {"ok": ok, "app_total": app_total, "independent_total": independent, + "detail": (f"Returned units {s.isoformat()} to {e.isoformat()}: App {app_total:g}, " + f"{RETURNS_TABLE} {independent:g}.")} + + +def on_app_removed(tenant_key, *, st=None): + """Rule 13: delete every document this App owns. No `amzn_*` table is touched.""" + st = _rt(tenant_key, st) + removed, errors = [], [] + for key in OWN_KEYS: + try: + present = bool(st.get(key)) + except Exception: # noqa: BLE001 + present = True + try: + if hasattr(st, "delete_document"): + st.delete_document(key) + else: + st.update(key, lambda _doc: None) + removed.append(key) + except Exception as exc: # noqa: BLE001 + if present: + errors.append(f"{key}: {type(exc).__name__}: {exc}") + return {"removed": removed, "errors": errors} diff --git a/platform/modules/three_statement.py b/platform/modules/three_statement.py new file mode 100644 index 0000000000000000000000000000000000000000..494b8cf38a99b116f765b78ee16f957e27a904ca --- /dev/null +++ b/platform/modules/three_statement.py @@ -0,0 +1,158 @@ +"""3-statement model: the read-only Odoo oracle and `validate()` for the Sheet template (S79-T57). + +The template itself is data in `core.sheet_templates` (core never imports up). This module ties +its method to Odoo (rule 8): the account-type grouping the template uses must reproduce Odoo's own +Profit and loss and Balance sheet, and its unit split must reproduce the analytic ledger. Odoo is +read only (rule 4): every call here is a read. +""" +from __future__ import annotations + +import json +import os +import re +import urllib.request +from collections import defaultdict +from contextlib import nullcontext + +from core import odoo as O +from core.sheet_templates import ( # noqa: F401 re-exported names the gate and callers read + THREE_STATEMENT, TS_ACCOUNTS as ACCOUNTS, TS_ASSET_GROUPS as ASSET_GROUPS, + TS_BS_TAB as BS_TAB, TS_BS_TYPES as BS_TYPES, TS_CASH_GROUPS as CASH_GROUPS, TS_CF_TAB as CF_TAB, + TS_EQUITY_GROUPS as EQUITY_GROUPS, TS_FULL_SHARE as FULL_SHARE, TS_GL as GL, TS_KEY as KEY, + TS_LIABILITY_GROUPS as LIABILITY_GROUPS, TS_OTHER_GROUP as OTHER_GROUP, TS_PL_GROUPS as PL_GROUPS, + TS_PL_TAB as PL_TAB, TS_PL_TYPES as PL_TYPES, TS_UNITS as UNITS, TsBuilder, + three_statement_descriptor as descriptor, ts_clean_chart as clean_chart, + ts_default_year as default_year, ts_month_bounds as month_bounds, ts_read_chart as read_chart) + +CENT = 0.005 +_REPORTS = {"pl": 7, "bs": 4, "cf": 5} # Odoo account.report ids on Royal's database +_READ_ONLY = frozenset({"get_report_informations"}) + + +def _report_creds(): + """(url, db, user, key) of the bound tenant's Odoo; the environment is tenant #0's only.""" + ctx = O._ctx() or {} + creds = ctx.get("creds") + if creds: + return (str(creds.get("url") or ""), str(creds.get("db") or ""), + str(creds.get("user") or ""), str(creds.get("api_key") or "")) + if str(ctx.get("slug") or O._DEFAULT_SLUG) != O._DEFAULT_SLUG: + raise O.TenantOdooUnavailable(f"tenant {ctx.get('slug')!r} has no Odoo credentials") + return os.environ["ODOO_URL"], os.environ["ODOO_DB"], os.environ["ODOO_USER"], os.environ["ODOO_API_KEY"] + + +def _report_call(report_id, options): + """Odoo's own report engine, read only. It answers over JSON-RPC; XML-RPC cannot marshal it.""" + base, db, user, key = _report_creds() + url = base.rstrip("/") + "/jsonrpc" + + def call(service, method, *args): + body = json.dumps({"jsonrpc": "2.0", "method": "call", "id": 1, + "params": {"service": service, "method": method, "args": list(args)}}).encode() + request = urllib.request.Request(url, body, {"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=300) as response: + out = json.loads(response.read()) + if out.get("error"): + raise RuntimeError(str(out["error"].get("data", {}).get("message") or out["error"])) + return out["result"] + method = "get_report_informations" + if method not in _READ_ONLY: + raise PermissionError(method) + uid = call("common", "login", db, user, key) + return call("object", "execute_kw", db, uid, key, "account.report", method, [report_id, options]) + + +_NAME = re.compile(r'class="account_report_line_name">\s*(.*?)\s*<', re.S) +_VALUE = re.compile(r'(?:=", date_from), + ("date", "<=", date_to)], ["balance:sum"], ["account_id"], lazy=False) + to_date = O.read_group("account.move.line", [("parent_state", "=", "posted"), ("date", "<=", date_to)], + ["balance:sum"], ["account_id"], lazy=False) + + def by_type(rows): + out = defaultdict(float) + for r in rows: + out[types.get(O.m2o_id(r["account_id"]))] += r["balance"] or 0.0 + return out + flow, stock = by_type(month), by_type(to_date) + group = lambda bag, groups: sum(bag[t] for _l, ts in groups for t in ts) # noqa: E731 + earnings = -sum(stock[t] for t in PL_TYPES) + return { + "net_income": -sum(flow[t] for t in PL_TYPES), + "assets": group(stock, ASSET_GROUPS), + "liabilities": -group(stock, LIABILITY_GROUPS), + "equity": -group(stock, EQUITY_GROUPS) + earnings, + "out_of_balance": round(sum(stock.values()), 2), + "cash": sum(stock[t] for label, ts in ASSET_GROUPS if label in CASH_GROUPS for t in ts), + } + + +def analytic_by_unit(date_from, date_to): + """{analytic account name: amount} on P&L accounts from Odoo's analytic ledger (R55's oracle).""" + types = _types_by_account() + pl_ids = [i for i, t in types.items() if t in PL_TYPES] + names = {r["id"]: r["name"] for r in O.search_read("account.analytic.account", [], ["name"])} + rows = O.read_group("account.analytic.line", [("date", ">=", date_from), ("date", "<=", date_to), + ("general_account_id", "in", pl_ids)], + ["amount:sum"], ["account_id"], lazy=False) + return {names.get(O.m2o_id(r["account_id"])): r["amount"] or 0.0 for r in rows} + + +def validate(year=None, month=6, *, runtime=None): + """Rule 8: the template's grouping ties Odoo's own statements for one month. + + Returns `{"ok", "checks": [(name, ours, odoo, ok)], "month"}`. Odoo's own out-of-balance gap + is a check of its own, so a ledger defect is visible and never absorbed. `runtime` binds every + read to that tenant's Odoo; None reads tenant #0, the gate's oracle. + """ + binding = nullcontext() + if runtime is not None: + source = runtime.odoo_source() if hasattr(runtime, "odoo_source") else None + if source is None: + raise ValueError("This workspace has no Odoo connection.") + binding = source._bound() + year = year or default_year() + first, last = month_bounds(year)[month - 1] + with binding: + ours = gl_totals(first, last) + pl, bs = report_lines("pl", first, last), report_lines("bs", last, last) + checks = [("net income", ours["net_income"], pl.get("Net Profit")), + ("total assets", ours["assets"], bs.get("ASSETS")), + ("total liabilities", ours["liabilities"], bs.get("LIABILITIES")), + ("total equity", ours["equity"], bs.get("EQUITY")), + ("bank and cash", ours["cash"], bs.get("Bank and Cash Accounts"))] + out = [(name, a, b, b is not None and abs(a - b) <= CENT) for name, a, b in checks] + gap = round((bs.get("ASSETS") or 0) - (bs.get("LIABILITIES + EQUITY") or 0), 2) + out.append(("ledger out of balance", ours["out_of_balance"], gap, ours["out_of_balance"] == gap)) + return {"ok": all(row[3] for row in out), "checks": out, "month": first[:7]} diff --git a/requirements.txt b/requirements.txt index a556d9b86e9f2b96224b74ac39a56ad89e13a47e..158f605b0762083de4f803efa3160db0e1e3fdb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ # wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the # import lives inside the handler, and the gate now asserts this line exists here. fastapi>=0.139 -starlette==1.3.1 +starlette==1.4.1 uvicorn[standard]>=0.30 python-dotenv>=1.2.2 pandas>=2.0 diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index dbd6efb4b62d676b7447c820c2724239669a1065..df359aabdff80ca5b00a60695e3ad8886586404a 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -6824a759adf3f7e8c1024e0b6018d935412d5746e2692cb95f9a35ad9cb12251 +85a9604dac43592a2c2e82e75b4e8526d8fc2b0763cd1415553dcb628464e8c5 diff --git a/web/src/app/FinanceWorkspace.tsx b/web/src/app/FinanceWorkspace.tsx index c3ea2488dab8669e3d6bba91b03f3c17f0e144c4..5b99e554330d0a1950a1f3dd637a6dcdc2aa4bdd 100644 --- a/web/src/app/FinanceWorkspace.tsx +++ b/web/src/app/FinanceWorkspace.tsx @@ -31,6 +31,7 @@ type MirrorWorklistContract = { entityField: string; documentField: string; amountField: string; + // C10: agingLabel prints "Overdue" from this field, so it binds the Odoo-matching `ar_overdue`, never the grace number. overdueAmountField?: string; dueDateField: string; ownerField: string; diff --git a/web/src/apps/CollectionPage.tsx b/web/src/apps/CollectionPage.tsx index 2878b65501ced5144b0b9340eca2de3ad1a8ed3a..66518abf311e763ca73ffd9931013548991b1b37 100644 --- a/web/src/apps/CollectionPage.tsx +++ b/web/src/apps/CollectionPage.tsx @@ -58,6 +58,7 @@ type BatchState = const money = new Intl.NumberFormat("en-US", { style: "currency", currency: "USD" }); const count = new Intl.NumberFormat("en-US"); +// C10: "Overdue" is Odoo's res.partner.total_overdue with no grace. A grace number reads "Overdue past grace". const columns = ["Customer", "Tier", "Overdue", "Receivable", "Oldest days", "Open docs", "Email", "Phone", "Status", "Reconcile"]; function plural(value: number, one: string, many: string): string { diff --git a/web/src/apps/account-health/AppView.tsx b/web/src/apps/account-health/AppView.tsx new file mode 100644 index 0000000000000000000000000000000000000000..ba921428ccd25d8284aec753c8438c2dd5f40457 --- /dev/null +++ b/web/src/apps/account-health/AppView.tsx @@ -0,0 +1,653 @@ +import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { TopLevelSpec } from "vega-lite"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { useEscapeClose } from "../../ui/useEscapeClose"; +import "../../styles/sdlc79/account-health.css"; + +// S79-T29 (C2, C4, R49): the Account health App screen. The dispatcher draws the App frame, so this view +// draws only the rating, the Metrics and Alerts tabs, their lists and the detail panel on the right. +const VegaLiteChart = lazy(() => import("../../viz/vega/VegaLiteChart")); + +/** C2: the App as the dispatcher passes it. Structurally equal to lane A's AppViewProps application. */ +interface AccountHealthApplication { + id: string; + name: string; + templateKind?: string; + templateView?: string; + templateMark?: string; + templateConnector?: string; + capabilities?: Record; +} + +interface AccountHealthViewProps { application: AccountHealthApplication; canEdit: boolean } + +interface Metric { + id: string; + key: string; + marketplaceId: string; + label: string; + unit: string; + value: number | null; + display: string; + target: number | null; + condition: string; + status: string; + windowFrom: string; + windowTo: string; +} + +interface Summary { + state: "ready" | "empty"; + message: string; + rating: number | null; + ratingStatus: string; + snapshotDate: string; + markets: number; + metrics: Metric[]; + canEdit: boolean; +} + +interface Alert { + id: string; + title: string; + status: string; + owner: string; + due: string; + reason: string; + metricId: string; + label: string; + level: string; + note: string; +} + +interface AlertPage { items: Alert[]; cursor: string | null; statuses: string[]; owners: string[]; canEdit: boolean } + +interface Point { day: string; value: number | null; target: number | null; status: string } +interface HistoryPage { points: Point[]; cursor: string | null; label: string; unit: string } + +type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; +type Picked = { kind: "metric" | "alert"; id: string }; +type Tab = "metrics" | "alerts"; + +const EMPTY = "No account health data"; +const ALERT_STATUSES = ["Open", "Investigating", "Resolved", "Dismissed"]; +const OPEN = new Set(["Open", "Investigating"]); +const PAGE = 200; +const HISTORY_PAGE = 400; + +const text = (value: unknown) => (typeof value === "string" ? value : ""); +const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); +const list = (value: unknown) => (Array.isArray(value) ? value : []); +const words = (value: unknown) => list(value).map(text).filter(Boolean); +const record = (value: unknown) => (value ?? {}) as Record; + +async function readJson(response: Response, fallback: string): Promise { + const body: unknown = await response.json().catch(() => null); + if (response.ok) return body; + // The server sends a refusal as top-level {error}; a bare router sends it under {detail: {error}}. + const shaped = body as { error?: { message?: unknown }; detail?: { error?: { message?: unknown } } } | null; + throw new Error(text(shaped?.error?.message) || text(shaped?.detail?.error?.message) || fallback); +} + +const failure = (reason: unknown, fallback: string) => (reason instanceof Error && reason.message + ? reason.message : fallback); + +function parseMetric(raw: unknown): Metric | null { + const row = record(raw); + const id = text(row.id); + if (!id) return null; + return { + id, key: text(row.key), marketplaceId: text(row.marketplaceId), label: text(row.label) || id, + unit: text(row.unit), value: num(row.value), display: text(row.display), target: num(row.target), + condition: text(row.condition), status: text(row.status), windowFrom: text(row.windowFrom), + windowTo: text(row.windowTo), + }; +} + +function parseSummary(raw: unknown): Summary { + const body = record(raw); + return { + state: body.state === "ready" ? "ready" : "empty", message: text(body.message), rating: num(body.rating), + ratingStatus: text(body.ratingStatus), snapshotDate: text(body.snapshotDate), + markets: list(body.marketplaces).length, + metrics: list(body.metrics).map(parseMetric).filter((row): row is Metric => row !== null), + canEdit: body.canEdit === true, + }; +} + +function parseAlert(raw: unknown): Alert | null { + const row = record(raw); + const id = text(row.id); + if (!id) return null; + return { + id, title: text(row.title) || text(row.label), status: text(row.status), owner: text(row.owner), + due: text(row.due), reason: text(row.reason), metricId: text(row.metricId), label: text(row.label), + level: text(row.level), note: text(row.note), + }; +} + +function parseAlerts(raw: unknown): AlertPage { + const body = record(raw); + const statuses = words(body.statuses); + return { + items: list(body.items).map(parseAlert).filter((row): row is Alert => row !== null), + cursor: text(body.cursor) || null, statuses: statuses.length ? statuses : ALERT_STATUSES, + owners: words(body.owners), canEdit: body.canEdit === true, + }; +} + +function parseHistory(raw: unknown): HistoryPage { + const body = record(raw); + return { + points: list(body.points).map((entry) => { + const row = record(entry); + return { day: text(row.day), value: num(row.value), target: num(row.target), status: text(row.status) }; + }).filter((point) => point.day), + cursor: text(body.cursor) || null, label: text(body.label), unit: text(body.unit), + }; +} + +/** "Oct 22" from an ISO day. A calendar day is read in UTC. */ +function day(iso: string) { + if (!iso) return ""; + const date = new Date(iso.length === 10 ? iso + "T00:00:00Z" : iso); + return Number.isNaN(date.getTime()) ? iso + : date.toLocaleDateString(undefined, { month: "short", day: "numeric", timeZone: "UTC" }); +} + +function amount(value: number | null, unit: string, status = "") { + if (value === null) return status === "No orders" ? "No orders" : "No data"; + if (unit === "percent") return `${value.toFixed(2)}%`; + return value.toLocaleString(undefined, { maximumFractionDigits: 2 }); +} + +const metricText = (row: Metric) => row.display || amount(row.value, row.unit, row.status); +const targetText = (row: Metric) => row.condition || "No target"; +const windowText = (row: Metric) => (row.windowFrom && row.windowTo + ? `${day(row.windowFrom)} to ${day(row.windowTo)}` : ""); + +/** A chip's tone class. Every chip also names its state in words. */ +function tone(status: string) { + switch (status) { + case "Good": case "Great": case "Resolved": return "is-good"; + case "Near target": case "Fair": case "Investigating": return "is-near"; + case "At risk": case "Bad": case "Critical": case "Poor": return "is-risk"; + case "Open": return "is-open"; + default: return "is-quiet"; + } +} + +function Chip({ status }: { status: string }) { + return status ? {status} : null; +} + +function Skeleton({ rows = 4, chart = false }: { rows?: number; chart?: boolean }) { + return
+ {chart ?
+ : Array.from({ length: rows }, (_v, index) =>
)} +
; +} + +function Failed({ message, onRetry }: { message: string; onRetry: () => void }) { + return

{message}{" "} +

; +} + +function chartColors() { + const style = getComputedStyle(document.documentElement); + const read = (token: string) => style.getPropertyValue(token).trim(); + return { series: read("--lp-blue-deep"), target: read("--lp-red-deep"), line: read("--lp-line"), + surface: read("--lp-surface"), muted: read("--lp-muted") }; +} + +function trendSpec(points: Point[], unit: string, label: string): TopLevelSpec { + const values = points.filter((point) => point.value !== null) + .map((point) => ({ day: point.day, value: point.value, dayLabel: day(point.day), + valueLabel: amount(point.value, unit) })); + const targets = points.filter((point) => point.target !== null) + .map((point) => ({ day: point.day, target: point.target, targetLabel: amount(point.target, unit) })); + const tokens = chartColors(); + const percent = unit === "percent"; + const x = { field: "day", type: "ordinal" as const, title: null, sort: "ascending" as const, + axis: { labelAngle: 0, labelExpr: "timeFormat(toDate(datum.value + 'T00:00:00'), '%b %-d')" } }; + const y = { type: "quantitative" as const, title: null, + axis: { gridColor: tokens.line, tickCount: 4, + labelExpr: percent ? "format(datum.value, '.1f') + '%'" : "format(datum.value, ',~r')" } }; + const layer: unknown[] = [ + { data: { values }, mark: { type: "line", strokeWidth: 2, color: tokens.series }, + encoding: { x, y: { ...y, field: "value" } } }, + { data: { values }, mark: { type: "point", filled: true, size: 60, color: tokens.series, + stroke: tokens.surface, strokeWidth: 1.5 }, + encoding: { x, y: { ...y, field: "value" }, + tooltip: [{ field: "dayLabel", title: "Day" }, { field: "valueLabel", title: label }] } }, + ]; + if (targets.length) { + layer.push({ data: { values: targets }, mark: { type: "line", strokeWidth: 1.5, strokeDash: [4, 4], + color: tokens.target }, encoding: { x, y: { ...y, field: "target" }, + tooltip: [{ field: "dayLabel", title: "Day" }, { field: "targetLabel", title: "Target" }] } }); + // Points come newest first: the label sits at the newest day, the right edge. + layer.push({ data: { values: targets.slice(0, 1) }, mark: { type: "text", align: "right", dy: -8, + color: tokens.target, fontSize: 11 }, encoding: { x, y: { ...y, field: "target" }, + text: { value: "Target" } } }); + } + return { + $schema: "https://vega.github.io/schema/vega-lite/v6.json", + width: "container", + height: 160, + autosize: { type: "fit", contains: "padding" }, + background: "transparent", + config: { view: { stroke: null }, axis: { domain: false, ticks: false, labelFontSize: 11, + labelColor: tokens.muted }, font: "Inter" }, + layer, + } as TopLevelSpec; +} + +function TrendChart({ points, unit, label }: { points: Point[]; unit: string; label: string }) { + const spec = useMemo(() => trendSpec(points, unit, label), [points, unit, label]); + const held = points.filter((point) => point.value !== null).length; + if (held < 2) return

The trend grows by one point each day.

; + return
+ }> + + +
; +} + +/** One metric's daily history, newest first, paged by cursor (rule 1). */ +function History({ metricId, label, url }: { + metricId: string; label: string; url: (path: string, params?: Record) => string; +}) { + const [page, setPage] = useState>({ phase: "loading" }); + const [attempt, setAttempt] = useState(0); + const [loadingMore, setLoadingMore] = useState(false); + const [moreError, setMoreError] = useState(""); + + useEffect(() => { + const controller = new AbortController(); + setPage({ phase: "loading" }); + setMoreError(""); + void fetch(url("/history", { metric: metricId, limit: String(HISTORY_PAGE) }), + { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "The history of this metric could not be read.")) + .then((body) => { if (!controller.signal.aborted) setPage({ phase: "ready", value: parseHistory(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) { + setPage({ phase: "error", message: failure(reason, "The history of this metric could not be read.") }); + } + }); + return () => controller.abort(); + }, [url, metricId, attempt]); + + const loadMore = async () => { + if (page.phase !== "ready" || !page.value.cursor) return; + setLoadingMore(true); + setMoreError(""); + try { + const response = await fetch(url("/history", { metric: metricId, limit: String(HISTORY_PAGE), + cursor: page.value.cursor }), { credentials: CREDENTIALS }); + const next = parseHistory(await readJson(response, "More days could not be read.")); + setPage((current) => current.phase !== "ready" ? current : { phase: "ready", value: { + ...current.value, points: [...current.value.points, ...next.points], cursor: next.cursor } }); + } catch (reason) { + setMoreError(failure(reason, "More days could not be read.")); + } finally { + setLoadingMore(false); + } + }; + + if (page.phase === "loading") return ; + if (page.phase === "error") return setAttempt((n) => n + 1)} />; + const { points, unit, cursor } = page.value; + if (points.length === 0) return

No history yet

; + return
+

Daily history

+ +
+ + + {points.map((point) => + + + + )} +
DayValueStatus
{day(point.day)}{amount(point.value, unit, point.status)}
+ {moreError &&

{moreError}

} + {cursor &&
+ +
} + ; +} + +/** Owner and status of one alert. An editor changes them with selects; the stored answer shows. */ +function AlertEditor({ alert, editable, owners, statuses, url, onSaved }: { + alert: Alert; editable: boolean; owners: string[]; statuses: string[]; + url: (path: string, params?: Record) => string; onSaved: (alert: Alert) => void; +}) { + const [saving, setSaving] = useState(false); + const [saveError, setSaveError] = useState(""); + const [savedAt, setSavedAt] = useState(""); + + const save = async (change: { owner?: string; status?: string }) => { + setSaving(true); + setSaveError(""); + setSavedAt(""); + try { + const response = await fetch(url(`/items/${encodeURIComponent(alert.id)}`), { + method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify(change), + }); + const stored = parseAlert(await readJson(response, "The change was not saved.")); + if (!stored) throw new Error("The change was not saved."); + onSaved(stored); + setSavedAt("Saved"); + } catch (reason) { + setSaveError(failure(reason, "The change was not saved.")); + } finally { + setSaving(false); + } + }; + + const ownerOptions = alert.owner && !owners.includes(alert.owner) ? [alert.owner, ...owners] : owners; + const statusOptions = alert.status && !statuses.includes(alert.status) ? [alert.status, ...statuses] : statuses; + return
+ + +
+ Due + {day(alert.due) || "No date"} +
+ {saveError &&

{saveError}

} + {savedAt &&

{savedAt}

} +
; +} + +interface PanelProps { + metric: Metric | null; + alert: Alert | null; + editable: boolean; + owners: string[]; + statuses: string[]; + url: (path: string, params?: Record) => string; + onSaved: (alert: Alert) => void; + onClose: () => void; +} + +/** The detail panel on the right: a picked metric with its daily history, or a picked alert. */ +function Panel({ metric, alert, editable, owners, statuses, url, onSaved, onClose }: PanelProps) { + const panel = useRef(null); + const close = useRef(null); + useEscapeClose(true, onClose, { container: () => panel.current }); + const focusKey = `${metric?.id ?? ""}:${alert?.id ?? ""}`; + useEffect(() => { close.current?.focus(); }, [focusKey]); + + const title = alert && !metric ? alert.title : metric?.label ?? alert?.title ?? ""; + const metricId = metric?.id ?? alert?.metricId ?? ""; + const label = metric?.label ?? alert?.label ?? ""; + return ; +} + +function MetricTable({ metrics, markets, picked, onPick }: { + metrics: Metric[]; markets: number; picked: string; onPick: (row: Metric) => void; +}) { + return + + + + + + + + + + + + + + {metrics.map((row) => + + + + + )} + +
MetricValueAmazon's targetStatus
+ + {markets > 1 && row.marketplaceId && {row.marketplaceId}} + {metricText(row)}{targetText(row)}
; +} + +function AlertTable({ alerts, picked, onPick }: { alerts: Alert[]; picked: string; onPick: (row: Alert) => void }) { + return + + + + + + + + + + + + + + {alerts.map((row) => + + + + + )} + +
AlertOwnerDueStatus
+ + {row.reason} + {row.owner || "Nobody"}{day(row.due)}
; +} + +export default function AccountHealthView({ application, canEdit }: AccountHealthViewProps) { + const appId = application.id; + const url = useCallback((path: string, params: Record = {}) => { + const query = new URLSearchParams({ app: appId, ...params }); + return `${API_V1}/app-kinds/account_health${path}?${query.toString()}`; + }, [appId]); + const [summary, setSummary] = useState>({ phase: "loading" }); + const [summaryAttempt, setSummaryAttempt] = useState(0); + const [alerts, setAlerts] = useState>({ phase: "loading" }); + const [alertsAttempt, setAlertsAttempt] = useState(0); + const [statusFilter, setStatusFilter] = useState(""); + const [loadingMore, setLoadingMore] = useState(false); + const [moreError, setMoreError] = useState(""); + const [tab, setTab] = useState("metrics"); + const [picked, setPicked] = useState(null); + const opener = useRef(null); + const alive = useRef(true); + useEffect(() => () => { alive.current = false; }, []); + + useEffect(() => { + const controller = new AbortController(); + setSummary({ phase: "loading" }); + void fetch(url("/summary"), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "Account health could not be read.")) + .then((body) => { if (!controller.signal.aborted) setSummary({ phase: "ready", value: parseSummary(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setSummary({ phase: "error", message: failure(reason, "Account health could not be read.") }); + }); + return () => controller.abort(); + }, [url, summaryAttempt]); + + const alertsUrl = useCallback((cursor?: string) => { + const params: Record = { limit: String(PAGE) }; + if (statusFilter) params.status = statusFilter; + if (cursor) params.cursor = cursor; + return url("/items", params); + }, [url, statusFilter]); + + useEffect(() => { + const controller = new AbortController(); + setAlerts({ phase: "loading" }); + setMoreError(""); + void fetch(alertsUrl(), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "The alerts could not be read.")) + .then((body) => { if (!controller.signal.aborted) setAlerts({ phase: "ready", value: parseAlerts(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setAlerts({ phase: "error", message: failure(reason, "The alerts could not be read.") }); + }); + return () => controller.abort(); + }, [alertsUrl, alertsAttempt]); + + const loadMore = async () => { + if (alerts.phase !== "ready" || !alerts.value.cursor) return; + setLoadingMore(true); + setMoreError(""); + try { + const response = await fetch(alertsUrl(alerts.value.cursor), { credentials: CREDENTIALS }); + const next = parseAlerts(await readJson(response, "More alerts could not be read.")); + if (!alive.current) return; + setAlerts((current) => current.phase !== "ready" ? current : { phase: "ready", value: { + ...current.value, items: [...current.value.items, ...next.items], cursor: next.cursor } }); + } catch (reason) { + if (alive.current) setMoreError(failure(reason, "More alerts could not be read.")); + } finally { + if (alive.current) setLoadingMore(false); + } + }; + + const pick = (next: Picked) => { + opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; + setPicked(next); + }; + const closePanel = useCallback(() => { + setPicked(null); + opener.current?.focus(); + }, []); + const saved = useCallback((alert: Alert) => { + setAlerts((current) => current.phase !== "ready" ? current : { phase: "ready", value: { + ...current.value, items: current.value.items.map((row) => (row.id === alert.id ? { ...row, ...alert } : row)), + } }); + }, []); + + const view = summary.phase === "ready" ? summary.value : null; + const page = alerts.phase === "ready" ? alerts.value : null; + const editable = canEdit === true && page?.canEdit === true; + const items = page?.items ?? []; + const metric = picked && view ? (picked.kind === "metric" + ? view.metrics.find((row) => row.id === picked.id) ?? null + : view.metrics.find((row) => row.id === items.find((a) => a.id === picked.id)?.metricId) ?? null) : null; + const alert = picked ? (picked.kind === "alert" ? items.find((row) => row.id === picked.id) ?? null + : items.find((row) => row.metricId === picked.id && OPEN.has(row.status)) + ?? items.find((row) => row.metricId === picked.id) ?? null) : null; + const alertCount = `${items.length}${page?.cursor ? "+" : ""}`; + + if (summary.phase === "loading") { + return
; + } + if (summary.phase === "error") { + return
+ setSummaryAttempt((n) => n + 1)} /> +
; + } + if (!view || view.state === "empty") { + return
+

{view?.message || EMPTY}

+
; + } + + const open = Boolean(picked && (metric || alert)); + return
+
+
+

Account health rating

+
+ {view.rating === null ? "No rating" + : view.rating.toLocaleString(undefined, { maximumFractionDigits: 0 })} + +
+ {view.snapshotDate && Updated {day(view.snapshotDate)}} +
+ +
+ + +
+ + {tab === "metrics" &&
+ {view.metrics.length === 0 + ?

{EMPTY}

+ : pick({ kind: "metric", id: row.id })} />} +
} + + {tab === "alerts" &&
+
+ +
+ {alerts.phase === "loading" && } + {alerts.phase === "error" && setAlertsAttempt((n) => n + 1)} />} + {page && items.length === 0 &&

No alerts

} + {page && items.length > 0 && pick({ kind: "alert", id: row.id })} />} + {moreError &&

{moreError}

} + {page?.cursor &&
+ +
} +
} +
+ {open && } +
; +} diff --git a/web/src/apps/aged-stock/AppView.tsx b/web/src/apps/aged-stock/AppView.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e65124f8eea0d9240f216df0373f20d8c33bafc4 --- /dev/null +++ b/web/src/apps/aged-stock/AppView.tsx @@ -0,0 +1,794 @@ +import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { TopLevelSpec } from "vega-lite"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { useEscapeClose } from "../../ui/useEscapeClose"; +import "../../styles/sdlc79/aged-stock.css"; + +// S79-T33 (C2, C4, R49): the Aged stock App screen. The dispatcher draws the App frame, so this view draws +// only the KPI strip, the monthly fee trend, the SKU list and the SKU drawer. No chat panel exists here (R18). +const VegaLiteChart = lazy(() => import("../../viz/vega/VegaLiteChart")); + +/** C2: the App as the dispatcher passes it. Structurally equal to lane A's TemplateApplication. */ +interface AgedStockApplication { + id: string; + name: string; + templateKind?: string; + templateView?: string; + templateMark?: string; + templateConnector?: string; + capabilities?: Record; +} + +interface AgedStockAppViewProps { application: AgedStockApplication; canEdit: boolean } + +interface Month { month: string; storage: number; longTerm: number; total: number } +interface Totals { storage: number; longTerm: number; total: number; atStake: number } + +/** One list row, from a Next tier entry or an item. */ +interface Row { + id: string; + sku: string; + fnsku: string; + asin: string; + title: string; + action: string; + recommended: string; + tier: string; + units: number | null; + atStake: number | null; + currency: string; + owner: string; + status: string; + due: string; + inReport: boolean; +} + +interface FeePoint { month: string; currency: string; storage: number; longTerm: number; tier: string; units: number | null } +interface Removal { snapshot: string; removal: number | null; sellable: number | null; unsellable: number | null } +interface Stranded { reasons: string[]; units: number | null; autoRemoval: string; since: string } +interface Detail { + row: Row; reason: string; fees: FeePoint[]; bands: [string, number | null][]; removal: Removal | null; + stranded: Stranded | null; owners: string[]; +} + +interface Summary { + ready: boolean; + notReady: string; + empty: boolean; + notices: string[]; + primary: string; + trend: Month[]; + totals: Totals | null; + others: [string, number][]; + nextTier: Row[]; + statusCounts: Record; + actionCounts: Record; + refreshedAt: string; + lastError: string; + canEdit: boolean; +} + +interface ListPage { items: Row[]; cursor: string | null; owners: string[] } + +type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; + +const NEXT_TIER = "Next tier"; +const ALL = "All"; +const ACTIONS = ["Remove", "Price down", "Push with ads"]; +/** The tab order: the Next tier list first, then one tab per action, then every SKU. */ +const TABS = [NEXT_TIER, ...ACTIONS, ALL]; +const STATUSES = ["Open", "Accepted", "In progress", "Done", "Dismissed", "Watching"]; +/** Age band stems in age order, as the module names them. */ +const BANDS: [string, string][] = [["0_to_90", "0 to 90 days"], ["91_to_180", "91 to 180 days"], + ["181_to_270", "181 to 270 days"], ["271_to_365", "271 to 365 days"], ["365_plus", "Over 365 days"]]; +const PAGE = 100; +const SEARCH_DELAY_MS = 250; +const SEARCH_LABEL = "Search SKU or product"; +const EMPTY_APP = "No aged stock"; +const EMPTY_TAB = "No SKUs here"; +const SERIES = { storage: "Storage", longTerm: "Long term" }; + +const text = (value: unknown) => (typeof value === "string" ? value : ""); +const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); +const list = (value: unknown) => (Array.isArray(value) ? value : []); +const words = (value: unknown) => list(value).map(text).filter(Boolean); +const record = (value: unknown) => (value && typeof value === "object" && !Array.isArray(value) + ? value as Record : {}); + +async function readJson(response: Response, fallback: string): Promise { + const body: unknown = await response.json().catch(() => null); + if (response.ok) return body; + // The server sends a refusal as top-level {error}; a bare router sends it under {detail: {error}}. + const shaped = body as { error?: { message?: unknown }; detail?: { error?: { message?: unknown } } } | null; + throw new Error(text(shaped?.error?.message) || text(shaped?.detail?.error?.message) || fallback); +} + +const failure = (reason: unknown, fallback: string) => (reason instanceof Error && reason.message + ? reason.message : fallback); + +/** A money amount in its own currency. An unknown currency code reads as a plain amount. */ +function money(value: number | null, currency: string, code = false) { + if (value === null) return ""; + try { + return value.toLocaleString(undefined, { style: "currency", currency, minimumFractionDigits: 2, + currencyDisplay: code ? "code" : "symbol" }); + } catch { + return value.toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + } +} + +const qty = (value: number | null) => (value === null ? "" + : value.toLocaleString(undefined, { maximumFractionDigits: 1 })); + +/** "Sep 2026" from "2026-09". */ +function monthWords(month: string) { + const date = new Date(`${month}-01T00:00:00Z`); + return Number.isNaN(date.getTime()) ? month + : date.toLocaleDateString(undefined, { month: "short", year: "numeric", timeZone: "UTC" }); +} + +/** "Oct 24" from an ISO day. A calendar day is read in UTC. */ +function day(iso: string) { + if (!iso) return ""; + const date = new Date(iso.length === 10 ? iso + "T00:00:00Z" : iso); + return Number.isNaN(date.getTime()) ? iso + : date.toLocaleDateString(undefined, { month: "short", day: "numeric", year: "numeric", timeZone: "UTC" }); +} + +/** Amazon's charged tier ("181-210", "365+") in the list's words ("181 to 210 days"). Anything else reads as sent. */ +function tierWords(tier: string) { + const range = /^(\d+)\s*-\s*(\d+)(\s*days)?$/i.exec(tier.trim()); + if (range) return `${range[1]} to ${range[2]} days`; + const over = /^(\d+)\s*\+(\s*days)?$/i.exec(tier.trim()); + return over ? `Over ${over[1]} days` : tier; +} + +const tone = (value: string) => value.toLowerCase().replace(/[^a-z]+/g, "-"); + +function parseItem(raw: unknown): Row | null { + const row = record(raw); + const id = text(row.id); + if (!id) return null; + const currency = text(row.currency); + const next = record(row.next_tier); + const stake = record(row.at_stake); + return { + id, sku: text(row.sku), fnsku: text(row.fnsku), asin: text(row.asin), + title: text(row.title) || text(row.sku) || text(row.fnsku), action: text(row.action), + recommended: text(row.recommended_action), tier: text(next.tier), units: num(row.units), + atStake: num(stake[currency]), currency, owner: text(row.owner), status: text(row.status), + due: text(row.due), inReport: row.inReport !== false, + }; +} + +function parseNextTier(raw: unknown): Row | null { + const row = record(raw); + const id = text(row.id); + if (!id) return null; + const item = record(row.item); + return { + id, sku: text(row.sku), fnsku: text(row.fnsku), asin: "", title: text(row.title) || text(row.sku), + action: text(item.action), recommended: text(item.recommended_action), tier: text(row.tier), + units: num(row.units), atStake: num(row.at_stake), currency: text(row.currency), owner: text(item.owner), + status: text(item.status), due: text(item.due), inReport: true, + }; +} + +function parseTotals(raw: unknown): Totals { + const t = record(raw); + return { storage: num(t.storage) ?? 0, longTerm: num(t.long_term) ?? 0, total: num(t.total) ?? 0, + atStake: num(t.at_stake) ?? 0 }; +} + +function counts(raw: unknown, names: string[]) { + const body = record(raw); + const out: Record = {}; + for (const name of names) out[name] = num(body[name]) ?? 0; + return out; +} + +function parseSummary(raw: unknown): Summary { + const body = record(raw); + const trend = record(body.trend); + const totals = record(body.totals); + const primary = text(body.primaryCurrency) || Object.keys(totals)[0] || ""; + return { + ready: body.ready === true, notReady: text(body.notReady), empty: body.empty === true, + notices: words(body.notices), primary, + trend: list(trend[primary]).map((entry) => { + const m = record(entry); + return { month: text(m.month), storage: num(m.storage) ?? 0, longTerm: num(m.long_term) ?? 0, + total: num(m.total) ?? 0 }; + }).filter((m) => m.month), + totals: primary && totals[primary] ? parseTotals(totals[primary]) : null, + // Other currencies are listed, never added to the primary one. + others: Object.keys(totals).filter((cur) => cur !== primary).sort() + .map((cur): [string, number] => [cur, parseTotals(totals[cur]).total]), + nextTier: list(body.nextTier).map(parseNextTier).filter((row): row is Row => row !== null), + statusCounts: counts(body.statusCounts, STATUSES), actionCounts: counts(body.actionCounts, ACTIONS), + refreshedAt: text(body.refreshedAt), lastError: text(body.lastError), canEdit: body.canEdit === true, + }; +} + +function parsePage(raw: unknown): ListPage { + const body = record(raw); + return { items: list(body.items).map(parseItem).filter((row): row is Row => row !== null), + cursor: text(body.cursor) || null, owners: words(body.owners) }; +} + +function parseDetail(raw: unknown, fallback: Row): Detail { + const body = record(raw); + const bands = record(body.age_bands); + const removal = body.removal ? record(body.removal) : null; + const stranded = body.stranded ? record(body.stranded) : null; + return { + row: parseItem(body) ?? fallback, + reason: text(body.reason), + fees: list(body.fee_history).map((entry) => { + const p = record(entry); + return { month: text(p.month), currency: text(p.currency), storage: num(p.storage) ?? 0, + longTerm: num(p.long_term) ?? 0, tier: text(p.tier), units: num(p.units) }; + }), + bands: BANDS.map(([stem, label]): [string, number | null] => [label, num(bands[stem])]), + removal: removal && { snapshot: text(removal.snapshot), removal: num(removal.sellable_removal_quantity), + sellable: num(removal.sellable_quantity), unsellable: num(removal.unsellable_quantity) }, + stranded: stranded && { reasons: words(stranded.reasons), units: num(stranded.units), + autoRemoval: text(stranded.auto_removal_date), since: text(stranded.date_stranded) }, + owners: words(body.owners), + }; +} + +function chartColors() { + const style = getComputedStyle(document.documentElement); + const read = (token: string) => style.getPropertyValue(token).trim(); + return { storage: read("--lp-blue-solid"), longTerm: read("--lp-orange-deep"), line: read("--lp-line"), + surface: read("--lp-surface"), muted: read("--lp-muted") }; +} + +/** Stacked monthly bars of storage and long term fees in one currency, on one axis. */ +function trendSpec(months: Month[], currency: string): TopLevelSpec { + const tokens = chartColors(); + const values = months.flatMap((m) => { + const shared = { month: m.month, monthLabel: monthWords(m.month), storageLabel: money(m.storage, currency), + longTermLabel: money(m.longTerm, currency), totalLabel: money(m.total, currency) }; + return [{ ...shared, series: SERIES.storage, amount: m.storage, order: 0 }, + { ...shared, series: SERIES.longTerm, amount: m.longTerm, order: 1 }]; + }); + return { + $schema: "https://vega.github.io/schema/vega-lite/v6.json", + width: "container", + height: 220, + autosize: { type: "fit", contains: "padding" }, + background: "transparent", + config: { view: { stroke: null }, axis: { domain: false, ticks: false, labelFontSize: 11, titleFontSize: 11, + titleFontWeight: 500, labelColor: tokens.muted, titleColor: tokens.muted }, font: "Inter" }, + data: { values }, + mark: { type: "bar", stroke: tokens.surface, strokeWidth: 2, cornerRadius: 2 }, + encoding: { + x: { field: "month", type: "ordinal", title: null, sort: "ascending", + axis: { labelAngle: 0, labelExpr: "timeFormat(toDate(datum.value + '-01T00:00:00'), '%b %Y')" }, + scale: { paddingInner: 0.35 } }, + y: { field: "amount", type: "quantitative", aggregate: "sum", stack: "zero", title: currency, + axis: { gridColor: tokens.line, tickCount: 4, format: ",.2~f" } }, + color: { field: "series", type: "nominal", legend: null, + scale: { domain: [SERIES.storage, SERIES.longTerm], range: [tokens.storage, tokens.longTerm] } }, + order: { field: "order", type: "quantitative" }, + tooltip: [{ field: "monthLabel", title: "Month" }, { field: "storageLabel", title: SERIES.storage }, + { field: "longTermLabel", title: SERIES.longTerm }, { field: "totalLabel", title: "Total" }], + }, + } as TopLevelSpec; +} + +function Skeleton({ rows = 4, chart = false }: { rows?: number; chart?: boolean }) { + if (chart) return
; + return
+ {Array.from({ length: rows }, (_v, index) =>
)} +
; +} + +function Failed({ message, onRetry }: { message: string; onRetry: () => void }) { + return

{message}{" "} +

; +} + +function Pill({ value }: { value: string }) { + return value ? {value} : null; +} + +function Kpi({ label, value, period }: { label: string; value: string; period: string }) { + return
+ {label} + {value} + {period} +
; +} + +function KpiStrip({ summary }: { summary: Summary }) { + const cur = summary.primary; + const t = summary.totals ?? { storage: 0, longTerm: 0, total: 0, atStake: 0 }; + const span = summary.trend.length === 0 ? "No months yet" + : summary.trend.length === 1 ? monthWords(summary.trend[0].month) + : `${monthWords(summary.trend[0].month)} to ${monthWords(summary.trend[summary.trend.length - 1].month)}`; + const open = summary.statusCounts.Open ?? 0; + return
+ + + + +
; +} + +function TrendChart({ months, currency }: { months: Month[]; currency: string }) { + const spec = useMemo(() => trendSpec(months, currency), [months, currency]); + return
+ }> + + +
; +} + +/** The chart's text equivalent: the same months, with the total row the fee total ties to. */ +function MonthTable({ months, totals, currency }: { months: Month[]; totals: Totals; currency: string }) { + return
+ + + + {months.map((m) => + + + )} + + + +
Month{SERIES.storage}{SERIES.longTerm}Total
{monthWords(m.month)}{money(m.storage, currency)}{money(m.longTerm, currency)}{money(m.total, currency)}
Total{money(totals.storage, currency)}{money(totals.longTerm, currency)}{money(totals.total, currency)}
+
; +} + +function Trend({ summary }: { summary: Summary }) { + const cur = summary.primary; + return
+
+

Monthly fees

+
    +
  • +
  • +
+
+ {summary.trend.length === 0 + ?

No fees charged yet

+ : <> + + {summary.totals && } + } +
; +} + +function Fact({ label, value, numeric = false }: { label: string; value: string; numeric?: boolean }) { + return
+
{label}
{value || "Not known"}
+
; +} + +interface DrawerProps { + row: Row; + editable: boolean; + owners: string[]; + url: (path: string, params?: Record) => string; + onSaved: (row: Row) => void; + onClose: () => void; +} + +type Change = { owner?: string; status?: string; due?: string; action?: string }; + +function Drawer({ row, editable, owners: listOwners, url, onSaved, onClose }: DrawerProps) { + const panel = useRef(null); + const close = useRef(null); + const [detail, setDetail] = useState>({ phase: "loading" }); + const [attempt, setAttempt] = useState(0); + const [saving, setSaving] = useState(false); + const [saveError, setSaveError] = useState(""); + // The row object changes after a save. Only a different SKU reloads the detail. + const latest = useRef(row); + latest.current = row; + useEscapeClose(true, onClose, { container: () => panel.current }); + useEffect(() => { close.current?.focus(); }, [row.id]); + + useEffect(() => { + const controller = new AbortController(); + setDetail({ phase: "loading" }); + void fetch(url(`/items/${encodeURIComponent(row.id)}`), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "This SKU could not be loaded.")) + .then((body) => { if (!controller.signal.aborted) setDetail({ phase: "ready", value: parseDetail(body, latest.current) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setDetail({ phase: "error", message: failure(reason, "This SKU could not be loaded.") }); + }); + return () => controller.abort(); + }, [url, row.id, attempt]); + + /** Store one change, then show the value the server answered. */ + const save = async (change: Change) => { + setSaving(true); + setSaveError(""); + try { + const response = await fetch(url(`/items/${encodeURIComponent(row.id)}`), { + method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify(change), + }); + const stored = parseItem(await readJson(response, "The change was not saved.")); + if (!stored) throw new Error("The change was not saved."); + const kept = { action: stored.action, owner: stored.owner, status: stored.status, due: stored.due }; + onSaved({ ...latest.current, ...kept }); + setDetail((current) => current.phase !== "ready" ? current + : { phase: "ready", value: { ...current.value, row: { ...current.value.row, ...kept } } }); + } catch (reason) { + setSaveError(failure(reason, "The change was not saved.")); + } finally { + setSaving(false); + } + }; + + const ready = detail.phase === "ready" ? detail.value : null; + const item = ready ? { ...ready.row, action: row.action, owner: row.owner, status: row.status, due: row.due } : row; + const tier = row.tier || item.tier; + const atStake = row.atStake ?? item.atStake; + const owners = ready && ready.owners.length ? ready.owners : listOwners; + const ownerOptions = item.owner && !owners.includes(item.owner) ? [item.owner, ...owners] : owners; + const actionOptions = item.action && !ACTIONS.includes(item.action) ? [item.action, ...ACTIONS] : ACTIONS; + const statusOptions = item.status && !STATUSES.includes(item.status) ? [item.status, ...STATUSES] : STATUSES; + return ; +} + +function SkuTable({ rows, onOpen }: { rows: Row[]; onOpen: (row: Row) => void }) { + return
+ + + + + + + + + + + + + + + + + + + + {rows.map((row) => + + + + + + + + + )} + +
ProductActionTierUnitsMoney at stakeOwnerStatusDue
+ + {row.sku || row.fnsku} + {row.action}{row.tier}{qty(row.units)}{money(row.atStake, row.currency)}{row.owner}{day(row.due)}
+
; +} + +export default function AgedStockView({ application, canEdit }: AgedStockAppViewProps) { + const appId = application.id; + const url = useCallback((path: string, params: Record = {}) => { + const query = new URLSearchParams({ app: appId, ...params }); + return `${API_V1}/app-kinds/aged_stock${path}?${query.toString()}`; + }, [appId]); + const [summary, setSummary] = useState>({ phase: "loading" }); + const [attempt, setAttempt] = useState(0); + const [tab, setTab] = useState(NEXT_TIER); + const [search, setSearch] = useState(""); + const [q, setQ] = useState(""); + const [page, setPage] = useState>({ phase: "loading" }); + const [loadingMore, setLoadingMore] = useState(false); + const [moreError, setMoreError] = useState(""); + const [refreshing, setRefreshing] = useState(false); + const [refreshError, setRefreshError] = useState(""); + const [openedId, setOpenedId] = useState(null); + const opener = useRef(null); + const alive = useRef(true); + useEffect(() => () => { alive.current = false; }, []); + + // Rule 1: the server filters every item by q before paging, so the search never runs over a loaded page. + useEffect(() => { + const timer = window.setTimeout(() => setQ(search.trim()), SEARCH_DELAY_MS); + return () => window.clearTimeout(timer); + }, [search]); + + useEffect(() => { + const controller = new AbortController(); + setSummary({ phase: "loading" }); + void fetch(url("/summary"), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "The aged stock summary could not be loaded.")) + .then((body) => { if (!controller.signal.aborted) setSummary({ phase: "ready", value: parseSummary(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) { + setSummary({ phase: "error", message: failure(reason, "The aged stock summary could not be loaded.") }); + } + }); + return () => controller.abort(); + }, [url, attempt]); + + const listQuery = useCallback((cursor?: string) => { + const params: Record = { limit: String(PAGE) }; + if (ACTIONS.includes(tab)) params.action = tab; + if (q) params.q = q; + if (cursor) params.cursor = cursor; + return url("/items", params); + }, [url, tab, q]); + + const itemsTab = tab !== NEXT_TIER; + useEffect(() => { + if (!itemsTab) return; + const controller = new AbortController(); + setPage({ phase: "loading" }); + setMoreError(""); + void fetch(listQuery(), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "The SKU list could not be loaded.")) + .then((body) => { if (!controller.signal.aborted) setPage({ phase: "ready", value: parsePage(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setPage({ phase: "error", message: failure(reason, "The SKU list could not be loaded.") }); + }); + return () => controller.abort(); + }, [itemsTab, listQuery, attempt]); + + const screen = summary.phase === "ready" ? summary.value : null; + const editable = canEdit === true && screen?.canEdit === true; + + const loadMore = async () => { + if (page.phase !== "ready" || !page.value.cursor) return; + setLoadingMore(true); + setMoreError(""); + try { + const response = await fetch(listQuery(page.value.cursor), { credentials: CREDENTIALS }); + const next = parsePage(await readJson(response, "More SKUs could not be loaded.")); + if (!alive.current) return; + setPage((current) => current.phase !== "ready" ? current : { phase: "ready", value: { + ...current.value, items: [...current.value.items, ...next.items], cursor: next.cursor, + } }); + } catch (reason) { + if (alive.current) setMoreError(failure(reason, "More SKUs could not be loaded.")); + } finally { + if (alive.current) setLoadingMore(false); + } + }; + + const refresh = async () => { + setRefreshing(true); + setRefreshError(""); + try { + const response = await fetch(url("/refresh"), { method: "POST", credentials: CREDENTIALS }); + await readJson(response, "The aged stock view could not be refreshed."); + if (alive.current) setAttempt((count) => count + 1); + } catch (reason) { + if (alive.current) setRefreshError(failure(reason, "The aged stock view could not be refreshed.")); + } finally { + if (alive.current) setRefreshing(false); + } + }; + + const openRow = (row: Row) => { + opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; + setOpenedId(row.id); + }; + const closeRow = useCallback(() => { + setOpenedId(null); + opener.current?.focus(); + }, []); + /** A stored change lands on the row in both lists, from the server's answer. */ + const saved = useCallback((stored: Row) => { + const patch = (row: Row) => (row.id === stored.id + ? { ...row, action: stored.action, owner: stored.owner, status: stored.status, due: stored.due } : row); + setSummary((current) => current.phase !== "ready" ? current + : { phase: "ready", value: { ...current.value, nextTier: current.value.nextTier.map(patch) } }); + setPage((current) => current.phase !== "ready" ? current + : { phase: "ready", value: { ...current.value, items: current.value.items.map(patch) } }); + }, []); + + const pickTab = (next: string) => { + setTab(next); + setOpenedId(null); + }; + + const needle = q.toLowerCase(); + // The Next tier list is the server's whole list, never a page, so filtering it here filters every row. + const nextRows = (screen?.nextTier ?? []).filter((row) => !needle + || [row.sku, row.fnsku, row.title].some((value) => value.toLowerCase().includes(needle))); + const rows: Row[] = itemsTab ? (page.phase === "ready" ? page.value.items : []) : nextRows; + const tabCount = (name: string) => { + if (!screen) return null; + if (name === NEXT_TIER) return screen.nextTier.length; + if (name === ALL) return Object.values(screen.statusCounts).reduce((sum, n) => sum + n, 0); + return screen.actionCounts[name] ?? 0; + }; + const openedRow = openedId ? rows.find((row) => row.id === openedId) ?? null : null; + const owners = page.phase === "ready" ? page.value.owners : []; + const listLoading = itemsTab && page.phase === "loading"; + + return
+
+ {summary.phase === "loading" && <> + + {openedRow && } +
; +} diff --git a/web/src/apps/amazon-operator/AppView.tsx b/web/src/apps/amazon-operator/AppView.tsx new file mode 100644 index 0000000000000000000000000000000000000000..17b911bd61e483642895e2fb4670f81fbe48fded --- /dev/null +++ b/web/src/apps/amazon-operator/AppView.tsx @@ -0,0 +1,644 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { useEscapeClose } from "../../ui/useEscapeClose"; +import "../../styles/sdlc79/amazon-operator.css"; + +// S79-T17 (C2, C4, R18): the Amazon operator App screen. The dispatcher draws the App frame, so this view +// draws only the Today batch, History and the item drawer. No chat panel exists here (R18). +// S79-T18 (C7): Today also lists the items other Apps posted. A decision on one goes back to its App. + +export interface OperatorApplication { + id: string; + name?: string; + capabilities?: Record; +} + +interface Proposal { + id: string; + status: string; + adType: string; + campaign: string; + sku: string; + asin: string; + kind: string; + /** The posting App's name and its action, on an App inbox item only. */ + app: string; + action: string; + change: string; + reason: string; + gate: string; + day: string; + dataDate: string; + spend: number | null; + decidedBy: string; + decidedAt: string; + note: string; + cause: string; + /** A skipped or unknown row's cause from the last approve, shown muted. */ + notice?: string; +} + +interface Held { adType: string; campaign: string; sku: string; asin: string; reason: string } + +interface TodayScreen { dataDate: string; items: Proposal[]; appItems: Proposal[]; held: Held[]; canEdit: boolean } + +interface HistoryPage { items: Proposal[]; cursor: string | null; total: number | null } + +type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; + +const AD_TAGS: Record = { sp: "SP", sb: "SB", sd: "SD" }; +/** The `kind` of an App inbox item. */ +const APP_KIND = "app"; +const HISTORY_PAGE = 50; +const HISTORY_FILTERS = [ + ["", "All"], ["proposed", "Open"], ["sent", "Sent"], ["approved", "Approved"], ["refused", "Refused"], + ["dismissed", "Dismissed"], +] as const; + +const text = (value: unknown) => (typeof value === "string" ? value : ""); +const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); + +async function readJson(response: Response, fallback: string): Promise { + const body: unknown = await response.json().catch(() => null); + if (response.ok) return body; + const error = (body as { detail?: { error?: { message?: unknown } } } | null)?.detail?.error; + throw new Error(text(error?.message) || fallback); +} + +function parseProposal(raw: unknown): Proposal | null { + const row = (raw ?? {}) as Record; + const id = text(row.id); + if (!id) return null; + return { + id, status: text(row.status) || "proposed", adType: text(row.ad_type), campaign: text(row.campaign) + || text(row.campaign_id), sku: text(row.sku), asin: text(row.asin), kind: text(row.kind), + app: text(row.app), action: text(row.action), change: text(row.change), reason: text(row.reason), gate: text(row.gate), day: text(row.day), + dataDate: text(row.data_date), spend: num(row.spend), decidedBy: text(row.decided_by), + decidedAt: text(row.decided_at), note: text(row.note), cause: text(row.cause), + }; +} + +const proposals = (raw: unknown) => (Array.isArray(raw) ? raw : []) + .map(parseProposal).filter((row): row is Proposal => row !== null); + +function parseToday(raw: unknown): TodayScreen { + const body = (raw ?? {}) as Record; + return { + dataDate: text(body.dataDate), + items: proposals(body.items), + appItems: proposals(body.appItems), + held: (Array.isArray(body.held) ? body.held : []).map((item) => { + const row = (item ?? {}) as Record; + return { adType: text(row.ad_type), campaign: text(row.campaign) || text(row.campaign_id), sku: text(row.sku), + asin: text(row.asin), reason: text(row.reason) }; + }).filter((row) => row.reason), + canEdit: body.canEdit === true, + }; +} + +function parseHistory(raw: unknown): HistoryPage { + const body = (raw ?? {}) as Record; + return { items: proposals(body.items), cursor: text(body.cursor) || null, total: num(body.total) }; +} + +/** "October 21" from an ISO day. The day is a calendar date, so it is read in UTC. */ +function dayWords(iso: string, month: "long" | "short" = "long") { + const date = new Date(iso.length === 10 ? iso + "T00:00:00Z" : iso); + return Number.isNaN(date.getTime()) ? iso + : date.toLocaleDateString(undefined, { month, day: "numeric", timeZone: "UTC" }); +} + +const money = (value: number | null) => (value === null ? "Not held" + : value.toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })); + +const isOpen = (row: Proposal) => row.status === "proposed"; + +/** The plain decision word for a proposal's status. */ +export function decisionLabel(status: string): string { + if (status === "proposed") return "Open"; + if (status === "dismissed") return "Dismissed"; + if (status === "stale") return "No longer open"; + return "Approved"; +} + +/** The outcome of a decided proposal: its words and whether it is a refusal. */ +export function outcomeOf(row: Pick): + { label: string; refused: boolean } | null { + if (row.status === "sent") return { label: "Sent", refused: false }; + if (row.status === "refused") return { label: row.cause || "Refused", refused: true }; + if (row.status === "approved") return { label: row.kind === APP_KIND ? "Approved" : "Sending", refused: false }; + if (row.status === "dismissed") return { label: "Dismissed", refused: false }; + if (row.status === "stale") return { label: row.cause || "No longer open in its App", refused: false }; + return null; +} + +function AdTag({ adType }: { adType: string }) { + const tag = AD_TAGS[adType] ?? adType.toUpperCase(); + return tag ? {tag} : null; +} + +function Outcome({ row }: { row: Proposal }) { + const outcome = outcomeOf(row); + return <> + {outcome && + {outcome.label}} + {row.status === "dismissed" && row.note && {row.note}} + {row.status === "proposed" && row.cause && {row.cause}} + {row.notice && {row.notice}} + ; +} + +function Skeleton() { + return
+
+
; +} + +function Failed({ message, onRetry }: { message: string; onRetry: () => void }) { + return

{message}{" "} +

; +} + +function Drawer({ row, onClose }: { row: Proposal; onClose: () => void }) { + const panel = useRef(null); + const close = useRef(null); + useEscapeClose(true, onClose, { container: () => panel.current }); + useEffect(() => { close.current?.focus(); }, [row.id]); + const outcome = outcomeOf(row); + const facts: [string, string][] = [ + ["App", row.app], ["Campaign", row.campaign], ["Ad type", AD_TAGS[row.adType] ?? row.adType], ["SKU", row.sku], + ["ASIN", row.asin], ["Change", row.change], ["Reason", row.reason], ["Gate", row.gate], + ["Data date", row.dataDate ? dayWords(row.dataDate) : ""], ["Decision", decisionLabel(row.status)], + ["Decided by", row.decidedBy], ["Note", row.note], + ]; + return ; +} + +interface TodayProps { + state: Load; + editable: boolean; + excluded: ReadonlySet; + settled: ReadonlySet; + sending: boolean; + decideError: string; + dismissing: Dismissing | null; + onToggle: (id: string) => void; + onToggleAll: (open: Proposal[], include: boolean) => void; + onApprove: (batch: Proposal[]) => void; + onDismissStart: (id: string | null) => void; + onDismissReason: (reason: string) => void; + onDismissCancel: () => void; + onDismiss: (ids: string[], reason: string) => void; + onOpen: (row: Proposal) => void; + onRetry: () => void; +} + +/** An open reason field: one row's (id) or the ticked batch's (id null). */ +interface Dismissing { id: string | null; reason: string } + +function ReasonForm({ label, submitLabel, ready, value, busy, inline, onChange, onCancel, onSubmit }: { + label: string; submitLabel: string; ready: boolean; value: string; busy: boolean; inline: boolean; + onChange: (value: string) => void; onCancel: () => void; onSubmit: () => void; +}) { + return
{ event.preventDefault(); if (value.trim() && ready && !busy) onSubmit(); }}> + onChange(event.target.value)} autoFocus + placeholder="Reason" aria-label={label} disabled={busy} /> + + + + +
; +} + +type RowProps = Pick; + +/** The select all tick for one group's open rows. */ +function TickAll({ open, excluded, sending, label, onToggleAll }: { + open: Proposal[]; excluded: ReadonlySet; sending: boolean; label: string; + onToggleAll: (open: Proposal[], include: boolean) => void; +}) { + const ticked = open.filter((row) => !excluded.has(row.id)).length; + const allIn = open.length > 0 && ticked === open.length; + return ; +} + +/** One row's result in place, then its own Dismiss while it is open. */ +function ResultCell({ row, live, props }: { row: Proposal; live: boolean; props: RowProps }) { + const { editable, sending, dismissing, onDismissStart, onDismissReason, onDismissCancel, onDismiss } = props; + const what = row.change || row.action; + return + + {editable && live && (dismissing?.id === row.id + ? onDismiss([row.id], dismissing.reason)} /> + : )} + ; +} + +function rowClass(live: boolean, leaving: boolean) { + return leaving ? "ao-row is-settled" : live ? "ao-row" : "ao-row is-decided"; +} + +function AdTable({ items, props }: { items: Proposal[]; props: RowProps }) { + const { editable, excluded, settled, sending, onToggle, onToggleAll, onOpen } = props; + const open = items.filter(isOpen); + return + + {editable && } + + + + + + {editable && } + + + + + + {items.map((row) => { + const live = isOpen(row); + const leaving = settled.has(row.id); + return + {editable && } + + + + + + + ; + })} + +
+ + CampaignSKUChangeReasonGateResult
+ {(live || leaving) && onToggle(row.id)} + aria-label={`Include ${row.change} for ${row.sku || row.campaign}`} />} + + + + {row.sku}{row.change}{row.reason}{row.gate}
; +} + +/** Items the other Apps posted: App, SKU, action and reason. A stale row shows why and takes no tick. */ +function AppTable({ items, props }: { items: Proposal[]; props: RowProps }) { + const { editable, excluded, settled, sending, onToggle, onToggleAll, onOpen } = props; + const open = items.filter(isOpen); + return + + {editable && } + + + + + + {editable && } + + + + + + {items.map((row) => { + const live = isOpen(row); + const leaving = settled.has(row.id); + return + {editable && } + + + + + + ; + })} + +
+ + AppSKUActionReasonResult
+ {(live || leaving) && onToggle(row.id)} + aria-label={`Include ${row.action} for ${row.sku || row.app}`} />} + + + {row.sku}{row.action}{row.reason}
; +} + +function GroupHeading({ title, count }: { title: string; count: number }) { + return

{title} {count}

; +} + +function TodayPanel(props: TodayProps) { + const { state, editable, excluded, settled, sending, decideError, dismissing, onApprove, onDismissStart, + onDismissReason, onDismissCancel, onDismiss, onRetry } = props; + if (state.phase === "loading") return ; + if (state.phase === "error") return ; + const { items, appItems, held, dataDate } = state.value; + const adOpen = items.filter(isOpen); + const appOpen = appItems.filter(isOpen); + const open = [...adOpen, ...appOpen]; + const batch = open.filter((row) => !excluded.has(row.id)); + const empty = open.length === 0 && settled.size === 0 && !appItems.some((row) => row.status === "stale"); + return
+
+ {dataDate && Ads data for {dayWords(dataDate)}} + {editable && open.length > 0 &&
+ {dismissing && dismissing.id === null + ? 0} value={dismissing.reason} + busy={sending} inline onChange={onDismissReason} onCancel={onDismissCancel} + onSubmit={() => onDismiss(batch.map((row) => row.id), dismissing.reason)} /> + : } + +
} +
+ {decideError &&

{decideError}

} + {empty &&

Nothing to change today

} + {items.length > 0 &&
+ + +
} + {appItems.length > 0 &&
+ + +
} + {held.length > 0 &&
+

Held

+
    + {held.map((row, index) =>
  • + {row.sku || row.asin} + {row.campaign} + {row.reason} +
  • )} +
+
} +
; +} + +interface HistoryProps { + state: Load; + filter: string; + loadingMore: boolean; + moreError: string; + onFilter: (status: string) => void; + onMore: () => void; + onOpen: (row: Proposal) => void; + onRetry: () => void; +} + +function HistoryPanel({ state, filter, loadingMore, moreError, onFilter, onMore, onOpen, onRetry }: HistoryProps) { + return
+
+ +
+ {state.phase === "loading" && } + {state.phase === "error" && } + {state.phase === "ready" && state.value.items.length === 0 &&

No proposals yet

} + {state.phase === "ready" && state.value.items.length > 0 && + + + + + + + + + + {state.value.items.map((row) => + + + + + + + )} + +
DaySourceSKUChangeDecisionOutcome
{dayWords(row.day, "short")} + + + {row.sku}{row.change} + + {decisionLabel(row.status)} +
} + {moreError &&

{moreError}

} + {state.phase === "ready" && state.value.cursor &&
+ +
} +
; +} + +export default function AmazonOperatorView({ application, canEdit }: { + application: OperatorApplication; canEdit?: boolean; +}) { + const appId = application.id; + const url = useCallback((path: string, params: Record = {}) => { + const query = new URLSearchParams({ app: appId, ...params }); + return `${API_V1}/app-kinds/amazon_operator${path}?${query.toString()}`; + }, [appId]); + const [tab, setTab] = useState<"today" | "history">("today"); + const [today, setToday] = useState>({ phase: "loading" }); + const [todayAttempt, setTodayAttempt] = useState(0); + const [history, setHistory] = useState>({ phase: "loading" }); + const [historyAttempt, setHistoryAttempt] = useState(0); + const [filter, setFilter] = useState(""); + const [loadingMore, setLoadingMore] = useState(false); + const [moreError, setMoreError] = useState(""); + const [excluded, setExcluded] = useState>(() => new Set()); + const [settled, setSettled] = useState>(() => new Set()); + const [sending, setSending] = useState(false); + const [decideError, setDecideError] = useState(""); + const [dismissing, setDismissing] = useState(null); + const [opened, setOpened] = useState<{ row: Proposal; from: "today" | "history" } | null>(null); + const opener = useRef(null); + const alive = useRef(true); + useEffect(() => () => { alive.current = false; }, []); + + useEffect(() => { + const controller = new AbortController(); + setToday({ phase: "loading" }); + setSettled(new Set()); + void fetch(url("/today"), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "Today's proposals could not be loaded.")) + .then((body) => { if (!controller.signal.aborted) setToday({ phase: "ready", value: parseToday(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setToday({ phase: "error", + message: reason instanceof Error ? reason.message : "Today's proposals could not be loaded." }); + }); + return () => controller.abort(); + }, [url, todayAttempt]); + + useEffect(() => { + const controller = new AbortController(); + setHistory({ phase: "loading" }); + setMoreError(""); + const params: Record = { limit: String(HISTORY_PAGE) }; + if (filter) params.status = filter; + void fetch(url("/history", params), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "History could not be loaded.")) + .then((body) => { if (!controller.signal.aborted) setHistory({ phase: "ready", value: parseHistory(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setHistory({ phase: "error", + message: reason instanceof Error ? reason.message : "History could not be loaded." }); + }); + return () => controller.abort(); + }, [url, filter, historyAttempt]); + + const screen = today.phase === "ready" ? today.value : null; + const editable = canEdit === true && screen?.canEdit === true; + const openCount = useMemo(() => (screen + ? screen.items.filter(isOpen).length + screen.appItems.filter(isOpen).length : null), [screen]); + const historyCount = history.phase === "ready" + ? history.value.total ?? history.value.items.length : null; + + const toggle = (id: string) => setExcluded((current) => { + const next = new Set(current); + if (next.has(id)) next.delete(id); else next.add(id); + return next; + }); + const toggleAll = (open: Proposal[], include: boolean) => setExcluded((current) => { + const next = new Set(current); + for (const row of open) { if (include) next.delete(row.id); else next.add(row.id); } + return next; + }); + + /** Approve or dismiss ids as one decision. Each row then shows its own result in place. */ + const decide = async (decision: "approve" | "dismiss", ids: string[], note = "") => { + if (!ids.length) return; + const fallback = decision === "approve" ? "The batch was not sent." : "The dismissal was not saved."; + setSending(true); + setDecideError(""); + try { + const response = await fetch(url(`/${decision}`), { + method: "POST", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify(decision === "approve" ? { item_ids: ids } : { item_ids: ids, reason: note.trim() }), + }); + const body = await readJson(response, fallback) as { rows?: unknown } | null; + const rows = new Map>(); + for (const item of Array.isArray(body?.rows) ? body.rows : []) { + const row = (item ?? {}) as Record; + if (text(row.id)) rows.set(text(row.id), row); + } + if (!alive.current) return; + // Each row keeps its place and shows its own result, so nothing in the table moves. + const settle = (item: Proposal): Proposal => { + const row = rows.get(item.id); + if (!row) return item; + const status = text(row.status); + if (status === "sent" || status === "refused" || status === "approved") { + return { ...item, status, cause: text(row.cause) }; + } + if (status === "dismissed") return { ...item, status, note: note.trim() }; + return { ...item, status: text(row.current) || item.status, notice: text(row.cause) || undefined }; + }; + setToday((current) => current.phase !== "ready" ? current : { phase: "ready", value: { + ...current.value, items: current.value.items.map(settle), appItems: current.value.appItems.map(settle), + } }); + setSettled((current) => new Set([...current, ...ids])); + setDismissing(null); + setHistoryAttempt((count) => count + 1); + } catch (reason) { + if (alive.current) setDecideError(reason instanceof Error ? reason.message : fallback); + } finally { + if (alive.current) setSending(false); + } + }; + + const loadMore = async () => { + if (history.phase !== "ready" || !history.value.cursor) return; + setLoadingMore(true); + setMoreError(""); + const params: Record = { limit: String(HISTORY_PAGE), cursor: history.value.cursor }; + if (filter) params.status = filter; + try { + const response = await fetch(url("/history", params), { credentials: CREDENTIALS }); + const page = parseHistory(await readJson(response, "More history could not be loaded.")); + if (!alive.current) return; + setHistory((current) => current.phase !== "ready" ? current : { phase: "ready", value: { + items: [...current.value.items, ...page.items], cursor: page.cursor, total: current.value.total, + } }); + } catch (reason) { + if (alive.current) setMoreError(reason instanceof Error ? reason.message : "More history could not be loaded."); + } finally { + if (alive.current) setLoadingMore(false); + } + }; + + const openRow = (row: Proposal, from: "today" | "history") => { + opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; + setOpened({ row, from }); + }; + const closeRow = useCallback(() => { + setOpened(null); + opener.current?.focus(); + }, []); + + // A Today drawer reads the live row, so a result that lands while it is open shows there too. + const openedRow = !opened ? null : opened.from === "today" + ? [...(screen?.items ?? []), ...(screen?.appItems ?? [])].find((row) => row.id === opened.row.id) ?? opened.row + : opened.row; + const tabs: [typeof tab, string, number | null][] = [["today", "Today", openCount], ["history", "History", historyCount]]; + return
+
+
+ {tabs.map(([key, label, count]) => )} +
+
+ {tab === "today" + ? void decide("approve", batch.map((row) => row.id))} + onDismissStart={(id) => setDismissing({ id, reason: "" })} + onDismissReason={(reason) => setDismissing((current) => current && { ...current, reason })} + onDismissCancel={() => setDismissing(null)} + onDismiss={(ids, reason) => void decide("dismiss", ids, reason)} + onOpen={(row) => openRow(row, "today")} + onRetry={() => setTodayAttempt((count) => count + 1)} /> + : void loadMore()} onOpen={(row) => openRow(row, "history")} + onRetry={() => setHistoryAttempt((count) => count + 1)} />} +
+
+ {openedRow && } +
; +} diff --git a/web/src/apps/catalog/useCatalogData.ts b/web/src/apps/catalog/useCatalogData.ts index c7b5d9b6a4721f79b2ba587c7c1c04c6fe400877..627d1eb79a2148de70fa3221a91b4a7432121f37 100644 --- a/web/src/apps/catalog/useCatalogData.ts +++ b/web/src/apps/catalog/useCatalogData.ts @@ -1,7 +1,8 @@ import { useCallback, useEffect, useState } from "react"; import { typedOpenApi, typedQuery } from "../../typed/client"; import type { QueryRequest, QueryResponse, TypedOpenApi } from "../../typed/client"; -import type { Field, Row, ViewConfig } from "../../customer-grid/types"; +import { FILTER_OPS, isFilterGroup } from "../../customer-grid/types"; +import type { Field, FilterNode, FilterOp, Row, ViewConfig } from "../../customer-grid/types"; export type CatalogReadScope = Pick; type Properties = Record; @@ -37,7 +38,8 @@ function projectRow(record: QueryResponse["records"][number], properties: Proper const value = record[key]; const types = typeof property.type === "string" ? [property.type] : property.type ?? []; const kind = value === null ? "null" : Array.isArray(value) ? "array" : typeof value; - if (!types.includes(kind) && !(kind === "number" && types.includes("integer") && Number.isInteger(value))) { + if (!types.includes(kind) && !["string", "number", "null"].includes(kind) && + !(kind === "number" && types.includes("integer") && Number.isInteger(value))) { throw new Error("A Catalog value does not match the typed document."); } if (Array.isArray(value) && types.includes("array") && value.every(item => typeof item === "string")) @@ -83,13 +85,29 @@ async function walkPages(database: string, body: QueryRequest, properties: Prope return { fields: fields ?? [], rows, returned }; } +function typedFilters(scope: CatalogReadScope): QueryRequest["filters"] { + if (!scope.filters.length) return null; + function node(value: FilterNode, depth: number): unknown { + if (isFilterGroup(value)) { + if (depth >= 3) throw new Error("This Catalog filter is nested too deeply. Use at most three levels."); + return { conj: value.conj, children: value.children.map((child) => node(child, depth + 1)) }; + } + if (!FILTER_OPS.includes(value.op as FilterOp) || (value.rhs && value.rhs.kind !== "field")) { + throw new Error("This Catalog filter cannot be used. Choose a column condition."); + } + const { colId, op, id, rhs, value: operand, value2, values } = value; + return { colId, op, id, rhs, value: operand, value2, values }; + } + return { conj: scope.filterConj ?? "and", nodes: scope.filters.map((value) => node(value, 1)) } as QueryRequest["filters"]; +} + export async function loadCatalogData(database: string, scope: CatalogReadScope, signal: AbortSignal): Promise { const properties = await documentProperties(await typedOpenApi(signal), database); signal.throwIfAborted(); // The limit sizes each page. The cursor walk has no total row limit. const body: QueryRequest = { limit: 100, - filters: scope.filters.length ? { conj: scope.filterConj ?? "and", nodes: scope.filters } : null, + filters: typedFilters(scope), sorts: scope.sorts.map((sort) => ({ field: sort.colId, direction: sort.dir })), }; const filtered = await walkPages(database, body, properties, signal); diff --git a/web/src/apps/competitor-watch/AppView.tsx b/web/src/apps/competitor-watch/AppView.tsx new file mode 100644 index 0000000000000000000000000000000000000000..c381c86ef2336f011a2758aa7a0da7d907bcbe74 --- /dev/null +++ b/web/src/apps/competitor-watch/AppView.tsx @@ -0,0 +1,9 @@ +import type { AppViewProps } from "../../customer-grid/viewModes"; +import CompetitorWatchView from "./CompetitorWatchView"; + +/** CompetitorWatchView draws its own AppFrame with its tabs, so the dispatcher draws no second header. */ + +/** C2: the Competitor watch screen, mounted by viewModes.tsx through the App glob. */ +export default function AppView({ application }: AppViewProps) { + return ; +} diff --git a/web/src/apps/competitor-watch/CompetitorWatchView.tsx b/web/src/apps/competitor-watch/CompetitorWatchView.tsx index f0ea9202bee972836a509ee952a8161ae94ce43c..314cbdfb2c758d465f24534a0feda4e7303cfc97 100644 --- a/web/src/apps/competitor-watch/CompetitorWatchView.tsx +++ b/web/src/apps/competitor-watch/CompetitorWatchView.tsx @@ -2,7 +2,6 @@ import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState } fro import type { FormEvent } from "react"; import type { TopLevelSpec } from "vega-lite"; import { API_V1, CREDENTIALS } from "../../apiContract"; -import { AppFrame } from "../../customer-grid/AppFrame"; import "../../styles/sdlc78/competitor-watch.css"; // S78-T17 (C4, R20): the Competitor watch App. Reads and writes go through the competitor watch @@ -471,15 +470,14 @@ export default function CompetitorWatchView({ application }: { application: Comp const canEditApp = application.capabilities?.edit === true; const tabs = [["terms", "Terms"], ["alerts", "Alerts"]] as const; return
- +
{tabs.map(([key, label]) => )} -
}> +
{tab === "terms" ? : }
-
; } diff --git a/web/src/apps/opportunity-finder/AppView.tsx b/web/src/apps/opportunity-finder/AppView.tsx new file mode 100644 index 0000000000000000000000000000000000000000..62af117f374cadd66a5328be408cbbcf3bca80f0 --- /dev/null +++ b/web/src/apps/opportunity-finder/AppView.tsx @@ -0,0 +1,564 @@ +import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { TopLevelSpec } from "vega-lite"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { useEscapeClose } from "../../ui/useEscapeClose"; +import "../../styles/sdlc79/opportunity-finder.css"; + +// S79-T35 (C2, C4, R18): the Opportunity finder App screen. The dispatcher draws the App frame, so this +// view draws only the week, the status tabs, the rising list and the term drawer. No chat panel. +const VegaLiteChart = lazy(() => import("../../viz/vega/VegaLiteChart")); + +export interface OpportunityFinderApplication { + id: string; + name?: string; + capabilities?: Record; +} + +const DEFAULT_STATUSES = ["Rising", "Opportunity", "Accepted", "Served", "Dismissed"]; +const ALL = "All terms"; +const EMPTY_WEEK = "No rising terms this week"; + +/** The one empty sentence per tab. */ +const EMPTY: Record = { + [ALL]: EMPTY_WEEK, Rising: EMPTY_WEEK, Opportunity: "No opportunities yet", Accepted: "No accepted terms", + Served: "No served terms", Dismissed: "No dismissed terms", +}; + +interface TopAsin { asin: string; rank: number | null; clickShare: number | null; conversionShare: number | null; isOurs: boolean } +interface Point { week: string; rank: number } + +interface Term { + id: string; + term: string; + priorRank: number | null; + rank: number | null; + risePct: number | null; + ours: boolean; + top3: TopAsin[]; + rising: boolean; + reason: string; + status: string; + owner: string; + due: string; + spark: Point[]; +} + +interface Screen { + canEdit: boolean; + week: string; + priorWeek: string; + counts: Record; + total: number; + items: Term[]; + cursor: string | null; + problems: string[]; + statuses: string[]; +} + +interface HistoryEvent { at: string; by: string; status: string; owner: string; due: string; note: string } +interface Detail { item: Term; history: HistoryEvent[]; trend: Point[]; top3: TopAsin[]; week: string } + +type Row = Record; +type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; + +const text = (value: unknown) => (typeof value === "string" ? value : typeof value === "number" ? String(value) : ""); +const num = (value: unknown) => { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim() !== "" && Number.isFinite(Number(value))) return Number(value); + return null; +}; +const rows = (raw: unknown): Row[] => (Array.isArray(raw) ? raw : []) + .filter((row): row is Row => !!row && typeof row === "object" && !Array.isArray(row)); +const strings = (raw: unknown) => (Array.isArray(raw) ? raw : []).map(text).filter(Boolean); + +async function readJson(response: Response, fallback: string): Promise { + const body: unknown = await response.json().catch(() => null); + if (response.ok) return body; + // main.py reshapes every refusal to a top-level `error`; a bare router answers under `detail`. + const shaped = body as { error?: { message?: unknown }; detail?: { error?: { message?: unknown } } } | null; + throw new Error(text(shaped?.error?.message) || text(shaped?.detail?.error?.message) || fallback); +} + +const failure = (reason: unknown, fallback: string) => (reason instanceof Error && reason.message ? reason.message : fallback); + +function parseTop(raw: unknown): TopAsin[] { + return rows(raw).map((row) => ({ + asin: text(row.asin), rank: num(row.rank), clickShare: num(row.clickShare), + conversionShare: num(row.conversionShare), isOurs: row.isOurs === true, + })).filter((row) => row.asin); +} + +function parsePoints(raw: unknown): Point[] { + return rows(raw).map((row) => ({ week: text(row.week), rank: num(row.rank) })) + .filter((point): point is Point => !!point.week && point.rank !== null) + .sort((a, b) => a.week.localeCompare(b.week)); +} + +function parseTerm(raw: unknown): Term | null { + const row = (raw ?? {}) as Row; + const id = text(row.id); + if (!id) return null; + return { + id, term: text(row.term) || text(row.title), priorRank: num(row.priorRank), rank: num(row.rank), + risePct: num(row.risePct), ours: row.ours === true, top3: parseTop(row.top3), rising: row.rising !== false, + reason: text(row.reason), status: text(row.status) || "Rising", owner: text(row.owner), due: text(row.due), + spark: parsePoints(row.spark), + }; +} + +function parseScreen(raw: unknown): Screen { + const body = (raw ?? {}) as Row; + const counts: Record = {}; + for (const [key, value] of Object.entries((body.counts ?? {}) as Row)) counts[key] = num(value) ?? 0; + const statuses = strings(body.statuses); + const items = (Array.isArray(body.items) ? body.items : []).map(parseTerm).filter((row): row is Term => row !== null); + return { + canEdit: body.canEdit === true, week: text(body.week), priorWeek: text(body.priorWeek), counts, + total: num(body.total) ?? items.length, items, cursor: text(body.cursor) || null, problems: strings(body.problems), + statuses: statuses.length ? statuses : DEFAULT_STATUSES, + }; +} + +function parseDetail(raw: unknown): Detail | null { + const body = (raw ?? {}) as Row; + const item = parseTerm(body.item); + if (!item) return null; + return { + item, trend: parsePoints(body.trend), top3: parseTop(body.top3), week: text(body.week), + history: rows(body.history).map((event) => ({ + at: text(event.at), by: text(event.by), status: text(event.status), owner: text(event.owner), + due: text(event.due), note: text(event.note), + })), + }; +} + +/** "Oct 18, 2026" from an ISO day or stamp. A calendar day is read in UTC. */ +function dayWords(raw: string) { + const value = raw.trim(); + const date = new Date(value.length === 10 ? value + "T00:00:00Z" : value); + return Number.isNaN(date.getTime()) ? value + : date.toLocaleDateString(undefined, { month: "short", day: "numeric", year: "numeric", timeZone: "UTC" }); +} + +const plural = (n: number, one: string, many: string) => `${n.toLocaleString()} ${n === 1 ? one : many}`; +const whole = (value: number) => Math.round(value).toLocaleString(); +const share = (value: number | null) => (value === null ? "Not reported" + : `${(value * 100).toLocaleString(undefined, { maximumFractionDigits: 1 })}%`); + +/** The rise as words, toned by sign. A lower rank number is a higher search rank. */ +function change(term: Term): { words: string; tone: "up" | "down" | "flat" } | null { + if (term.risePct === null) return null; + const pct = Math.round(term.risePct); + if (pct === 0) return { words: "No change", tone: "flat" }; + return pct > 0 ? { words: `Up ${pct}%`, tone: "up" } : { words: `Down ${-pct}%`, tone: "down" }; +} + +function Ours() { + return Ours; +} + +function Skeleton({ chart }: { chart?: boolean }) { + return
+ {chart ?
: <> +
+ } +
; +} + +function Failed({ message, onRetry }: { message: string; onRetry: () => void }) { + return

{message}{" "} +

; +} + +const SPARK_W = 80; +const SPARK_H = 24; + +/** A row's weekly rank as a small line. A better rank sits higher. */ +function Spark({ points, term }: { points: Point[]; term: string }) { + if (points.length === 0) return No history; + const ranks = points.map((point) => point.rank); + const low = Math.min(...ranks); + const span = Math.max(...ranks) - low || 1; + const step = points.length > 1 ? (SPARK_W - 4) / (points.length - 1) : 0; + const xy = points.map((point, index) => [2 + index * step, 2 + ((point.rank - low) / span) * (SPARK_H - 4)]); + const label = `Rank by week for ${term}: ${ranks.map(whole).join(", ")}`; + return + {xy.length > 1 && `${x},${y}`).join(" ")} />} + + ; +} + +function TopList({ top3 }: { top3: TopAsin[] }) { + if (top3.length === 0) return None reported; + return
    + {top3.map((entry) =>
  1. + {entry.asin}{entry.isOurs && } +
  2. )} +
; +} + +function TermTable({ items, onOpen }: { items: Term[]; onOpen: (term: Term) => void }) { + return + + + + + + + + + + + + + + {items.map((term) => { + const delta = change(term); + return + + + + + + + + ; + })} + +
Search termTrendRankChangeTop 3 clickedOwnerStatus
+ + {term.ours && } + + {term.rank === null ? Not rising : whole(term.rank)} + {term.priorRank !== null && from {whole(term.priorRank)}} + {delta && {delta.words}}{term.owner || No owner}{term.status}
; +} + +function chartColors() { + const style = getComputedStyle(document.documentElement); + const read = (token: string) => style.getPropertyValue(token).trim(); + return { series: read("--lp-blue-solid"), mark: read("--lp-blue-deep"), line: read("--lp-line"), + surface: read("--lp-surface"), muted: read("--lp-muted") }; +} + +const RANK_TITLE = "Search frequency rank"; + +function trendSpec(points: Point[], week: string): TopLevelSpec { + const values = points.map((point) => ({ + week: point.week, rank: point.rank, weekLabel: dayWords(point.week), rankLabel: whole(point.rank), + picked: point.week === week, + })); + const tokens = chartColors(); + const x = { field: "week", type: "ordinal" as const, title: null, sort: "ascending" as const, + axis: { labelAngle: 0, labelExpr: "timeFormat(toDate(datum.value + 'T00:00:00'), '%b %-d')" } }; + // A lower rank number is a better rank, so the axis runs upside down. Bare "," prints 4000 as "4e+3". + const y = { field: "rank", type: "quantitative" as const, title: RANK_TITLE, scale: { zero: false, reverse: true }, + axis: { gridColor: tokens.line, tickCount: 4, format: ",d" } }; + const tooltip = [{ field: "weekLabel", title: "Week of" }, { field: "rankLabel", title: RANK_TITLE }]; + return { + $schema: "https://vega.github.io/schema/vega-lite/v6.json", + width: "container", + height: 200, + autosize: { type: "fit", contains: "padding" }, + background: "transparent", + config: { view: { stroke: null }, axis: { domain: false, ticks: false, labelFontSize: 11, titleFontSize: 11, + titleFontWeight: 500, labelColor: tokens.muted, titleColor: tokens.muted }, font: "Inter" }, + data: { values }, + layer: [ + { mark: { type: "line", strokeWidth: 2, color: tokens.series }, encoding: { x, y } }, + { mark: { type: "point", filled: true, size: 70, stroke: tokens.surface, strokeWidth: 1.5 }, + encoding: { x, y, tooltip, + color: { condition: { test: "datum.picked", value: tokens.mark }, value: tokens.series }, + size: { condition: { test: "datum.picked", value: 130 }, value: 70 } } }, + ], + } as TopLevelSpec; +} + +function TrendChart({ points, term, week }: { points: Point[]; term: string; week: string }) { + const spec = useMemo(() => trendSpec(points, week), [points, week]); + if (points.length < 2) return

The trend shows after two weekly syncs.

; + return }> + + ; +} + +interface DrawerProps { + opened: Term; + url: (path: string) => string; + editable: boolean; + statuses: string[]; + onClose: () => void; + onSaved: () => void; +} + +function Drawer({ opened, url, editable, statuses, onClose, onSaved }: DrawerProps) { + const panel = useRef(null); + const close = useRef(null); + const [detail, setDetail] = useState>({ phase: "loading" }); + const [attempt, setAttempt] = useState(0); + const [status, setStatus] = useState(""); + const [owner, setOwner] = useState(""); + const [saving, setSaving] = useState(false); + const [saveError, setSaveError] = useState(""); + const [saved, setSaved] = useState(false); + useEscapeClose(true, onClose, { container: () => panel.current }); + useEffect(() => { close.current?.focus(); }, [opened.id]); + + useEffect(() => { + const controller = new AbortController(); + setDetail({ phase: "loading" }); + void fetch(url(`/items/${encodeURIComponent(opened.id)}`), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "This search term could not be loaded.")) + .then((body) => { + if (controller.signal.aborted) return; + const value = parseDetail(body); + if (!value) { setDetail({ phase: "error", message: "This search term could not be loaded." }); return; } + setDetail({ phase: "ready", value }); + setStatus(value.item.status); + setOwner(value.item.owner); + }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) { + setDetail({ phase: "error", message: failure(reason, "This search term could not be loaded.") }); + } + }); + return () => controller.abort(); + }, [url, opened.id, attempt]); + + const value = detail.phase === "ready" ? detail.value : null; + const item = value?.item ?? opened; + // The stored owner, never the default: the server gives an ownerless Opportunity the default owner. + const changed = value !== null && status !== "" && (status !== item.status || owner.trim() !== item.owner); + + const save = async () => { + if (!value || !changed) return; + setSaving(true); + setSaveError(""); + setSaved(false); + try { + const body: Record = { status }; + if (owner.trim()) body.owner = owner.trim(); + const response = await fetch(url(`/items/${encodeURIComponent(item.id)}`), { + method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + await readJson(response, "The status was not saved."); + setSaved(true); + setAttempt((count) => count + 1); + onSaved(); + } catch (reason) { + setSaveError(failure(reason, "The status was not saved.")); + } finally { + setSaving(false); + } + }; + + const delta = change(item); + const top3 = value ? (value.top3.length ? value.top3 : value.item.top3) : opened.top3; + return ; +} + +export default function OpportunityFinderView({ application, canEdit }: { + application: OpportunityFinderApplication; canEdit?: boolean; +}) { + const appId = application.id; + const url = useCallback((path: string, params: Record = {}) => { + const query = new URLSearchParams({ app: appId, ...params }); + return `${API_V1}/app-kinds/opportunity_finder${path}?${query.toString()}`; + }, [appId]); + const [tab, setTab] = useState(ALL); + const [screen, setScreen] = useState>({ phase: "loading" }); + const [attempt, setAttempt] = useState(0); + const [held, setHeld] = useState<{ week: string; priorWeek: string; counts: Record | null }>( + { week: "", priorWeek: "", counts: null }); + const [statuses, setStatuses] = useState(DEFAULT_STATUSES); + const [problems, setProblems] = useState([]); + const [serverEdit, setServerEdit] = useState(false); + const [loadingMore, setLoadingMore] = useState(false); + const [moreError, setMoreError] = useState(""); + const [opened, setOpened] = useState(null); + const opener = useRef(null); + const alive = useRef(true); + useEffect(() => () => { alive.current = false; }, []); + + const params = useCallback((extra: Record = {}) => ( + tab === ALL ? { ...extra } : { ...extra, status: tab }), [tab]); + + useEffect(() => { + const controller = new AbortController(); + setScreen({ phase: "loading" }); + setMoreError(""); + void fetch(url("/screen", params()), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "The rising terms could not be loaded.")) + .then((body) => { + if (controller.signal.aborted) return; + const value = parseScreen(body); + setScreen({ phase: "ready", value }); + setHeld({ week: value.week, priorWeek: value.priorWeek, counts: value.counts }); + setStatuses(value.statuses); + setProblems(value.problems); + setServerEdit(value.canEdit); + }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) { + setScreen({ phase: "error", message: failure(reason, "The rising terms could not be loaded.") }); + } + }); + return () => controller.abort(); + }, [url, params, attempt]); + + const editable = canEdit === true && serverEdit; + + const loadMore = async () => { + if (screen.phase !== "ready" || !screen.value.cursor) return; + setLoadingMore(true); + setMoreError(""); + try { + const page = parseScreen(await readJson(await fetch(url("/screen", params({ cursor: screen.value.cursor })), + { credentials: CREDENTIALS }), "More could not be loaded.")); + if (alive.current) setScreen((now) => now.phase !== "ready" ? now : { phase: "ready", value: { + ...now.value, items: [...now.value.items, ...page.items], cursor: page.cursor } }); + } catch (reason) { + if (alive.current) setMoreError(failure(reason, "More could not be loaded.")); + } finally { + if (alive.current) setLoadingMore(false); + } + }; + + const open = (term: Term) => { + opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; + setOpened(term); + }; + const closeDrawer = useCallback(() => { + setOpened(null); + opener.current?.focus(); + }, []); + const urlFor = useCallback((path: string) => url(path), [url]); + const onSaved = useCallback(() => setAttempt((count) => count + 1), []); + + const counts = held.counts; + const allTotal = counts ? statuses.reduce((sum, status) => sum + (counts[status] ?? 0), 0) : null; + const tabs: [string, number | null][] = [[ALL, allTotal], + ...statuses.map((status): [string, number | null] => [status, counts ? counts[status] ?? 0 : null])]; + const listed = screen.phase === "ready" ? screen.value.items.length : 0; + const total = screen.phase === "ready" ? screen.value.total : 0; + const cursor = screen.phase === "ready" ? screen.value.cursor : null; + return
+
+ {held.week &&
+

Week of {dayWords(held.week)}

+ {held.priorWeek &&

Against the week of {dayWords(held.priorWeek)}

} +
} +
+ {tabs.map(([key, count]) => )} +
+ {problems.length > 0 &&
+ {problems.map((problem) =>

{problem}

)} +
} +
+ {screen.phase === "loading" && } + {screen.phase === "error" && setAttempt((count) => count + 1)} />} + {/* One empty sentence: a problem line already says why the list is empty. */} + {screen.phase === "ready" && listed === 0 && problems.length === 0 + &&

{EMPTY[tab] ?? EMPTY_WEEK}

} + {screen.phase === "ready" && listed > 0 && } + {screen.phase === "ready" && listed > 0 &&
+ {plural(total, "term", "terms")} + {cursor && } +
} + {moreError &&

{moreError}

} +
+
+ {opened && } +
; +} diff --git a/web/src/apps/payouts/AppView.tsx b/web/src/apps/payouts/AppView.tsx new file mode 100644 index 0000000000000000000000000000000000000000..0c6f40adc93206efd016df85904b55c87f921e27 --- /dev/null +++ b/web/src/apps/payouts/AppView.tsx @@ -0,0 +1,508 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { useEscapeClose } from "../../ui/useEscapeClose"; +import "../../styles/sdlc79/payouts.css"; + +// S79-T27 (C2): the Payouts App body. The dispatcher draws the App frame; this view never does. +// Every read and write names the App, so the server applies may_read and may_edit to this App record. + +export interface PayoutsApplication { + id: string; + name?: string; + capabilities?: Record; +} + +interface ListRow { + id: string; + title: string; + currency: string; + start: string | null; + end: string | null; + depositDate: string | null; + amazonTotal: number | null; + reconciled: boolean; +} + +interface NextDeposit { expected: boolean; date: string | null; amount: number | null; currency: string; overdue: boolean } +interface OpenSettlement { start: string | null; expectedClose: string | null; currency: string } +interface LastDeposit { date: string | null; amount: number | null; currency: string; change: number | null } + +interface Screen { + empty: boolean; + currencies: string[]; + counts: { all: number; unreconciled: number }; + rows: ListRow[]; + cursor: string | null; + next: NextDeposit | null; + open: OpenSettlement | null; + last: LastDeposit | null; + unreconciled: number; + total: number; +} + +interface Part { transactionType: string; amountType: string; description: string; amount: number; lines: number } +interface Bucket { key: string; label: string; amount: number | null; parts: Part[] } +interface Exclusion { key: string; label: string; monthAmount: number | null } +interface Reference { message: string } +interface Tie { + available: boolean; + final: boolean; + month: string; + monthLabel: string; + currency: string; + basis: string; + settlements: number | null; + economics: number | null; + gap: number | null; + gapPct: number | null; + matched: boolean | null; + exclusions: Exclusion[]; + reference: Reference | null; + message: string; +} +interface Item { id: string; status: string; owner: string; due: string } +interface Detail { + canEdit: boolean; + statuses: string[]; + id: string; + title: string; + currency: string; + reconciled: boolean; + gap: number | null; + reason: string; + waterfall: Bucket[]; + tie: Tie; + item: Item | null; +} + +type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; + +const text = (value: unknown) => (typeof value === "string" ? value : ""); +const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); +const rec = (value: unknown) => (value && typeof value === "object" ? value as Record : {}); +const list = (value: unknown) => (Array.isArray(value) ? value : []); + +/** The server's own cause: `error` at the top on the real server, `detail.error` on a bare router. */ +export function refusalMessage(body: unknown): string { + const top = rec(rec(body).error); + const nested = rec(rec(rec(body).detail).error); + return text(top.message) || text(nested.message); +} + +async function readJson(response: Response, fallback: string): Promise { + const body: unknown = await response.json().catch(() => null); + if (response.ok) return body; + throw new Error(refusalMessage(body) || fallback); +} + +function parseScreen(raw: unknown): Screen { + const body = rec(raw); + const kpis = rec(body.kpis); + const counts = rec(body.counts); + const next = kpis.nextDeposit ? rec(kpis.nextDeposit) : null; + const open = kpis.openSettlement ? rec(kpis.openSettlement) : null; + const last = kpis.lastDeposit ? rec(kpis.lastDeposit) : null; + return { + empty: body.empty === true, + currencies: list(body.currencies).map(text).filter(Boolean), + counts: { all: num(counts.all) ?? 0, unreconciled: num(counts.unreconciled) ?? 0 }, + rows: list(body.settlements).map((item) => { + const row = rec(item); + return { + id: text(row.id), title: text(row.title), currency: text(row.currency), + start: text(row.start) || null, end: text(row.end) || null, depositDate: text(row.depositDate) || null, + amazonTotal: num(row.amazonTotal), reconciled: row.reconciled === true, + }; + }).filter((row) => row.id), + cursor: text(body.cursor) || null, + next: next && { expected: next.expected === true, date: text(next.date) || null, amount: num(next.amount), + currency: text(next.currency), overdue: next.overdue === true }, + open: open && { start: text(open.start) || null, expectedClose: text(open.expectedClose) || null, + currency: text(open.currency) }, + last: last && { date: text(last.date) || null, amount: num(last.amount), currency: text(last.currency), + change: num(last.change) }, + unreconciled: num(kpis.unreconciled) ?? 0, + total: num(kpis.settlements) ?? 0, + }; +} + +function parseItem(raw: unknown): Item | null { + const row = rec(raw); + return text(row.id) ? { id: text(row.id), status: text(row.status), owner: text(row.owner), due: text(row.due) } : null; +} + +function parseDetail(raw: unknown): Detail { + const body = rec(raw); + const s = rec(body.settlement); + const t = rec(s.tie); + const ref = t.reference ? rec(t.reference) : null; + return { + canEdit: body.canEdit === true, + statuses: list(body.statuses).map(text).filter(Boolean), + id: text(s.id), title: text(s.title), currency: text(s.currency), reconciled: s.reconciled === true, + gap: num(s.gap), reason: text(s.reason), + waterfall: list(s.waterfall).map((item) => { + const row = rec(item); + return { + key: text(row.key), label: text(row.label), amount: num(row.amount), + parts: list(row.parts).map((part) => { + const p = rec(part); + return { transactionType: text(p.transactionType), amountType: text(p.amountType), + description: text(p.description), amount: num(p.amount) ?? 0, lines: num(p.lines) ?? 0 }; + }), + }; + }).filter((row) => row.key), + tie: { + available: t.available === true, final: t.final === true, month: text(t.month), monthLabel: text(t.monthLabel), + currency: text(t.currency), basis: text(t.basis), settlements: num(t.settlements), economics: num(t.economics), + gap: num(t.gap), gapPct: num(t.gapPct), matched: typeof t.matched === "boolean" ? t.matched : null, + exclusions: list(t.exclusions).map((item) => { + const row = rec(item); + return { key: text(row.key), label: text(row.label), monthAmount: num(row.monthAmount) }; + }), + reference: ref && text(ref.message) ? { message: text(ref.message) } : null, + message: text(t.message), + }, + item: parseItem(body.item), + }; +} + +/** Money with its currency, for example "$1,234.50" or "CA$255.00"; a bare number when the code is unknown. */ +export function money(amount: number | null, currency: string): string { + if (amount === null) return "Not stated"; + try { + return new Intl.NumberFormat("en-US", { style: "currency", currency: currency || "USD", + currencyDisplay: "symbol" }).format(amount); + } catch { + return amount.toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + } +} + +const signed = (amount: number, currency: string) => (amount > 0 ? "+" : "") + money(amount, currency); +const tone = (amount: number | null) => (amount === null || Math.abs(amount) < 0.005 ? "" : amount < 0 ? " is-neg" : " is-pos"); + +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +/** "24 Sep 2026" in UTC, the module's own date words; "24 Sep" without the year. */ +function day(iso: string | null, withYear = true): string { + if (!iso) return "Not known"; + const date = new Date(iso.length === 10 ? iso + "T00:00:00Z" : iso); + if (Number.isNaN(date.getTime())) return iso; + const short = `${date.getUTCDate()} ${MONTHS[date.getUTCMonth()]}`; + return withYear ? `${short} ${date.getUTCFullYear()}` : short; +} + +function Kpi({ label, value, delta, deltaTone, period }: { + label: string; value: string; delta?: string; deltaTone?: string; period: string; +}) { + return
+ {label} + {value} + + {delta && {delta}} + {period} + +
; +} + +function KpiStrip({ screen }: { screen: Screen }) { + const { next, open, last } = screen; + return
+ + + + +
; +} + +function Skeleton({ rows = 4 }: { rows?: number }) { + return
+ {Array.from({ length: rows }, (_, index) =>
)} +
; +} + +function Chevron({ open }: { open: boolean }) { + return ; +} + +function Waterfall({ rows, currency }: { rows: Bucket[]; currency: string }) { + const [openKey, setOpenKey] = useState(null); + return
    + {rows.map((row) => { + const deposit = row.key === "deposit"; + const expandable = row.parts.length > 0; + const expanded = openKey === row.key; + const amount = {money(row.amount, currency)}; + return
  • + {expandable + ? + :
    } + {expandable && expanded &&
      + {row.parts.map((part, index) =>
    • + {part.description || part.amountType || part.transactionType} + {[part.transactionType, part.amountType].filter(Boolean).join(", ")} + + {money(part.amount, currency)} +
    • )} +
    } +
  • ; + })} +
; +} + +function TieLine({ tie, currency }: { tie: Tie; currency: string }) { + const cur = tie.currency || currency; + if (!tie.available) { + return
+

{tie.message}

+ {tie.reference &&

{tie.reference.message}

} +
; + } + const pct = tie.gapPct === null ? "" : ` (${Math.abs(tie.gapPct).toFixed(1)} percent)`; + return
+
+
Month
{tie.monthLabel}
+
Settlements without loan repayment and withheld tax
+
{money(tie.settlements, cur)}
+
{tie.basis ? `Economics ${tie.basis}` : "Economics"}
+
{money(tie.economics, cur)}
+ {tie.final &&
Gap
+
{money(tie.gap, cur)}{pct}
} + {tie.exclusions.map((ex) =>
+
Excluded {ex.label.toLowerCase()}
{money(ex.monthAmount, cur)}
)} +
+ {tie.final + ?

+ {tie.matched ? "Matched" : "Not matched"}

+ : <> +

{tie.message}

+ {tie.reference &&

{tie.reference.message}

} + } +
; +} + +function useAppUrl(appId: string) { + return useCallback((path: string, params: Record = {}) => { + const query = new URLSearchParams({ app: appId, ...params }); + return `${API_V1}/app-kinds/payouts${path}?${query.toString()}`; + }, [appId]); +} + +function SettlementDrawer({ appId, settlementId, canEditApp, onClose }: { + appId: string; settlementId: string; canEditApp: boolean; onClose: () => void; +}) { + const url = useAppUrl(appId); + const [state, setState] = useState>({ phase: "loading" }); + const [attempt, setAttempt] = useState(0); + const [saving, setSaving] = useState(false); + const [saveError, setSaveError] = useState(""); + const panel = useRef(null); + const alive = useRef(true); + useEffect(() => () => { alive.current = false; }, []); + useEscapeClose(true, onClose, { container: () => panel.current }); + + useEffect(() => { + const controller = new AbortController(); + setState({ phase: "loading" }); + setSaveError(""); + void fetch(url(`/screen/settlements/${encodeURIComponent(settlementId)}`), + { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "The settlement could not be loaded.")) + .then((body) => { if (!controller.signal.aborted) setState({ phase: "ready", value: parseDetail(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setState({ phase: "error", + message: reason instanceof Error ? reason.message : "The settlement could not be loaded." }); + }); + return () => controller.abort(); + }, [url, settlementId, attempt]); + + useEffect(() => { panel.current?.focus(); }, [settlementId]); + + const setStatus = async (item: Item, status: string) => { + setSaving(true); + setSaveError(""); + try { + const response = await fetch(url(`/items/${encodeURIComponent(item.id)}`), { + method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ status }), + }); + const saved = parseItem(await readJson(response, "The status was not saved.")); + if (!alive.current) return; + setState((current) => current.phase !== "ready" ? current + : { phase: "ready", value: { ...current.value, item: saved ?? { ...item, status } } }); + } catch (reason) { + if (alive.current) setSaveError(reason instanceof Error ? reason.message : "The status was not saved."); + } finally { + if (alive.current) setSaving(false); + } + }; + + const detail = state.phase === "ready" ? state.value : null; + const canEdit = canEditApp && detail?.canEdit === true; + return ; +} + +export default function PayoutsAppView({ application, canEdit = false }: { + application: PayoutsApplication; canEdit?: boolean; +}) { + const appId = application.id; + const url = useAppUrl(appId); + const [view, setView] = useState<"all" | "unreconciled">("all"); + const [currency, setCurrency] = useState(""); + const [attempt, setAttempt] = useState(0); + const [state, setState] = useState>({ phase: "loading" }); + const [more, setMore] = useState<{ busy: boolean; error: string }>({ busy: false, error: "" }); + const [selected, setSelected] = useState(null); + const alive = useRef(true); + useEffect(() => () => { alive.current = false; }, []); + + useEffect(() => { + const controller = new AbortController(); + setState({ phase: "loading" }); + setMore({ busy: false, error: "" }); + void fetch(url("/screen", { view, ...(currency ? { currency } : {}) }), + { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "Payouts could not be loaded.")) + .then((body) => { if (!controller.signal.aborted) setState({ phase: "ready", value: parseScreen(body) }); }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setState({ phase: "error", + message: reason instanceof Error ? reason.message : "Payouts could not be loaded." }); + }); + return () => controller.abort(); + }, [url, view, currency, attempt]); + + const screen = state.phase === "ready" ? state.value : null; + const close = useCallback(() => setSelected(null), []); + + const showMore = async () => { + if (!screen?.cursor) return; + setMore({ busy: true, error: "" }); + try { + const response = await fetch(url("/screen", { view, cursor: screen.cursor, ...(currency ? { currency } : {}) }), + { credentials: CREDENTIALS }); + const page = parseScreen(await readJson(response, "More settlements could not be loaded.")); + if (!alive.current) return; + setState((current) => current.phase !== "ready" ? current : { phase: "ready", value: { + ...current.value, rows: [...current.value.rows, ...page.rows], cursor: page.cursor } }); + setMore({ busy: false, error: "" }); + } catch (reason) { + if (alive.current) setMore({ busy: false, + error: reason instanceof Error ? reason.message : "More settlements could not be loaded." }); + } + }; + + const tabs = [["all", "All", screen?.counts.all ?? 0], ["unreconciled", "Unreconciled", screen?.counts.unreconciled ?? 0]] as const; + return
+
+ {state.phase === "loading" && <> +