diff --git a/RELEASES.json b/RELEASES.json index 836ae5ce0cc4b64d3034f5f9844b03a1aab760ae..d4e5d6472d0afb427aca71acca71ff36ab099dcf 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,53 +1,4 @@ { - "current": "d0c6e7945", - "releases": [ - { - "version": "v84", - "sha": "7bb3a2e09", - "date": "2026-10-24", - "subject": "release(sdlc78): v84 QA records the staging build change and what v84 excludes" - }, - { - "version": "v83", - "sha": "9b7471968", - "date": "2026-10-21", - "subject": "release v83" - }, - { - "version": "v82", - "sha": "9c61b1ee9", - "date": "2026-10-17", - "subject": "release v82" - }, - { - "version": "v81", - "sha": "e141ccc85", - "date": "2026-10-15", - "subject": "release v81" - }, - { - "version": "v80", - "sha": "869b8fcc6", - "date": "2026-10-13", - "subject": "release v80" - }, - { - "version": "v79", - "sha": "406417b8d", - "date": "2026-10-13", - "subject": "release v79" - }, - { - "version": "v78", - "sha": "aad6561d6", - "date": "2026-10-09", - "subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs" - }, - { - "version": "v77", - "sha": "e741e5ca6", - "date": "2026-10-05", - "subject": "release v77" - } - ] + "current": "2990461bb", + "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index 793a877899b3a5a9112b7cfa4ed1769be3c0c49d..8b9ee6e0d59fda10fa8276894d287d4f23eb96c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -d0c6e7945 +2990461bb diff --git a/api/amazon_relational.py b/api/amazon_relational.py index 0ee4f8fbb41d93c41b14023bba40c6a866f92a1e..f9b8990489be5139c9ef46d3279d1cb7aecb78ec 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -133,6 +133,113 @@ def source_attribution(key): } +def credential_table_owner(_key, _rt): + """Combined Amazon tables have row owners and never belong to one credential.""" + return None + + +CONNECTION_REVOCATIONS_KEY = "amazon_connection_revocations" + + +def record_connection_revocation(rt, connection_id, *, entries_before): + """Keep metadata for retrying scoped cleanup after the credential itself has been removed.""" + cid = str(connection_id or "") + entries = [{key: e[key] for key in ("id", "connection_id", "type", "slot", "label") if key in e} + for e in entries_before if e.get("type") == "amazon_sp"] + if not cid or not any(e.get("connection_id") == cid for e in entries): + raise ValueError("Amazon revocation requires the original connection metadata") + def save(doc): + doc[cid] = {"state": "pending", "entries": entries} + return doc + rt.update(CONNECTION_REVOCATIONS_KEY, save, flush="sync") + + +def pending_connection_revocation(rt, connection_id): + """Return retry metadata only while this connection's cleanup remains unfinished.""" + row = (rt.get(CONNECTION_REVOCATIONS_KEY) or {}).get(str(connection_id)) or {} + return row.get("entries") if row.get("state") == "pending" else None + + +def connection_footprint(rt, connection_id, *, entries_before=None): + """Disclose this connection's rows without claiming ownership of a combined table.""" + from contextlib import closing + from core import keychain + entries = amazon_store._ordered_account_entries( + entries_before if entries_before is not None else keychain.list_entries(rt)) + cid = str(connection_id or "") + if not cid or not any(e["connection_id"] == cid for e in entries): + raise ValueError("Amazon footprint requires an explicit stored connection") + path = amazon_store._tenant_mirror_path(rt.key, rt) + out = {"type": "amazon_sp", "connectionId": cid, "action": "purge", "purge": [], "keep": [], + "rows": 0, "rowsKnown": path.is_file(), "why": "Only this connection's rows are deleted. Shared tables remain."} + projection = getattr(rt, "get_projection", None) + definitions = projection("user_tables", drop=("rows",)) if callable(projection) else rt.get("user_tables") + if definitions is None: + definitions = {} + if not isinstance(definitions, dict): + raise RuntimeError("Amazon database definitions are unreadable") + if not path.is_file(): + out["purge"] = [{"key": key, "label": defn.get("label") or key, "rows": 0, "rowsKnown": False} + for key, defn in definitions.items() if key in SOURCE and key not in _ADS_SPEC] + return out + import duckdb + cursor = datastore.ro_cursor() if path.resolve() == Path(datastore.DB_PATH).resolve() else datastore.bounded_connection( + duckdb.connect(str(path), read_only=True)) + with closing(cursor) as cur: + bound = amazon_store._read_state(cur, "amzn_account_binding") + legacy_owner = bound["detail"] if bound else entries[0]["connection_id"] + for key in SOURCE.keys() - _ADS_SPEC.keys(): + defn = definitions.get(key) or {} + columns = amazon_store._columns_of(cur, SOURCE[key]) + if not columns and key not in definitions: + continue + if not columns: + count = 0 + elif amazon_schema.CONNECTION_ID in columns: + count = cur.execute(f'SELECT count(*) FROM {SOURCE[key]} WHERE "_connection_id"=?', [cid]).fetchone()[0] + else: + count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0] if cid == legacy_owner else 0 + out["purge"].append({"key": key, "label": defn.get("label") or LABELS.get(key) or key, + "rows": count, "rowsKnown": True}) + out["rows"] += count + out["purge"].sort(key=lambda table: table["key"]) + return out + + +def purge_connection(rt, connection_id, *, entries_before=None): + """Remove this account's stored snapshots and observations while retaining shared definitions.""" + cid = str(connection_id or "") + if entries_before is None: + entries_before = pending_connection_revocation(rt, cid) + if not entries_before: + raise ValueError("Amazon scoped cleanup requires saved revocation metadata") + record_connection_revocation(rt, cid, entries_before=entries_before) + footprint = connection_footprint(rt, cid, entries_before=entries_before) + from contextlib import closing + con, _path = amazon_store._connect_tenant_mirror(rt.key, rt) + with closing(con): + binding = amazon_store._read_state(con, "amzn_account_binding") + entries = amazon_store._ordered_account_entries(entries_before) + legacy_owner = binding["detail"] if binding else entries[0]["connection_id"] + def sweep(doc): + for key, table in doc.items(): + if key not in SOURCE or key in _ADS_SPEC or not isinstance(table, dict): + continue + rows = table.get("rows") or {} + for rid, row in list(rows.items()): + if row.get(amazon_schema.CONNECTION_ID, legacy_owner) == cid: + rows.pop(rid) + return doc + rt.update("user_tables", sweep, flush="sync") + removed = amazon_store.purge_connection(rt.key, cid, rt=rt, entries_before=entries_before) + def complete(doc): + doc[cid] = {"state": "done", "snapshotId": removed["snapshotId"]} + return doc + rt.update(CONNECTION_REVOCATIONS_KEY, complete, flush="sync") + return {"purged": [table["key"] for table in footprint["purge"]], "refused": [], "failed": [], + "rows": removed["rows"], "rowsKnown": True, "footprint": footprint} + + #: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601. _DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date", "settlement_end_date", "deposit_date", "posted_date", "open_date", "date", @@ -261,12 +368,14 @@ def mirror_columns(cur, table, *, strict=False): """[(name, type)] for a mirror table, or [] when it has never been synced.""" if strict: # A successful empty metadata query proves absence on this cursor only. - return cur.execute( + return [(name, kind) for name, kind in cur.execute( "SELECT column_name, data_type FROM information_schema.columns " "WHERE table_catalog = current_database() AND table_schema = current_schema() " "AND table_name = ? ORDER BY ordinal_position", [table]).fetchall() + if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS] try: - return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()] + return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall() + if r[1] not in amazon_schema.MIRROR_INTERNAL_COLUMNS] except Exception: # noqa: BLE001 return [] @@ -322,6 +431,14 @@ def _latest_listings_sql(listing_names): if not {"seller_sku", "asin1"} <= set(listing_names): return None table = SOURCE[LISTINGS] + if amazon_schema.CONNECTION_ID in listing_names: + latest = (', max("_snapshot_date") OVER (PARTITION BY "_connection_id") AS latest_date' + if "_snapshot_date" in listing_names else "") + date = ', "_snapshot_date"' if latest else "" + predicate = ' AND "_snapshot_date" IS NOT DISTINCT FROM latest_date' if latest else "" + return ('SELECT DISTINCT "_connection_id", sku, asin FROM (SELECT "_connection_id", ' + 'TRIM(CAST(seller_sku AS VARCHAR)) AS sku, TRIM(CAST(asin1 AS VARCHAR)) AS asin' + f'{date}{latest} FROM {table}) WHERE sku <> \'\' AND asin <> \'\'{predicate}') latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})' if "_snapshot_date" in listing_names else "") return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, " @@ -369,6 +486,27 @@ def _parent_lookup_sql(cur): JSON is a read-only SQL relation here, with SQL quotes escaped separately from JSON. No temporary table or tenant mirror write is needed, and the complete mapping is retained. """ + sources = {"amzn_" + source: amazon_store._columns_of(cur, "amzn_" + source) + for source in _PARENT_SOURCES} + scoped = [table for table, names in sources.items() if amazon_schema.CONNECTION_ID in names] + if scoped: + base = amazon_store.asin_parents_sql(cur) + if base is None: + return None + ids = cur.execute(" UNION ".join(f'SELECT DISTINCT "_connection_id" FROM {table}' + for table in scoped)).fetchall() + parts = [] + for (cid,) in ids: + literal = "NULL::VARCHAR" if cid is None else "'" + str(cid).replace("'", "''") + "'" + ctes = [] + for table, names in sources.items(): + if not names: + continue + where = (f'"_connection_id" IS NOT DISTINCT FROM {literal}' + if amazon_schema.CONNECTION_ID in names else "FALSE") + ctes.append(f'{table} AS (SELECT * FROM main.{table} WHERE {where})') + parts.append(f'(WITH {", ".join(ctes)} SELECT {literal} AS "_connection_id", p.* FROM ({base}) p)') + return " UNION ALL ".join(parts) if parts else None identity = _parent_revision(cur) if identity is None: return amazon_store.asin_parents_sql(cur) @@ -427,11 +565,21 @@ def lookups(cur, key, names=None, *, allowed=True): "expr": "CAST(NULL AS VARCHAR)", "opens": {"via": via} if via else None, "description": _LOOKUP_DESCRIPTIONS[name]} for name, after, via in keys if name not in names] - listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])} + listing_names = amazon_store._columns_of(cur, SOURCE[LISTINGS]) parents = None if "parent_asin" in names else _parent_lookup_sql(cur) latest = _latest_listings_sql(listing_names) table = SOURCE[key] out = [] + scoped = amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, table) + parent_scoped = any(amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, "amzn_" + source) + for source in _PARENT_SOURCES) + def account_match(alias): + tagged = parent_scoped if alias == "_p" else amazon_schema.CONNECTION_ID in listing_names + if scoped and tagged: + return f' AND {alias}."_connection_id" IS NOT DISTINCT FROM {table}."_connection_id"' + if scoped: + return " AND FALSE" + return f' AND {alias}."_connection_id" IS NULL' if tagged else "" def add(lookup_key, after, expr, via): if lookup_key in names: @@ -445,11 +593,11 @@ def lookups(cur, key, names=None, *, allowed=True): via = asin_via if "asin1" in listing_names else None if parents: add(LOOKUP_PARENT, asin, - f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref})", via) + f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref}{account_match('_p')})", via) if latest and not sku: add(LOOKUP_SKU, asin, f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) " - f"FROM ({latest}) AS _l WHERE _l.asin = {ref})", via) + f"FROM ({latest}) AS _l WHERE _l.asin = {ref}{account_match('_l')})", via) elif sku and latest: ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))" via = sku_via if "seller_sku" in listing_names else None @@ -457,12 +605,14 @@ def lookups(cur, key, names=None, *, allowed=True): if not names & _NATIVE_ASIN: add(LOOKUP_ASIN, sku, f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) " - f"FROM ({latest}) AS _l WHERE _l.sku = {ref})", via) + f"FROM ({latest}) AS _l WHERE _l.sku = {ref}{account_match('_l')})", via) if parents: add(LOOKUP_PARENT, sku, f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) " f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin " - f"WHERE _l.sku = {ref})", via) + + ('AND _p."_connection_id" IS NOT DISTINCT FROM _l."_connection_id" ' + if parent_scoped and amazon_schema.CONNECTION_ID in listing_names else "") + + f"WHERE _l.sku = {ref}{account_match('_l')})", via) return out @@ -494,7 +644,7 @@ def _fields_for(cur, key, *, read_through=False): "label": declared.get(name, {}).get("label") or written.get(name) or _label(name), "type": _ftype(name, dtype), "source": "overlay", - "default": name == pinned or name in shown or name in product, + "default": name == pinned or name in shown or name in product or name == amazon_schema.STORE, "pinned": name == pinned, **({"format": fmt} if fmt else {}), }, flow="amazon_relational")) @@ -514,6 +664,13 @@ def _fields_for(cur, key, *, read_through=False): live_columns[target] = {name for name, _dtype in columns} out.extend(_odoo._preset(field, flow="amazon_relational") for field in connector_spec.system_link_fields(key, live_columns).get(key, ())) + if amazon_schema.STORE in names: + excess = max(0, sum(bool(field.get("default")) for field in out) - 12) + for field in reversed(out): + if excess and field.get("default") and field["key"] not in { + pinned, amazon_schema.STORE, *product, *LOOKUP_KEYS}: + field["default"] = False + excess -= 1 if not read_through and "refreshed" not in names: out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational")) return out @@ -636,17 +793,24 @@ def _integer(value): def grid_source(cur, key, *, columns=None, allow_lookups=True): """One binding from the current mirror schema, or an explicit unloaded-mirror reason.""" columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns + columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS] if not columns: return missing_source(key) names = {name for name, _dtype in columns} table = key.removeprefix(PREFIX) - identity = _identity_columns(table, names) + prepared = {row[0] for row in cur.execute( + "SELECT column_name FROM information_schema.columns WHERE table_catalog = current_database() " + "AND table_schema = current_schema() AND table_name = ? AND column_name IN (?, ?, ?)", + [SOURCE[key], amazon_schema.IDENTITY_OCCURRENCE, amazon_schema.IDENTITY_PID, + amazon_schema.CONNECTION_ID]).fetchall()} + identity_names = names | prepared + identity = _identity_columns(table, identity_names) fallback = sorted(set((*identity, *_fallback_columns(table)))) source = _quoted(SOURCE[key]) def value(name, owner=source): return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')" - if name in names else "NULL::VARCHAR") + if name in identity_names else "NULL::VARCHAR") def digest(values): return f"sha256(to_json(list_value({', '.join(values)})))" @@ -655,13 +819,16 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True): peer = '"_amazon_identity_peer"' peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity) partition = ", ".join(value(name, peer) for name in fallback) - # Rank only incomplete identities; earlier equal peers are exactly row_number minus one. + # Unprepared mirrors rank once in a materialized relation, outside the row lookup. occurrence = ( - '(SELECT CAST("_identity_occurrence" AS VARCHAR) FROM ' + '(WITH "_amazon_missing_identity" AS MATERIALIZED ' f'(SELECT {peer}.rowid AS "_identity_rowid", ' f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 ' f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) ' - f'AS "_amazon_missing_identity" WHERE "_identity_rowid" = {source}.rowid)') + 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_missing_identity" ' + f'WHERE "_identity_rowid" = {source}.rowid)') + if amazon_schema.IDENTITY_OCCURRENCE in prepared: + occurrence = f'CAST({source}.{_quoted(amazon_schema.IDENTITY_OCCURRENCE)} AS VARCHAR)' fallback_digest = digest(["'__amazon_missing_identity__'", *(value(name) for name in fallback), occurrence]) digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} " @@ -672,6 +839,31 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True): # The shared grid, relation and selection contracts all use numeric pids. Thirteen # hex digits retain a stable 52-bit identity that JavaScript represents exactly. ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))" + if amazon_schema.IDENTITY_PID in prepared: + ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}' + elif amazon_schema.CONNECTION_ID in prepared: + legacy_owner = amazon_store.legacy_identity_owner(cur, table) + if legacy_owner: + literal = "'" + str(legacy_owner).replace("'", "''") + "'" + old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID}) + old_fallback = sorted(set((*old_identity, *_fallback_columns(table)))) + old_complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in old_identity) + old_peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in old_identity) + old_partition = ", ".join(value(name, peer) for name in old_fallback) + old_occurrence = ( + '(WITH "_amazon_legacy_identity" AS MATERIALIZED ' + f'(SELECT {peer}.rowid AS "_identity_rowid", ' + f'row_number() OVER (PARTITION BY {old_partition} ORDER BY {peer}.rowid) - 1 ' + f'AS "_identity_occurrence" FROM {source} AS {peer} ' + f'WHERE {peer}."_connection_id"={literal} AND NOT ({old_peer_complete})) ' + 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_legacy_identity" ' + f'WHERE "_identity_rowid" = {source}.rowid)') + old_missing = digest(["'__amazon_missing_identity__'", *(value(name) for name in old_fallback), + old_occurrence]) + old_digest = (f"CASE WHEN {old_complete} THEN {digest([value(name) for name in old_identity])} " + f"ELSE {old_missing} END") + old_ident = f"greatest(1, CAST(('0x' || substr(({old_digest}), 1, 13)) AS BIGINT))" + ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END' cols = {} personal = set(amazon_store.personal_columns(table)) for name, dtype in columns: @@ -767,9 +959,12 @@ def _read(cur, key): # expressions the read-through binding uses, in this one SELECT (never per row). derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)] extra = "".join(f", {expr}" for _k, expr in derived) + owner_columns = ([amazon_schema.CONNECTION_ID] + if amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, SOURCE[key]) else []) + extra += "".join(f", {_quoted(name)}" for name in owner_columns) rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]} ' f'ORDER BY {_quoted(SOURCE[key])}.rowid').fetchall() - names = cols + [k for k, _expr in derived] + names = cols + [k for k, _expr in derived] + owner_columns out = [] for r in rows: row = {} @@ -792,6 +987,8 @@ def _read(cur, key): clauses.append(f'CAST({_quoted(column)} AS VARCHAR) IN ' f'({",".join("?" for _ in values)})') params.extend(values) + if amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, SOURCE[target]): + columns.append(amazon_schema.CONNECTION_ID) projection = ", ".join(_quoted(column) for column in columns) found = cur.execute( f'SELECT {target_spec["id"]}, {projection} FROM {SOURCE[target]} ' @@ -838,13 +1035,20 @@ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refu Neither path stores history rows or caps the matching target set. """ edges = link_edges(fields) + scoped = {target for field, target, _frm, _on in edges + if (field.get("link") or {}).get("connectionScope") == amazon_schema.CONNECTION_ID} + def match_key(target, row, value): + if target not in scoped: + return value + cid = row.get(amazon_schema.CONNECTION_ID) + return (str(cid), value) if cid not in (None, "") else None requests, weights = {}, {} for _field, target, frm, on in edges: values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")} requests.setdefault(target, {}).setdefault(on, set()).update(values) for row in rows: if row.get(frm) not in (None, ""): - slot = (target, on, str(row[frm])) + slot = (target, on, match_key(target, row, str(row[frm]))) weights[slot] = weights.get(slot, 0) + 1 indexes = {} remaining = max_bytes @@ -858,17 +1062,20 @@ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refu value = str(row[column]) if row.get(column) not in (None, "") else None if value not in wanted: continue - bucket = indexes[target][column].get(value) + matched = match_key(target, row, value) + if matched is None or not weights.get((target, column, matched)): + continue + bucket = indexes[target][column].get(matched) if bucket is not None and pid in bucket: continue if remaining is not None: # Charge every output occurrence before allocating its distinct ID. - cost = (len(str(pid)) + (1 if bucket else 2)) * weights[target, column, value] + cost = (len(str(pid)) + (1 if bucket else 2)) * weights.get((target, column, matched), 0) if cost > remaining: raise capacity_refusal() remaining -= cost if bucket is None: - bucket = indexes[target][column][value] = set() + bucket = indexes[target][column][matched] = set() bucket.add(pid) finally: close = getattr(iterator, "close", None) @@ -877,7 +1084,7 @@ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refu for field, target, frm, on in edges: for row in rows: row[field["key"]] = ",".join(str(pid) for pid in sorted( - indexes[target][on].get(str(row.get(frm)), ()))) + indexes[target][on].get(match_key(target, row, str(row.get(frm))), ()))) def available(tenant_key="royal-imports"): @@ -994,9 +1201,12 @@ def _lead_with_identity(stored, report, pinned): def _listing_identity_rows(cur): - """Capture SKU targets and recoverable numeric IDs from the previous listing declaration.""" + """Capture connection/SKU targets and recoverable numeric IDs from earlier listing declarations.""" key = PREFIX + "listing_items" names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])} + owner = (_quoted(amazon_schema.CONNECTION_ID) + if amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, SOURCE[key]) + else "NULL::VARCHAR") old = ["_snapshot_date", "sku", "_report_row_number"] if "_report_document_id" in names: old.append("_report_document_id") @@ -1006,16 +1216,23 @@ def _listing_identity_rows(cur): digest = f"sha256(to_json(list_value({', '.join(values)})))" legacy = f"CASE WHEN {complete} THEN greatest(1, CAST(('0x' || substr({digest}, 1, 13)) AS BIGINT)) END" sku = _quoted("sku") if "sku" in names else "NULL::VARCHAR" + unscoped = (f"greatest(1, CAST(('0x' || substr(sha256(to_json(list_value(CAST({sku} AS VARCHAR)))), " + "1, 13)) AS BIGINT))") spec = grid_source(cur, key) - return cur.execute(f'SELECT {sku}, {spec["id"]}, {legacy} FROM {SOURCE[key]}').fetchall() + return cur.execute(f'SELECT {owner}, {sku}, {spec["id"]}, {legacy}, {unscoped} ' + f'FROM {SOURCE[key]}').fetchall() -def _migrate_listing_identity(rt, current, identities, source_fields, read_through): - """Copy authored listing cells only when stored SKU or retained lineage proves their target.""" +def _migrate_listing_identity(rt, current, identities, source_fields, read_through, legacy_owner=None): + """Copy authored listing cells only when connection/SKU or retained lineage proves their target.""" from core import shared_overlay key = PREFIX + "listing_items" - targets, aliases, target_ids = {}, {}, set() + targets, aliases, target_ids, by_sku, candidates = {}, {}, set(), {}, {} + version = "connection_sku" if any(row[0] not in (None, "") for row in identities) else "sku" + previous_version = current.get("amazonListingIdentity") + migrating = previous_version != version + legacy_owner = str(legacy_owner) if legacy_owner not in (None, "") else None def refuse(): raise Refused("Amazon listing SKU migration is ambiguous. Existing cells were retained.") @@ -1026,44 +1243,69 @@ def _migrate_listing_identity(rt, current, identities, source_fields, read_throu refuse() aliases[old] = target - for sku, pid, previous in identities: + for owner, sku, pid, previous, unscoped in identities: if sku in (None, ""): continue - sku = str(sku) - if sku in targets or str(pid) in target_ids: + owner = str(owner) if owner not in (None, "") else None + identity = (owner, str(sku)) + if identity in targets or str(pid) in target_ids: refuse() - targets[sku] = str(pid) + targets[identity] = str(pid) + by_sku.setdefault(str(sku), []).append(identity) target_ids.add(str(pid)) alias(pid, pid) - if previous is not None and current.get("amazonListingIdentity") != "sku": - alias(previous, pid) + old_ids = [] + if migrating and previous_version not in ("sku", "connection_sku") and previous is not None: + old_ids.append(previous) + if migrating and owner is not None: + old_ids.append(unscoped) + for old in old_ids: + candidates.setdefault(str(old), set()).add(identity) authored = {field["key"] for field in current.get("fields", ()) if (field.get("automation") or {}).get("flowId") != "amazon_relational" or _odoo._ut().user_edited(field)} stored = current.get("rows") or {} - source_names = {*source_fields, "refreshed"} + source_names = {*source_fields, "refreshed", amazon_schema.CONNECTION_ID} moved, notes, seen = {}, {}, set() for old, row in stored.items(): sku = str(row.get("sku") or "") values = {name: value for name, value in row.items() - if name in authored or name not in source_names} - if sku not in targets: + if name != amazon_schema.CONNECTION_ID and (name in authored or name not in source_names)} + owner = row.get(amazon_schema.CONNECTION_ID) + if owner not in (None, ""): + target = targets.get((str(owner), sku)) + elif str(old) in target_ids: + target = str(old) if any(targets[identity] == str(old) for identity in by_sku.get(sku, ())) else None + elif legacy_owner is not None and version == "connection_sku" and previous_version != "connection_sku": + target = targets.get((legacy_owner, sku)) + else: + matches = by_sku.get(sku, ()) + target = targets[matches[0]] if len(matches) == 1 else None + if target is None: if values: refuse() continue - if sku in seen: + if target in seen: refuse() - seen.add(sku) - target = targets[sku] + seen.add(target) alias(old, target) moved[target] = dict(row) if read_through and values: notes[target] = values + # Stored ownership is stronger evidence than an old numeric alias shared by two accounts. + for old, matches in candidates.items(): + if old in aliases: + continue + owned = {identity for identity in matches if identity[0] == legacy_owner} if legacy_owner else set() + resolved = owned if legacy_owner is not None and version == "connection_sku" else matches + if len(resolved) == 1: + alias(old, targets[next(iter(resolved))]) + # Read strictly before declaring migration complete. A failed read is never an empty overlay. captured = dict((rt.get(shared_overlay.bucket(key)) or {}).get("cells") or {}) - if current.get("amazonListingIdentity") != "sku" and any( + if migrating and any( values and str(pid) not in aliases for pid, values in captured.items()): refuse() for old, target in aliases.items(): @@ -1093,7 +1335,7 @@ def _migrate_listing_identity(rt, current, identities, source_fields, read_throu if any(delta.values()): shared_overlay.put_rows(key, delta, st=Writer()) current["rows"] = moved - current["amazonListingIdentity"] = "sku" + current["amazonListingIdentity"] = version def apply_plan(rt, built, username="amazon", today=None): @@ -1119,6 +1361,12 @@ def apply_plan(rt, built, username="amazon", today=None): for key, _label, _fields, _rows in plans} listing_ids = (_listing_identity_rows(cur_mirror) if PREFIX + "listing_items" in built else ()) + listing_owner = None + if PREFIX + "listing_items" in built: + from core import keychain + binding = amazon_store._read_state(cur_mirror, "amzn_account_binding") + listing_owner = ((binding or {}).get("detail") + or keychain.default_connection_id(rt, "amazon_sp")) # S79-T13: each Ads table's description states its history window and any recorded limit. ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror) for key, *_rest in plans if key in _ADS_SPEC} @@ -1187,7 +1435,10 @@ def apply_plan(rt, built, username="amazon", today=None): if key == PREFIX + "listing_items": current = doc.get(key) or {} _migrate_listing_identity(rt, current, listing_ids, report_order[key], - isinstance(rows, dict) and rows.get("readThrough") is True) + isinstance(rows, dict) and rows.get("readThrough") is True, + legacy_owner=listing_owner) + listing_version = ("connection_sku" if any(row[0] not in (None, "") for row in listing_ids) + else "sku") fields = [field for field in fields if not (field.get("type") == "link" and (field.get("automation") or {}).get("flowId") == "amazon_relational")] @@ -1261,7 +1512,7 @@ def apply_plan(rt, built, username="amazon", today=None): if ads_descriptions.get(key): doc[key]["description"] = ads_descriptions[key] if key == PREFIX + "listing_items": - doc[key]["amazonListingIdentity"] = "sku" + doc[key]["amazonListingIdentity"] = listing_version return doc rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync") diff --git a/api/app_routes.py b/api/app_routes.py index db51ab342f7534f7b750e00985ab9e333eada381..8d4703dc03a2a3a20590ebb9ea5f386d9331b00b 100644 --- a/api/app_routes.py +++ b/api/app_routes.py @@ -1,14 +1,25 @@ """C3: mount every C1 App router by name, recording a failed import instead of raising it.""" from __future__ import annotations -import sys from importlib import import_module import app_templates +from fastapi import APIRouter, Depends +from deps import Session, require_session, running_main + + +availability_router = APIRouter(prefix="/api/v1/applications", tags=["applications"]) + + +@availability_router.get("/{app_id}/availability") +def app_availability(app_id: str, session: Session = Depends(require_session)): + """Read one App's availability through the authenticated tenant and App walls.""" + import application_registry + return application_registry.opening(session, app_id) def _failures_record(): - main = sys.modules.get("main") + main = running_main() record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None return record if isinstance(record, dict) else app_templates.ROUTER_MOUNT_FAILURES @@ -44,6 +55,8 @@ def held_dependencies(kind): def mount_app_routers(app, failures=None): """Include each C1 `router` once, with main.py `_mount_router` semantics; returns those mounted.""" record = failures if isinstance(failures, dict) else _failures_record() + if not _included(app, availability_router): + app.include_router(availability_router) mounted = [] for row in app_templates.TEMPLATES: name = row["router"] @@ -58,7 +71,7 @@ def mount_app_routers(app, failures=None): failure = {"error": type(exc).__name__, "missing_module": exc.name if isinstance(exc, ModuleNotFoundError) else ""} record.setdefault(name, []).append(failure) - if record is not _failures_record(): + if record is not app_templates.ROUTER_MOUNT_FAILURES: # The offer predicate reads main.py's record and this one, never a caller's own dict. app_templates.ROUTER_MOUNT_FAILURES.setdefault(name, []).append(failure) # Exception messages can contain configuration values; report the class/module only. diff --git a/api/app_templates.py b/api/app_templates.py index 85b8db32a6967cbfb0f46ea112168ddf7d463156..f1c6b2763940ddce9c4f8f37dbd758b8f5de4096 100644 --- a/api/app_templates.py +++ b/api/app_templates.py @@ -18,6 +18,7 @@ CONNECTOR_ENTRY_TYPES = MappingProxyType({"odoo": "odoo", "amazon": "amazon_sp", def _amazon(kind, label, purpose, process, sources, *, connector="amazon", proposes=True, tools=()): return {"kind": kind, "label": label, "purpose": purpose, "connector": connector, + "tags": ["Amazon Ads" if connector == "amazon_ads" else "Amazon"], "sources": tuple(sources), "process": process, "mark": kind, "module": "modules." + kind, "router": "routes_" + kind, "view": kind.replace("_", "-"), "tools": tuple(tools), "proposes": proposes} @@ -27,10 +28,10 @@ TEMPLATES = ( # Routing and Catalog render through Map mode; their routes are mounted by main.py. # They read FIELD_BINDINGS over their source, so an App of either kind stores no field map. {"kind": "route", "label": "Routing", "purpose": "Plan delivery routes for your customers.", - "connector": "odoo", "sources": ("customer_data",), "process": "None", "mark": "route", + "connector": "odoo", "tags": ["Odoo"], "sources": ("customer_data",), "process": "None", "mark": "route", "module": None, "router": None, "view": None, "tools": (), "proposes": False}, {"kind": "catalog", "label": "Catalog", "purpose": "Build and print product catalogs.", - "connector": "odoo", "sources": ("product_data",), "process": "None", "mark": "catalog", + "connector": "odoo", "tags": ["Odoo"], "sources": ("product_data",), "process": "None", "mark": "catalog", "module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"), "proposes": False}, _amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.", @@ -64,13 +65,12 @@ TEMPLATES = ( BY_KIND = MappingProxyType({row["kind"]: row for row in TEMPLATES}) KINDS = tuple(BY_KIND) -#: R11 (owner 2026-10-25): every Amazon App is held out of the product until SDLC 80 rebuilds it. -#: A held kind is never offered, listed or opened. Its stored App records and data stay untouched. -HELD_KINDS = frozenset(row["kind"] for row in TEMPLATES if row["connector"] in ("amazon", "amazon_ads")) +# SDLC 80 ships the rebuilt Apps through their existing connector, grant and source walls. +HELD_KINDS = frozenset() def held(kind_or_app): - """True when a template kind, or an App record's `templateKind`, is held out (R11).""" + """True when a template kind, or an App record's `templateKind`, has a release hold.""" kind = kind_or_app.get("templateKind") if isinstance(kind_or_app, dict) else kind_or_app return isinstance(kind, str) and kind in HELD_KINDS diff --git a/api/application_registry.py b/api/application_registry.py index 0aa77e949c15e8c2dd4c42915c183d92987813bc..a98f7c30b20ea0597311c796242e9b6e22d89175 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -20,6 +20,7 @@ APPLICATION_TYPES = ("chart", "route", "map", "code") # Finance descriptors remain available to the governed fact routes. FINANCE_TEMPLATE_KIND = "finance" _KEY = "applications_v1" +_RECORD_PREFIX = "applications_v1_app/" #: D-8392 (R58): the most live Apps one account may create. The cap counts per account, never per tenant. _MAX = 500 _MAX_DISPLAY_BYTES = 512 * 1024 @@ -417,11 +418,11 @@ def _stored_doc(raw): return doc -def _update_document(runtime, mutate): +def _update_document(runtime, mutate, *, key=_KEY): """Keep every registry write, including migrations, safe for JSONB storage.""" def apply(raw): return _stored_doc(mutate(raw)) - return runtime.update(_KEY, apply) + return runtime.update(key, apply) def _clean_doc(raw): @@ -444,15 +445,197 @@ def _clean_doc(raw): return doc -def load(runtime): +def _app_projection(app): + row = _app_templates.bound(app) + if isinstance(row, dict): + contract = _template_contract(str(row.get("templateKind") or "")) + if contract and contract.get("view") and not contract.get("fieldRoles"): + row = dict(row) + row.setdefault("fieldMap", {}) + return row + + +def _record_documents(runtime): + """Read independent App records through the tenant's existing document doors.""" + listing = getattr(runtime, "list_keys", None) + if not callable(listing): + return {} # Legacy stores without document enumeration retain their original format. + keys = listing(_RECORD_PREFIX) + reader = getattr(runtime, "get_many", None) + records = reader(keys) if keys and callable(reader) else None + if records is None: + records = {key: _read_strict(runtime, key) for key in keys} + result = {} + for key in keys: + doc = _clean_doc(records.get(key)) + if len(doc["applications"]) > 1: + raise ApplicationError("an App record must contain one App") + for identity, app in doc["applications"].items(): + if (not isinstance(app, dict) or app.get("id") != identity + or app.get("tenant") != runtime.key): + raise ApplicationError("the App record does not match its tenant") + result[key] = doc + return result + + +def _application_key(runtime, app_id): + for key, doc in _record_documents(runtime).items(): + if app_id in doc["applications"]: + return key + return _KEY + + +def load(runtime, *, strict=False): """The read door: a Routing or Catalog App carries its C1 binding (`app_templates.bound`).""" - try: doc = _clean_doc(runtime.get(_KEY)) - except Exception: return blank_doc() - doc["applications"] = {key: _app_templates.bound(app) + try: + doc = _clean_doc(runtime.get(_KEY)) + for record in _record_documents(runtime).values(): + doc["applications"].update(record["applications"]) + except Exception as exc: + if strict: + raise + raise_if_resource_error(exc) + return blank_doc() + doc["applications"] = {key: _app_projection(app) for key, app in doc["applications"].items()} return doc +_AVAILABILITY_MESSAGES = { + "router_unavailable": "This App cannot open because its routes are unavailable.", + "no_grant": "Your access limit prevents this App from reading its source.", + "not_connected": "Connect the App's connector to open this App.", + "no_rows": "This App has no source rows yet.", + "source_unavailable": "This App cannot read its source right now.", + "held": "This App is not available yet.", +} + + +def availability_reason(state, connector=None): + """The public reason vocabulary contains no exception or configuration text.""" + message = _AVAILABILITY_MESSAGES.get(state) + if state == "not_connected": + label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector) + if label: + message = f"Connect {label} to open this App." + return {"state": state, "message": message, + "lock": "Access limit" if state == "no_grant" else None} + + +def _mount_reason(kind): + catalogue = _app_templates.BY_KIND.get(str(kind or "")) + if catalogue and catalogue["router"] in _app_templates.router_failures(): + return availability_reason("router_unavailable") + return None + + +def _source_rows_exist(session, sources): + """Probe at most one row per declared mirror source in the session's tenant.""" + import contextlib + import stat + import duckdb + from pathlib import Path + from core import connector_spec + from harness import amazon_store, datastore + + specs = [connector_spec.spec_for(source) for source in sources] + if any(spec is None for spec in specs): + raise RuntimeError("App source is not a declared mirror table") + path = amazon_store._tenant_mirror_path(session.tenant, rt=session.runtime) + try: + mode = path.stat().st_mode + except FileNotFoundError: + return False + if not stat.S_ISREG(mode): + raise RuntimeError("App mirror is not a database file") + shared = path.resolve() == Path(datastore.DB_PATH).resolve() + cursor = (datastore.ro_cursor() if shared else + datastore.bounded_connection(duckdb.connect(str(path), read_only=True))) + with contextlib.closing(cursor): + for spec in specs: + present = cursor.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'main' " + "AND table_name = ? LIMIT 1", [spec.source]).fetchone() + if present: + table = '"' + spec.source.replace('"', '""') + '"' + if cursor.execute(f"SELECT 1 FROM {table} LIMIT 1").fetchone(): + return True + return False + + +def _readable_template_app(session, app_id): + """Resolve App identity before reading any source or exposing a refusal.""" + from deps import err + if (session.tenant != getattr(session.runtime, "key", None) + or session.tenant != getattr(getattr(session.runtime, "tenant", None), "key", None)): + raise err(404, "app_not_found", "This App is unavailable or is not shared with you.") + try: + app = load(session.runtime, strict=True)["applications"].get(app_id) + except Exception: + raise err(503, "app_unavailable", "This App cannot be read right now.") from None + if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant + or app.get("retired") is True or not may_read(app, session.uname, session.admin)): + raise err(404, "app_not_found", "This App is unavailable or is not shared with you.") + kind = app.get("templateKind") + catalogue = _app_templates.BY_KIND.get(str(kind or "")) + if not catalogue or not catalogue["view"]: + raise err(404, "app_not_found", "This App is unavailable or is not shared with you.") + return app, catalogue + + +def _availability_for_app(session, app, catalogue): + from core import perm_scope + from routes_connectors import _entry_types + + kind = app["templateKind"] + if _app_templates.held(kind): + return availability_reason("held") + mount = _mount_reason(kind) + if mount: + return mount + if not _app_templates.seams_ready(kind): + return availability_reason("router_unavailable") + try: + sources = catalogue["sources"] + for source in sources: + if not perm_scope.may_read(session.user, source, st=session.runtime): + return availability_reason("no_grant") + # These App readers cannot apply a row or field wall, so refuse before probing. + if perm_scope.wall_declared(session.user, source, st=session.runtime): + return availability_reason("no_grant") + types, _scopes, failure = _entry_types(session.runtime, session.uname) + if failure is not None: + return availability_reason("source_unavailable") + connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ()) + if not connected and catalogue["connector"] == "odoo": + from routes_keychain import env_odoo_available + connected = env_odoo_available(session.runtime) + if not connected: + return availability_reason("not_connected", catalogue["connector"]) + if sources and not _source_rows_exist(session, sources): + return availability_reason("no_rows") + except Exception: + return availability_reason("source_unavailable") + return availability_reason("ready") + + +def availability(session, app_id): + """Resolve a named App without weakening its tenant, sharing or source permissions.""" + app, catalogue = _readable_template_app(session, app_id) + return _availability_for_app(session, app, catalogue) + + +def opening(session, app_id): + """Return only frame metadata and availability for a standalone template App.""" + app, catalogue = _readable_template_app(session, app_id) + return {"application": { + "id": app["id"], "name": app["name"], "templateKind": catalogue["kind"], + "templateView": catalogue["view"], "templateMark": catalogue["mark"], + "templateConnector": catalogue["connector"], "fieldMap": {}, + "capabilities": {"read": True, "edit": may_edit(app, session.uname, session.admin)}, + }, "availability": _availability_for_app(session, app, catalogue)} + + def _server_origin(origin_door=None, via=None): """Stamp the authenticated server door without inferring it from a display label.""" door = "app" if origin_door is None and via is None else origin_door @@ -925,7 +1108,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None, is_default=False): - row=copy.deepcopy(_app_templates.bound(app)) + row=copy.deepcopy(_app_projection(app)) row.setdefault("origin", None) if row.get("type") == "code": # Only the authenticated code door returns the body. @@ -944,6 +1127,12 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas # C2: the client mounts apps//AppView.tsx and fills the two marks. row.update(templateView=catalogue["view"], templateMark=catalogue["mark"], templateConnector=catalogue["connector"]) + row["deepLink"] = f"/app/template/{row['id']}" + mount = _mount_reason(row.get("templateKind")) + if mount: + row["availability"] = mount + elif not _database_free(app) and not compatible: + row["availability"] = availability_reason("no_grant") if row.get("templateKind") == FINANCE_TEMPLATE_KIND: # A Finance App is a user's own copy: owner controls apply. Defaults bind a database. row["capabilities"]["default"] = False @@ -990,6 +1179,9 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre doc = load(runtime) rows=[] for app in doc["applications"].values(): + if (isinstance(app, dict) and getattr(runtime, "key", None) + and app.get("tenant") != runtime.key): + continue if _app_templates.held(app): continue if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND): @@ -1000,7 +1192,11 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre if database not in (app.get("compatibleDatabases") or ()): continue elif not _listed(app, user): continue if (not _database_free(app) - and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue + and not _allowed(app.get("compatibleDatabases"), allowed_databases)): + catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or "")) + # An owned or shared template may explain a lost source grant without serving data. + if database or not catalogue or not catalogue["view"]: + continue if kind and app.get("type") != kind: continue if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue default_key = f"{database}::{app.get('type')}" if database else "" @@ -1077,17 +1273,22 @@ def _code_audit_config(app): "htmlSha256": hashlib.sha256(html).hexdigest()}} -def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_of=None): +def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_of=None, + document_key=_KEY): """Log the final committed App change once, excluding previews and unbound legacy stores.""" from core.user_tables import _activity_writer append = _activity_writer(runtime, user) if append is None: - return _update_document(runtime, mutate) + return _update_document(runtime, mutate, key=document_key) app_ids = (app_id,) if isinstance(app_id, str) else tuple(app_id) pending = {} def snapshot(doc, identity): app = copy.deepcopy((doc.get("applications") or {}).get(identity) or {}) + if not app and document_key == _KEY: + key = _application_key(runtime, identity) + if key != _KEY: + app = _clean_doc(_read_strict(runtime, key))["applications"].get(identity, {}) if app: if app.get("type") == "code": app["config"] = _code_audit_config(app) @@ -1114,7 +1315,7 @@ def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_ pending[identity] = changes return result - result = _update_document(runtime, apply) + result = _update_document(runtime, apply, key=document_key) if result is not False: for identity, changes in pending.items(): append(kind="app", object={"app": identity, "id": identity}, by=user, @@ -1122,8 +1323,59 @@ def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_ return result +class _AppSlotOccupied(Exception): + pass + + +def _create_record(runtime, app, user, *, via=None, on_behalf_of=None): + """Claim one independently locked App slot without a registry or index write.""" + legacy = _clean_doc(_read_strict(runtime, _KEY)) + records = _record_documents(runtime) + combined = _clean_doc(legacy) + for record in records.values(): + combined["applications"].update(record["applications"]) + refusal = capacity_refusal(combined, user) + if refusal: + raise ApplicationError(refusal) + owner = hashlib.sha256(user.encode("utf-8")).hexdigest() + prefix = f"{_RECORD_PREFIX}{owner}/" + occupied = {int(key[len(prefix):]) for key, doc in records.items() + if key.startswith(prefix) and key[len(prefix):].isdigit() and doc["applications"]} + legacy_count = sum(isinstance(row, dict) and row.get("createdBy") == user + and row.get("retired") is not True for row in legacy["applications"].values()) + # Existing slots above a lowered capacity still consume that account's allowance. + limit = account_capacity() - legacy_count + while True: + reduced = account_capacity() - legacy_count - sum(slot >= limit for slot in occupied) + if reduced == limit: + break + limit = reduced + slots = [slot for slot in range(max(0, limit)) if slot not in occupied] + # Different creates start at different records, including when their reads race. + slots.sort(key=lambda slot: hashlib.sha256(f"{app['id']}:{slot}".encode()).digest()) + for slot in slots: + def apply(raw): + doc = _clean_doc(raw) + if doc["applications"]: + raise _AppSlotOccupied() + doc["applications"][app["id"]] = app + return doc + try: + saved = _update_with_activity(runtime, app["id"], user, apply, via=via, + on_behalf_of=on_behalf_of, document_key=prefix + str(slot)) + except _AppSlotOccupied: + continue + if saved is False: + raise ApplicationError("the App was not saved") + return + cap = account_capacity() + raise ApplicationError(f"Your account can hold up to {cap} App{'' if cap == 1 else 's'}. Delete an App to add another.") + + def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None, allowed_databases=None, via=None, on_behalf_of=None, origin_door=None): + if getattr(runtime, "key", tenant) != tenant or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant: + raise PermissionError("an Application cannot be created for another tenant") app = _build_application( tenant, user, payload, semantic_records=semantic_records, field_contracts=field_contracts, allowed_databases=allowed_databases, origin_door=origin_door, via=via) @@ -1131,14 +1383,16 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr advice = _validate_chart_rules( app, semantic_records=semantic_records, field_contracts=field_contracts) _validate_catalog_rules(app) - identity = template_identity(app) def apply(raw): doc=_clean_doc(raw) refusal = capacity_refusal(doc, user) if refusal: raise ApplicationError(refusal) # R216 and R247 allow every template to have several independent copies. doc["applications"][app_id]=app; return doc - _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of) + if callable(getattr(runtime, "list_keys", None)): + _create_record(runtime, app, user, via=via, on_behalf_of=on_behalf_of) + else: + _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of) result = _public(app,user,allowed_databases=allowed_databases) if app["type"] == "chart": result["advice"] = advice @@ -1194,6 +1448,7 @@ def _ensure_route_seed(runtime, tenant): if not isinstance(routes, list): raise ApplicationError("route store routes must be a list") route_count = len(routes) + records = _record_documents(runtime) except Exception as exc: return {"outcome": "skipped", "reason": f"route read failed: {type(exc).__name__}: {exc}"} @@ -1205,12 +1460,15 @@ def _ensure_route_seed(runtime, tenant): if "routeApplication" in migrations: result = {"outcome": "skipped", "reason": "route Application migration already applied"} return raw - existing = next((app for app in doc["applications"].values() + applications = dict(doc["applications"]) + for record in records.values(): + applications.update(record["applications"]) + existing = next((app for app in applications.values() if isinstance(app, dict) and app.get("templateKind") == "route"), None) app_id = existing.get("id") if existing is not None else None outcome = "existing" if existing is not None else "no_routes" if existing is None and route_count: - refusal = capacity_refusal(doc, "loopable") + refusal = capacity_refusal({"applications": applications}, "loopable") if refusal: raise ApplicationError(refusal) app = _build_application( @@ -1285,6 +1543,7 @@ def migrate_c1_bindings(runtime, tenant, *, app_ids=None): scope = None if app_ids is None else set(app_ids) try: doc = _clean_doc(_read_strict(runtime, _KEY)) + records = _record_documents(runtime) except Exception as exc: return {"outcome": "skipped", "reason": f"registry read failed: {type(exc).__name__}: {exc}", "rewritten": [], "kept": []} @@ -1310,6 +1569,34 @@ def migrate_c1_bindings(runtime, tenant, *, app_ids=None): result.update(outcome="migrated", rewritten=sorted(planned)) return current _update_document(runtime, apply) + for key, record in records.items(): + planned, held = _c1_plan(record, scope) + result["kept"].extend(held) + if not planned: + continue + rewritten = [] + + def apply_record(raw): + rewritten.clear() + current = _clean_doc(raw) + fresh, _held = _c1_plan(current, scope) + if not fresh: + return raw + marker = current.setdefault("migrations", {}).setdefault( + C1_BINDINGS_MARKER, {"rewritten": {}, "kept": []}) + for identity in fresh: + before = current["applications"][identity] + marker["rewritten"][identity] = {field: copy.deepcopy(before[field]) for field in + ("sourceDatabase", "fieldMap", "compatibleDatabases") + if field in before} + marker["at"] = write_execution.utcnow().isoformat() + current["applications"].update(fresh) + rewritten.extend(sorted(fresh)) + return current + _update_document(runtime, apply_record, key=key) + if rewritten: + result["rewritten"].extend(rewritten) + result["outcome"] = "migrated" if result["rewritten"]: try: from catalog_store import migrate_snapshots @@ -1590,7 +1877,8 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie app["revision"]=int(app.get("revision") or 0)+1 app["updatedAt"]=write_execution.utcnow().isoformat(); landed.update(app) return doc - _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of) + _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of, + document_key=_application_key(runtime, app_id)) result = _public(landed,user,is_admin,allowed_databases=allowed_databases, target_database=target_database) if landed["type"] == "chart" and not set(patch) <= {"starred"}: @@ -1932,7 +2220,16 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None, for key in list(doc["defaults"]): if doc["defaults"][key]==app_id: doc["defaults"].pop(key) return doc - _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of) + document_key = _application_key(runtime, app_id) + _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of, + document_key=document_key) + if document_key != _KEY: + def clear_defaults(raw): + doc = _clean_doc(raw) + doc["defaults"] = {key: value for key, value in doc["defaults"].items() if value != app_id} + return doc + if app_id in _clean_doc(_read_strict(runtime, _KEY))["defaults"].values(): + _update_document(runtime, clear_defaults) result = {"deleted": app_id} # After the registry write: a failed clear is reported, never an undone delete. cleared, failed = clear_view_references(runtime, app_id, removed.get("compatibleDatabases")) @@ -1965,7 +2262,9 @@ def set_default(runtime, database, kind, app_id, user, is_admin=False, *, allowe raise PermissionError("that database is not available") key=f"{database}::{kind}" def apply(raw): - doc=_clean_doc(raw); app=doc["applications"].get(app_id) + doc=_clean_doc(raw) + record_key = _application_key(runtime, app_id) + app = (doc if record_key == _KEY else _clean_doc(_read_strict(runtime, record_key)))["applications"].get(app_id) if (not isinstance(app, dict) or not may_edit(app,user,is_admin) or database not in (app.get("compatibleDatabases") or ()) or app.get("type")!=kind): diff --git a/api/assistant_mirror.py b/api/assistant_mirror.py index 19f9f7ff6abd16ccb1cd019042edb87b9d7a28b2..2f186f8b5568994bf680d346b7bde2a2aec2303f 100644 --- a/api/assistant_mirror.py +++ b/api/assistant_mirror.py @@ -441,7 +441,7 @@ def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys, *, table_ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None, - filter_keys=None, defn=None): + filter_keys=None, defn=None, serialize=True): """Every permitted row of a mirror-served grid, or a refusal that names its cause. Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape @@ -516,6 +516,8 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None), defn=defn if isinstance(defn, dict) else None) rows = rot._pool_from_window(spec, win, stamp=stamp) + if serialize: + rows = _typed_rows(rot, fields, rows) rows.sort(key=lambda row: row["pid"]) return rows, stamp, int(win["total"]) @@ -543,6 +545,30 @@ def _json_value(rot, value): return value +def field_value(rot, field, value): + """Serialize a native checkbox as a Boolean while preserving other JSON values.""" + if field.get("type") == "checkbox" and value is not None: + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "t", "yes", "y", "on"} + return bool(value) + return _json_value(rot, value) + + +def _typed_rows(rot, fields, rows, *, preserve_blank=False): + """Apply the declared field codecs without changing source records.""" + flags = [field for field in fields if field.get("type") == "checkbox"] + if not flags: + return rows + + def encode(field, value): + if preserve_blank and (value is None or isinstance(value, str) and value.strip() in {'', '(none)'}): + return value + return field_value(rot, field, value) + + return [{**row, **{field["key"]: encode(field, row[field["key"]]) + for field in flags if field["key"] in row}} for row in rows] + + def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None, filter_tree=None, filter_keys=None): """Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631). @@ -606,7 +632,7 @@ def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None, def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None, - filter_tree=None, filter_keys=None, defn=None, sorts=None): + filter_tree=None, filter_keys=None, defn=None, sorts=None, serialize=True): """Return one SQL-ordered permitted page, its source stamp and its full population count.""" from harness import datastore rot = _rot() @@ -639,4 +665,6 @@ def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None), defn=defn if isinstance(defn, dict) else None) rows = rot._pool_from_window(spec, win, stamp=stamp) + if serialize: + rows = _typed_rows(rot, fields, rows) return rows, stamp, int(win["total"]) diff --git a/api/automation_engine.py b/api/automation_engine.py index c81ec873f3cf3ac5ec4ce2852d35531b1fcc8654..724f0033668f0796753f946ed0b6d372b0843248 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -14958,55 +14958,54 @@ def _refresh_relations_inplace(tables, log=print, rt=None): def refresh_relations(rt, log=print, tables=None, persist=True): - """The tick half of the relational pass — the twin of `refresh_metrics`. - - ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale - when table B gains a row, and A has no way to know that happened. Cheap by construction — a - table declaring neither kind costs one dict scan. - """ - if tables is not None: - local = _refresh_relations_inplace(tables, log=log, rt=rt) - if not local or not persist: - return local - actual = [0] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - actual[0] = _refresh_relations_inplace(cur, log=log, rt=rt) - return cur - - rt.update(UT_STORE_KEY, _up, flush="async") - return actual[0] - - snapshot = { - str(key): {**(table or {}), - "rows": {str(rid): dict(row or {}) - for rid, row in ((table or {}).get("rows") or {}).items()}} - for key, table in (ut_all(rt) or {}).items() - } - if not _refresh_relations_inplace(snapshot, log=log, rt=rt): + """Compute relations outside the write lock and merge only unchanged derived cells.""" + snapshot = tables if tables is not None else ut_all(rt) + # Keep authored inputs before computation, including nested cell values. + before = copy.deepcopy(snapshot) + local = _refresh_relations_inplace(snapshot, log=log, rt=rt) + if not local or not persist: + return local + patches = {} + for key, old in before.items(): + if not isinstance(old, dict): + continue + fields = old.get('fields') or [] + derived = {str(f['key']) for f in fields if f.get('key') + and (f.get('type') in ('rollup', 'if') or _ut().is_derived_link(f))} + new_rows = (snapshot.get(key) or {}).get('rows') or {} + for rid, row in (old.get('rows') or {}).items(): + after = new_rows.get(rid) + if not isinstance(row, dict) or not isinstance(after, dict): + continue + changed = {fk: copy.deepcopy(after[fk]) for fk in derived + if fk in after and (fk not in row or row[fk] != after[fk])} + if changed: + patches.setdefault(str(key), {})[str(rid)] = (row, changed) + if not patches: return 0 actual = [0] def _up(cur): - cur = cur if isinstance(cur, dict) else {} - actual[0] = _refresh_relations_inplace(cur, log=log, rt=rt) + actual[0] = 0 + lock = getattr(cur, 'lock_records', None) + if callable(lock): + lock({key: list(rows) for key, rows in patches.items()}) + for key, rows in patches.items(): + table = (cur or {}).get(key) + if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'): + continue + current_rows = table.get('rows') + if current_rows is None: + continue + for rid, (old_row, changes) in rows.items(): + current = current_rows.get(rid) + # A changed authored input or a deleted row waits for the next refresh. + if not isinstance(current, dict) or current != old_row: + continue + current.update(changes) + actual[0] += 1 return cur - # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). - # - # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and - # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. - # A derived-cell recompute has no business forcing a commit on someone typing. - # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new - # row lives only in the store cache for 2-20s, and this call — on the SAME key - # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the - # result. `POST /rows` answered 201 and the row was gone. - # - # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it - # is what makes the other ~39 sync writers of this key safe. This line is not that fix and - # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both - # landed together on purpose: one is correctness, one is the hot path. rt.update(UT_STORE_KEY, _up, flush="async") return actual[0] diff --git a/api/catalog_store.py b/api/catalog_store.py index 7c5fff4c02798f045d065c42a79bacabf863e9ca..3b112d124b44d72b6db1cfbdaf103d2d54d70f42 100644 --- a/api/catalog_store.py +++ b/api/catalog_store.py @@ -212,7 +212,7 @@ def require_app(session, app_id, *, write=False, allow_orphan=False): raise err(403, "catalog_tenant_forbidden", "This catalog belongs to another workspace. Open the correct workspace.") if not runtime.available(): raise err(503, "store_unavailable", "Catalog storage is unavailable. Try again when the store is ready.") - app = applications._clean_doc(applications._read_strict(runtime, applications._KEY))["applications"].get(app_id) + app = applications.load(runtime, strict=True)["applications"].get(app_id) if app is None and allow_orphan and session.admin and not write: app = _load(runtime)["appSnapshots"].get(app_id) if (not isinstance(app, dict) or app.get("tenant") != session.tenant @@ -231,6 +231,9 @@ def require_app(session, app_id, *, write=False, allow_orphan=False): _assert_write_scope(session, "catalogs") contracts = applications.chart_field_contracts(session, [source]) fields = applications._contract_fields(source, field_contracts=contracts) + # Projected C1 fields retain their optional roles; authored legacy maps stay strict. + if not app_templates.own_binding(app): + app = {key: value for key, value in app.items() if key not in {"sourceDatabase", "fieldMap"}} try: app = applications.bound_application(app, fields) except applications.ApplicationError: @@ -260,7 +263,7 @@ def _binding_wall(record, app, fields): def _record(session, catalog_id, *, write=False): record = _load(session.runtime)["catalogs"].get(catalog_id) if record is None: - apps = applications._clean_doc(applications._read_strict(session.runtime, applications._KEY))["applications"] + apps = applications.load(session.runtime, strict=True)["applications"] for app in apps.values(): if not isinstance(app, dict): continue diff --git a/api/deps.py b/api/deps.py index 27453c6c05de46f90922ec297ca918418fb43716..3f5a465c32d3a3a4a59e5f0fd1062bbbd18fdf80 100644 --- a/api/deps.py +++ b/api/deps.py @@ -642,14 +642,14 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No rows, stamp, total = assistant_mirror.read_page( session, database, cur, spec, fields, offset=mode[1], limit=mode[2], wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys, - defn=definition, sorts=mode[3] if len(mode) > 3 else None) + defn=definition, sorts=mode[3] if len(mode) > 3 else None, serialize=False) if isinstance(extra, dict): extra["total"] = total else: rows, stamp, total = assistant_mirror.read_rows( session, database, cur, spec, fields, wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys, - defn=definition) + defn=definition, serialize=False) except assistant_mirror.MirrorRefusal as exc: _assistant_refuse(exc.status, exc.code, exc.message) raise # unreachable; see above @@ -717,7 +717,8 @@ def _assistant_mirror_groups(session, database, cur, spec, fields, specs, option f'"a{i}"' for i in range(len(specs))), params=tuple(params), order_by=", ".join(f'"g{i}"' for i in range(len(group_exprs))), offset=options["offset"], limit=options["limit"], cur=cur) - groups = [{"keys": {group["field"]: mirror._json_value(rot, row[i]) + by_key = {field["key"]: field for field in fields} + groups = [{"keys": {group["field"]: mirror.field_value(rot, by_key[group["field"]], row[i]) for i, group in enumerate(options["groupBy"])}, "aggregates": [{**item, "value": mirror._json_value( rot, row[len(group_exprs) + i])} for i, item in enumerate(specs)]} @@ -808,8 +809,34 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity" +def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False): + """Separate one opaque identity equality from visible field operands.""" + if database not in _ASSISTANT_BUILT_INS: + if identity_pid is not None: + _assistant_refuse(400, "assistant_bad_filter", "Opaque identity reads require a built-in database.") + return filters, identity_pid + if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"} + and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list) + and len(filters["nodes"]) == 1): + leaf = filters["nodes"][0] + if (isinstance(leaf, dict) and leaf.get("colId") == "pid" + and leaf.get("op") == "eq" and "value" in leaf + and not set(leaf) - {"id", "colId", "op", "value"}): + value = leaf["value"] + if value is None: + _assistant_refuse(400, "assistant_bad_filter", "Use one non-null opaque record identity.") + if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid): + _assistant_refuse(400, "assistant_bad_filter", "Opaque identity pins must agree.") + identity_pid, filters = value, None + if identity_pid is not None and (type(identity_pid) not in (int, str) + or isinstance(identity_pid, str) + and (not identity_pid.strip() or len(identity_pid) > 200)): + _assistant_refuse(400, "assistant_bad_filter", "Use one opaque integer or string record identity.") + return filters, identity_pid + + def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None, - also_as=()): + also_as=(), identity_pid=None): """Return a detached, permissioned snapshot for one Assistant tool call. Signature (the C8 contract for E): @@ -836,11 +863,17 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None, `core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and unsupported operands before a row is read. The returned dictionaries are deep copies, never a live cache/store object, cursor, connector, route, provider, or credential. + `identity_pid` intersects built-in records by opaque identity after every permission wall. + Only a verified browser session may express that pin through a single pid equality filter. """ import core.perm_scope as perm_scope from harness import filter_eval key = str(database or "").strip() + original_filters = filters + filters, identity_pid = _assistant_identity_request( + key, filters, identity_pid, + trusted_filters=(session.claims or {}).get("external_via") == "session") members = tuple(also_as) if members: # C15 members belong to this session's tenant. A record from another tenant @@ -951,6 +984,15 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None, else: _assistant_refuse(404, "assistant_unknown_database", "that database is not available") + evaluation_fields = canonical_fields + if source_kind == "mirror_user_table": + import assistant_mirror + # Preserve raw blanks through both row predicates, then encode the response once. + rows = assistant_mirror._typed_rows(None, canonical_fields, rows, preserve_blank=True) + # Numeric flag comparisons accept Boolean operands without changing raw blank checks. + evaluation_fields = [dict(field, type="int") if field.get("type") == "checkbox" else field + for field in canonical_fields] + # Permanent permissions run against the full source contract. A permission filter may name # a field that is intentionally hidden from the returned schema; resolving it after the field # projection would make that filter unanswerable and (depending on the evaluator) could widen. @@ -1003,7 +1045,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None, # C16: with no grant there is no row wall, but a declared recipient list still binds # every reader, administrators included (S68-T57). permitted_rows = (perm_scope.assistant_apply_row_scope( - rows, session.user, key, canonical_fields, st=getattr(session, "runtime", None)) + rows, session.user, key, evaluation_fields, st=getattr(session, "runtime", None)) if grant is not None else perm_scope.apply_audience_scope( rows, session.user, key, canonical_fields, st=getattr(session, "runtime", None), defn=_audience_defn)) @@ -1016,12 +1058,12 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None, if members: hidden = set(access["hidden"]) member_rows = builtin_scope.get("rows", permitted_rows) - member_fields = builtin_scope.get("fields", canonical_fields) + member_fields = builtin_scope.get("fields", evaluation_fields) for member in members: hidden.update(perm_scope.hidden_keys(member, key, member_fields, st=session.runtime)) if mode not in ("schema", "query_schema"): member_rows = perm_scope.apply_row_scope( - member_rows, member, key, builtin_scope.get("row_fields", canonical_fields), + member_rows, member, key, builtin_scope.get("row_fields", evaluation_fields), st=session.runtime) if builtin_scope and mode not in ("schema", "query_schema"): allowed_pids = {row.get("pid") for row in member_rows} @@ -1039,15 +1081,24 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None, except ValueError as exc: _assistant_refuse(400, "assistant_bad_filter", str(exc)) if requested_filter is not None: + visible_keys = {field["key"] for field in visible} + evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys] + permitted_rows = [row for row in permitted_rows + if filter_eval.permits(requested_filter, row, evaluation_visible)] + if identity_pid is not None: permitted_rows = [row for row in permitted_rows - if filter_eval.permits(requested_filter, row, visible)] + if row.get("pid") is not None and str(row["pid"]) == str(identity_pid)] records = [] + checkbox_fields = ({field["key"]: field for field in visible if field.get("type") == "checkbox"} + if source_kind == "mirror_user_table" else {}) for row in permitted_rows: if not isinstance(row, dict) or row.get("pid") is None: continue records.append({"pid": copy.deepcopy(row["pid"]), - **{field: copy.deepcopy(row.get(field)) for field in selected_keys}}) + **{field: copy.deepcopy(assistant_mirror.field_value( + None, checkbox_fields[field], row.get(field)) if field in checkbox_fields + else row.get(field)) for field in selected_keys}}) out = { "database": key, @@ -1062,7 +1113,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None, set(hidden) | ({field.get("key") for field in canonical_fields} - {field.get("key") for field in visible})), definition=_audience_defn, members=members), - "filters": copy.deepcopy(requested_filter), + "filters": copy.deepcopy(original_filters if identity_pid is not None else requested_filter), "records": records, } if "aggregates" in extra: @@ -1597,6 +1648,15 @@ def require_session(request: Request, response: Response) -> Session: user = _user_for(claims) if not user: raise err(401, "invalid_session", "Your session ended. Sign in again.") + identity = {_TENANT_HEADER: str(claims["t"]), + _PRINCIPAL_HEADER: _principal(claims["t"], user["username"])} + if request.method not in {"GET", "HEAD", "OPTIONS"} and any( + header in request.headers and request.headers[header] != value + for header, value in identity.items()): + refused = err(409, "mutation_identity_changed", + "Your account or workspace changed. Reload before retrying this edit.") + refused.headers = identity + raise refused try: rt = runtime.get_runtime(claims["t"]) except KeyError: diff --git a/api/external_openapi.py b/api/external_openapi.py index 2b07983798c663f7be94d198a459ab8c3378753b..c62743f1e80d77b08e41c3dffd7b3f6a95acd200 100644 --- a/api/external_openapi.py +++ b/api/external_openapi.py @@ -6,7 +6,7 @@ import re from copy import copy, deepcopy -# Checkbox, structured and linked cells use the grid's string wire representation. +# Authored checkboxes and structured cells retain the grid's string representation. _JSON_TYPE = { "text": "string", "status": "string", "select": "string", "user": "string", "multiselect": "string", "phone": "string", "email": "string", "url": "string", @@ -40,7 +40,8 @@ def _record_name(database): def _field_schema(field, source_kind): if field.get("type") == "multiselect": return {"type": ["string", "array", "null"], "items": {"type": "string"}} - kind = _JSON_TYPE.get(field.get("type"), "string") + kind = ("boolean" if field.get("type") == "checkbox" and source_kind == "mirror_user_table" + else _JSON_TYPE.get(field.get("type"), "string")) types = [kind] # Materialised table writes retain numeric cells as strings; reads preserve those bytes. if kind in {"number", "integer"}: @@ -119,7 +120,8 @@ def _schemas(databases, records): "QueryRequest": _object({ "fields": _array({"type": "string"}, maxItems=200), "filters": _ref("Filter"), "sorts": _array(_ref("Sort"), maxItems=8), - "limit": {"type": "integer", "minimum": 1, "maximum": 500, "default": 100}, + "limit": {"type": "integer", "minimum": 1, "default": 100, + "description": "Each response returns at most 500 rows. Follow next_cursor to read all requested rows."}, "cursor": {"type": ["string", "null"], "maxLength": 768}, "aggregates": _array(_ref("AggregateRequest"), maxItems=16), "groupBy": _array(_ref("GroupBy"), minItems=1, maxItems=4), diff --git a/api/intelligence_store.py b/api/intelligence_store.py index 9845a72138cf1dd45283fc9fe44a8673c1cb5e5b..bdfdf08aaf42a72b1a2550315f175b79e3bb1b98 100644 --- a/api/intelligence_store.py +++ b/api/intelligence_store.py @@ -86,6 +86,8 @@ def normalize_path(value, *, folder=False): def _assert_authored_path(path): + if _database_note_path(path): + raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) import wiki_log if wiki_log.reserved(path): raise IntelligenceError("wiki_generated", "The system maintains wiki/log/ and wiki/index/.", 403) @@ -275,8 +277,7 @@ def _files(session, *, include_generated=False, wiki_only=False, raw=None): _check_file_capacity(session, raw) import wiki_log files = _project_templates(session, raw) - if not (include_generated or wiki_only): - files = _project_database_notes(session, files) + files = {path: record for path, record in files.items() if not _database_note_path(path)} files = {path: record for path, record in files.items() if not wiki_log.reserved(path)} files = {path: record for path, record in files.items() if not wiki_log.purpose_file(path) or wiki_log.purpose_record(session, path, files) is not None} @@ -294,140 +295,6 @@ def _database_note_path(path): return path.casefold().startswith(DATABASE_NOTES_PREFIX) -def _database_note_definition(session, key): - """Resolve a note's title and permission key from current database definitions.""" - from core import registry, user_tables - - if key.startswith("ut_"): - definition = user_tables.all_defs(st=session.runtime).get(key) - if definition: - return {"key": key, "label": definition["label"], "owner": definition.get("createdBy") or "role:admin"} - for topic, definition in registry.governable_modules().items(): - # Registry grids store notes under their scope, such as customer, rather than their module key. - if key in (topic, definition["key"]): - return {**definition, "owner": "role:admin"} - return None - - -def _database_note_can_edit(session, key): - """Use the grid's Database-note edit authority after current reader admission.""" - from core import external_access, user_tables - - definition = _database_note_definition(session, key) - if not definition or not can_read(session, {"reads": [definition["key"]]}): - return False - if not external_access.may_write(credential_scope(session)[0]): - return False - return (user_tables.may_edit_database(key, session.uname, is_admin=session.admin, st=session.runtime) - if key.startswith("ut_") else bool(session.admin)) - - -def _database_note_record(session, key, *, stored=None, with_body=False): - """Project canonical note metadata without storing a Brain version or a body copy.""" - from core import user_tables - - path = DATABASE_NOTES_PREFIX + key + ".md" - if normalize_path(path) != path or "/" in key or not key: - raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) - definition = _database_note_definition(session, key) - if not definition or not can_read(session, {"reads": [definition["key"]]}): - raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) - note = user_tables.shared_note("database", key, st=session.runtime) - if not note or note.get("deleted"): - raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) - meta = {"title": note.get("title") or definition["label"], "section": "database-notes", "owner": definition["owner"], - "description": f"Shared note for {definition['label']}.", "reads": [definition["key"]], - "approvers": [], "approvals_needed": 1, "tools": [], "databaseKey": key} - text = note.get("body") or "" - entry = {"metadata": meta, "version": int(note.get("version") or 0), - "bytes": len(text.encode("utf-8")), "author": note.get("author") or "", - "authorKind": "user", "created": note.get("updated")} - if with_body: - entry["body"] = text - revisions = note.get("revisions") or [] - return {"path": path, "databaseKey": key, "canEdit": _database_note_can_edit(session, key), - "revisionHistory": "database-note", "versions": [entry], - "created": revisions[0].get("at") if revisions else entry["created"], - "proposals": (stored or {}).get("proposals") or {}, - **({"note": note} if with_body else {})} - - -def _project_database_notes(session, raw): - """Reserve the note namespace and expose only current, readable Database rows.""" - from core import user_tables - - files = {path: record for path, record in raw.items() if not _database_note_path(path)} - notes = _document(session.runtime.get(user_tables.SHARED_NOTES_KEY)) - for subject in notes: - if not isinstance(subject, str) or not subject.startswith("database:"): - continue - key = subject[len("database:"):] - path = DATABASE_NOTES_PREFIX + key + ".md" - try: - files[path] = _database_note_record(session, key, stored=raw.get(path)) - except IntelligenceError as exc: - if exc.code not in ("no_file", "bad_path"): - raise - return files - - -def _database_note_file(session, path, *, version=0): - if not path.startswith(DATABASE_NOTES_PREFIX) or not path.endswith(".md"): - raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) - record = _database_note_record(session, path[len(DATABASE_NOTES_PREFIX):-3], with_body=True) - note = record["note"] - current = _head(record) - entry = current - if version and version != current["version"]: - revision = next((row for row in note.get("revisions") or [] if row.get("version") == version), None) - if revision is None: - raise IntelligenceError("no_version", "This file has no such version.", 404) - entry = {**current, "body": revision["body"], "version": version, - "author": revision["author"], "created": revision["at"], - "bytes": len(revision["body"].encode("utf-8"))} - history = [{"version": row["version"], "author": row["author"], "authorKind": "user", - "created": row["at"], "bytes": len(row["body"].encode("utf-8"))} - for row in reversed(note.get("revisions") or [])] - return {**_row(record, entry, session=session), "body": entry["body"], "atVersion": entry["version"], - "latestVersion": current["version"], "history": history, "versions": len(history), "limits": limits()} - - -def _put_database_note(session, key, body, actor, base_version, base_updated, *, title=None, deleted=False): - """Let the canonical writer enforce admission and its exact base on every atomic attempt.""" - from core import user_tables - - class CheckedRuntime: - def __getattr__(self, name): - return getattr(session.runtime, name) - - def update(self, bucket, callback, **kwargs): - if bucket != user_tables.SHARED_NOTES_KEY: - raise IntelligenceError("database_note_reserved", "Database notes use their shared store.", 403) - - def checked(raw): - if not _database_note_can_edit(session, key): - raise IntelligenceError("note_read_only", "Your Database role does not allow note edits.", 403) - prior = (_document(raw).get("database:" + key) or {}) - if int(prior.get("version") or 0) != base_version or prior.get("updated") != base_updated: - raise IntelligenceError("version_conflict", "The note changed. Reload it before saving.", 409) - updated = callback(raw) - note = updated["database:" + key] - if title is not None or prior.get("title"): - note["title"] = title if title is not None else prior["title"] - if deleted: - note["deleted"] = True - return updated - - return session.runtime.update(bucket, checked, **kwargs) - - try: - return user_tables.put_shared_note("database", key, body, actor, st=CheckedRuntime()) - except IntelligenceError: - raise - except Exception: - raise IntelligenceError("store_unavailable", "The note change could not be confirmed. Reload and try again.", 503) from None - - @lru_cache(maxsize=1) def _template_catalog(): """Cache only shipped text, never tenant connections, grants, schemas or overrides.""" @@ -753,7 +620,7 @@ class _FolderReadRuntime: if name not in self._documents: if name == STORE_KEY: reader = getattr(self._runtime, "get_many", None) - values = reader([STORE_KEY, TEMPLATE_USAGE_KEY, "keychain", "shared_notes"]) if callable(reader) else None + values = reader([STORE_KEY, TEMPLATE_USAGE_KEY, "keychain"]) if callable(reader) else None if values is not None: self._documents.update(values) if name not in self._documents: @@ -1012,12 +879,7 @@ def linked_files(ctx, database, field=None): entry = _head(record) if not entry or not can_read(session, entry["metadata"]): continue - if _database_note_path(path): - keys = [record["databaseKey"], *entry["metadata"]["reads"]] - if field is not None: - keys += _LINK_TOKENS.findall(_database_note_file(session, path)["body"]) - else: - keys = _link_index(record)["keys"] + keys = _link_index(record)["keys"] if database in keys and (field is None or field in keys): meta = entry["metadata"] rows.append({"path": path, "title": meta["title"], "section": meta["section"], @@ -1064,7 +926,7 @@ def read_file(session, path, *, version=0, record_use=True): if type(version) is not int or version < 0: raise IntelligenceError("bad_version", "Use a positive version number, or zero for the latest version.") if _database_note_path(path): - return _database_note_file(session, path, version=version) + raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) def readable(raw): record = _project_templates(session, _document(raw)).get(path) @@ -1202,12 +1064,7 @@ def _proposal_readable(session, record, proposal): or not wiki_log.purpose_object(session, proposal["metadata"])): return False if _database_note_path(record.get("path", "")): - try: - _database_note_file(session, record["path"]) - except IntelligenceError as exc: - if exc.code == "no_file": - return False - raise + return False template = _template_catalog().get(record.get("path")) if template and record["path"] not in _project_templates(session, {record["path"]: record}): return False @@ -1244,13 +1101,6 @@ def _recipients(session, policy, *, record, proposal): if wiki_log.purpose_file(record.get("path", "")): obj = wiki_log.purpose_object(recipient, proposal["metadata"]) matches = bool(obj and obj["owner"] == name) - if _database_note_path(record.get("path", "")): - try: - matches = _database_note_can_edit(recipient, record["path"][len(DATABASE_NOTES_PREFIX):-3]) - except HTTPException as exc: - if exc.status_code != 403: - raise - matches = False if matches: try: if not _proposal_readable(recipient, record, proposal): @@ -1284,10 +1134,7 @@ def _public_proposal(session, proposal, record): row["approvedBy"] = [name for name in proposal["approvedBy"] if name in eligible] row["recipients"] = eligible row["canDecide"] = (not trusted_identity(session)[1].get("agent") and session.uname in eligible - and (proposal["status"] == "pending" - or (_database_note_path(record.get("path", "")) and proposal["status"] == "approving")) - and (session.uname not in row["approvedBy"] - or (_database_note_path(record.get("path", "")) and proposal["status"] == "approving"))) + and proposal["status"] == "pending" and session.uname not in row["approvedBy"]) return row @@ -1329,7 +1176,7 @@ def _notify_proposal(session, proposal): send = getattr(module, "send_request", None) title = f"Review {proposal['title']}" body = f"{proposal['path']}\nProposal {proposal['id']}\n\n{proposal['diff']}" - if "personal_scope" in proposal["metadata"] or _database_note_path(proposal["path"]): + if "personal_scope" in proposal["metadata"]: # Inbox copies cannot keep private content after its source access is revoked. if "personal_scope" in proposal["metadata"]: title = "Review a personal change" @@ -1397,11 +1244,7 @@ def file_history(session, path, *, version=0): if type(version) is not int or version < 0: raise IntelligenceError("bad_version", "Choose a version number.") if _database_note_path(path): - file = _database_note_file(session, path, version=version) - return {"path": path, "history": file["history"], "trimmed": 0, - "latestVersion": file["latestVersion"], "selectedVersion": file["atVersion"], - "previousVersion": None, "diff": None, "canRestore": False, - "databaseKey": file["databaseKey"], "revisionHistory": "database-note", "limits": limits()} + raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) documents = _files(session, include_generated=wiki_log.reserved(path), wiki_only=path.startswith(WIKI_PREFIX)) if wiki_log.purpose_file(path): purpose = wiki_log.purpose_record(session, path, _project_templates(session, _document(session.runtime.get(STORE_KEY)))) @@ -1437,9 +1280,6 @@ def restore_file(session, path, version, *, expected_version): _authorize(session) path = normalize_path(path) _assert_authored_path(path) - if _database_note_path(path): - _database_note_file(session, path) - raise IntelligenceError("database_note_reserved", "Edit the Database note through its shared note history.", 403) if type(version) is not int or version < 1 or type(expected_version) is not int or expected_version < 1: raise IntelligenceError("bad_version", "Choose a saved version and the current file version.") identity = trusted_identity(session)[1] @@ -1548,158 +1388,6 @@ def insights(session, *, now=None): {"key": "sheets", "label": "Sheets"}, {"key": "apps", "label": "Apps"}]} -def _write_database_note(session, identity, path, body, *, expected_version=None): - """Save a person's exact note text or store an agent proposal without a Brain head.""" - from core import external_access - - current = _database_note_file(session, path) - key = current["databaseKey"] - if not isinstance(body, str): - raise IntelligenceError("no_body", "A Database note needs text.") - if len(body) > 100000 or len(body.encode("utf-8")) > MAX_BODY_BYTES: - raise IntelligenceError("body_too_long", "A Database note can be at most 100,000 characters and 128 KB.", 413) - if expected_version is not None and expected_version != current["version"]: - raise IntelligenceError("version_conflict", "The note changed. Reload it before saving.", 409) - if not identity.get("agent"): - note = _put_database_note(session, key, body, identity["author"], current["version"], current["updated"]) - _log_write(session, path, identity, current["version"], note["version"]) - return _database_note_file(session, path) - if not external_access.may_write(credential_scope(session)[0]): - raise IntelligenceError("key_read_only", "This credential is read only.", 403) - from routes_agent_harness import MAX_PENDING_PROPOSALS, MAX_PROPOSALS_PER_AGENT - - proposal_id = "ip_" + write_execution.new_id()[:24] - saved = {} - - def propose(raw): - saved.clear() - documents = dict(_document(raw)) - old = documents.get(path) or {} - record = _database_note_record(session, key, stored=old, with_body=True) - head = _head(record) - if expected_version is not None and head["version"] != expected_version: - raise IntelligenceError("version_conflict", "The note changed. Reload it before saving.", 409) - proposals = dict(old.get("proposals") or {}) - if sum(row["status"] in ("pending", "approving") for row in proposals.values()) >= MAX_PENDING_PROPOSALS: - raise IntelligenceError("too_many_proposals", "Review the pending changes before adding another.", 409) - if len(proposals) >= MAX_PROPOSALS_PER_AGENT: - terminal = next((name for name, row in proposals.items() if row["status"] in ("approved", "rejected", "stale")), None) - if terminal: - del proposals[terminal] - proposal = _new_proposal(session, record, body, head["metadata"], identity, proposal_id) - proposals[proposal_id] = proposal - # This envelope holds proposals only. Canonical text and revisions stay in shared_notes. - documents[path] = {"path": path, "databaseKey": key, "versions": [], "proposals": proposals} - saved.update(proposal) - return documents - - _update(session, propose) - try: - delivery = _notify_proposal(session, saved) - except Exception: - delivery = "failed" - record = _files(session).get(path, {}) - if not record or not _proposal_readable(session, record, saved): - raise IntelligenceError("no_proposal", "This proposal is unavailable.", 404) - return {"proposal": _public_proposal(session, saved, record), "delivery": delivery} - - -def _decide_database_note(session, path, proposal_id, decision): - """Claim a decision, write through the canonical note's atomic guard, then retain its receipt.""" - key = path[len(DATABASE_NOTES_PREFIX):-3] - saved = {} - - def claim(raw): - saved.clear() - documents = dict(_document(raw)) - old = documents.get(path) or {} - proposal = deepcopy((old.get("proposals") or {}).get(proposal_id)) - if not proposal or not _proposal_readable(session, old, proposal): - raise IntelligenceError("no_proposal", "This proposal is unavailable.", 404) - eligible = _recipients(session, proposal["policy"], record=old, proposal=proposal) - if session.uname not in eligible: - raise IntelligenceError("not_approver", "Your Database role does not allow note approval.", 403) - record = _database_note_record(session, key, with_body=True) - head = _head(record) - if proposal["status"] == "approving" and head["version"] == proposal["baseVersion"] + 1 \ - and head["body"] == proposal["body"] and head["author"] == proposal["author"][:120]: - # Recover a canonical write whose decision receipt was interrupted. - proposal.update(status="approved", approvedVersion=head["version"], approvedAt=head["created"]) - if proposal["status"] == "approved" and decision == "approve": - pass - elif proposal["status"] == "rejected" and decision == "reject": - pass - elif proposal["status"] in ("pending", "approving") and _stale(record, proposal): - proposal["status"] = "stale" - saved["stale"] = True - elif proposal["status"] == "stale": - saved["stale"] = True - elif proposal["status"] not in ("pending", "approving"): - raise IntelligenceError("proposal_not_pending", "This proposal already has a decision.", 409) - elif decision == "reject": - proposal.update(status="rejected", rejectedBy=session.uname) - else: - proposal["approvedBy"] = list(dict.fromkeys([ - *[name for name in proposal["approvedBy"] if name in eligible], session.uname])) - proposal["status"] = "approving" - saved["write"] = True - documents[path] = {"path": path, "databaseKey": key, "versions": [], - "proposals": {**(old.get("proposals") or {}), proposal_id: proposal}} - saved["proposal"] = proposal - return documents - - _update(session, claim) - if saved.get("stale"): - raise IntelligenceError("proposal_stale", "The note changed. Ask the agent for a new proposal.", 409) - proposal = saved["proposal"] - if saved.get("write"): - wrote = True - try: - note = _put_database_note(session, key, proposal["body"], proposal["author"], - proposal["baseVersion"], proposal["baseCreated"]) - except IntelligenceError as exc: - if exc.code != "version_conflict": - raise - accepted = ((_document(session.runtime.get(STORE_KEY)).get(path) or {}).get("proposals") or {}).get(proposal_id) or {} - current = _database_note_record(session, key, with_body=True) - head = _head(current) - if accepted.get("status") == "approved": - return {"proposal": _public_proposal(session, accepted, current), "file": _database_note_file(session, path)} - if (head["version"] == proposal["baseVersion"] + 1 and head["body"] == proposal["body"] - and head["author"] == proposal["author"][:120]): - note = current["note"] - wrote = False - else: - - def stale(raw): - documents = deepcopy(_document(raw)) - documents[path]["proposals"][proposal_id]["status"] = "stale" - return documents - - _update(session, stale) - raise IntelligenceError("proposal_stale", "The note changed. Ask the agent for a new proposal.", 409) from None - - def receipt(raw): - documents = deepcopy(_document(raw)) - proposals = documents[path]["proposals"] - proposals[proposal_id].update(status="approved", approvedVersion=note["version"], approvedAt=note["updated"]) - for name, other in proposals.items(): - if name != proposal_id and other["status"] in ("pending", "approving"): - other["status"] = "stale" - return documents - - _update(session, receipt) - identity = {name: proposal[name] for name in ("author", "authorKind", "agent", "onBehalfOf")} - if wrote: - _log_write(session, path, identity, proposal["baseVersion"], note["version"]) - record = _files(session).get(path, {}) - proposal = (record.get("proposals") or {}).get(proposal_id) - if not proposal or not _proposal_readable(session, record, proposal): - raise IntelligenceError("no_proposal", "This proposal is unavailable.", 404) - return {"proposal": _public_proposal(session, proposal, record), - **({"file": _database_note_file(session, path)} if proposal["status"] == "approved" else {})} - - def _purpose_metadata(session, path, metadata, documents=None): """Keep purpose ownership and authorization bound to the current full object identity.""" import wiki_log @@ -1808,8 +1496,6 @@ def write_file(caller, path, body, *, author_kind="user", expected_version=None, raise IntelligenceError("bad_author_kind", "authorKind must be user or agent.") if expected_version is not None and (type(expected_version) is not int or expected_version < 0): raise IntelligenceError("bad_version", "expectedVersion must be a whole number of zero or more.") - if _database_note_path(path): - return _write_database_note(session, identity, path, body, expected_version=expected_version) text, metadata = prepare_markdown(body, session.uname) import wiki_log if wiki_log.purpose_file(path): @@ -1942,10 +1628,6 @@ def decide_proposal(session, proposal_id, decision): raise IntelligenceError("person_required", "A person must review this change.", 403) if decision not in ("approve", "reject"): raise IntelligenceError("bad_decision", "Choose Approve or Decline.") - for path, record in _document(session.runtime.get(STORE_KEY)).items(): - if _database_note_path(path) and proposal_id in (record.get("proposals") or {}): - return _decide_database_note(session, path, proposal_id, decision) - saved = {} def decide(raw): saved.clear() @@ -2024,18 +1706,6 @@ def rename_file(caller, path, title, *, expected_version): raise IntelligenceError("bad_title", "Use a title of 1 to 200 characters.") if type(expected_version) is not int or expected_version < 1: raise IntelligenceError("bad_version", "Reload the file before trying again.") - if _database_note_path(path): - if identity.get("agent"): - raise IntelligenceError("approval_required", "An agent cannot rename a shared note.", 403) - record = _database_note_record(session, path[len(DATABASE_NOTES_PREFIX):-3], with_body=True) - if record["path"] != path: - raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) - entry = _head(record) - _put_database_note(session, record["databaseKey"], entry["body"], session.uname, - expected_version, entry["created"], title=title.strip()) - saved = _database_note_file(session, path) - _log_write(session, path, identity, expected_version, saved["version"]) - return saved saved = {} def rename(raw): @@ -2065,25 +1735,13 @@ def rename_file(caller, path, title, *, expected_version): def delete_file(caller, path, *, expected_version=None): - """Delete an editable workspace file or clear its canonical database note.""" + """Delete an editable workspace file.""" session, identity = trusted_identity(caller) _authorize(session) path = normalize_path(path) _assert_authored_path(path) if expected_version is not None and (type(expected_version) is not int or expected_version < 1): raise IntelligenceError("bad_version", "Reload the file before trying again.") - if _database_note_path(path): - if identity.get("agent"): - raise IntelligenceError("approval_required", "An agent cannot delete a shared note.", 403) - record = _database_note_record(session, path[len(DATABASE_NOTES_PREFIX):-3], with_body=True) - if record["path"] != path: - raise IntelligenceError("no_file", "This workspace has no file at that path.", 404) - entry = _head(record) - _put_database_note(session, record["databaseKey"], "", session.uname, - expected_version if expected_version is not None else entry["version"], - entry["created"], deleted=True) - _log_write(session, path, identity, entry["version"], None) - return {"deleted": path} removed = {} def remove(raw): diff --git a/api/main.py b/api/main.py index f12191b9d89abf89bfd42f4e2d293dde299ee1c6..d4fc10c1159cf53ee59fe3ba8f12516793107ce7 100644 --- a/api/main.py +++ b/api/main.py @@ -38,7 +38,7 @@ from dotenv import load_dotenv # noqa: E402 load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed import process_role # noqa: E402 (before any DuckDB connect: the web role sets its memory cap) -from fastapi import Body, Depends, FastAPI, Request # noqa: E402 +from fastapi import Body, Depends, FastAPI, Request, Response # noqa: E402 from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 from fastapi.responses import JSONResponse # noqa: E402 from fastapi.staticfiles import StaticFiles # noqa: E402 @@ -46,13 +46,11 @@ from starlette.exceptions import HTTPException as StarletteHTTPException # noqa import aios_session # noqa: E402 import routes_admin # noqa: E402 -import routes_competitor_watch # noqa: E402 (S78-T16 / S78-T17 competitor watch) import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) import routes_auth # noqa: E402 import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) -import routes_notes # noqa: E402 (wave 47 instr 19 / W47-T16 — shared-note read state, lane C's router) import routes_charts # noqa: E402 (wave 47 instr 18 / W47-T30 / R7 — saved charts as records, lane H's router) import routes_sheet # noqa: E402 (wave 49 instr 1 / W49-T02 / C1 — the Sheet workbook doors, lane A's router) import routes_customers # noqa: E402 @@ -238,13 +236,13 @@ async def _unexpected_error(request: Request, exc: Exception): status_code=500) -from core.odoo import OdooPoolTimeout, OdooRequestTimeout +from core.odoo import OdooPoolTimeout, OdooRequestTimeout, OdooBreakerOpen from core.store_pg import SharedReadTimeout, PostgresOperationTimeout from core.request_timing import RequestDeadlineExceeded from request_timing import DeadlineCapacityExceeded from duckdb import OutOfMemoryException # Each busy cause needs its own handler: the catch-all `Exception` one answers, then re-raises to uvicorn. -for _resource_error in (OdooPoolTimeout, OdooRequestTimeout, SharedReadTimeout, PostgresOperationTimeout, +for _resource_error in (OdooPoolTimeout, OdooRequestTimeout, OdooBreakerOpen, SharedReadTimeout, PostgresOperationTimeout, RequestDeadlineExceeded, DeadlineCapacityExceeded, OutOfMemoryException): app.add_exception_handler(_resource_error, _resource_unavailable) try: @@ -410,24 +408,59 @@ def _mirror_answers(): return False -@app.get("/api/v1/ready") -def readiness_check(): - """C9: 200 only when the mirror is installed, the pools are open and the first reads answered. +def _tenant_readiness(rt): + """Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo.""" + checks = {"mirror": False, "pools": False, "first_reads": False} + detail = {"mirror": "The tenant mirror check did not answer.", + "pools": "The tenant store did not answer.", + "first_reads": "The tenant's first read did not answer."} + stage = "mirror" + try: + if rt.odoo_source() is not None: + rt.assert_datastore_matches() + return process_role.readiness(mirror_live=_mirror_answers()) + checks["mirror"] = True + stage = "pools" + if not rt.available(): + raise RuntimeError("store unavailable") + checks["pools"] = True + stage = "first_reads" + # HF display reads hide outages, so readiness uses the tenant's strict reader. + rt._store()._read_strict(rt.store_key("sync_settings")) + checks["first_reads"] = True + except Exception as exc: # noqa: BLE001 + detail[stage] += f" ({type(exc).__name__})" + failing = [name for name in checks if not checks[name]] + return {"ready": not failing, "checks": checks, "failing": failing, + "detail": {name: detail[name] for name in failing}} - Unauthenticated, because the deploy switch polls it before anyone signs in. It carries the - version on purpose (C9), unlike /api/health. - """ - report = process_role.readiness(mirror_live=_mirror_answers()) - body = {"ready": report["ready"], "version": os.environ.get("AIOS_VERSION") or "unknown", - "checks": report["checks"]} - headers = {"Cache-Control": "no-store"} + +@app.get("/api/v1/ready") +def readiness_check(request: Request, response: Response): + """Report tenant readiness publicly and reveal the version only after session verification.""" + session = (deps_require_session(request, response) + if request.cookies.get(aios_session.COOKIE_NAME) else None) + try: + from harness import runtime as _ready_runtime + rt = (session.runtime if session else + _ready_runtime.get_runtime(_ready_runtime.tenant_for_store_path())) + report = _tenant_readiness(rt) + except Exception as exc: # noqa: BLE001 + report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"], + "detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}} + body = {"ready": report["ready"], "checks": report["checks"]} + if session is not None: + body["version"] = os.environ.get("AIOS_VERSION") or "unknown" + response.headers["Cache-Control"] = "no-store" if report["ready"]: - return JSONResponse(body, headers=headers) + response.status_code = 200 + return body body["failing"] = report["failing"] body["error"] = {"code": "not_ready", "message": "Not ready: " + ", ".join(report["failing"]) + ". " + " ".join(report["detail"].values())} - return JSONResponse(body, status_code=503, headers=headers) + response.status_code = 503 + return body # W59-T43: boot stays available while a cross-lane module is absent, but never silently. @@ -554,9 +587,7 @@ app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's lin # `main.py` can sit in exactly ONE lane's fence, so a router built in any other lane has nowhere to # be mounted from, and the line gets deferred to a session that no longer exists. Mounting empty # routers at plan time removes the failure mode instead of warning about it a sixth time. -# `routes_notes` is lane C's (W47-T16, the Notes unread badge); `routes_charts` is lane H's -# (W47-T30 / R7, saved charts as their own records). Neither lane may edit this file. -app.include_router(routes_notes.router) +# Saved charts mount before the static application. app.include_router(routes_charts.router) # ⭐ WAVE 49 (instruction 1 / C1, W49-T02) — THE SHEET WORKBOOK DOORS. Lane A owns both this file # and that router, so this is the one mount in this block that crosses no fence. It is still @@ -655,12 +686,10 @@ app.include_router(routes_tickets.router) app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) app.include_router(routes_external.router) import app_routes -app.include_router(routes_competitor_watch.router, # S78-T16 / S78-T17 competitor watch; held by R11 - dependencies=app_routes.held_dependencies("competitor_watch")) _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount. _mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors. _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount. -app_routes.mount_app_routers(app) +app_routes.mount_app_routers(app, ROUTER_MOUNT_FAILURES) # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) @@ -950,7 +979,7 @@ def _pull_meta(why, tenant_key=None): outcomes[key] = False try: rt = runtime.get_runtime(key) - if not _meta.token(rt, tenant_key=key): + if not _meta._connection_rows(rt, key, {"problems": []}): continue rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key, log=lambda *_a: None, insights_days=7) @@ -964,6 +993,65 @@ def _pull_meta(why, tenant_key=None): return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False) +def _drain_amazon_connection_revocations(rt): + """Finish scoped deletions on the resident writer before any Amazon restore or pull.""" + try: + pending = rt.get("amazon_connection_revocations", fresh=True) or {} + if not isinstance(pending, dict): + raise ValueError("Unreadable Amazon revocation metadata") + pending = {cid: row for cid, row in pending.items() + if isinstance(row, dict) and row.get("state") == "pending"} + if pending: + import process_role as _role + from harness import datastore as _ds + if (not _role.may_write() or rt.datastore_path.resolve() != _ds.DB_PATH.resolve() + or _ds.path_for(rt.key).resolve() != _ds.DB_PATH.resolve()): + raise RuntimeError("Amazon cleanup requires the tenant's resident writer") + import amazon_relational as _ar + failures = [] + for cid, row in pending.items(): + try: + result = _ar.purge_connection(rt, cid, entries_before=row.get("entries")) + if not isinstance(result, dict) or result.get("failed") or result.get("refused"): + raise RuntimeError("Amazon connection cleanup did not finish") + except Exception as exc: + failures.append(type(exc).__name__) + if failures: + raise RuntimeError("Amazon connection cleanup did not finish") + RESYNC_STATE.setdefault("amazonRevocationErrors", {}).pop(rt.key, None) + except Exception as exc: + message = "Amazon connection cleanup is pending. Retry on the tenant's resident sync worker." + RESYNC_STATE.setdefault("amazonRevocationErrors", {})[rt.key] = { + "type": type(exc).__name__, "message": message} + print(f"[aios-api] amazon connection cleanup pending ({rt.key}): {type(exc).__name__}") + raise _SyncNowRefused(message) from None + + +def _drain_pending_amazon_revocations(): + """Handle cleanup-only wakes without requiring a credential, definition or sync receipt.""" + import process_role as _role + if not _role.may_write(): + return + from harness import runtime, datastore + try: + tenants = runtime.known_tenants() + RESYNC_STATE.pop("amazonRevocationDiscoveryError", None) + except Exception as exc: + RESYNC_STATE["amazonRevocationDiscoveryError"] = { + "type": type(exc).__name__, "message": "Amazon cleanup tenants could not be read. Retry the sync worker."} + return + for key in tenants: + try: + rt = runtime.get_runtime(key) + if rt.key == key and rt.datastore_path.resolve() == datastore.DB_PATH.resolve(): + _drain_amazon_connection_revocations(rt) + except KeyError: + continue + except Exception as exc: + print(f"[aios-api] amazon cleanup wake unavailable ({key}): {type(exc).__name__}") + continue + + def _pull_amazon(why, vendor=True): """Produce Amazon rows in this container before projecting them, on the caller's thread. @@ -983,6 +1071,7 @@ def _pull_amazon(why, vendor=True): rt.datastore_path.resolve() != _ds.DB_PATH.resolve() or _ds.path_for(tenant_key).resolve() != _ds.DB_PATH.resolve()): raise RuntimeError("Amazon sync requires the requesting tenant's resident mirror.") + _drain_amazon_connection_revocations(rt) if not _amazon.credential(tenant_key, rt=rt): print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential " "for royal-imports; the connector is idle, not broken") @@ -1075,6 +1164,7 @@ def _restore_amazon_mirror_once(why, tenant_key): if (rt.key != tenant_key or rt.datastore_path.resolve() != _ds.path_for(tenant_key).resolve()): raise RuntimeError("Amazon restore requires the requesting tenant's runtime.") + _drain_amazon_connection_revocations(rt) if rt.download_bytes(_amazon.MIRROR_PREFIX + "manifest.json") is None: RESYNC_STATE.setdefault("amazonRestoreErrors", {}).pop(tenant_key, None) return True @@ -1172,6 +1262,20 @@ class _SyncNowRefused(Exception): def _sync_now_cause(cause): """Classify actual provider failures without returning vendor text or exception reprs.""" import re + if isinstance(cause, dict): + sentence = "The connector could not read an expected value in the sync response." + if cause.get("classifier") == "unreadable_value": + table = cause.get("table") + if isinstance(table, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", table): + sentence = f"Amazon {table}: unreadable value." + filename, line, function = (cause.get(key) for key in ("file", "line", "function")) + if (isinstance(filename, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}\.py", filename) + and type(line) is int and 0 < line < 10000000 + and isinstance(function, str) and re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_]{0,79}|<(?:lambda|listcomp|dictcomp|setcomp|genexpr|module)>", function)): + sentence += f" Raised at {filename}:{line}:{function}." + return "unreadable_value", sentence + return "unclassified", "The connector reported a failure. Inspect its sync report and retry." raw = str(cause) text = raw.lower() causes = ( @@ -1213,7 +1317,15 @@ def _sync_now_cause(cause): if isinstance(cause, TypeError): return "bad_arguments", "The connector failed because its provider call used incompatible arguments." if isinstance(cause, (KeyError, ValueError)): - return "unreadable_value", "The connector could not read an expected value in the sync response." + sentence = "The connector could not read an expected value in the sync response." + import traceback + frames = traceback.extract_tb(cause.__traceback__) + if frames: + frame = frames[-1] + filename = frame.filename.replace("\\", "/").rsplit("/", 1)[-1] + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}\.py", filename): + sentence += f" Raised at {filename}:{frame.lineno}." + return "unreadable_value", sentence if isinstance(cause, str): # amazon_store._sync_problem appends the application-authored fix LAST. The # exception message may itself contain delimiters; none of it is display text. @@ -1244,6 +1356,27 @@ def _sync_now_failure(cause): return _sync_now_cause(cause)[1] +def _amazon_sync_receipt(report): + """Project table windows and resync counts without vendor diagnostics or exception values.""" + import re + details = [] + for table, row in (report.get("tables") or {}).items(): + if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)): + continue + parts = [] + progress = row.get("resync") + if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0 + for k in ("landed", "left")): + parts.append(f"{progress['landed']} windows landed, {progress['left']} left") + window = row.get("window") + if isinstance(window, str): + # Window descriptions are authored by the loader, never copied from its diagnostics. + parts.append(" ".join(window.split())) + if parts: + details.append(f"{table}: " + "; ".join(parts)) + return " | ".join(details) + + def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None): """Record one classified failure; leave successful receipt text untouched.""" if ok: @@ -1255,6 +1388,8 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None): locations = " > ".join(f"{os.path.basename(frame.filename)}:{frame.lineno}:{frame.name}" for frame in traceback.extract_tb(cause.__traceback__)) print(f"[aios-api] sync failure location {type(cause).__name__}: {locations}") + if detail: + sentence += " " + detail suffix = f" (cause code: {classifier})" # finish_sync_request stores only 300 characters. Reserve the suffix before it clips. budget = 300 - len(suffix) @@ -1272,9 +1407,82 @@ def _run_sync_requests(outcome=None, *, during_odoo=False): project only what a desk wake actually changed. """ from core import connector_sync + _drain_pending_amazon_revocations() + + def progress_log(provider, tenant, receipt, rt): + """Publish only authored progress shapes, with one bounded detail per running receipt.""" + import re + from collections import deque + lines = deque(maxlen=2) + + def publish_progress(*parts, **_kwargs): + text = " ".join(str(part) for part in parts).strip() + if provider == "odoo": + match = re.fullmatch( + r"([a-z][a-z0-9_]*): (column backfill|column refresh|m2m) " + r"([a-z][a-z0-9_]*(?:\+[a-z][a-z0-9_]*)*) " + r"(starting at id|resuming at id|-> id) ([0-9,]+) " + r"\(([0-9,]+) rows(?: already done)?\)", text) + if match: + entity, kind, fields, step, last_id, count = match.groups() + safe = f"{entity}: {kind} {fields} {step} {last_id} ({count} rows)" + elif re.fullmatch(r"[a-z][a-z0-9_]*: backfill …[0-9,]+ rows \(id≤[0-9,]+\)", text): + safe = text + elif re.fullmatch(r"[a-z][a-z0-9_]*: backfill starting at id [0-9,]+ \([0-9,]+ rows\)", text): + safe = text + else: + return + else: + match = re.fullmatch( + r"([a-z][a-z0-9_]*): resync of ([0-9]+) stored windows(?: from [0-9TZ:+. -]+)?", text) + summary = re.fullmatch( + r"Amazon resync ([a-z][a-z0-9_]*): ([0-9]+) windows landed, ([0-9]+) left", text) + if match: + safe = f"Amazon {match[1]}: resync of {match[2]} stored windows" + elif summary: + safe = f"Amazon {summary[1]}: {summary[2]} windows landed, {summary[3]} left" + else: + return + lines.append(safe[:220]) + detail = " | ".join(lines)[-300:] + print(f"[aios-api] {provider} sync desk: {safe[:220]}", flush=True) + # The desk owns lifecycle state. Progress must not finish or replace another claim. + with connector_sync._SYNC_REQUEST_LOCK: + current = next((row for row in connector_sync._SYNC_REQUESTS.values() + if row["provider"] == provider and row["tenant"] == tenant + and row["state"] == "running" + and row.get("id") == receipt.get("id") + and row.get("startedAt") == receipt.get("startedAt")), None) + if current is None: + return + current["detail"] = detail + if current.get("tables") is not None: + identity, token = current["id"], current["claimToken"] + + def publish(document): + doc = connector_sync._sync_request_document(document) + row = doc["requests"].get(identity) + if (row and row.get("tenant") == tenant and row.get("state") == "running" + and row.get("claimToken") == token): + row["detail"] = detail + return doc + + rt.update(connector_sync.SYNC_REQUESTS_KEY, publish, flush="sync") + # Ordinary requests cross processes through the existing tenant-bound desk publisher. + connector_sync.poll_remote_requests(rt) + + def log(*parts, **kwargs): + try: + publish_progress(*parts, **kwargs) + except Exception as exc: + print(f"[aios-api] sync progress unavailable ({type(exc).__name__})", flush=True) + + return log, lines + odoo_requested = False - pending = globals().setdefault("_SYNC_CLAIMED", []) - pending.extend(connector_sync.take_sync_requests()) + # Claim only work this turn executes. A later provider remains queued with an honest wait. + pending = connector_sync.take_sync_requests(limit=1, exclude_providers=("odoo",) if during_odoo else (), + refresh_remote=True) while True: receipt = next((row for row in pending if not during_odoo or row["provider"] != "odoo"), None) @@ -1287,6 +1495,8 @@ def _run_sync_requests(outcome=None, *, during_odoo=False): ok, detail = False, "" first_load_progress, continuing = None, False floor_wait = None + progress = () + amazon_detail, amazon_failure = "", None try: from harness import runtime, datastore, mirror_store rt = runtime.get_runtime(tenant) @@ -1298,61 +1508,37 @@ def _run_sync_requests(outcome=None, *, during_odoo=False): datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()): raise _SyncNowRefused( "This tenant's mirror is not resident on this server. Sync requires its tenant worker.") + connector_sync.poll_remote_requests(rt) if provider == "odoo": import core.odoo as odoo import odoo_relational as rel import routes_connectors as _sync_routes if rt.odoo_paused() or rel.frozen(rt): raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.") - requested_cid = (_sync_routes.sync_request_connection(provider, tenant) + requested_cid = ((receipt.get("connectionId") if "connectionId" in receipt + else _sync_routes.sync_request_connection(provider, tenant)) or _sync_routes._odoo_connection_slot(rt, None)) - default_cid = _sync_routes._odoo_connection_slot(rt, None) - if requested_cid != default_cid: - # Named connections have their own provider reader and must never be pulled - # through the process-global default mirror under another connection's id. - from types import SimpleNamespace - report = _sync_routes._odoo_sync_runner( - SimpleNamespace(runtime=rt, tenant=tenant, - uname=receipt.get("requestedBy") or "automation"), - requested_cid) - problems = list(report.get("errors") or ()) - if report.get("state") in ("error", "partial") and not problems: - problems.append("The named Odoo connection did not finish cleanly.") - else: - source = rt.odoo_source() - if source is None: - raise _SyncNowRefused( - "This tenant has no Odoo credential. Connect Odoo before syncing.") - odoo_requested = True - previous_client = getattr(odoo._tlocal, "client", None) - previous_context = getattr(odoo._tlocal, "tenant_ctx", None) - try: - # Raw mirror pulls use the resolved tenant credential and a dedicated - # transport. Do not borrow a request's cached XML-RPC client. - odoo._tlocal.client = _DeskAwareOdooClient( - lambda: odoo.OdooClient(creds=source._creds)) - odoo._tlocal.tenant_ctx = None - from connector_onboarding import selected_mirror_entities - selected_entities = selected_mirror_entities(rt) - # Sync now counts each entity's source beside its copy, as the full pass does. - # Looked up by name: extracted test fixtures of this function omit the counter. - _counter = globals().get("_entity_source_counter") - _after_entity = _counter(datastore) if _counter else None - report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities, - log=lambda *a, **k: None, - **({"after_entity": _after_entity} if _after_entity else {})) - finally: - odoo._tlocal.client = previous_client - odoo._tlocal.tenant_ctx = previous_context - if not report and selected_entities != []: - raise _SyncNowRefused( - "Odoo did not run a sync. Check whether the connection is paused.") - if any(row.get("phase") != "live" for row in report.values()): - raise _SyncNowRefused( - "Odoo backfill is incomplete. Run Sync now again to continue it.") - problems = [] + # Named and default requests share one all-connection publication cohort. + odoo_requested = True + from connector_onboarding import selected_mirror_entities + selected_entities = selected_mirror_entities(rt) + _counter = globals().get("_entity_source_counter") + _after_entity = _counter(datastore) if _counter else None + _log, progress = progress_log(provider, tenant, receipt, rt) + report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities, + connection_id=requested_cid, log=_log, + client_factory=lambda creds: _DeskAwareOdooClient(lambda: odoo.OdooClient(creds=creds)), + **({"after_entity": _after_entity} if _after_entity else {})) + if not report and selected_entities != []: + raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.") + if any(row.get("phase") != "live" for row in report.values()): + raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.") + if mirror_store.pending_sync_entities("odoo", tenant, rt) is not None: + raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.") + problems = [] elif provider == "amazon_sp": from harness import amazon_store as amazon + _drain_amazon_connection_revocations(rt) if not amazon.credential(tenant, rt=rt): raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.") from connectors_amazon import notification_sync_options @@ -1361,10 +1547,11 @@ def _run_sync_requests(outcome=None, *, during_odoo=False): first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt) options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS, "refresh_loaded": True} if first_load else {} + _log, progress = progress_log(provider, tenant, receipt, rt) report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt, scheduled=False if targeted else receipt.get("requestedBy") == "first load", **target_options, **options, - log=lambda message: print(f"[aios-api] amazon sync desk: {message}", flush=True)) + log=_log) print("[aios-api] amazon sync done (sync desk): " + ", ".join( f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "") + (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left" @@ -1378,13 +1565,16 @@ def _run_sync_requests(outcome=None, *, during_odoo=False): floor_wait = float(report["nextDueSeconds"]) problems = list(report.get("problems") or []) + list(report.get("audit_problems") or []) tables = report.get("tables") or {} + amazon_detail = _amazon_sync_receipt(report) + amazon_failure = next((row["failure"] for row in tables.values() + if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None) if not problems and not tables and not continuing: raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.") if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()): raise _SyncNowRefused("All Amazon databases are disabled. Enable a database before syncing.") elif provider == "meta_ads": from harness import meta_store as meta - if not meta.token(rt): + if not meta._connection_rows(rt, tenant, {"problems": []}): raise _SyncNowRefused("Meta Ads has no access token. Connect Meta Ads before syncing.") report = mirror_store.tracked_sync("meta", rt, meta.sync, tenant, log=lambda *a, **k: None, insights_days=7) @@ -1400,15 +1590,27 @@ def _run_sync_requests(outcome=None, *, during_odoo=False): detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; " f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.") if problems: - detail += " " + _sync_now_failure(problems[0]) + detail = _sync_now_detail(provider, tenant, ok=False, + detail=amazon_detail or detail, + cause=amazon_failure or problems[0]) elif problems: - detail = _sync_now_detail(provider, tenant, ok=False, cause=problems[0]) + detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail, + cause=amazon_failure or problems[0]) else: ok, detail = True, "Sync finished." + if amazon_detail and not problems: + detail += " " + amazon_detail except _SyncNowRefused as exc: detail = str(exc) except Exception as exc: # noqa: BLE001 detail = _sync_now_detail(provider, tenant, ok=False, cause=exc) + if progress and not amazon_detail: + import re + suffix = re.search(r" \(cause code: [a-z_]+\)$", detail) + cause_code = suffix[0] if suffix else "" + sentence = detail[:suffix.start()] if suffix else detail + latest = progress[-1][:140] + detail = sentence[:299 - len(latest) - len(cause_code)] + " " + latest + cause_code if first_load_progress is not None: connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, first_load=first_load_progress, request_id=receipt.get("id")) @@ -2273,6 +2475,8 @@ def _resync_mirror_pass(manual, passes, entities=None): try: first = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all, entities=absent, log=_resync_log, + client_factory=lambda creds: _DeskAwareOdooClient( + lambda: _odoo.OdooClient(creds=creds)), _catch_up_complete=False) _log_entities("absent", first) print(f"[aios-api] store resync: synced {', '.join(absent)} first, " @@ -2284,11 +2488,18 @@ def _resync_mirror_pass(manual, passes, entities=None): _after_entity = _entity_source_counter(_ds) sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all, entities=entities, log=_resync_log, + client_factory=lambda creds: _DeskAwareOdooClient( + lambda: _odoo.OdooClient(creds=creds)), **({"after_entity": _after_entity} if _after_entity else {})) _log_entities("full", sync_report) for key, row in first.items(): if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict): sync_report[key]["pulled"] = sync_report[key].get("pulled", 0) + row.get("pulled", 0) + _held_check = globals().get("_resync_publication_held") + if (_held_check(sync_report) if _held_check else + any(row.get("phase") == "backfill" for row in (sync_report or {}).values())): + _resync_log("publication held; the next admitted pass resumes only its unfinished cohort") + return sync_report, None, None # Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has # the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER. # ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was @@ -2330,6 +2541,22 @@ def _absent_mirror_entities(ds, entities=None): return [] +def _resync_publication_held(sync_report, tenant=None): + """Keep projections behind the durable source boundary, including validation and transport failures.""" + if any(isinstance(row, dict) and row.get("phase") == "backfill" + for row in (sync_report or {}).values()): + return True + try: + from harness import datastore, mirror_store, runtime + if not datastore.DB_PATH.with_suffix(".odoo.candidate.duckdb").exists(): + return False + rt = runtime.get_runtime(tenant) if tenant else datastore._resident_odoo_runtime() + return rt is not None and mirror_store.pending_sync_entities("odoo", rt.key, rt) is not None + except Exception as exc: + print(f"[aios-api] Odoo publication status unavailable ({type(exc).__name__}); projections held") + return True + + def _smallest_first(ds, keys): """`keys` ordered by their Odoo row count, so a small table answers before a large backfill.""" def size(key): @@ -2499,17 +2726,26 @@ def _entity_source_counter(ds): if not tenant: return None counted = _PASS_SOURCE_COUNTS[tenant] = {} + per_connection = {} def after_entity(key): # A volatile table (stock_quant) drifts between its copy and a count at the end of the pass. try: import odoo_relational as _rel_counts model = (ds.ENTITIES.get(key) or {}).get("model") - buckets = _rel_counts.source_count_buckets(model) - set(counted) + binding = getattr(getattr(ds, "_SYNC_TLS", None), "connection", None) + cid = binding["id"] if binding else None + seen = per_connection.setdefault(cid, {}) + buckets = _rel_counts.source_count_buckets(model) - set(seen) if not buckets: return - got = _rel_counts.capture_source_counts(rt, buckets=buckets) - counted.update(got) + got = _rel_counts.capture_source_counts(rt, buckets=buckets, + **({"connection_id": cid} if cid and cid != "legacy-env" else {})) + seen.update(got) + for bucket, (count, mode) in got.items(): + members = [values[bucket] for values in per_connection.values() if bucket in values] + if all(other_mode == mode for _, other_mode in members): + counted[bucket] = (sum(value for value, _ in members), mode) for bucket, (count, _mode) in sorted(got.items()): print(f"[aios-api] store resync: counted the Odoo source for {bucket} " f"beside the {key} copy: {count:,} rows", flush=True) @@ -2772,13 +3008,17 @@ def _store_resync_loop(): sync_report, delete_report, _err = None, None, _classify_resync_error(exc) if _err is None and sync_report: RESYNC_STATE["passes"] = RESYNC_STATE.get("passes", 0) + 1 - source_fingerprints = _resync_source_fingerprints(sync_report) + _held_check = globals().get("_resync_publication_held") + _publication_held = (_held_check(sync_report, _cad["tenant"]) if _held_check else + any(row.get("phase") == "backfill" for row in (sync_report or {}).values())) + RESYNC_STATE["publicationHeld"] = _publication_held + source_fingerprints = {} if _publication_held else _resync_source_fingerprints(sync_report) RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()) RESYNC_STATE["lastPassOk"] = _err is None RESYNC_STATE["lastError"] = _err if _err is None: RESYNC_STATE["consecutiveFailures"] = 0 - if not _manual: + if not _manual and not _publication_held: RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"] # A published pull is the Odoo source stage's success, so the receipt's source stamp follows it. try: @@ -2796,6 +3036,10 @@ def _store_resync_loop(): print(f"[aios-api] store resync failed ({_err['kind']}, " f"{RESYNC_STATE['consecutiveFailures']} in a row): " f"{_err['type']}: {_err['message']}") + if _publication_held: + print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; " + "the next admitted pass resumes the unfinished Odoo cohort") + continue if _boot_catch_up and not _odoo_requested: # Restore only: the saved Amazon mirror and its read-through registry, no vendor call. if not catch_up_retries: diff --git a/api/meta_relational.py b/api/meta_relational.py index f0168ed1e5b552d7af8c085708517ab9bd585289..9fce810f2c0c8a890cc2d7759db80c8ab60f63d5 100644 --- a/api/meta_relational.py +++ b/api/meta_relational.py @@ -146,7 +146,7 @@ def _fields_for(cur, key): for name, dtype in cols: f = {"key": name, "label": _label(name), "type": _ftype(name, dtype), "source": "overlay", "default": name in (pinned, "id", "name", "status", "spend", - "impressions", "clicks", "date_start"), + "impressions", "clicks", "date_start", "store"), "pinned": name == pinned} out.append(_odoo._preset(f, flow="meta_relational")) # Links require the stored and planned schemas held by apply_plan's atomic updater. @@ -163,6 +163,14 @@ def _machine_preset(field): def _row_id(key, row): """Use the declared source identity, preserving existing Graph object IDs.""" + if "_row_key" in row: + value = str(row.get("_row_key") or "").strip() + if not value: + raise Refused(f"{key}: missing connection-aware row identity. Nothing was written.") + payload = json.dumps([key, [value]], ensure_ascii=True, separators=(',', ':')) + return str(int(hashlib.sha256(payload.encode('utf-8')).hexdigest()[:13], 16)) + if row.get("_connection_id"): + raise Refused(f"{key}: missing connection-aware row identity. Nothing was written.") identity = IDENTITY[key] values = [str(row.get(name) if row.get(name) is not None else '').strip() for name in identity] @@ -310,6 +318,17 @@ def apply_plan(rt, built, username="meta", today=None): live_columns.update({key: {field['key'] for field in fields} for key, (fields, _rows) in plans.items()}) generated = connector_spec.system_link_fields('meta', live_columns) + scoped = {"account_id": "_account_key", "campaign_id": "_campaign_key", + "adset_id": "_adset_key", "ad_id": "_ad_key", "id": "_row_key"} + for source, links in generated.items(): + for field in links: + link = field["link"] + target = link["table"] + if "_connection_id" in live_columns[source] or "_connection_id" in live_columns[target]: + left, right = scoped.get(link["from"]), scoped.get(link["on"]) + if left not in live_columns[source] or right not in live_columns[target]: + raise Refused("Meta links need connection-aware endpoints. Nothing was written.") + link.update({"from": left, "on": right}) table = doc.get(INSIGHTS_KEY) if rekeys and isinstance(table, dict): rows = table.get('rows') or {} diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 9e3bbb4cf851fef20f59b344912120ca22d6229f..71e094d81c060abe19ef30276b36e8386c6602da 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -246,6 +246,7 @@ _AUDIT_AGENT_SQL = { } _INVOICE_AUDIT = ( + _af("state", "Posting state", "text", "account.move.state", "am.state"), _af("amount_total", "Total", "currency", "account.move.amount_total", "am.amount_total", "num", "sum"), _af("amount_total_signed", "Total signed", "currency", "account.move.amount_total_signed", diff --git a/api/registry_source.py b/api/registry_source.py index 3f9b103d1e7640a5e1c82fd31924fbf72f4370e3..09d6ebc1e6ce4376cd9edbed6467896ed2f20a0a 100644 --- a/api/registry_source.py +++ b/api/registry_source.py @@ -354,7 +354,7 @@ class BatchStore: raise_if_resource_error(exc) raise err(503, 'canonical_source_unavailable', 'Saved view membership changed or is unavailable.') from exc cur = getattr(self, '_canonical_source_cursor', None) - if cur is not None and selected: + if cur is not None and (selected or self.metadata.get('_scan')): from routes_odoo_tables import _membership_relation cache_key = ('view_members', name, owner, vid) if cache_key not in self.metadata: @@ -366,7 +366,7 @@ class BatchStore: kept = [int(row[0]) for row in cur.execute( f'SELECT DISTINCT * FROM ({operand.sql}) members(pid) ' f'WHERE pid IN ({",".join("?" for _ in selected)}) ORDER BY pid', - [*operand.params, *selected]).fetchall()] + [*operand.params, *selected]).fetchall()] if selected else [] else: # Metadata consumes no row memberships; retain only the curated witness. witness_key = ('view_witness', name, owner, vid) @@ -408,11 +408,12 @@ class BatchStore: if name.endswith('_table_workspace'): value = self._workspace(name) elif name.endswith('__shared'): - selected = paths(self._runtime, name, [['fields'], - *[['cells', pid] for pid in self.row_ids]]) - fields, values = selected[0] or {}, selected[1:] - value = {'fields': fields, 'cells': {pid: cell for pid, cell in zip(self.row_ids, values) - if isinstance(cell, dict)}} + key = ('shared_fields', name) + if key not in self.metadata: + self.metadata[key] = paths(self._runtime, name, [['fields']])[0] or {} + values = self._selected_json(name, ['cells']) + value = {'fields': self.metadata[key], 'cells': { + pid: cell for pid, cell in values.items() if isinstance(cell, dict)}} elif name in {self.table_key + '_docs', 'customer_docs', 'product_docs'}: values = (paths(self._runtime, name, [[pid] for pid in self.row_ids]) if self._canonical_needed is None else [None] * len(self.row_ids)) @@ -516,12 +517,12 @@ class CanonicalStore(BatchStore): return super().get_entry(name, entry) -def canonical_metadata(session, table_key, *, metadata=None): +def canonical_metadata(session, table_key, *, metadata=None, cur=None): """Read the canonical permitted definitions without constructing source rows.""" import routes_customers as customers import routes_products as products session.require(table_key) - lent = CanonicalStore(session, table_key, [], cur=None, + lent = CanonicalStore(session, table_key, [], cur=cur, metadata={} if metadata is None else metadata, needed=frozenset()) actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True)) build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly @@ -558,7 +559,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric team, agent = perm_scope.derive_pool_scope(session.user, table_key) before = generation(session, table_key) metadata = {'_scan': ids is None} - definition = canonical_metadata(session, table_key, metadata=metadata) + definition = canonical_metadata(session, table_key, metadata=metadata, cur=cur) # Every permanent wall operand is populated before apply_row_scope judges the batch. tree = perm_scope.entry(session.user, table_key) or {} wall = tree.get('filter') or {} @@ -632,31 +633,34 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric batches = frozen() else: raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.') - elif table_key == 'product_data': + else: from harness import canonical_source def mirrored(): # Stock, sales and product reads come from the Odoo mirror, never live Odoo. A lent # cursor that cannot see it (Sheet's in-memory one) gets a child cursor on the mirror. source_cur, owned = cur, None - if canonical_source.mirror_missing(cur): + kind = 'product' if table_key == 'product_data' else 'customer' + if canonical_source.mirror_missing(cur, kind): import routes_odoo_tables as rot base, mismatch = rot._mirror_cur(session) if mismatch: raise err(409, 'cross_tenant_store', mismatch) source_cur = owned = base.cursor() try: - source = product_data.source_batches(source_cur, st=runtime, team_id=team, ids=ids, - needed=required, raw_fields=[field['rawName'] for field in raw]) - with canonical_source.mirrored(source_cur), closing(source): + if table_key == 'product_data': + source = product_data.source_batches(source_cur, st=runtime, team_id=team, ids=ids, + needed=required, raw_fields=[field['rawName'] for field in raw]) + else: + source = customer_data.source_batches(source_cur, st=runtime, agent_name=agent, + team_id=team, ids=ids, needed=required, + ledger_batches=items(session.runtime, customer_data.LEDGER_KEY, selected=ids) + if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY) else ()) + with canonical_source.mirrored(source_cur, runtime, kind=kind), closing(source): yield from source finally: if owned is not None: owned.close() batches = mirrored() - else: - batches = customer_data.source_batches(cur, st=runtime, agent_name=agent, team_id=team, - ids=ids, needed=required, ledger_batches=items(session.runtime, customer_data.LEDGER_KEY, - selected=ids) if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY) else ()) with closing(batches): for rows in batches: runtime = CanonicalStore(session, table_key, rows, cur=cur, metadata=metadata, needed=required) diff --git a/api/request_timing.py b/api/request_timing.py index cb9b81cb999d39d568bbfd893ed2e05ceb950fb9..d44e28982d7661a677d44f106b1034a6c2836d6b 100644 --- a/api/request_timing.py +++ b/api/request_timing.py @@ -20,12 +20,18 @@ _STATS = OrderedDict() _ROUTE_LIMIT = 512 -def request_timeout_s(): +def request_timeout_s(scope=None): + """Give cold pool reads a bounded budget unless the operator sets an explicit timeout.""" + pool_read = bool(scope and (scope.get("method"), scope.get("path")) in { + ("GET", "/api/v1/products"), ("GET", "/api/v1/customers"), + ("POST", "/api/v1/products/aggregates"), ("POST", "/api/v1/customers/aggregates"), + }) + default = 120.0 if pool_read else 45.0 try: - value = float(os.environ.get("AIOS_REQUEST_TIMEOUT_S", "45")) - return max(.01, value) if math.isfinite(value) else 45.0 + value = float(os.environ.get("AIOS_REQUEST_TIMEOUT_S", default)) + return max(.01, value) if math.isfinite(value) else default except ValueError: - return 45.0 + return default _BUSY_CAUSES = { @@ -375,7 +381,7 @@ class RequestTiming: if scope.get("type") != "http": return await self.app(scope, receive, send) with timing.request_scope() as acc: - acc.deadline = time.monotonic() + request_timeout_s() + acc.deadline = time.monotonic() + request_timeout_s(scope) status, failed, disconnected, complete = None, False, False, False threshold = slow_threshold_ms() _resolve_route(scope) diff --git a/api/routes_account_health.py b/api/routes_account_health.py index 9daec2157a9857b26f710fa22d79819396c58436..43ebcb4c1f98c26a12a154f5e1382cec90f38018 100644 --- a/api/routes_account_health.py +++ b/api/routes_account_health.py @@ -96,9 +96,12 @@ def owners(session): @router.get("/summary") -def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): +def read_summary(app: str = Query(..., min_length=1), search: str = Query(""), + offset: int = Query(0, ge=0), limit: int = Query(50, ge=1), + metric: str | None = Query(None), session: Session = Depends(_app_reader)): ah = _module() - out = _call(ah.summary, _tenant(session), st=session.runtime) + out = _call(ah.summary_page, _tenant(session), search=search, offset=offset, limit=limit, + metric=metric, st=session.runtime) return {**out, "canEdit": _may_edit(session, _app(session, app))} @@ -111,11 +114,12 @@ def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Qu @router.get("/items") def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None), - cursor: str | None = Query(None), limit: int = Query(200, ge=1), + search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None), + cursor: str | None = Query(None), limit: int = Query(50, ge=1), session: Session = Depends(_app_reader)): ah = _module() - listed = _call(ah.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit, - st=session.runtime) + listed = _call(ah.items, _tenant(session), session.user, status=status, search=search, offset=offset, + cursor=cursor, limit=limit, metric=metric, st=session.runtime) return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} diff --git a/api/routes_aged_stock.py b/api/routes_aged_stock.py index 94a5e2165474b2296c01b0ff40ee5e516e029199..6e2ed10987e323645d73a9eae0556d8e85ea8917 100644 --- a/api/routes_aged_stock.py +++ b/api/routes_aged_stock.py @@ -1,5 +1,9 @@ """Aged stock App routes (C3): summary, items, one SKU, settings, refresh and the rule 8 tie.""" +import time +import uuid + from fastapi import APIRouter, Body, Depends, Query +from fastapi.responses import JSONResponse from deps import Session, err, require_session @@ -95,40 +99,33 @@ def owners(session): return sorted((n for n in names if n), key=str.lower) -#: The item fields a Next tier row carries, so the screen lists it with no second read. -ROW_FIELDS = ("action", "recommended_action", "owner", "status", "due", "units") - - -def _summary_rows(tenant_key, st): - """The module summary with each Next tier row joined to its item's action, owner, status and due.""" - mod = _module() - out = mod.summary(tenant_key, st=st) - stored = mod._doc(st, mod.KEY_ITEMS).get("items") or {} - for row in out.get("nextTier") or (): - item = stored.get(row.get("id")) or {} - row["item"] = {key: item.get(key) for key in ROW_FIELDS} - return out - - @router.get("/summary") -def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): - out = _call(_summary_rows, _tenant(session), session.runtime) +def read_summary(app: str = Query(..., min_length=1), offset: int = Query(0, ge=0), + limit: int = Query(24, ge=1, description="Requests above 200 use 200 months. monthLimit reports the effective size."), + session: Session = Depends(_app_reader)): + out = _call(_module().summary, _tenant(session), offset=offset, + limit=min(limit, _module().MAX_PAGE_SIZE), st=session.runtime) return {**out, "canEdit": _may_edit(session, _app(session, app))} @router.get("/items") def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None), action: str | None = Query(None), cursor: str | None = Query(None), - limit: int = Query(200, ge=1), q: str | None = Query(None), + limit: int = Query(200, ge=1, description="Requests above 200 use 200 items. limit reports the effective size."), + q: str | None = Query(None), + nextTier: bool = Query(False), session: Session = Depends(_app_reader)): listed = _call(_module().items, _tenant(session), session.uname, status=status, action=action, - cursor=cursor, limit=limit, q=q, st=session.runtime) + cursor=cursor, limit=min(limit, _module().MAX_PAGE_SIZE), q=q, next_tier=nextTier, st=session.runtime) return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} @router.get("/items/{item_id}") -def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): - detail = _call(_module().item_detail, _tenant(session), item_id, st=session.runtime) +def read_item(item_id: str, app: str = Query(..., min_length=1), offset: int = Query(0, ge=0), + limit: int = Query(50, ge=1, description="Requests above 200 use 200 entries per history. detailLimit reports the effective size."), + session: Session = Depends(_app_reader)): + detail = _call(_module().item_detail, _tenant(session), item_id, offset=offset, + limit=min(limit, _module().MAX_PAGE_SIZE), st=session.runtime) return {**detail, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))} @@ -158,7 +155,80 @@ def write_settings(adsFromDays: int | None = Body(None), priceDownFromDays: int @router.post("/refresh") def refresh(session: Session = Depends(_app_writer)): - return _call(_module().refresh, _tenant(session), st=session.runtime) + identity, stamp = uuid.uuid4().hex, time.time() + + def enqueue(doc): + doc = doc if isinstance(doc, dict) else {} + if doc.get("state") in ("queued", "running") and stamp - float(doc.get("requestedAt") or 0) < 600: + return doc + return {"state": "queued", "requestId": identity, "requestedAt": stamp} + + queued = _call(session.runtime.update, _module().KEY_REFRESH, enqueue) + return JSONResponse(status_code=202, content=queued) + + +@router.get("/refresh") +def refresh_status(session: Session = Depends(_app_reader)): + return _call(session.runtime.get, _module().KEY_REFRESH) or {"state": "idle"} + + +def _drain_refresh(rt, *, now=None): + """Claim one durable refresh intent in the sync worker and retain its terminal result.""" + mod = _module() + stamp = time.time() if now is None else now + pending = rt.get(mod.KEY_REFRESH) or {} + if pending.get("state") != "queued" and not ( + pending.get("state") == "running" and stamp - float(pending.get("startedAt") or 0) >= 600): + return + token, claimed = uuid.uuid4().hex, {} + + def claim(doc): + if (doc or {}).get("requestId") == pending.get("requestId") and ( + doc.get("state") == "queued" or (doc.get("state") == "running" + and stamp - float(doc.get("startedAt") or 0) >= 600)): + doc.update(state="running", startedAt=stamp, claim=token) + claimed.update(doc) + return doc + + rt.update(mod.KEY_REFRESH, claim) + if not claimed: + return + try: + result = mod.refresh(rt.key, st=rt) + completion = {"state": "done", "result": result} + except Exception: # noqa: BLE001 + completion = {"state": "failed", "message": "The aged stock view could not be refreshed. Try again shortly."} + + def finish(doc): + if (doc or {}).get("claim") == token: + doc.update(completion, finishedAt=time.time()) + return doc + rt.update(mod.KEY_REFRESH, finish) + + +def _refresh_worker(): + import threading + import process_role + from harness import runtime + stop = threading.Event() + while not stop.wait(1): + if not process_role.may_write(): + continue + for tenant in runtime.known_tenants(): + try: + _drain_refresh(runtime.get_runtime(tenant)) + except Exception: # noqa: BLE001 + continue + + +def _start_refresh_worker(): + import process_role + from core import data_binding + if not data_binding.gate_process(): + process_role.start("aged-stock-refresh", "heavy", _refresh_worker) + + +_start_refresh_worker() @router.get("/validate") diff --git a/api/routes_amazon_deletion.py b/api/routes_amazon_deletion.py index d44bf2377879a8a2ea565bef2deb87aa6b0cf6a0..21915c5c3eced43795dc5a48ef3ddac2796383d2 100644 --- a/api/routes_amazon_deletion.py +++ b/api/routes_amazon_deletion.py @@ -13,15 +13,27 @@ ATTENTION_MESSAGE = "Amazon data deletion needs attention from an administrator. @router.get("/connectors/amazon_sp/deletion") def deletion_status(session: Session = Depends(admin_gate)): """Read metadata and lifecycle evidence without decrypting credentials or running purge.""" + from amazon_relational import CONNECTION_REVOCATIONS_KEY from core import keychain from harness import amazon_store rt = session.runtime try: - # A saved credential suppresses a stale revocation schedule even if cancellation failed. - if keychain.first_entry_of_type(rt, "amazon_sp") is not None: + revocations = rt.get(CONNECTION_REVOCATIONS_KEY) + if revocations is None: + revocations = {} + if any(not isinstance(row, dict) or row.get("state") != "done" for row in revocations.values()): + return {"state": "failed", "message": ATTENTION_MESSAGE} + connected = keychain.first_entry_of_type(rt, "amazon_sp") is not None + if connected: + # Reconnection cannot cancel an independent notice or termination obligation. + ledger = rt.get(amazon_store.LIFECYCLE_KEY) or {} + if not any(row["kind"] in ("notice", "termination") for row in + amazon_store._active_deletion_triggers(rt.key, ledger, amazon_store._utc())): + return {"state": "none"} + plan = amazon_store.scheduled_purge(rt.key, rt, ignore_credential_loss=connected) + if connected and plan is None: return {"state": "none"} - plan = amazon_store.scheduled_purge(rt.key, rt) ledger = amazon_store._ledger(rt.key, rt) now = amazon_store._utc() checks = [row for row in ledger.get("checks", []) diff --git a/api/routes_amazon_operator.py b/api/routes_amazon_operator.py index 6326065aedcdc649548bec19033e116611517ddf..7b7297b1ee471aaa94542a3825e96b283be6b140 100644 --- a/api/routes_amazon_operator.py +++ b/api/routes_amazon_operator.py @@ -78,24 +78,21 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends( @router.get("/today") -def read_today(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): +def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None), + app_cursor: str | None = Query(None), held_cursor: str | None = Query(None), + limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)): op = _module() - out = _call(op.today, _tenant(session), st=session.runtime) + out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor, + app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q) return {**out, "canEdit": _may_edit(session, _app(session, app))} @router.get("/history") def read_history(status: str | None = Query(None), cursor: str | None = Query(None), - limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)): + limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)): op = _module() - page = _call(op.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit, + return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q, st=session.runtime) - if cursor: - return page - # The first page carries the full count for the History tab. Uncapped: every row is counted (rule 1). - every = _call(op.items, _tenant(session), session.user, status=status, cursor=None, limit=None, - st=session.runtime) - return {**page, "total": len(every["items"])} @router.get("/settings") @@ -131,3 +128,50 @@ def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length= op = _module() return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason, st=session.runtime) + + +def _refresh_inboxes_once(): + """Refresh installed Operator inboxes independently of Ads data and screen requests.""" + import application_registry + from harness import runtime + for tenant in runtime.known_tenants(): + try: + st = runtime.get_runtime(tenant) + apps = application_registry.load(st)["applications"].values() + if any(a.get("templateKind") == TEMPLATE_KIND and not a.get("retired") for a in apps): + _module().pull_app_proposals(tenant, st=st) + except Exception: + import logging + logging.getLogger(__name__).exception("Operator inbox refresh failed for %s", tenant) + + +_inbox_thread = None +_inbox_stop = None + + +@router.on_event("startup") +def _start_inbox_refresh(): + global _inbox_thread, _inbox_stop + import process_role + import threading + if _inbox_thread is not None and _inbox_thread.is_alive(): + return + _inbox_stop = threading.Event() + + def run(): + while not _inbox_stop.is_set(): + _refresh_inboxes_once() + _inbox_stop.wait(30) + _inbox_thread = process_role.start("amazon-operator-inbox", "heavy", run) + + +@router.on_event("shutdown") +def _stop_inbox_refresh(): + if _inbox_stop is not None: + _inbox_stop.set() + + +# The sync worker imports routers without invoking FastAPI startup hooks. +import process_role as _process_role +if _process_role.role() == "worker": + _start_inbox_refresh() diff --git a/api/routes_competitor_watch.py b/api/routes_competitor_watch.py index 2cd5c0b407d8914c618f6fce811aed3398b77990..cdc82a28f67550154c59e65a6290f83bc000e78b 100644 --- a/api/routes_competitor_watch.py +++ b/api/routes_competitor_watch.py @@ -38,6 +38,19 @@ def _call(fn, *args, **kwargs): raise err(400, "competitor_watch_invalid", str(exc)) from exc except KeyError as exc: raise err(404, "competitor_watch_alert_not_found", "That alert does not exist.") from exc + except _module().SourceNotReady as exc: + cw = _module() + cause = cw.READ_FAILED.format(cause=exc) + page = exc.page or {"termsReady": False} + page = {**page, "problems": list(dict.fromkeys([*(page.get("problems") or []), cause]))} + screen = screen_payload(cw, {}, page.get("terms", []), page.get("term"), False, page=page) + raise err(503, "competitor_watch_source_not_ready", cause, {"screen": screen}) from exc + except _module().TermsNotReady as exc: + screen = screen_payload(_module(), {}, [], None, False, + page={"problems": exc.problems, "termsReady": False}) + raise err(503, "competitor_watch_terms_not_ready", str(exc), {"screen": screen}) from exc + except _module().AlertsNotReady as exc: + raise err(503, "competitor_watch_alerts_not_ready", str(exc)) from exc except Exception as exc: # noqa: BLE001 raise err(503, "competitor_watch_unavailable", "The competitor watch could not be read. Try again shortly.") from exc @@ -80,9 +93,10 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends( @router.get("/terms") -def list_terms(session: Session = Depends(_app_reader)): +def list_terms(q: str = "", offset: int = 0, limit: int = 50, + session: Session = Depends(_app_reader)): cw = _module() - return _call(cw.terms_summary, _tenant(session), st=session.runtime) + return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit) @router.get("/terms/detail") @@ -94,13 +108,13 @@ def term_detail(term: str = Query(..., min_length=1), session: Session = Depends @router.post("/terms") def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)): cw = _module() - return {"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime)} + return {"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)} @router.post("/terms/remove") def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)): cw = _module() - return {"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime)} + return {"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)} @router.get("/settings") @@ -119,9 +133,11 @@ def write_settings(undercutPct: float | None = Body(None), rankSurgePct: float | @router.get("/alerts") def list_alerts(app: str = Query(..., min_length=1), status: str | None = Query(None), + q: str = "", offset: int = 0, limit: int = 50, check_offset: int = 0, check_limit: int = 50, session: Session = Depends(_app_reader)): cw = _module() - listed = _call(cw.alerts, _tenant(session), status=status, st=session.runtime) + listed = _call(cw.alerts, _tenant(session), status=status, st=session.runtime, search=q, + offset=offset, limit=limit, check_offset=check_offset, check_limit=check_limit) return {**listed, "canEdit": _may_edit(session, _app(session, app))} @@ -196,6 +212,7 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None): comparison = detail.get("comparison") or {} ready = comparison.get("ready") is True return {"canEdit": bool(can_edit), "terms": list(terms), "term": term, + "termsReady": (page or {}).get("termsReady", True), "week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None, "previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None, "comparisonReady": ready, diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 16a61af892ec65c0a89a05441ab6084db096064d..a4af7ae2d690195aa87d0d43a884ae7ef2c1f49d 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -131,6 +131,7 @@ class AiClientStepAction(BaseModel): class AiClientStep(BaseModel): + title: str | None = Field(default=None, min_length=1, max_length=48, strict=True) text: str = Field(min_length=1) block: str | None = None action: AiClientStepAction | None = None @@ -150,19 +151,31 @@ class AiClientGuide(BaseModel): record: str = Field(min_length=1) provenance: str = Field(min_length=1) need: str = Field(min_length=1, strict=True) - steps: list[AiClientStep] = Field(min_length=1) + steps: list[AiClientStep] = Field(min_length=2, max_length=3) keySteps: list[AiClientStep] = Field(default_factory=list) caveat: str = Field(min_length=1) failureAction: str = Field(min_length=1, strict=True) - testNote: str = Field(min_length=1, strict=True) + testNote: str = Field(strict=True) - @field_validator("need", "failureAction", "testNote") + @field_validator("need", "failureAction") @classmethod def guide_text(cls, value: str) -> str: if not value.strip(): raise ValueError("A guide needs all four fields.") return value + @field_validator("steps", "keySteps") + @classmethod + def short_steps(cls, steps: list[AiClientStep]) -> list[AiClientStep]: + if steps and not 2 <= len(steps) <= 3: + raise ValueError("An AI guide needs two or three steps.") + for step in steps: + if not step.title or not step.title.strip() or not step.block or not step.block.strip(): + raise ValueError("An AI step needs a short title and a copy box.") + if re.search(r"\bTest\s*:", " ".join((step.title, step.text, step.block)), re.I): + raise ValueError("An AI guide has no Test line.") + return steps + class AiClientVerification(BaseModel): mcp: str = Field(min_length=1) @@ -1590,7 +1603,10 @@ def connectors_directory(session: Session = Depends(require_session)): # number of cards behind it by construction, and never a sum of endpoint counts (the ticket's # trap; `chip_counts` carries the argument). served = connector_directory.to_client_rows(rows) + sync_providers = connector_sync.providers() for row in served: + # Registry support lets the modal start its fresh status read without another round trip. + row["syncSupported"] = row.get("key") in sync_providers if str(row.get("key") or "") == "scraper": row["spawns_tables"] = True _safe_connector_status(row.get("key"), row) @@ -3679,17 +3695,17 @@ _CONNECT_TUTORIALS = { "odoo": { "need": "An Odoo URL, database, user and API key.", "steps": ["Copy your Odoo URL and ask your administrator for the database name.", - "Sign in as the API user. Open My profile, then Account security and API keys.", - "Create a key and copy it now. Keep this user's login with the key."], + "Sign in as the API user. Open My profile, Account security and API keys. " + "Create and copy a key now. Keep this user's login with the key."], "failureAction": "Open the server URL in a browser and confirm it is reachable. " "Then check the database name and that the API key belongs to the API user.", "testNote": "Checks server reachability only. It does not verify the user or API key.", }, "amazon_sp": { "need": "A Production app client and access to your Amazon seller account.", - "steps": ["Open Solution Provider Portal, then Apps. Create or open your Production app.", - "Open its Login with Amazon credentials. Copy the client ID and client secret.", - "Authorize the app for your own seller account and copy the refresh token.", + "steps": ["In Solution Provider Portal, open Apps and create or open your Production app. " + "Copy its Login with Amazon client ID and client secret.", + "Authorize that app for your own seller account and copy the refresh token. " "In Seller Central, open Settings, Account info and copy your selling partner ID. " "Choose the marketplace that account sells in."], "failureAction": "In Solution Provider Portal, confirm the client ID and secret belong to " @@ -3699,9 +3715,9 @@ _CONNECT_TUTORIALS = { }, "amazon_ads": { "need": "An Amazon Ads API app and access to your advertising account.", - "steps": ["Open your Amazon Ads API app's Login with Amazon security profile.", + "steps": ["Open your Amazon Ads API app's Login with Amazon security profile. " "Copy its client ID and client secret.", - "Authorize the app for your advertising account and copy the refresh token.", + "Authorize the app for your advertising account and copy the refresh token. " "Copy the profile ID of the marketplace you advertise in."], "failureAction": "Authorize the app again for your advertising account, then check " "that the profile ID belongs to it.", @@ -3716,7 +3732,7 @@ _CONNECT_TUTORIALS = { }, "shopify": { "need": "Permission to create a custom app in your Shopify store.", - "steps": ["In your store admin, open Settings, Apps and sales channels, then Develop apps.", + "steps": ["In your store admin, open Settings, Apps and sales channels, then Develop apps. " "Create a custom app, configure its read permissions and install it.", "Open API credentials and copy its Admin API access token."], "failureAction": "Open the installed custom app's API credentials and copy its Admin API token.", @@ -3724,7 +3740,7 @@ _CONNECT_TUTORIALS = { }, "meta_ads": { "need": "A Meta app and access to the ad account you want to read.", - "steps": ["Open Meta for Developers and select your app.", + "steps": ["Open Meta for Developers and select your app. " "Use Graph API Explorer to get a Marketing API token with ads_read.", "Confirm the token's user can access your ad account, then copy the token."], "failureAction": "Generate a current Marketing API token with ads_read and confirm its " @@ -4048,9 +4064,17 @@ def _detail_keychain(rt, uname, is_admin, row): continue entries.append({ "id": str(entry.get("id") or ""), + "type": entry_type, + "connectionId": str(entry.get("connectionId") or entry.get("id") or ""), + "slot": int(entry.get("slot") or 1), "label": str(entry.get("label") or entry.get("id") or ""), "masked": str(entry.get("preview") or ""), "scope": str(entry.get("scope") or rk.DEFAULT_SCOPE), + "status": ("Paused" if entry_type == "odoo" and + ((rt.get(rk._CONNECTOR_FLAGS_KEY) or {}).get( + entry.get("connectionId") or entry.get("id")) or {}).get("paused") + else "Connected" if entry_type in ("amazon_sp", "amazon_ads") + and rk._kc().entry_verified(rt, entry.get("id")) else "Needs a test"), # `keychain.test_entry` answers for every stored entry: a registered prober for # the type when the connector layer filled that slot, and a read-only shape or # xmlrpc check otherwise. The flag says whether a Test control would have @@ -4067,6 +4091,7 @@ def _detail_keychain(rt, uname, is_admin, row): }) except Exception: # noqa: BLE001 entries = [] + entries.sort(key=lambda entry: (entry["slot"], entry["id"])) # ⭐ THE SECOND SOURCE. An OAuth card has no keychain entry type at all, so `entries` above is # empty for it by construction and this is the only reading that can fill the tab. if not entry_type and manage == "oauth": @@ -4077,12 +4102,21 @@ def _detail_keychain(rt, uname, is_admin, row): "canConnect": bool(row.get("startUrl")) and connect_available, "canDisconnect": bool(entries), "connect": None} + connect = _detail_connect(entry_type, bool(is_admin)) if can_connect else None + if connect: + from core import keychain as kc + connect["defaultLabel"] = f"{kc.kind_label(entry_type)} {kc.next_slot(rt, entry_type)}" + migration_pending = False + if entry_type == "odoo": + import routes_keychain as rk + migration_pending = rk.env_odoo_migration_pending(rt) return {"entries": entries, "canConnect": can_connect, "canDisconnect": bool(can_connect and entries), + "migrationPending": migration_pending, # What the Connect form asks for. `None` when this card has no credential door at all, # which is what stops the client inventing one (Webhooks, the Scraper, a planned row). - "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None} + "connect": connect} def _odoo_catalog_by_model(rt, connection_id=None, *, models=None): diff --git a/api/routes_customers.py b/api/routes_customers.py index b30e4e612b719fdd9231c46b17bd4369f52c3f90..e5d8db621f2ab61d0a9d61179415f54e90a13096 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -29,10 +29,12 @@ import os import threading import time -from fastapi import APIRouter, Body, Depends +from fastapi import APIRouter, Body, Depends, Response import scope_cache -from deps import Session, err, module_gate +from deps import Session, err, module_gate, require_session +from routes_grid import row_patch_receipt +from routes_tables import _row_patch_identity, _note_delta, _patch_cell_notes, _project_cell_notes, _builtin_note_context router = APIRouter(prefix="/api/v1") @@ -40,6 +42,50 @@ router = APIRouter(prefix="/api/v1") #: 403 rather than an empty table that looks like "you have no customers". MODULE = "customer_data" + +def _customer_fields(fields): + """Expose the mirror's Store label without adding an upstream Odoo field.""" + from routes_odoo_tables import mirror_store_field + declared = {field['key']: field for field in fields} + declared['Store'] = mirror_store_field() + declared['odoo_id'] = {'key': 'odoo_id', 'label': 'Odoo ID', 'type': 'int', 'source': 'odoo', + 'default': False, 'readOnly': True, 'mirrorOnly': True} + return list(declared.values()) + + +def _customer_store_rows(runtime, rows): + """Project Store onto admitted customer identities with bounded mirror queries.""" + if not rows: + return rows + from harness import datastore + cur = getattr(runtime, '_canonical_source_cursor', None) + admitted = callable(getattr(runtime, 'assert_datastore_matches', None)) + if cur is None and not admitted: + return rows + from routes_odoo_tables import _table_cursor + cursor = cur.cursor() if cur is not None else _table_cursor(runtime) + try: + columns = datastore.columns_of('res_partner', cur=cursor) + if not columns and admitted and cur is not None: + cursor.close() + cursor = _table_cursor(runtime) + columns = datastore.columns_of('res_partner', cur=cursor) + if 'store' not in columns: + return rows + identities = list(dict.fromkeys(int(row['pid']) for row in rows)) + stores = {} + source_ids = {} + for offset in range(0, len(identities), 256): + batch = identities[offset:offset + 256] + source_id = '_source_id' if '_source_id' in columns else 'id' + cursor.execute('SELECT id, Store, ' + source_id + ' FROM res_partner WHERE id IN (' + ','.join('?' for _ in batch) + ')', batch) + for pid, label, upstream in cursor.fetchmany(256): + stores[int(pid)] = label or '' + source_ids[int(pid)] = upstream if upstream is not None else '' + return [dict(row, Store=stores.get(int(row['pid']), ''), odoo_id=source_ids.get(int(row['pid']), '')) for row in rows] + finally: + cursor.close() + _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before # Waiters share one cold build (about 4 min on 2 vCPU); a private rebuild per waiter multiplies it. @@ -664,10 +710,15 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = def own_copy(result): # Both aliases lend immutable source rows. Copy the request-shaped metadata together. borrowed = {id(result['rows_src']): result['rows_src']} + note_runtime = result.get('_note_runtime') + if note_runtime is not None: + borrowed[id(note_runtime)] = note_runtime wall_rows = (result.get('_permission_scope') or {}).get('rows') if wall_rows is not None: borrowed[id(wall_rows)] = wall_rows out = deepcopy(result, borrowed) + if note_runtime is not None: + out['_note_runtime'] = session.runtime out["workspace"]["storageKey"] = storage_key return out @@ -840,6 +891,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke rt = session.runtime team_id, agent = _team_agent(session) rows_src = _pool_for(rt, team_id, agent) if with_rows else [] + rows_src = _customer_store_rows(rt, rows_src) # ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that # frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so # scoping here means a row this account may not see never enters ANY of them, rather than @@ -863,7 +915,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke # WRITE wall to this one's READ wall; lending it here alone would make a user-generated rule # narrow what an account SEES while leaving what it may PATCH untouched. if with_rows: - rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS, + rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, _customer_fields(aios_grid.FIELDS), st=rt) pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None) # ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because @@ -895,7 +947,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells} ws["overlays"] = _ov import odoo_relational - fields_base = list(aios_grid.FIELDS) + odoo_relational.module_link_fields(MODULE) + fields_base = _customer_fields(aios_grid.FIELDS) + odoo_relational.module_link_fields(MODULE) workspace, fields, views, lists = aios_grid.workspace_wire( ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key, fields_base=fields_base, st=rt) @@ -1145,6 +1197,7 @@ def _payload(session: Session, related=None): from routes_tables import scope_linked_metrics, linked_metric_limits rows, g['fields'], metric_refusals = scope_linked_metrics( session, MODULE, rows, g['fields'], g['measures'], related) + rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime) if related: g["pids"] = frozenset(row["pid"] for row in rows) # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door @@ -1213,7 +1266,7 @@ def _hidden_for(session: Session, defs=None): if defs is None: defs = shared_fields(st=session.runtime) return perm_scope.hidden_keys( - session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), defs), + session.user, MODULE, _merge_shared_fields(_customer_fields(aios_grid.FIELDS), defs), st=session.runtime) def allowed_pids(session: Session): @@ -1239,8 +1292,8 @@ def allowed_pids(session: Session): import core.perm_scope as perm_scope import aios_grid - rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE, - aios_grid.FIELDS, st=session.runtime) + rows = perm_scope.apply_row_scope(_customer_store_rows(session.runtime, _pool_rows(session)), session.user, MODULE, + _customer_fields(aios_grid.FIELDS), st=session.runtime) return frozenset(r["pid"] for r in rows if r.get("pid") is not None) @@ -1276,6 +1329,7 @@ def assigned_view_rows(view_id: str, session: Session = Depends(module_gate(MODU rows = aios_grid.rows_from_pool( [row for row in g["rows_src"] if row.get("pid") in pids], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) + rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime) return {"fields": g["fields"], "rows": rows, "viewId": view_id, "locked": True, "today": g["today"]} @@ -1388,7 +1442,7 @@ def migrate_routing_app_routes(runtime, tenant): """Run R20 explicitly during integration, using the tenant's complete registry snapshot.""" import application_registry as registry from core import route_system - doc = registry._read_strict(runtime, registry._KEY) or {} + doc = registry.load(runtime, strict=True) if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict): raise ValueError("the Application registry is unreadable") apps = [app for app in doc.get("applications", {}).values() @@ -1420,7 +1474,7 @@ def _route_application(session, application, *, write=False): import application_registry as registry from core import route_system try: - doc = registry._read_strict(session.runtime, registry._KEY) or {} + doc = registry.load(session.runtime, strict=True) if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict): raise ValueError("invalid Application registry") except Exception: @@ -1480,7 +1534,7 @@ def _home_route_application(session): if not me: return None try: - doc = registry._read_strict(session.runtime, registry._KEY) or {} + doc = registry.load(session.runtime, strict=True) apps = doc.get("applications", {}) if isinstance(doc, dict) else None if not isinstance(apps, dict): return None @@ -2301,8 +2355,10 @@ def remove_zone(zone_id: str, session: Session = Depends(module_gate(MODULE)), a @router.patch("/customers/{pid}") +@_row_patch_identity(module=MODULE) +@row_patch_receipt('customer') def patch_customer(pid: int, body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): + session: Session = Depends(require_session), response: Response = None): """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever. Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing @@ -2311,9 +2367,10 @@ def patch_customer(pid: int, body: dict = Body(default=None), them to keep in step. The response reports what was ACCEPTED, which is not always what was asked for. """ - from core import grid_events + from core import grid_events, edit_receipts updates = dict(body or {}) + notes = _note_delta(updates) if not updates: raise err(400, "empty_patch", "no fields to update") from core import route_system @@ -2325,18 +2382,23 @@ def patch_customer(pid: int, body: dict = Body(default=None), if retired.intersection(updates): raise err(403, "legacy_route_hidden", "this legacy Destination field was migrated; edit the named Route instead") - pool = allowed_pids(session) + note_page = _builtin_note_context(session, MODULE, pid) if notes is not None else None + pool = note_page['pids'] if note_page is not None else allowed_pids(session) if pid not in pool: # 403, not 404: the pid may well exist — it is simply not in this session's book, and # saying "no such customer" would confirm the opposite to anyone who guessed right. raise err(403, "out_of_scope", "that customer is not in your book") - payload = _payload(session) + payload = note_page if note_page is not None else _payload(session) ctx = grid_events.EventCtx( uname=session.uname, allowed_pids=pool, fields=payload["fields"], admin=session.admin, fallback_ws=None, seen_ids={}, - hidden_keys=_hidden_for(session), table=_customer_table(session), + hidden_keys=note_page['hidden'] if note_page is not None else _hidden_for(session), table=_customer_table(session), st=session.runtime) + if notes is not None: + return _patch_cell_notes(MODULE, pid, notes, ctx, + source_row_exists=lambda key, rid, admitted: key == MODULE and int(rid) in pool) + # ⭐⭐ OWNER ITEM 7 (2026-08-23) — A ROUTE RANK IS EDITED HERE, NOT THROUGH `overlay_patch`. # # ⚠ AND **NOT** BECAUSE THE ORDINARY PATH WRITES THE WRONG STRATUM. It does not: @@ -2359,7 +2421,8 @@ def patch_customer(pid: int, body: dict = Body(default=None), if updates: try: grid_events.handle_one( - {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch", + {"id": edit_receipts.current_op_id() or f"patch:{pid}:{time.time_ns()}", + "type": "overlay_patch", "pid": pid, "updates": updates}, ctx) except grid_events.StoreUnavailable: raise err(503, "store_unavailable", diff --git a/api/routes_external.py b/api/routes_external.py index 98c1dddc80449c9a8fb4e436a7d6c9f98f2e5450..b02a0c248227d9379ea834743422d8e6a7101ff9 100644 --- a/api/routes_external.py +++ b/api/routes_external.py @@ -27,7 +27,8 @@ import aios_session from deps import Session, err, require_session from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets from harness import runtime -from tool_groups.data import (QUERY_SYNTAX_CODES, UNKNOWN_DATABASE_FIX, bad_filter_message, +from tool_groups.data import (DISCOVERY_HELP, QUERY_PAGING_HELP, READ_REFUSAL_FIXES, + QUERY_SYNTAX_CODES, UNKNOWN_DATABASE_FIX, bad_filter_message, query_syntax_section, with_query_syntax) from tool_groups.sheet import WORKBOOK_MISSING_FIX @@ -38,6 +39,7 @@ router = APIRouter(prefix=EXTERNAL_PREFIX, tags=["external"]) _CREDENTIAL_PREFIXES = ("lpb1.", "lpc1.", "lpo1.", "lpr1.") _QUERY_DEFAULT_LIMIT = 100 _QUERY_MAX_LIMIT = 500 +_QUERY_PAGING_HELP = QUERY_PAGING_HELP _QUERY_MAX_SORTS = 8 _QUERY_MAX_AGGREGATES = 16 _QUERY_MAX_CURSOR = 768 @@ -45,6 +47,19 @@ _QUERY_MAX_FILTER_DEPTH = 3 _QUERY_MAX_FILTER_NODES = 100 _QUERY_MAX_FILTER_SIBLINGS = 50 _QUERY_KEYS = frozenset({"fields", "filters", "sorts", "limit", "cursor", "aggregates", "groupBy"}) + + +def _register_query_paging_contract(): + """Keep the legacy and grouped query schemas aligned with the API's bounded page reader.""" + from tool_groups.data import GROUP + for schema in (external_tools.TOOLS["query_database"]["inputSchema"], + GROUP["actions"]["query_database"]["schema"]): + schema["properties"]["limit"] = { + "type": "integer", "minimum": 1, "description": _QUERY_PAGING_HELP} + + +_register_query_paging_contract() + _MCP_FIXES = { "query_filter_limit": "split it into separate queries", "query_cursor_stale": "run the same query again without a cursor", @@ -197,6 +212,13 @@ class _PreviewRuntime: def get_many(self, names): return {str(name): self.get(name) for name in names or ()} + def list_keys(self, prefix=""): + prefix = str(prefix) + listing = getattr(self._real, "list_keys", None) + keys = set(listing(prefix)) if callable(listing) else set() + keys.update(self._docs) + return sorted(key for key in keys if key.startswith(prefix)) + def get_entry(self, name, entry): return copy.deepcopy((self.get(name) or {}).get(entry)) @@ -2663,6 +2685,26 @@ def _external_image_urls(session, database, rows, records): return records +def _typed_selected_stop_identity(session, database, filters): + """Recognize one typed built-in identity equality without declaring a readable column.""" + if ((session.claims or {}).get("external_via") != "session" + or database not in {"customer_data", "product_data"} + or not isinstance(filters, dict) or set(filters) != {"conj", "nodes"} + or filters.get("conj") != "and"): + return None + nodes = filters.get("nodes") + if not isinstance(nodes, list) or len(nodes) != 1: + return None + leaf = nodes[0] + if (not isinstance(leaf, dict) or set(leaf) - {"id", "colId", "op", "value"} + or leaf.get("colId") != "pid" or leaf.get("op") != "eq"): + return None + identity = leaf.get("value") + if type(identity) not in (int, str) or not str(identity).strip() or len(str(identity)) > 200: + return None + return identity + + def _query_database_page(session: Session, database: str, payload: dict): if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS: raise err(400, "query_arguments_invalid", @@ -2672,15 +2714,16 @@ def _query_database_page(session: Session, database: str, payload: dict): cursor = (_read_query_cursor(session, payload["cursor"]) if payload.get("cursor") is not None else None) limit = payload.get("limit", _QUERY_DEFAULT_LIMIT) - if type(limit) is not int or not 1 <= limit <= _QUERY_MAX_LIMIT: + if type(limit) is not int or limit < 1: requested = limit try: json.dumps(requested, allow_nan=False) except (TypeError, ValueError, OverflowError, RecursionError): requested = None raise err(400, "query_limit_invalid", - f"limit must be between 1 and {_QUERY_MAX_LIMIT}.", + _QUERY_PAGING_HELP, {"limit": _QUERY_MAX_LIMIT, "requested": requested}) + limit = min(limit, _QUERY_MAX_LIMIT) raw_sorts = payload.get("sorts") or [] if not isinstance(raw_sorts, list) or len(raw_sorts) > _QUERY_MAX_SORTS: raise err(400, "query_sorts_invalid", @@ -2691,8 +2734,15 @@ def _query_database_page(session: Session, database: str, payload: dict): # database, field and row walls on every call before this adapter sorts, caps, or aggregates. # ⭐ S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so # its aggregates run in SQL over every permitted row and its rows are read only as needed. + identity = _typed_selected_stop_identity(session, database, payload.get("filters")) + scope_filters = None if identity is not None else payload.get("filters") scoped = _external_read_scope( - session, database, fields=None, filters=payload.get("filters"), mode="query_schema") + session, database, fields=None, filters=scope_filters, mode="query_schema") + if identity is not None and ( + scoped.get("source_kind") != "runtime_cache" + or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters): + raise err(409, "assistant_snapshot_unavailable", + "Selected record reads are unavailable in this deployment. Retry after an update.") mirror = scoped.get("source_kind") == "mirror_user_table" visible_fields = list(scoped.get("fields") or []) visible_keys = { @@ -2865,7 +2915,11 @@ def _query_database_page(session: Session, database: str, payload: dict): if not mirror: scoped = _external_read_scope( - session, database, fields=None, filters=payload.get("filters")) + session, database, fields=None, filters=scope_filters, + **({"identity_pid": identity} if identity is not None else {})) + if identity is not None: + # Keep the identity predicate in the response and signed cursor's query digest. + scoped["filters"] = copy.deepcopy(payload.get("filters")) rows = list(scoped.get("records") or []) inspected_keys = (set(selected_keys) | {rule["field"] for rule in sorts} @@ -3469,6 +3523,54 @@ def _database_value_plan(session, database, record_id, values, fields, *, apply_ return planned +def _database_nav_body(args): + """Preserve omitted metadata fields when addressing the app's navigation writer.""" + return {"key": str(args.get("database") or "").strip(), + **{key: copy.deepcopy(args[key]) for key in ("name", "icon") if key in args}} + + +def _plan_database_rename(session, args): + """Run the app's metadata writer on a private copy and bind its navigation revision.""" + import routes_nav + from core import user_tables + + preview = _PreviewRuntime(session.runtime) + # Metadata validation needs definitions, so do not load record rows into a fresh preview. + source = session.runtime + while isinstance(source, _PreviewRuntime) and user_tables.STORE_KEY not in source._docs: + source = source._real + definitions = user_tables.all_defs(st=source, schema_only=True) + preview._initial[user_tables.STORE_KEY] = copy.deepcopy(definitions) + preview._docs[user_tables.STORE_KEY] = copy.deepcopy(definitions) + before = preview.get(routes_nav._NAV_META_KEY) + revision = preview.revision(routes_nav._NAV_META_KEY) + if type(revision.get("rev")) is not int: + raise err(503, "confirmation_revision_unavailable", + "The database metadata version could not be checked. Nothing was saved.") + preview_session = Session(session.tenant, session.user, session.claims, preview) + result = routes_nav.save_nav_meta(_database_nav_body(args), preview_session) + diff, truncated = _diff_documents({routes_nav._NAV_META_KEY: { + "before": before, "after": preview.get(routes_nav._NAV_META_KEY)}}) + database = result["key"] + return {"diff": diff, "affected": {"ids": [database], "count": 1}, + "warnings": ["The preview diff reached its display cap; split this change."] if truncated else [], + "revision": revision, + "planned": {"database": database, "action": "rename", "meta": result["meta"]}} + + +def _database_record_write_guard(session, database, *, field_keys=None): + """Pass update fields to the shared locked-record wall without unlocking row creation or deletion.""" + import routes_tables + from core import user_tables + + guard = routes_tables._records_or_refuse + if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime): + # A deployment without the shared field-aware wall keeps its existing refusal. + if "field_keys" in inspect.signature(guard).parameters: + return guard(session, database, field_keys=field_keys) + return guard(session, database) + + def _plan_database_write(session, args): import routes_tables from core import user_tables @@ -3476,10 +3578,12 @@ def _plan_database_write(session, args): action = args.get("action") database = str(args.get("database") or "").strip() if action not in {"record_create", "record_update", "record_delete", - "field_create", "field_update", "field_delete"} or not database: - raise err(400, "tool_arguments_invalid", "Name a database and an action that changes records or fields.") + "field_create", "field_update", "field_delete", "rename"} or not database: + raise err(400, "tool_arguments_invalid", "Name a database and an action that changes records, fields or metadata.") _assert_database_scope(session, database) _assert_write_scope(session, "database_object") + if action == "rename": + return _plan_database_rename(session, args) routes_tables._managed_or_refuse(session, database, mutation=True) bucket_before = session.runtime.get(user_tables.STORE_KEY) if not isinstance(bucket_before, dict): @@ -3496,7 +3600,13 @@ def _plan_database_write(session, args): bucket_after = copy.deepcopy(bucket_before) if action.startswith("record_"): - routes_tables._records_or_refuse(session, database) + field_keys = None + if action == "record_update": + raw_values = args.get("values") + if not isinstance(raw_values, dict) or not raw_values: + raise err(400, "tool_arguments_invalid", "Record values must be a non-empty object.") + field_keys = tuple(str(key) for key in raw_values) + _database_record_write_guard(session, database, field_keys=field_keys) g = routes_tables.ut_write_ctx(session, database) rows = table.get("rows") or {} if action == "record_create": @@ -3649,6 +3759,19 @@ def _execute_database_object(session, args, *, plan=None): if action == "query": return _query_database(session, database, args.get("query") or {}) plan = plan or _plan_database_write(session, args) + if action == "rename": + import routes_nav + + guarded_session = Session(session.tenant, session.user, session.claims, + _ConfirmationRuntime(session.runtime, { + routes_nav._NAV_META_KEY: plan.get("revision")})) + try: + result = routes_nav.save_nav_meta(_database_nav_body(args), guarded_session) + except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc: + raise err(409, "confirmation_stale", + "The database metadata changed after the preview, " + _STALE_FIX) from exc + return {"database": database, "action": action, "meta": result["meta"], + "affected": copy.deepcopy(plan.get("affected") or {})} expected_rev = plan.get("revision", {}).get("rev") expected_digest = plan.get("bucket_before_digest") replacement = copy.deepcopy(plan.get("bucket_after")) @@ -4206,12 +4329,11 @@ def _build_about(value): else "Send about as an object with title, description and tags." if not isinstance(value, dict) else "Add " + ", ".join(missing) + " to about." if missing else "Fix " + ", ".join(invalid) + ".") - if not isinstance(value, dict) or "about.tags" in invalid or "about.tags" in missing: - details["allowed"] = tags - message += " Use tags from this list, each once: " + ", ".join(tags) + "." - if "about.title" in invalid or "about.description" in invalid: - message += (" Use a title of 1 to 240 characters and a description of 20 to 240 characters," - " without em or en dashes.") + details["allowed"] = tags + message += (" Use only title (1 to 240 characters), description (20 to 240 characters), and tags. " + "Use no control characters or long dashes in title or description. " + "Use one or more tags from this list, each once: " + ", ".join(tags) + ". " + "Valid metadata: " + json.dumps(example, separators=(", ", ": ")) + ".") raise err(400, "build_about_required" if missing else "build_about_invalid", message, details) from None return cleaned @@ -4561,7 +4683,10 @@ def _operation_access(session, item, *, confirming=False): _external_read_scope(session, database, fields=[], mode="schema") else: _external_read_scope(session, database, fields=[]) - if name == "database_object": + if name == "database_object" and args.get("action") == "rename": + import routes_nav + routes_nav._nav_meta_write_patch(_database_nav_body(args), session) + elif name == "database_object": import routes_tables g = routes_tables.ut_write_ctx(session, args["database"]) action = args["action"] @@ -4768,8 +4893,9 @@ def _committed_build_evidence(session, writer, arguments, *, build_kind=None): for identity in before.keys() | after.keys(): if before.get(identity) != after.get(identity): add("app", (after.get(identity) or before.get(identity) or {}).get("appId")) - elif key in (application_registry._KEY, automation_engine.STORE_KEY): - kind = "app" if key == application_registry._KEY else "automation" + elif (key in (application_registry._KEY, automation_engine.STORE_KEY) + or key.startswith("applications_v1_app/")): + kind = "automation" if key == automation_engine.STORE_KEY else "app" if kind == "app": before, after = before.get("applications") or {}, after.get("applications") or {} for identity in before.keys() | after.keys(): @@ -4947,9 +5073,10 @@ def _confirm_change(session, args): before_revision = copy.deepcopy(recomputed.get("current_revision") or {}) if tool == "database_object": from core import user_tables + import routes_nav + bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY after_revision = _bucket_revision( - fresh_session.runtime, user_tables.STORE_KEY, - fresh_session.runtime.get(user_tables.STORE_KEY)) + fresh_session.runtime, bucket, fresh_session.runtime.get(bucket)) produced_rev = before_revision["rev"] + 1 if after_revision.get("rev") != produced_rev: after_revision = {"rev": produced_rev} @@ -5004,7 +5131,7 @@ _MCP_CODE_GROUPS = ( ("The arguments do not match this action. Read its guide and correct them.", """ tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid - bad_application bad_entries bad_value bad_values bad_request bad_recurrence invalid_field_contract + bad_application bad_entries bad_value bad_values bad_request bad_recurrence bad_icon name_not_allowed invalid_field_contract external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid """), ("You do not have permission to do this with this connection.", """ @@ -5144,6 +5271,7 @@ MCP_RESULT_CODES.update({ "request_method_unknown": "The requested method is not available.", "request_rate_limited": "This AI has reached its hourly request cap. Try again later.", }) +MCP_RESULT_CODES.update(READ_REFUSAL_FIXES) _MCP_INTERNAL_PROSE = re.compile( r"\b(?:DuckDB|Postgres(?:ql)?|SQLite|Polars|Pandas|SQLAlchemy|PyArrow|HyperFormula)\b|" r"\b(?:api|web|platform)_[a-z_]+\b|\b(?:verify_\w+|\w+Error)\b|" @@ -5444,7 +5572,7 @@ MCP_INSTRUCTIONS = "\n".join([ "A build that breaks a required rule is refused with its fix.", "2. Discover before you query: use table.list_databases, then table.describe_database for the database you " "need. Use only field keys that describe_database returns. Never guess a field name. " - "State the age of data older than one day before giving a number.", + "State the age of data older than one day before giving a number. " + DISCOVERY_HELP, "3. Read with table.query_database. Filter and sort on the server, ask only for the fields you need, " "and follow the cursor for more pages instead of reading everything. Report totals from the " "counts the tool returns, not from the rows you happened to read. " @@ -5592,28 +5720,41 @@ def _mcp_database_actions(session, rows): """Resolve database grants through read-only permission predicates.""" from core import grid_events, user_tables import routes_grid + import routes_nav import routes_tables session = Session(session.tenant, session.user, session.claims, user_tables.lend_defs(session.runtime)) allowed = {action: False for action in external_tools._DATABASE_ACTIONS if action not in {"list", "describe", "query"}} for row in rows: database = row["database"] + try: + routes_nav._nav_meta_write_patch({"key": database, "icon": None}, session) + allowed["rename"] = True + except HTTPException: + pass try: routes_tables._managed_or_refuse(session, database, mutation=True) if not user_tables.is_user_table(database, st=session.runtime): continue fields = row.get("fields", []) + ctx = routes_grid._ctx(session, fields, frozenset(), scope_key=database) + editable = [field["key"] for field in fields + if not user_tables.is_computed_cell(field) and field.get("type") != "automation" + and grid_events._may_edit_field_value(ctx, field["key"], field)] try: - routes_tables._records_or_refuse(session, database) - ctx = routes_grid._ctx(session, fields, frozenset(), scope_key=database) - editable = any( - not user_tables.is_computed_cell(field) and field.get("type") != "automation" - and grid_events._may_edit_field_value(ctx, field["key"], field) - for field in fields) - allowed["record_create"] |= editable - allowed["record_update"] |= editable + _database_record_write_guard(session, database) + allowed["record_create"] |= bool(editable) + allowed["record_update"] |= bool(editable) allowed["record_delete"] = True except HTTPException: pass + if not allowed["record_update"]: + for key in editable: + try: + _database_record_write_guard(session, database, field_keys=(key,)) + allowed["record_update"] = True + break + except HTTPException: + pass try: routes_tables._field_or_refuse(session, database) allowed["field_create"] = True @@ -5754,6 +5895,7 @@ def _mcp_capabilities(session, groups, rows): "record_update": "edit record fields", "record_delete": "delete records", "field_create": "add fields as the database owner or an Administrator", "field_update": "edit a visible field", "field_delete": "delete a visible field", + "rename": "change database names or icons as an Administrator", }[operation] + " in a writable database") granted = not required row.update(state=("not supported here" if not supported else @@ -5993,8 +6135,11 @@ def _mcp_tool_specs() -> list[dict]: spec["inputSchema"]["properties"]["about"] = copy.deepcopy(external_tools.ABOUT_SCHEMA) spec["inputSchema"]["properties"]["action"]["enum"] = list(groups[spec["name"]]["actions"]) if spec["name"] == "sheet": - from tool_groups.sheet import WORKBOOK_RESOURCE_URI + from tool_groups.sheet import WORKBOOK_RESOURCE_URI, write_guide + spec["description"] += " " + write_guide() spec["_meta"] = {"ui": {"resourceUri": WORKBOOK_RESOURCE_URI}} + if spec["name"] == "table": + spec["description"] += " " + _QUERY_PAGING_HELP spec["description"] = _mcp_plain(spec["description"], "Read this group's guide for actions.") spec["inputSchema"] = _mcp_schema_copy(spec["inputSchema"]) if "title" in spec.get("annotations", {}): @@ -6233,22 +6378,32 @@ def _mcp_hoist_call_metadata(name, arguments): def _mcp_intent_fix(name): """Name the intent field and where this tool takes it.""" import tool_groups + example = ' Example: {"intent": "Prepare the weekly sales review"}.' if name == "confirm_change": return ("Send intent beside confirmation_token: 1 to 500 characters on why you are confirming this change. " - "A client that sends only action and arguments may put both inside arguments.") + "A client that sends only action and arguments may put both inside arguments." + example) if tool_groups.canonical_call(name, {})[0] in tool_groups.groups(): - return ("Send intent beside action and arguments: 1 to 500 characters on why you are making this write. " - "If your client sends only action and arguments, put intent inside arguments.") - return "Send intent beside this tool's other arguments: 1 to 500 characters on why you are making this write." + fix = ("Send intent beside action and arguments: 1 to 500 characters on why you are making this write. " + "If your client sends only action and arguments, put intent inside arguments." + example) + else: + fix = "Send intent beside this tool's other arguments: 1 to 500 characters on why you are making this write." + example + if name == "sheet" or isinstance(name, str) and name.startswith("sheet_"): + from tool_groups.sheet import write_guide + fix += " " + write_guide() + return fix def _mcp_about_fix(name): """Name where a group call takes about, or nothing for a flat tool.""" import tool_groups + fix = "" if isinstance(name, str) and tool_groups.canonical_call(name, {})[0] in tool_groups.groups(): - return (" Put about beside action and arguments. " - "If your client sends only action and arguments, put about inside arguments.") - return "" + fix = (" Put about beside action and arguments. " + "If your client sends only action and arguments, put about inside arguments.") + if name == "sheet" or isinstance(name, str) and name.startswith("sheet_"): + from tool_groups.sheet import write_guide + fix += " " + write_guide() + return fix def _mcp_requires_intent(name, arguments): @@ -6286,12 +6441,13 @@ def _dispatch_mcp(req, session: Session, base_url: str = ""): return reply -#: Raw refusal codes whose MCP answer names the call that succeeds next (S79-T11). -_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX} +#: Known refusal codes whose MCP answer carries a repair. +_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX, + **READ_REFUSAL_FIXES} def _mcp_next_call(code, message): - """Append the call that succeeds next for a refusal that names a missing item (S79-T11).""" + """Append the supported repair once for a known read or missing-item refusal.""" text = str(message or "").rstrip() fix = _MCP_NEXT_CALL.get(code, "") if code == "assistant_unknown_database": @@ -6507,7 +6663,7 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""): if info.get("code") == "workbook_name_ambiguous" and isinstance(details, dict): retry["candidates"] = details.get("candidates", []) message = info.get("message", "The app refused this operation.") - if info.get("code") == "build_about_required" and isinstance(message, str): + if info.get("code") in {"build_about_required", "build_about_invalid"} and isinstance(message, str): message += _mcp_about_fix(called) if info.get("code") in _MCP_NEXT_CALL: message = _mcp_next_call(info["code"], message) diff --git a/api/routes_geo.py b/api/routes_geo.py index c43bf9441efa92d5c66f52b35ea5b528391d396c..2f049b8f979dbbe2903d8571e80fc57e77318495 100644 --- a/api/routes_geo.py +++ b/api/routes_geo.py @@ -338,7 +338,7 @@ def _zone_application(session, table_key, application, *, write=False): import application_registry as registry from core import route_system try: - doc = registry._read_strict(session.runtime, registry._KEY) or {} + doc = registry.load(session.runtime, strict=True) if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict): raise ValueError("invalid Application registry") except Exception: diff --git a/api/routes_grid.py b/api/routes_grid.py index bc0fda74833bdc8c8e0288702f7e06586b3d7c39..48d781d98405eafc69dc4173bbce6e45ea008a08 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -6,16 +6,7 @@ same `core.grid_events.handle_events` the Streamlit adapter runs. That is the wh EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated anything here would be a second wall to keep in step, and the two would drift on the first change. -DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's -`seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends -its recent 24-event window on every emit inside one page session. A stateless API has no such -dict, and inventing a per-user server-side one would be exactly the resident per-tenant state -EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole -purpose — a batch that repeats an id processes it once), and a genuinely replayed request is -handled by the operations being idempotent. The one event where a replay is observable is -`add_to_list` (it would add the same pids to the same cohort twice — a set union, so the -membership is unchanged, but the toast count repeats). Noted rather than papered over; a -server-side idempotency key belongs with D2's session mirror (C-2). +Edit receipts retain each event's first result in the tenant store across requests and restarts. STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises `StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read @@ -23,6 +14,8 @@ back. A 200 over a write that evaporated is the failure this rule exists to prev """ import datetime as dt import time +from functools import wraps +from inspect import signature from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query @@ -37,6 +30,70 @@ MODULE = "customer_data" _MAX_EVENTS = 24 +def row_patch_receipt(scope): + """Retain the complete row PATCH response and skip its writes and hooks on replay.""" + def decorate(fn): + sig = signature(fn) + + @wraps(fn) + def wrapped(*args, **kwargs): + from core import edit_receipts + bound = sig.bind(*args, **kwargs) + body = bound.arguments.get('body') + if not isinstance(body, dict) or 'opId' not in body: + return fn(*args, **kwargs) + try: + identity = edit_receipts.op_id(body['opId']) + except ValueError as exc: + raise err(400, 'bad_op_id', str(exc)) from exc + session = bound.arguments['session'] + scope_key = scope if isinstance(scope, str) else scope(bound.arguments) + bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'} + + def apply(): + try: + answer = dict(fn(*bound.args, **bound.kwargs), opId=identity) + return {'answer': answer, 'event': edit_receipts.current_event()} + except HTTPException as exc: + if exc.status_code >= 500: + raise + detail = exc.detail + if isinstance(detail, dict): + detail = dict(detail, opId=identity) + error = (detail.get('error') or {}) if isinstance(detail, dict) else {} + refusal = {'event': 'overlay_patch', 'code': error.get('code') or 'write_refused', + 'message': error.get('message') or str(detail)} + event = edit_receipts.current_event() or { + 'rerender': True, 'write': {'ok': False, 'value': None, 'refused': True, + 'reason': refusal['code'], 'opId': identity}, + 'refusals': [refusal], 'doc': None, 'toast': None} + return {'status': exc.status_code, 'detail': detail, + 'headers': exc.headers, 'event': event} + + try: + result = edit_receipts.run_once( + session.runtime, session.uname, + edit_receipts.operation(scope_key, 'overlay_patch', bound.arguments['pid']), + identity, apply) + except edit_receipts.ReceiptUnavailable as exc: + raise err(503, 'store_unavailable', str(exc)) from exc + if 'status' in result: + raise HTTPException(result['status'], result['detail'], headers=result['headers']) + if 'answer' in result: + return result['answer'] + # A grid event may have committed this operation before its row PATCH replay. + write = result['write'] + answer = {'pid': bound.arguments['pid'], 'updates': write.get('value') or {}, + 'opId': identity} + if write.get('refused'): + answer['refused'] = list(bound.arguments['body']) + answer['reason'] = write.get('reason') or 'write_refused' + answer['refusals'] = result.get('refusals') or [] + return answer + return wrapped + return decorate + + def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw): from core import grid_events import aios_grid @@ -611,59 +668,6 @@ def workspace(scope: str = "customer", return {"workspace": workspace} -def _shared_note_subject(session, scope, view_id): - """Resolve a note subject only after the ordinary workspace wall admits it.""" - scope = _scope_or_400(scope) - payload = workspace(scope=scope, session=session)["workspace"] - if view_id: - view = next((item for item in (payload.get("views") or []) - if str(item.get("id") or "") == str(view_id)), None) - if view is None: - raise err(404, "view_not_found", "that View is not visible in this database") - return "database", _database_key_for_scope(scope), view - return "database", _database_key_for_scope(scope), None - - -@router.get("/grid/shared-note") -def get_shared_note(scope: str = "customer", viewId: str = "", - session: Session = Depends(require_session)): - """Return the database note from any visible View.""" - from core import user_tables - - kind, subject_id, _view = _shared_note_subject(session, scope, viewId) - return {"note": user_tables.shared_note(kind, subject_id, st=session.runtime)} - - -@router.put("/grid/shared-note") -def update_shared_note(scope: str = "customer", viewId: str = "", - body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Edit a shared note through the same database/View permission wall as the grid.""" - from core import shares, user_tables - - kind, subject_id, view = _shared_note_subject(session, scope, viewId) - may_edit = bool(session.admin) - if subject_id.startswith("ut_"): - may_edit = user_tables.may_edit_database( - subject_id, session.uname, is_admin=session.admin, st=session.runtime) - if view is not None and not may_edit: - owner = str(view.get("createdBy") or view.get("owner") or "").strip().lower() - may_edit = bool( - owner == str(session.uname or "").strip().lower() - or shares.may_edit("view", str(view.get("id") or viewId), session.uname, - is_admin=session.admin, st=session.runtime) - ) - if not may_edit: - raise err(403, "note_read_only", "your role does not allow note edits") - try: - note = user_tables.put_shared_note( - kind, subject_id, (body or {}).get("body"), session.uname, st=session.runtime) - except ValueError as exc: - raise err(400, "note_too_long", str(exc), - {"limit": user_tables.SHARED_NOTE_MAX_CHARS, "unit": "characters"}) from exc - return {"note": note} - - _RECURRENCE_CONDITION_OPS = frozenset({"eq", "neq", "isNotEmpty", "between"}) _RECURRENCE_ORDERED_TYPES = frozenset({ "date", "datetime", "int", "integer", "number", "currency", "pct", "rating", @@ -2120,6 +2124,7 @@ def grid_events_route(body: dict = Body(default=None), # simply had not been given them — the standalone shell could read measures it could # never write. ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, via=via, + durable_receipts=True, permission_fields=(g.get('_permission_scope') or {}).get('fields'), measure_keys=frozenset(m["key"] for m in g["measures"]), resolved_ids=frozenset(g["measure_sets"]), @@ -2134,7 +2139,7 @@ def grid_events_route(body: dict = Body(default=None), results = [] try: for one in events: - eid = str(one.get("id") or "") if isinstance(one, dict) else "" + eid = str(one.get("opId", one.get("id")) or "") if isinstance(one, dict) else "" # ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler # appends to a shared list, so the refusals belonging to THIS event are exactly the # ones that appeared across THIS call. Reading the list once after the loop would @@ -2142,20 +2147,20 @@ def grid_events_route(body: dict = Body(default=None), # the same class of unfalsifiable answer the channel exists to end. _before = len(ctx.out.refusals) rerender = grid_events.handle_one(one, ctx) - row = {**ctx.out.writes.get(eid, {}), "id": eid, "rerender": bool(rerender)} + row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid, + "rerender": bool(rerender)} mine = ctx.out.refusals[_before:] if mine: # `refused` is the SHAPE a caller branches on; the first reason is the one that # stopped this write (a handler returns at its first refusal). row["refused"] = mine[0] results.append(row) - except shared_overlay.WriteUnavailable as exc: - refusal = err(503, "store_unavailable", str(exc)) - refusal.detail.update(ok=False, value=None, refused=True, reason="store_unavailable") + except (shared_overlay.WriteUnavailable, grid_events.StoreUnavailable) as exc: + refusal = err(503, "store_unavailable", + "This edit could not be confirmed as saved. Retry with the same op id. " + "Earlier operations in this batch may have been saved.") + refusal.detail.update(ok=False, value=None, refused=True, reason="store_unavailable", opId=eid) raise refusal from exc - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable, so none of your changes were saved") out = {"results": results, "rerender": any(r["rerender"] for r in results)} if ctx.out.refusals: diff --git a/api/routes_keychain.py b/api/routes_keychain.py index f51c39b057c6e58a0c894836d533a1b85cc8bc9c..c75eaaf1b04fc3489e93b1724fb66b2666c568c0 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -592,11 +592,12 @@ def migrate_env_odoo(rt): ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a third-party failure must not take the boot down. """ - out = {"done": False, "entry": "", "carried_pause": False, "why": ""} + out = {"done": False, "entry": "", "carried_pause": False, "why": "", "pending": False} if not env_odoo_available(rt): # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. out["why"] = "this tenant has no environment Odoo credential to migrate" return out + out["pending"] = _kc().first_entry_of_type(rt, "odoo") is None fields, why = _env_odoo_fields() if not fields: out["why"] = why @@ -612,9 +613,11 @@ def migrate_env_odoo(rt): row, why = _kc().ensure_entry_of_type( rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") if row is None: - out["why"] = why + out["why"] = ("Odoo credential migration is pending. Check the keychain and try again." + if out["pending"] else why) return out out["done"], out["entry"] = True, row["id"] + out["pending"] = False if was_paused: # The freeze followed the credential. Without this the connector silently RESUMES pulling # live Odoo at the first boot after the migration. @@ -629,16 +632,36 @@ def migrate_env_odoo(rt): return cur try: - rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + flags = rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + saved = flags.get(row["id"]) if isinstance(flags, dict) else None + if not isinstance(saved, dict) or saved.get("paused") is not True: + raise RuntimeError("The migrated connection pause was not confirmed.") out["carried_pause"] = True - except Exception as exc: # noqa: BLE001 - # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse - # than one that did not run, so this is the one failure that must never be silent. - out["why"] = (f"the credential migrated but the PAUSE could not be carried over " - f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") + except Exception: # noqa: BLE001 + # Keep the paused environment source authoritative when pause transfer fails. + try: + _kc().delete_entry(rt, row["id"]) + out["done"], out["entry"] = False, "" + out["why"] = "Odoo credential migration is pending. The paused environment connection was kept." + except Exception: + out["why"] = "Odoo credential migration is pending. An administrator must restore the connection pause." + out["pending"] = True return out +def env_odoo_migration_pending(rt): + """Disclose an environment-only connection without decrypting or migrating during a read.""" + if not env_odoo_available(rt): + return False + entries = _kc().list_entries(rt, "odoo") + if not entries: + return True + flags = rt.get(_CONNECTOR_FLAGS_KEY) or {} + return bool(_rel().frozen(rt) and (flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused") and any( + row.get("createdBy") == "system" and row.get("label") == "Odoo (migrated from this deployment)" + and not (flags.get(row["connection_id"]) or {}).get("paused") for row in entries)) + + def _own_row(session, entry_id): """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the caller cannot see: telling a member that somebody else's personal credential EXISTS is the @@ -735,9 +758,11 @@ def _register_credential_tables(): try: from core import user_tables as ut import routes_connectors as rc + import amazon_relational # Instruction 30 / C13 names the Amazon Data Protection Policy, not every connector. ut.register_credential_tables( "amazon_sp", rc.CONNECTED_PREFIXES["amazon_sp"], on_revoke=ut.REVOKE_PURGE, + owner=amazon_relational.credential_table_owner, why="The Amazon Data Protection Policy requires deleting the seller data " "authorised by a revoked credential.") # W32 R10's second sentence and D-259 require these grids to survive a reconnect. @@ -763,9 +788,13 @@ def _register_credential_tables(): def _credential_revocation(row, rt): """A failed census is UNKNOWN, never evidence of an empty seller footprint.""" registered = _register_credential_tables() - etype, cid = str(row.get("type") or ""), str(row["id"]) + etype = str(row.get("type") or "") + cid = str(row.get("connection_id") or row.get("connectionId") or row["id"]) if registered: try: + if etype == "amazon_sp": + import amazon_relational + return registered, amazon_relational.connection_footprint(rt, cid) from core import user_tables as ut return registered, ut.credential_footprint(etype, cid, rt=rt, st=rt) except Exception as exc: @@ -787,7 +816,7 @@ def _revocation_residue(entry_type): return [] return [{ "where": "The tenant DuckDB mirror", - "why": "Connection removal clears product databases but not the raw Amazon mirror.", + "why": "Connection removal queues cleanup of this connection's Amazon data.", "fix": "If automatic deletion could not be scheduled, after every Amazon credential is revoked an operator must run " "platform/harness/amazon_store.py --purge --tenant " "--confirm-tenant and retain the residue-free receipt.", @@ -822,18 +851,22 @@ def _amazon_lifecycle_warning(session, action): return {"warning": warning} -def _amazon_deletion_receipt(session, *, product_purge_succeeded=True): +def _amazon_deletion_receipt(session): try: - if not product_purge_succeeded: - raise RuntimeError("product cleanup is incomplete") - from harness import amazon_store as az rt = session.runtime - az.record_deletion_trigger(rt.key, "revocation", rt=rt, principal=session.uname) + if any(entry.get("type") == "amazon_sp" for entry in _kc().list_entries(rt)): + return {} + from harness import amazon_store as az + active = az._active_deletion_triggers(rt.key, az._ledger(rt.key, rt), az._utc()) + if not any(trigger["kind"] == "revocation" for trigger in active): + az.record_deletion_trigger(rt.key, "revocation", rt=rt, principal=session.uname) plan = az.scheduled_purge(rt.key, rt) receipt = {"purgeScheduledAt": plan["purgeAt"], "deadlineAt": plan["deadlineAt"], "residue": [{"where": "Amazon data in this workspace", "why": az.deletion_schedule_message(plan), - "fix": "Add the Amazon credential again before the scheduled date to cancel deletion."}]} + "fix": ("Add the Amazon credential again before the scheduled date to cancel deletion." + if plan["trigger"]["kind"] in ("revocation", "authorization_loss") else + "Contact an administrator to track this deletion obligation.")}]} deadline = az._utc(plan["deadlineAt"]).date().isoformat() except Exception: return {**_amazon_lifecycle_warning(session, "revocation"), @@ -855,6 +888,22 @@ def _amazon_deletion_receipt(session, *, product_purge_succeeded=True): return receipt +def _queue_amazon_cleanup(session, connection_id): + """Wake the resident writer while durable revocation metadata retains retry ownership.""" + result = {"connectionId": connection_id, "cleanupPending": True, + "purged": {"tables": [], "rows": 0, "rowsKnown": False, + "refused": [], "failed": []}} + try: + from core import connector_sync + try: + connector_sync.request_sync("amazon_sp", session.runtime.key, requested_by=session.uname) + finally: + connector_sync.sync_wake_event().set() + except Exception: + result["warning"] = "Amazon cleanup is pending. Retry removal to wake the cleanup worker." + return result + + def _write_revocation_warning(rt, entry_id, warning=None): """Prune expired consent on every write; failed persistence requires explicit confirmation.""" now = int(time.time()) @@ -965,6 +1014,7 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_ raise err(400, "bad_initial_selection", str(exc)) except Exception: raise err(503, "store_unavailable", "The database choices were not saved. Try again.") + existing_odoo = etype == "odoo" and kc.resolve_connection_id(session.runtime, "odoo") is not None try: row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), body.get("fields"), session.uname) @@ -986,10 +1036,8 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_ # of a side bucket is to publish a credential, not to hide one. So the second write is not # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was # stored. `business` needs no row at all, so this branch is the only one that can be partial. - # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing - # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin - # find a second switch. The tables were never dropped, so "resume" is one flag. - if etype == "odoo": + # The first replacement key resumes frozen data. Adding a sibling preserves the existing freeze. + if etype == "odoo" and not existing_odoo: try: _rel_reconnect(session.runtime) except Exception: # noqa: BLE001 @@ -1108,6 +1156,18 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi # another person's personal entry's 404, but ids have 12 random hex characters and are not # enumerable. A 404 on a lost response's retry would break this policy path's idempotence. if not any(entry["id"] == str(entry_id) for entry in entries): + import amazon_relational + revocations = rt.get(amazon_relational.CONNECTION_REVOCATIONS_KEY) or {} + for cid, pending in revocations.items(): + if (isinstance(pending, dict) and pending.get("state") == "pending" + and any(str(entry.get("id")) == str(entry_id) + and str(entry.get("connection_id")) == str(cid) + for entry in pending.get("entries", []))): + if not session.admin: + raise err(403, "admin_only", "only an administrator may retry Amazon cleanup") + lifecycle = _amazon_deletion_receipt(session) + cleanup = _queue_amazon_cleanup(session, cid) + return {"ok": True, "alreadyGone": True, **lifecycle, **cleanup} return {"ok": True, "alreadyGone": True, "purged": {"tables": [], "rows": 0, "rowsKnown": True, "refused": [], "failed": []}} @@ -1145,7 +1205,9 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi if stored else "The warning could not be saved. " "Pass confirm=purge to confirm explicitly.") raise err(409, "confirm_purge", - f"Removing this connection also deletes {len(tables):,} {kind} {noun}" + (f"Removing this connection also deletes its rows from {len(tables):,} {kind} {noun}" + if row["type"] == "amazon_sp" else + f"Removing this connection also deletes {len(tables):,} {kind} {noun}") + count + "That data cannot be recovered. " + next_step) class _Ownership: @@ -1158,26 +1220,44 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi return {"entries": {entry["id"]: entry for entry in entries}} return rt.get(name) - # Credential FIRST, data second. A crash leaves revoked access and residual tables that can - # be swept again; the reverse leaves live credentials over deleted data we cannot read back. - # The absent-id retry above is only a no-op receipt, not proof that a crashed sweep completed. + # Save Amazon ownership before removal so a restart can resume cleanup without credentials. + if row["type"] == "amazon_sp": + import amazon_relational + try: + cid = str(next(entry["connection_id"] for entry in entries + if entry["id"] == str(entry_id))) + amazon_relational.record_connection_revocation(rt, cid, entries_before=entries) + except Exception: + raise err(503, "store_unavailable", "Amazon cleanup was not recorded. The credential was kept. Try again.") + + removal = _pause_odoo_removal(session, row) if row["type"] == "odoo" else {} try: kc.delete_entry(rt, entry_id) - _write_scope(rt, entry_id, DEFAULT_SCOPE, "") + if row["type"] != "amazon_sp": + _write_scope(rt, entry_id, DEFAULT_SCOPE, "") except Exception: raise err(503, "store_unavailable", "The delete did not land. Try again.") + if row["type"] == "amazon_sp": + scope_warning = {} + try: + _write_scope(rt, entry_id, DEFAULT_SCOPE, "") + except Exception: + scope_warning["warning"] = "The credential was removed, but its scope metadata could not be cleared." + _write_revocation_warning(rt, entry_id) + lifecycle = _amazon_deletion_receipt(session) + cleanup = _queue_amazon_cleanup(session, cid) + return {"ok": True, "registry": registered, "footprint": footprint, + **scope_warning, **lifecycle, **cleanup} if not purges: _write_revocation_warning(rt, entry_id) - lifecycle = (_amazon_deletion_receipt(session, product_purge_succeeded=footprint["rowsKnown"]) - if row["type"] == "amazon_sp" else {}) if footprint["rowsKnown"]: - return {"ok": True, **lifecycle} + return {"ok": True, **removal} # Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes # that incomplete cleanup explicit instead of returning an invented "nothing to purge". return {"ok": True, "registry": registered, "footprint": footprint, "purged": {"tables": [], "rows": 0, "rowsKnown": False, "refused": [], "failed": []}, - "residue": _revocation_residue(row["type"]), **lifecycle} + "residue": _revocation_residue(row["type"]), **removal} from core import user_tables as ut try: result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt) @@ -1186,14 +1266,11 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [], "failed": [{"key": key, "error": type(exc).__name__} for key in tables]} _write_revocation_warning(rt, entry_id) - lifecycle = (_amazon_deletion_receipt( - session, product_purge_succeeded=not result["failed"] and not result["refused"]) - if row["type"] == "amazon_sp" else {}) return {"ok": True, "purged": {"tables": result["purged"], "rows": result["rows"], "rowsKnown": result["rowsKnown"], "refused": result["refused"], "failed": result["failed"]}, - "residue": _revocation_residue(row["type"]), **lifecycle} + "residue": _revocation_residue(row["type"])} @router.post("/admin/keychain/{entry_id}/test") @@ -1625,8 +1702,47 @@ def _rewrite_server_db(session, server_db): return f"the connection now points at the {server_db} database" +def _pause_odoo_removal(session, entry): + """Pause the addressed mirror before deletion and block the last key's environment fallback.""" + rt = session.runtime + cid = str(entry.get("connectionId") or entry.get("connection_id") or entry["id"]) + siblings = [row for row in _kc().list_entries(rt, "odoo") if row["id"] != entry["id"] + and entry_scope(rt, row["id"])[0] != "personal"] + last = not siblings and entry.get("scope") != "personal" + keys = {cid} + if last and str(rt.key) == ENV_ODOO_TENANT: + keys.add(_ENV_ODOO_FLAG_KEY) + + def pause(cur): + cur = dict(cur or {}) + for key in keys: + cur[key] = {**(cur.get(key) or {}), "paused": True} + return cur + + try: + flags = rt.update(_CONNECTOR_FLAGS_KEY, pause, flush="sync") + if not isinstance(flags, dict) or not all( + isinstance(flags.get(key), dict) and flags[key].get("paused") is True for key in keys): + raise RuntimeError("The Odoo pause was not confirmed.") + if last: + import datetime as dt + + def freeze(cur): + return {**(cur or {}), "frozen": True, + "frozenAt": dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + "frozenBy": str(session.uname)} + + config = rt.update(_rel().CONFIG_KEY, freeze, flush="sync") + if not isinstance(config, dict) or config.get("frozen") is not True: + raise RuntimeError("The Odoo freeze was not confirmed.") + except Exception: + raise err(503, "store_unavailable", "The Odoo pause could not be saved. No key was removed. Try again.") from None + return {"pausedMirror": True, "connectionId": cid, "frozen": last or _rel().frozen(rt), + "note": "This Odoo connection is paused. Its saved data stays available."} + + @router.post("/admin/connectors/odoo/disconnect") -def odoo_disconnect(session: Session = Depends(require_session)): +def odoo_disconnect(session: Session = Depends(require_session), connection_id: str | None = None): """R10 — remove the credential and FREEZE the grids as static data. ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the @@ -1640,6 +1756,11 @@ def odoo_disconnect(session: Session = Depends(require_session)): careful. """ _odoo_admin(session) + if connection_id is not None: + entry = _odoo_entry(session, connection_id) + if entry is None: + raise err(404, "no_entry", "no such Odoo connection") + return delete_key(entry["id"], session) rel = _rel() import datetime as _dt entry = _odoo_entry(session) @@ -1695,14 +1816,21 @@ def odoo_disconnect(session: Session = Depends(require_session)): paused_mirror = False if flag_key: + pause_keys = {str(flag_key)} | { + str(row["connection_id"]) for row in _kc().list_entries(session.runtime, "odoo")} + if str(session.runtime.key) == ENV_ODOO_TENANT: + pause_keys.add(_ENV_ODOO_FLAG_KEY) + def _pause(cur): cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": True} + for key in pause_keys: + cur[key] = {**(cur.get(key) or {}), "paused": True} return cur try: flags = session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") - saved = flags.get(str(flag_key)) if isinstance(flags, dict) else None - if not isinstance(saved, dict) or saved.get("paused") is not True: + if not isinstance(flags, dict) or not all( + isinstance(flags.get(key), dict) and flags[key].get("paused") is True + for key in pause_keys): raise RuntimeError("the Odoo pause was not confirmed") paused_mirror = True except Exception: # noqa: BLE001 diff --git a/api/routes_nav.py b/api/routes_nav.py index 2999091b63ba1ad058a3d3588a4483c6aa600216..ab7a03198d344f5a0c6adc0786381e6b2fa46040 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -18,6 +18,7 @@ from datetime import datetime, timezone from fastapi import APIRouter, Body, Depends from deps import Session, err, perms, require_session +from core import view_audit import application_registry router = APIRouter(prefix="/api/v1") @@ -879,15 +880,14 @@ def nav(session: Session = Depends(require_session)): # # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the # de5037f share-grant admission widened `may_open`, so a row's presence now includes - # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are - # therefore answered from the DEFINITION: creator-or-admin strictly, matching the - # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no - # controls — the honest shape (the old code offered rename to users the route 403'd). + # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` + # use separate walls: metadata requires an administrator, while deletion also allows + # the creator. A grantee sees the row without either control. key = str(p.get("key", "")) if key.startswith("ut_"): _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") _mine = bool(session.admin or (_creator and _creator == session.uname)) - p["manage"] = _mine + p["manage"] = bool(session.admin) # S73-T64: user_tables refuses to delete the activity log, so the rail offers no Delete. p["canDelete"] = _mine and key != "ut_activity" # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. @@ -1017,44 +1017,14 @@ def nav_opened(body: dict = Body(default=None), return {"key": key, "at": stamp} -@router.post("/nav/meta") -def save_nav_meta(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. - - A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the - asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so - replacing the document whole is what makes a deleted folder stay deleted. This bucket is - shared by every admin in the tenant: a wholesale write here means whoever saves last - silently erases what the other one named while their tab was open. - - THREE WALLS, all fail-closed: - · the SESSION must be able to open the key at all (the same predicate the nav uses, so - the rail and this route cannot disagree about what exists); - · the WRITE is admin-only — renaming a database is a change every user in the tenant - sees, which is the definition of an administrative act here; - · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry - literals: honouring an override would leave this payload and `core/registry.py` - calling the same module two different things, and the wave-16 lesson is that the - client must not be the place that decides what a payload meant. - - `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an - icon change two independent writes rather than a race between them. - """ +def _nav_meta_write_patch(body, session): + """Validate tenant-wide database metadata through the app's administration wall.""" body = body if isinstance(body, dict) else {} key = str(body.get("key") or "").strip() if not key: raise err(400, "bad_request", "no database was named") - # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── - # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. - # - # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what - # `user_tables.may_open` already means. A database you made is yours to name; requiring - # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` - # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, - # because a user table is not a module. Same split `/nav/schema/{key}` makes. - # · everything else — admin only, and icon only. There is no owner of `customer_data` to - # defer to, and its label is a compiled registry literal (refused below regardless). + if not session.admin: + raise err(403, "forbidden", "only an administrator can change a database's name or icon") if key.startswith("ut_"): import core.user_tables as user_tables # This write edits metadata, never a record. The table wall therefore needs its @@ -1065,9 +1035,6 @@ def save_nav_meta(body: dict = Body(default=None), key, session.uname, session.admin, st=definitions): raise err(403, "forbidden", "that database belongs to another user") else: - if not session.admin: - raise err(403, "forbidden", - "only an administrator can change a built-in database's icon") session.require(key) patch = {} @@ -1090,6 +1057,14 @@ def save_nav_meta(body: dict = Body(default=None), patch["name"] = name if not patch: raise err(400, "bad_request", "nothing to change") + return key, patch + + +@router.post("/nav/meta") +def save_nav_meta(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Patch supplied tenant-wide name/icon fields, with an explicit null icon clearing its override.""" + key, patch = _nav_meta_write_patch(body, session) if not session.runtime.available(): raise err(503, "store_unavailable", @@ -1114,6 +1089,8 @@ def save_nav_meta(body: dict = Body(default=None), try: session.runtime.update(_NAV_META_KEY, _up) + except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable): + raise except Exception: raise err(503, "store_unavailable", "the change was not saved: the store refused the write.") diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index 53c1634bc375c7206acceda87dd8d2a342249aae..d34a6584726499f6532e1cd485fde44233d5e865 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -25,6 +25,8 @@ import json import math import threading import time +from collections import OrderedDict +from copy import deepcopy from contextlib import contextmanager from fastapi import Body, Depends, HTTPException @@ -108,7 +110,7 @@ def _table_cursor(rt, table_key=None): `rt=None` is the unguarded legacy caller `whole_pool`/`population` already documents. """ from harness import datastore - if str(table_key or "").startswith("ut_amzn_"): + if str(table_key or "").startswith(("ut_amzn_", "ut_meta_")): if rt is not None: rt.assert_datastore_matches() return datastore.ro_cursor() @@ -219,8 +221,9 @@ def odoo_tables_status(brief: bool = Query(False), definitions = ut.all_defs(st=session.runtime) lent = ut.lend(session.runtime, **{ut.STORE_KEY: definitions}) for key, definition in definitions.items(): - if (str(key).startswith("ut_amzn_") and isinstance(definition, dict) - and definition.get("readThrough") is True + if (isinstance(definition, dict) + and ((str(key).startswith('ut_amzn_') and definition.get('readThrough') is True) + or (str(key).startswith('ut_meta_') and key in sources)) and perm_scope.may_read(session.user, key, st=lent)): tables[key] = {"readThrough": True} return {"ok": True, "tables": tables, "catchUp": catch_up, @@ -1260,6 +1263,11 @@ def _odoo_sources(): # S72-T43 / R107: the audited Odoo columns. Their presets are declared in `odoo_relational`. GRID_SOURCES = rel.merge_audit_cols(GRID_SOURCES) for key, spec in GRID_SOURCES.items(): + owner = spec['id'].rsplit('.', 1)[0] + '.' if '.' in spec['id'] else '' + spec['cols']['Store'] = (owner + 'Store', _s) + spec['cols']['_connection_id'] = (owner + '_connection_id', _s) + if 'odoo_id' in spec['cols']: + spec['cols']['odoo_id'] = (owner + '_source_id', spec['cols']['odoo_id'][1]) if key in rel.READ_THROUGH_KEYS: spec["mirror_only"] = True return GRID_SOURCES @@ -1291,6 +1299,69 @@ def _amazon_sources(): return {} +def _meta_source(cur, key): + """Bind Meta's current mirror schema and the native connection-aware row identity.""" + import meta_relational as meta + from core import connector_spec + + if key not in meta.SOURCE: + return None + table = meta.SOURCE[key] + columns = meta.mirror_columns(cur, table) + if not columns: + return None + quote = lambda name: '"' + name.replace('"', '""') + '"' + names = {name for name, _kind in columns} + cols = {name: (quote(name), lambda value: '' if value is None else value) + for name, _kind in columns} + if '_row_key' in names: + payload = "'[\"" + key + "\",[' || to_json(CAST(\"_row_key\" AS VARCHAR)) || ']]'" + identity = f"CAST('0x' || substr(sha256({payload}), 1, 13) AS BIGINT)" + elif '_connection_id' in names: + raise err(409, 'meta_identity_unavailable', 'This Meta mirror needs connection-aware row identities.') + elif meta.IDENTITY[key] == ('id',): + identity = 'TRY_CAST("id" AS BIGINT)' + else: + values = ', '.join(f'COALESCE(CAST({quote(name)} AS VARCHAR), \'\')' for name in meta.IDENTITY[key]) + payload = "'[\"" + key + "\",' || to_json(list_value(" + values + ")) || ']'" + identity = f"CAST('0x' || substr(sha256({payload}), 1, 13) AS BIGINT)" + live = {target: {name for name, _kind in meta.mirror_columns(cur, source)} + for target, source in meta.SOURCE.items()} + links = connector_spec.system_link_fields('meta', live).get(key, []) + scoped = {'account_id': '_account_key', 'campaign_id': '_campaign_key', + 'adset_id': '_adset_key', 'ad_id': '_ad_key', 'id': '_row_key'} + for field in links: + bag = field['link'] + target = bag['table'] + if '_connection_id' in names or '_connection_id' in live[target]: + left, right = scoped.get(bag['from']), scoped.get(bag['on']) + if left not in names or right not in live[target]: + raise err(409, 'meta_identity_unavailable', 'This Meta relation needs connection-aware identities.') + bag.update({'from': left, 'on': right}) + return {'table': table, 'id': identity, 'page_key': table + '.rowid', 'where': '', + 'cols': cols, 'mirror_only': True, '_fields': meta._fields_for(cur, key) + links} + + +def _meta_sources(): + """Declare Meta grids without retaining another tenant's mirror schema.""" + import meta_relational as meta + return {key: {'table': table, 'id': 'id', 'where': '', 'cols': {}, 'mirror_only': True} + for key, table in meta.SOURCE.items()} + + +def meta_source_fields(spec, fields): + """Merge current Meta-owned columns while retaining authored field settings.""" + import meta_relational as meta + declared = {field['key']: field for field in spec.get('_fields') or []} + for field in fields: + if field['key'] not in declared or not meta._machine_preset(field): + declared[field['key']] = field + else: + declared[field['key']] = _ut().keep_user_edits(field, dict( + declared[field['key']], automation=field.get('automation'))) + return list(declared.values()) + + def require_amazon_mirror(table_key, session, definition=None, st=None): """Admit one Amazon grid and return its binding, or None for a stored snapshot.""" if not str(table_key).startswith("ut_amzn_"): @@ -1317,6 +1388,7 @@ def require_amazon_mirror(table_key, session, definition=None, st=None): register_source_provider(_odoo_sources) register_source_provider(_amazon_sources) +register_source_provider(_meta_sources) register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned) @@ -1543,6 +1615,10 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku "This Amazon database could not be resolved. Retry the request.") from exc if spec is not None and spec.get("missing"): raise err(503, "amazon_mirror_not_loaded", spec["missing"]) + if spec is not None and '_connection_id' in { + name for name, _kind in amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)}: + spec = {**spec, 'cols': {**spec['cols'], '_connection_id': ( + '"_connection_id"', lambda value: '' if value is None else str(value))}} return spec @@ -1573,6 +1649,8 @@ def _source_for(cur, table_key, *, allow_lookups=True): # table and turns Products ordered into the wrong 404. if module_cache_source(key) is not None: return None + if key.startswith('ut_meta_'): + return _meta_source(cur, key) if key.startswith("ut_amzn_"): import amazon_relational as amazon if key in amazon.SOURCE: @@ -1632,6 +1710,8 @@ def _source_for(cur, table_key, *, allow_lookups=True): excluded = ", ".join(str(i) for i in ids) or "-1" cols = {} for key, (expr, cast) in spec["cols"].items(): + if key == 'odoo_id' and expr.endswith('_source_id') and _degrade(expr, have_by_alias) == 'NULL': + expr = spec['id'] cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr, have_by_alias), cast) if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"): @@ -1667,7 +1747,9 @@ FILTERED_SELECTION_SECONDS = 5 * 60 #: `filter_sql` speaks the CLIENT's type vocabulary (`types.ts isNumericType`), and two of our #: field kinds are spelled differently there. Mapped in one place so the pushdown and the grid #: cannot disagree about whether a column is text or a number. -_FILTER_TYPE = {"select": "status", "checkbox": "text", "float": "int"} +_FILTER_TYPE = {"select": "status", "float": "int"} +_SOURCE_PRIVATE = frozenset({'_connection_id', '_row_key', '_source_id', '_account_key', + '_campaign_key', '_adset_key', '_ad_key'}) def _if_references(defn): @@ -1743,6 +1825,8 @@ def _filter_columns(spec, fields): by_key = {f["key"]: f for f in fields} out = {} for key, (expr, _cast) in spec["cols"].items(): + if key in _SOURCE_PRIVATE: + continue ftype = str((by_key.get(key) or {}).get("type") or "text") out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False} if key == "customer_type" and (by_key.get(key) or {}).get("source") == "odoo": @@ -1864,6 +1948,12 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=( trail = (*trail, table_key) handle = (st if isinstance(st, _ut()._Lent) and getattr(st, "schema_only", False) else _ut().lend_defs(st, schema_only=True)) + # Source declarations can advance before the saved schema refreshes. + definitions = dict(handle.get(_ut().STORE_KEY) or {}) + if isinstance(definitions.get(table_key), dict): + definitions[table_key] = dict(definitions[table_key], fields=fields) + handle = _ut().lend(handle, **{_ut().STORE_KEY: definitions}) + handle.schema_only = True declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")} columns = _filter_columns(spec, fields) guards, resolved, targets = [], {}, {} @@ -1894,7 +1984,10 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=( allow_lookups=_may_open(session, amazon_relational.LISTINGS, st=handle)) if not definition or target_spec is None or target_spec.get("missing"): refuse(field) - target_fields = definition.get("fields") or [] + target_fields = resolved_fields(key, definition) + if target_spec.get('_fields') is not None: + target_fields = meta_source_fields(target_spec, target_fields) + definitions[key] = dict(definition, fields=target_fields) hidden = ps.hidden_keys(session.user, key, target_fields, st=handle) tree = ((ps.entry(session.user, key) or {}).get("filter") if ps.row_scope_applies(session.user, key) else None) @@ -2031,8 +2124,16 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=( # Relation cells compare string identities. Blank source keys never match. match = (f"NULLIF(CAST({left['sql']} AS VARCHAR), '') = " f"NULLIF(CAST({right['sql']} AS VARCHAR), '')") + connection_ok = 'TRUE' + if bag.get('connectionScope') == '_connection_id': + source_connection = context['cols'].get('_connection_id', ('NULL',))[0] + target_connection = target_context['cols'].get('_connection_id', ('NULL',))[0] + match += (f" AND NULLIF(CAST({source_connection} AS VARCHAR), '') = " + f"NULLIF(CAST({target_connection} AS VARCHAR), '')") + connection_ok = (f"NULLIF(CAST({source_connection} AS VARCHAR), '') IS NOT NULL" + if target_connection.strip().upper() != 'NULL' else 'FALSE') allowed = f'{alias}."__wall_allowed"' - good = (f"({left_ok}) AND NOT EXISTS (SELECT 1 FROM {source} WHERE {match} " + good = (f"({left_ok}) AND ({connection_ok}) AND NOT EXISTS (SELECT 1 FROM {source} WHERE {match} " f"AND NOT COALESCE(({allowed}) AND ({right_ok}), FALSE))") join = f"{match} AND {allowed} AND ({right_ok})" pid = target_context["id"] @@ -2545,7 +2646,7 @@ class _RelationRows: def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None, traversal_of=None, st=None, rev=..., normalize_match=False, - compiled=None, stream=False, materialised=None): # `...` IS `_UNREAD` + compiled=None, stream=False, materialised=None, connection_scope=None): # `...` IS `_UNREAD` """Internal join values, after the ordinary row wall, with no row-count ceiling. Join columns may be hidden on the wire. Only identities leave the resolver; the native @@ -2571,6 +2672,8 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non _compiled_read(table, session, traversal_intermediate=True, st=st) if traversal_of is not None else _compiled_read(table, session, st=st)) spec = read["spec"] + if '_connection_id' in spec['cols']: + columns.add('_connection_id') bindings = dict(spec["cols"], pid=(spec["id"], int)) if columns - set(bindings): raise err(409, "relation_target_unavailable", "A required relation column is unavailable.") @@ -2580,6 +2683,12 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non if values is None: continue expr, cast = bindings[key] + if connection_scope and key != 'pid': + connection = bindings.get(connection_scope) + if connection is None: + raise err(409, 'relation_target_unavailable', 'This relation needs its account identity.') + expr = f'to_json(list_value(CAST({connection[0]} AS VARCHAR), CAST({expr} AS VARCHAR)))' + cast = str if normalize_match and key != "pid": expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))" cast = lambda value: str(value).strip().lower() @@ -2659,12 +2768,16 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non normalize = ((lambda value: str(value).strip().lower()) if normalize_match else str) wanted = {normalize(value) for value in values} - rows = [row for row in rows if normalize(row.get(key)) in wanted] + rows = [row for row in rows if normalize( + json.dumps([str(row.get(connection_scope)), str(row.get(key))], separators=(',', ':'), ensure_ascii=False) + if connection_scope and row.get(connection_scope) not in (None, '') else row.get(key)) in wanted] if matches is not None: normalize = (lambda value: str(value).strip().lower()) if normalize_match else str wanted = {key: {normalize(value) for value in values} for key, values in matches.items()} rows = [row for row in rows if any(normalize(row.get(key)) in values for key, values in wanted.items())] + if any('_connection_id' in row for row in rows): + columns.add('_connection_id') return [{key: row.get(key) for key in columns} for row in rows] @@ -2679,6 +2792,7 @@ def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_ """Follow a scoped leg with SQL operands at every population-sized boundary.""" import core.relations as relations target, link, via = str(entry['target']), entry.get('link') or {}, entry.get('via') or {} + connection = link.get('connectionScope') observed = {'rows': 0, 'complete': True, 'available': False} def source_values(): for row in _relation_rows(session, source, [source_col], ids=source_ids, stream=True): @@ -2689,7 +2803,11 @@ def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_ if (manual and not reverse) or (inverse and reverse): yield from _relation_cell_ids(row[source_col]) else: - yield str(row[source_col]) + if connection: + if row.get(connection) not in (None, ''): + yield json.dumps([str(row[connection]), str(row[source_col])], separators=(',', ':'), ensure_ascii=False) + else: + yield str(row[source_col]) values = _membership_relation(cur, source_values(), normalize=str, sql_type='VARCHAR') if source_col != relations.IDENTITY_COL and not observed['available']: if (source.startswith('ut_amzn_') and target.startswith('ut_amzn_') @@ -2711,7 +2829,7 @@ def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_ rows = _relation_rows(session, target, [target_col], stream=True) return _relation_cell_membership(cur, rows, target_col, values) rows = _relation_rows(session, target, [target_col], match=(target_col, values), - where=entry.get('where'), stream=True) + where=entry.get('where'), stream=True, connection_scope=connection) return _membership_relation(cur, (row['pid'] for row in rows)) @@ -2737,7 +2855,10 @@ def _relation_leg_ids(session, source, source_ids, entry, membership_cur=None): return _relation_leg_membership(session, source, source_ids, entry, membership_cur, source_col=source_col, target_col=target_col, manual=manual, inverse=inverse, reverse=reverse) source_rows = _relation_rows(session, source, [source_col], ids=source_ids) - values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")} + connection = link.get('connectionScope') + values = {(json.dumps([str(row[connection]), str(row[source_col])], separators=(',', ':'), ensure_ascii=False) + if connection else str(row[source_col])) for row in source_rows + if row.get(source_col) not in (None, '') and (not connection or row.get(connection) not in (None, ''))} # A column-keyed edge cannot fall back to the wire record id. In particular, # product_data.pid is a stable hash while sale_order_line.product_id is an Odoo # identity. Treating a missing/hidden/blank source value as an empty match would @@ -2764,7 +2885,7 @@ def _relation_leg_ids(session, source, source_ids, entry, membership_cur=None): rows = (row for row in rows if _relation_cell_ids(row.get(target_col)) & values) else: rows = _relation_rows(session, target, [target_col], match=(target_col, values), - where=entry.get("where"), stream=True) + where=entry.get("where"), stream=True, connection_scope=connection) return {int(row["pid"]) for row in rows} @@ -3673,6 +3794,12 @@ def _may_see_view(session, vid, store): return shares.may_see("view", vid, session.uname, is_admin=session.admin, st=store.st) +def mirror_store_field(): + """The Odoo Store label is a read-only mirror value, never an upstream field request.""" + return {'key': 'Store', 'label': 'Store', 'type': 'text', 'source': 'odoo', + 'default': True, 'readOnly': True, 'mirrorOnly': True} + + def resolved_fields(table_key, defn): """Resolve presets and authored fields for the same connection as the rows door.""" rel = _rel() @@ -3691,11 +3818,13 @@ def resolved_fields(table_key, defn): if not connected: raise err(404, "not_connected", "that database is not a connected grid") fields = (defn or {}).get("fields") or [] - if not fields: + if not fields and not str(table_key).startswith('ut_meta_'): raise err(409, "schema_not_loaded", "This connected database has no saved fields yet.") declared = {field["key"]: field for field in fields if field.get("key")} declared.update({field["key"]: field for field in (defn or {}).get("fields") or [] if field.get("key")}) + if str(base or '').startswith('ut_odoo_'): + declared['Store'] = mirror_store_field() return list(declared.values()) @@ -3783,7 +3912,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= # try, not inside it. try: cur, mismatch = (_mirror_cur(session, table_key) - if (table_key.startswith("ut_amzn_") + if (table_key.startswith(("ut_amzn_", "ut_meta_")) or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY) else _mirror_cur(session)) except RuntimeError as e: @@ -3818,6 +3947,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= "or ask for a read-through binding to be added for it") st = st if st is not None else session.runtime + if spec.get('_fields') is not None: + fields = meta_source_fields(spec, fields) shared_fields = _so().fields(table_key, st=st) if shared_fields: declared = {field["key"]: field for field in fields} @@ -3869,7 +4000,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= _wall_nodes, _wall_conj = _fe().tree_parts(_wall_tree) if _wall_nodes: try: - if str(table_key).startswith(("ut_odoo_", "ut_amzn_")): + if str(table_key).startswith(("ut_odoo_", "ut_amzn_", "ut_meta_")): _wall_cols, _wall_guards = _read_wall_columns( table_key, session, cur, spec, fields, _wall_nodes, st if st is not None else session.runtime) @@ -4242,8 +4373,103 @@ def _restrict_row_ids(read, ids): return narrowed +_DRILL_PAGES = OrderedDict() +_DRILL_PAGES_LOCK = threading.Lock() +_DRILL_PAGES_MAX = 64 + + +def _materialised_revision(session, table_key): + """Bind reusable selection IDs to current policy, authorship and mirror revisions.""" + import registry_source + from core import sheet_binding, store_pg + from harness import datastore, mirror_tokens + + generation = registry_source.generation(session, table_key) + if generation is None: + return None + runtime = session.runtime + reader = registry_source.backend(runtime) + revisions = None + if isinstance(reader, store_pg.PgStore): + # Derived fields can read another database's authored cells or workspace. + revisions = store_pg._read(store_pg._pool(), store_pg._q( + 'SELECT key, rev, updated_at FROM {tbl} ORDER BY key', 'store_kv', reader.tenant_slug), ()) + mirror = None + if callable(getattr(runtime, 'assert_datastore_matches', None)): + from harness.runtime import mirror_cursor + try: + cur = mirror_cursor(runtime) + except RuntimeError as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + return None # A stored table remains readable while its mirror is unavailable. + mirror = mirror_tokens._revision(cur, mirror_tokens._file_marks(cur), + datastore.mirror_generation(), []) + if mirror is None: + return None + elif table_key in {'customer_data', 'product_data'}: + return None # Unmanaged mirrors cannot prove an unchanged projection. + return sheet_binding.fingerprint([ + generation, revisions, mirror, _today(), getattr(runtime, 'key', None), + getattr(runtime, 'store_key', str)('user_tables'), getattr(reader, 'tenant_slug', None), + getattr(getattr(runtime, 'tenant', None), 'config', None)]) + + def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related, groupBy=None, groupKey=None, ids=None): + """Reuse an unchanged page's selection, then hydrate it through current scoped readers.""" + import registry_source + from core.sheet_binding import fingerprint + from harness import datastore + + if table_key == 'ut_activity': + return _materialised_window_read(table_key, session, offset, limit, filters, filterConj, + sorts, search, related, groupBy, groupKey, ids=ids) + revision = _materialised_revision(session, table_key) + owner = registry_source.backend(session.runtime) + key = fingerprint([id(owner), table_key, revision, offset, limit, filters, filterConj, + sorts, search, related, groupBy, groupKey, ids]) if revision is not None else None + with _DRILL_PAGES_LOCK: + cached = _DRILL_PAGES.get(key) if key is not None else None + if cached is not None: + _DRILL_PAGES.move_to_end(key) + if cached is not None and cached[0] is owner: + page_ids, summary = cached[1], deepcopy(cached[2]) + if table_key in {'customer_data', 'product_data'}: + session.require(table_key) + cur, mismatch = _mirror_cur(session) + if mismatch: + raise err(409, 'cross_tenant_store', mismatch) + if cur is None: + raise err(503, 'canonical_source_unavailable', 'The tenant mirror is unavailable.') + with cur.cursor() as page_cur: + payload = registry_source.canonical_page(session, page_cur, table_key, page_ids, related=related) + else: + payload = registry_source.stored_page(session, table_key, page_ids, related=related) + if _materialised_revision(session, table_key) != revision: + return _materialised_window_read(table_key, session, offset, limit, filters, filterConj, + sorts, search, related, groupBy, groupKey, ids=ids) + limits = list(payload.get('limits') or []) + limits.extend(report for report in summary['limits'] if report not in limits) + return {**payload, **summary, 'limits': limits} + + answer = _materialised_window_read(table_key, session, offset, limit, filters, filterConj, + sorts, search, related, groupBy, groupKey, ids=ids) + if (key is not None and len(answer['rows']) <= datastore.WINDOW_MAX + and _materialised_revision(session, table_key) == revision): + summary = {name: deepcopy(answer[name]) for name in + ('ok', 'total', 'totalUnfiltered', 'offset', 'limit', 'limits', 'scope')} + entry = (owner, tuple(int(row['pid']) for row in answer['rows']), summary) + with _DRILL_PAGES_LOCK: + _DRILL_PAGES[key] = entry + _DRILL_PAGES.move_to_end(key) + while len(_DRILL_PAGES) > _DRILL_PAGES_MAX: + _DRILL_PAGES.popitem(last=False) + return answer + + +def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related, + groupBy=None, groupKey=None, ids=None): from harness import datastore query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj, @@ -4488,6 +4714,8 @@ def odoo_table_rows(table_key: str, # as /related pins. No unscoped SQL subquery can leak a hidden target's id. amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets, max_bytes=LINK_VALUE_MAX_BYTES, capacity_refusal=_link_capacity_refusal) + if table_key.startswith('ut_meta_'): + _fill_meta_link_cells(session, rows_src, fields, st=page_st, rev=page_rev) overlays = {} # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID. @@ -4536,12 +4764,23 @@ def odoo_table_rows(table_key: str, cells.update(overlays.get(str(row["pid"]), {})) cells.update({key: row.get(key, "") for key in if_keys if key not in _hidden}) rows.extend(aios_grid.rows_from_pool([row], fields, {str(row["pid"]): cells})) + fields = [field for field in fields if field['key'] not in _SOURCE_PRIVATE] + rows = [{key: value for key, value in row.items() if key not in _SOURCE_PRIVATE} + for row in rows] unfiltered = win["total"] if where == base_where else datastore.count_rows( where=base_where, params=tuple(wall_params), cur=cur, **frm) from core import connector_spec rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime) + if rows: + user_tables = _ut() + notes = user_tables.cell_notes( + table_key, [row["pid"] for row in rows], + field_keys={field["key"] for field in fields} - _hidden, st=session.runtime) + for row in rows: + if str(row["pid"]) in notes: + row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])] return {"ok": True, "fields": fields, "rows": rows, # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`. "total": win["total"], "totalUnfiltered": unfiltered, @@ -4610,6 +4849,52 @@ def _sql_link_cells(session, target, on, values, *, st, single, remaining, read= lease.close() +def _fill_meta_link_cells(session, rows, fields, st=None, rev=...): + """Resolve Meta's scoped hierarchy keys within the existing complete-cell byte budget.""" + from routes_grid import _may_open + remaining = LINK_VALUE_MAX_BYTES + requests, edges = {}, [] + for field in fields: + bag = field.get('link') or {} + if field.get('type') != 'link' or not bag.get('from') or not bag.get('on'): + continue + target, source, on = bag['table'], bag['from'], bag['on'] + edges.append((field, target, source, on)) + values = {str(row[source]) for row in rows if row.get(source) not in (None, '')} + requests.setdefault(target, {}).setdefault(on, set()).update(values) + indexes = {} + for target, matches in requests.items(): + index = {key: {} for key in matches} + if _may_open(session, target, st=st) and any(matches.values()): + iterator = iter(_relation_rows(session, target, list(matches), matches=matches, + st=st, rev=rev, stream=True)) + try: + for row in iterator: + for key, values in matches.items(): + value = str(row.get(key)) + if value not in values: + continue + ids = index[key].setdefault(value, set()) + if int(row['pid']) not in ids: + remaining -= len(str(row['pid'])) + 1 + if remaining < 0: + raise _link_capacity_refusal() + ids.add(int(row['pid'])) + finally: + close = getattr(iterator, 'close', None) + if callable(close): + close() + indexes[target] = index + remaining = LINK_VALUE_MAX_BYTES + for field, target, source, on in edges: + for row in rows: + value = ','.join(str(pid) for pid in sorted(indexes[target][on].get(str(row.get(source)), ()))) + remaining -= len(value) + (2 if value else 0) + if remaining < 0: + raise _link_capacity_refusal() + row[field['key']] = value + + def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...): """Resolve page link cells through scoped SQL and a complete-value byte budget.""" from routes_grid import _may_open diff --git a/api/routes_payouts.py b/api/routes_payouts.py index db44b7632035f0a4a27826d6e0cf71b3df8ade55..30440086429f073decf3feb059f224008085dc04 100644 --- a/api/routes_payouts.py +++ b/api/routes_payouts.py @@ -171,48 +171,35 @@ def _last_deposits(rows, as_of): @router.get("/screen") def read_screen(app: str = Query(..., min_length=1), currency: str | None = Query(None), view: str = Query("all", pattern="^(all|unreconciled)$"), cursor: str | None = Query(None), - limit: int = Query(SCREEN_PAGE, ge=1), session: Session = Depends(_app_reader)): + limit: int = Query(SCREEN_PAGE, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)): """The App screen in one read: the KPI strip, tab counts and one page of settlements, newest first. No cap.""" pay = _module() - offset = _offset(cursor) tenant, st = _tenant(session), session.runtime can_edit = _may_edit(session, _app(session, app)) base = {"canEdit": can_edit, "statuses": list(pay.STATUSES), "view": view, "currency": currency or None} try: - over = _call(pay.overview, tenant, st=st) + page = _call(pay.screen, tenant, st=st, currency=currency, view=view, q=q, cursor=cursor, limit=limit) except HTTPException as exc: if not _empty(exc): raise return {**base, "empty": True, "currencies": [], "counts": {"all": 0, "unreconciled": 0}, "settlements": [], "cursor": None, "kpis": {}} - listed = _call(pay.settlements, tenant, st=st, currency=currency or None, limit=None)["settlements"] - counts = {"all": len(listed), "unreconciled": sum(1 for r in listed if not r["reconciled"])} - rows = listed if view == "all" else [r for r in listed if not r["reconciled"]] - page = rows[offset:offset + limit] - opened = over.get("openSettlements") or [] - open_now = next((o for o in opened if currency and o.get("currency") == currency.strip().upper()), - opened[0] if opened else None) - return {**base, "empty": not over.get("settlementCount"), "asOf": over.get("asOf"), - "currencies": over.get("currencies") or [], "economicsCurrency": over.get("economicsCurrency"), - "counts": counts, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page], - "cursor": str(offset + limit) if offset + limit < len(rows) else None, - "kpis": {"nextDeposit": over.get("nextDeposit"), "openSettlement": open_now, - "lastDeposit": _last_deposits(listed, over.get("asOf") or ""), - "unreconciled": counts["unreconciled"], "settlements": counts["all"]}} + return {**base, **page, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page["settlements"]]} @router.get("/screen/settlements/{settlement_id}") def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1), + bucket: str | None = Query(None), cursor: str | None = Query(None), + limit: int = Query(SCREEN_PAGE, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)): """One settlement for the drawer: its waterfall, the tie line and, when unreconciled, its owned item.""" pay = _module() tenant, st = _tenant(session), session.runtime can_edit = _may_edit(session, _app(session, app)) - detail = _call(pay.settlement, tenant, settlement_id, st=st) + detail = _call(pay.screen_settlement, tenant, settlement_id, st=st, bucket_key=bucket, cursor=cursor, limit=limit, q=q) item = None if not detail.get("reconciled"): - held = _call(pay.items, tenant, session.user, st=st, limit=None)["items"] - item = next((i for i in held if str(i.get("id")) == str(settlement_id)), None) + item = _call(pay.screen_item, tenant, detail, st=st) return {"canEdit": can_edit, "statuses": list(pay.STATUSES), "settlement": detail, "item": item} @@ -226,3 +213,49 @@ def read_settings(session: Session = Depends(_app_reader)): def write_settings(defaultOwner: str | None = Body(None, embed=True), session: Session = Depends(_app_writer)): pay = _module() return _call(pay.set_settings, _tenant(session), default_owner=defaultOwner, st=session.runtime) + + +def _refresh_owned_items_once(): + """Persist settlement work independently of GETs and preserve its first-seen due date.""" + import application_registry + from harness import runtime + for tenant in runtime.known_tenants(): + try: + st = runtime.get_runtime(tenant) + apps = application_registry.load(st)["applications"].values() + if any(a.get("templateKind") == TEMPLATE_KIND and not a.get("retired") for a in apps): + _module().refresh_items(tenant, st=st) + except Exception: + import logging + logging.getLogger(__name__).exception("Payouts item refresh failed for %s", tenant) + + +_items_thread = None +_items_stop = None + + +@router.on_event("startup") +def _start_item_refresh(): + global _items_thread, _items_stop + import process_role + import threading + if _items_thread is not None and _items_thread.is_alive(): + return + _items_stop = threading.Event() + + def run(): + while not _items_stop.is_set(): + _refresh_owned_items_once() + _items_stop.wait(30) + _items_thread = process_role.start("payouts-items", "heavy", run) + + +@router.on_event("shutdown") +def _stop_item_refresh(): + if _items_stop is not None: + _items_stop.set() + + +import process_role as _process_role +if _process_role.role() == "worker": + _start_item_refresh() diff --git a/api/routes_products.py b/api/routes_products.py index 489554450c93f6fb7deebb0ebbe43e0bccffde70..e12e3e417483eedcfa691fa27bfbc3b220545821 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -24,13 +24,15 @@ import json import re import time -from fastapi import Body, Depends +from fastapi import Body, Depends, Response from fastapi import APIRouter import scope_cache from core import grid_events, user_tables -from deps import Session, err, module_gate +from deps import Session, err, module_gate, require_session +from routes_grid import row_patch_receipt +from routes_tables import _row_patch_identity, _note_delta, _patch_cell_notes, _project_cell_notes, _builtin_note_context router = APIRouter(prefix="/api/v1") @@ -614,6 +616,8 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_ "fields": fields, "views": views, "lists": lists, "_permission_scope": permission_scope, "derived": derived, + "_note_runtime": session.runtime, + "_note_hidden": hidden, "measures": measures, "measure_sets": measure_sets, "today": today, "team_id": team_id} @@ -679,19 +683,23 @@ def product_aggregates(body: dict = Body(default=None), @router.patch("/products/{pid}") +@_row_patch_identity(module=MODULE) +@row_patch_receipt('product') def patch_product(pid: int, body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): + session: Session = Depends(require_session), response: Response = None): """Write the product table's EDITABLE overlay stratum — the customers PATCH, on the product topic's ctx. Routed through `core.grid_events.handle_one` so the per-key `permissions.edit` wall, the pid wall and the truncation rules stay ONE implementation; the ctx's `table` ops aim the write at the PRODUCT bucket.""" import modules.product_data as pd - from core import grid_events + from core import grid_events, edit_receipts updates = dict(body or {}) + notes = _note_delta(updates) if not updates: raise err(400, "empty_patch", "no fields to update") - g = product_assembly(session, consume_corrections=False) + g = (_builtin_note_context(session, MODULE, pid) if notes is not None else + product_assembly(session, consume_corrections=False)) if pid not in g["pids"]: # 403, not 404 — the code may exist; it is simply not in this session's catalogue. raise err(403, "out_of_scope", "that product is not in your catalogue") @@ -703,9 +711,13 @@ def patch_product(pid: int, body: dict = Body(default=None), hidden_keys=perm_scope.hidden_keys( session.user, MODULE, g["fields"], st=session.runtime), scope_key="product", table=pd.table_ops(session.runtime), st=session.runtime) + if notes is not None: + return _patch_cell_notes(MODULE, pid, notes, ctx, + source_row_exists=lambda key, rid, admitted: key == MODULE and int(rid) in g['pids']) try: grid_events.handle_one( - {"id": f"patch:product:{pid}:{time.time_ns()}", "type": "overlay_patch", + {"id": edit_receipts.current_op_id() or f"patch:product:{pid}:{time.time_ns()}", + "type": "overlay_patch", "pid": pid, "updates": updates}, ctx) except grid_events.StoreUnavailable: raise err(503, "store_unavailable", @@ -744,7 +756,11 @@ def product_rows(assembly): import aios_grid rows = aios_grid.rows_from_pool(assembly["rows_src"], assembly["fields"], assembly["ws"].get("overlays"), derived=assembly["derived"]) - return _seed_image(_seed_shared(rows, assembly["rows_src"], assembly["fields"]), assembly["fields"]) + rows = _seed_image(_seed_shared(rows, assembly["rows_src"], assembly["fields"]), assembly["fields"]) + if assembly.get('_note_runtime') is not None: + rows = _project_cell_notes(MODULE, rows, assembly['fields'], assembly['_note_runtime'], + hidden=assembly.get('_note_hidden') or ()) + return rows def _seed_shared(rows, rows_src, fields): diff --git a/api/routes_ranking_diagnostic.py b/api/routes_ranking_diagnostic.py index 361372d9671e1fb0edc7dfb53f7ceec4630aae54..7bc6e7d9d9f8d896c3b5e27090a54dc48f18bd9f 100644 --- a/api/routes_ranking_diagnostic.py +++ b/api/routes_ranking_diagnostic.py @@ -85,17 +85,19 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends( @router.get("/screen") def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None), week: str | None = Query(None), cursor: str | None = Query(None), + limit: int = Query(200, ge=1, le=200), q: str = Query("", max_length=200), session: Session = Depends(_app_reader)): """The week's drop list (the newest week by default), with counts, problems and settings.""" rd = _module() record = _app(session, app) st = session.runtime held = _call(rd.weeks, _tenant(session), st=st) - listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, + listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, q=q, week=week or (held[-1] if held else None), st=st) return {"canEdit": _may_edit(session, record), "week": listed["week"], "weeks": listed["weeks"], "counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"], - "problems": listed["problems"], "statuses": list(rd.STATUSES), + "problems": listed["problems"], "statuses": list(rd.STATUSES), "total": listed["total"], + "sourceTotals": _call(rd.screen_totals, _tenant(session), listed["week"], st=st), "settings": _call(rd.settings, _tenant(session), st=st)} @@ -116,10 +118,12 @@ def update_item(item_id: str, status: str = Body(...), note: str | None = Body(N @router.get("/asins") def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None), + limit: int = Query(200, ge=1, le=200), q: str = Query("", max_length=200), session: Session = Depends(_app_reader)): """The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card.""" rd = _module() - page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, st=session.runtime) + page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, limit=limit, q=q, + st=session.runtime) labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)} for row in page["asins"]: flagged = bool(row.get("cause") and page["week"]) diff --git a/api/routes_reimbursements.py b/api/routes_reimbursements.py index dd84d7f2805d71a2520528b783b5d6f5de7eafc8..616729ab000bd9474d446bdede710f7b6fa14d2d 100644 --- a/api/routes_reimbursements.py +++ b/api/routes_reimbursements.py @@ -83,18 +83,26 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends( @router.get("/screen") def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None), - cursor: str | None = Query(None), session: Session = Depends(_app_reader)): + cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1), + session: Session = Depends(_app_reader)): rb = _module() record = _app(session, app) - listed = _call(rb.items, _tenant(session), session.uname, status=status, cursor=cursor, st=session.runtime) + listed = _call(rb.items, _tenant(session), session.uname, status=status, search=search, cursor=cursor, + limit=limit, st=session.runtime) return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"], - "cursor": listed["cursor"], "problems": listed["problems"], "statuses": list(rb.STATUSES)} + "cursor": listed["cursor"], "total": listed["total"], "offset": listed["offset"], + "limit": listed["limit"], "paidTotal": listed["paidTotal"], + "problems": listed["problems"], "statuses": list(rb.STATUSES)} @router.get("/items/{item_id}") -def item_detail(item_id: str, session: Session = Depends(_app_reader)): +def item_detail(item_id: str, events_cursor: str | None = Query(None), returns_cursor: str | None = Query(None), + reimbursements_cursor: str | None = Query(None), history_cursor: str | None = Query(None), + limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)): rb = _module() - return _call(rb.item_detail, _tenant(session), session.uname, item_id, st=session.runtime) + return _call(rb.item_detail, _tenant(session), session.uname, item_id, events_cursor=events_cursor, + returns_cursor=returns_cursor, reimbursements_cursor=reimbursements_cursor, + history_cursor=history_cursor, limit=limit, st=session.runtime) @router.patch("/items/{item_id}") @@ -106,9 +114,10 @@ def update_item(item_id: str, status: str = Body(...), caseId: str | None = Body @router.get("/paid") -def paid_rows(cursor: str | None = Query(None), session: Session = Depends(_app_reader)): +def paid_rows(cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1), + session: Session = Depends(_app_reader)): rb = _module() - return _call(rb.paid, _tenant(session), session.uname, cursor=cursor, st=session.runtime) + return _call(rb.paid, _tenant(session), session.uname, search=search, cursor=cursor, limit=limit, st=session.runtime) @router.get("/validate") diff --git a/api/routes_restock_planner.py b/api/routes_restock_planner.py index 02415e8c9f3b49e8b62d6118d3657493607c5219..7c67333c9dce78c0558e3f0626a71e3661396f15 100644 --- a/api/routes_restock_planner.py +++ b/api/routes_restock_planner.py @@ -1,5 +1,9 @@ """Restock planner App routes (C3): items, one SKU, settings, refresh and the rule 8 tie.""" +import time +import uuid + from fastapi import APIRouter, Body, Depends, Query +from fastapi.responses import JSONResponse from deps import Session, err, require_session @@ -104,19 +108,23 @@ def thresholds(session): @router.get("/items") def list_items(app: str = Query(..., min_length=1), band: str | None = Query(None), status: str | None = Query(None), cursor: str | None = Query(None), - limit: int = Query(200, ge=1), q: str | None = Query(None), + limit: int = Query(200, ge=1, description="Requests above 200 use 200 items. limit reports the effective size."), + q: str | None = Query(None), session: Session = Depends(_app_reader)): rp = _module() - listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, - band=band, q=q, st=session.runtime) + listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor, + limit=min(limit, rp.MAX_PAGE_SIZE), band=band, q=q, st=session.runtime) return {**listed, "owners": owners(session), "thresholds": thresholds(session), "canEdit": _may_edit(session, _app(session, app))} @router.get("/items/{item_id}") -def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)): +def read_item(item_id: str, app: str = Query(..., min_length=1), offset: int = Query(0, ge=0), + limit: int = Query(50, ge=1, description="Requests above 200 use 200 entries per collection. detailLimit reports the effective size."), + session: Session = Depends(_app_reader)): rp = _module() - detail = _call(rp.item_detail, _tenant(session), item_id, st=session.runtime) + detail = _call(rp.item_detail, _tenant(session), item_id, offset=offset, + limit=min(limit, rp.MAX_PAGE_SIZE), st=session.runtime) return {**detail, "thresholds": thresholds(session), "canEdit": _may_edit(session, _app(session, app))} @@ -147,7 +155,80 @@ def write_settings(sendToFbaDays: float | None = Body(None), overstockDays: floa @router.post("/refresh") def refresh(session: Session = Depends(_app_writer)): rp = _module() - return _call(rp.refresh, _tenant(session), st=session.runtime) + identity, stamp = uuid.uuid4().hex, time.time() + + def enqueue(doc): + doc = doc if isinstance(doc, dict) else {} + if doc.get("state") in ("queued", "running") and stamp - float(doc.get("requestedAt") or 0) < 600: + return doc + return {"state": "queued", "requestId": identity, "requestedAt": stamp} + + queued = _call(session.runtime.update, rp.KEY_REFRESH, enqueue) + return JSONResponse(status_code=202, content=queued) + + +@router.get("/refresh") +def refresh_status(session: Session = Depends(_app_reader)): + return _call(session.runtime.get, _module().KEY_REFRESH) or {"state": "idle"} + + +def _drain_refresh(rt, *, now=None): + """Claim one durable refresh intent in the sync worker and retain its terminal result.""" + rp = _module() + stamp = time.time() if now is None else now + pending = rt.get(rp.KEY_REFRESH) or {} + if pending.get("state") != "queued" and not ( + pending.get("state") == "running" and stamp - float(pending.get("startedAt") or 0) >= 600): + return + token, claimed = uuid.uuid4().hex, {} + + def claim(doc): + if (doc or {}).get("requestId") == pending.get("requestId") and ( + doc.get("state") == "queued" or (doc.get("state") == "running" + and stamp - float(doc.get("startedAt") or 0) >= 600)): + doc.update(state="running", startedAt=stamp, claim=token) + claimed.update(doc) + return doc + + rt.update(rp.KEY_REFRESH, claim) + if not claimed: + return + try: + result = rp.refresh(rt.key, st=rt) + completion = {"state": "failed" if result.get("state") == "read_failed" else "done", "result": result} + except Exception: # noqa: BLE001 + completion = {"state": "failed", "message": "The restock plan could not be refreshed. Try again shortly."} + + def finish(doc): + if (doc or {}).get("claim") == token: + doc.update(completion, finishedAt=time.time()) + return doc + rt.update(rp.KEY_REFRESH, finish) + + +def _refresh_worker(): + import threading + import process_role + from harness import runtime + stop = threading.Event() + while not stop.wait(1): + if not process_role.may_write(): + continue + for tenant in runtime.known_tenants(): + try: + _drain_refresh(runtime.get_runtime(tenant)) + except Exception: # noqa: BLE001 + continue + + +def _start_refresh_worker(): + import process_role + from core import data_binding + if not data_binding.gate_process(): + process_role.start("restock-refresh", "heavy", _refresh_worker) + + +_start_refresh_worker() @router.get("/validate") diff --git a/api/routes_returns_repeat.py b/api/routes_returns_repeat.py index 9c82c4f39bb3bef67b5b8c298d3fc1a264e7707e..626ad878dd267dbf23b5897f473b34f51949298d 100644 --- a/api/routes_returns_repeat.py +++ b/api/routes_returns_repeat.py @@ -85,15 +85,16 @@ def _settings_writer(app_id: str = Path(..., min_length=1), session: Session = D @router.get("/{app_id}/summary") def read_summary(start: str | None = Query(None), end: str | None = Query(None), + cursor: str | None = Query(None), limit: int = Query(24, ge=1), session: Session = Depends(_app_reader)): rr = _module() - out = _call(rr.summary, _tenant(session), start=start, end=end, st=session.runtime) + out = _call(rr.summary, _tenant(session), start=start, end=end, cursor=cursor, limit=limit, st=session.runtime) return {**out, "canEdit": _can_edit_items(session) and not _viewer(session)} @router.get("/{app_id}/skus") def list_skus(segment: str = Query("highest"), start: str | None = Query(None), end: str | None = Query(None), - cursor: str | None = Query(None), limit: int = Query(200, ge=1), q: str | None = Query(None), + cursor: str | None = Query(None), limit: int = Query(100, ge=1), q: str | None = Query(None), session: Session = Depends(_app_reader)): rr = _module() return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor, @@ -102,9 +103,10 @@ def list_skus(segment: str = Query("highest"), start: str | None = Query(None), @router.get("/{app_id}/sku") def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(None), - end: str | None = Query(None), session: Session = Depends(_app_reader)): + end: str | None = Query(None), cursor: str | None = Query(None), limit: int = Query(25, ge=1), + session: Session = Depends(_app_reader)): rr = _module() - return _call(rr.detail, _tenant(session), key, start=start, end=end, st=session.runtime) + return _call(rr.detail, _tenant(session), key, start=start, end=end, cursor=cursor, limit=limit, st=session.runtime) def owners(session): @@ -127,7 +129,7 @@ def owners(session): @router.get("/{app_id}/items") def list_items(status: str | None = Query(None), cursor: str | None = Query(None), - limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)): + limit: int = Query(100, ge=1), session: Session = Depends(_app_reader)): rr = _module() listed = _call(rr.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit, st=session.runtime) diff --git a/api/routes_sheet.py b/api/routes_sheet.py index a563befc833b462f664a620f46b00d0512f5a2e5..1924f4881daff9d62fd3e45626f41f232df86dc6 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -3444,7 +3444,9 @@ def _serve_open_tab(session, book, target, *, databases=None): @router.patch("/sheets/{workbook_id}/tabs/{tab}/cells") def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None), session: Session = Depends(require_session)): - """Merge a RANGE of cells into one tab: body `{cells: {A1: Cell | null}, rev}` (contract C3). + """Merge a RANGE of cells into one tab: body `{cells: {A1: Cell | null}, rev, opId}` (contract C3). + + A supplied opId retains the complete response for retries, including refusals and revisions. ⭐ `{"A1": null}` DELETES A1 and leaves its neighbours alone. After C2 an empty cell can carry a style (`kind: "empty"`), so a client needs a way to say "nothing at all lives here" that is @@ -3470,10 +3472,49 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None whole workbook is recomputed IN MEMORY (never a second write, never a second `rev`, see `_recomputed_tab_cells`) and every FORMULA cell on THIS tab is handed back with its answer. """ - from core import sheets + from core import edit_receipts raw = dict(body or {}) + # Older clients omit opId until their outbox is available. + if "opId" not in raw: + return _apply_sheet_cell_patch(workbook_id, tab, raw, session, _store(session)) try: - _require_editor(session, sheets.get_workbook(_store(session), workbook_id)) + identity = edit_receipts.op_id(raw.pop("opId")) + except ValueError as exc: + raise err(400, "bad_op_id", str(exc)) from exc + + def apply(): + from dataclasses import replace + checked_session = replace( + session, runtime=edit_receipts.CheckedStore(_store(session))) + try: + answer = _apply_sheet_cell_patch( + workbook_id, tab, raw, checked_session, checked_session.runtime) + return {"answer": dict(answer, opId=identity)} + except HTTPException as exc: + if exc.status_code >= 500: + raise + detail = exc.detail if isinstance(exc.detail, dict) else {"message": exc.detail} + return {"status": exc.status_code, "detail": dict(detail, opId=identity), + "headers": exc.headers} + + try: + result = edit_receipts.run_once( + _store(session), session.uname, + edit_receipts.operation(workbook_id, "sheet_cells", tab), identity, apply) + except edit_receipts.ReceiptUnavailable as exc: + failure = err(503, "store_unavailable", str(exc)) + failure.detail["opId"] = identity + raise failure from exc + if "status" in result: + raise HTTPException(result["status"], result["detail"], headers=result["headers"]) + return result["answer"] + + +def _apply_sheet_cell_patch(workbook_id, tab, raw, session, st): + """Apply and compute a cell patch before retaining its complete response.""" + from core import sheets + try: + _require_editor(session, sheets.get_workbook(st, workbook_id)) except sheets.SheetRefusal as exc: raise _refuse(exc) #: ⚠ `rev` ABSENT and `rev: null` are the same request and both opt out. A client that has @@ -3481,7 +3522,7 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None #: imported workbook impossible. expect = raw.get("rev") try: - out = sheets.patch_cells(_store(session), workbook_id, tab, raw.get("cells"), + out = sheets.patch_cells(st, workbook_id, tab, raw.get("cells"), expect_rev=expect, actor=session.uname, actor_name=session.user.get("name") or session.uname) except sheets.SheetRefusal as exc: diff --git a/api/routes_tables.py b/api/routes_tables.py index 7b47fa562ad7d2acb2c8fa4109a7db63bd665901..3a5e283f7a6f09678869c40fec674044bf16b3d0 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -41,6 +41,7 @@ import time import base64 import hmac from functools import wraps +from inspect import signature from fastapi import Body, Depends, Header, Query from fastapi import APIRouter @@ -49,13 +50,114 @@ from fastapi import APIRouter # `rev_current` so the client can refetch that revision and retry rather than only toast. # Bending `err` to take a fourth member would change the error shape of every route in the # app; naming the exception here changes one. -from fastapi import HTTPException, Request +from fastapi import HTTPException, Request, Response from deps import Session, err, require_session +from routes_grid import row_patch_receipt router = APIRouter(prefix="/api/v1") +def _row_patch_identity(fn=None, *, module=None): + """Stamp authenticated row acknowledgements and refusals, including receipt replays.""" + if fn is None: + return lambda handler: _row_patch_identity(handler, module=module) + sig = signature(fn) + + @wraps(fn) + def wrapped(*args, **kwargs): + from deps import _TENANT_HEADER, _PRINCIPAL_HEADER, _principal + bound = sig.bind(*args, **kwargs) + session = bound.arguments['session'] + headers = {_TENANT_HEADER: str(session.tenant), + _PRINCIPAL_HEADER: _principal(session.tenant, session.uname)} + response = bound.arguments.get('response') + if response is not None: + response.headers.update(headers) + try: + if module is not None: + session.require(module) + return fn(*args, **kwargs) + except HTTPException as exc: + exc.headers = {**(exc.headers or {}), **headers} + raise + return wrapped + + +def _note_delta(updates): + """Reserve one note field per operation and refuse mixed scalar writes.""" + key = _ut().CELL_NOTES_KEY + if key not in updates: + return None + notes = updates[key] + if (set(updates) != {key} or not isinstance(notes, dict) or len(notes) != 1 + or any(not isinstance(field, str) or not field or field.startswith('_') + or (text is not None and not isinstance(text, str)) for field, text in notes.items())): + raise err(400, 'bad_cell_notes', 'Send one changed field note as text or null.') + return notes + + +def _patch_cell_notes(table_key, pid, notes, ctx, *, source_row_exists=None): + """Use the outer row operation's runtime and receipt for the canonical note merge.""" + from core import edit_receipts + identity = edit_receipts.current_op_id() + if identity is None: + raise err(400, 'bad_op_id', 'A note edit needs its opId.') + try: + return _ut().patch_cell_notes(table_key, pid, notes, ctx, op_id=identity, + source_row_exists=source_row_exists) + except _ut().CellNoteRefused as exc: + raise err(400 if exc.code == 'bad_cell_notes' else 403, exc.code, str(exc)) from exc + except edit_receipts.ReceiptUnavailable as exc: + raise err(503, 'store_unavailable', str(exc)) from exc + + +def _connected_note_context(session, table_key, pid): + """Admit one mirrored note record through the shared SQL row and field walls.""" + import core.perm_scope as perm_scope + import routes_odoo_tables as connected + read = connected._compiled_read(table_key, session) + try: + rows = connected._relation_rows(session, table_key, [], ids=[pid], compiled=read, + materialised=False) + return {'pids': frozenset(row['pid'] for row in rows), 'fields': read['fields'], 'limits': [], + 'hidden': perm_scope.hidden_keys(session.user, table_key, read['fields'], st=session.runtime)} + finally: + read['cur'].close() + + +def _builtin_note_context(session, table_key, pid): + """Admit one canonical customer or deduplicated product without reading its whole pool.""" + from core import perm_scope + from registry_source import canonical_page + from routes_odoo_tables import _mirror_cur + try: + cursor, mismatch = _mirror_cur(session) + except RuntimeError as exc: + raise err(503, 'store_not_ready', str(exc)) from exc + if mismatch: + raise err(409, 'cross_tenant_store', mismatch) + try: + page = canonical_page(session, cursor, table_key, [pid]) + return {'pids': frozenset(row['pid'] for row in page['rows']), 'fields': page['fields'], + 'hidden': perm_scope.hidden_keys(session.user, table_key, page['fields'], st=session.runtime)} + finally: + cursor.close() + + +def _project_cell_notes(table_key, rows, fields, st, *, hidden=()): + """Attach metadata only for this authorized page and its readable fields.""" + if not rows: + return rows + visible = {field['key'] for field in fields} - set(hidden) + notes = _ut().cell_notes(table_key, [row['pid'] for row in rows], field_keys=visible, st=st) + for row in rows: + row.pop(_ut().CELL_NOTES_KEY, None) + if str(row['pid']) in notes: + row[_ut().CELL_NOTES_KEY] = notes[str(row['pid'])] + return rows + + def _app_write_via(session, application_id): """Resolve write attribution through the viewer's stored App permissions.""" # Direct Python callers omit the header and receive FastAPI's parameter descriptor. @@ -475,8 +577,52 @@ def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied= return defn -def _records_or_refuse(session, table_key, st=None): - """Require readable editor authority and human record mutability before record writes.""" +def _locked_user_field(field): + """Admit authored values on locked databases without admitting source or computed cells.""" + if not isinstance(field, dict): + return False + automation = field.get("automation") or {} + return (field.get("source") == "overlay" + and field.get("origin", "user") == "user" + and field.get("valueAuthority", "user-entered") == "user-entered" + and not field.get("preset") + and not field.get("derived") + and not (isinstance(automation, dict) + and (automation.get("flowId") or automation.get("preset"))) + and field.get("type") in _ut().UT_FIELD_TYPES - {"automation"} + and not field.get("measure") + and not _ut().is_computed_cell(field)) + + +def _records_lock_message(table_key, definition, st): + """Name the database's declared source when its record lock refuses a write.""" + from core import connector_spec, connector_sync, keychain + identity = _ut().connector_database_identity(table_key, st=st, definition=definition) + source = "" + if identity: + provider = identity["provider"] + provider = next((key for key, spec in connector_spec.CONNECTOR_SPEC_KEYS.items() + if spec == provider), provider) + source = (connector_sync.providers().get(provider) or {}).get("label", "") + if not source: + label = keychain.kind_label(provider) + source = label if label != "Connection" else "" + if not source: + import automation_engine as engine + for platform in engine.PLATFORMS: + try: + schema = engine.platform_schema(platform) + except ValueError: + continue + if table_key in schema["locked_tables"]: + source = schema["platform"] + break + name = f"{source} database" if source else "database" + return f"This {name} is locked, so only user-created field values can be edited." + + +def _records_or_refuse(session, table_key, st=None, *, cell_updates=None): + """Require editor authority and reserve locked database edits for authored field values.""" # Share one snapshot across the read, editor and record-mode checks. st = st if st is not None else _ut().lend(session.runtime) # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a @@ -487,9 +633,14 @@ def _records_or_refuse(session, table_key, st=None): if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st): raise err(403, "database_read_only", "Can edit access is required to change this database.") if not _ut().records_mutable(table_key, st=st): - raise err(403, "records_read_only", - "records in this automation-owned database are read-only. Add Instagram " - "handles in a Profile database and let enrichment populate this database") + if cell_updates and table_key != "ut_activity": + from core import shared_overlay + fields = dict(shared_overlay.fields(table_key, st=st)) + # A shared field cannot redefine a connector-owned column with the same key. + fields.update({field["key"]: field for field in defn.get("fields") or []}) + if all(_locked_user_field(fields.get(key)) for key in cell_updates): + return defn + raise err(403, "records_read_only", _records_lock_message(table_key, defn, st)) return defn @@ -906,7 +1057,8 @@ def scoped_pool(session: Session, table_key: str, st=None): `workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does. """ defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] + fields_base = _source_contract_fields(table_key, [dict(f) for f in (defn.get("fields") or [])], + st=st if st is not None else session.runtime) field_keys = {f["key"] for f in fields_base} if table_key == 'ut_activity': rows_src = _activity_grid_rows(session) @@ -956,6 +1108,46 @@ def scoped_pool(session: Session, table_key: str, st=None): return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn +def _new_locked_shared_field(session, table_key, pid, key, body): + """Admit an explicit authored declaration and its first value before creating any artifacts.""" + _managed_or_refuse(session, table_key, mutation=True) + if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, + st=session.runtime): + raise err(403, "database_read_only", "Can edit access is required to add a column.") + proposed = dict(body, key=key) + proposed.setdefault("source", "overlay") + proposed.setdefault("type", "text") + if (key in {"pid", "zone", _ut().ZONE_FIELD_KEY} + or not _locked_user_field(proposed) + or proposed.get("scope", "tenant") != "tenant" + or any(proposed.get(name) for name in ( + "formula", "rollup", "if", "automation", "metric", "measure", "link"))): + raise err(400, "bad_field", "Only a user-created value column can be added here.") + fields = _ut().clean_fields([proposed]) + if not fields or fields[0]["key"] != key or not _locked_user_field(fields[0]): + raise err(400, "bad_field", "The column name or type is invalid.") + field = fields[0] + reason = _ut().cell_type_refusal(field, body.get("value")) + if reason: + raise err(400, "bad_value", reason) + if field["type"] == "json" and body.get("value") not in (None, ""): + value = body["value"] + try: + json.loads(value if isinstance(value, str) else json.dumps(value)) + except (TypeError, ValueError): + raise err(400, "bad_value", "This column holds JSON. Enter a valid JSON value.") + # The write context resolves connected membership through the same row wall as existing edits. + g = ut_write_ctx(session, table_key) + if g.get("limits"): + raise err(409, "pid_scope_unresolved", + "The permitted record set could not be resolved. This cell was not saved.") + if pid not in g["pids"]: + raise err(403, "out_of_scope", "That row is not in this database.") + if key in (g.get("hidden") or ()) or key in {f["key"] for f in g["fields"]}: + raise err(403, "field_read_only", "This field is outside your edit access.") + return field + + @router.patch("/tables/{table_key}/shared/{pid}") @_c8_write def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), @@ -981,8 +1173,9 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), key = str(body.get("field") or "").strip() if not key: raise err(400, "bad_request", "a field key is required") - definition = _defn_or_refuse( - session, table_key, scope_applied=_ut().is_connected(table_key, st=session.runtime)) + connected = _ut().is_connected(table_key, st=session.runtime) + locked = not _ut().records_mutable(table_key, st=session.runtime) + definition = _defn_or_refuse(session, table_key, scope_applied=connected or locked) if table_key == 'ut_activity': raise err(403, 'records_read_only', 'Activity records are read-only.') import core.shares as shares @@ -996,8 +1189,13 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), except ValueError as exc: raise err(400, 'bad_value', str(exc)) declared = {f.get('key') for f in definition.get('fields') or []} - if _ut().is_connected(table_key, st=session.runtime) and ( - key in declared or shared_overlay.is_shared(table_key, key, st=session.runtime)): + shared = shared_overlay.is_shared(table_key, key, st=session.runtime) + new_field = None + # A label explicitly declares a new column; a bare unknown-key edit still fails closed. + if locked and key not in declared and not shared and body.get('label'): + new_field = _new_locked_shared_field(session, table_key, pid, key, body) + if new_field is None and (locked or (connected and (key in declared or shared))): + _records_or_refuse(session, table_key, cell_updates={key: body.get('value')}) from core import grid_events from routes_grid import _ctx import uuid @@ -1005,7 +1203,15 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), if g.get('limits'): raise err(409, 'pid_scope_unresolved', 'The permitted record set could not be resolved. This cell was not saved.') + if pid not in g['pids']: + raise err(403, 'out_of_scope', 'That row is not in this database.') + if key not in {field['key'] for field in g['fields']} or key in (g.get('hidden') or ()): + raise err(403, 'field_read_only', 'This field is outside your access.') ctx = _ctx(session, g['fields'], g['pids'], scope_key=table_key) + ctx.hidden_keys = g.get('hidden') or frozenset() + field = next(field for field in g['fields'] if field['key'] == key) + if locked and not grid_events._may_edit_field_value(ctx, key, field): + raise err(403, 'field_read_only', 'This field is outside your edit access.') event_id = uuid.uuid4().hex grid_events.handle_one({'id': event_id, 'type': 'overlay_patch', 'pid': pid, 'updates': {key: body.get('value')}}, ctx) @@ -1034,8 +1240,9 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), # ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading # their absence as "granted to nobody" would blank every existing shared column at once. shared_overlay.put_field(table_key, key, { - "key": key, "label": str(body.get("label") or key), "source": "overlay", - "type": str(body.get("type") or "text"), "shared": True, + **(new_field or {"key": key, "label": str(body.get("label") or key), + "source": "overlay", "type": str(body.get("type") or "text")}), + "shared": True, perm_scope.FIELD_GRANT_MARK: True, "createdBy": session.uname}, st=session.runtime) try: @@ -1241,7 +1448,8 @@ def scoped_pids(session: Session, table_key: str, limits=None, st=None): # D-213 bought on the database-switch path is that this wall answers from the 0.1% document, # and handing it `lend()` would quietly take that back while looking like an optimisation. defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] + fields_base = _source_contract_fields(table_key, [dict(f) for f in (defn.get("fields") or [])], + st=st if st is not None else session.runtime) if table_key == 'ut_activity': if limits is not None: limits.append({'subject': 'pids', 'effect': 'unresolved', @@ -1333,6 +1541,24 @@ def _ut_hidden(session, table_key, fields, st=None): return perm_scope.hidden_keys(session.user, table_key, fields, st=st) +def _source_contract_fields(table_key, fields, st=None): + """Keep the Odoo mirror label on old saved definitions and fresh source contracts.""" + if str(table_key).startswith('ut_odoo_'): + from routes_odoo_tables import _rel, _sources, mirror_store_field + if _rel().base_table_key(table_key) in _sources(): + return [field for field in fields if field.get('key') != 'Store'] + [mirror_store_field()] + if str(table_key).startswith('ut_meta_') and callable(getattr(st, 'assert_datastore_matches', None)): + from routes_odoo_tables import _table_cursor, _meta_source, meta_source_fields, _SOURCE_PRIVATE + cursor = _table_cursor(st, table_key) + try: + spec = _meta_source(cursor, table_key) + if spec is not None: + return [field for field in meta_source_fields(spec, fields) if field['key'] not in _SOURCE_PRIVATE] + finally: + cursor.close() + return fields + + def _ut_contract_fields(table_key, fields, st=None): """The base contract every `ut_*` read door presents before its field wall. @@ -1341,7 +1567,7 @@ def _ut_contract_fields(table_key, fields, st=None): Keeping this boundary named prevents `/nav/schema` and the grid from attaching it on opposite sides of that wall. """ - return _ut().with_zone_field(table_key, fields, st=st) + return _ut().with_zone_field(table_key, _source_contract_fields(table_key, fields, st=st), st=st) def _ut_shared_fields(session, table_key, fields, workspace_document=None, @@ -2941,6 +3167,8 @@ def table_rows(table_key: str, session: Session = Depends(require_session), for zone in _ut().zones(table_key, st=_lent)] from core import connector_spec rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime) + rows = _project_cell_notes(table_key, rows, projection_fields, session.runtime, + hidden=g.get('hidden') or ()) return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"], "refusals": metric_refusals, "zones": _zones, @@ -3710,24 +3938,43 @@ def delete_row(table_key: str, rid: str, session: Session = Depends(require_sess @router.patch("/tables/{table_key}/rows/{pid}") +@_row_patch_identity +@row_patch_receipt(lambda args: args['table_key']) def patch_row(table_key: str, pid: int, body: dict = Body(default=None), session: Session = Depends(require_session), - application_id: str | None = Header(default=None, alias="X-Loopable-App")): + application_id: str | None = Header(default=None, alias="X-Loopable-App"), + response: Response = None): """Cell edits — the products PATCH on the user-table ctx. Routed through `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; the accepted values are read BACK from the bucket, never echoed from the request.""" - from core import grid_events + from core import grid_events, edit_receipts via = _app_write_via(session, application_id) updates = dict(body or {}) + notes = _note_delta(updates) if not updates: raise err(400, "empty_patch", "no fields to update") if _ut().ZONE_FIELD_KEY in updates or "zone" in updates: raise err(403, "zone_membership_read_only", "Zone membership is worked out from the record's current location") _managed_or_refuse(session, table_key, mutation=True) - _records_or_refuse(session, table_key) - g = ut_assembly(session, table_key, consume_corrections=False) + definition = _records_or_refuse(session, table_key, + st=_table_lend(session.runtime, table_key, defs_only=True) if notes is not None else None, + cell_updates=notes if notes is not None else updates) + connected = _ut().is_connected(table_key, st=session.runtime) + if notes is not None and connected: + from routes_odoo_tables import _sources + mirror_note = (table_key in _sources() + or not _ut().materialises(table_key, st=session.runtime, defn=definition)) + else: + mirror_note = False + g = (_connected_note_context(session, table_key, pid) if mirror_note else + ut_write_ctx(session, table_key) if connected else + ut_assembly(session, table_key, consume_corrections=False)) + if any(limit.get('subject') == 'pids' and limit.get('effect') == 'unresolved' + for limit in g.get('limits') or ()): + raise err(409, 'pid_scope_unresolved', + 'The permitted record set could not be resolved. This edit was not saved.') if pid not in g["pids"]: raise err(403, "out_of_scope", "that row is not in this database") ctx = grid_events.EventCtx( @@ -3739,9 +3986,19 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), hidden_keys=g.get("hidden") or frozenset(), st=session.runtime, # R6b (D-16) scope_key=table_key, table=_ops(session, table_key), via=via) + if notes is not None: + return _patch_cell_notes(table_key, pid, notes, ctx, + source_row_exists=lambda key, rid, admitted: key == table_key and int(rid) in g['pids']) + if not _ut().records_mutable(table_key, st=session.runtime): + fields = {field['key']: field for field in g['fields']} + if any(key not in fields or key in ctx.hidden_keys + or not grid_events._may_edit_field_value(ctx, key, fields[key]) + for key in updates): + raise err(403, 'field_read_only', 'This field is outside your edit access.') try: refused = grid_events.handle_one( - {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", + {"id": edit_receipts.current_op_id() or f"patch:{table_key}:{pid}:{time.time_ns()}", + "type": "overlay_patch", "pid": pid, "updates": updates}, ctx) except grid_events.StoreUnavailable: raise err(503, "store_unavailable", @@ -3755,21 +4012,12 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), "cell_write_refused", "row_outside_audience", "bad_row_id"): raise err(400, _refusal.get("code") or "bad_value", _refusal.get("message") or "the edit was refused") - # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. - # - # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH - # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. - # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's - # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began - # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` - # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring - # claiming otherwise is what makes the next reader look for a merge bug that is not here. - # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY - # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` - # asks whether THIS write landed, and this write goes to the definition. - # `get_record` answers the TABLE entry holding this one record; the read-back is the record. + # Connected edits live in the authored overlay, while ordinary edits live in definition rows. stored = dict(((_ut().get_record(table_key, pid, st=session.runtime) or {}).get("rows") or {}) .get(str(pid)) or {}) + if connected: + from core import shared_overlay + stored.update(shared_overlay.cells(table_key, [pid], st=session.runtime).get(str(pid)) or {}) accepted = {k: stored.get(k) for k in updates if k in stored} def _took(k): diff --git a/api/routes_templates.py b/api/routes_templates.py index 79e58eb751f6dbed0bda40e772bbecb9ebf747f3..11d490d5c19153df18e56f6b19c345d9edf588eb 100644 --- a/api/routes_templates.py +++ b/api/routes_templates.py @@ -37,12 +37,12 @@ router = APIRouter(prefix="/api/v1") _TEMPLATE_CATEGORIES = { "sheet": "Sheet", "data": "Table", "app": "App", "agents": "Automation", "saved": "Saved", - "route": "Operations", "catalog": "Sales", "competitor_watch": "Sales", + "route": "Operations", "catalog": "Odoo", "competitor_watch": "Amazon", } # C1 connector keys for a shipped Sheet descriptor's `source` (F25: Amazon is `amazon_sp` there). _SHEET_CONNECTORS = {"odoo": "odoo", "amazon_sp": "amazon"} # C1 fields each App row carries so the client can pick its screen and marks. -_APP_WIRE_KEYS = ("view", "mark", "connector", "process", "purpose") +_APP_WIRE_KEYS = ("view", "mark", "connector", "process", "purpose", "tags") _TEMPLATE_PREVIEW_LABELS = {**_TEMPLATE_CATEGORIES, "route": "Routing", "catalog": "Catalog", "competitor_watch": "Competitor watch"} @@ -249,7 +249,7 @@ def apply_template(key: str, body: dict = Body(default=None), # keeps the reference (the read projection in `grid_events.table_workspace` still masks it). try: import application_registry - raw_registry = session.runtime.get(application_registry._KEY) + raw_registry = application_registry.load(session.runtime, strict=True) live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict) and isinstance(raw_registry.get("applications"), dict) else None) except Exception: @@ -515,11 +515,12 @@ def _module_templates(session): connector = "odoo" if row["source"].startswith("odoo_") else row["source"] data.append(_catalog_row(row["key"], row["label"], row["desc"], [requirement(database, connector)], module="data")) - # These App contracts map fields in the existing wizard. A connector-gated kind is listed - # only when the registry's offer predicate holds, so it needs no requirement row here. - # S79-T05: an App card reads its C1 purpose. R33 removed the database and field steps. - app = [{**_catalog_row(row["kind"], row["label"], row.get("purpose") or row["label"], [], module="app"), - **{key: row.get(key) for key in _APP_WIRE_KEYS}} + # Bound kinds keep their wizard callback; connector-gated Apps post through their template door. + # The registry's offer predicate checks the connector and source walls before listing an App. + app = [{**_catalog_row(row["kind"], row["label"], row.get("purpose") or row["label"], [], module="app", + creation_mode="wizard" if row["kind"] in app_templates.BOUND_KINDS else "post"), + **{key: (app_templates.BY_KIND[row["kind"]] if key == "tags" else row).get(key) + for key in _APP_WIRE_KEYS}} for row in application_templates(session=session)["templates"]] # Owned copies are optional links. They never replace the template's create action. owned = _own_template_apps(session) diff --git a/api/tool_groups/data.py b/api/tool_groups/data.py index 42cdbdbca797b7ad5b2670d8c3cb43a1d272c449..c4441b75946d803119c1fe842382e7d504950c90 100644 --- a/api/tool_groups/data.py +++ b/api/tool_groups/data.py @@ -5,11 +5,40 @@ from core import external_tools from . import legacy_action +QUERY_PAGING_HELP = ( + 'Send limit as a positive integer, for example {"limit": 100}. ' + 'A limit of 501 or more requests pages of at most 500 rows. ' + 'Send page.next_cursor as cursor with the same query until page.has_more is false. ' + 'There is no total row cap.') +DISCOVERY_HELP = ( + "If your client exposes only the previous data tool, send the same action and arguments to that tool. " + "Refresh or reconnect the client to discover table. Read loopable_guide with group table for its actions." +) +READ_REFUSAL_FIXES = { + "assistant_snapshot_unavailable": ( + "This database's current data is unavailable. Describe it to read its fields. " + "Try the same read again after its source sync completes."), + "assistant_store_not_ready": ( + "This database is temporarily unavailable. Try the same read again shortly."), + "assistant_cross_tenant_store": ( + "This connection cannot read that workspace. Reconnect to the intended workspace before reading."), + "assistant_scope_not_applicable": ( + "This database's access restriction could not be applied. Ask an Administrator to repair it. " + "Do not remove or widen the restriction."), + "assistant_filter_not_answerable": ( + "This filter cannot be applied. Use only the fields and operators returned by describe_database. " + "Keep the intended restriction when correcting the filter."), + "query_group_type": ( + "Month grouping requires a date field. Use a date field returned by describe_database, " + "or omit unit to group by the field's values."), +} + + READ_ARGUMENT_HELP = ( "Call table with action describe_database or query_database and an arguments object. " "arguments.database must be a string. Put query options inside arguments. " - "fields must be an array of field keys. limit must be an integer from 1 to 500. " - "Read loopable_guide with group table for the selected action's input schema." + "fields must be an array of field keys. " + QUERY_PAGING_HELP + " " + "Read loopable_guide with group table for the selected action's input schema. " + DISCOVERY_HELP ) @@ -89,8 +118,8 @@ def query_syntax_section(): "one aggregate and no sorts (query_groups_invalid otherwise). Only a connected database, such as an " "Odoo sales lines mirror, groups. Any other database answers 409 query_groups_unavailable: read its " "records by page, or ask for aggregates without groupBy.", - "- Paging: limit is 1 to 500, default 100. Send page.next_cursor as cursor with the same arguments " - "until page.has_more is false. Only fields, filters, sorts, limit, cursor, aggregates and groupBy are " + "- Paging: default 100. " + QUERY_PAGING_HELP + + " Only fields, filters, sorts, limit, cursor, aggregates and groupBy are " "accepted (query_arguments_invalid otherwise).", "- Full pulls: no row cap applies to a connected source. Read every page at limit 500.", ] @@ -212,7 +241,7 @@ def _delivery_photos(ctx, args): GROUP = { "name": "table", "title": "Table", - "guide": "Table: list databases, read fields, group and page records, propose changes, list an Odoo delivery's photos with delivery_photos, or report a limitation. Read loopable_guide(group='table') for actions and a grouping example.", + "guide": "Table: list databases, read fields, group and page records, propose changes, list an Odoo delivery's photos with delivery_photos, or report a limitation. Read loopable_guide(group='table') for actions and a grouping example. " + DISCOVERY_HELP, "actions": { "operation_status": { "read": True, "destructive": False, "confirm": False, diff --git a/api/tool_groups/sheet.py b/api/tool_groups/sheet.py index f7073bf52dd7a7e8c1ae9adc216e4b0679f30b0c..2750d60ac019b31697903ec994c5ed748313dc28 100644 --- a/api/tool_groups/sheet.py +++ b/api/tool_groups/sheet.py @@ -1,5 +1,6 @@ """Sheet actions over the existing workbook permissions and writers.""" from difflib import get_close_matches +import json from fastapi import HTTPException from core import external_access, external_tools @@ -8,6 +9,14 @@ from . import legacy_action WORKBOOK_RESOURCE_URI = "ui://loopable/workbook.html" WORKBOOK_RESOURCE_MIME = "text/html;profile=mcp-app" +WRITE_ABOUT_EXAMPLE = { + "title": "Sheet write", "description": "Update the requested workbook cells.", "tags": ["reporting"]} +WRITE_INTENT_EXAMPLE = "Update the requested workbook cells" +WRITE_CALL_EXAMPLE = { + "action": "write_cells", "arguments": { + "workbook_id": "", "tab": "", + "cells": {"A1": {"kind": "literal", "value": "Weekly review"}}}, + "intent": WRITE_INTENT_EXAMPLE, "about": WRITE_ABOUT_EXAMPLE} #: The next call after workbook_missing (S79-T11): list readable workbooks, then read one by its id. WORKBOOK_MISSING_FIX = ( 'Call sheet with {"action": "workbook.list", "arguments": {"limit": 100}} and follow cursor until it is null. ' @@ -103,6 +112,23 @@ def read_guide(): 'A missing or hidden workbook returns workbook_missing. Readable workbooks return their current tabs.') +def write_guide(): + """Publish the metadata, argument shape and signed confirmation required for Sheet writes.""" + return ( + "Every Sheet write needs intent and about beside action and arguments. " + "If your client sends only action and arguments, put intent and about inside arguments. " + "Use intent as 1 to 500 characters explaining the write. " + "Use about with only title (1 to 240 characters), description (20 to 240 characters), " + "and tags (one or more distinct values). Use no control characters or long dashes in title or description. " + "Allowed tags: " + ", ".join(external_tools.BUILD_TAGS) + ". " + "Use workbook_id and tab from workbook.get. Call sheet with " + + json.dumps(WRITE_CALL_EXAMPLE, separators=(", ", ": ")) + ". " + "The write returns preview_ready without changing cells. After the user approves, call confirm_change " + 'with {"confirmation_token": "", "intent": "Apply the approved Sheet write"}. ' + "Confirmation uses the saved about." + ) + + def _workbook_view(ctx, args): """Project native workbook cells without exposing bindings or private workbook metadata.""" import copy @@ -210,6 +236,7 @@ GROUP["guide"] += ( "Use write_batch for a checked sequence of supported writes. " "Workbook export and named ranges have no grouped replacement in this build.") GROUP["guide"] += ' Under MCP rule 9, ask "Do you want me to build a Sheet workbook in Loopable?".' +GROUP["guide"] += " " + write_guide() _BATCH_GUIDE = ( " A batch has 1 to 50 operations for one workbook. Use name and arguments for each operation. " "Use sheet_add_tab, sheet_write_cells, sheet_set_binding, sheet_set_styles or sheet_set_dimensions " diff --git a/platform/ai_client_steps.json b/platform/ai_client_steps.json index 7114f564a2460c4d04b9446b41a405ef3b716642..63a089b9751b96786d058a53052c48e94d9594c7 100644 --- a/platform/ai_client_steps.json +++ b/platform/ai_client_steps.json @@ -24,116 +24,104 @@ "claude": { "need": "Access to Claude.ai and your Loopable sign in.", "failureAction": "Finish the Loopable sign in from Claude, then select Verify sign in again. Check your workspace policy if the connector is unavailable.", - "testNote": "Select Verify sign in to check for a completed Claude sign in with Loopable.", + "testNote": "", "verified": "2026-10-11", "documentation": "https://support.claude.com/en/articles/11175166-getting-started-with-custom-connectors-using-remote-mcp", "record": "AutoWiki/research/chatgpt-side-view-and-connector-setup.md#4-claude", "provenance": "Documentation reviewed on 2026-10-11. No account walkthrough was performed.", "steps": [ - { "text": "In Claude, open Customize, then Connectors.", "action": { "label": "Open Claude", "url": "https://claude.ai/" } }, - { "text": "Click + Add, then Add custom connector." }, - { "text": "Enter Loopable as the name and this remote MCP server URL. Click Continue.", "block": "" }, - { "text": "Review the detected authentication settings. Click Continue." }, - { "text": "Under Authentication, choose Sign in now to connect now. Sign in when needed and No sign in are also listed." }, - { "text": "Under OAuth client, keep Use Claude’s published identity (Recommended)." }, - { "text": "Click Add and complete any Loopable sign in requested." } + { "title": "Open Connectors", "text": "In Claude, open Customize, then Connectors. Click + Add, then Add custom connector.", "block": "Loopable", "action": { "label": "Open Claude", "url": "https://claude.ai/" } }, + { "title": "Add Loopable", "text": "Enter Loopable and this remote MCP server URL, then Continue.", "block": "" }, + { "title": "Sign in", "text": "Keep Claude’s published identity, choose Sign in now, then Add.", "block": "" } ], - "caveat": "For Team or Enterprise, an Owner first adds the connector through Organization settings > Connectors > Add > Custom > Web. Members then click Connect. Free users are limited to one custom connector." + "caveat": "Review the detected authentication settings and click Continue. Authentication also lists Sign in when needed and No sign in. Keep Use Claude’s published identity (Recommended). Click Add and complete any Loopable sign in requested. For Team or Enterprise, an Owner first adds the connector through Organization settings > Connectors > Add > Custom > Web. Members then click Connect. Free users are limited to one custom connector." }, "chatgpt": { "need": "Access to ChatGPT Plugins and your Loopable sign in.", "failureAction": "Finish the Loopable sign in from ChatGPT, then select Verify sign in again. Check your workspace policy if the plugin is unavailable.", - "testNote": "Select Verify sign in to check for a completed ChatGPT sign in with Loopable.", + "testNote": "", "verified": "2026-10-11", "documentation": "https://developers.openai.com/plugins/deploy/connect-chatgpt", "additionalDocumentation": ["https://developers.openai.com/plugins/quickstart"], "record": "AutoWiki/research/chatgpt-side-view-and-connector-setup.md#3-chatgpt-setup-steps-today", "provenance": "Documentation reviewed on 2026-10-11. No account walkthrough was performed.", "steps": [ - { "text": "Open ChatGPT Plugins at chatgpt.com/plugins.", "action": { "label": "Open ChatGPT Plugins", "url": "https://chatgpt.com/plugins" } }, - { "text": "Select the plus button, then Add custom MCP server." }, - { "text": "Enter Loopable as the name and a description." }, - { "text": "Under Connection, choose the public URL method and enter this MCP server URL.", "block": "" }, - { "text": "Configure OAuth authentication, review the risk warning, and select I understand and want to continue." }, - { "text": "Select Create as a plugin." }, - { "text": "Review the tools and metadata discovered from Loopable." }, - { "text": "Open Loopable under personal plugins and click the plus button to install it." }, - { "text": "In a new chat, type @ and select Loopable. Complete any Loopable sign in requested." } + { "title": "Open Plugins", "text": "Open ChatGPT Plugins at chatgpt.com/plugins. Choose Add custom MCP server.", "block": "Loopable", "action": { "label": "Open ChatGPT Plugins", "url": "https://chatgpt.com/plugins" } }, + { "title": "Add Loopable", "text": "Enter a name and description, then this public URL under Connection.", "block": "" }, + { "title": "Install plugin", "text": "Choose OAuth, Create as a plugin, then install Loopable and sign in.", "block": "@Loopable" } ], - "caveat": "Account and workspace policies apply to adding and using custom MCP servers." + "caveat": "Select the plus button to add the server. Review the risk warning and select I understand and want to continue before Create as a plugin. Review the discovered tools and metadata, then open Loopable under personal plugins and click the plus button to install it. In a new chat, type @ and select Loopable. Complete any Loopable sign in requested. Account and workspace policies apply to adding and using custom MCP servers." }, "claude-code": { "need": "Claude Code on your computer and your Loopable sign in or an active API key.", "failureAction": "Check the server URL and finish signing in, or check your saved API key. Select the verification button for your chosen mode again.", - "testNote": "Select Verify sign in for MCP or Verify API key for a saved key. This checks Loopable access for the selected mode.", + "testNote": "", "verified": "2026-10-11", "documentation": "https://code.claude.com/docs/en/mcp", "record": "AutoWiki/sdlc/sdlc64/lanes/E/context/S64-T12.md", "provenance": "Checked against retained Loopable instructions on 2026-10-11. Provider documentation and account setup were not rechecked. The 2026-09-23 research also did not recheck this command.", "steps": [ - { "text": "In your terminal, run:", "block": "claude mcp add --transport http loopable " }, - { "text": "In Claude Code, run /mcp and finish signing in to Loopable in the browser." } + { "title": "Add server", "text": "Run this command in your terminal.", "block": "claude mcp add --transport http loopable " }, + { "title": "Sign in", "text": "Run /mcp in Claude Code and finish the Loopable browser sign in.", "block": "/mcp" } ], "keySteps": [ - { "text": "If Loopable is already configured, remove its old entry first.", "block": "claude mcp remove loopable" }, - { "text": "Run in your terminal, replacing with your saved key.", "block": "claude mcp add --transport http loopable --header \"Authorization: Bearer \"" }, - { "text": "Open /mcp to check the server. Keep the key out of chat messages." } + { "title": "Remove old entry", "text": "Run this only if Loopable is already configured.", "block": "claude mcp remove loopable" }, + { "title": "Add server", "text": "Replace with your saved key locally, never in chat.", "block": "claude mcp add --transport http loopable --header \"Authorization: Bearer \"" }, + { "title": "Open server", "text": "Open /mcp to check the server.", "block": "/mcp" } ], "caveat": "Use a current Claude Code release. Organization policy can restrict MCP servers." }, "codex": { "need": "Codex on your computer and your Loopable sign in or an active API key.", "failureAction": "Check the server URL and finish signing in, or check the API key in your local environment. Select the verification button again.", - "testNote": "Select Verify sign in for MCP or Verify API key for a saved key. This checks Loopable access for the selected mode.", + "testNote": "", "verified": "2026-09-23", "documentation": "https://learn.chatgpt.com/docs/extend/mcp", "record": "AutoWiki/research/chatgpt-and-codex-mcp-connections.md", "provenance": "Documentation reviewed on 2026-09-23. Remote add flags were measured with codex-cli 0.155.1 on the owner’s laptop that day. No new account walkthrough was performed.", "steps": [ - { "text": "In your terminal, run:", "block": "codex mcp add loopable --url " }, - { "text": "Then sign in to Loopable:", "block": "codex mcp login loopable" } + { "title": "Add server", "text": "Run this command in your terminal.", "block": "codex mcp add loopable --url " }, + { "title": "Sign in", "text": "Sign in to Loopable in the browser.", "block": "codex mcp login loopable" } ], "keySteps": [ - { "text": "In your terminal, run:", "block": "codex mcp add loopable --url --bearer-token-env-var LOOPABLE_API_KEY" }, - { "text": "Set LOOPABLE_API_KEY to your saved key locally before opening Codex. Keep the key out of chat messages." } + { "title": "Set saved key", "text": "Set LOOPABLE_API_KEY locally before opening Codex, never in chat.", "block": "LOOPABLE_API_KEY" }, + { "title": "Add server", "text": "Run this command in your terminal.", "block": "codex mcp add loopable --url --bearer-token-env-var LOOPABLE_API_KEY" } ], "caveat": "Set up once for Codex CLI, the ChatGPT desktop app and the IDE extension on the same host through one ~/.codex/config.toml." }, "cursor": { "need": "Cursor on your computer and your Loopable sign in or an active API key.", "failureAction": "Check the server configuration and sign in, or replace an inactive API key. Reopen the server and select the verification button again.", - "testNote": "Select Verify sign in for MCP or Verify API key for a saved key. This checks Loopable access for the selected mode.", + "testNote": "", "verified": "2026-10-11", "documentation": "https://cursor.com/docs/context/mcp", "record": "AutoWiki/sdlc/sdlc64/lanes/E/context/S64-T12.md", "provenance": "Checked against retained Loopable instructions on 2026-10-11. Provider documentation and account setup were not rechecked. Cursor was added in the SDLC 64 setup record.", "steps": [ - { "text": "In Cursor, open Settings, then Tools and Integrations, and click New MCP Server." }, - { "text": "Add this server to ~/.cursor/mcp.json, keeping any existing servers.", "block": "{\n \"mcpServers\": {\n \"loopable\": {\n \"url\": \"\"\n }\n }\n}" }, - { "text": "Reopen the server and finish signing in to Loopable." } + { "title": "Open settings", "text": "Open Settings, Tools and Integrations, then New MCP Server.", "block": "~/.cursor/mcp.json" }, + { "title": "Add server", "text": "Add this server to ~/.cursor/mcp.json, keeping existing servers.", "block": "{\n \"mcpServers\": {\n \"loopable\": {\n \"url\": \"\"\n }\n }\n}" }, + { "title": "Sign in", "text": "Reopen the server and finish signing in to Loopable.", "block": "" } ], "keySteps": [ - { "text": "In ~/.cursor/mcp.json, add or update this server, keeping any other servers.", "block": "{\n \"mcpServers\": {\n \"loopable\": {\n \"url\": \"\",\n \"headers\": {\n \"Authorization\": \"Bearer \"\n }\n }\n }\n}" }, - { "text": "Replace with your saved key locally, then reopen the server. Keep the key out of chat messages." } + { "title": "Add server", "text": "Add or update Loopable in ~/.cursor/mcp.json, keeping other servers.", "block": "{\n \"mcpServers\": {\n \"loopable\": {\n \"url\": \"\",\n \"headers\": {\n \"Authorization\": \"Bearer \"\n }\n }\n }\n}" }, + { "title": "Use saved key", "text": "Replace locally, then reopen the server. Keep the key out of chat.", "block": "~/.cursor/mcp.json" } ], "caveat": "Cursor reads project servers from .cursor/mcp.json and personal servers from the home folder. An organization policy can block a server." }, "muse": { "need": "Access to Muse and an active Loopable API key.", "failureAction": "Check the MCP address and locally saved bearer key. Select Verify API key again.", - "testNote": "Verify API key checks Loopable access. Ask Muse a question to check its connection.", + "testNote": "", "verified": "2026-10-16", "record": "AutoWiki/sdlc/sdlc76/PRD.md", "provenance": "Loopable API key setup guide. Muse account setup has not been independently verified.", "steps": [ - { "text": "Create a Loopable API key in API key settings. Save its secret locally." }, - { "text": "Ask Muse to add this Loopable MCP address over HTTP, using your locally saved key as a bearer token.", "block": "" }, - { "text": "Set Authorization: Bearer with your saved key locally. Never paste the key into chat." } + { "title": "Save API key", "text": "Create or choose a Loopable API key. Never paste the key into chat.", "block": "Loopable" }, + { "title": "Add server", "text": "Add this MCP address in Muse over HTTP with your locally saved bearer key.", "block": "" } ], "keySteps": [ - { "text": "Create a Loopable API key in API key settings. Save its secret locally." }, - { "text": "Ask Muse to add this Loopable MCP address over HTTP, using your locally saved key as a bearer token.", "block": "" }, - { "text": "Set Authorization: Bearer with your saved key locally. Never paste the key into chat." } + { "title": "Save API key", "text": "Create or choose a Loopable API key. Never paste the key into chat.", "block": "Loopable" }, + { "title": "Add server", "text": "Add this MCP address in Muse over HTTP with your locally saved bearer key.", "block": "" } ], "caveat": "Muse uses the same tenant bound MCP address and grants as other AI clients. Restricted and buyer data stay excluded." } diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 5f6e744b03bd5573dc2bfc6bfd1b16273c4d4308..e2f385ee7a00b80a13b2dd703c9db9acf672347c 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -1928,6 +1928,7 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): odoo_fields = [field for field in fields if field["source"] == "odoo"] overlay_fields = [field for field in fields if field["source"] == "overlay"] derived_keys = [field["key"] for field in fields if field.get("derived")] + readable_keys = {field["key"] for field in fields} overlays = overlays or {} derived = derived or {} out = [] @@ -1940,6 +1941,14 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): # the same way: the Map VIEW's data, nullable, deliberately not a column. row = {"pid": pid, "_created": r.get("_created") or "", "lat": r.get("lat"), "lon": r.get("lon")} + notes = {} + for source in (r, saved): + mapped = source.get('__cell_notes__') + if isinstance(mapped, dict): + notes.update({key: text for key, text in mapped.items() + if key in readable_keys and isinstance(text, str)}) + if notes: + row['__cell_notes__'] = notes for field in odoo_fields: k = field["key"] if field.get("derived"): diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index 0001c086c36895c5eb487dc367d9eeabd64d9461..5dcd47e1a054889436bae81b45ecdf2983466241 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -205,7 +205,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Odoo Aged Receivable overdue: open receivable lines past their due date, no grace, New Customer Hold due at once, credits and unapplied payments netted per customer. Ties to Odoo to the cent." + "description": "Odoo open receivables past due, no grace; New Customer Hold due now. Credits and unapplied payments net per customer, tying to Odoo to the cent." }, { "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", diff --git a/platform/core/activity_log.py b/platform/core/activity_log.py index e704380f2387500cf850adcb516b61e97da42f83..b91224628513cf3232c715e55d43ca525cae218f 100644 --- a/platform/core/activity_log.py +++ b/platform/core/activity_log.py @@ -98,7 +98,7 @@ def _object(value, *, allow_key_only=False): return kind, key, encoded -def _bounded_changes(changes): +def _bounded_changes(changes, *, cell_edit=False): if changes is None: changes = {} if not isinstance(changes, dict): @@ -110,7 +110,7 @@ def _bounded_changes(changes): if '\\u0000' in raw: raise ValueError('changes cannot contain a null character.') original_bytes = len(raw.encode('utf-8')) - if original_bytes <= CHANGES_MAX_BYTES: + if cell_edit or original_bytes <= CHANGES_MAX_BYTES: return json.loads(raw), False, original_bytes kept = {} for key, pair in changes.items(): @@ -203,7 +203,8 @@ def append(rt, *, kind, object, by, via=None, on_behalf_of=None, changes=None, _ by = _text(by, 'by') via = _text(via, 'via', optional=True) on_behalf_of = _text(on_behalf_of, 'on_behalf_of', optional=True) - bounded, truncated, original_bytes = _bounded_changes(changes) + bounded, truncated, original_bytes = _bounded_changes( + changes, cell_edit=kind == 'cell' or bool(_delivery and _delivery.get('_cell_edit'))) now = _time(_delivery['occurred_at']) if _delivery else _now() values = (now.isoformat(), kind, object_type, object_key, object_id, by, via, on_behalf_of, _json(bounded), truncated, original_bytes) @@ -240,6 +241,10 @@ def drain(rt, *, keys=None): finally: if delivered: def acknowledge(document): + # Delivery priority differs by writer. Lock every delivered row before reading any. + lock_records = getattr(document, 'lock_records', None) + if callable(lock_records): + lock_records({DATABASE_KEY: sorted(delivered)}) queue = (document.get(DATABASE_KEY) or {}).get(OUTBOX_KEY) or {} for key, version in delivered.items(): entry = queue.get(key) @@ -303,7 +308,8 @@ def _pending_page(backend, pending, *, target, actor, since, until, before, limi if prior is not None and prior[0] >= delivery['version']: yield prior[1] continue - changes, truncated, original_bytes = _bounded_changes(entry.get('changes')) + changes, truncated, original_bytes = _bounded_changes( + entry.get('changes'), cell_edit=entry['kind'] == 'cell' or delivery.get('_cell_edit', False)) yield _event((event_id, stamp, entry['kind'], kind, name, identity, entry['by'], entry.get('via'), entry.get('on_behalf_of'), changes, truncated, original_bytes), @@ -447,6 +453,7 @@ class PgActivityStore: def _connection(self): from core import store_pg + store_pg._effective_slug(self.tenant_slug) return store_pg._pool().connection() def _exists(self, con): @@ -457,6 +464,20 @@ class PgActivityStore: if self._ready: return with self._connection() as con: + from core import store_pg + if store_pg._CALLER_TRANSACTION.get() is not None and self._exists(con): + columns = con.execute('SELECT attname FROM pg_attribute WHERE attrelid = %s::regclass ' + 'AND attnum > 0 AND NOT attisdropped', (self.table,)).fetchall() + indexes = con.execute('SELECT indexname FROM pg_indexes WHERE schemaname = %s ' + 'AND tablename = %s', (self.schema, self.name)).fetchall() + constraint = con.execute('SELECT pg_get_constraintdef(oid) FROM pg_constraint ' + 'WHERE conrelid = %s::regclass AND conname = %s', + (self.table, f'{self.name}_kind_check')).fetchone() + needed = {f'{self.name}_{suffix}' for suffix in ( + 'delivery', 'time', 'object_time', 'object_id_time', 'person_time')} + if (set(_COLUMNS) | {'delivery_key', 'delivery_version'} <= {r[0] for r in columns} + and needed <= {r[0] for r in indexes} and constraint and "'view'" in constraint[0]): + return con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} ( id BIGSERIAL PRIMARY KEY, occurred_at TIMESTAMPTZ NOT NULL, @@ -487,7 +508,10 @@ class PgActivityStore: ): con.execute(f'CREATE INDEX IF NOT EXISTS "{self.name}_{suffix}" ' f'ON {self.table} ({columns})') - self._ready = True + # A caller rollback can remove newly created indexes and the table itself. + from core import store_pg + if store_pg._CALLER_TRANSACTION.get() is None: + self._ready = True def _prune(self, con, cutoff, limit): return con.execute(f'''DELETE FROM {self.table} WHERE id IN ( diff --git a/platform/core/connector_directory.py b/platform/core/connector_directory.py index 6fd6266f37a146f435a8da6dd23bd79fdd671edc..9e6a6e8c00be8ba2c714229a48f4160c3a8b2e7d 100644 --- a/platform/core/connector_directory.py +++ b/platform/core/connector_directory.py @@ -235,6 +235,7 @@ NATIVE_CATEGORIES = { # same way Odoo is: it reaches orders, fees and listings, which is Data plus the two shelves # Odoo itself sits under for the commerce side of the same story. "amazon_sp": ("Data", "E-commerce", "Finance"), + "amazon_ads": ("Data", "E-commerce", "Finance"), "webhooks": ("Utilities", "Web Tools"), "scraper": ("Scrapers", "Social Media"), "stripe": ("Finance", "E-commerce"), diff --git a/platform/core/connector_spec.py b/platform/core/connector_spec.py index 22bc350ef0ef178328f7fa0c25d9440b42ab7d25..85ce6765676e5c90bcde2414cae62143c769cb0b 100644 --- a/platform/core/connector_spec.py +++ b/platform/core/connector_spec.py @@ -308,7 +308,10 @@ def system_link_fields(key, live_columns): "key": edge["key"], "label": _link_field_label(edge["label"]), "type": "link", "source": "overlay", "default": True, "link": {"table": target, "on": edge["to_col"], "from": edge["from_col"], - "single": edge["cardinality"] == "many_to_one"}, + "single": edge["cardinality"] == "many_to_one", + **({"connectionScope": "_connection_id"} + if source.startswith("ut_amzn_") and not source.startswith("ut_amzn_ads_") + and "store" in live_columns[source] else {})}, }) return out @@ -336,7 +339,7 @@ AMAZON_TABLES = ( key=_AMZN + "orders", native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", source="amzn_orders", - label="Amazon orders", + label="Amazon Sales: Orders", pinned="amazon_order_id", identity=("amazon_order_id",), links=( @@ -347,7 +350,7 @@ AMAZON_TABLES = ( key=_AMZN + "order_items", native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", source="amzn_order_items", - label="Amazon order items", + label="Amazon Sales: Order items", pinned="amazon_order_id", identity=("order_item_id", "amazon_order_id"), links=( @@ -361,7 +364,7 @@ AMAZON_TABLES = ( key=_AMZN + "listings", native="GET_MERCHANT_LISTINGS_ALL_DATA", source="amzn_listings", - label="Amazon listings", + label="Amazon Listings: All SKUs", pinned="seller_sku", identity=("seller_sku", "asin1", "asin"), links=(), # the product spine's HEAD: everything points at it, it points nowhere @@ -370,7 +373,7 @@ AMAZON_TABLES = ( key=_AMZN + "settlements", native="GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2", source="amzn_settlements", - label="Amazon settlements", + label="Amazon Money: Settlements", pinned="settlement_id", identity=("settlement_id",), # ⚠ THE MONEY-TO-ORDER BRIDGE — what turns "an order happened" into "this is what we were @@ -384,7 +387,7 @@ AMAZON_TABLES = ( key=_AMZN + "inventory", native="GET_LEDGER_SUMMARY_VIEW_DATA", source="amzn_inventory", - label="Amazon inventory", + label="Amazon Inventory: All", pinned="asin", identity=("asin", "fnsku", "msku"), # ⚠ THE LEDGER KEYS ON `msku`, NOT `sku` — FBA calls the seller's own SKU the MSKU. Same @@ -398,7 +401,7 @@ AMAZON_TABLES = ( key=_AMZN + "pricing", native="GET_SALES_AND_TRAFFIC_REPORT", source="amzn_pricing", - label="Amazon pricing and Buy Box", + label="Amazon Listings: Pricing and Buy Box", pinned="parent_asin", identity=("child_asin", "parent_asin"), links=( @@ -410,7 +413,7 @@ AMAZON_TABLES = ( key=_AMZN + "brand_analytics", native="GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", source="amzn_brand_analytics", - label="Amazon brand analytics", + label="Amazon Marketing: Brand analytics", pinned="search_term", identity=("department_name", "search_term", "clicked_asin", "click_share_rank", "_period_start", "_period_end"), @@ -421,14 +424,14 @@ AMAZON_TABLES = ( ), ConnectorTableSpec( key=_AMZN + "economics", native="analytics_economics_2024_03_15", - source="amzn_economics", label="Amazon economics", pinned="msku", + source="amzn_economics", label="Amazon Sales: Unit economics", pinned="msku", identity=("start_date", "msku", "marketplace_id"), links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"), ("child_asin", _AMZN + "listings", "asin1", "many_to_one")), ), ConnectorTableSpec( key=_AMZN + "transactions", native="finances/2024-06-19/transactions", - source="amzn_transactions", label="Amazon financial transactions", pinned="transaction_id", + source="amzn_transactions", label="Amazon Money: Transactions", pinned="transaction_id", identity=("transaction_id", "item_position"), links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), @@ -436,325 +439,325 @@ AMAZON_TABLES = ( ), ConnectorTableSpec( key=_AMZN + "fba_fee_estimates", native="GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA", - source="amzn_fba_fee_estimates", label="Amazon FBA fee estimates", pinned="sku", + source="amzn_fba_fee_estimates", label="Amazon Fees: FBA", pinned="sku", identity=("_snapshot_date", "sku", "amazon_store"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one")), ), ConnectorTableSpec( key=_AMZN + "referral_fees", native="GET_REFERRAL_FEE_PREVIEW_REPORT", - source="amzn_referral_fees", label="Amazon referral fee preview", pinned="seller_sku", + source="amzn_referral_fees", label="Amazon Fees: Referral", pinned="seller_sku", identity=("_snapshot_date", "seller_sku"), links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one")), ), ConnectorTableSpec( key=_AMZN + "storage_fees", native="GET_FBA_STORAGE_FEE_CHARGES_DATA", - source="amzn_storage_fees", label="Amazon storage fees", pinned="asin", + source="amzn_storage_fees", label="Amazon Fees: Storage", pinned="asin", identity=("month", "asin", "fnsku", "fulfillment_center"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "search_query_performance", native="GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT", - source="amzn_search_query_performance", label="Amazon search query performance", + source="amzn_search_query_performance", label="Amazon Marketing: Search terms", pinned="search_query_data_search_query", identity=("start_date", "end_date", "asin", "search_query_data_search_query"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "long_term_storage_fees", native="GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA", - source="amzn_long_term_storage_fees", label="Amazon long term storage fees", pinned="sku", + source="amzn_long_term_storage_fees", label="Amazon Fees: Long term storage", pinned="sku", identity=("_period_start", "sku", "fnsku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "storage_overage_fees", native="GET_FBA_OVERAGE_FEE_CHARGES_DATA", - source="amzn_storage_overage_fees", label="Amazon storage overage fees", pinned="storage_type", + source="amzn_storage_overage_fees", label="Amazon Fees: Storage overage", pinned="storage_type", identity=("_period_start", "storage_type", "_report_row_number"), links=(), ), ConnectorTableSpec( key=_AMZN + "reimbursements", native="GET_FBA_REIMBURSEMENTS_DATA", - source="amzn_reimbursements", label="Amazon reimbursements", pinned="reimbursement_id", + source="amzn_reimbursements", label="Amazon Money: Reimbursements", pinned="reimbursement_id", identity=("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"), ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "fba_customer_returns", native="GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", - source="amzn_fba_customer_returns", label="Amazon FBA customer returns", pinned="order_id", + source="amzn_fba_customer_returns", label="Amazon Returns: FBA", pinned="order_id", identity=("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"), links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "replacements", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA", - source="amzn_replacements", label="Amazon replacements", pinned="replacement_amazon_order_id", + source="amzn_replacements", label="Amazon Returns: Replacements", pinned="replacement_amazon_order_id", identity=("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "fba_shipped_sales", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA", - source="amzn_fba_shipped_sales", label="Amazon FBA shipped sales", pinned="amazon_order_id", + source="amzn_fba_shipped_sales", label="Amazon Sales: FBA shipments", pinned="amazon_order_id", identity=("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"), links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "fba_shipment_promotions", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA", - source="amzn_fba_shipment_promotions", label="Amazon FBA shipment promotions", pinned="amazon_order_id", + source="amzn_fba_shipment_promotions", label="Amazon Marketing: FBA shipment promotions", pinned="amazon_order_id", identity=("_period_start", "amazon_order_id", "sku", "_report_row_number"), links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "inventory_ledger_detail", native="GET_LEDGER_DETAIL_VIEW_DATA", - source="amzn_inventory_ledger_detail", label="Amazon inventory ledger detail", pinned="msku", + source="amzn_inventory_ledger_detail", label="Amazon Inventory: Ledger", pinned="msku", identity=("_period_start", "msku", "fnsku", "asin", "_report_row_number"), links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "inventory_planning", native="GET_FBA_INVENTORY_PLANNING_DATA", - source="amzn_inventory_planning", label="Amazon inventory planning", pinned="sku", + source="amzn_inventory_planning", label="Amazon Inventory: Planning", pinned="sku", identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "manage_inventory", native="GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA", - source="amzn_manage_inventory", label="Amazon manage inventory", pinned="sku", + source="amzn_manage_inventory", label="Amazon Inventory: Manage inventory", pinned="sku", identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "reserved_inventory", native="GET_RESERVED_INVENTORY_DATA", - source="amzn_reserved_inventory", label="Amazon reserved inventory", pinned="sku", + source="amzn_reserved_inventory", label="Amazon Inventory: Reserved", pinned="sku", identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "restock_recommendations", native="GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", - source="amzn_restock_recommendations", label="Amazon restock recommendations", pinned="merchant_sku", + source="amzn_restock_recommendations", label="Amazon Inventory: Restock", pinned="merchant_sku", identity=("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"), links=(("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one")), ), ConnectorTableSpec( key=_AMZN + "stranded_inventory", native="GET_STRANDED_INVENTORY_UI_DATA", - source="amzn_stranded_inventory", label="Amazon stranded inventory", pinned="sku", + source="amzn_stranded_inventory", label="Amazon Inventory: Stranded", pinned="sku", identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "inbound_noncompliance", native="GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA", - source="amzn_inbound_noncompliance", label="Amazon inbound noncompliance", pinned="shipment_id", + source="amzn_inbound_noncompliance", label="Amazon Inventory: Inbound issues", pinned="shipment_id", identity=("_period_start", "shipment_id", "sku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "inventory_expiration_dates", native="GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA", - source="amzn_inventory_expiration_dates", label="Amazon inventory expiration dates", pinned="sku", + source="amzn_inventory_expiration_dates", label="Amazon Inventory: Expiration dates", pinned="sku", identity=("_snapshot_date", "sku", "fnsku", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "recommended_removals", native="GET_FBA_RECOMMENDED_REMOVAL_DATA", - source="amzn_recommended_removals", label="Amazon recommended removals", pinned="sku", + source="amzn_recommended_removals", label="Amazon Inventory: Recommended removals", pinned="sku", identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "removal_orders", native="GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA", - source="amzn_removal_orders", label="Amazon removal orders", pinned="order_id", + source="amzn_removal_orders", label="Amazon Inventory: Removal orders", pinned="order_id", identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "removal_shipments", native="GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA", - source="amzn_removal_shipments", label="Amazon removal shipments", pinned="order_id", + source="amzn_removal_shipments", label="Amazon Inventory: Removal shipments", pinned="order_id", identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "subscribe_save_forecast", native="GET_FBA_SNS_FORECAST_DATA", - source="amzn_subscribe_save_forecast", label="Amazon Subscribe and Save forecast", pinned="sku", + source="amzn_subscribe_save_forecast", label="Amazon Marketing: Subscribe and Save forecast", pinned="sku", identity=("_snapshot_date", "sku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "subscribe_save_performance", native="GET_FBA_SNS_PERFORMANCE_DATA", - source="amzn_subscribe_save_performance", label="Amazon Subscribe and Save performance", pinned="sku", + source="amzn_subscribe_save_performance", label="Amazon Marketing: Subscribe and Save", pinned="sku", identity=("_period_start", "sku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "remote_fulfillment_eligibility", native="GET_REMOTE_FULFILLMENT_ELIGIBILITY", - source="amzn_remote_fulfillment_eligibility", label="Amazon remote fulfillment eligibility", pinned="sku", + source="amzn_remote_fulfillment_eligibility", label="Amazon Inventory: Remote fulfillment", pinned="sku", identity=("_snapshot_date", "sku", "asin", "_report_row_number"), links=(), # Measured header has no product key; do not declare a guessed join. ), ConnectorTableSpec( key=_AMZN + "search_catalog_performance", native="GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT", - source="amzn_search_catalog_performance", label="Amazon search catalog performance", pinned="asin", + source="amzn_search_catalog_performance", label="Amazon Marketing: Search funnel by ASIN", pinned="asin", identity=("_period_start", "asin", "_report_row_number"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "market_basket", native="GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT", - source="amzn_market_basket", label="Amazon market basket", pinned="asin", + source="amzn_market_basket", label="Amazon Marketing: Bought together", pinned="asin", identity=("_period_start", "asin", "purchased_with_asin", "_report_row_number"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "repeat_purchase", native="GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT", - source="amzn_repeat_purchase", label="Amazon repeat purchase", pinned="asin", + source="amzn_repeat_purchase", label="Amazon Marketing: Repeat purchases", pinned="asin", identity=("_period_start", "asin", "_report_row_number"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "promotion_performance", native="GET_PROMOTION_PERFORMANCE_REPORT", - source="amzn_promotion_performance", label="Amazon promotion performance", pinned="promotion_id", + source="amzn_promotion_performance", label="Amazon Marketing: Promotions", pinned="promotion_id", identity=("_period_start", "promotion_id", "_report_row_number"), links=(), ), ConnectorTableSpec( key=_AMZN + "coupon_performance", native="GET_COUPON_PERFORMANCE_REPORT", - source="amzn_coupon_performance", label="Amazon coupon performance", pinned="coupon_id", + source="amzn_coupon_performance", label="Amazon Marketing: Coupons", pinned="coupon_id", identity=("_period_start", "coupon_id", "_report_row_number"), links=(), ), ConnectorTableSpec( key=_AMZN + "account_health", native="GET_V2_SELLER_PERFORMANCE_REPORT", - source="amzn_account_health", label="Amazon account health", pinned="marketplace_id", + source="amzn_account_health", label="Amazon Account health", pinned="marketplace_id", identity=("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"), links=(), ), ConnectorTableSpec( key=_AMZN + "inactive_listings", native="GET_MERCHANT_LISTINGS_INACTIVE_DATA", - source="amzn_inactive_listings", label="Amazon inactive listings", pinned="seller_sku", + source="amzn_inactive_listings", label="Amazon Listings: Inactive", pinned="seller_sku", identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"), links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin1", _AMZN + "listings", "asin1", "many_to_one")), ), ConnectorTableSpec( key=_AMZN + "cancelled_listings", native="GET_MERCHANT_CANCELLED_LISTINGS_DATA", - source="amzn_cancelled_listings", label="Amazon cancelled listings", pinned="seller_sku", + source="amzn_cancelled_listings", label="Amazon Listings: Cancelled", pinned="seller_sku", identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"), links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin1", _AMZN + "listings", "asin1", "many_to_one")), ), ConnectorTableSpec( key=_AMZN + "suppressed_listings", native="GET_MERCHANTS_LISTINGS_FYP_REPORT", - source="amzn_suppressed_listings", label="Amazon suppressed listings", pinned="sku", + source="amzn_suppressed_listings", label="Amazon Listings: Suppressed", pinned="sku", identity=("_snapshot_date", "sku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "listing_defects", native="GET_MERCHANT_LISTINGS_DEFECT_DATA", - source="amzn_listing_defects", label="Amazon listing defects", pinned="sku", + source="amzn_listing_defects", label="Amazon Listings: Defects", pinned="sku", identity=("_snapshot_date", "sku", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "sold_listings", native="GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA", - source="amzn_sold_listings", label="Amazon sold listings", pinned="sku", + source="amzn_sold_listings", label="Amazon Listings: Sold", pinned="sku", identity=("_snapshot_date", "sku", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "mfn_returns", native="GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE", - source="amzn_mfn_returns", label="Amazon MFN returns", pinned="order_id", + source="amzn_mfn_returns", label="Amazon Returns: Seller fulfilled", pinned="order_id", identity=("_period_start", "order_id", "sku", "asin", "_report_row_number"), links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "b2b_recommended", native="GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU", - source="amzn_b2b_recommended", label="Amazon B2B product opportunities", pinned="asin", + source="amzn_b2b_recommended", label="Amazon Marketing: B2B product opportunities", pinned="asin", identity=("_snapshot_date", "asin", "_report_row_number"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "b2b_not_yet_on_amazon", native="GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON", - source="amzn_b2b_not_yet_on_amazon", label="Amazon B2B products not yet on Amazon", pinned="ean", + source="amzn_b2b_not_yet_on_amazon", label="Amazon Marketing: B2B products not yet on Amazon", pinned="ean", identity=("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"), links=(), ), ConnectorTableSpec( key=_AMZN + "geo_opportunities", native="GET_FLAT_FILE_GEO_OPPORTUNITIES", - source="amzn_geo_opportunities", label="Amazon geo opportunities", pinned="asin", + source="amzn_geo_opportunities", label="Amazon Marketing: Geo opportunities", pinned="asin", identity=("_snapshot_date", "asin", "_report_row_number"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "out_of_stock_mfn", native="GET_OUT_OF_STOCK_MFN_INFORMATION", - source="amzn_out_of_stock_mfn", label="Amazon out of stock MFN listings", pinned="sku", + source="amzn_out_of_stock_mfn", label="Amazon Listings: Out of stock MFN", pinned="sku", identity=("_snapshot_date", "sku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "b2b_quotes", native="RFQD_BULK_DOWNLOAD", - source="amzn_b2b_quotes", label="Amazon B2B quantity discounts", pinned="sku", + source="amzn_b2b_quotes", label="Amazon Listings: B2B quantity discounts", pinned="sku", identity=("_snapshot_date", "sku", "asin", "_report_row_number"), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "fee_discounts", native="FEE_DISCOUNTS_REPORT", - source="amzn_fee_discounts", label="Amazon fee discounts", pinned="sku", + source="amzn_fee_discounts", label="Amazon Fees: Discounts", pinned="sku", identity=("_snapshot_date", "sku", "asin", "_report_row_number"), links=(), # The measured empty report supplies metadata only. ), ConnectorTableSpec( key=_AMZN + "order_metrics", native="sales/v1/orderMetrics", - source="amzn_order_metrics", label="Amazon order metrics", pinned="interval", + source="amzn_order_metrics", label="Amazon Sales: Metrics by period", pinned="interval", identity=("_period_start", "interval", "_report_row_number"), links=(), ), ConnectorTableSpec( key=_AMZN + "fba_inventory", native="fba/inventory/v1/summaries", - source="amzn_fba_inventory", label="Amazon FBA inventory summaries", pinned="seller_sku", + source="amzn_fba_inventory", label="Amazon Inventory: FBA", pinned="seller_sku", identity=("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"), links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "listing_items", native="listings/2021-08-01/items/{sellerId}", - source="amzn_listing_items", label="Amazon listing status and issues", pinned="sku", + source="amzn_listing_items", label="Amazon Listings: Status and issues", pinned="sku", identity=("sku",), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "catalog_items", native="catalog/2022-04-01/items", - source="amzn_catalog_items", label="Amazon catalog attributes", pinned="asin", + source="amzn_catalog_items", label="Amazon ASINs", pinned="asin", identity=("_snapshot_date", "asin", "_report_row_number"), links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "fee_estimates_api", native="products/fees/v0/feesEstimate", - source="amzn_fee_estimates_api", label="Amazon per-price fee estimates", pinned="sku", + source="amzn_fee_estimates_api", label="Amazon Fees: Per-price estimates", pinned="sku", identity=("estimate_id",), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "outbound_fulfillment", native="fulfillmentOutbound/2020-07-01/fulfillmentOrders", - source="amzn_outbound_fulfillment", label="Amazon outbound fulfillment orders", pinned="seller_fulfillment_order_id", + source="amzn_outbound_fulfillment", label="Amazon Inventory: Outbound fulfillment orders", pinned="seller_fulfillment_order_id", identity=("seller_fulfillment_order_id", "seller_fulfillment_order_item_id"), links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "inbound_shipments", native="inbound/fba/2024-03-20/inboundPlans", - source="amzn_inbound_shipments", label="Amazon inbound shipments", pinned="sku", + source="amzn_inbound_shipments", label="Amazon Inventory: Inbound shipments", pinned="sku", identity=("shipment_item_id",), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "replenishment", native="replenishment/2022-11-07/offers/metrics/search", - source="amzn_replenishment", label="Amazon replenishment metrics", pinned="sku", + source="amzn_replenishment", label="Amazon Inventory: Replenishment", pinned="sku", identity=("metric_id",), links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance", - source="amzn_sku_keywords", label="Amazon keywords per SKU", pinned="sku", + source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku", identity=("keyword_id",), links=(("sku", _AMZN + "listing_items", "sku", "many_to_one"),), ), ConnectorTableSpec( key=_AMZN + "keyword_ranks", native="derived:apify:khadinakbar/amazon-search-scraper+sku_keywords", - source="amzn_keyword_ranks", label="Amazon keyword ranks", pinned="keyword", + source="amzn_keyword_ranks", label="Amazon Marketing: Keyword ranks", pinned="keyword", identity=("rank_id",), links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),), ), @@ -3570,6 +3573,11 @@ def table_group(key): return None +for _amazon_table in AMAZON_TABLES: + AMAZON_FIELDS[_amazon_table.key] = (*AMAZON_FIELDS.get(_amazon_table.key, ()), { + "name": "store", "type": "VARCHAR", "label": "Store", "source": "keychain.label", + }) + register("amazon", AMAZON_TABLES, mirror=ConnectorMirrorSpec(tuple(s.source for s in AMAZON_TABLES if not s.native.startswith("derived:")) diff --git a/platform/core/connector_sync.py b/platform/core/connector_sync.py index 14e3220658941d5ab31a6c0ab9e13b4a63e36355..a7a6c18a955cb73e76518654e130868ff1859764 100644 --- a/platform/core/connector_sync.py +++ b/platform/core/connector_sync.py @@ -206,10 +206,14 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non with _SYNC_REQUEST_LOCK: previous = _SYNC_REQUESTS.get((provider, tenant)) if previous and previous["state"] in ("queued", "running"): - return {**previous, "accepted": False} - receipt = {"provider": provider, "tenant": tenant, "state": "queued", + if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load" + and requested_by != "first load"): + return {**_request_wait(previous), "accepted": False} + receipt = {"id": uuid.uuid4().hex, "provider": provider, "tenant": tenant, "state": "queued", "requestedAt": time.time() if now is None else now, - "notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0)) or None, + "notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0), + float(previous.get("notBefore") or 0) if previous + and previous["state"] == "queued" else 0) or None, "startedAt": None, "finishedAt": None, "detail": "", "requestedBy": str(requested_by or ""), "accepted": True} if requested_by == "first load" and previous and previous.get("firstLoad"): @@ -220,7 +224,39 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non _arm_sync_wake(stamp) if (receipt.get("notBefore") or 0) <= stamp: _SYNC_WAKE.set() - return dict(receipt) + return _request_wait(receipt, now=stamp) + + +def _request_order(receipt): + """Manual work precedes first-load continuations, with FIFO within each class.""" + return receipt.get("requestedBy") == "first load", receipt.get("requestedAt") or 0 + + +def _request_wait(receipt, *, now=None): + """Describe a queued receipt's executing competitor or provider floor without claiming a start.""" + result = deepcopy(receipt) + result.setdefault("waitingFor", None) + if receipt["state"] != "queued": + result["waitingFor"] = None + return result + running = [row for row in _SYNC_REQUESTS.values() + if row["tenant"] == receipt["tenant"] and row["state"] == "running" + and row.get("startedAt") is not None] + if running: + active = max(running, key=lambda row: row["startedAt"]) + started = active["startedAt"] + identity = active.get("id") or f"{active['provider']}:{started}" + result["waitingFor"] = {"id": identity, "provider": active["provider"], "startedAt": started} + stamp = _dt.datetime.fromtimestamp(started, _dt.timezone.utc).isoformat() + result["detail"] = f"Waiting for {active['provider']} receipt {identity}, started at {stamp}." + elif result.get("waitingFor"): + return result + elif (receipt.get("notBefore") or 0) > (time.time() if now is None else now): + stamp = _dt.datetime.fromtimestamp(receipt["notBefore"], _dt.timezone.utc).isoformat() + result["detail"] = f"Waiting for the provider not-before floor at {stamp}." + elif result.get("detail", "").startswith("Waiting for "): + result["detail"] = "" + return result def sync_request(provider, tenant): @@ -231,22 +267,34 @@ def sync_request(provider, tenant): receipt = next((row for row in rows if row["state"] == "running"), None) if receipt is None: queued = [row for row in rows if row["state"] == "queued"] - receipt = min(queued, key=lambda row: row["requestedAt"]) if queued else _SYNC_REQUESTS.get((provider, tenant)) + receipt = min(queued, key=_request_order) if queued else _SYNC_REQUESTS.get((provider, tenant)) if receipt is None: return None - return {**deepcopy(receipt), "accepted": False} + return {**_request_wait(receipt), "accepted": False} -def take_sync_requests(*, now=None): - """Claim at most one receipt per provider/tenant, preserving request order and siblings.""" +def take_sync_requests(*, now=None, limit=None, exclude_providers=(), refresh_remote=False): + """Claim manual work first, preserving provider floors, tenant claims and event siblings.""" + if refresh_remote: + with _SYNC_REQUEST_LOCK: + runtimes = list(_SYNC_REQUEST_RUNTIMES.values()) + for rt in runtimes: + try: + poll_remote_requests(rt, now=now) + except Exception as exc: + logging.getLogger(__name__).warning("Sync desk refresh unavailable for %s (%s)", + rt.key, type(exc).__name__) with _SYNC_REQUEST_LOCK: _SYNC_WAKE.clear() claimed = [] stamp = time.time() if now is None else now busy = {(row["provider"], row["tenant"]) for row in _SYNC_REQUESTS.values() if row["state"] == "running"} - for receipt in sorted(list(_SYNC_REQUESTS.values()), key=lambda row: row["requestedAt"]): + for receipt in sorted(list(_SYNC_REQUESTS.values()), key=_request_order): + if limit is not None and len(claimed) >= limit: + break pair = (receipt["provider"], receipt["tenant"]) - if pair in busy or receipt["state"] != "queued" or (receipt.get("notBefore") or 0) > stamp: + if (receipt["provider"] in exclude_providers or pair in busy or receipt["state"] != "queued" + or max(receipt.get("notBefore") or 0, _SYNC_NOT_BEFORE.get(pair, 0)) > stamp): continue if receipt.get("tables") is not None: rt = _SYNC_REQUEST_RUNTIMES[receipt["tenant"]] @@ -260,7 +308,7 @@ def take_sync_requests(*, now=None): return document eligible = [row for row in rows if row["state"] == "queued" and (row.get("notBefore") or 0) <= stamp] - row = min(eligible, key=lambda row: row["requestedAt"]) if eligible else None + row = min(eligible, key=_request_order) if eligible else None if row is not None and row["id"] == identity: row.update(state="running", startedAt=stamp, claimToken=token, accepted=False) return document @@ -272,9 +320,12 @@ def take_sync_requests(*, now=None): continue else: receipt.update(state="running", startedAt=stamp, accepted=False) + receipt["waitingFor"] = None + if receipt.get("detail", "").startswith("Waiting for "): + receipt["detail"] = "" claimed.append(deepcopy(receipt)) busy.add(pair) - _arm_sync_wake(stamp) + _wake_sync_requests(stamp) return claimed @@ -310,6 +361,7 @@ def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load else: receipt.update(completion) result = deepcopy(receipt) + result["waitingFor"] = None _wake_sync_requests(stamp) connector = {"amazon_sp": "amazon", "meta_ads": "meta"}.get(provider) if connector: @@ -334,7 +386,7 @@ def sync_wake_event(): def queue_remote_request(rt, receipt, connection_id=None, *, now=None): """Web side: durably hand one accepted ordinary request to the sync worker process.""" stamp = time.time() if now is None else now - identity = uuid.uuid4().hex + identity = receipt.get("id") or uuid.uuid4().hex entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key, "requestedBy": str(receipt.get("requestedBy") or ""), "requestedAt": float(receipt.get("requestedAt") or stamp), @@ -365,10 +417,13 @@ def fail_remote_request(provider, tenant, detail, *, now=None): def _remote_progress(entry, receipt): """The worker receipt's progress for one entry, or None when that run does not cover it.""" started = receipt.get("startedAt") if receipt else None - if not started or entry["requestedAt"] > started or receipt["state"] not in ("running", "done", "failed"): + if (started is None or entry["requestedAt"] > started + or receipt["state"] not in ("running", "done", "failed") + or (entry.get("requestedBy") != "first load" and receipt.get("requestedBy") == "first load")): return None progress = {"state": receipt["state"], "startedAt": started, - "finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or ""} + "finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or "", + "waitingFor": None} if receipt.get("firstLoad"): progress["firstLoad"] = dict(receipt["firstLoad"]) return progress @@ -384,8 +439,7 @@ def poll_remote_requests(rt, *, now=None): admitted, changes, pruned = [], {}, [] with _SYNC_REQUEST_LOCK: document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True)) - for identity, entry in sorted(document[REMOTE_MEMBER].items(), - key=lambda item: item[1].get("requestedAt") or 0): + for identity, entry in sorted(document[REMOTE_MEMBER].items(), key=lambda item: _request_order(item[1])): key = (entry.get("provider"), rt.key) if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS: continue @@ -394,6 +448,16 @@ def poll_remote_requests(rt, *, now=None): pruned.append(identity) continue receipt = _SYNC_REQUESTS.get(key) + # Promote under the same lock as claims, so a continuation cannot win a manual hand-off race. + if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load" + and entry.get("requestedBy") != "first load"): + receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "", + requestedAt=entry["requestedAt"], + connectionId=entry.get("connectionId"), + notBefore=max(float(receipt.get("notBefore") or 0), + float(entry.get("notBefore") or 0), + _SYNC_NOT_BEFORE.get(key, 0)) or None) + admitted.append(deepcopy(entry)) progress = _remote_progress(entry, receipt) if progress is not None: if any(entry.get(name) != value for name, value in progress.items()): @@ -402,17 +466,30 @@ def poll_remote_requests(rt, *, now=None): # A "running" entry with no local run was abandoned by a worker restart: admit it again. if (entry["state"] in ("queued", "running") and not (receipt and receipt["state"] in ("queued", "running"))): - local = {"provider": key[0], "tenant": rt.key, "state": "queued", + local = {"id": identity, "provider": key[0], "tenant": rt.key, "state": "queued", "requestedAt": entry["requestedAt"], "notBefore": max(float(entry.get("notBefore") or 0), _SYNC_NOT_BEFORE.get(key, 0)) or None, "startedAt": None, "finishedAt": None, "detail": "", + "connectionId": entry.get("connectionId"), "requestedBy": entry.get("requestedBy") or "", "accepted": True} if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load": local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"])) _SYNC_REQUESTS[key] = local _SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt) admitted.append(deepcopy(entry)) + receipt = local + waiting = _request_wait({**entry, "state": "queued", "startedAt": None, + "finishedAt": None, "waitingFor": None, + "notBefore": max(float(entry.get("notBefore") or 0), + _SYNC_NOT_BEFORE.get(key, 0), + float(receipt.get("notBefore") or 0) + if receipt and receipt["state"] == "queued" else 0) or None}, + now=stamp) + fields = {name: waiting.get(name) for name in + ("state", "startedAt", "finishedAt", "notBefore", "detail", "waitingFor")} + if any(entry.get(name) != value for name, value in fields.items()): + changes[identity] = fields if changes or pruned: def publish(current): doc = _sync_request_document(current) @@ -438,10 +515,11 @@ def apply_remote_progress(rt): receipt = _SYNC_REQUESTS.get(key) if _REMOTE_IDS.get(key) != identity or receipt is None: continue - fields = {name: entry.get(name) for name in ("state", "startedAt", "finishedAt", "detail")} + fields = {name: entry.get(name) for name in + ("state", "startedAt", "finishedAt", "detail", "notBefore", "waitingFor")} if entry.get("firstLoad"): fields["firstLoad"] = dict(entry["firstLoad"]) - if entry["state"] != "queued" and any(receipt.get(k) != v for k, v in fields.items()): + if any(receipt.get(k) != v for k, v in fields.items()): receipt.update(fields) applied += 1 # Event receipts the worker claimed or finished, refreshed without requeueing anything. diff --git a/platform/core/edit_receipts.py b/platform/core/edit_receipts.py new file mode 100644 index 0000000000000000000000000000000000000000..c30e1c73bb45218d7384579c20b54afb3317f174 --- /dev/null +++ b/platform/core/edit_receipts.py @@ -0,0 +1,216 @@ +"""Retain edit responses in the tenant store and serialize replays before applying them.""" +from contextlib import contextmanager +from contextvars import ContextVar +from copy import deepcopy +import hashlib +import json +import threading +import time + + +TABLE_NAME = 'edit_receipts' +_OFFLINE_LOCK = threading.RLock() +_ACTIVE = ContextVar('edit_receipt_operation', default=None) +# Reserve half of store_pg's eight connections for callbacks across all tenant handles. +_PG_SLOTS = threading.BoundedSemaphore(4) + + +class ReceiptUnavailable(RuntimeError): + """The edit cannot receive a durable acknowledgement.""" + + +class CheckedStore: + """Require synchronous acknowledged writes from the event's existing store handle.""" + + def __init__(self, st): + self.st = st + + def __getattr__(self, name): + return getattr(self.st, name) + + def _store(self): + return self.st._store() if callable(getattr(self.st, '_store', None)) else self.st + + def update(self, name, fn, flush='sync', **kwargs): + result = self.st.update(name, fn, flush='sync', **kwargs) + if result is None or result is False: + raise ReceiptUnavailable('The edit write was not saved. Retry this opId.') + return result + + +def op_id(value): + """Keep the complete client identity without truncation or normalization.""" + if not isinstance(value, str) or not value.strip(): + raise ValueError('opId must be a non-empty string.') + return value + + +def operation(scope, kind, pid=None): + return json.dumps([str(scope), str(kind), None if pid is None else str(pid)], + separators=(',', ':')) + + +def current_op_id(): + active = _ACTIVE.get() + return active['key'][3] if active is not None else None + + +def observe_event(result): + active = _ACTIVE.get() + if active is not None: + active['event'] = deepcopy(result) + + +def current_event(): + active = _ACTIVE.get() + return deepcopy(active.get('event')) if active is not None else None + + +def _backend(st): + from core import store, store_pg + bound = st._store() if callable(getattr(st, '_store', None)) else st + # The default runtime can hold the module facade instead of a PgStore. + if bound is store and store.backend() == 'pg': + bound = store._d() + if isinstance(bound, store_pg.PgStore): + if getattr(st, 'store_namespace', ''): + raise ReceiptUnavailable('Edit receipts require a dedicated tenant schema.') + if getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug: + raise ReceiptUnavailable('The edit receipt store belongs to another tenant.') + return PgReceipts(bound) + # Detached stores retain their normal get/update contract without opening Postgres. + return DocumentReceipts(st) + + +def run_once(st, principal, action, identity, apply): + """Return the first committed response and skip every later application of its identity.""" + identity = op_id(identity) + key = (id(st), str(principal), str(action), identity) + if _ACTIVE.get() is not None and _ACTIVE.get()['key'] == key: + return apply() + + def invoke(): + token = _ACTIVE.set({'key': key}) + try: + return apply() + finally: + _ACTIVE.reset(token) + + return _backend(st).run(str(principal), str(action), identity, invoke) + + +class DocumentReceipts: + """Keep offline and legacy stores hermetic, with serialization within their process.""" + + def __init__(self, st): + self.st = st + + def run(self, principal, action, identity, apply): + key = json.dumps([principal, action, identity], separators=(',', ':')) + with _OFFLINE_LOCK: + prior = (self.st.get(TABLE_NAME) or {}).get(key) + if prior is not None: + return deepcopy(prior['result']) + result = apply() + + def save(current): + current = dict(current or {}) + current[key] = {'result': deepcopy(result)} + return current + + stored = self.st.update(TABLE_NAME, save, flush='sync') + if stored is True: + stored = self.st.get(TABLE_NAME) + if not isinstance(stored, dict) or key not in stored: + raise ReceiptUnavailable('The edit receipt was not saved. Retry this opId.') + return deepcopy(stored[key]['result']) + + +class PgReceipts: + """Use the bound tenant schema and a transaction lock for each principal and operation.""" + + def __init__(self, bound): + from core import store_pg + self.slug = bound.tenant_slug + self.schema = store_pg._schema_for(self.slug) + self.table = f'"{self.schema}"."{TABLE_NAME}"' + + def _connection(self): + from core import store_pg + return store_pg._pool().connection() + + def ensure(self): + self._write_guard() + with self._connection() as con: + con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) + con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} ( + receipt_key BYTEA PRIMARY KEY, + principal TEXT NOT NULL, operation TEXT NOT NULL, op_id TEXT NOT NULL, + result JSONB NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now())''') + + def _write_guard(self): + from core import store_pg + store_pg.check_write(self.slug, 'store edit receipt') + + @contextmanager + def _admission(self): + from core import request_timing, store_pg + timeout = request_timing.remaining_budget(store_pg._pg_limits()['acquire']) + if not _PG_SLOTS.acquire(timeout=timeout): + request_timing.remaining_budget() + raise store_pg.PostgresOperationTimeout('Edit receipt admission timed out. Retry this opId.') + try: + request_timing.remaining_budget() + yield + finally: + _PG_SLOTS.release() + + def run(self, principal, action, identity, apply): + from core import request_timing, store_pg + limits = store_pg._pg_limits() + deadline = time.monotonic() + limits['acquire'] + limits['statement'] + with request_timing.budget_scope(deadline, store_pg.PostgresOperationTimeout): + with self._admission(): + return self._run(principal, action, identity, apply) + + def _run(self, principal, action, identity, apply): + from core import request_timing, store_pg + self.ensure() + store = store_pg.PgStore(self.slug) + lock = json.dumps([self.table, principal, action, identity], separators=(',', ':')) + receipt_key = hashlib.sha256(lock.encode('utf-8')).digest() + # Poll without keeping a pool connection that the current writer may need. + while True: + request_timing.remaining_budget() + with self._connection() as con: + acquired = con.execute( + 'SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0))', + (lock,)).fetchone()[0] + if acquired: + # The callback may leave this transaction idle, within the existing database budget. + idle_ms = max(1, int(request_timing.remaining_budget( + store_pg._pg_limits()['idle']) * 1000)) + con.execute("SELECT set_config('idle_in_transaction_session_timeout', %s, true)", + (f'{idle_ms}ms',)) + prior = con.execute(f'SELECT result FROM {self.table} ' + 'WHERE receipt_key=%s', (receipt_key,)).fetchone() + if prior is not None: + result = deepcopy(prior[0]) + break + with store.transaction(con): + result = apply() + request_timing.remaining_budget() + con.execute(f'INSERT INTO {self.table} ' + '(receipt_key, principal, operation, op_id, result) ' + 'VALUES (%s, %s, %s, %s, %s::jsonb)', + (receipt_key, principal, action, identity, json.dumps(result))) + # The connection context commits before the caller receives the answer. + break + time.sleep(request_timing.remaining_budget(0.01)) + # Deliver only committed activity. The durable intent survives a delivery outage. + from core import activity_log + try: + activity_log.drain(store) + except Exception as error: + print('[edit_receipts] activity delivery deferred: ' + type(error).__name__) + return deepcopy(result) diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py index 26f392c14b02207c72f81ca01a6b32749bcbd972..02ab141ccc729247e5d9c9de170fb66eca4bfbf2 100644 --- a/platform/core/external_tools.py +++ b/platform/core/external_tools.py @@ -519,7 +519,7 @@ def _callback(ctx, name, args): _DATABASE_ACTIONS = ( "list", "describe", "query", "record_create", "record_update", "record_delete", - "field_create", "field_update", "field_delete", + "field_create", "field_update", "field_delete", "rename", ) _VIEW_ACTIONS = ("upsert", "delete", "share_preview") _SHEET_OBJECT_ACTIONS = { @@ -670,10 +670,14 @@ TOOLS = { additionalProperties=False), lambda ctx, args: ctx.app_delete(args), write=True, grants=("ai_apps",)), "database_object": _tool( - "Read a database or preview changes to its records and fields, then confirm them.", + "Read a database or preview changes to its records, fields, name or icon, then confirm them. Name and icon changes require an Administrator.", _object({ "action": {"enum": list(_DATABASE_ACTIONS)}, "database": _STRING, + "name": _STRING, + "icon": {"oneOf": [{"type": "null"}, _object({ + "shape": _STRING, "tone": _STRING, + }, ("shape", "tone"), additionalProperties=False)]}, "record_id": {"oneOf": [_STRING, {"type": "integer"}]}, "values": {"type": "object"}, "field": {"type": "object"}, diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 3c6e3563d4fbab82a7eaad229000adcccc7ddc8f..d34299650c4ba079fccead22c5dbed0904b49af6 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -132,9 +132,8 @@ class EventCtx: is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity — never anything the browser said about itself. - `seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every - emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP - passes the caller's per-session mapping. + `seen_ids` deduplicates trusted/internal and fallback events within their context. + HTTP outbox delivery opts into `durable_receipts`; an active client PATCH captures its event too. `fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username (exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes @@ -192,6 +191,7 @@ class EventCtx: # Trusted adapters supply activity attribution. Browser event payloads cannot override it. by: Optional[str] = None via: Optional[str] = None + durable_receipts: bool = False def __post_init__(self): # A caller that forgets `seen_ids` gets per-call dedup rather than a crash — the SET is @@ -644,21 +644,46 @@ def _grant_marker(ctx, ws, uname=None): #: S67-T39 / R36: the Application registry's store key. READ here, never imported: #: `application_registry` lives in the api layer and core never imports up (ARCHITECTURE). _APPLICATIONS_KEY = 'applications_v1' +_APPLICATION_RECORD_PREFIX = 'applications_v1_app/' def _live_application_ids(ctx): """The set of App ids the tenant registry holds, or None when that is not KNOWN. - None (read failed, or no registry document with an `applications` map) means "strip - nothing": an unknown answer must never turn a live App binding into a plain view. + None means a read failed or a document is malformed, so no binding can be stripped. """ try: - doc = _store_of(ctx).get(_APPLICATIONS_KEY) + st = _store_of(ctx) + doc = st.get(_APPLICATIONS_KEY) + apps = {} if doc is None else doc.get('applications', {}) if isinstance(doc, dict) else None + if not isinstance(apps, dict): + return None + live = {str(key) for key in apps} + listing = getattr(st, 'list_keys', None) + if not callable(listing): + return live + keys = listing(_APPLICATION_RECORD_PREFIX) + reader = getattr(st, 'get_many', None) + records = reader(keys) if keys and callable(reader) else None + if records is None: + records = {key: st.get(key) for key in keys} + tenant = getattr(st, 'key', None) or getattr(st, 'tenant_slug', None) + for key in keys: + record = records.get(key) + if record is None: + continue + applications = record.get('applications', {}) if isinstance(record, dict) else None + if not isinstance(applications, dict) or len(applications) > 1: + return None + for identity, app in applications.items(): + if (not isinstance(app, dict) or app.get('id') != identity + or (tenant is not None and app.get('tenant') != tenant)): + return None + live.update(str(identity) for identity in applications) + return live except Exception as exc: _tel.error('applications:live', exc) return None - apps = doc.get('applications') if isinstance(doc, dict) else None - return {str(key) for key in apps} if isinstance(apps, dict) else None def _application_ref_id(view): @@ -1363,6 +1388,71 @@ def _activity_action(fn): return wrapped +def _edit_receipt(fn): + @wraps(fn) + def wrapped(event, ctx): + from copy import copy, deepcopy + from dataclasses import replace + from core import edit_receipts + if not isinstance(event, dict): + return False + identity = event.get('opId', event.get('id')) + active_id = edit_receipts.current_op_id() + # Trusted staged previews own their confirmation ledger and must not gain transport metadata. + if ctx.fallback_ws is not None or not (ctx.durable_receipts or ( + active_id is not None and active_id == identity)): + identity = str(identity or '')[:180] + if not identity or identity in ctx.seen_ids: + return False + ctx.seen_ids[identity] = True + if len(ctx.seen_ids) > 400: + for stale in list(ctx.seen_ids)[:200]: + ctx.seen_ids.pop(stale, None) + return fn(dict(event, id=identity), ctx) + if not identity: + return False + try: + identity = edit_receipts.op_id(identity) + except ValueError as exc: + ctx.out.write_id = '' + return _refuse(ctx, 'bad_op_id', str(exc), event=event.get('type')) + event = dict(event, id=identity) + + def apply(): + before = len(ctx.out.refusals) + checked = edit_receipts.CheckedStore(_store_of(ctx)) + table = copy(ctx.table) if ctx.table is not None else None + if table is not None: + table._st = checked + write_ctx = replace(ctx, st=checked, table=table) + rerender = fn(event, write_ctx) + write = dict(ctx.out.writes.get(identity) or {}) + write['opId'] = identity + result = deepcopy({'rerender': bool(rerender), 'write': write, + 'refusals': ctx.out.refusals[before:], + 'doc': ctx.out.doc, 'toast': ctx.out.toast}) + edit_receipts.observe_event(result) + return result + + before = len(ctx.out.refusals) + try: + receipt = edit_receipts.run_once( + _store_of(ctx), ctx.uname, + edit_receipts.operation(ctx.scope_key, event.get('type'), event.get('pid')), + identity, apply) + except edit_receipts.ReceiptUnavailable as exc: + raise StoreUnavailable(str(exc)) from exc + receipt = receipt.get('event') or receipt + # Replace this event's observations so first delivery and replay have the same shape. + ctx.out.refusals[before:] = receipt['refusals'] + ctx.out.writes[identity] = receipt['write'] + ctx.out.write_id = identity + ctx.out.doc, ctx.out.toast = receipt['doc'], receipt['toast'] + return receipt['rerender'] + return wrapped + + +@_edit_receipt @_activity_action def handle_one(event, ctx): """Validate ONE untrusted component event. @@ -1399,8 +1489,7 @@ def handle_one(event, ctx): cohort_* / add_to_list — True: the lists payload is server truth, and these are not the hot path. - The id dedupe is a SET (not a last-id slot): the event log resends the whole recent - window on every emit, so every id must stay consumed, not just the latest one. + Transport receipts retain the first result; trusted/internal callers use their context's seen ids. """ # The ctx fields the moved body reads as plain locals — bound once here so every branch # below is textually the app.py body (which is what makes this move reviewable). @@ -1417,15 +1506,8 @@ def handle_one(event, ctx): if not isinstance(event, dict): return False - event_id = str(event.get('id') or '')[:180] - seen = ctx.seen_ids - if not event_id or event_id in seen: - return False - seen[event_id] = True + event_id = event['id'] ctx.out.write_id = event_id - if len(seen) > 400: # a session-lifetime set, kept bounded - for stale in list(seen)[:200]: - seen.pop(stale, None) kind = event.get('type') if kind == 'overlay_patch' and str(scope_key) == 'ut_activity': return _refuse(ctx, 'activity_read_only', 'Activity records cannot be edited.', @@ -2448,9 +2530,12 @@ def handle_one(event, ctx): _fp.grant_entries(field.get('permissions')), owner=field.get('createdBy') or uname, st=ctx.table.st) elif _store_of(ctx).available(): - field = (_tops(ctx).save_field( + saved_field = _tops(ctx).save_field( uname, field, reserved_names=_reserved_field_names, - correction_id=event_id) or field) + correction_id=event_id) + if not saved_field: + raise StoreUnavailable('The field definition was not saved.') + field = saved_field else: _session_ready() _field_names = list(_reserved_field_names) @@ -2474,6 +2559,15 @@ def handle_one(event, ctx): ws['fields'].pop(key, None) else: ws['fields'][key] = field + if ctx.fallback_ws is None and not _tstore.source_override_is_empty(field): + if shared_field: + fresh_fields = _shared_overlay.fields(shared_storage_key, st=ctx.table.st) or {} + else: + fresh_fields = (table_workspace( + ctx, allowed_pids=allowed_pids, consume_corrections=False).get('fields') or {}) + durable_field = fresh_fields.get(key) + if not isinstance(durable_field, dict) or not _field_echo_equal(durable_field, field): + raise StoreUnavailable('The saved field definition could not be read back.') from core.if_field import project_fields ctx.out.writes[event_id] = { 'ok': not refused_perms, diff --git a/platform/core/keychain.py b/platform/core/keychain.py index d233d35a7a328a9eaeaee6717c388a6d32b56cdd..ddf4b928bf9e48210b99426041b8c4df64b72867 100644 --- a/platform/core/keychain.py +++ b/platform/core/keychain.py @@ -36,6 +36,12 @@ KEY = 'keychain' ORIGINAL_ENVELOPE_VERSION = 1 ENVELOPE_VERSION = ORIGINAL_ENVELOPE_VERSION AMAZON_ROTATION_DEADLINE_FIELD = 'client_secret_rotation_deadline' +# Required Amazon fields belong below the clients that consume credentials. +_AMAZON_OAUTH_FIELDS = ('client_id', 'client_secret', 'refresh_token') +AMAZON_CREDENTIAL_FIELDS = { + 'amazon_sp': _AMAZON_OAUTH_FIELDS + ('seller_id', 'marketplace_id'), + 'amazon_ads': _AMAZON_OAUTH_FIELDS + ('profile_id',), +} #: ⭐ WAVE 23 (C11/R8) — `stripe` and `shopify` joined. They are TOKEN connectors, not OAuth #: ones, and that is the whole reason they could ship in this wave: both issue a long-lived #: restricted key / custom-app token an admin pastes in, so they need no OAuth client, no @@ -224,9 +230,10 @@ def slot_of(entry, ordinal): return n if n >= 1 else int(ordinal) -def list_entries(rt): +def list_entries(rt, etype=None): """Metadata ONLY — never a decrypted field, never `enc` itself. + Optional `etype` filters metadata without changing the unfiltered reader or slot identities. Every row carries `connection_id` and `slot` (W54-T44 / C4). Rows may GAIN keys here and must never LOSE one: `verify_api` asserts the listing has no `enc` / `url` / `api_key` / `fields`, and several callers read these rows by key. @@ -236,16 +243,19 @@ def list_entries(rt): for eid, e in sorted((_bucket(rt).get('entries') or {}).items()): if not isinstance(e, dict): continue - etype = e.get('type') or 'generic' - seen[etype] = ordinal = seen.get(etype, 0) + 1 + entry_type = e.get('type') or 'generic' + seen[entry_type] = ordinal = seen.get(entry_type, 0) + 1 out.append({'id': eid, 'connection_id': connection_id_of(e, eid), 'slot': slot_of(e, ordinal), 'label': e.get('label') or eid, - 'type': etype, + 'type': entry_type, 'preview': e.get('preview') or '', 'created': e.get('created') or '', 'createdBy': e.get('createdBy') or ''}) + if etype is not None: + wanted = str(etype or '').strip().lower() + return [row for row in out if row['type'] == wanted] return out @@ -260,7 +270,7 @@ def next_slot(rt, etype): DEFAULT rather than at zero: a tenant with two un-stamped Odoo entries gets 3, not 1. """ etype = str(etype or '').strip().lower() - top = 0 + top = int((_bucket(rt).get('slot_high_water') or {}).get(etype) or 0) for row in list_entries(rt): if row.get('type') == etype: try: @@ -308,13 +318,8 @@ def add_entry(rt, label, etype, fields, username, *, scope=None, write_scope=Non continue if k and isinstance(v, (str, int, float, bool)): clean[k] = str(v)[:MAX_FIELD_LEN] - if etype in ('amazon_sp', 'amazon_ads'): - # S79-T13: Amazon Ads stores its four fields whole, like Amazon SP stores its five. - if etype == 'amazon_sp': - from harness import amazon_client as roster - else: - from harness import amazon_ads_client as roster - missing = [key for key in roster.CREDENTIAL_FIELDS + if etype in AMAZON_CREDENTIAL_FIELDS: + missing = [key for key in AMAZON_CREDENTIAL_FIELDS[etype] if not isinstance(fields.get(key), str) or not clean.get(key, '').strip()] if missing: raise ValueError('Missing ' + ', '.join(missing) + '. Enter every required field.') @@ -359,6 +364,9 @@ def add_entry(rt, label, etype, fields, username, *, scope=None, write_scope=Non if len(entries) >= MAX_ENTRIES: raise ValueError(f'this keychain is at its {MAX_ENTRIES}-entry cap') entries[eid] = row + high_water = dict(cur.get('slot_high_water') or {}) + high_water[etype] = max(int(high_water.get(etype) or 0), slot) + cur['slot_high_water'] = high_water return cur stored = rt.update(KEY, _up, flush='sync') @@ -403,7 +411,14 @@ def rename_entry(rt, entry_id, label): def delete_entry(rt, entry_id): + retired = next((row for row in list_entries(rt) if row['id'] == str(entry_id)), None) + def _drop(cur): + if retired: + high_water = dict(cur.get('slot_high_water') or {}) + kind = retired['type'] + high_water[kind] = max(int(high_water.get(kind) or 0), int(retired['slot'])) + cur['slot_high_water'] = high_water (cur.get('entries') or {}).pop(str(entry_id), None) return cur rt.update(KEY, _drop, flush='sync') diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index c5e7b9be1cfb843d5fe85092481c9e8c3d059513..97b4bb0ee374ff391873ad2e63ec7e2948703f6a 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -446,6 +446,11 @@ def hidden_keys(user, module, fields, st=None): e = entry(user, module) if e: hidden |= {str(k) for k in (e.get('hiddenFields') or ()) if k} + if module == 'product_data': + team_id, _agent = derive_pool_scope(user, module) + families = {5: ('royal1', 'royal2'), 6: ('fisch',)}.get(team_id, ()) + hidden.update(f'{kind}_{family}_t{tier}' for family in families + for kind in ('price', 'unit') for tier in (1, 2, 3)) if not hidden: return frozenset() # ⭐⭐ W40-T04 / I13 — A THIRD SOURCE, SEEDED BEFORE THE FIXPOINT FOR THE SAME REASON THE diff --git a/platform/core/shared_overlay.py b/platform/core/shared_overlay.py index 7186401ad834ed0001f70ed22793f2720162f021..df63259af98f67232998bffbe5cf223e2b914611 100644 --- a/platform/core/shared_overlay.py +++ b/platform/core/shared_overlay.py @@ -146,6 +146,29 @@ def _st(st): return st if st is not None else store +def _cell_backend(table_key, st, *, write=False): + """Select native cell storage only for a directly bound tenant store.""" + from core import store_pg + from core.user_tables import _Lent + target = _st(st) + while isinstance(target, _Lent): + if not write: + return None + target = target._runtime + if (getattr(target, 'activity_preview', False) is True + or callable(getattr(target, 'changed', None))): + return None + if target is store: + target = store._d() + resolver = getattr(target, '_document_store', None) + bound = resolver() if callable(resolver) else target + if not isinstance(bound, store_pg.PgStore): + return None + name = bucket(table_key) + keyer = getattr(target, 'store_key', None) + return bound, keyer(name) if callable(keyer) else name + + def _read(table_key, st=None): """The whole stratum, always in its full two-key shape. Lenient like every other display read: an unreachable store degrades to "nothing shared yet", never to an exception on a page @@ -353,6 +376,10 @@ def fields(table_key, st=None, document=None): tenant-wide by definition — that is the whole feature, and it is what stops a shared view naming a column half the workspace lacks. Its VALUES are scoped by `cells()`. """ + if not isinstance(document, dict): + native = _cell_backend(table_key, st) + if native is not None: + return dict(native[0].get_entry(native[1], 'fields') or {}) data = document if isinstance(document, dict) else _read(table_key, st) return dict(data.get('fields') or {}) @@ -361,7 +388,7 @@ def is_shared(table_key, field_key, st=None): """Is this column's value tenant-wide? The predicate a caller uses to decide WHICH stratum to read and write — one evaluator, so the read path and the write path cannot disagree about where a given column lives ([[one-evaluator-per-question]]).""" - return str(field_key or '') in _read(table_key, st)['fields'] + return str(field_key or '') in fields(table_key, st) def _clean_route(bag): @@ -566,7 +593,9 @@ def cells(table_key, pids, st=None): wanted = {_pid(p) for p in pids} if not wanted: return {} - stored = _read(table_key, st)['cells'] + native = _cell_backend(table_key, st) + stored = ((native[0].shared_document(native[1], wanted).get('cells') or {}) if native is not None + else _read(table_key, st)['cells']) return {pid: dict(row) for pid, row in stored.items() if pid in wanted and isinstance(row, dict) and row} @@ -589,8 +618,12 @@ def snapshot(table_key, pids, st=None): if pids is None: raise TypeError('shared_overlay.snapshot: pids is required. Pass the row set you have ' 'already scoped for this reader (there is deliberately no "all" call)') - stored = _read(table_key, st) wanted = {_pid(p) for p in pids} + native = _cell_backend(table_key, st) + stored = (native[0].shared_document(native[1], wanted) if native is not None + else _read(table_key, st)) + if native is not None and not stored: + stored = {'fields': {}, 'cells': {}} served = {pid: dict(row) for pid, row in stored['cells'].items() if pid in wanted and isinstance(row, dict) and row} if wanted else {} return dict(stored['fields']), served @@ -624,6 +657,12 @@ def put_cells(table_key, pid, values, st=None): if not clean: return {} + native = _cell_backend(table_key, st, write=True) + if native is not None: + native[0].shared_patch(native[1], {row_id: clean}, + lambda data: _validate_link_values(data, {row_id: clean}, st)) + return dict(clean) + def _patch(data): _validate_link_values(data, {row_id: clean}, st) data['cells'].setdefault(row_id, {}).update(clean) @@ -647,6 +686,8 @@ def put_cells_and_choice_options(table_key, pid, values, choice_values, st=None) (choice_values or {}).items() if str(key) and labels} if not clean: return {} + if not additions: + return put_cells(table_key, pid, clean, st=st) def _patch(data): _validate_link_values(data, {row_id: clean}, st) diff --git a/platform/core/sheet_binding.py b/platform/core/sheet_binding.py index 69747418c9ff00f0f00c3e3f8bb9f8aaf4c29235..30be48e534cb2b13b9507dda5cb7d251c9caeb67 100644 --- a/platform/core/sheet_binding.py +++ b/platform/core/sheet_binding.py @@ -35,8 +35,8 @@ def database_change_token(st, database): shared = database if database.startswith("ut_") else workspace buckets = [workspace, shared + "__shared"] handle = st._store() if callable(getattr(st, "_store", None)) else st + key = getattr(st, "store_key", str) if isinstance(handle, store_pg.PgStore): - key = getattr(st, "store_key", str) # Values stay on the server. No row JSON crosses the connection on this path. with store_pg._pool().connection() as con: row = con.execute(store_pg._q( @@ -71,6 +71,7 @@ def database_change_token(st, database): store_pg._table("user_table_records", handle.tenant_slug)), (database, database)).fetchone() token = fingerprint([token, row_token]) + token = fingerprint([token, handle.revision(key(shared + "__shared"))["token"]]) else: definition = (st.get(user_tables.STORE_KEY, fresh=True) or {}).get(database) or {} jobs = st.get(connector_sync.JOBS_KEY, fresh=True) or {} diff --git a/platform/core/store.py b/platform/core/store.py index 967ff7f8cdea425a9eac24c963ed3bede09b216e..c76c2c5dbbd7bf222aacc798d55ecc149b56e8d8 100644 --- a/platform/core/store.py +++ b/platform/core/store.py @@ -494,6 +494,63 @@ class Store: self._cache[name], self._base[name] = data, sha return selected(data) + def _json_document(self, name): + """Return a cached document without copying unrelated members.""" + with self._lock: + if name in self._cache: + return self._cache[name] + data, sha = self._download(name) + with self._lock: + if name not in self._cache: + self._cache[name], self._base[name] = data, sha + return self._cache[name] + + @staticmethod + def _json_value(document, path): + for part in path: + if isinstance(document, dict): + document = document.get(str(part)) + elif isinstance(document, list): + try: + document = document[int(part)] + except (ValueError, TypeError, IndexError): + return None + else: + return None + return document + + def get_json_paths(self, name, paths): + """Copy at most 4096 selected nested values from the cached HF document.""" + from itertools import islice + paths = [list(map(str, path)) for path in islice(iter(paths), 4097)] + if len(paths) > 4096: + raise ValueError("A JSON path read accepts at most 4096 paths per batch") + if not paths: + return [] + document = self._json_document(name) + return [json.loads(json.dumps(self._json_value(document, path))) for path in paths] + + def iter_json_items(self, name, path=(), *, batch_size=1024): + """Yield detached object members in batches without copying the containing document.""" + size = int(batch_size) + if not 1 <= size <= 4096: + raise ValueError("A JSON object batch accepts between 1 and 4096 entries") + document = self._json_document(name) + value = self._json_value(document, path) + if value is None: + yield [] + return + if not isinstance(value, dict): + raise RuntimeError("canonical_source_invalid: the stored rows are not an object") + batch = [] + for key, item in value.items(): + batch.append((key, json.loads(json.dumps(item)))) + if len(batch) == size: + yield batch + batch = [] + if batch or not value: + yield batch + def _key_lock(self, name): """The write lock for ONE key (W31-T13). Created on demand; never removed.""" n = str(name) diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index 7eeadeeae182683da3ff766e091333e4b3f97d2a..7985fd98bf7f1f13cc09b057b4408ba989092c23 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -49,6 +49,7 @@ import threading import time from collections import OrderedDict from contextlib import contextmanager, nullcontext +from contextvars import ContextVar from functools import lru_cache from pathlib import Path from urllib.parse import urlparse @@ -63,6 +64,63 @@ _URL_ENV = 'DATABASE_URL' _LOCK = threading.RLock() _POOL = {'pool': None, 'url': None, 'schema': None} _DATABASE_CA_ENV = 'AIOS_DATABASE_CA_PEM_B64' +_CALLER_TRANSACTION = ContextVar('store_pg_caller_transaction', default=None) + + +class _EnlistedPool: + """Lend the caller's connection without committing or returning it to a pool.""" + + _aios_pipelines = False + + def __init__(self, con, slug): + self.con, self.slug = con, slug + + @contextmanager + def connection(self, **kwargs): + _request_timing.remaining_budget() + yield self.con + + autocommit = connection + + +@contextmanager +def _caller_transaction(con, tenant_slug): + """Enlist bounded tenant writes in an already active caller-owned PostgreSQL transaction.""" + from psycopg.pq import TransactionStatus + from psycopg.conninfo import conninfo_to_dict + slug = _effective_slug(tenant_slug) + check_write(slug, 'enlist edit receipt transaction') + current = _CALLER_TRANSACTION.get() + if current is not None: + if current.con is not con or current.slug != slug: + raise ValueError('A different store transaction is already enlisted.') + yield + return + if con.info.transaction_status != TransactionStatus.INTRANS: + raise ValueError('Enlist an active caller-owned transaction.') + if not _url(): + raise ValueError('The store database endpoint is not configured.') + expected = conninfo_to_dict(_url()) + if any(str(getattr(con.info, key)) != str(expected[key]) + for key in ('host', 'port', 'dbname') if key in expected): + raise ValueError('The receipt connection belongs to another database endpoint.') + limits = _pg_limits() + deadline = time.monotonic() + limits['statement'] + with _request_timing.budget_scope(deadline, PostgresOperationTimeout): + # Preserve a caller's tighter timeout and never permit an unlimited enlisted write. + for name, seconds in (('statement_timeout', limits['statement']), + ('idle_in_transaction_session_timeout', limits['idle']), + ('lock_timeout', _lock_timeout_s())): + con.execute("SELECT set_config(%s, LEAST(COALESCE(NULLIF(" + "EXTRACT(epoch FROM current_setting(%s)::interval) * 1000, 0), %s), %s)" + "::bigint::text || 'ms', true)", + (name, name, int(seconds * 1000), int(seconds * 1000))) + token = _CALLER_TRANSACTION.set(_EnlistedPool(con, slug)) + try: + yield + finally: + _CALLER_TRANSACTION.reset(token) + _request_timing.clear_read_memo() def _url(): @@ -79,7 +137,11 @@ def _effective_slug(tenant_slug=None): data — and the only code that knew had already turned it into an identifier. Resolving once, here, is what lets `write_refusal` below refuse the DEFAULT rather than only an explicit slug. """ - return (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower() + slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower() + enlisted = _CALLER_TRANSACTION.get() + if enlisted is not None and enlisted.slug != slug: + raise ValueError('The enlisted transaction belongs to another tenant.') + return slug def _schema_for(tenant_slug=None): @@ -526,6 +588,9 @@ def _pool(): psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency to be installed at all, which is what keeps this file safe to commit before B-3 unblocks. """ + enlisted = _CALLER_TRANSACTION.get() + if enlisted is not None: + return enlisted with _LOCK: url = _url() if not url: @@ -777,7 +842,7 @@ def _request_key(name, shape, tenant_slug, pool=None): @_request_timing.memo_read def _request_text(name, shape, tenant_slug, read, *, fresh=False): """Share only this request's validated backing read, including independently acquired handles.""" - if _request_timing.CURRENT.get() is None: + if _CALLER_TRANSACTION.get() is not None or _request_timing.CURRENT.get() is None: return read() return _request_timing.memo_text(_request_key(name, shape, tenant_slug), read, fresh=fresh) @@ -795,6 +860,8 @@ def _seed_read_token(key, checkpoint, revision, stamp): def _validated_content(name, drops, tenant_slug): """Reuse retained content only against this request's checked revision AND timestamp.""" + if _CALLER_TRANSACTION.get() is not None: + return False, None checkpoint = _request_timing.memo_checkpoint() if checkpoint is None: return False, None @@ -824,6 +891,8 @@ def get(name, fresh=False, tenant_slug=None): Row mode: a `fresh` read assembles the whole document now. Otherwise the answer is a detached `RowsDocument` that reads one table per first access and the rest in one statement. """ + if str(name).endswith('__shared'): + return _shared_document(name, tenant_slug) if _ut_rows(name, tenant_slug): if fresh: return _utr.read_document(name, tenant_slug, fresh=True) @@ -908,6 +977,8 @@ def _forget_whole(key): def _remember_whole(key, generation, revision, stamp, text): """Retain bounded immutable text only while its local write generation remains current.""" global _WHOLE_TEXT_BYTES + if _CALLER_TRANSACTION.get() is not None: + return _request_timing.remaining_budget() size = sys.getsizeof(text) if isinstance(text, str) else 0 with _READ_FLIGHT_LOCK: @@ -1146,6 +1217,8 @@ _PROJECTION_TEXT_LIMIT = 16 * 1024 * 1024 def _remember_projection(key, revision, stamp, text): """Retain bounded immutable content whose revision must pass the next database read.""" global _PROJECTION_TEXT_BYTES + if _CALLER_TRANSACTION.get() is not None: + return _request_timing.remaining_budget() size = len(text.encode('utf-8')) if isinstance(text, str) else 0 with _PROJECTION_TEXT_LOCK: @@ -1165,6 +1238,8 @@ def _remember_projection(key, revision, stamp, text): def get_projection(name, drop=(), tenant_slug=None): """Parse independent projected definitions from the current request's backing read.""" import core.store as _hf + if str(name).endswith('__shared'): + return _hf._project(_shared_document(name, tenant_slug), frozenset(map(str, drop))) if _ut_rows(name, tenant_slug): return _rows_projection(name, drop, tenant_slug) drops = tuple(sorted(set(str(d) for d in drop))) @@ -1368,6 +1443,11 @@ def _delete_document(name, *, tenant_slug=None): if rows: # Every table goes with the document; the empty row tables stay for a later migration. _utr.clear_rows(con, tenant_slug) + if str(name).endswith('__shared'): + _shared_lock(con, name, tenant_slug, exclusive=True) + if _shared_exists(con, tenant_slug): + con.execute(_q('DELETE FROM {tbl} WHERE bucket = %s', + 'shared_overlay_cells', tenant_slug), (str(name),)) con.execute(_q('DELETE FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), (str(name),)) if rows: _utr.remember_mode(name, tenant_slug, value=False) @@ -1396,6 +1476,11 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name))) if not keys: return {} + shared = [name for name in keys if name.endswith('__shared')] + if shared: + ordinary = [name for name in keys if name not in shared] + return {**_get_projections(ordinary, drop, tenant_slug, timeout_s), + **{name: get_projection(name, drop, tenant_slug) for name in shared}} drops = tuple(str(d) for d in drop) rows_keys = {key for key in keys if _ut_rows(key, tenant_slug)} pool = _pool() @@ -1465,6 +1550,8 @@ def _get_entry(name, entry, tenant_slug=None): `user_tables` document (~60-100 MB on staging, ~3 s per read, holding a pool connection). In row mode it reads one definition row; `rows` and the row maps load records on first touch. """ + if str(name).endswith('__shared') and str(entry) == 'cells': + return _shared_document(name, tenant_slug)['cells'] if _ut_rows(name, tenant_slug): value = _utr.lazy_table(name, tenant_slug, str(entry)) return dict(value) if isinstance(value, dict) else None @@ -1783,6 +1870,9 @@ def _get_many(names, tenant_slug=None): found = {} for key, text, rev, stamp in rows: found[str(key)] = _parsed(text) + if str(key).endswith('__shared'): + found[str(key)] = _shared_document(str(key), tenant_slug) + continue _request_timing.record_store('get_many', str(key), bytes_read=_request_timing.text_bytes(text), count=0) _seed_read_token(request_keys[str(key)], checkpoint, rev, stamp) if _ut_marker(key, tenant_slug, found[str(key)]): @@ -1896,6 +1986,8 @@ def put(name, data, tenant_slug=None): # pool stubbed to raise, "refused" and "reached the database" are two distinguishable answers, # which is what makes the negative control in `verify_store_pg.py` bite without a server. check_write(tenant_slug, f'put {name!r}') + if str(name).endswith('__shared'): + return _shared_update(name, lambda _old: data, tenant_slug) if _ut_rows(name, tenant_slug): _utr.settle_lazy(name, tenant_slug) try: @@ -1959,6 +2051,8 @@ def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row= what it loses is locality, not rows. """ check_write(tenant_slug, f'update {name!r}') # D-353 — see `put`, and before `_pool()` + if str(name).endswith('__shared'): + return _shared_update(name, fn, tenant_slug, expect_rev=expect_rev, audit_row=audit_row) try: if _ut_rows(name, tenant_slug): _utr.settle_lazy(name, tenant_slug) @@ -2094,8 +2188,8 @@ def _update_rows(name, fn, tenant_slug, expect_rev, audit_row, *, table_mode=Fal Record mode locks one record row per `rows[rid]` access under a shared definition lock, so writers on different records of one table do not wait on each other. A change record mode - cannot write (a definition change, a full-table access) reruns `fn` once with those tables - in table mode, then once with every table in table mode. A lock or SQL error inside a lazy + cannot write reruns `fn` under an exclusive definition lock or in table mode as needed. + Repeated escalation falls back to table mode. A lock or SQL error inside a lazy load is re-raised after `fn` returns, even when `fn` swallowed it. """ from psycopg.errors import DeadlockDetected, LockNotAvailable @@ -2105,14 +2199,18 @@ def _update_rows(name, fn, tenant_slug, expect_rev, audit_row, *, table_mode=Fal from core.view_audit import ViewRevisionStale raise ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) modes, plan, deadlocks, escalations = (True if table_mode else set()), {}, 0, 0 + definition_mode = set() while True: try: - doc, result = _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan) + doc, result = _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan, + definition_mode=definition_mode) break except _utr.Escalate as esc: escalations += 1 plan = esc.plan - modes = True if (esc.all or escalations >= 2 or modes is True) else modes | set(esc.tables) + definitions = {key for key, reason in esc.tables.items() if reason == 'definition changed'} + definition_mode.update(definitions) + modes = True if (esc.all or escalations >= 3 or modes is True) else modes | (set(esc.tables) - definitions) except DeadlockDetected: # Two writers crossed; Postgres aborted one. Rerun `fn` on fresh state. deadlocks += 1 @@ -2124,7 +2222,8 @@ def _update_rows(name, fn, tenant_slug, expect_rev, audit_row, *, table_mode=Fal f'{_effective_slug(tenant_slug)}/{name} (rows); holder: ' f'{exc.aios_holder["label"]}', flush=True) raise - _utr.carry_memo(name, tenant_slug, *getattr(doc, 'committed', (None, None))) + if _CALLER_TRANSACTION.get() is None: + _utr.carry_memo(name, tenant_slug, *getattr(doc, 'committed', (None, None))) if result is not None and result is not doc: return result doc.detach(lambda key: _utr.read_table_split(tenant_slug, key, name), @@ -2133,7 +2232,7 @@ def _update_rows(name, fn, tenant_slug, expect_rev, audit_row, *, table_mode=Fal return doc -def _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan): +def _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan, *, definition_mode=()): """One row-mode write attempt. Raises `Escalate` (rolled back) when table mode is needed.""" from psycopg.errors import LockNotAvailable from core import user_tables as _ut @@ -2147,17 +2246,22 @@ def _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan) + ', '.join(f'{k} ({v / 1048576:.1f} MB)' for k, v in sorted(report['evicted'].items()))) - with _pool().autocommit() as con: + enlisted = _CALLER_TRANSACTION.get() + with _pool().autocommit() as con, (con.transaction() if enlisted is not None else nullcontext()): # BEGIN, the limits, lock_timeout, the backend pid and the table keys: one round trip. with con.pipeline(): - con.execute('BEGIN') + if enlisted is None: + con.execute('BEGIN') head = con.execute(_SETTINGS_SQL + ", set_config('lock_timeout', %s, true), pg_backend_pid()", _settings_params() + (f'{int(wait_s * 1000)}ms',)) listed = _utr.queue_table_keys(con, tenant_slug) pid_row = head.fetchone()[3:] access = _utr.RecordAccess(con, tenant_slug) - doc = _utr.RowsDocument(_utr.ordered_keys(r[0] for r in listed.fetchall()), access.table, - access=access, table_mode=modes) + keys = _utr.ordered_keys(r[0] for r in listed.fetchall()) + _request_timing.record_store('ut_keys', str(name), + bytes_read=sum(_request_timing.text_bytes(key) for key in keys)) + doc = _utr.RowsDocument(keys, access.table, + access=access, table_mode=modes, definition_mode=definition_mode) try: if plan: doc.lock_records(plan) @@ -2192,7 +2296,7 @@ def _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan) 'view.' + str(entry.get('action') or ''), f"{entry.get('table_key') or ''}/{entry.get('view_id') or ''}", json.dumps(entry))) - if expect_rev is None: + if expect_rev is None and enlisted is None: con.execute('COMMIT') except LockNotAvailable: # A record write that waited has no result; past every write, the wait was the marker. @@ -2204,7 +2308,7 @@ def _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan) from core.view_audit import ViewRevisionStale raise ViewRevisionStale(rev_current=_utr.marker_rev(con, tenant_slug, name), rev_sent=int(expect_rev)) - if expect_rev is not None: + if expect_rev is not None and enlisted is None: con.execute('COMMIT') finish() doc.rows_written = n @@ -2215,8 +2319,10 @@ def _rows_transaction(name, fn, tenant_slug, expect_rev, audit_row, modes, plan) exc._aios_lock_target = access.target raise _request_timing.record_store('update', str(name), count=0, - bytes_written=sum(len(v or '') + len(m or '') - for _k, _r, v, m in changes['records'])) + bytes_written=sum(_request_timing.text_bytes(v) + for v in changes['defs'].values()) + + sum(_request_timing.text_bytes(v) + _request_timing.text_bytes(m) + for _k, _r, v, m in changes['records'])) if doc.whole_read: _request_timing.record_store('ut_whole', str(name), count=1) return doc, result @@ -2308,6 +2414,9 @@ def delete_path(path_in_repo, tenant_slug=None): def revision(name, tenant_slug=None): """Return an independent revision object, reusing only this request's backing validation.""" + if str(name).endswith('__shared'): + with _pool().connection() as con: + return _shared_revision(con, name, tenant_slug) held = _kv_revision(name, tenant_slug) if held is not None: return held @@ -2319,6 +2428,10 @@ def revision(name, tenant_slug=None): def _revision_many(names, tenant_slug=None): """Batch only unchecked keys and lend complete tokens to every request read shape.""" keys = tuple(sorted(set(map(str, names)))) + shared = [name for name in keys if name.endswith('__shared')] + if shared: + return {**_revision_many([name for name in keys if name not in shared], tenant_slug), + **{name: revision(name, tenant_slug) for name in shared}} checkpoint = _request_timing.memo_checkpoint() request_keys = {name: _request_key(name, 'revision', tenant_slug) for name in keys} found, missing = {}, [] @@ -2420,6 +2533,190 @@ def close(): # for the last request. That is the same property `TenantRuntime.store_handle` already gives the # HF backend (EXIT-4b proof #3), reached a different way. # --------------------------------------------------------------------------------------------- +_SHARED_LAYOUT = '_shared_cell_layout' +_SHARED_VERSION = '_shared_cell_version' + + +def _shared_exists(con, tenant_slug): + return con.execute('SELECT to_regclass(%s)', + (_schema_for(tenant_slug) + '.shared_overlay_cells',)).fetchone()[0] is not None + + +def _shared_load(con, name, tenant_slug, pids=None): + """Read definitions and scoped cells from one PostgreSQL statement snapshot.""" + if not _shared_exists(con, tenant_slug): + row = con.execute(_q('SELECT value::text FROM {tbl} WHERE key = %s', + 'store_kv', tenant_slug), (str(name),)).fetchone() + data = _parsed(row[0] if row else None) + else: + from psycopg import sql + # _q owns tenant identifiers; this additional table has the same binding. + statement = sql.SQL('SELECT kv.value::text, (SELECT jsonb_agg(jsonb_build_array(' + 'row_id, field_key, value)) FROM {} WHERE bucket = %s ' + 'AND (%s::text[] IS NULL OR row_id = ANY(%s::text[])))::text ' + 'FROM {} kv WHERE kv.key = %s').format( + _table('shared_overlay_cells', tenant_slug), _table('store_kv', tenant_slug)) + wanted = None if pids is None else [str(pid) for pid in pids] + row = con.execute(statement, (str(name), wanted, wanted, str(name))).fetchone() + data = _parsed(row[0] if row else None) + if data.get(_SHARED_LAYOUT) == 1: + data['cells'] = {} + for rid, field, value in json.loads(row[1] or '[]'): + data['cells'].setdefault(rid, {})[field] = value + if not row: + return {} + data.pop(_SHARED_LAYOUT, None) + data.pop(_SHARED_VERSION, None) + data.setdefault('fields', {}) + data.setdefault('cells', {}) + if pids is not None: + wanted = {str(pid) for pid in pids} + data['cells'] = {rid: values for rid, values in data['cells'].items() if rid in wanted} + return data + + +def _shared_document(name, tenant_slug, pids=None): + pids = None if pids is None else tuple(str(pid) for pid in pids) + with _pool().connection() as con: + return _shared_load(con, name, tenant_slug, pids) + + +def _shared_lock(con, name, tenant_slug, *, exclusive): + lock = 'pg_advisory_xact_lock' if exclusive else 'pg_advisory_xact_lock_shared' + con.execute(f'SELECT {lock}(hashtextextended(%s, 0))', + (_schema_for(tenant_slug) + '.shared.' + str(name),)) + + +def _shared_ensure(con, tenant_slug): + """Create cell storage before taking any bucket lock.""" + if not _shared_exists(con, tenant_slug): + con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', + (_schema_for(tenant_slug) + '.shared_overlay_cells.ddl',)) + if not _shared_exists(con, tenant_slug): + con.execute(_q('CREATE TABLE {tbl} (bucket text NOT NULL, row_id text NOT NULL, ' + 'field_key text NOT NULL, value jsonb NOT NULL, ' + 'version bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 1000000000000), ' + 'updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), ' + 'PRIMARY KEY (bucket, row_id, field_key))', 'shared_overlay_cells', tenant_slug)) + con.execute(_q('CREATE INDEX shared_overlay_cells_version ON {tbl} (bucket, version DESC)', + 'shared_overlay_cells', tenant_slug)) + + +def _shared_prepare(con, name, tenant_slug): + """Move legacy cells once, under a guarded transaction, preserving authored metadata.""" + _shared_ensure(con, tenant_slug) + row = con.execute(_q('SELECT value ->> %s FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), + (_SHARED_LAYOUT, str(name))).fetchone() + if row and row[0] == '1': + return + _shared_lock(con, name, tenant_slug, exclusive=True) + row = con.execute(_q('SELECT value::text FROM {tbl} WHERE key = %s FOR UPDATE', + 'store_kv', tenant_slug), (str(name),)).fetchone() + data = _parsed(row[0] if row else None) + if data.get(_SHARED_LAYOUT) == 1: + return + for rid, values in (data.pop('cells', {}) or {}).items(): + for key, value in (values or {}).items(): + _shared_cell(con, name, rid, key, value, tenant_slug) + data.setdefault('fields', {}) + data[_SHARED_LAYOUT] = 1 + data[_SHARED_VERSION] = _shared_next_version(con, tenant_slug) + con.execute(_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ON CONFLICT (key) ' + 'DO UPDATE SET value = EXCLUDED.value, rev = {bare}.rev + 1, updated_at = now()', + 'store_kv', tenant_slug), (str(name), json.dumps(data))) + + +def _shared_cell(con, name, rid, key, value, tenant_slug): + con.execute(_q('INSERT INTO {tbl} (bucket, row_id, field_key, value) VALUES (%s, %s, %s, %s::jsonb) ' + 'ON CONFLICT (bucket, row_id, field_key) DO UPDATE SET value = EXCLUDED.value, ' + 'version = nextval(pg_get_serial_sequence(%s, %s)), updated_at = clock_timestamp()', + 'shared_overlay_cells', tenant_slug), + (str(name), str(rid), str(key), json.dumps(value), + _schema_for(tenant_slug) + '.shared_overlay_cells', 'version')) + + +def _shared_next_version(con, tenant_slug): + return con.execute('SELECT nextval(pg_get_serial_sequence(%s, %s))', + (_schema_for(tenant_slug) + '.shared_overlay_cells', 'version')).fetchone()[0] + + +def _shared_revision(con, name, tenant_slug): + from psycopg import sql + if not _shared_exists(con, tenant_slug): + row = con.execute(_q('SELECT rev, updated_at FROM {tbl} WHERE key = %s', + 'store_kv', tenant_slug), (str(name),)).fetchone() + else: + row = con.execute(sql.SQL('SELECT GREATEST(kv.rev, (kv.value ->> %s)::bigint, cell.version), ' + 'GREATEST(kv.updated_at, cell.updated_at) FROM {} kv ' + 'LEFT JOIN LATERAL (SELECT version, updated_at FROM {} ' + 'WHERE bucket = kv.key ORDER BY version DESC LIMIT 1) cell ON true ' + 'WHERE kv.key = %s').format( + _table('store_kv', tenant_slug), + _table('shared_overlay_cells', tenant_slug)), + (_SHARED_VERSION, str(name))).fetchone() + return _revision_value(row[0] if row else 0, row[1] if row else None) + + +def _shared_patch(name, rows, validate, tenant_slug): + """Write addressed cells without acquiring or updating the whole bucket row.""" + check_write(tenant_slug, 'patch shared cells') + try: + with _pool().connection() as con, con.transaction(): + if _CALLER_TRANSACTION.get() is None: + con.execute("SELECT set_config('lock_timeout', %s, true)", + (f'{int(_lock_timeout_s() * 1000)}ms',)) + _shared_prepare(con, name, tenant_slug) + _shared_lock(con, name, tenant_slug, exclusive=False) + row = con.execute(_q('SELECT value::text FROM {tbl} WHERE key = %s', + 'store_kv', tenant_slug), (str(name),)).fetchone() + validate(_parsed(row[0] if row else None)) + for rid in sorted(rows): + for key in sorted(rows[rid]): + _shared_cell(con, name, rid, key, rows[rid][key], tenant_slug) + finally: + _bump_write_generation(name, tenant_slug) + return rows + + +def _shared_update(name, fn, tenant_slug, *, expect_rev=None, audit_row=None): + """Keep definition, deletion and legacy document writers coherent with independent cells.""" + check_write(tenant_slug, 'update shared definitions') + try: + with _pool().connection() as con, con.transaction(): + if _CALLER_TRANSACTION.get() is None: + con.execute("SELECT set_config('lock_timeout', %s, true)", + (f'{int(_lock_timeout_s() * 1000)}ms',)) + _shared_ensure(con, tenant_slug) + _shared_lock(con, name, tenant_slug, exclusive=True) + if expect_rev is not None: + current = _shared_revision(con, name, tenant_slug)['rev'] + if current != int(expect_rev): + from core.view_audit import ViewRevisionStale + raise ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) + _shared_prepare(con, name, tenant_slug) + data = _shared_load(con, name, tenant_slug) + old = {rid: dict(values) for rid, values in data['cells'].items()} + result = fn(data) + data = result if result is not None else data + new = data.get('cells') or {} + for rid in sorted(old.keys() | new.keys()): + left, right = old.get(rid, {}), new.get(rid, {}) + for key in sorted(left.keys() | right.keys()): + if key not in right: + con.execute(_q('DELETE FROM {tbl} WHERE bucket = %s AND row_id = %s AND field_key = %s', + 'shared_overlay_cells', tenant_slug), (str(name), rid, key)) + elif key not in left or left[key] != right[key]: + _shared_cell(con, name, rid, key, right[key], tenant_slug) + metadata = {key: value for key, value in data.items() if key != 'cells'} + metadata[_SHARED_LAYOUT] = 1 + metadata[_SHARED_VERSION] = _shared_next_version(con, tenant_slug) + with _caller_transaction(con, tenant_slug): + _update_locked_body(name, lambda _old: metadata, tenant_slug, None, audit_row) + return data + finally: + _bump_write_generation(name, tenant_slug) + + class PgStore: """`core.store.Store`'s interface for ONE tenant's schema. @@ -2452,6 +2749,16 @@ class PgStore: def get(self, name, fresh=False): return get(name, fresh=fresh, tenant_slug=self.tenant_slug) + def transaction(self, connection): + """Join this tenant's writes to the caller's active receipt connection.""" + return _caller_transaction(connection, self.tenant_slug) + + def shared_document(self, name, pids=None): + return _shared_document(name, self.tenant_slug, pids) + + def shared_patch(self, name, rows, validate): + return _shared_patch(name, rows, validate, self.tenant_slug) + def get_projection(self, name, drop=()): """`Store.get_projection`'s contract on Postgres — same ANSWER, different economics. diff --git a/platform/core/user_table_rows.py b/platform/core/user_table_rows.py index 9081fc4a982bb653675e82de6d32cfec56378589..475c8571381c5d53bfbf57d80f09a96d2a2a0312 100644 --- a/platform/core/user_table_rows.py +++ b/platform/core/user_table_rows.py @@ -23,7 +23,8 @@ DOCUMENT_KEY = 'user_tables' DEFS = 'user_table_defs' RECORDS = 'user_table_records' #: Definition keys whose value is `{row id: entry}`. Each entry lives in `meta` of that record row. -ROW_MAPS = ('_rowAudit', '_rowCreationOrder', 'ticketSourceRows', '_activity_outbox') +CELL_NOTES_KEY = '__cell_notes__' +ROW_MAPS = ('_rowAudit', '_rowCreationOrder', 'ticketSourceRows', '_activity_outbox', CELL_NOTES_KEY) LAYOUT_KEY = '__layout__' MAPS_KEY = '__rowMaps__' NO_ROWS_KEY = '__noRows__' @@ -399,7 +400,7 @@ class _TableState: return self.doc._any_record(self.key, kind, list(self.orig)) def changes(self, entry, out): - """Append this table's record writes to `out`; return a reason when it needs table mode.""" + """Append record writes to `out` or request a stronger definition or table lock.""" if entry is not self.entry: return 'table replaced' for kind, view in self.views.items(): @@ -409,24 +410,32 @@ class _TableState: return f'{kind} removed' if entry[kind] is not view: return f'{kind} replaced' - definition = {} + definition, added_maps = {}, {} for key, value in entry.items(): key = _json_key(key) if key in self.views and value is self.views[key]: continue if key in ROW_MAPS and isinstance(value, dict): - return f'{key} added' + added_maps[key] = value + continue definition[key] = value for key in INTERNAL_KEYS: if key in self.definition: definition[key] = self.definition[key] + if added_maps: + definition[MAPS_KEY] = ordered_keys(set(self.maps) | set(added_maps)) if _canon(definition) != self.def_canon: - return 'definition changed' + if self.key not in self.doc._definition_mode: + return 'definition changed' + out['defs'][self.key] = _dump(definition) + for values in added_maps.values(): + self.load([_json_key(rid) for rid in values]) rows = self.views.get('rows') for rid in ordered_keys(self.orig): value = dict.get(rows, rid, ABSENT) if rows is not None else ABSENT meta = {m: dict.__getitem__(self.views[m], rid) for m in self.maps if dict.__contains__(self.views[m], rid)} + meta.update({m: values[rid] for m, values in added_maps.items() if rid in values}) pair = _record_json([value, meta or None]) if pair != (self.orig[rid] or (None, None)): out['records'].append((self.key, rid) + pair) @@ -449,7 +458,7 @@ class RowsDocument(dict): """ lazy_tables = True - def __init__(self, table_keys, loader, *, access=None, table_mode=()): + def __init__(self, table_keys, loader, *, access=None, table_mode=(), definition_mode=()): super().__init__() self._keys = dict.fromkeys(str(k) for k in table_keys) self._listed = frozenset(self._keys) @@ -460,6 +469,7 @@ class RowsDocument(dict): self.loads = [] self._access = access self._table_mode = table_mode if table_mode is True else set(table_mode) + self._definition_mode = set(definition_mode) self._states = {} self.escalated = {} self.escalate_all = None @@ -530,6 +540,45 @@ class RowsDocument(dict): if state is not None and plan.get(key): state.load(ordered_keys(_json_key(r) for r in plan[key])) + def lock_record_metadata(self, table_key, row_id, *, default=None): + """Read the current definition and metadata after locking an identity, including absent tables.""" + key, rid = str(table_key), str(row_id) + if not self._live() or key in self._table_mode: + if self._access is not None and not self._detached: + self._keys.setdefault(key, None) + self._load(key) + return + try: + self._access.lock_metadata_record(key, rid) + definition = self._access.definition(key, exclusive=key in self._definition_mode) + missing = definition is None + if missing and default is not None: + definition, _records = split_table(default) + if definition is None: + self._keys.pop(key, None) + self._fill() + return + if not isinstance(definition, dict): + self._orig[key] = (_canon(definition), {}) + self._keys[key] = None + dict.__setitem__(self, key, definition) + self._fill() + return + # The transaction's key list and any preloaded maps can predate the identity lock. + state = _TableState(self, key, definition) + if missing: + state.def_canon = _canon(None) + self._states[key] = state + self._orig[key] = None + self._keys[key] = None + self.loads.append(key) + dict.__setitem__(self, key, state.entry) + self._fill() + state.load((rid,)) + except Exception as exc: + self._fail(exc) + raise + # -- storage ------------------------------------------------------------------------------ def _is_loaded(self, key): return dict.__contains__(self, key) and dict.__getitem__(self, key) is not _PENDING @@ -557,7 +606,11 @@ class RowsDocument(dict): self._load_all() return self._is_loaded(key) try: - split = self._access.definition(key) if record else self._loader(key) + if record: + split = (self._access.definition(key, exclusive=True) + if key in self._definition_mode else self._access.definition(key)) + else: + split = self._loader(key) except Exception as exc: self._fail(exc) raise @@ -1110,6 +1163,10 @@ def rows_mode(name, tenant_slug=None): return False pg = _pg() pool = pg._pool() + if pg._CALLER_TRANSACTION.get() is not None: + row = pg._read(pool, _sql('SELECT value ->> %s FROM {kv} WHERE key = %s', tenant_slug), + (LAYOUT_KEY, str(name)), one=True) + return bool(row and row[0] == 'rows') # A pool this module did not build is a gate's stub: stubs stay in document mode. if type(pool) not in tuple(pg._HOLDER_POOL_CLASS.values()): return False @@ -1169,7 +1226,8 @@ def _split_from_rows(raw): def _raw_bytes(raw): if raw is None: return 0 - return len(raw[0] or '') + sum(len(v or '') + len(m or '') for _rid, v, m in raw[1]) + return _request_timing.text_bytes(raw[0]) + sum( + _request_timing.text_bytes(v) + _request_timing.text_bytes(m) for _rid, v, m in raw[1]) def load_locked(con, tenant_slug, table_key): @@ -1177,6 +1235,7 @@ def load_locked(con, tenant_slug, table_key): con.execute(_sql('SELECT 1 FROM {defs} WHERE table_key = %s FOR UPDATE', tenant_slug), (str(table_key),)) raw = _table_rows(con, tenant_slug, table_key) + _request_timing.record_store('ut_table', DOCUMENT_KEY, bytes_read=_raw_bytes(raw)) return None if raw is None else _split_from_rows(raw) @@ -1234,16 +1293,18 @@ class RecordAccess: self._run(('advisory', str(table_key), None), f'SELECT {call}(hashtextextended(%s, 0))', (table_lock_name(self.tenant_slug, table_key),)) - def definition(self, table_key): - """The definition row `FOR SHARE` (record writers share it), or None when absent.""" + def definition(self, table_key, exclusive=False): + """Read the definition under a shared lock, or an exclusive lock for definition changes.""" lock = read = None self.target = ('advisory', str(table_key), None) try: # The advisory lock, then the row lock, in one round trip. + advisory = 'pg_advisory_xact_lock' if exclusive else 'pg_advisory_xact_lock_shared' + row_lock = 'UPDATE' if exclusive else 'SHARE' with self.con.pipeline(): - lock = self.con.execute(_sql('SELECT pg_advisory_xact_lock_shared(hashtextextended(%s, 0))', + lock = self.con.execute(_sql(f'SELECT {advisory}(hashtextextended(%s, 0))', self.tenant_slug), (table_lock_name(self.tenant_slug, table_key),)) - read = self.con.execute(_sql('SELECT value::text FROM {defs} WHERE table_key = %s FOR SHARE', + read = self.con.execute(_sql('SELECT value::text FROM {defs} WHERE table_key = %s FOR ' + row_lock, self.tenant_slug), (str(table_key),)) except Exception: if lock is not None and lock.pgresult is not None: @@ -1251,6 +1312,8 @@ class RecordAccess: raise self.target = None rows = read.fetchall() + _request_timing.record_store('ut_def', DOCUMENT_KEY, + bytes_read=sum(_request_timing.text_bytes(r[0]) for r in rows)) return _parse(rows[0][0]) if rows else None def table(self, table_key): @@ -1269,15 +1332,32 @@ class RecordAccess: rids = [str(r) for r in rids] rows = self._run((RECORDS, str(table_key), rids[0] if rids else None), _RECORDS_SQL + ' FOR UPDATE', (str(table_key), rids)) + _request_timing.record_store('ut_records', DOCUMENT_KEY, + bytes_read=sum(_request_timing.text_bytes(v) + _request_timing.text_bytes(m) + for _rid, v, m in rows)) return _records_from(rows) + def lock_metadata_record(self, table_key, row_id): + """Lock a metadata identity even when no record row exists for SELECT FOR UPDATE.""" + # Definition retries take the table lock first, so metadata writers use the same order. + self._table_lock(table_key, shared=True) + identity = json.dumps([table_lock_name(self.tenant_slug, table_key), str(row_id), CELL_NOTES_KEY]) + self._run((RECORDS, str(table_key), str(row_id)), + 'SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (identity,)) + def any_record(self, table_key, kind, exclude): params = (str(table_key), [str(r) for r in exclude]) + (() if kind == 'rows' else (kind,)) return bool(self._run(None, _any_sql(kind), params)[0][0]) def document(self): """The whole document in one unlocked statement (a record-mode attempt that escalates).""" - return _assemble_rows(self._run(None, _DOCUMENT_SQL, ())) + rows = self._run(None, _DOCUMENT_SQL, ()) + # The completed transaction counts the whole read after any table-mode retry. + _request_timing.record_store('ut_whole', DOCUMENT_KEY, + count=0, + bytes_read=sum(_request_timing.text_bytes(v) + _request_timing.text_bytes(m) + for _kind, _key, _rid, v, m in rows)) + return _assemble_rows(rows) class DetachedRecords: @@ -1627,6 +1707,8 @@ def _table_memo(): text, `('ut_defs',)` every `(key, text)`, `('ut_keys',)` the table keys, `('ut_maps', key, wanted)` the `(rid, meta)` texts of one table's records that hold a wanted row map. """ + if _pg()._CALLER_TRANSACTION.get() is not None: + return None, None acc = _request_timing.CURRENT.get() if acc is None or acc.closed or acc.cancelled: return None, None @@ -2202,12 +2284,16 @@ def queue_changes(con, tenant_slug, changes, written=None): written['records'][(key, rid)] = None def finish(): + returned_bytes = 0 for kind, key, cur in reads: if kind == 'def': written['defs'][key] = cur.fetchone()[0] + returned_bytes += _request_timing.text_bytes(written['defs'][key]) else: for table_key, rid, value, meta in cur.fetchall(): written['records'][(table_key, rid)] = (value, meta) + returned_bytes += _request_timing.text_bytes(value) + _request_timing.text_bytes(meta) + _record_bytes('update', DOCUMENT_KEY, returned_bytes) finish.cursors = cursors return n, finish @@ -2494,7 +2580,8 @@ def verify(tenant_slug, name=DOCUMENT_KEY, statement_timeout_s=STATEMENT_TIMEOUT started = time.monotonic() with _pg()._pool().connection() as con: _settings(con, statement_timeout_s, read_only=True) - row = con.execute(_sql('SELECT CASE WHEN pg_column_size(value) < 8192 THEN value::text END, rev ' + row = con.execute(_sql("SELECT jsonb_build_object('__layout__', value -> '__layout__', " + "'source', value -> 'source')::text, rev " 'FROM {kv} WHERE key = %s', tenant_slug), (str(name),)).fetchone() stamp = _parse(row[0]) if row and row[0] else None rows = is_marker(stamp) @@ -2505,6 +2592,15 @@ def verify(tenant_slug, name=DOCUMENT_KEY, statement_timeout_s=STATEMENT_TIMEOUT 'SELECT count(*) FILTER (WHERE NOT EXISTS (SELECT 1 FROM {defs} AS d ' 'WHERE d.table_key = r.table_key)), count(*) FILTER (WHERE r.value IS NULL ' 'AND r.meta IS NULL) FROM {recs} AS r', tenant_slug)).fetchone() + embedded_defs = con.execute(_sql( + "SELECT count(*) FROM {defs} AS d WHERE jsonb_typeof(d.value -> 'rows') = 'object' " + 'OR EXISTS (SELECT 1 FROM unnest(%s::text[]) AS m ' + "WHERE jsonb_typeof(d.value -> m) = 'object')", tenant_slug), + (list(ROW_MAPS),)).fetchone()[0] + embedded_document = con.execute(_sql( + 'SELECT count(*) FROM {kv} AS kv CROSS JOIN LATERAL jsonb_each(kv.value) AS e ' + "WHERE kv.key = %s AND jsonb_typeof(e.value -> 'rows') = 'object'", tenant_slug), + (str(name),)).fetchone()[0] report = compare_digests(digest, digest) out = {'mode': 'rows' if rows else ('document' if row else 'absent'), 'tables': report['tables'], 'records': report['records'], 'bytes': report['bytes'], 'md5': report['md5'], 'ok': True, @@ -2514,8 +2610,10 @@ def verify(tenant_slug, name=DOCUMENT_KEY, statement_timeout_s=STATEMENT_TIMEOUT writes = int(row[1]) - int(source['rev']) if source.get('rev') is not None else None matches = (source.get('md5') == report['md5']) if source.get('md5') else None out.update(source=source, writes_since_migration=writes, matches_source=matches, - orphan_records=int(orphans), empty_rows=int(empty)) - out['ok'] = orphans == 0 and empty == 0 and not (matches is False and writes == 0) + orphan_records=int(orphans), empty_rows=int(empty), + embedded_definitions=int(embedded_defs), embedded_document_tables=int(embedded_document)) + out['ok'] = (orphans == 0 and empty == 0 and embedded_defs == 0 and embedded_document == 0 + and not (matches is False and writes == 0)) return out diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 884a844bf1e522d66e62949036da39ff262f592f..7afadc42384310568254dfee6ee18d5bec6ac1d2 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -57,6 +57,7 @@ from contextvars import ContextVar import core.store as store from core.route_system import MAX_OPTIONS +from core.user_table_rows import CELL_NOTES_KEY, ROW_MAPS STORE_KEY = 'user_tables' ZONE_STORE_KEY = 'table_zones' @@ -1295,7 +1296,7 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, result = {**result, 'ut_activity': activity} return result - def mutate(current): + def lock_current(current): lock = getattr(current, 'lock_records', None) if callable(lock): # Row mode: lock this write's records (or the table) first, in one sorted order. @@ -1303,6 +1304,10 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, if append is not None: plan.setdefault('ut_activity', []) lock(plan) + + def mutate(current, *, locked=False): + if not locked: + lock_current(current) prior_table = (current or {}).get(str(table_key)) or {} selected = prior_table.get('rows') if selected is None: @@ -1357,8 +1362,9 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, def apply(current): # Stores may retry a callback. Only the final successful attempt supplies events. pending.clear() + lock_current(current) before = snapshot(current) - after_document = mutate(current) + after_document = mutate(current, locked=True) after = snapshot(after_document) for kind, old, new in zip(('record', 'field'), before, after): for identity in sorted(old.keys() | new.keys()): @@ -1384,11 +1390,13 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, 'occurred_at': activity_log._now().isoformat()} entry = dict(kind=kind, object={'database': str(table_key), 'id': identity}, by=by, via=via, changes=changes, _delivery=delivery) + if kind == 'record' and identity in old and identity in new: + delivery['_cell_edit'] = True # Validate before committing either the row or its durable delivery intent. activity_log._object(entry['object']) activity_log._text(by, 'by') activity_log._text(via, 'via', optional=True) - activity_log._bounded_changes(changes) + activity_log._bounded_changes(changes, cell_edit=delivery.get('_cell_edit', False)) pending[key] = entry if pending: queue = after_document.setdefault(activity_log.DATABASE_KEY, {}).setdefault( @@ -1399,9 +1407,11 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None, result = target.update(STORE_KEY, apply, flush=flush, **update_kwargs) if result is not False: - if batch is not None: + from core import store_pg + enlisted = store_pg._CALLER_TRANSACTION.get() is not None + if batch is not None and not enlisted: batch.update({key: (batch_owner, entry) for key, entry in pending.items()}) - elif pending: + elif pending and not enlisted: activity_log.drain(batch_owner, keys={entry['_delivery']['key'] for entry in pending.values()}) return public_result(result) @@ -1952,7 +1962,7 @@ def all_tables(st=None): #: The keys `all_defs` refuses to read. `rows` is 99.9% of tenant #0's document by bytes and is #: touched by no permission check, no nav render and no workspace envelope (D-185, W32-T01). -DEF_ONLY_DROPS = ('rows', '_activity_outbox', '_rowCreationOrder', '_rowAudit') +DEF_ONLY_DROPS = ('rows', '_activity_outbox', '_rowCreationOrder', '_rowAudit', CELL_NOTES_KEY) def definition_counts(st=None): @@ -4802,7 +4812,7 @@ def add_field(table_key, raw, st=None, recurrence_validator=None, *, by=None, vi raise ValueError('recurrence needs the server field-authority validator') if not is_user_table(table_key, st): return None - defn = get_one(table_key, st) or {} + defn = get_definition(table_key, st) or {} have = [f for f in (defn.get('fields') or [])] if len(have) >= MAX_FIELDS: return None @@ -4908,7 +4918,8 @@ def sync_reciprocal_link(table_key, field_key, st=None): if f.get('key') == field_key), None) # Remove every old inverse for this source identity first. This makes retargeting and # deleting deterministic instead of leaving a live-looking backlink on the old table. - for candidate in cur.values(): + for candidate_key in cur: + candidate = cur.get(candidate_key) if not isinstance(candidate, dict): continue candidate['fields'] = [f for f in (candidate.get('fields') or []) @@ -5138,7 +5149,8 @@ def delete_field(table_key, fkey, st=None, *, by=None, via=None): if f.get('key') != source_field] # An ordinary link owns its reciprocal field. Delete the reciprocal in the same schema # update so no target database can retain a live-looking backlink to a missing source. - for candidate in cur.values(): + for candidate_key in cur: + candidate = cur.get(candidate_key) if not isinstance(candidate, dict): continue candidate['fields'] = [f for f in (candidate.get('fields') or []) @@ -5222,17 +5234,159 @@ def rename_choice_values(table_key, fkey, renames, st=None, *, by=None, via=None return counts +class CellNoteRefused(ValueError): + """A named record, database or field wall refused a note write.""" + + def __init__(self, code, message): + super().__init__(message) + self.code = code + + +def _refuse_reserved_cell_keys(values): + """Scalar writers cannot replace reserved record metadata.""" + if any(str(key).startswith('__') or str(key) in ROW_MAPS for key in values or {}): + raise ValueError('Reserved record metadata requires its own write door.') + + +def cell_notes(table_key, row_ids, *, field_keys, st=None): + """Read notes for named records and explicitly readable fields, without reading source values.""" + ids, visible = tuple(str(rid) for rid in row_ids), frozenset(field_keys) + table = get_records(table_key, ids, st) or {} + mapped = table.get(CELL_NOTES_KEY) + if mapped is None: + return {} + out = {} + for rid in ids: + notes = mapped.get(rid) or {} + shown = {key: text for key, text in notes.items() + if key in visible and isinstance(text, str)} + if shown: + out[rid] = shown + return out + + +def patch_cell_notes(table_key, row_id, notes, ctx, *, op_id, source_row_exists=None): + """Merge a note delta using a trusted value-write context and the tenant's C1 receipts. + + Source adapters must verify the scoped source row without copying it into this store. + The context supplies authenticated identity, readable fields, hidden keys and allowed pids. + """ + from core import grid_events, edit_receipts + + key, rid = str(table_key), str(row_id) + scope = {'product_data': 'product', 'customer_data': 'customer'}.get(key, key) + if ctx.st is None or not ctx.uname or ctx.scope_key != scope: + raise CellNoteRefused('note_context_required', 'This database needs a tenant-bound edit context.') + identity = edit_receipts.op_id(op_id) + if not isinstance(notes, dict) or not notes: + raise CellNoteRefused('bad_cell_notes', 'Supply the changed field notes.') + delta = deepcopy(notes) + if any(not isinstance(field, str) or not field or field.startswith('_') + or (text is not None and not isinstance(text, str)) for field, text in delta.items()): + raise CellNoteRefused('bad_cell_notes', 'Each field note must be text or null.') + + def apply(): + permitted = (rid in ctx.allowed_pids + or (rid.isdigit() and int(rid) in ctx.allowed_pids)) + if not permitted: + raise CellNoteRefused('row_outside_audience', + f'This record is outside your access to database {key}.') + if key == 'ut_activity': + raise CellNoteRefused('database_read_only', 'The Activity database is read only.') + fields = {field['key']: field for field in ctx.fields if isinstance(field, dict) and field.get('key')} + saved = False + + def merge(current): + nonlocal saved + saved = False + lock = getattr(current, 'lock_record_metadata', None) + if callable(lock): + lock(key, rid, default={'readThrough': True, 'connected': True} + if key in LINKABLE_MODULE_KEYS else None) + table = current.get(key) + external = (key in LINKABLE_MODULE_KEYS + or (isinstance(table, dict) and _mirror_only_definition(key, table))) + if external: + if not callable(source_row_exists): + raise CellNoteRefused('note_source_adapter_required', + f'Database {key} needs its scoped source record reader.') + if source_row_exists(key, rid, ctx) is not True: + raise CellNoteRefused('row_outside_audience', + f'This record is outside your access to database {key}.') + if table is None: + table = current.setdefault(key, {'readThrough': True, 'connected': True}) + elif not isinstance(table, dict) or rid not in table.get('rows', {}): + raise CellNoteRefused('record_not_found', f'This record no longer exists in database {key}.') + + stored_fields = {field['key']: field for field in table.get('fields') or [] + if isinstance(field, dict) and field.get('key')} + for field_key in delta: + field = fields.get(field_key) + if not field or field_key in (ctx.hidden_keys or ()): + raise CellNoteRefused('field_read_only', + f'Field {field_key} is outside your edit access in database {key}.') + # Keep current stored restrictions and the adapter's effective field grants. + definition = {**field, **stored_fields.get(field_key, {})} + label = definition.get('label') or field_key + if (definition.get('source') not in (None, '', 'overlay') + and not grid_events._may_override_source_value(definition)): + raise CellNoteRefused('source_field_read_only', + f'Field {label} is a locked source column in database {key}.') + if (is_computed_cell(definition) or definition.get('type') in ('metric', 'lookup', 'created_time') + or definition.get('valueAuthority') == 'computed' or definition.get('readOnly') is True): + raise CellNoteRefused('calculated_field_read_only', f'Field {label} is a calculated read-only cell.') + automation = definition.get('automation') + if (definition.get('type') == 'automation' + or (isinstance(automation, dict) and (automation.get('flowId') or automation.get('preset')) + and not isinstance(definition.get('profile'), dict) and not automation.get('stageField'))): + raise CellNoteRefused('automation_field_read_only', f'Field {label} is filled by an automation.') + if (not grid_events._may_edit_field_value(ctx, field_key, definition) + or not grid_events._may_edit_field_value(ctx, field_key, field)): + raise CellNoteRefused('member_field_read_only', + f'Member {ctx.uname} cannot edit field {label} in database {key}.') + + mapped = table.setdefault(CELL_NOTES_KEY, {}) + merged = dict(mapped.get(rid) or {}) + for field_key, text in delta.items(): + if text is None: + merged.pop(field_key, None) + else: + merged[field_key] = text + if merged: + mapped[rid] = merged + else: + mapped.pop(rid, None) + saved = True + return current + + result = edit_receipts.CheckedStore(ctx.st).update(STORE_KEY, merge, flush='sync') + if not saved or result is None or result is False: + raise edit_receipts.ReceiptUnavailable('The cell note was not saved. Retry this opId.') + return {'pid': row_id, 'opId': identity, 'value': {CELL_NOTES_KEY: deepcopy(delta)}} + + # Match the row PATCH receipt identity and envelope so its active transaction is reused. + result = edit_receipts.run_once(ctx.st, ctx.uname, + edit_receipts.operation(scope, 'overlay_patch', rid), identity, + lambda: {'answer': apply(), 'event': None}) + return deepcopy(result['answer']) + + def patch_cells(table_key, row_id, values, st=None, *, by=None, via=None): """Write shared definition-row cells for trusted server-side writers.""" + _refuse_reserved_cell_keys(values) if str(table_key) == 'ut_activity' or not is_user_table(table_key, st): return False rows = (get_record(table_key, row_id, st) or {}).get('rows') or {} if str(row_id) not in rows: return False + applied = False + def _set(cur): + nonlocal applied + applied = False t = cur.get(str(table_key)) - if t is not None: + if t is not None and str(row_id) in (t.get('rows') or {}): _refuse_autonumber_values(t.get('fields') or [], values or {}) _refuse_audit_values(t.get('fields') or [], values or {}) _validate_number_values(t.get('fields') or [], values or {}) @@ -5245,13 +5399,14 @@ def patch_cells(table_key, row_id, values, st=None, *, by=None, via=None): row.pop(key, None) else: row[key] = value + applied = True _hydrate_choice_options(t.get('fields') or [], [ {key: value for key, value in clean.items() if value is not None} ]) return cur - update_with_activity(table_key, _set, st=st, by=by, via=via, row_ids=(row_id,)) - return True + result = update_with_activity(table_key, _set, st=st, by=by, via=via, row_ids=(row_id,)) + return applied and result is not False def patch_many_cells(table_key, updates, st=None, *, by=None, via=None): @@ -5267,6 +5422,8 @@ def patch_many_cells(table_key, updates, st=None, *, by=None, via=None): rather than created: this is a cell writer, not an insert door, and inventing a row here would let a stale plan resurrect a record somebody deleted mid-run. """ + for values in (updates or {}).values(): + _refuse_reserved_cell_keys(values) if str(table_key) == 'ut_activity' or not is_user_table(table_key, st): return 0 have = set((get_records(table_key, list(updates or {}), st) or {}).get('rows') or {}) @@ -5436,6 +5593,9 @@ def delete_row(table_key, row_id, st=None, *, by=None, via=None): if t is not None: (t.get('rows') or {}).pop(str(row_id), None) (t.get('_rowCreationOrder') or {}).pop(str(row_id), None) + notes = t.get(CELL_NOTES_KEY) + if notes is not None: + notes.pop(str(row_id), None) return cur update_with_activity(table_key, _drop, st=st, by=by, via=via, row_ids=(row_id,), flush='sync') @@ -6135,8 +6295,12 @@ def nav_entries(viewer=None, is_admin=False, st=None): tables = all_defs(st) lent = lend(st, **{STORE_KEY: tables}) out = [] - for key, t in sorted(tables.items(), + definitions = ((key, table) for key, table in tables.items() if isinstance(table, dict)) + for key, t in sorted(definitions, key=lambda kv: (kv[1].get('label') or '').lower()): + # Source-topic metadata belongs to the registry's existing database entry. + if key in LINKABLE_MODULE_KEYS: + continue if viewer is not None and not may_open(key, viewer, is_admin, st=lent): continue from core import route_system diff --git a/platform/evals/analyst_golden.yml b/platform/evals/analyst_golden.yml index 5028f284d518b8c816d3b6d2c311c602747ef56b..17975b7e39828e7627ae7c46b4e08e68b7b2fde7 100644 --- a/platform/evals/analyst_golden.yml +++ b/platform/evals/analyst_golden.yml @@ -1,24 +1,27 @@ -# Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the -# Jul-15 Odoo mass-recompute + hard-delete reconciliation — closed windows DO drift when docs -# are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances). -# Original header: truths computed from the PARITY-PROVEN -# semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot -# break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use -# assertions) — the validate() culture applied to AI answers. Run: harness/evals.py. +# Analyst truths bind to analyst_snapshot.json, including its checksum and clock. +# Closed windows can drift when source documents change. Never substitute the serving mirror. # expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive). # expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools). # expect.not_tools: tool names that must NOT appear (the over-refusal guard — an answerable ask # must never end in report_gap; a gap ask must never end in a forced wrong query). -# Truths below come from the semantic layer over this seed, with no LLM. A live Odoo read after the -# cut can differ (documents edited later); the seed value is the truth and the drift is logged. +# Independent raw-table SQL verifies numeric and name truths against these bytes, without an LLM. +# Evidence: AutoWiki/sdlc/sdlc80/lanes/P/proofs/independent_{seed,remaining}_truths.py. +# Unchanged values retain their original tolerances. A later source edit can change live totals. +# Supply the exact private seed through ANALYST_EVAL_SNAPSHOT_FILE or the pinned HF revision. +# An unavailable seed fails before any model call. No customer rows belong in this repository. seed: - contentSha: 3857b63c11148c7565463cbae21b24f54602f1106624ab7337b115c278828a5b - cutDate: "2026-09-03" + contentSha: f15564602b9c5b85abeafae2ff79ce6ba4b48640f6530f7954d27dabe9227a7e + cutDate: "2026-10-23" dataset: royal-imports/cfo-os-data - revision: e91188cbb12e6d13e48b7d754afd9bfdd6e84ae4 - truthsComputed: "2026-10-09" -# The OpenRouter model the paid eval measured. analyst.py PROVIDERS defaults to it. -measured: + revision: 581e549c14108c693af2bcd91154cbb5c9182b71 + filename: store_seed/royal.duckdb + retainedFilename: analyst_snapshots/f15564602b9c5b85abeafae2ff79ce6ba4b48640f6530f7954d27dabe9227a7e.duckdb + bytes: 610021376 + truthsComputed: "2026-10-26" +# Historical measurement only. It does not score the replacement snapshot. +historicalMeasured: + goldenSeedSha: 3857b63c11148c7565463cbae21b24f54602f1106624ab7337b115c278828a5b + dataBinding: unpinned-serving-mirror model: z-ai/glm-4.7-flash openWeights: zai-org/GLM-4.7-Flash score: "20/26" @@ -28,25 +31,25 @@ measured: items: - id: rev_h1 ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?" - expect: {value: 3502957.34, rel: 0.001, tools: [run_semantic_query]} + expect: {value: 3493861.41, rel: 0.001, tools: [run_semantic_query]} - id: rev_h1_fisch ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?" expect: {value: 2489001.85, rel: 0.001, tools: [run_semantic_query]} - id: rev_h1_royal ask: "And Royal's revenue for the same H1 2026 window?" - expect: {value: 1013955.49, rel: 0.001, tools: [run_semantic_query]} + expect: {value: 1005606.87, rel: 0.001, tools: [run_semantic_query]} - id: rev_june ask: "Revenue for June 2026?" expect: {value: 536944.56, rel: 0.001, tools: [run_semantic_query]} - id: margin_h1 ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?" - expect: {value: 2069536.47, rel: 0.001, tools: [run_semantic_query]} + expect: {value: 2061906.77, rel: 0.001, tools: [run_semantic_query]} - id: margin_pct_h1 ask: "What gross margin percentage did we run in H1 2026?" expect: {value: 0.5908, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]} - id: orders_h1 ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?" - expect: {value: 4985, abs: 1, tools: [run_semantic_query]} + expect: {value: 4979, abs: 1, tools: [run_semantic_query]} - id: customers_h1 ask: "How many distinct customers bought from us in H1 2026?" expect: {value: 1036, abs: 1, tools: [run_semantic_query]} @@ -89,7 +92,7 @@ items: not_tools: [report_gap]} - id: agent_revenue ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?" - expect: {value: 880329.39, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]} + expect: {value: 872080.37, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]} # --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a # per-invoice-line commission agent; they name different people, and the commission table also # carries internal SALESPEOPLE who are not agents. These three lock the distinction: the diff --git a/platform/harness/amazon_ads_client.py b/platform/harness/amazon_ads_client.py index 0322b7b36da1cc7669fa3c2b68f23093c1692ce5..fd43ca52202abb59519ce1b19a76b8e7176154b6 100644 --- a/platform/harness/amazon_ads_client.py +++ b/platform/harness/amazon_ads_client.py @@ -291,10 +291,10 @@ def send_named_write(tenant_key, kind, payload, *, approval_id, rt=None): "response": reply} -def request(tenant_key, method, path, *, body=None, query=None, rt=None, profile_id=None): +def request(tenant_key, method, path, *, body=None, query=None, rt=None, profile_id=None, connection_id=None): """A report call by method and path. Anything else raises `AdsWriteRefused` before any I/O.""" return _send(build_request(method, path, body=body, query=query), - str(tenant_key or "").strip().lower(), rt=rt, profile_id=profile_id) + str(tenant_key or "").strip().lower(), rt=rt, profile_id=profile_id, connection_id=connection_id) def _admitted(prepared): @@ -307,21 +307,24 @@ def _admitted(prepared): return bool(rule) and rule["endpoints"].get(prepared.named["ad_type"]) == (prepared.method, prepared.path) -def _send(prepared, tenant_key, *, rt=None, profile_id=None): +def _send(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None): if not _admitted(prepared): raise AdsWriteRefused("the request did not come from the request builder") if fixture_mode(): return FIXTURE.answer(prepared, tenant_key, profile_id) - return _live(prepared, tenant_key, rt=rt, profile_id=profile_id) + return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id) # --------------------------------------------------------------------------- live transport -def credential(tenant_key, rt=None): +def credential(tenant_key, rt=None, connection_id=None): """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source.""" try: import core.keychain as kc - fields = kc._creds_for(rt, ENTRY_TYPE, None) + bound = getattr(rt, "key", None) or getattr(getattr(rt, "tenant", None), "key", None) + if bound != tenant_key: + return None + fields = kc.amazon_ads_creds(rt, connection_id=connection_id) except Exception: # noqa: BLE001 return None if not isinstance(fields, dict) or not all(fields.get(f) for f in CREDENTIAL_FIELDS): @@ -343,11 +346,11 @@ def _token(fields): raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None -def _live(prepared, tenant_key, *, rt=None, profile_id=None): - fields = credential(tenant_key, rt) +def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None): + fields = credential(tenant_key, rt, connection_id=connection_id) if fields is None: raise AdsError("No Amazon Ads credential is stored for this tenant. " - "Connect Amazon Ads under Settings, Connectors.") + "Connect Amazon Ads under Amazon Ads Setup.") host = HOSTS.get(str(fields.get("region") or DEFAULT_REGION).lower(), HOSTS[DEFAULT_REGION]) url = f"{host}/{prepared.path}" if prepared.query: @@ -385,7 +388,7 @@ def _report_lock(tenant_key, profile_id): return _REPORT_LOCKS.setdefault((tenant_key, str(profile_id or "")), threading.Lock()) -def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, sleep=time.sleep, +def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep, now=time.monotonic): """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).""" from harness import amazon_ads_schema as schema @@ -398,14 +401,14 @@ def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, sleep # One report in flight per advertiser (F11): the lock serialises this process's reports. with _report_lock(tenant_key, profile_id): created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body, - rt=rt, profile_id=profile_id) + rt=rt, profile_id=profile_id, connection_id=connection_id) report_id = str(created.get("reportId") or "") if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id): raise AdsError("Amazon Ads returned no report id") started = now() status_path = REPORT_OPERATIONS["getReport"][1].replace("{reportId}", report_id) while True: - status = request(tenant_key, "GET", status_path, rt=rt, profile_id=profile_id) + status = request(tenant_key, "GET", status_path, rt=rt, profile_id=profile_id, connection_id=connection_id) state = str(status.get("status") or "").upper() if state == "COMPLETED": break @@ -421,7 +424,7 @@ def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, sleep rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id) while True: page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None, - rt=rt, profile_id=profile_id) + rt=rt, profile_id=profile_id, connection_id=connection_id) rows.extend(page.get("rows") or []) token = page.get("nextToken") if not token: diff --git a/platform/harness/amazon_ads_store.py b/platform/harness/amazon_ads_store.py index 44da574b37d012d85f471c8822ce37612c0dc8a0..71659481efa8308d4e98312e3eafeedb8ac0ba6c 100644 --- a/platform/harness/amazon_ads_store.py +++ b/platform/harness/amazon_ads_store.py @@ -19,6 +19,7 @@ Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_` is the ne `amzn_sync_floor_ads_
` the oldest, `amzn_ads_
` the table state. """ import argparse +import hashlib import importlib import json import sys @@ -45,6 +46,7 @@ RESTATEMENT_DAYS = 14 CONNECT_DAY_KEY = amazon_store.entity_key("ads_connect_day") BUDGET_KEY = amazon_store.entity_key("ads_report_budget") INSERT_BATCH_ROWS = 500 +ORIGIN_COLUMNS = ("_connection_id", "store") def table_name(table): @@ -83,15 +85,28 @@ def _utc_today(now=None): return now.date() if isinstance(now, datetime) else now -def _connect_day(con, today): +def _state_key(key, connection_id=None): + """Keep the public table markers and give each connection its own continuation markers.""" + return f"{key}::connection::{connection_id}" if connection_id else key + + +def _connect_day(con, today, connection_id=None, *, legacy=False): """The connect day, stamped once on the first sync and never moved.""" - state = amazon_store._read_state(con, CONNECT_DAY_KEY) + key = _state_key(CONNECT_DAY_KEY, connection_id) + state = amazon_store._read_state(con, key) + inherited = False + if state is None and legacy and connection_id: + state = amazon_store._read_state(con, CONNECT_DAY_KEY) + inherited = state is not None if state and state.get("detail"): try: - return date.fromisoformat(str(state["detail"])[:10]) + day = date.fromisoformat(str(state["detail"])[:10]) + if inherited: + amazon_store._record_state(con, key, "done", detail=day.isoformat()) + return day except ValueError: pass - amazon_store._record_state(con, CONNECT_DAY_KEY, "done", detail=today.isoformat()) + amazon_store._record_state(con, key, "done", detail=today.isoformat()) return today @@ -103,14 +118,14 @@ def _day_marker(con, key): return None -def windows(con, table, connect_day, today): +def windows(con, table, connect_day, today, connection_id=None): """The report windows one sync asks for, oldest first, each at most 31 days.""" spec = schema.TABLE_SPECS[schema.short_name(table)] last = today - timedelta(days=1) target_floor = max(connect_day, today) - timedelta(days=spec["backfill_days"]) mark_name = _marker_table(table) - newest = _day_marker(con, amazon_store.entity_key(amazon_store.WATERMARK_PREFIX + mark_name)) - oldest = _day_marker(con, amazon_store.entity_key(amazon_store.BACKFILL_FLOOR_PREFIX + mark_name)) + newest = _day_marker(con, _state_key(amazon_store.entity_key(amazon_store.WATERMARK_PREFIX + mark_name), connection_id)) + oldest = _day_marker(con, _state_key(amazon_store.entity_key(amazon_store.BACKFILL_FLOOR_PREFIX + mark_name), connection_id)) spans = [] if newest is None or oldest is None: spans.append((target_floor, last)) @@ -128,9 +143,13 @@ def windows(con, table, connect_day, today): return out +def _columns_for(table): + return tuple(dict.fromkeys(schema.columns_for(table) + schema.ORIGIN_COLUMNS + ORIGIN_COLUMNS)) + + def _ensure_table(con, table): name = table_name(table) - cols = schema.columns_for(table) + schema.ORIGIN_COLUMNS + cols = _columns_for(table) have = {r[0] for r in con.execute( "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [name]).fetchall()} if not have: @@ -149,7 +168,7 @@ def _upsert(con, table, rows): if not rows: return 0 name = table_name(table) - cols = list(schema.columns_for(table) + schema.ORIGIN_COLUMNS) + cols = _columns_for(table) by_key = {r["_row_key"]: r for r in rows} keys = list(by_key) for i in range(0, len(keys), INSERT_BATCH_ROWS): @@ -162,8 +181,10 @@ def _upsert(con, table, rows): return len(by_key) -def _budget(con, today): - state = amazon_store._read_state(con, BUDGET_KEY) +def _budget(con, today, connection_id=None, *, legacy=False): + state = amazon_store._read_state(con, _state_key(BUDGET_KEY, connection_id)) + if state is None and legacy and connection_id: + state = amazon_store._read_state(con, BUDGET_KEY) try: detail = json.loads(state["detail"]) if state and state.get("detail") else {} except (ValueError, TypeError): @@ -171,22 +192,98 @@ def _budget(con, today): return int(detail.get("used") or 0) if detail.get("day") == today.isoformat() else 0 -def _spend_budget(con, today, used): - amazon_store._record_state(con, BUDGET_KEY, "done", rows=used, +def _spend_budget(con, today, used, connection_id=None): + amazon_store._record_state(con, _state_key(BUDGET_KEY, connection_id), "done", rows=used, detail=json.dumps({"day": today.isoformat(), "used": used})) -def _record_limit(con, table, reason, **extra): - amazon_store._record_state(con, limit_key(table), "done", +def _record_limit(con, table, reason, connection_id=None, **extra): + amazon_store._record_state(con, _state_key(limit_key(table), connection_id), "done", detail=json.dumps({"reason": reason, **extra})) -def _clear_limit(con, table): - con.execute("DELETE FROM _sync_state WHERE entity = ?", [limit_key(table)]) +def _clear_limit(con, table, connection_id=None): + con.execute("DELETE FROM _sync_state WHERE entity = ?", [_state_key(limit_key(table), connection_id)]) + + +def _count(con, table, connection_id=None): + where = ' WHERE "_connection_id" = ?' if connection_id is not None else '' + return con.execute(f'SELECT count(*) FROM "{table_name(table)}"' + where, + [connection_id] if connection_id is not None else []).fetchone()[0] + + +def _runtime_for(key): + try: + from harness import runtime + return runtime.get_runtime(key) + except Exception: + return None + + +def _connections(rt, key, profile_id, report): + """Read named business credentials without changing process state or the client's resolver.""" + from core import keychain + if rt is None and client.fixture_mode(): + return [("", "Amazon Ads", profile_id, True)] + try: + entries = sorted((row for row in keychain.list_entries(rt) if row.get("type") == "amazon_ads"), + key=lambda row: (int(row.get("slot") or 1), str(row["id"]))) + default = keychain.resolve_connection_id(rt, "amazon_ads") + except Exception: + report["problems"].append("The Amazon Ads connections could not be read. Retry this sync.") + return [] + out = [] + for row in entries: + cid = str(row.get("connection_id") or row["id"]) + label = str(row.get("label") or cid) + try: + fields = keychain.amazon_ads_creds(rt, connection_id=cid) + except Exception: + report["problems"].append(f"{label}: the Amazon Ads credential could not be read. Test this connection again.") + continue + if fields is None: + continue # The public resolver excludes personal credentials. + if not all(fields.get(name) for name in client.CREDENTIAL_FIELDS): + report["problems"].append(f"{label}: the Amazon Ads credential is incomplete. Check its four fields.") + continue + out.append((cid, label, str(fields["profile_id"]), cid == default)) + return out -def _count(con, table): - return con.execute(f'SELECT count(*) FROM "{table_name(table)}"').fetchone()[0] +def _row_key(connection_id, key): + if not connection_id: + return key + payload = json.dumps([connection_id, key], separators=(",", ":")).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _refresh_origin(con, table, connection_id, label, legacy): + """Migrate the former default's rows once and refresh labels across retained history.""" + name = table_name(table) + changed = con.execute(f'SELECT count(*) FROM "{name}" WHERE ' + '("_connection_id" = ? AND store IS DISTINCT FROM ?) ' + 'OR ("_connection_id" IS NULL AND ?)', [connection_id, label, legacy]).fetchone()[0] + if legacy: + rows = con.execute(f'SELECT "_row_key" FROM "{name}" WHERE "_connection_id" IS NULL').fetchall() + for (key,) in rows: + con.execute(f'UPDATE "{name}" SET "_row_key" = ?, "_connection_id" = ?, store = ? ' + 'WHERE "_row_key" = ? AND "_connection_id" IS NULL', + [_row_key(connection_id, key), connection_id, label, key]) + con.execute(f'UPDATE "{name}" SET store = ? WHERE "_connection_id" = ?', [label, connection_id]) + return changed + + +def _run_report(key, table, start, end, rt, profile_id, connection_id): + """Require named transport support instead of sending a sibling's credential.""" + options = {"rt": rt, "profile_id": profile_id} + if connection_id: + options["connection_id"] = connection_id + try: + return client.run_report(key, table, start, end, **options) + except TypeError as exc: + if connection_id and "unexpected keyword argument 'connection_id'" in str(exc): + raise client.AdsError("The Amazon Ads client needs named connection support before this connection can sync.") from None + raise def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print, @@ -197,79 +294,101 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None through `amazon_store.persist_mirror`. A caller that lends `con` persists itself. """ key = amazon_store._tenant_key(tenant_key) - report = {"tenant": key, "tables": {}, "problems": [], "limits": {}} - fields = None if client.fixture_mode() else client.credential(key, rt) - if not client.fixture_mode() and fields is None: - report["problems"].append("No Amazon Ads credential is stored for this tenant, so nothing " - "was pulled. Connect Amazon Ads under Settings, Connectors.") + report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": []} + if rt is None and not client.fixture_mode(): + rt = _runtime_for(key) + if rt is not None and str(getattr(rt, "key", "") or "").strip().lower() != key: + report["problems"].append("The runtime does not match the requested tenant. No Amazon Ads rows were pulled.") + return report + connections = _connections(rt, key, profile_id, report) + if not connections: + report["problems"].append("No Amazon Ads credential is stored for this tenant. Open Amazon Ads Setup to add a connection.") return report - profile_id = profile_id or (fields or {}).get("profile_id") own = con is None if own: con, _path = amazon_store._connect_tenant_mirror(key, rt) try: datastore._ensure_sync_state(con) today = _utc_today(now) - report["connectDay"] = _connect_day(con, today).isoformat() - connect_day = date.fromisoformat(report["connectDay"]) - used = _budget(con, today) synced = [] - for table in (tables or TABLES): - table = schema.short_name(table) - spec = schema.TABLE_SPECS[table] - _ensure_table(con, table) - if not spec["offered"]: - _record_limit(con, table, spec["not_offered_reason"], state="not_offered") - report["tables"][table] = {"state": "not_offered", "pulled": 0, "in_mirror": _count(con, table)} - report["limits"][table] = spec["not_offered_reason"] - continue - pulled, state, mark = 0, "done", _marker_table(table) - for start, end in windows(con, table, connect_day, today): - if used >= client.REPORTS_PER_DAY: - state = "deferred" - reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} " - f"reports are used, so days from {start.isoformat()} load on the next sync.") - _record_limit(con, table, reason, state="deferred") - report["limits"][table] = reason - report["problems"].append(f"{spec['label']}: waiting for tomorrow's report limit") - break - used += 1 - _spend_budget(con, today, used) - try: - report_id, rows = client.run_report(key, table, start, end, rt=rt, profile_id=profile_id) - except client.AdsReportUnavailable as exc: - state = "not_offered" - reason = (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser " - f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, " - f"contact Amazon Ads support with the report type {spec['report_type']}.") - _record_limit(con, table, reason, state="not_offered") - report["limits"][table] = reason - break - except client.AdsError as exc: - state = "failed" - report["problems"].append(f"{spec['label']}: {str(exc)[:200]}") - break - stored = [schema.row_from_vendor(table, r, profile_id=profile_id, report_id=report_id) - for r in rows] - pulled += _upsert(con, table, stored) - # The receipts move only after a window lands. - newest = _day_marker(con, amazon_store.entity_key(amazon_store.WATERMARK_PREFIX + mark)) - oldest = _day_marker(con, amazon_store.entity_key(amazon_store.BACKFILL_FLOOR_PREFIX + mark)) - if newest is None or end > newest: - amazon_store._record_state(con, amazon_store.entity_key(amazon_store.WATERMARK_PREFIX + mark), - "done", _count(con, table), end.isoformat()) - if oldest is None or start < oldest: - amazon_store._record_state(con, amazon_store.entity_key(amazon_store.BACKFILL_FLOOR_PREFIX + mark), - "done", detail=start.isoformat()) - if state == "done": + for cid, label, profile, legacy in connections: + connect_day = _connect_day(con, today, cid, legacy=legacy) + report.setdefault("connectDay", connect_day.isoformat()) + entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}} + report["connections"].append(entry) + used = _budget(con, today, cid, legacy=legacy) + for table in (tables or TABLES): + table = schema.short_name(table) + spec = schema.TABLE_SPECS[table] + _ensure_table(con, table) + origin_changed = _refresh_origin(con, table, cid, label, legacy) + pulled, state, mark, reason = 0, "done", _marker_table(table), "" + if not spec["offered"]: + state, reason = "not_offered", spec["not_offered_reason"] + else: + for start, end in windows(con, table, connect_day, today, cid): + if used >= client.REPORTS_PER_DAY: + state = "deferred" + reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} " + f"reports are used, so days from {start.isoformat()} load on the next sync.") + report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit") + break + used += 1 + _spend_budget(con, today, used, cid) + try: + report_id, rows = _run_report(key, table, start, end, rt, profile, cid) + except client.AdsReportUnavailable as exc: + state = "not_offered" + reason = (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser " + f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, " + f"contact Amazon Ads support with the report type {spec['report_type']}.") + break + except client.AdsError as exc: + state = "failed" + reason = str(exc)[:200] + report["problems"].append(f"{label}: {spec['label']}: {reason}") + break + stored = [] + for raw in rows: + row = schema.row_from_vendor(table, raw, profile_id=profile, report_id=report_id) + row.update(_connection_id=cid, store=label, + _row_key=_row_key(cid, row["_row_key"])) + stored.append(row) + pulled += _upsert(con, table, stored) + # Only this connection's landed window advances its continuation. + for prefix, day, later in ((amazon_store.WATERMARK_PREFIX, end, True), + (amazon_store.BACKFILL_FLOOR_PREFIX, start, False)): + public = amazon_store.entity_key(prefix + mark) + for marker in dict.fromkeys((_state_key(public, cid), public)): + previous = _day_marker(con, marker) + if previous is None or (day > previous if later else day < previous): + amazon_store._record_state(con, marker, "done", _count(con, table), day.isoformat()) + if reason: + _record_limit(con, table, reason, cid, state=state) + shown = f"{label}: {reason}" if cid else reason + report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown))) + elif state == "done": + _clear_limit(con, table, cid) + total = _count(con, table) + entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)} + amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid), + "done" if state == "done" else "partial", _count(con, table, cid), state) + previous = report["tables"].get(table, {}) + severity = {"done": 0, "not_offered": 1, "deferred": 2, "failed": 3} + aggregate = max((state, previous.get("state", "done")), key=severity.get) + report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled, + "in_mirror": total} + if (pulled or origin_changed) and table_name(table) not in synced: + synced.append(table_name(table)) + log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}") + for table, state in report["tables"].items(): + amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)), + "done" if state["state"] == "done" else "partial", + state["in_mirror"], state["state"]) + if table in report["limits"]: + _record_limit(con, table, report["limits"][table], state=state["state"]) + else: _clear_limit(con, table) - total = _count(con, table) - amazon_store._record_state(con, amazon_store.entity_key(mark), - "done" if state == "done" else "partial", total, state) - report["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": total} - if pulled: - synced.append(table_name(table)) - log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}") if own and rt is not None and synced: _persist(key, rt, con, synced, report, log) finally: diff --git a/platform/harness/amazon_schema.py b/platform/harness/amazon_schema.py index 7da0a12e0bb9a64265e41491a448333d1b7755f1..a0454467319de225d1ab7bb499906ca2dcc43365 100644 --- a/platform/harness/amazon_schema.py +++ b/platform/harness/amazon_schema.py @@ -296,8 +296,11 @@ def catalogue(): label, native = descriptors[key].label, descriptors[key].native table[key] = {"label": label, "native": native} if provenance in {"derived", "apify_dataset"}: - table["columns"] = [dict(field) for field in AMAZON_FIELDS[key]] + table["columns"] = [dict(field) for field in AMAZON_FIELDS[key] + if field["name"] != STORE] table["note"] = PROVENANCE_COPY[provenance] + table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ()) + if field["name"] == STORE] return result @@ -315,6 +318,13 @@ def columns_for(table): #: A-C1-KEYS preserves stored enablement keys and route paths while lane D adopts this map. DATABASE_PREFIX = "ut_amzn_" +# Internal mirror metadata preserves legacy duplicate ordinals across SQL pages. +IDENTITY_OCCURRENCE = "_amazon_identity_occurrence" +IDENTITY_PID = "_amazon_identity_pid" +CONNECTION_ID = "_connection_id" +STORE = "store" +MIRROR_INTERNAL_COLUMNS = frozenset((IDENTITY_OCCURRENCE, IDENTITY_PID, CONNECTION_ID)) + def _table_spec(table, identity, relationships=(), *, grain="entity", line_discriminator=None): """Build the common descriptor fields from the LOCAL pin, never a vendor call.""" @@ -356,7 +366,9 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr "Standing rule 1: these are connector records with records_mutable=False. " "Serve history read-through from the mirror without a record cap."), } - spec["columns"].extend(dict(field) for field in AMAZON_FIELDS.get(DATABASE_PREFIX + table, ())) + fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ()) + spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE) + spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE] if PROVENANCE[table] == "derived": spec["columns_source"] = { "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS", @@ -558,7 +570,7 @@ def row_key_columns_for(table, columns=()): if ("_report_row_number" in identity and "_report_document_id" not in identity and "_report_document_id" in columns): identity = (*identity, "_report_document_id") - return identity + return (CONNECTION_ID, *identity) if CONNECTION_ID in columns else identity def row_fallback_columns_for(table): @@ -579,6 +591,8 @@ def row_identity_for(table, row): identity = spec["row_identity"] if not identity or spec["row_identity_alternatives"]: raise ValueError(f"{table}: exactly one complete row identity is required") + if CONNECTION_ID in row: + identity = (CONNECTION_ID, *identity) missing = [name for name in identity if row.get(name) in (None, "")] if missing: raise ValueError(f"{table}: missing declared row identity components: {', '.join(missing)}") diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index ff6059c8617a25e1cab0fbb59714d6dedf1dd977..1ec05c9859fda32cabf1d02161f5f50cba9ef9aa 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -110,6 +110,330 @@ ENABLEMENT_KEY = "amazon_connector_enablement" MIRROR_PREFIX = "amazon_mirror/" _MIRROR_CACHE = {} _MIRROR_LOCK = threading.RLock() +_ACCOUNT = threading.local() + + +def _account_scope(): + return getattr(_ACCOUNT, "scope", None) + + +def _ordered_account_entries(entries): + """Order Amazon metadata by keychain slot and entry ID, including saved revocation snapshots.""" + return sorted((dict(row) for row in entries if row.get("type") == "amazon_sp"), + key=lambda row: (int(row.get("slot") or 1), str(row["id"]))) + + +def account_entries(tenant_key, rt): + """List every tenant Amazon connection without decrypting credentials.""" + _tenant_mirror_path(tenant_key, rt) + if rt is None: + raise RuntimeError("Amazon sync requires a tenant runtime and stored credentials") + from core import keychain + rows = _ordered_account_entries(keychain.list_entries(rt)) + identities = [str(row.get("connection_id") or "") for row in rows] + if not rows or any(not cid for cid in identities) or len(set(identities)) != len(rows): + raise RuntimeError("Amazon sync requires distinct stored amazon_sp connections") + return rows + + +def _account_prefix(connection_id): + return "amzn_account_" + str(connection_id).encode("utf-8").hex() + "__" + + +def _sp_objects(con): + return [(name, kind) for name, kind in _amazon_objects(con) + if name.startswith(PREFIX) and not name.startswith("amzn_ads_") + and name != "amzn_apify_rank_observations"] + + +def legacy_identity_owner(con, table): + """Return the immutable owner of a table's pre-connection numeric identity namespace.""" + return (_read_state(con, "amzn_account_legacy_identity_" + table) or {}).get("detail") + + +def _adopt_legacy_account(con, entry): + """Bind untagged single-account rows and markers once to the designated connection.""" + if _read_state(con, "amzn_account_binding"): + return set() + con.execute("BEGIN TRANSACTION") + try: + changed = _bind_legacy_account(con, entry) + con.execute("COMMIT") + return changed + except BaseException: + con.execute("ROLLBACK") + raise + + +def _bind_legacy_account(con, entry): + cid, label = str(entry["connection_id"]), str(entry["label"]) + changed = set() + if _read_state(con, "amzn_account_binding"): + return changed + for name, kind in _sp_objects(con): + if kind != "BASE TABLE": + continue + columns = _columns_of(con, name) + if amazon_schema.CONNECTION_ID not in columns: + if con.execute(f'SELECT 1 FROM {name} LIMIT 1').fetchone(): + _record_state(con, "amzn_account_legacy_identity_" + name.removeprefix(PREFIX), "done", detail=cid) + con.execute(f'ALTER TABLE {name} ADD COLUMN "_connection_id" VARCHAR') + if amazon_schema.STORE not in columns: + con.execute(f'ALTER TABLE {name} ADD COLUMN store VARCHAR') + if con.execute(f'SELECT 1 FROM {name} WHERE "_connection_id" IS NULL LIMIT 1').fetchone(): + con.execute(f'UPDATE {name} SET "_connection_id"=?, store=? WHERE "_connection_id" IS NULL', + [cid, label]) + changed.add(name) + prefix = _account_prefix(cid) + con.execute("INSERT OR REPLACE INTO _sync_state SELECT ? || entity, phase, last_id, cursor_wd, rows, updated_at " + "FROM _sync_state WHERE starts_with(entity, 'amzn_') " + "AND NOT starts_with(entity, 'amzn_account_') AND NOT starts_with(entity, 'amzn_ads_')", [prefix]) + con.execute("DELETE FROM _sync_state WHERE starts_with(entity, 'amzn_') " + "AND NOT starts_with(entity, 'amzn_account_') AND NOT starts_with(entity, 'amzn_ads_')") + _record_state(con, "amzn_account_binding", "done", detail=cid) + return changed + + +def _refresh_account_status(con, entries, rt): + """Project account progress for existing status readers without lending it to another account.""" + enabled = read_database_enablement(rt)["databases"] + tables = [t for t in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + t)] + for table in TABLES: + for family in ("", "sync_tick_", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "refused_"): + key = entity_key(family + table) + states = [_read_state(con, _account_prefix(e["connection_id"]) + key) for e in entries] + present = [s for s in states if s] + con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) + if not present: + continue + if family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX) and len(present) != len(entries): + continue + state = next((s for s in present if s["state"] != "done"), present[0]) + phase = state["state"] if len(present) == len(entries) else "pending" + detail = (min(s["detail"] for s in present if s.get("detail")) + if family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX) and all(s.get("detail") for s in present) + else state.get("detail")) + _record_state(con, key, phase, sum(s.get("rows", 0) for s in present), detail) + updated = min(s["updated_at"] for s in present) + con.execute("UPDATE _sync_state SET updated_at=? WHERE entity=?", [updated, key]) + loads, remaining, eta, waits = [], 0, 0, [] + for entry in entries: + prefix = _account_prefix(entry["connection_id"]) + state = _read_state(con, prefix + entity_key(FIRST_LOAD_END_KEY)) + progress = _read_state(con, prefix + entity_key(FIRST_LOAD_STATUS_KEY)) + if state and progress: + values = json.loads(progress["detail"]) + loads.append(state["state"]) + remaining += int(values["remaining"]) + eta += int(values["etaSeconds"]) + else: + owed = sum(not _read_state(con, prefix + entity_key(WATERMARK_PREFIX + t)) + and (_read_state(con, prefix + entity_key(t)) or {}).get("state") != "done" for t in tables) + loads.append("partial" if owed else "done") + remaining += owed + if loads[-1] != "done": + due = _read_state(con, prefix + entity_key(FIRST_LOAD_DUE_KEY)) + waits.append(due.get("detail") if due and due.get("state") == "pending" else None) + phase = "failed" if "failed" in loads else "partial" if any(s != "done" for s in loads) else "done" + _record_state(con, entity_key(FIRST_LOAD_END_KEY), phase, detail=_iso(_utc())) + _record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), phase, + detail=json.dumps({"remaining": remaining, "etaSeconds": eta})) + due = min(waits) if waits and all(waits) else None + _record_state(con, entity_key(FIRST_LOAD_DUE_KEY), "pending" if due else "done", detail=due) + + +@contextmanager +def _account_workspace(con, tenant_key, rt, entry): + """Stage one connection with SQL transfers, retaining all rows and its durable resume markers.""" + import duckdb + path = _tenant_mirror_path(tenant_key, rt) + cid, label = str(entry["connection_id"]), str(entry["label"]) + prefix = _account_prefix(cid) + with tempfile.TemporaryDirectory(prefix="amazon-account-", dir=path.parent) as directory: + workspace = Path(directory) / "account.duckdb" + alias = "amazon_account_" + uuid.uuid4().hex + con.execute(f"ATTACH '{str(workspace).replace(chr(39), chr(39)*2)}' AS {alias}") + try: + con.execute(f"CREATE TABLE {alias}._sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " + "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") + con.execute(f"INSERT INTO {alias}._sync_state SELECT substr(entity, ?), phase, last_id, " + "cursor_wd, rows, updated_at FROM _sync_state WHERE starts_with(entity, ?)", + [len(prefix) + 1, prefix]) + for name, kind in _sp_objects(con): + if kind != "BASE TABLE": + continue + columns = _columns_of(con, name) + if amazon_schema.CONNECTION_ID not in columns: + continue + con.execute(f'CREATE TABLE {alias}.{name} AS SELECT * FROM main.{name} ' + 'WHERE "_connection_id"=?', [cid]) + con.execute(f'UPDATE {alias}.{name} SET store=?', [label]) + # Public keyword captures belong to Apify, and each seller derives its own SKU rows. + if _columns_of(con, "amzn_apify_rank_observations"): + con.execute(f'CREATE TABLE {alias}.amzn_apify_rank_observations AS ' + 'SELECT * FROM main.amzn_apify_rank_observations') + finally: + con.execute(f"DETACH {alias}") + root = duckdb.connect(str(workspace)) + scope = {"tenant": tenant_key, "entry": entry, "root": root, "path": path, + "workspace": workspace, "changed": set()} + previous = _account_scope() + _ACCOUNT.scope = scope + try: + datastore.apply_limits(root) + refresh_sku_keywords(root) + yield scope + finally: + _ACCOUNT.scope = previous + root.close() + + +def _merge_account(con, scope): + """Replace only this account's population and state, including empty snapshots.""" + entry = scope["entry"] + cid, label, prefix = str(entry["connection_id"]), str(entry["label"]), _account_prefix(entry["connection_id"]) + alias = "amazon_account_" + uuid.uuid4().hex + path = str(scope["workspace"]).replace("'", "''") + # Close the workspace writer before attaching it to the combined candidate. + scope["root"].close() + con.execute(f"ATTACH '{path}' AS {alias} (READ_ONLY)") + changed = set() + try: + objects = con.execute("SELECT table_name FROM information_schema.tables " + "WHERE table_catalog=? AND table_schema='main' AND table_type='BASE TABLE' " + "AND starts_with(table_name, 'amzn_') " + "AND table_name != 'amzn_apify_rank_observations'", [alias]).fetchall() + con.execute("BEGIN TRANSACTION") + try: + for (name,) in objects: + columns = {row[0]: row[1] for row in con.execute(f'DESCRIBE {alias}.{name}').fetchall()} + # Sync loaders stamp metadata before writes. Empty inherited tables retain it too. + if amazon_schema.CONNECTION_ID not in columns: + raise RuntimeError("Amazon connection attribution is missing") + target_columns = {r[0]: r[1] for r in con.execute("SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_catalog=current_database() AND table_schema='main' AND table_name=?", [name]).fetchall()} + if not target_columns: + con.execute(f'CREATE TABLE main.{name} AS SELECT * FROM {alias}.{name} WHERE FALSE') + else: + for column, kind in columns.items(): + if column not in target_columns: + con.execute(f'ALTER TABLE main.{name} ADD COLUMN {_quoted_identifier(column)} {kind}') + elif target_columns[column] != kind: + numeric = {"BIGINT", "INTEGER", "DOUBLE", "FLOAT", "HUGEINT"} + widened = "DOUBLE" if {kind, target_columns[column]} <= numeric else "VARCHAR" + if target_columns[column] != widened: + con.execute(f'ALTER TABLE main.{name} ALTER COLUMN {_quoted_identifier(column)} TYPE {widened}') + con.execute(f'DELETE FROM main.{name} WHERE "_connection_id"=?', [cid]) + con.execute(f'INSERT INTO main.{name} BY NAME SELECT * FROM {alias}.{name}') + con.execute(f'UPDATE main.{name} SET store=? WHERE "_connection_id"=?', [label, cid]) + changed.add(name) + con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [prefix]) + con.execute(f"INSERT INTO _sync_state SELECT ? || entity, phase, last_id, cursor_wd, rows, updated_at " + f"FROM {alias}._sync_state WHERE starts_with(entity, 'amzn_')", [prefix]) + con.execute("COMMIT") + except Exception: + con.execute("ROLLBACK") + raise + finally: + con.execute(f"DETACH {alias}") + return changed + + +def _purge_connection_rows(con, connection_id): + """Delete one immutable connection while preserving sibling rows and resume markers.""" + con.execute("BEGIN TRANSACTION") + try: + result = _delete_connection_rows(con, connection_id) + con.execute("COMMIT") + return result + except BaseException: + con.execute("ROLLBACK") + raise + + +def _delete_connection_rows(con, connection_id): + changed, deleted = set(), 0 + for name, kind in _sp_objects(con): + if kind != "BASE TABLE": + continue + if amazon_schema.CONNECTION_ID not in _columns_of(con, name): + if con.execute(f"SELECT 1 FROM {name} LIMIT 1").fetchone(): + raise RuntimeError("Amazon legacy ownership must be bound before scoped deletion") + continue + deleted += con.execute(f'SELECT count(*) FROM {name} WHERE "_connection_id"=?', [connection_id]).fetchone()[0] + con.execute(f'DELETE FROM {name} WHERE "_connection_id"=?', [connection_id]) + changed.add(name) + con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [_account_prefix(connection_id)]) + refresh_sku_keywords(con) + return changed, deleted + + +def purge_connection(tenant_key, connection_id, *, rt, entries_before): + """Revoke one account's serving, saved and private observations without deleting combined tables.""" + previous = getattr(_ACCOUNT, "purging", False) + _ACCOUNT.purging = True + try: + return _purge_connection(tenant_key, connection_id, rt=rt, entries_before=entries_before) + finally: + _ACCOUNT.purging = previous + + +def _purge_connection(tenant_key, connection_id, *, rt, entries_before): + key, cid = _tenant_key(tenant_key), str(connection_id or "") + _tenant_mirror_path(key, rt) + entries = _ordered_account_entries(entries_before) + if not cid or not any(e.get("connection_id") == cid for e in entries): + raise ValueError("Amazon scoped deletion requires the original connection metadata") + if _account_scope() or getattr(datastore._SYNC_TLS, "candidate", None) is not None: + raise RuntimeError("Amazon scoped deletion requires the resident writer outside a sync pass") + obligation = retention_check(key, rt=rt, raise_overdue=True) + if obligation["earliestTriggerAt"]: + raise RuntimeError("Amazon tenant deletion is outstanding; complete its tenant-wide lifecycle purge") + remaining = [e for e in entries if e["connection_id"] != cid] + with _MIRROR_LOCK: + restored = restore_mirror(key, rt, log=lambda *_: None) + if restored["state"] == "refused": + raise RuntimeError("Amazon scoped deletion could not restore its saved observations") + con, _path = _connect_tenant_mirror(key, rt) + try: + changed = _adopt_legacy_account(con, entries[0]) + removed, count = _purge_connection_rows(con, cid) + changed.update(removed) + _refresh_account_status(con, remaining, rt) + report = {"problems": []} + manifest = _MIRROR_STORE.persist_mirror(key, rt, con, changed, report=report, log=lambda *_: None, + _publication={"snapshot_id": uuid.uuid4().hex, "started_at": _control_stamp(), "state": "complete"}) + if not manifest or report["problems"]: + raise RuntimeError("Amazon scoped deletion could not save its retained observations") + datastore._write_publication(con, "amazon", manifest) + private_path = Path(rt.datastore_path).with_suffix(".amazon.candidate.duckdb") + private_deleted = 0 + if private_path.exists(): + with closing(datastore.apply_limits(datastore.duckdb.connect(str(private_path)))) as private: + _adopt_legacy_account(private, entries[0]) + _unused, private_deleted = _purge_connection_rows(private, cid) + _refresh_account_status(private, remaining, rt) + if _columns_of(private, "_candidate_base"): + private.execute("DELETE FROM _candidate_base") + private.execute("INSERT INTO _candidate_base VALUES (?,?)", + [manifest["snapshot_id"], _control_stamp()]) + current = set(_manifest_keys(manifest, strict=False)) + ledger = _MIRROR_STORE._read_key_ledger(rt, key) + candidates = set(ledger["keys"]) | {name + ".parquet" for name in changed} | {"sync_state.json"} + removed_keys = set() + for blob in sorted(candidates - current): + if blob.startswith("sync_state.") or any(blob.startswith(name + ".") for name in changed): + path = MIRROR_PREFIX + blob + if rt.delete_path(path) is False or rt.download_bytes(path) is not None: + raise RuntimeError("Amazon scoped deletion left superseded saved bytes") + removed_keys.add(blob) + ledger["keys"] = {k: v for k, v in ledger["keys"].items() if k not in removed_keys} + rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger)) + _purge_runtime_caches(rt, key, key) + return {"rows": count, "privateRows": private_deleted, "tables": sorted(changed), + "connectionId": cid, "snapshotId": manifest["snapshot_id"]} + finally: + con.close() def read_database_enablement(rt): @@ -564,12 +888,24 @@ def _quoted_header(header): for cell in cells) -def _unquote_cells(raw): +def _unquote_cells(raw, width=None): """One quoted document's split line -> its values: wrapping quotes off, `""` -> `"`. A tab inside a quoted value split it in two; the pieces join again up to the closing quote. - A value never closed on its line keeps its original pieces unchanged. + Measured-width rows keep their boundaries, including Amazon's unescaped terminal inch marks. + A supplied width refuses incomplete or ambiguous rows instead of padding or truncating them. """ + if width is not None and len(raw) == width: + out = [] + for cell in raw: + if cell.startswith('"'): + if len(cell) < 2 or not cell.endswith('"'): + raise ValueError("Amazon quoted report has an unclosed cell; no rows were replaced") + cell = cell[1:-1].replace('""', '"') + elif cell: + raise ValueError("Amazon quoted report has an unwrapped cell; no rows were replaced") + out.append(cell) + return out out, open_cells = [], None for cell in raw: if open_cells is not None: @@ -583,7 +919,11 @@ def _unquote_cells(raw): elif cell.startswith('"'): open_cells = [cell] else: + if width is not None and cell: + raise ValueError("Amazon quoted report has an unwrapped cell; no rows were replaced") out.append(cell) + if width is not None and (open_cells is not None or len(out) != width): + raise ValueError("Amazon quoted report row does not match its header width; no rows were replaced") return out + (open_cells or []) @@ -599,6 +939,8 @@ def _flat_rows(lines, document_id, report_type): header = _unquote_cells(header) cols = _dedupe([_snake(h) for h in header]) width = len(cols) + ledger = UNQUOTE_QUOTED_DOCUMENTS and (report_type == "GET_LEDGER_DETAIL_VIEW_DATA" or + {"event_type", "quantity", "fulfillment_center"} <= set(cols)) # The read-only connectors_amazon report probe measures headers without capture metadata. # Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies # BOTH keywords, and a missing document ID with a report type is an error, never a fallback. @@ -609,10 +951,16 @@ def _flat_rows(lines, document_id, report_type): if not raw or (len(raw) == 1 and not str(raw[0]).strip()): continue if quoted: - raw = _unquote_cells(raw) + raw = _unquote_cells(raw, width) + if ledger and len(raw) != width: + raise ValueError("Amazon ledger row does not match its header width; no rows were replaced") vals = list(raw[:width]) + [None] * max(0, width - len(raw)) row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals) if not buyer_column(c)} + if ledger and any(row.get(c) is not None and not re.fullmatch( + r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?", str(row[c]).strip()) + for c in ("quantity", "reconciled_quantity", "unreconciled_quantity")): + raise ValueError("Amazon ledger has a nonnumeric quantity; no rows were replaced") yield _origin(row, document_id, row_number, report_type) if attributed else row return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows() @@ -1173,6 +1521,12 @@ def _ensure(con, table, cols, rows=None, declared=None, *, preserve_rows=False): # type, that type is the authority and one sample cannot overrule it. # ⚠ A column the schema does not declare still sniffs -- that is how the union's extra fields # (the vendor's own examples/properties disagreement) get a usable type at all. + scope = _account_scope() + if scope is not None: + for column in (amazon_schema.CONNECTION_ID, amazon_schema.STORE): + if column not in cols: + cols.append(column) + declared = {**(declared or {}), amazon_schema.CONNECTION_ID: "VARCHAR", amazon_schema.STORE: "VARCHAR"} personal = set(personal_columns(table)).intersection(cols) if personal: from harness.pii_cipher import PiiCipher @@ -1309,6 +1663,10 @@ def _replace(con, table, cols, types, rows, *, source_table=None): # keeps the window all-or-nothing, and values bind and cast exactly as `executemany` did. count, batch = 0, [] for r in rows: + scope = _account_scope() + if scope is not None: + r = {**r, amazon_schema.CONNECTION_ID: str(scope["entry"]["connection_id"]), + amazon_schema.STORE: str(scope["entry"]["label"])} batch.append([cipher.encrypt(r.get(c), table=name, column=c) if c in personal else _cell(r.get(c), types.get(c, "VARCHAR")) for c in cols]) if len(batch) == per: @@ -1351,8 +1709,14 @@ def replace_window(con, table, cols, types, rows, keys, after=None): con.execute("BEGIN TRANSACTION") try: count = _replace(con, temporary, cols, types, rows, source_table=table) + inserted = cols + if table == table_name("brand_analytics") and {"_period_start", "_period_end"} <= set(keys): + _normalise_week_periods(con, target) + _normalise_week_periods(con, temporary) + inserted = _preserve_brand_week_identity(con, target, temporary, cols, keys) con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})') - con.execute(f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {temporary}") + inserted_columns = ", ".join(quote(c) for c in inserted) + con.execute(f"INSERT INTO {target} ({inserted_columns}) SELECT {inserted_columns} FROM {temporary}") if after is not None: after(con) con.execute("COMMIT") @@ -1364,6 +1728,43 @@ def replace_window(con, table, cols, types, rows, keys, after=None): con.execute(f"DROP TABLE IF EXISTS {temporary}") +def _weekly_date_sql(column): + """Read an ISO period's calendar date without interpreting unknown legacy text as a date.""" + text = f"CAST({column} AS VARCHAR)" + return (f"CASE WHEN regexp_full_match({text}, '\\d{{4}}-\\d{{2}}-\\d{{2}}([T ].+)?') " + f"AND TRY_CAST({column} AS TIMESTAMP) IS NOT NULL " + f"THEN TRY_CAST(substr({text}, 1, 10) AS DATE) END") + + +def _normalise_week_periods(con, relation): + """Canonicalize known ISO week dates without filling unknown periods or changing saved PIDs.""" + for name in ("_period_start", "_period_end"): + col = _quoted_identifier(name) + day = _weekly_date_sql(col) + con.execute(f"UPDATE {relation} SET {col} = CAST({day} AS VARCHAR) " + f"WHERE {day} IS NOT NULL AND {col} IS DISTINCT FROM CAST({day} AS VARCHAR)") + + +def _preserve_brand_week_identity(con, target, incoming, cols, keys): + """Carry prepared PIDs across calendar-equivalent report refreshes with one set-based join.""" + pid = amazon_schema.IDENTITY_PID + if pid not in {row[0] for row in con.execute(f"DESCRIBE {target}").fetchall()}: + return cols + qpid = _quoted_identifier(pid) + if pid not in cols: + con.execute(f"ALTER TABLE {incoming} ADD COLUMN {qpid} BIGINT") + con.execute(f"UPDATE {incoming} SET {qpid} = NULL") + predicate = " AND ".join(f'"old".{_quoted_identifier(c)} IS NOT DISTINCT FROM "k".{_quoted_identifier(c)}' + for c in keys) + complete = " AND ".join(f'NULLIF(CAST("k".{_quoted_identifier(c)} AS VARCHAR), \'\') IS NOT NULL' + for c in keys) + con.execute(f'UPDATE {incoming} SET {qpid} = saved.pid FROM ' + f'(SELECT "k".rowid AS rid, min("old".{qpid}) AS pid FROM {incoming} AS "k" ' + f'JOIN {target} AS "old" ON {predicate} WHERE {complete} GROUP BY "k".rowid) saved ' + f'WHERE {incoming}.rowid = saved.rid') + return [*cols, *(() if pid in cols else (pid,))] + + def _windows(start, end, size=MAX_WINDOW_DAYS): """Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden.""" out, cur = [], start @@ -1412,10 +1813,72 @@ def _period_windows(spec, start, end): def stamp_week(cols, rows, w0, w1): """Stamp a window's report period on every row, for a table whose identity names the period.""" - period = {"_period_start": _iso(w0), "_period_end": _iso(w1)} + period = {"_period_start": _utc(w0).date().isoformat(), "_period_end": _utc(w1).date().isoformat()} return [*cols, *(c for c in period if c not in cols)], _stamp_rows(rows, period) +def prepare_brand_analytics(con): + """Add nullable period fields and precompute stable numeric identities on the writer connection.""" + table = table_name("brand_analytics") + names = set(_columns_of(con, table)) + if not names: + return False + ordinal = amazon_schema.IDENTITY_OCCURRENCE + changed = False + for name in ("_period_start", "_period_end"): + if name not in names: + con.execute(f'ALTER TABLE {table} ADD COLUMN "{name}" VARCHAR') + names.add(name) + changed = True + if ordinal not in names: + con.execute(f'ALTER TABLE {table} ADD COLUMN "{ordinal}" BIGINT DEFAULT 0') + changed = True + pid = amazon_schema.IDENTITY_PID + if pid not in names: + con.execute(f'ALTER TABLE {table} ADD COLUMN "{pid}" BIGINT') + changed = True + if not changed and not con.execute(f'SELECT 1 FROM {table} WHERE "{pid}" IS NULL LIMIT 1').fetchone(): + return False + identity = amazon_schema.row_key_columns_for("brand_analytics", names) + fallback = sorted(set((*identity, *amazon_schema.row_fallback_columns_for("brand_analytics")))) + def value(name): + return f'NULLIF(CAST("{name}" AS VARCHAR), \'\')' if name in names else "NULL::VARCHAR" + complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity) + partition = ", ".join(value(name) for name in fallback) + if con.execute(f'SELECT 1 FROM {table} WHERE "{pid}" IS NULL AND NOT ({complete}) LIMIT 1').fetchone(): + # Saved ordinals survive reordered Parquet parts. New equal rows follow their largest saved ordinal. + con.execute(f'UPDATE {table} SET "{ordinal}" = ranked.occurrence FROM ' + f'(SELECT rowid AS rid, coalesce(max(CASE WHEN "{pid}" IS NOT NULL THEN "{ordinal}" END) ' + f'OVER (PARTITION BY {partition}), -1) + row_number() OVER ' + f'(PARTITION BY {partition}, "{pid}" IS NULL ORDER BY rowid) AS occurrence ' + f'FROM {table} WHERE NOT ({complete})) ranked ' + f'WHERE {table}.rowid = ranked.rid AND {table}."{pid}" IS NULL') + def digest(values): + return f"sha256(to_json(list_value({', '.join(values)})))" + fallback_digest = digest(["'__amazon_missing_identity__'", *(value(name) for name in fallback), + f'CAST("{ordinal}" AS VARCHAR)']) + digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} " + f"ELSE {fallback_digest} END") + con.execute(f'UPDATE {table} SET "{pid}" = ' + f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT)) WHERE \"{pid}\" IS NULL") + return True + + +def _weekly_period_repair(con, table, end): + """Choose an evidenced completed week or the preceding completed week with a full week of processing time.""" + saved = _read_state(con, entity_key("reconcile_" + table)) + try: + proof = json.loads(saved["detail"]) if saved and saved.get("state") == "done" else {} + start, stop = _utc(proof["start"]), _utc(proof["end"]) + if (proof.get("count", 0) > 0 and start == _week_start(start) + and stop == start + timedelta(days=7) and stop <= _week_start(end)): + return start, stop + except (KeyError, TypeError, ValueError, AttributeError, OverflowError): + pass + stop = _week_start(end) - timedelta(days=7) + return stop - timedelta(days=7), stop + + def watched_terms(tenant_key): """R18's watched set -> `(terms, None)`, or `(None, problem)` when the reader fails.""" module_name, _, name = WATCHED_TERMS.partition(":") @@ -1445,13 +1908,14 @@ def _retain_weekly(con, mirror, watched): quote = lambda name: '"' + str(name).replace('"', '""') + '"' terms = quote("amazon_watched_terms_" + uuid.uuid4().hex) target = quote(mirror) + period = _weekly_date_sql('"_period_start"') con.execute(f"CREATE TEMP TABLE {terms} (term VARCHAR)") try: if watched: con.executemany(f"INSERT INTO {terms} VALUES (?)", [[term] for term in sorted(watched)]) row = con.execute( - f'DELETE FROM {target} WHERE "_period_start" IS DISTINCT FROM ' - f'(SELECT MAX("_period_start") FROM {target}) ' + f'DELETE FROM {target} WHERE {period} IS NOT NULL AND {period} IS DISTINCT FROM ' + f'(SELECT MAX({period}) FROM {target}) ' f"AND (search_term IS NULL OR lower(trim(search_term)) NOT IN (SELECT term FROM {terms}))").fetchone() return int(row[0]) if row else 0 finally: @@ -1465,8 +1929,13 @@ def write_week(con, table, cols, types, rows, tenant_key, report, log=print): report.setdefault("problems", []).append(problem) log(problem) mirror = table_name(table) + if table == "brand_analytics": + prepare_brand_analytics(con) after = None if watched is None else (lambda c: _retain_weekly(c, mirror, watched)) - return replace_window(con, mirror, cols, types, rows, replace_keys_for(table), after=after) + result = replace_window(con, mirror, cols, types, rows, replace_keys_for(table), after=after) + if table == "brand_analytics": + prepare_brand_analytics(con) + return result def _record_brand_analytics_limit(con): @@ -1513,7 +1982,8 @@ _ASIN_RE = re.compile(r"[A-Z0-9]{10}") def _columns_of(con, table): return {r[0] for r in con.execute( - "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()} + "SELECT column_name FROM information_schema.columns WHERE table_catalog=current_database() " + "AND table_schema=current_schema() AND table_name=?", [table]).fetchall()} _NEWEST_ORDER = ("start_date", "_period_start", "_snapshot_date", "_report_row_number") @@ -1880,6 +2350,20 @@ def refresh_sku_keywords(con): summaries = "TRY_CAST(summaries AS JSON)" if "summaries" in listing_columns else "NULL::JSON" keywords = "CAST(backend_keywords AS VARCHAR)" if "backend_keywords" in listing_columns else "NULL::VARCHAR" raw_columns = _columns_of(con, table_name(SQP_TABLE)) + scoped = amazon_schema.CONNECTION_ID in listing_columns + listing_meta = ', "_connection_id", store' if scoped else "" + mapping_meta = ', l."_connection_id"' if scoped else "" + account_group = ', "_connection_id"' if scoped else "" + raw_meta = (', "_connection_id"' if amazon_schema.CONNECTION_ID in raw_columns else + ', NULL::VARCHAR AS "_connection_id"') if scoped else "" + mapping_join = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" + query_join = ' AND q."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" + count_join = ' AND c."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" + grain_meta = ', "_connection_id"' if scoped else "" + matched_meta = ', l.store' if scoped else "" + output_meta = ', "_connection_id", store' if scoped else "" + backend_meta = ', l."_connection_id", l.store' if scoped else "" + backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" raw_fields = { "start_date": "start_date", "end_date": "end_date", "asin": "asin", "search_query": "search_query_data_search_query", @@ -1898,34 +2382,34 @@ def refresh_sku_keywords(con): CREATE OR REPLACE VIEW amzn_sku_keywords AS WITH listings AS ( SELECT CAST(sku AS VARCHAR) AS sku, COALESCE({keywords}, '') AS backend_keywords, - {summaries} AS summaries + {summaries} AS summaries{listing_meta} FROM amzn_listing_items WHERE sku IS NOT NULL AND CAST(sku AS VARCHAR) <> '' ), mapping AS ( - SELECT DISTINCT l.sku, json_extract_string(j.value, '$.asin') AS asin + SELECT DISTINCT l.sku, json_extract_string(j.value, '$.asin') AS asin{mapping_meta} FROM listings l, json_each(l.summaries) j WHERE NULLIF(json_extract_string(j.value, '$.asin'), '') IS NOT NULL ), mapped_counts AS ( - SELECT asin, COUNT(DISTINCT sku) AS mapped_sku_count FROM mapping GROUP BY asin - ), raw AS (SELECT {select}{raw_from}), queries AS ( - SELECT *, CAST(to_json([start_date, end_date, asin, search_query]) AS VARCHAR) AS source_grain + SELECT asin{account_group}, COUNT(DISTINCT sku) AS mapped_sku_count FROM mapping GROUP BY asin{account_group} + ), raw AS (SELECT {select}{raw_meta}{raw_from}), queries AS ( + SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain FROM raw ), matched AS ( - SELECT l.sku, q.*, l.backend_keywords, c.mapped_sku_count - FROM listings l JOIN mapping m ON m.sku=l.sku - JOIN queries q ON q.asin=m.asin JOIN mapped_counts c ON c.asin=m.asin + SELECT l.sku, q.*, l.backend_keywords, c.mapped_sku_count{matched_meta} + FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join} + JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join} ) SELECT md5(CAST(to_json([sku, source_grain]) AS VARCHAR)) AS keyword_id, sku, asin, substr(start_date, 1, 7) || '-01' AS month, start_date, end_date, search_query, impressions, clicks, cart_adds, purchases, backend_keywords, source_grain, mapped_sku_count, 'Amazon ASIN/query metrics shared across ' || CAST(mapped_sku_count AS VARCHAR) || - ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis + ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis{output_meta} FROM matched UNION ALL - SELECT md5(CAST(to_json([l.sku, 'backend_keywords']) AS VARCHAR)), + SELECT md5(CAST(to_json([l.sku, 'backend_keywords'{', l."_connection_id"' if scoped else ''}]) AS VARCHAR)), l.sku, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, l.backend_keywords, NULL, NULL, - 'Listing backend keywords only. No matching Search Query Performance rows.' - FROM listings l WHERE NOT EXISTS (SELECT 1 FROM matched m WHERE m.sku=l.sku) + 'Listing backend keywords only. No matching Search Query Performance rows.'{backend_meta} + FROM listings l WHERE NOT EXISTS (SELECT 1 FROM matched m WHERE m.sku=l.sku{backend_match}) """) refresh_keyword_ranks(con) return True @@ -1938,24 +2422,32 @@ def refresh_keyword_ranks(con): return False summaries = ("TRY_CAST(l.summaries AS JSON)" if "summaries" in _columns_of(con, "amzn_listing_items") else "NULL::JSON") + scoped = amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_sku_keywords") + partition_meta = ', k."_connection_id"' if scoped else "" + matched_meta = ', k."_connection_id" AS _rank_connection_id, k.store AS _rank_store' if scoped else "" + observation_match = (' AND r."_connection_id" IS NOT DISTINCT FROM k."_connection_id"' + if scoped and amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_apify_rank_observations") else "") + listing_match = ' AND l."_connection_id" IS NOT DISTINCT FROM k."_connection_id"' if scoped else "" + rank_meta = ', _rank_connection_id' if scoped else "" + output_meta = ', _rank_connection_id AS "_connection_id", _rank_store AS store' if scoped else "" con.execute(f""" CREATE OR REPLACE VIEW amzn_keyword_ranks AS WITH matched AS ( - SELECT r.*, k.keyword_id, k.sku, - row_number() OVER (PARTITION BY r.observation_id, k.sku + SELECT r.*, k.keyword_id, k.sku{matched_meta}, + row_number() OVER (PARTITION BY r.observation_id, k.sku{partition_meta} ORDER BY k.start_date DESC, k.keyword_id) AS recency FROM amzn_apify_rank_observations r JOIN amzn_sku_keywords k ON ((r.asin=k.asin AND lower(trim(r.keyword))=lower(trim(k.search_query))) OR (k.search_query IS NULL AND EXISTS ( SELECT 1 FROM amzn_listing_items l, json_each({summaries}) j - WHERE l.sku=k.sku AND json_extract_string(j.value, '$.asin')=r.asin) + WHERE l.sku=k.sku AND json_extract_string(j.value, '$.asin')=r.asin{listing_match}) AND EXISTS (SELECT 1 FROM unnest(string_split(k.backend_keywords, ';')) t(term) WHERE lower(trim(term))=lower(trim(r.keyword))))) - AND (k.start_date <= r.date OR k.start_date IS NULL) + AND (k.start_date <= r.date OR k.start_date IS NULL){observation_match} ) - SELECT sha256(CAST(to_json([observation_id, keyword_id]) AS VARCHAR)) AS rank_id, + SELECT sha256(CAST(to_json([observation_id, keyword_id{rank_meta}]) AS VARCHAR)) AS rank_id, observation_id, keyword_id, sku, keyword, asin, date, organic_rank, - sponsored_rank, global_position, is_sponsored, sponsored_rank_basis, dataset_id + sponsored_rank, global_position, is_sponsored, sponsored_rank_basis, dataset_id{output_meta} FROM matched WHERE recency=1 """) return True @@ -1967,7 +2459,7 @@ def import_keyword_ranks(con, rows, dataset_id): fields = {field["name"]: field["type"] for field in AMAZON_FIELDS["ut_amzn_keyword_ranks"] - if field["name"] not in {"rank_id", "keyword_id", "sku"}} + if field["name"] not in {"rank_id", "keyword_id", "sku", amazon_schema.STORE}} fields["source_json"] = "VARCHAR" if not dataset_id or any(row.get("dataset_id") != dataset_id for row in rows): raise ValueError("Amazon rank dataset attribution is missing or inconsistent") @@ -2141,7 +2633,7 @@ def _runtime_for(tenant_key): return None -def credential(tenant_key="royal-imports", rt=None): +def credential(tenant_key="royal-imports", rt=None, connection_id=None): """The tenant's stored Amazon credential, or None. ⛔ THE KEYCHAIN IS THE ONLY RUNTIME SOURCE, AND THERE IS NO ENVIRONMENT FALLBACK. `AMAZON_SP_*` @@ -2160,7 +2652,9 @@ def credential(tenant_key="royal-imports", rt=None): f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}") try: import core.keychain as kc - fields = kc.amazon_creds(runtime) + scope = _account_scope() + connection_id = connection_id or (scope["entry"]["connection_id"] if scope else None) + fields = kc.amazon_creds(runtime, connection_id) if connection_id else kc.amazon_creds(runtime) if runtime is not None: # Delivery clocks keep running while credentials remain stored or cannot decrypt. # Claim the interval before work, including failures; no recursion via credential(). @@ -2207,6 +2701,11 @@ def _connect_tenant_mirror(tenant_key, rt=None): credentialed Amazon sync can write its first row. """ path = _tenant_mirror_path(tenant_key, rt) + scope = _account_scope() + if scope is not None: + if tenant_key != scope["tenant"] or path != scope["path"]: + raise RuntimeError("Amazon account workspace does not match the tenant") + return scope["root"].cursor(), path candidate = datastore._candidate_connection(tenant_key, path) if candidate is not None: return candidate, path @@ -2576,7 +3075,7 @@ def cancel_deletion_trigger(tenant_key, *, rt=None, occurred_at=None, principal= return cancelled -def scheduled_purge(tenant_key, rt, *, now=None): +def scheduled_purge(tenant_key, rt, *, now=None, ignore_credential_loss=False): """Read-only policy schedule for the earliest uncancelled, unpurged mirror obligation.""" key, clock = _tenant_key(tenant_key), _utc(now) ledger = _ledger(key, rt) @@ -2584,6 +3083,8 @@ def scheduled_purge(tenant_key, rt, *, now=None): if r["tenant"] == key and r["kind"] == "purge" and r["success"] and _utc(r["checked_at"]) <= clock] for trigger in _active_deletion_triggers(key, ledger, clock): + if ignore_credential_loss and trigger["kind"] in ("revocation", "authorization_loss"): + continue when = _utc(trigger["occurred_at"]) if not any(stamp >= when for stamp in checks): return {"purgeAt": _control_stamp(when + timedelta(days=PURGE_AFTER_DAYS)), @@ -2864,7 +3365,7 @@ def _amazon_objects(con): """Every live mirror object reserved to Amazon by the `amzn_` naming contract.""" rows = con.execute( "SELECT table_name, table_type FROM information_schema.tables " - "WHERE table_schema='main'").fetchall() + "WHERE table_catalog=current_database() AND table_schema='main'").fetchall() out = [] for name, kind in rows: name = str(name) @@ -2922,6 +3423,16 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): def _connect(self, tenant_key, rt): return _connect_tenant_mirror(tenant_key, rt) + def restore_mirror(self, tenant_key, rt, log=None): + if rt is not None and not getattr(_ACCOUNT, "purging", False): + obligations = rt.get("amazon_connection_revocations", fresh=True) + if obligations is None: + obligations = {} + if not isinstance(obligations, dict) or any( + not isinstance(row, dict) or row.get("state") != "done" for row in obligations.values()): + raise RuntimeError("Amazon connection cleanup is unfinished; retry its scoped deletion before restoring or syncing") + return super().restore_mirror(tenant_key, rt, log=log) + def _validated_snapshot(self, *args, **kwargs): result = super()._validated_snapshot(*args, **kwargs) if not any(personal_columns(name) for name in result[2]): @@ -2934,6 +3445,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print, **kwargs): changed_tables = tuple(changed_tables) + if table_name("brand_analytics") in changed_tables: + prepare_brand_analytics(con) for name in changed_tables: _assert_personal_storage(con, name) return super().persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log, **kwargs) @@ -2963,8 +3476,10 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): partner = {pairs[name]: name for name in unfinished if name in pairs and pairs[name] not in unfinished} unfinished |= set(partner) owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))} + # An account failure holds the combined source, including successful sibling tables. + multi_account = len(report.get("accounts") or {}) > 1 blocked = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or [] - if str(problem).split(":", 1)[0].strip() not in owners] + if multi_account or str(problem).split(":", 1)[0].strip() not in owners] held = {} for name in sorted(unfinished): if name in partner: @@ -2985,6 +3500,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): names = sorted({table_name(t) for t in TABLES} | tables, key=lambda name: (-len(name), name)) def owner(entity): + if entity.startswith("amzn_account_") and "__" in entity: + entity = entity.split("__", 1)[1] if entity.startswith(entity_key("settlement_doc_")): return table_name("settlements") for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_", RESYNC_PREFIX): @@ -3007,6 +3524,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): _heal_snapshots(con, result, log) _heal_quoted_values(con, result, log) _heal_identity_duplicates(con, result, log) + if prepare_brand_analytics(con): + result["dirty"].add(table_name("brand_analytics")) refresh_sku_keywords(con) def _heal_restored(self, con, result, log): @@ -3014,6 +3533,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): healed = {"dirty": set(), "problems": result["problems"]} for heal in (_heal_snapshots, _heal_quoted_values, _heal_identity_duplicates): heal(con, healed, log) + prepare_brand_analytics(con) def _restored_state_rows(self, rows, tables, refused): return _restored_state_rows(rows, tables, refused) @@ -3050,6 +3570,8 @@ _validated_snapshot = _MIRROR_STORE._validated_snapshot def restore_mirror(tenant_key, rt, log=None): + if _account_scope() is not None: + return {"state": "scoped", "dirty": set(), "problems": []} result = _MIRROR_STORE.restore_mirror(tenant_key, rt, log=log) if result["state"] != "refused": con, _path = _connect_tenant_mirror(tenant_key, rt) @@ -3062,6 +3584,11 @@ def restore_mirror(tenant_key, rt, log=None): def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print): + scope = _account_scope() + if scope is not None: + scope["changed"].update(changed_tables) + prepare_brand_analytics(con) + return None return _MIRROR_STORE.persist_mirror( tenant_key, rt, con, changed_tables, report=report, log=log) @@ -3078,6 +3605,8 @@ def _restored_state_rows(rows, tables, refused): def keep(row): entity = row[0] + if entity.startswith("amzn_account_") and "__" in entity: + return keep((entity.split("__", 1)[1], *row[1:])) if entity in shared or ("amzn_settlements" in refused and entity.startswith("amzn_settlement_doc_")): return False if entity.startswith("amzn_pending_report_"): @@ -3127,6 +3656,31 @@ def _collapse_snapshots(con, log=None): continue scope = ("COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" and "inventory_source" in cols else "TRUE") + if amazon_schema.CONNECTION_ID in cols: + before = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] + for (cid,) in con.execute(f'SELECT DISTINCT "_connection_id" FROM {mirror}').fetchall(): + value = "NULL" if cid is None else "'" + str(cid).replace("'", "''") + "'" + account_scope = f'{scope} AND "_connection_id" IS NOT DISTINCT FROM {value}' + dates = [r[0] for r in con.execute(f'SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) ' + f'FROM {mirror} WHERE {account_scope}').fetchall()] + present = sorted(d for d in dates if d is not None) + if len(dates) <= 1 or not present: + continue + keep = present[-1:] + if table == "inventory": + key = entity_key("reconcile_" + table) + saved = _read_state(con, (_account_prefix(cid) if not _account_scope() and cid else "") + key) + try: + marked = json.loads(saved["detail"])["dates"] if saved and saved.get("detail") else {} + except (ValueError, KeyError, TypeError): + marked = {} + keep = [d for d in present if d in marked] or keep + con.execute(f'DELETE FROM {mirror} WHERE {account_scope} AND (_snapshot_date IS NULL ' + f'OR CAST(_snapshot_date AS VARCHAR) NOT IN ({",".join("?" for _ in keep)}))', keep) + after = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] + if before != after: + collapsed[mirror] = (before, after, 0) + continue dates = [row[0] for row in con.execute( f"SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) FROM {mirror} WHERE {scope}").fetchall()] present = sorted(d for d in dates if d is not None) @@ -3211,7 +3765,8 @@ def _unquote_mirror_values(con, log=None): types = dict(con.execute( "SELECT column_name, data_type FROM information_schema.columns " "WHERE table_schema='main' AND table_name=?", [mirror]).fetchall()) - text = [c for c, t in types.items() if str(t).upper() == "VARCHAR"] + text = [c for c, t in types.items() if str(t).upper() == "VARCHAR" + and c not in (amazon_schema.CONNECTION_ID, amazon_schema.STORE)] identity = [c for c in _QUOTE_IDENTITY if c in text] if not identity: continue @@ -3244,9 +3799,49 @@ def _unquote_mirror_values(con, log=None): if resync: log(f"Amazon resync {mirror}: {resync[0]:,} rows lost cells to the old parse; " f"{len(resync[1])} report windows from {resync[1][0][0]} download again on the next passes") + if _queue_invalid_ledger_rows(con, log): + healed.setdefault(table_name("inventory_ledger_detail"), 0) return healed +_INVALID_LEDGER_QUANTITY = "quantity IS NOT NULL AND TRY_CAST(quantity AS DOUBLE) IS NULL" + + +def ledger_parse_diagnosis(con): + """Read invalid ledger counts and their evidenced report windows without exposing row values or writing.""" + mirror = table_name("inventory_ledger_detail") + cols = _columns_of(con, mirror) + if "quantity" not in cols: + return {"invalid_rows": 0, "undated_rows": 0, "windows": []} + invalid = con.execute(f"SELECT count(*) FROM {mirror} WHERE {_INVALID_LEDGER_QUANTITY}").fetchone()[0] + if not {"_period_start", "_period_end"} <= cols: + return {"invalid_rows": invalid, "undated_rows": invalid, "windows": []} + found = con.execute( + f"SELECT CAST(_period_start AS VARCHAR), CAST(_period_end AS VARCHAR), count(*) FROM {mirror} " + f"WHERE {_INVALID_LEDGER_QUANTITY} GROUP BY 1, 2 ORDER BY 1, 2").fetchall() + return {"invalid_rows": invalid, + "undated_rows": sum(n for start, end, n in found if start is None or end is None), + "windows": [(start, end) for start, end, _n in found if start is not None and end is not None]} + + +def _queue_invalid_ledger_rows(con, log=None): + """Queue only evidenced windows of already-unquoted shifted ledger rows, once per queue change.""" + diagnosis = ledger_parse_diagnosis(con) + if not diagnosis["windows"]: + return False + mirror = table_name("inventory_ledger_detail") + key = entity_key(RESYNC_PREFIX + "inventory_ledger_detail") + def markers(): + return [row for row in _sync_state_rows(con) if row[0] == key or row[0].endswith("__" + key)] + before = markers() + queued = _queue_resync(con, mirror, _columns_of(con, mirror), _INVALID_LEDGER_QUANTITY) + changed = before != markers() + if changed and queued and log: + log(f"Amazon ledger parse repair: {diagnosis['invalid_rows']:,} invalid rows; " + f"{len(diagnosis['windows'])} evidenced report windows queued for resync") + return changed + + def _unhealable_sql(types, text): """SQL true for a quoted row the unquote cannot repair, or "" when no such test applies. @@ -3271,6 +3866,31 @@ def _queue_resync(con, mirror, types, where): if (table not in PERIOD_TABLES or not {"_period_start", "_period_end"} <= set(types) or (_client().REPORT_TYPES.get(table) or {}).get("snapshot")): return None + if amazon_schema.CONNECTION_ID in types and _account_scope() is None: + results = [] + for (cid,) in con.execute(f'SELECT DISTINCT "_connection_id" FROM {_quoted_identifier(mirror)} ' + f'WHERE ({where})').fetchall(): + if cid is None: + continue + literal = "'" + str(cid).replace("'", "''") + "'" + unscoped = ({k: v for k, v in types.items() if k != amazon_schema.CONNECTION_ID} + if hasattr(types, "items") else set(types) - {amazon_schema.CONNECTION_ID}) + result = _queue_resync(con, mirror, unscoped, + f'({where}) AND "_connection_id"={literal}') + if result: + key = entity_key(RESYNC_PREFIX + table) + row = _read_state(con, key) + scoped_key = _account_prefix(cid) + key + saved = _read_state(con, scoped_key) + windows = {tuple(w) for w in json.loads(row["detail"])["windows"]} + if saved: + windows |= {tuple(w) for w in json.loads(saved["detail"])["windows"]} + detail = json.dumps({"windows": sorted(windows)}) + if not saved or (saved["state"], saved["rows"], saved["detail"]) != ("pending", result[0], detail): + _record_state(con, scoped_key, "pending", result[0], detail) + con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) + results.append((result[0], windows)) + return (sum(n for n, _w in results), sorted(set().union(*(w for _n, w in results)))) if results else None found = con.execute( f"SELECT CAST(_period_start AS VARCHAR), CAST(_period_end AS VARCHAR), count(*) " f"FROM {_quoted_identifier(mirror)} WHERE ({where}) AND _period_start IS NOT NULL " @@ -3285,7 +3905,9 @@ def _queue_resync(con, mirror, types, where): queued = set() windows = sorted(queued | {(start, end) for start, end, _n in found}) rows = sum(n for _s, _e, n in found) - _record_state(con, key, "pending", rows, json.dumps({"windows": [list(w) for w in windows]})) + detail = json.dumps({"windows": [list(w) for w in windows]}) + if not saved or (saved["state"], saved["rows"], saved["detail"]) != ("pending", rows, detail): + _record_state(con, key, "pending", rows, detail) return rows, windows @@ -3338,13 +3960,17 @@ def _collapse_identity(con, log=None): continue cols = _columns_of(con, mirror) keys = [c for c in replace_keys_for(table) if c in cols] + if amazon_schema.CONNECTION_ID in cols: + keys.insert(0, amazon_schema.CONNECTION_ID) if not keys or "_report_document_id" not in cols: continue q, target = _quoted_identifier, _quoted_identifier(mirror) part = ", ".join(q(c) for c in keys) doomed = (f"SELECT rid FROM (SELECT rid, d, first_value(d) OVER (PARTITION BY {part} " f"ORDER BY landed DESC, d DESC) AS keep FROM (SELECT rowid AS rid, {part}, " - f"_report_document_id AS d, max(rowid) OVER (PARTITION BY _report_document_id) AS landed " + f"_report_document_id AS d, max(rowid) OVER (PARTITION BY " + + ('"_connection_id", ' if amazon_schema.CONNECTION_ID in cols else "") + + "_report_document_id) AS landed " f"FROM {target})) WHERE d IS DISTINCT FROM keep") if "_period_start" in keys: same = " AND ".join(f"o.{q(c)} IS NOT DISTINCT FROM t.{q(c)}" for c in keys @@ -3740,10 +4366,11 @@ def _client(): def _record_state(con, key, phase, rows=0, detail=None): """Write `_sync_state` in its declared column order, never by a misleading positional guess.""" + payload_key = str(key).split("__", 1)[-1] if str(key).startswith("amzn_account_") else str(key) con.execute( "INSERT OR REPLACE INTO _sync_state " "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", - [str(key), str(phase), None, (str(detail) if str(key).startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_", + [str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_", "amzn_resync_")) else str(detail)[:300]) if detail else None, int(rows or 0), _iso(datetime.now(timezone.utc))]) @@ -3817,6 +4444,8 @@ def _first_load_due(con): def _restore_first_load_wait(tenant_key, con): from core import connector_sync due = _first_load_due(con) + if _account_scope() is not None: + return due connector_sync.defer_sync("amazon_sp", tenant_key, due.timestamp() if due else None) return due @@ -4178,8 +4807,32 @@ def _sync_event(kind, tenant, table, detail, rt, report, log): log(message) +def _sync_failure_context(exc, table): + """Keep unreadable-value locations without exception values, source text or frame locals.""" + if not isinstance(exc, (KeyError, ValueError)): + return None + frame = exc.__traceback__ + while frame and frame.tb_next: + frame = frame.tb_next + return {"classifier": "unreadable_value", "table": table, + "file": os.path.basename(frame.tb_frame.f_code.co_filename) if frame else None, + "line": frame.tb_lineno if frame else None, + "function": frame.tb_frame.f_code.co_name if frame else None} + + def _sync_problem(exc, table, operation, tenant, rt, report, log): from harness.amazon_client import AmazonThrottled, AmazonOperationRefused, AmazonReportPending + failure = _sync_failure_context(exc, table) + if failure: + location = (f"{failure['file']}:{failure['line']}:{failure['function']}" if failure["file"] + else "raising frame unavailable") + fix = "check the connector's expected response fields and value types, then retry sync" + message = (f"{table}: {operation} REFUSED: {type(exc).__name__}: " + f"unreadable_value at {location}; fix: {fix}") + report["problems"].append(message) + log(message) + return {"state": "refused", "reason": f"Amazon could not read {table} at {location}. {fix}.", + "diagnostic": message, "amazonStatus": "REFUSED", "error": "", "failure": failure} cause = f"{type(exc).__name__}: {exc}" if isinstance(exc, AmazonReportPending): message = (f"{table}: Amazon report is {exc.status}; its saved report ID will be polled on the next pass. " @@ -5117,7 +5770,16 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None max_wait=900, *, scheduled=False, first_load=False, report_budget=None, refresh_loaded=False): """Publish a whole Amazon source after its requested work completes.""" rt = rt if rt is not None else _runtime_for(tenant_key) - _sync_credential(tenant_key, rt) + entries = account_entries(tenant_key, rt) + admitted = False + for entry in entries: + try: + _sync_credential(tenant_key, rt, connection_id=entry["connection_id"]) + admitted = True + except Exception: + continue + if not admitted: + raise RuntimeError("Amazon sync requires a stored production amazon_sp credential; SANDBOX credentials cannot sync") if (tables is None and os.environ.get("AIOS_AMAZON_BRAND_ANALYTICS") != "1" and os.environ.get("AIOS_PROCESS_ROLE") != "worker"): # Large weekly reports belong to the dedicated sync worker. @@ -5127,14 +5789,14 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None scheduled=scheduled, first_load=first_load, report_budget=report_budget, refresh_loaded=refresh_loaded) if getattr(datastore._SYNC_TLS, "candidate", None) is not None: - return _sync_candidate(tenant_key, **options) + return _sync_accounts_candidate(tenant_key, **options) report = {} expected = [name for name in (TABLES if tables is None else tables) if name in TABLES] if scheduled and not first_load and expected and "settlements" not in expected: expected.append("settlements") _PUBLISHED.flag = False with _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected): - report.update(_sync_candidate(tenant_key, **options)) + report.update(_sync_accounts_candidate(tenant_key, **options)) _refuse_unreconciled(tenant_key, rt, report, log) if getattr(_PUBLISHED, "flag", False): _after_sync(tenant_key, report, log) @@ -5172,6 +5834,8 @@ def _after_sync(tenant_key, report, log): def _refuse_unreconciled(tenant_key, rt, report, log): """Standing rule 8: a table synced in this pass whose reconciliation refuses keeps its previous rows.""" + if "accounts" in report: + return # Each isolated account was reconciled before it joined the combined candidate. synced = _synced_tables(report) if not synced or report.get("state") in {"partial", "failed", "done"}: return @@ -5193,9 +5857,9 @@ def _refuse_unreconciled(tenant_key, rt, report, log): log(f"{table}: {reason}") -def _sync_credential(tenant_key, rt): +def _sync_credential(tenant_key, rt, connection_id=None): """Refuse unusable credentials before storage or vendor work at either sync entry.""" - fields = credential(tenant_key, rt) + fields = credential(tenant_key, rt, connection_id=connection_id) if not fields: raise RuntimeError( f"tenant `{tenant_key}` has no stored `amazon_sp` credential. Add one under " @@ -5222,6 +5886,109 @@ def _sync_credential(tenant_key, rt): return fields +def _summarize_account_table(row, total_accounts): + """Sum all reported resync progress while retaining each account's window and safe failure details.""" + accounts = row["accounts"] + progress = [r["resync"] for r in accounts.values() if isinstance(r.get("resync"), dict)] + if progress: + row["resync"] = {key: sum(p.get(key, 0) for p in progress) for key in ("landed", "left")} + if total_accounts > 1: + complete = sum(_mirrors._table_sync_complete("amazon", r) for r in accounts.values()) + summary = [f"{len(accounts)} of {total_accounts} stores reported", + f"{complete} complete, {total_accounts - complete} incomplete"] + if progress: + summary.append(f"resync: {row['resync']['landed']} windows landed, {row['resync']['left']} left") + row["window"] = "; ".join(summary) + + +def _completed_account_tables(con, result, tables): + """Retain this account's completed first-load tables when a later pass only reports its remaining tasks.""" + if result.get("state") != "done": + return + for table in TABLES if tables is None else tables: + if table not in TABLES or table in result["tables"]: + continue + state = _read_state(con, entity_key(table)) + if not state or state["state"] != "done": + continue + columns = _columns_of(con, table_name(table)) + if not columns: + continue + result["tables"][table] = {"state": "done", "rows": con.execute( + f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0], + "columns": len(columns), "provenance": amazon_schema.PROVENANCE[table], + "reason": "This connection completed this table in an earlier first-load pass"} + + +def _sync_accounts_candidate(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, + max_wait=900, *, scheduled=False, first_load=False, report_budget=None, refresh_loaded=False): + """Attempt every admitted account before publishing one combined source.""" + rt = rt if rt is not None else _runtime_for(tenant_key) + entries = account_entries(tenant_key, rt) + restored = restore_mirror(tenant_key, rt, log=log) + con, _path = _connect_tenant_mirror(tenant_key, rt) + changed = _adopt_legacy_account(con, entries[0]) + report = {"tenant": tenant_key, "tables": {}, "accounts": {}, "started": _iso(_utc()), + "problems": list(restored["problems"]), "audit_problems": []} + try: + for entry in entries: + cid = entry["connection_id"] + try: + with _account_workspace(con, tenant_key, rt, entry) as scope: + _sync_credential(tenant_key, rt, connection_id=cid) + result = _sync_candidate(tenant_key, tables=tables, since=since, log=log, rt=rt, + max_wait=max_wait, scheduled=scheduled, first_load=first_load, + report_budget=report_budget, refresh_loaded=refresh_loaded) + if first_load: + _completed_account_tables(scope["root"], result, tables) + _refuse_unreconciled(tenant_key, rt, result, log) + changed.update(_merge_account(con, scope)) + report["accounts"][cid] = {"store": entry["label"], **result} + except Exception as exc: + # Credential and vendor exception text can contain source secrets. + report["accounts"][cid] = {"store": entry["label"], "state": "failed", "tables": {}, + "problems": [f"Amazon account sync failed ({type(exc).__name__})"]} + for cid, result in report["accounts"].items(): + for key in ("problems", "audit_problems"): + report[key].extend(result.get(key, ())) + for table, row in result.get("tables", {}).items(): + aggregate = report["tables"].setdefault(table, {**row, "rows": 0, "accounts": {}}) + aggregate["accounts"][cid] = row + aggregate["rows"] += row.get("rows") or 0 + if not _mirrors._table_sync_complete("amazon", row): + aggregate.update({k: v for k, v in row.items() if k != "rows"}) + aggregate["state"] = row.get("state") or "failed" + expected = list(TABLES if tables is None else tables) + for table in expected: + row = report["tables"].setdefault(table, {"rows": 0, "state": "failed", "accounts": {}}) + if any(cid not in row["accounts"] for cid in (e["connection_id"] for e in entries)): + row.update(state="failed", reason="An Amazon account did not complete this table") + for row in report["tables"].values(): + _summarize_account_table(row, len(entries)) + if first_load: + states = [r.get("state") for r in report["accounts"].values()] + report.update(state="failed" if "failed" in states else "partial" if "partial" in states else "done", + remaining=sum(r.get("remaining", 0) for r in report["accounts"].values()), + landedTasks=sum(r.get("landedTasks", 0) for r in report["accounts"].values()), + etaSeconds=sum(r.get("etaSeconds", 0) for r in report["accounts"].values())) + waits = [r["nextDueSeconds"] for r in report["accounts"].values() if r.get("nextDueSeconds")] + if waits: + report["nextDueSeconds"] = min(waits) + _refresh_account_status(con, entries, rt) + _restore_first_load_wait(tenant_key, con) + report["available"] = all(result.get("available", False) for result in report["accounts"].values()) + if first_load: + progress = first_load_status(tenant_key, rt) or {} + report["firstLoad"] = {**progress, "state": report["state"]} + report["remaining"] = progress.get("remaining", report["remaining"]) + refresh_sku_keywords(con) + persist_mirror(tenant_key, rt, con, changed, report=report, log=log) + report["finished"] = _iso(_utc()) + return report + finally: + con.close() + + def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, max_wait=900, *, scheduled=False, first_load=False, report_budget=None, refresh_loaded=False): """Pull into private progress; refresh loaded tables before bounded history and waits.""" @@ -5369,6 +6136,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri except Exception as exc: # noqa: BLE001 report["problems"].append(f"Amazon identity collapse failed ({type(exc).__name__}); the next " "resync of each doubled table replaces its earlier document's rows.") + if prepare_brand_analytics(con): + changed_tables.add(table_name("brand_analytics")) def write_window(table, cols, types, rows): if cx.REPORT_TYPES[table].get("weekly"): @@ -5514,14 +6283,18 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri continue prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync") root = cx.REPORT_TYPES[table].get("json_root") - if not first_load and cx.REPORT_TYPES[table].get("weekly") and not { - "_period_start", "_period_end"} <= set(_columns_of(con, mirror)): - # Legacy rows have no trustworthy week. Fetch a real completed period before replacing them. - repair_weeks = _week_windows(end - timedelta(days=21), end) - if repair_weeks: - repair_start, repair_end = repair_weeks[-1] + # History completion does not prove that legacy rows carry a report period. + period_repair_mark = None + if not first_load and cx.REPORT_TYPES[table].get("weekly"): + names = set(_columns_of(con, mirror)) + dated = ({"_period_start", "_period_end"} <= names and con.execute( + f'SELECT 1 FROM {mirror} WHERE "_period_start" IS NOT NULL ' + 'AND "_period_end" IS NOT NULL LIMIT 1').fetchone()) + if not dated: + period_repair_mark = read_watermark(con, table) + repair_start, repair_end = _weekly_period_repair(con, table, end) plan = {**plan, "start": repair_start, "end": repair_end, - "cause": "repair missing period fields from the latest completed week"} + "cause": "repair missing period fields from a completed earlier week"} log(f"{table}: {plan['cause']}") sibling = SHARED_REPORT_SIBLINGS.get(table) sibling_plan = None @@ -5897,6 +6670,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri _record_state(con, entity_key("sync_tick_" + table), "done", total_rows) if not any(v["state"] in ("refused", "throttled") for v in operations.values()): mark = settlement_mark if table == "settlements" else plan["end"] + if mark is not None and period_repair_mark is not None: + # Repairing an earlier week cannot rewind completed history. + mark = max(mark, period_repair_mark) has_window = (cx.REPORT_TYPES[table].get("snapshot") or table in ("settlements", SQP_TABLE) or landed_window or (first_load and floors[table] is not None)) if mark is not None and has_window: @@ -5978,6 +6754,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri resync_left = len(queued_resyncs(con, table)) report["tables"].setdefault(table, {})["resync"] = { "landed": resync_count, "left": resync_left} + report["tables"][table].setdefault("window", plan["cause"]) log(f"Amazon resync {table}: {resync_count} windows landed, {resync_left} left") if previous_proof: proof = _read_state(con, proof_key) or {} @@ -6073,6 +6850,35 @@ def _first_row_per_order(rows): def validate(tenant_key="royal-imports", rt=None, log=print): + """Reconcile each connection against its own source summaries before combining verdicts.""" + if _account_scope() is not None: + return _validate_account(tenant_key, rt=rt, log=log) + runtime = rt if rt is not None else _runtime_for(tenant_key) + con, _path = _connect_tenant_mirror(tenant_key, runtime) + try: + if not _read_state(con, "amzn_account_binding"): + return _validate_account(tenant_key, rt=runtime, log=log) + entries = account_entries(tenant_key, runtime) + results = {} + for entry in entries: + with _account_workspace(con, tenant_key, runtime, entry): + results[entry["connection_id"]] = _validate_account(tenant_key, rt=runtime, log=log) + out = {} + for table in DATABASE_TABLES: + accounts = {cid: rows[table] for cid, rows in results.items()} + if len(accounts) == 1: + out[table] = {**next(iter(accounts.values())), "accounts": accounts} + continue + state = ("refused" if any(row["state"] == "refused" for row in accounts.values()) else + "ok" if all(row["state"] == "ok" for row in accounts.values()) else "unproven") + out[table] = {"state": state, "rows": sum(row.get("rows", 0) for row in accounts.values()), + "accounts": accounts} + return out + finally: + con.close() + + +def _validate_account(tenant_key="royal-imports", rt=None, log=print): """⭐⭐ R3 — RECONCILE EACH TABLE TO AMAZON'S OWN SUMMARY, NOT TO ODOO. -> a report dict. Standing rule 8 says a number that does not tie to an independent aggregate does not ship. @@ -6162,6 +6968,7 @@ def validate(tenant_key="royal-imports", rt=None, log=print): out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")} for t, r in out.items(): log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}") + con.close() return out @@ -6214,6 +7021,15 @@ class _FixtureMirrorRuntime: getattr(self, "blobs", {}).pop(path, None) +def _fixture_account_cursor(con, rt): + """Lend owned cursors from the active account, publication candidate or fixture publication in that order.""" + scope = _account_scope() + if scope is not None: + return scope["root"].cursor(), rt.datastore_path + candidate = datastore._candidate_connection(rt.key, rt.datastore_path) + return (candidate if candidate is not None else con.cursor()), rt.datastore_path + + def snapshot_selftest(log=print, check=None): """S66-T47: real sync with in-memory storage and fixture-only vendor transports.""" # Window tests inspect private progress. S73-T32 separately exercises public atomic publication. @@ -7462,8 +8278,13 @@ def margin_selftest(log=print, check=None): return end.astimezone(tz) class Runtime(_FixtureMirrorRuntime): key = "margin-fixture" - def get(self, *_args, **_kwargs): + def get(self, name, *_args, **_kwargs): + if name == "keychain": + return {"entries": {"margin-entry": { + "connection_id": "margin-account", "type": "amazon_sp", "slot": 1, + "label": "Margin store"}}} return None + rt = Runtime() def dynamic_economics(_fields, date_start, date_end, **_kwargs): row = {**deepcopy(economy), "startDate": date_start, "endDate": date_start} return json.dumps(row), {"reportDocumentId": "economics-" + date_start} @@ -7477,23 +8298,28 @@ def margin_selftest(log=print, check=None): module = sys.modules[__name__] with patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 40), \ patch.object(module, "credential", return_value={"fixture": True}), \ - patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \ + patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: + _fixture_account_cursor(con, rt)), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), \ patch.object(cx, "economics_document", side_effect=dynamic_economics) as kiosk, \ patch.object(cx, "transactions", side_effect=dynamic_transactions) as finances, \ patch.object(cx, "report_document", side_effect=fake_report) as reports: - first = sync(Runtime.key, tables=MARGIN_TABLES, rt=Runtime(), first_load=True, report_budget=2, log=lambda *_: None) + first = sync(Runtime.key, tables=MARGIN_TABLES, rt=rt, first_load=True, report_budget=2, log=lambda *_: None) ck("margin API windows consume the first-load pass budget", first["state"] == "partial" and kiosk.call_count == 2 and finances.call_count == 1 and reports.call_count == 0) for _ in range(3): - last = sync(Runtime.key, tables=MARGIN_TABLES, rt=Runtime(), first_load=True, report_budget=2, log=lambda *_: None) + last = sync(Runtime.key, tables=MARGIN_TABLES, rt=rt, first_load=True, report_budget=2, log=lambda *_: None) + checked = validate(Runtime.key, rt=rt, log=lambda *_: None) ck("margin bounded passes resume to completion across all five table kinds", last["state"] == "done" and last["remaining"] == 0 and all(read_watermark(con, t) for t in MARGIN_TABLES) - and all(_validate_margin(con, t)["state"] == "ok" for t in MARGIN_TABLES)) + and all(checked[t]["state"] == "ok" for t in MARGIN_TABLES)) + ck("margin final account pass retains tables completed by earlier passes", + all(_mirrors._table_sync_complete("amazon", last["tables"][t]) + and "margin-account" in last["tables"][t]["accounts"] for t in MARGIN_TABLES)) calls = reports.call_count - deferred = sync(Runtime.key, tables=["referral_fees", "fba_fee_estimates", "storage_fees"], rt=Runtime(), log=lambda *_: None) + deferred = sync(Runtime.key, tables=["referral_fees", "fba_fee_estimates", "storage_fees"], rt=rt, log=lambda *_: None) ck("margin manual sync also honors vendor report floors", reports.call_count == calls and all(r["state"] == "deferred" for r in deferred["tables"].values())) finally: @@ -7944,13 +8770,18 @@ def group_a_selftest(log=print, check=None): selected = ("inventory_planning", "long_term_storage_fees", "order_metrics", "listing_items", "account_health") class Runtime(_FixtureMirrorRuntime): key = "group-a-sync-fixture" - def get(self, key, *args): + def get(self, key, *args, **_kwargs): + if key == "keychain": + return {"entries": {"group-a-entry": { + "connection_id": "group-a-account", "type": "amazon_sp", "slot": 1, + "label": "Group A store"}}} return {"databases": {amazon_schema.DATABASE_PREFIX + t: t in selected for t in TABLES}} if key == ENABLEMENT_KEY else None rt = Runtime() try: with patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 62), \ patch.object(module, "credential", return_value={"fixture": True}), \ - patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \ + patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: + _fixture_account_cursor(con, rt)), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \ patch.object(cx, "group_a_records", side_effect=records), \ diff --git a/platform/harness/analyst.py b/platform/harness/analyst.py index 938b14cc1fc02c778201e0975d059aa10190f6e8..18ec97c5c6fce53e01494943ad4946f9ea4d44e0 100644 --- a/platform/harness/analyst.py +++ b/platform/harness/analyst.py @@ -169,7 +169,7 @@ def _get_recipe(arguments): return {"ok": False, "error": str(error)} -def _tool_schemas(): +def _tool_schemas(data_source=None): """Keep complete argument contracts; skill bodies and catalogs load on demand.""" descriptions = { "list_topics": "List topics with metrics, dims and grain. The system prompt's TOPIC " @@ -197,7 +197,7 @@ def _tool_schemas(): return [{**tool, "function": {**tool["function"], "description": descriptions.get(tool["function"]["name"], tool["function"]["description"])}} - for tool in T.openai_tools()] + [{ + for tool in (data_source if data_source is not None else T).openai_tools()] + [{ "type": "function", "function": { "name": "get_recipe", "description": "Read exact local skill context and recipe/guard by 1-based " @@ -235,10 +235,10 @@ def _ops_index(): return " ".join(sigs) or "(read get_recipe(skill=analytics, section=ops))" -def system_prompt(): +def system_prompt(data_source=None): """The small model's standing instructions: the platform rules + the skill recipes. Stable content (cache-friendly); per-question content goes in the user turn.""" - topics = T.list_topics() + topics = (data_source if data_source is not None else T).list_topics() topic_summary = _topic_index(topics) return f"""AIOS Analyst: wholesale data questions via governed TOOLS. Non-negotiable rules: 1. NEVER write SQL or invent fields/joins/numbers. Use registered schema keys only; recipes @@ -246,6 +246,8 @@ def system_prompt(): topic from it, describe_topic it once for its scope rules, then query. Wholesale sales, margin, units and distinct customers come from sales_lines; order counts from sales_orders. Invoice-line basis, commission agent, allocated or unallocated sales: invoice_lines. + An explicit revenue ask uses revenue, including a short ask such as "Revenue for June". + Resolve its stated window and query it even when there is no earlier conversation. 2. Resolve every typed name or place (customer, product, agent, salesperson, city, state) with get_field_values BEFORE filtering, even when the spelling looks exact. Every filter value of any dim comes from get_field_values or an earlier result, never typed from the ask. A @@ -501,8 +503,8 @@ def _date_group(arguments, topic_dims): def _window_gap(arguments): """True when a gap report claims a missing date dimension or window.""" - missing = str(_parsed(arguments).get("missing") or "").lower() - return any(w in missing for w in ("date dim", "date dimension", "no date", "date field", + missing = str(_parsed(arguments).get("missing") or "").strip().lower() + return missing == "date" or any(w in missing for w in ("date dim", "date dimension", "no date", "date field", "time dimension", "date filter", "filtered by 20", "filter by year", "by date")) @@ -860,8 +862,9 @@ class _AskContext: # The default only. ask() passes the provider's ceiling from request_budget(). MAX_REQUEST_BYTES = DEFAULT_REQUEST_BYTES - def __init__(self, messages, tools, max_request_bytes=None): + def __init__(self, messages, tools, max_request_bytes=None, data_source=None): self.messages, self.tools = messages, tools + self.data_source = data_source if data_source is not None else T if max_request_bytes: self.MAX_REQUEST_BYTES = max_request_bytes self.cache, self.entries, self.rules, self.results = {}, [], {}, {} @@ -904,7 +907,8 @@ class _AskContext: key = self._key(name, args) if name in self._READS and key in self.cache: return deepcopy(self.cache[key]) - res = _get_recipe(args) if name == "get_recipe" else T.dispatch(name, args) + res = (_get_recipe(args) if name == "get_recipe" + else self.data_source.dispatch(name, args)) if name in self._READS and res.get("ok"): self.cache[key] = deepcopy(res) return res @@ -925,9 +929,9 @@ class _AskContext: if rules: self.rules[key] = rules data = response.get("data") - if isinstance(data, dict) and data.get("result_id") in T._RESULTS: + if isinstance(data, dict) and data.get("result_id") in self.data_source._RESULTS: rid = data["result_id"] - self.results[rid] = deepcopy(T._RESULTS[rid]) + self.results[rid] = deepcopy(self.data_source._RESULTS[rid]) if name in ("describe_topic", "run_semantic_query"): parsed = json.loads(args) if isinstance(args, str) else args self.current_topic = parsed.get("topic") @@ -992,7 +996,7 @@ class _AskContext: def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=None, - telemetry_kind="analyst", user=None): + telemetry_kind="analyst", user=None, data_source=None): """Run one Analyst turn. Returns {answer, artifacts, tool_trace, iterations, exhausted, usage}. chat_fn(messages, tools, model) -> (assistant_message_dict, usage_dict). @@ -1009,23 +1013,24 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=Non if max_iters is None: max_iters = iter_budget() chat = chat_fn or _live_chat - tools = _tool_schemas() - messages = [{"role": "system", "content": system_prompt()}] + backend = data_source if data_source is not None else T + tools = _tool_schemas(data_source) + messages = [{"role": "system", "content": system_prompt(data_source)}] messages += list(history or []) # The model has no clock: anchor relative windows ("last week", "this month") to today. # Lives in the USER turn so the system prompt stays byte-stable (prompt-cache friendly). import datetime as _dt - messages.append({"role": "user", - "content": f"(today is {_dt.date.today().isoformat()})\n{question}"}) + today = backend.today if data_source is not None else _dt.date.today().isoformat() + messages.append({"role": "user", "content": f"(today is {today})\n{question}"}) trace, artifacts, usage_total = [], [], {"prompt_tokens": 0, "completion_tokens": 0} # The live ladder's first provider sets the ceiling. An injected chat_fn has no provider. provider = (available_providers() or [{}])[0].get("name") if chat is _live_chat else None - context = _AskContext(messages, tools, request_budget(provider)) + context = _AskContext(messages, tools, request_budget(provider), data_source) calls, failed_calls, retries = {}, set(), 0 cut_results, queries, field_values = set(), [], {} yoy_pending, yoy_done, reasked = set(), False, set() try: - topic_dims = {t["topic"]: set(t.get("dims") or []) for t in T.list_topics()} + topic_dims = {t["topic"]: set(t.get("dims") or []) for t in backend.list_topics()} except Exception: # noqa: BLE001 - guard only topic_dims = {} @@ -1108,7 +1113,8 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=Non # principle. Read defensively off `user` because this function is deliberately # callable WITHOUT one — `harness/evals.py` and `harness/routines.py` run as the # system, and telemetry omits both keys for them rather than inventing a subject. - _tel.log(telemetry_kind, question=question[:200], + log = data_source.log if data_source is not None else _tel.log + log(telemetry_kind, question=question[:200], user=(user or {}).get("username") if isinstance(user, dict) else None, tenant=(user or {}).get("tenant") if isinstance(user, dict) else None, model=model or (available_providers() or [{}])[0].get("name", "auto"), diff --git a/platform/harness/canonical_source.py b/platform/harness/canonical_source.py index 07390d4da62056b0d8c408bc1da8292d5b81caca..be3d2737adf933ec79db5815e6449e634dcc3b5f 100644 --- a/platform/harness/canonical_source.py +++ b/platform/harness/canonical_source.py @@ -89,7 +89,8 @@ class Source: domain = list(domain or []) if self.kind == 'customer': field = {'res.partner': 'id', 'sale.order': 'partner_id', - 'sale.order.line': 'order_partner_id', 'account.move': 'partner_id'}.get(model) + 'sale.order.line': 'order_partner_id', 'account.move': 'partner_id', + 'account.move.line': 'partner_id'}.get(model) # Explicit partner lookup domains also resolve agents and must retain their ids. if model == 'res.partner' and any(isinstance(d, (tuple, list)) and d[0] in {'id', 'name'} for d in domain): @@ -135,14 +136,15 @@ class Source: def customer_mix(self, groups): def products(): - last = 0 + last = None while True: ids = [r[0] for r in self.cur.execute(f"""SELECT DISTINCT try_cast(json_extract_string(body, '/product_id/0') AS BIGINT) id - FROM "{groups.table}" WHERE id > ? ORDER BY id LIMIT ?""", [last, BATCH]).fetchall()] + FROM "{groups.table}" WHERE id IS NOT NULL AND (? IS NULL OR id > ?) + ORDER BY id LIMIT ?""", [last, last, BATCH]).fetchall()] if not ids: return - yield from _odoo.search_read('product.product', [('id', 'in', ids)], + yield from odoo.search_read('product.product', [('id', 'in', ids)], ['id', 'name', 'categ_id'], limit=BATCH, order='id asc') last = ids[-1] prods = self.snapshot(products()) @@ -371,6 +373,10 @@ class Source: #: Tables a product pool needs before it may read the mirror instead of Odoo. MIRROR_TABLES = ('product_product', 'stock_quant', 'stock_location', 'sale_order', 'sale_order_line', 'res_partner') +CUSTOMER_MIRROR_TABLES = ('product_product', 'sale_order', 'sale_order_line', 'res_partner', + 'account_move', 'account_move_line', 'account_account') +CUSTOMER_PARTNER_FIELDS = ('property_payment_term_id', 'property_product_pricelist', + 'create_date', 'partner_latitude', 'partner_longitude', 'category_source_ids') _MIRROR_MODELS = { 'product.product': 'product_product', 'sale.order.line': 'sale_order_line', @@ -378,6 +384,13 @@ _MIRROR_MODELS = { 'stock.quant': 'stock_quant', 'stock.location': 'stock_location', 'res.partner': 'res_partner', + 'account.move': 'account_move', + 'account.move.line': 'account_move_line', + 'account.account': 'account_account', + 'res.partner.category': ('(SELECT DISTINCT try_cast(ids.value AS BIGINT) AS id, ' + 'json_extract_string(names.value, \'$\') AS name FROM res_partner p, ' + 'json_each(p.category_source_ids) ids, json_each(p.category_id) names ' + 'WHERE ids.key = names.key)'), 'product.pricelist': 'product_pricelist', 'product.pricelist.item': 'product_pricelist_item', 'product.tag': 'product_tag', @@ -385,7 +398,9 @@ _MIRROR_MODELS = { 'product.category': ('(SELECT DISTINCT categ_id AS id, categ_name AS complete_name ' 'FROM product_product WHERE categ_id IS NOT NULL)'), } -# The only many2one hops a mirrored domain may cross, one level deep. +_CUSTOMER_MIRROR_MODELS = {'account.move', 'account.move.line', 'account.account', + 'res.partner.category'} +# Every many2one hop must be declared before a mirrored domain can cross it. _MIRROR_RELATIONS = { ('sale.order.line', 'order_id'): 'sale.order', ('sale.order.line', 'product_id'): 'product.product', @@ -393,9 +408,16 @@ _MIRROR_RELATIONS = { ('stock.quant', 'product_id'): 'product.product', ('stock.quant', 'location_id'): 'stock.location', ('sale.order', 'partner_id'): 'res.partner', + ('account.move.line', 'account_id'): 'account.account', + ('account.move.line', 'move_id'): 'account.move', + ('account.move.line', 'partner_id'): 'res.partner', + ('account.move', 'partner_id'): 'res.partner', } # Odoo's line state is the order's state, and the mirror stores it only on sale_order. -_MIRROR_ALIASES = {('sale.order.line', 'state'): 'order_id.state'} +_MIRROR_ALIASES = {('sale.order.line', 'state'): 'order_id.state', + ('res.partner', 'category_id'): 'category_source_ids', + ('res.partner', 'property_product_pricelist'): 'property_product_pricelist_id', + ('account.move.line', 'invoice_date'): 'move_id.invoice_date'} _NUMERIC = ('DOUBLE', 'FLOAT', 'REAL', 'INTEGER', 'BIGINT', 'SMALLINT', 'TINYINT', 'HUGEINT', 'UBIGINT', 'UINTEGER', 'USMALLINT', 'UTINYINT', 'DECIMAL') _CODE_FIELD = '__code__' @@ -444,17 +466,44 @@ class _Unmirrored(Exception): """A model, field, path or operator the mirror cannot answer.""" -def mirror_missing(cur): +def mirror_missing(cur, kind='product'): """The `MIRROR_TABLES` this cursor cannot see, in declaration order.""" + required = CUSTOMER_MIRROR_TABLES if kind == 'customer' else MIRROR_TABLES found = {r[0] for r in cur.execute( 'SELECT DISTINCT table_name FROM information_schema.tables WHERE table_name IN ' - '(SELECT unnest($1::VARCHAR[]))', [list(MIRROR_TABLES)]).fetchall()} - return [t for t in MIRROR_TABLES if t not in found] + '(SELECT unnest($1::VARCHAR[]))', [list(required)]).fetchall()} + missing = [t for t in required if t not in found] + if kind != 'customer' or missing: + return missing + from harness import datastore + try: + phases = dict(cur.execute('SELECT entity, phase FROM _sync_state').fetchall()) + for table in required: + if phases.get(table) != 'live': + missing.append(table + '.sync') + states = datastore._column_backfill_states(cur, 'res_partner') + missing.extend('res_partner.' + f for f in CUSTOMER_PARTNER_FIELDS + if states.get(f, (None,))[0] != 'done') + for table in ('account_move', 'account_move_line', 'account_account'): + if not datastore._columns_backfilled(cur, table, datastore.ENTITIES[table]): + missing.append(table + '.columns') + cols = {r[0] for r in cur.execute('DESCRIBE res_partner').fetchall()} + for field in CUSTOMER_PARTNER_FIELDS: + missing.extend('res_partner.' + col for col in datastore._field_cols(field) + if col not in cols) + if 'category_source_ids' not in cols or cur.execute( + 'SELECT 1 FROM res_partner WHERE category_source_ids IS NULL LIMIT 1').fetchone(): + missing.append('res_partner.category_source_ids') + except Exception as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + missing.append('customer.sync_state') + return list(dict.fromkeys(missing)) def _unready(missing): return ('canonical_source_mirror_unavailable: the Odoo mirror has not synced ' - + ', '.join(missing) + ' yet, and the product pool does not read Odoo live') + + ', '.join(missing) + ' yet, and the canonical pool does not read Odoo live') def _sql_list_type(values): @@ -478,7 +527,8 @@ class _Query: """Translate one Odoo model's domain and fields into SQL over the mirror tables.""" def __init__(self, source, model, params=None): - if model not in _MIRROR_MODELS: + if model not in _MIRROR_MODELS or (model in _CUSTOMER_MIRROR_MODELS + and source.required_kind != 'customer'): raise _Unmirrored('model ' + model) self.source, self.model = source, model self.params = params if params is not None else [] @@ -517,18 +567,22 @@ class _Query: if not isinstance(path, str) or not path.replace('.', '').replace('_', '').isalnum(): raise _Unmirrored('path ' + repr(path)) path = _MIRROR_ALIASES.get((self.model, path), path) - head, _, rest = path.partition('.') - if not rest: - expr, kind = self._column(self.model, head, 'b') - return expr, kind, None - comodel = _MIRROR_RELATIONS.get((self.model, head)) - if comodel is None or '.' in rest: - raise _Unmirrored(f'{self.model}.{path}') - self._column(self.model, head, 'b') - alias = 'j_' + head - self.joins[head] = (comodel, alias) - expr, kind = self._column(comodel, rest, alias) - return expr, kind, f'{alias}.id IS NOT NULL' + model, alias, traversed, guards = self.model, 'b', [], [] + parts = path.split('.') + for index, head in enumerate(parts): + expr, kind = self._column(model, head, alias) + if index == len(parts) - 1: + return expr, kind, ' AND '.join(guards) or None + if parts[index + 1:] == ['name'] and kind == 'm2o': + return expr[1], 'str', ' AND '.join(guards) or None + comodel = _MIRROR_RELATIONS.get((model, head)) + if comodel is None: + raise _Unmirrored(f'{self.model}.{path}') + traversed.append(head) + target = 'j_' + '_'.join(traversed) + self.joins[target] = (comodel, target, alias, head) + model, alias = comodel, target + guards.append(f'{alias}.id IS NOT NULL') def leaf(self, term): if not isinstance(term, (list, tuple)) or len(term) != 3: @@ -537,6 +591,19 @@ class _Query: op = {'==': '=', '<>': '!='}.get(str(op).lower(), str(op).lower()) expr, kind, guard = self.field(path) if kind == 'm2m': + if self.source.required_kind != 'customer' or self.model != 'res.partner' or path not in { + 'agent_ids', 'category_id', 'category_source_ids'}: + raise _Unmirrored(f'many2many filter on {self.model}.{path}') + empty = value is False or value is None + if op in {'=', '!='} and empty: + cond = f'coalesce(json_array_length({expr}), 0) = 0' + return cond if op == '=' else f'NOT ({cond})' + if op in {'in', 'not in'}: + values = list(value) if isinstance(value, (list, tuple, set)) else [value] + cond = (f'EXISTS (SELECT 1 FROM json_each({expr}) member WHERE ' + f'try_cast(member.value AS BIGINT) IN ' + f'(SELECT unnest({self.param(values)}::BIGINT[])))') + return cond if op == 'in' else f'NOT ({cond})' raise _Unmirrored(f'many2many filter on {self.model}.{path}') if kind == 'str' and op in {'<=', '>'} and isinstance(value, str) and _DATE_ONLY.fullmatch(value): # Odoo reads a bare date as its midnight, so a value of exactly midnight is <= that day. @@ -598,8 +665,8 @@ class _Query: return ' AND '.join(reversed(stack)) or 'true' def from_sql(self): - joins = ''.join(f' LEFT JOIN {self.source.relation(comodel)} {alias} ON {alias}.id = b.{head}' - for head, (comodel, alias) in self.joins.items()) + joins = ''.join(f' LEFT JOIN {self.source.relation(comodel)} {alias} ON {alias}.id = {parent}.{head}' + for comodel, alias, parent, head in self.joins.values()) return f'{self.source.relation(self.model)} b{joins}' @@ -627,7 +694,7 @@ def _refuse(method, model, cause): class MirrorSource(Source): - """A `Source` that answers every product pool read from the tenant's Odoo mirror, never live. + """Answer canonical customer and product reads from the tenant's Odoo mirror. `live_reads` stays `[]`. `pending` names each declared `table.column` or `table` the sync has not filled yet: those read blank or empty, logged once per root build. @@ -641,6 +708,7 @@ class MirrorSource(Source): self.parent = None self._columns = {} self._relations = {} + self.required_kind = 'product' def _note(self, bucket, name): if name not in bucket: @@ -654,7 +722,8 @@ class MirrorSource(Source): 'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1', [target]).fetchone(): declared = declared_columns(target) - if target in MIRROR_TABLES: + required = CUSTOMER_MIRROR_TABLES if self.required_kind == 'customer' else MIRROR_TABLES + if target in required: raise RuntimeError(_unready([target])) if not declared: raise _Unmirrored('table ' + target) @@ -702,7 +771,7 @@ class MirrorSource(Source): if self._cur is None: opened = self._factory() try: - missing = mirror_missing(opened) + missing = mirror_missing(opened, self.required_kind) except BaseException: opened.close() raise @@ -778,13 +847,17 @@ class MirrorSource(Source): name, _, agg = str(spec).partition(':') if name in groupby or name in {'id', '__count'}: continue - expr, kind, _ = query.field(name) + named = re.fullmatch(r'(sum|min|max|avg|count|count_distinct)\(([\w.]+)\)', agg) + field = name + if named: + agg, field = named.groups() + expr, kind, _ = query.field(field) if kind == 'm2m': raise _Unmirrored('many2many aggregate ' + spec) ident = expr[0] if kind == 'm2o' else expr agg = agg or ('sum' if kind == 'num' else '') if agg in {'sum', 'min', 'max', 'avg'}: - select.append((name, f'coalesce({agg}({ident}), 0)')) + select.append((name, _json_value(f'{agg}({ident})', kind))) elif agg == 'count': select.append((name, f'count({ident})')) elif agg == 'count_distinct': @@ -807,6 +880,9 @@ class MirrorSource(Source): return self.cur.execute(f'SELECT count(*) FROM {query.from_sql()} WHERE {where}', query.params).fetchone()[0] + def sum_field(self, model, domain, field): + return sum(row.get(field) or 0 for row in self.read_group(model, domain, [field + ':sum'], [])) + def excluded_partner_ids(self): """`core.odoo.excluded_partner_ids` resolved from res_partner, shared through its cache.""" ctx = _odoo._ctx() @@ -930,6 +1006,7 @@ def bounded(cur, kind, identities, store=None): source = MirrorSource(cur, kind, identities, store) source.live_reads, source.pending, source.dropped = parent.live_reads, parent.pending, parent.dropped source.parent = parent + source.required_kind = parent.required_kind else: source = Source(cur, kind, identities, store) with _activate(source) as entered: @@ -937,16 +1014,18 @@ def bounded(cur, kind, identities, store=None): @contextmanager -def mirrored(cur, store=None): +def mirrored(cur, store=None, *, kind='product'): """Serve every proxied read in this block from the Odoo mirror; never reads Odoo live. `cur` is a mirror cursor, or a zero-argument factory opened on the first read and closed on exit. """ if not callable(cur) or hasattr(cur, 'execute'): - missing = mirror_missing(cur) + missing = mirror_missing(cur, kind) if missing: raise RuntimeError(_unready(missing)) - with _activate(MirrorSource(cur, 'all', [], store)) as entered: + source = MirrorSource(cur, 'all', [], store) + source.required_kind = kind + with _activate(source) as entered: yield entered @@ -957,7 +1036,7 @@ class _Odoo: return getattr(source, name) if source is not None and name == 'get_odoo': return lambda: self - if isinstance(source, MirrorSource) and name in {'excluded_partner_ids', 'sale_line_domain'}: + if isinstance(source, MirrorSource) and name in {'excluded_partner_ids', 'sale_line_domain', 'sum_field'}: return getattr(source, name) return getattr(_odoo, name) diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index 206e00a5bdb589bc342e51e92a1f2973f5d95e37..ff1a5936b47411a3b5754e38891e5e7de3ba95f1 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -275,6 +275,8 @@ ENTITIES = { "agent", "salesman_as_agent", "customer_rank", "street", "street2", "zip", "email", "phone", "mobile", "website", "vat", "ref", + "property_payment_term_id", "property_product_pricelist", "create_date", + "partner_latitude", "partner_longitude", "write_date"], # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next # sync_all() instead of needing a reseed. @@ -287,7 +289,9 @@ ENTITIES = { "customer_rank", # added 2026-08-11 (W29-T54) "street", "street2", "zip", # added 2026-08-12 (W30-T33) "email", "phone", "mobile", # added 2026-08-14 (W33-T48) - "website", "vat", "ref", "category_id"], + "website", "vat", "ref", "category_id", + "property_payment_term_id", "property_product_pricelist", "create_date", + "partner_latitude", "partner_longitude", "category_source_ids"], # Rank drift behind the write_date cursor requires recurring observation; # schema backfill alone cannot maintain the customer population predicate. "refresh_fields": ["customer_rank"], @@ -298,6 +302,7 @@ ENTITIES = { "m2m": ["agent_ids"], # Named many2many fields store source names despite their singular `_id` suffix. "named_m2m": {"category_id": "res.partner.category"}, + "raw_m2m": {"category_source_ids": "category_id"}, }, "product_product": { "model": "product.product", "archivable": True, @@ -736,6 +741,9 @@ def _odoo_mirror_store(): from harness import mirror_store class OdooMirrorStore(mirror_store.MirrorStore): + def _valid_table(self, name): + return name in {"odoo_connections", "odoo_record_ids"} or super()._valid_table(name) + def _connect(self, tenant_key, rt): self._mirror_cache_key(tenant_key, rt) return _mirror_connection(tenant_key, rt) @@ -1679,11 +1687,46 @@ def _required_columns_ready(cur, key, spec): def _columns_backfilled(cur, key, spec): - """Read the completion marker without requiring readers to wait for the migration.""" - fields = sorted(f for f in spec.get("backfill_fields", ()) if f != "write_date") - marker = f"{key}.cols.{','.join(fields)}" - migration = cur.execute("SELECT phase FROM _sync_state WHERE entity = ?", [marker]).fetchone() - return bool(migration) and migration[0] == "done" + """Require completion of each declared field, including completed legacy cohorts.""" + states = _column_backfill_states(cur, key) + return all(states.get(field, (None,))[0] == "done" + for field in spec.get("backfill_fields", ()) if field != "write_date") + + +def _column_backfill_states(cur, key): + """Expand legacy cohorts while giving each field's own checkpoint precedence.""" + prefix = _connection_state_key(f"{key}.cols.") + prior = cur.execute("SELECT entity, phase, last_id, rows FROM _sync_state " + "WHERE starts_with(entity, ?)", [prefix]).fetchall() + states, individual = {}, {} + for name, phase, last, count in prior: + fields = name[len(prefix):].split(",") + point = (phase, last or 0, count or 0) + if len(fields) == 1: + individual[fields[0]] = point + for field in fields: + previous = states.get(field) + if previous is None or (phase == "done", point[1:]) > (previous[0] == "done", previous[1:]): + states[field] = point + states.update(individual) + return states + + +def _checkpoint_column_fields(con, key, fields, phase, last, count): + """Save field progress and settle obsolete cohort markers once their fields finish.""" + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + con.executemany("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [[_connection_state_key(f"{key}.cols.{field}"), phase, last, None, count, stamp] for field in fields]) + if phase == "done": + states = _column_backfill_states(con, key) + prefix = _connection_state_key(f"{key}.cols.") + prior = con.execute("SELECT entity FROM _sync_state WHERE starts_with(entity, ?) " + "AND phase != 'done'", [prefix]).fetchall() + for (name,) in prior: + cohort = name[len(prefix):].split(",") + if all(states[field][0] == "done" for field in cohort): + con.execute("UPDATE _sync_state SET phase='done', updated_at=? WHERE entity=?", + [stamp, name]) def entity_live(key): @@ -2526,7 +2569,167 @@ def _ensure_sync_state(con): cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") -def _flatten(rec, fields, m2m=(), *, named_m2m=None, analytic_names=None, named_m2m_names=None): +def _connection_state_key(key): + """Keep legacy checkpoints and qualify other connections after the source table's first dot.""" + binding = getattr(_SYNC_TLS, "connection", None) + if binding is None or binding["legacy"]: + return key + table, dot, tail = key.partition(".") + token = hashlib.sha256(binding["id"].encode()).hexdigest() + return f"{table}.connection.{token}" + (dot + tail if dot else "") + + +def _connection_where(): + """Restrict replacement and reconciliation to the active stable connection identity.""" + binding = getattr(_SYNC_TLS, "connection", None) + return ("_connection_id = '" + binding["id"].replace("'", "''") + "'" + if binding else "TRUE") + + +def _connection_ids(con, model, ids): + """Allocate durable negative mirror IDs without changing upstream cursors or legacy IDs.""" + binding = getattr(_SYNC_TLS, "connection", None) + ids = sorted({int(i) for i in ids if i is not None and i is not False}) + if binding is None or binding["legacy"]: + return {i: i for i in ids} + if not ids: + return {} + found = dict(con.execute("SELECT source_id, id FROM odoo_record_ids " + "WHERE connection_id=? AND model=? AND source_id IN (SELECT unnest(?::BIGINT[]))", + [binding["id"], model, ids]).fetchall()) + missing = [i for i in ids if i not in found] + if missing: + last = con.execute("SELECT coalesce(min(id), 0) FROM odoo_record_ids").fetchone()[0] + rows = [(binding["id"], model, i, last - n) for n, i in enumerate(missing, 1)] + con.execute("INSERT INTO odoo_record_ids SELECT ?, ?, unnest(?::BIGINT[]), unnest(?::BIGINT[])", + [binding["id"], model, missing, [row[3] for row in rows]]) + found.update((row[2], row[3]) for row in rows) + return found + + +def _connection_rows(rows, spec): + """Translate row and relation identities once, after source names resolve against raw IDs.""" + binding = getattr(_SYNC_TLS, "connection", None) + if binding is None: + return rows + con, model = binding["con"], spec["model"] + identities = _connection_ids(con, model, [r["id"] for r in rows]) + relations = binding["relations"].get(model, {}) + for row in rows: + source_id = row["id"] + row.update(id=identities[source_id], _source_id=source_id, + _connection_id=binding["id"], Store=binding["label"]) + for field, (target, kind) in relations.items(): + # team_id is the tenant's BU enum, not a mirrored crm.team record identity. + if field == "team_id": + continue + output_field = field if field.endswith("_id") or field not in MANY2ONE_FIELDS else field + "_id" + outputs = [output_field] + outputs += [out for out, source in (spec.get("raw_m2m") or {}).items() if source == field] + for output in outputs: + if field in (spec.get("named_m2m") or {}) and output == output_field: + continue + is_list = kind in {"many2many", "one2many"} + values = [i for row in rows for i in (json.loads(row.get(output) or "[]") if is_list + else [row.get(output)]) if i is not None] + mapped = _connection_ids(con, target, values) + for row in rows: + if output not in row: + continue + if is_list: + raw = json.loads(row[output] or "[]") + row[output] = json.dumps([mapped[i] for i in raw]) if row[output] is not None else None + first = output.removesuffix("_ids") + "_id" + if first in row: + row[first] = mapped[raw[0]] if raw else None + elif row[output] is not None: + row[output] = mapped[row[output]] + return rows + + +def _odoo_entries(rt): + """Resolve every tenant Odoo credential by its stable keychain connection identity.""" + from core import keychain + entries = sorted((row for row in keychain.list_entries(rt) if row.get("type") == "odoo"), + key=lambda row: (int(row.get("slot") or 1), str(row["id"]))) + resolved = [] + for row in entries: + cid = str(row.get("connection_id") or row["id"]) + creds = keychain.odoo_creds(rt, cid) + if not creds: + raise RuntimeError("An Odoo connection has no usable tenant credential") + resolved.append({"id": cid, "label": str(row.get("label") or row["id"]), "creds": creds}) + if not resolved and rt.key == "royal-imports": + resolved.append({"id": "legacy-env", "label": "Odoo 1", "creds": None}) + return resolved + + +@contextmanager +def _odoo_connection(con, rt, entry, client_factory=None): + """Bind a dedicated read-only transport and the saved identity namespace for one connection.""" + con.execute("CREATE TABLE IF NOT EXISTS odoo_connections " + "(connection_id VARCHAR PRIMARY KEY, legacy BOOLEAN, label VARCHAR)") + con.execute("CREATE TABLE IF NOT EXISTS odoo_record_ids " + "(connection_id VARCHAR, model VARCHAR, source_id BIGINT, id BIGINT UNIQUE, " + "PRIMARY KEY(connection_id, model, source_id))") + known = dict(con.execute("SELECT connection_id, legacy FROM odoo_connections").fetchall()) + if entry["id"] not in known: + # The saved binding survives a default change and disconnection. + legacy = not known + if set(known) == {"legacy-env"} and rt.key == "royal-imports": + con.execute("UPDATE odoo_connections SET connection_id=? WHERE connection_id='legacy-env'", [entry["id"]]) + for table in ENTITIES: + if "_connection_id" in (columns_of(table, cur=con) or ()): + con.execute(f"UPDATE {table} SET _connection_id=? WHERE _connection_id='legacy-env'", [entry["id"]]) + legacy = True + else: + con.execute("INSERT INTO odoo_connections VALUES (?,?,?)", [entry["id"], legacy, entry["label"]]) + else: + legacy = known[entry["id"]] + con.execute("UPDATE odoo_connections SET label=? WHERE connection_id=?", [entry["label"], entry["id"]]) + previous = getattr(_SYNC_TLS, "connection", None) + client, context = getattr(O._tlocal, "client", None), getattr(O._tlocal, "tenant_ctx", None) + try: + O._tlocal.client = (client_factory(entry["creds"]) if client_factory else O.OdooClient(creds=entry["creds"])) + O._tlocal.tenant_ctx = None + _SYNC_TLS.connection = {**entry, "legacy": legacy, "con": con, "relations": {}} + for table in ENTITIES: + cols = set(columns_of(table, cur=con) or ()) + if cols: + for name, typ in (("Store", "VARCHAR"), ("_connection_id", "VARCHAR"), ("_source_id", "BIGINT")): + if name.lower() not in cols: + con.execute(f"ALTER TABLE {table} ADD COLUMN {name} {typ}") + if legacy: + con.execute(f"UPDATE {table} SET _connection_id=?, _source_id=id, Store=? WHERE _connection_id IS NULL", + [entry["id"], entry["label"]]) + con.execute(f"UPDATE {table} SET Store=? WHERE _connection_id=? AND Store IS DISTINCT FROM ?", + [entry["label"], entry["id"], entry["label"]]) + yield _SYNC_TLS.connection + finally: + _SYNC_TLS.connection = previous + O._tlocal.client, O._tlocal.tenant_ctx = client, context + + +def _connection_model(spec): + """Read relation metadata for the active source without treating every integer as a relation.""" + binding = getattr(_SYNC_TLS, "connection", None) + if binding is not None and spec["model"] not in binding["relations"]: + metadata = O.get_odoo().fields_get(spec["model"], attributes=("type", "relation")) + binding["relations"][spec["model"]] = { + field: (info["relation"], info["type"]) for field, info in metadata.items() + if info.get("type") in {"many2one", "many2many", "one2many"} and info.get("relation")} + + +def _connection_pending(con, key): + """Read this connection's markers without mixing the legacy namespace with its siblings.""" + prefix = _connection_state_key(key) + "." + return [(name, phase) for name, phase in con.execute( + "SELECT entity, phase FROM _sync_state WHERE starts_with(entity, ?)", [prefix]).fetchall() + if not (prefix == key + "." and name.startswith(key + ".connection."))] + + +def _flatten(rec, fields, m2m=(), *, named_m2m=None, analytic_names=None, named_m2m_names=None, + list_fields=()): out = {"id": rec["id"]} for f in m2m: v = rec.get(f) or [] @@ -2553,7 +2756,7 @@ def _flatten(rec, fields, m2m=(), *, named_m2m=None, analytic_names=None, named_ for account_id in key.split(",")]} for key, share in v.items() ], ensure_ascii=False) - elif f in ID_LIST_FIELDS: # a 2-id list would otherwise read as an m2o pair + elif f in ID_LIST_FIELDS or f in list_fields: out[f] = json.dumps(list(v or [])) elif isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int): out[f"{f.removesuffix('_id')}_id"] = v[0] @@ -2567,7 +2770,7 @@ def _flatten(rec, fields, m2m=(), *, named_m2m=None, analytic_names=None, named_ return out -def _flatten_records(recs, fields, m2m=(), *, named_m2m=None): +def _flatten_records(recs, fields, m2m=(), *, named_m2m=None, raw_m2m=None, spec=None): """Resolve analytic and tag names once per source page, without a shared tenant cache.""" named_m2m_names = {} for field, model in (named_m2m or {}).items(): @@ -2591,8 +2794,16 @@ def _flatten_records(recs, fields, m2m=(), *, named_m2m=None): [("id", "in", ids[start:start + 1000]), ("active", "in", [True, False])], ["name"], limit=1000, order="id asc") names.update({account["id"]: account.get("name") for account in accounts}) - return [_flatten(rec, fields, m2m, named_m2m=named_m2m, analytic_names=names, - named_m2m_names=named_m2m_names) for rec in recs] + binding = getattr(_SYNC_TLS, "connection", None) + relations = binding["relations"].get(spec["model"], {}) if binding and spec is not None else {} + list_fields = {field for field, (_target, kind) in relations.items() if kind in {"many2many", "one2many"}} + rows = [_flatten(rec, fields, m2m, named_m2m=named_m2m, analytic_names=names, + named_m2m_names=named_m2m_names, list_fields=list_fields) for rec in recs] + for rec, row in zip(recs, rows): + for output, source in (raw_m2m or {}).items(): + if source in rec: + row[output] = json.dumps(list(rec[source] or [])) + return _connection_rows(rows, spec) if spec is not None else rows #: ⛔⛔ A HARD-CODED NAME LIST DECIDES WHETHER A COLUMN IS A NUMBER, and a field missing from it @@ -2615,10 +2826,11 @@ NUMERIC_FIELDS = ("amount_untaxed", "amount_untaxed_signed", "amount_residual_si "standard_price", "debit", "credit", "balance", "amount", # W37-T13 — the stock-movement quantity. See the note above. "quantity_done", "quantity", "value", "reserved_quantity", - "amount_total", "product_qty", "qty_received", "price_unit", "factor") + "amount_total", "product_qty", "qty_received", "price_unit", "factor", + "partner_latitude", "partner_longitude") # Odoo also names this many2one without the usual `_id` suffix. -MANY2ONE_FIELDS = frozenset({"product_uom", "create_uid"}) +MANY2ONE_FIELDS = frozenset({"product_uom", "create_uid", "property_product_pricelist"}) # Plain x2many fields stored whole as a JSON id list, `[]` for none. ID_LIST_FIELDS = frozenset({"product_tag_ids"}) @@ -2686,8 +2898,8 @@ for _entity, _added in AUDIT_FIELD_ADDITIONS.items(): if _new: _tail = ["write_date"] if "write_date" in _spec["fields"] else [] _spec["fields"] = [f for f in _spec["fields"] if f != "write_date"] + _new + _tail - _backfill = list(_spec.get("backfill_fields") or []) - _spec["backfill_fields"] = _backfill + [f for f in _new if f not in _backfill] + _backfill = list(_spec.get("backfill_fields") or []) + _spec["backfill_fields"] = _backfill + [f for f in _added if f not in _backfill] BOOL_FIELDS += ("deprecated", "enable_max_invoice_age", "include_initial_balance", "is_company", "is_delivery", "is_inventory", "is_locked", "is_move_sent", "online_payment", "propagate_cancel", "recompute_delivery_price", "reconcile", "reconciled", @@ -2781,7 +2993,7 @@ _connector_spec.register("odoo", _connector_spec.tables_for("odoo"), def _cols(spec): - cols = ["id BIGINT PRIMARY KEY"] + cols = ["id BIGINT PRIMARY KEY", "Store VARCHAR", "_connection_id VARCHAR", "_source_id BIGINT"] for f in spec["fields"]: base = f.removesuffix("_id") if f in (spec.get("named_m2m") or {}): @@ -2801,6 +3013,7 @@ def _cols(spec): for f in spec.get("m2m") or []: base = f.removesuffix("_ids") cols += [f"{base}_id BIGINT", f"{f} VARCHAR"] + cols += [f"{output} VARCHAR" for output in spec.get("raw_m2m") or {}] return cols @@ -2818,6 +3031,10 @@ def _ensure_columns(con, key, spec): if name not in have: con.execute(f"ALTER TABLE {key} ADD COLUMN {c}") added.append(name) + binding = getattr(_SYNC_TLS, "connection", None) + if binding and binding["legacy"]: + con.execute(f"UPDATE {key} SET _connection_id=?, _source_id=id, Store=? WHERE _connection_id IS NULL", + [binding["id"], binding["label"]]) return added @@ -2878,31 +3095,28 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal if f != "write_date"] if not fields: return 0 - marker = f"{key}.{'refresh' if repeat else 'cols'}.{','.join(sorted(fields))}" + marker = _connection_state_key(f"{key}.{'refresh' if repeat else 'cols'}.{','.join(sorted(fields))}") full_marker, remaining_progress = marker, [] operation = "column refresh" if repeat else "column backfill" st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?", [marker]).fetchone() - if st and st[0] == "done": - if not repeat: - return 0 + if repeat and st and st[0] == "done": st = None if not repeat: - # A field addition does not invalidate completed migrations. Legacy combined markers - # prove each field they name; unfinished supersets retain their existing ID checkpoint. - prefix = f"{key}.cols." - prior = con.execute("SELECT entity, phase, last_id, rows FROM _sync_state " - "WHERE starts_with(entity, ?)", [prefix]).fetchall() - completed = {field for name, phase, _last, _rows in prior if phase == "done" - for field in name[len(prefix):].split(",")} + # Field checkpoints own progress. Joined markers remain summaries for older readers. + prefix = _connection_state_key(f"{key}.cols.") + states = _column_backfill_states(con, key) + for field in fields: + phase, last_id, count = states.get(field, ("running", 0, 0)) + _checkpoint_column_fields(con, key, [field], phase, last_id, count) + completed = {field for field, point in states.items() if point[0] == "done"} fields = [field for field in fields if field not in completed] if not fields: + last_id, count = max((point[1:] for point in states.values()), default=(0, 0)) con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")]) + [marker, "done", last_id, None, count, time.strftime("%Y-%m-%d %H:%M:%S")]) return 0 - progress = {field: max(((last_id or 0, count or 0) for name, phase, last_id, count in prior - if phase != "done" and field in name[len(prefix):].split(",")), - default=(0, 0)) for field in fields} + progress = {field: states.get(field, ("running", 0, 0))[1:] for field in fields} # Finish the furthest-ahead cohort first. A new field at zero must not rewind # another field's running migration; each cohort keeps its own durable marker. last_id, count = max(progress.values()) @@ -2916,41 +3130,54 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal # interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring # completion it never reached. last, n = (st[1] or 0, st[2] or 0) if st else (0, 0) + log(f" {key}: {operation} {'+'.join(fields)} starting at id {last:,} ({n:,} rows)") if last: - log(f" {key}: {operation} resuming at id {last:,} ({n:,} rows already done)") + log(f" {key}: {operation} {'+'.join(fields)} resuming at id {last:,} ({n:,} rows already done)") + raw_m2m = spec.get("raw_m2m") or {} + requested = list(dict.fromkeys(raw_m2m.get(f, f) for f in fields)) + flattened = [f for f in fields if f not in raw_m2m] cols = [c for f in fields for c in _field_cols(f, named_m2m=spec.get("named_m2m"))] pages, changed = 0, 0 while True: recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], - fields, limit=5000, order="id asc") + requested, limit=5000, order="id asc") if not recs: break - if any(field not in rec for rec in recs for field in fields): + if any(field not in rec for rec in recs for field in requested): raise RuntimeError(f"{key}: column backfill response omitted a requested field") - rows = _flatten_records(recs, fields, named_m2m=spec.get("named_m2m")) + rows = _flatten_records(recs, flattened, named_m2m=spec.get("named_m2m"), raw_m2m=raw_m2m, spec=spec) # One set-based write per source page also bounds million-row migration elapsed time; # executemany paid a DuckDB transaction for each individual row. values = ','.join('(' + ','.join('?' for _ in range(len(cols) + 1)) + ')' for _ in rows) assignments = ', '.join(f'{c}=src.{c}' for c in cols) differs = ' OR '.join(f'dest.{c} IS DISTINCT FROM src.{c}' for c in cols) - updated = con.execute( - f'UPDATE {key} AS dest SET {assignments} ' - f'FROM (VALUES {values}) AS src(id, {", ".join(cols)}) ' - f'WHERE dest.id=src.id AND ({differs}) RETURNING dest.id', - [value for row in rows for value in [row['id']] + [row.get(c) for c in cols]] - ).fetchall() + next_id, next_count = recs[-1]["id"], n + len(recs) + con.execute("BEGIN TRANSACTION") + try: + updated = con.execute( + f'UPDATE {key} AS dest SET {assignments} ' + f'FROM (VALUES {values}) AS src(id, {", ".join(cols)}) ' + f'WHERE dest.id=src.id AND ({differs}) RETURNING dest.id', + [value for row in rows for value in [row['id']] + [row.get(c) for c in cols]] + ).fetchall() + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "running", next_id, None, next_count, time.strftime("%Y-%m-%d %H:%M:%S")]) + if not repeat: + _checkpoint_column_fields(con, key, fields, "running", next_id, next_count) + if full_marker != marker: + full_last, full_count = min([*remaining_progress, (next_id, next_count)]) + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [full_marker, "running", full_last, None, full_count, + time.strftime("%Y-%m-%d %H:%M:%S")]) + con.execute("COMMIT") + except BaseException: + con.execute("ROLLBACK") + raise if repeat: changed += len(updated) - last, n = recs[-1]["id"], n + len(recs) + last, n = next_id, next_count pages += 1 - con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) - if full_marker != marker: - full_last, full_count = min([*remaining_progress, (last, n)]) - con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - [full_marker, "running", full_last, None, full_count, - time.strftime("%Y-%m-%d %H:%M:%S")]) log(f" {key}: {operation} {'+'.join(fields)} -> id {last:,} ({n:,} rows)") # ⛔ RETURN, NOT `break`: falling out of the loop writes 'done', and declaring completion # a bounded pass never reached is the one failure this whole helper's checkpointing @@ -2966,47 +3193,65 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", [full_marker, phase, full_last, None, full_count, time.strftime("%Y-%m-%d %H:%M:%S")]) + if not repeat: + _checkpoint_column_fields(con, key, fields, "done", last, n) log(f" {key}: {operation} complete -> {n:,} rows") return changed -def _backfill_m2m(con, key, spec, log=print): - """One-time targeted backfill after a schema migration: existing rows never re-sync (their - write_date didn't move when WE added a column), so pull every record where the m2m field is - SET and update in place. Rows with the field empty keep NULL — correct and free. Completion - is a durable _sync_state marker ('.m2m.') — NOT 'column just added': a crash - between the ALTER and this backfill (seen live: the SSL-failed first run) must retry.""" +def _backfill_m2m(con, key, spec, log=print, max_batches=None): + """Commit each relation page with its cursor and resume until an empty page proves completion.""" + pages = 0 for f in spec.get("m2m") or []: - marker = f"{key}.m2m.{f}" - if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone(): + marker = _connection_state_key(f"{key}.m2m.{f}") + st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?", + [marker]).fetchone() + if st and st[0] == "done": continue base = f.removesuffix("_ids") - # PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line - # has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone - # missing with no error — a partial backfill that reads as a complete one (the - # [[no-unverifiable-aggregates]] no-silent-caps rule). - rows, last, page = [], 0, 20000 + last, count = (st[1] or 0, st[2] or 0) if st else (0, 0) + log(f" {key}: m2m {f} starting at id {last:,} ({count:,} rows)") + if last: + log(f" {key}: m2m {f} resuming at id {last:,} ({count:,} rows)") while True: recs = O.search_read(spec["model"], _base_domain(spec) + [(f, "!=", False), ("id", ">", last)], - [f], limit=page, order="id asc") + [f], limit=20000, order="id asc") if not recs: break - rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)] - last = recs[-1]["id"] - log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)") - if rows: - con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows) + if any(f not in rec for rec in recs): + raise RuntimeError(f"{key}: m2m backfill response omitted {f}") + rows = _flatten_records(recs, [], m2m=[f], spec=spec) + values = ','.join('(?,?,?)' for _ in rows) + next_id, next_count = recs[-1]["id"], count + len(recs) + con.execute("BEGIN TRANSACTION") + try: + con.execute(f"UPDATE {key} AS dest SET {base}_id=src.first_id, {f}=src.ids " + f"FROM (VALUES {values}) AS src(id, first_id, ids) WHERE dest.id=src.id", + [v for row in rows for v in (row["id"], row[f"{base}_id"], row[f])]) + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "running", next_id, None, next_count, + time.strftime("%Y-%m-%d %H:%M:%S")]) + con.execute("COMMIT") + except BaseException: + con.execute("ROLLBACK") + raise + last, count = next_id, next_count + pages += 1 + log(f" {key}: m2m {f} -> id {last:,} ({count:,} rows)") + if max_batches is not None and pages >= max_batches: + log(f" {key}: m2m {f} PAUSED at {count:,} rows (bounded; resumes)") + return con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - [marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")]) - log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated") + [marker, "done", last, None, count, time.strftime("%Y-%m-%d %H:%M:%S")]) + log(f" {key}: m2m backfill {f} complete -> {count:,} rows") def _upsert(con, key, spec, recs): if not recs: return rows = _flatten_records(recs, spec["fields"], spec.get("m2m") or (), - named_m2m=spec.get("named_m2m")) + named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec) colnames = [c.split()[0] for c in _cols(spec)] ids = [r["id"] for r in rows] con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})") @@ -3017,14 +3262,14 @@ def _upsert(con, key, spec, recs): def _state(con, key): row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?", - [key]).fetchone() + [_connection_state_key(key)]).fetchone() return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None def _save_state(con, key, phase, last_id, cursor_wd): - n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + n = con.execute(f"SELECT count(*) FROM {key} WHERE {_connection_where()}").fetchone()[0] con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""", - [key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")]) + [_connection_state_key(key), phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")]) def _base_domain(spec): @@ -3040,21 +3285,15 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print): phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log) # A recurring refresh may report changed pages to a bespoke cursor, but a # tenant pass cannot claim completion until those private pages finish too. - pending = candidate["con"].execute( - "SELECT phase FROM _sync_state WHERE starts_with(entity, ?)", [key + "."] - ).fetchall() - if phase == "live" and any(row[0] != "done" for row in pending): + pending = _connection_pending(candidate["con"], key) + if phase == "live" and any(row[1] != "done" for row in pending if not row[0].endswith(".pass")): phase = "backfill" return phase, pulled rt = _resident_odoo_runtime() if rt is None: return _sync_one(key, max_batches=max_batches, batch=batch, log=log) - from harness import mirror_store - report = {} - with mirror_store.sync_mirror("odoo", rt.key, rt, report=report, log=log, expected_tables=[key]): - phase, pulled = sync_entity(key, max_batches=max_batches, batch=batch, log=log) - report[key] = {"phase": phase, "pulled": pulled} - return phase, pulled + report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch) + return report[key]["phase"], report[key]["pulled"] def _sync_entity(key, max_batches=200, batch=BATCH, log=print): @@ -3071,21 +3310,22 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print): _paused_notice(f"sync of {key}", log) return PAUSED_PHASE, 0 spec = ENTITIES[key] + _connection_model(spec) con = connect() - _ensure_table(con, key, spec) - _ensure_columns(con, key, spec) - # ⭐ W50-T29 — THE BOUND IS PASSED HERE, so `bounded pass` means all four steps and not three. - # See `COL_BACKFILL_MAX_BATCHES`: an unattended hourly resync must not inherit a `while True`. - _backfill_columns(con, key, spec, log=log, max_batches=COL_BACKFILL_MAX_BATCHES) - if spec.get("backfill_before_delta") and not _columns_backfilled(con, key, spec): - con.close() - return "backfill", 0 - columns_ready = (not spec.get("backfill_fields") or _columns_backfilled(con, key, spec)) - _backfill_m2m(con, key, spec, log=log) - st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0} - fields = spec["fields"] + list(spec.get("m2m") or []) - pulled, refreshed = 0, 0 try: + _ensure_table(con, key, spec) + _ensure_columns(con, key, spec) + _backfill_columns(con, key, spec, log=log, max_batches=COL_BACKFILL_MAX_BATCHES) + if spec.get("backfill_before_delta") and not _columns_backfilled(con, key, spec): + return "backfill", 0 + columns_ready = (not spec.get("backfill_fields") or _columns_backfilled(con, key, spec)) + _backfill_m2m(con, key, spec, log=log, max_batches=COL_BACKFILL_MAX_BATCHES) + columns_ready = columns_ready and all( + con.execute("SELECT phase FROM _sync_state WHERE entity=?", [_connection_state_key(f"{key}.m2m.{field}")]).fetchone() + == ("done",) for field in spec.get("m2m") or []) + st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0} + fields = spec["fields"] + list(spec.get("m2m") or []) + pulled, refreshed = 0, 0 if st["phase"] != "backfill": _save_state(con, key, "catchup", st["last_id"], st["cursor_wd"]) if st["phase"] != "backfill": @@ -3093,13 +3333,14 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print): max_batches=COL_BACKFILL_MAX_BATCHES, repeat=True) if st["phase"] == "backfill": last_id = st["last_id"] or 0 + log(f" {key}: backfill starting at id {last_id:,} ({st['rows']:,} rows)") for i in range(max_batches): dom = _base_domain(spec) + [("id", ">", last_id)] recs = O.search_read(spec["model"], dom, ["id"] + fields, limit=batch, order="id asc") if not recs: # backfill complete → switch to live mode anchored at max write_date seen - wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0] + wd = con.execute(f"SELECT max(write_date) FROM {key} WHERE {_connection_where()}").fetchone()[0] _save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00") log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode") return ("live" if columns_ready else "backfill"), pulled @@ -3107,8 +3348,7 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print): last_id = recs[-1]["id"] pulled += len(recs) _save_state(con, key, "backfill", last_id, None) - if (i + 1) % 10 == 0: - log(f" {key}: backfill …{pulled:,} rows (id≤{last_id})") + log(f" {key}: backfill …{pulled:,} rows (id≤{last_id})") log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)") return "backfill", pulled # live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP @@ -3181,12 +3421,11 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print): if pulled: log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}" + (f" walking id>{tie_id}" if walking else "") + ")") - pending = con.execute("SELECT entity, phase FROM _sync_state WHERE starts_with(entity, ?)", - [key + "."]).fetchall() + pending = _connection_pending(con, key) # Recurring refreshes have their own bounded cycle, separate from schema # readiness. The public source wrapper also requires that cycle to finish. columns_ready = columns_ready and all(phase == "done" for marker, phase in pending - if not marker.startswith(key + ".refresh.")) + if not marker.startswith(_connection_state_key(key + ".refresh."))) return ("live" if columns_ready and delta_complete else "backfill"), pulled + refreshed finally: con.close() @@ -3194,6 +3433,7 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print): def _sync_one(key, max_batches=200, batch=BATCH, log=print): """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace.""" + _connection_model(ENTITIES[key]) plan = REPLACE_EACH_PASS.get(key) if plan is None or source_paused(): return _sync_entity(key, max_batches=max_batches, batch=batch, log=log) @@ -3206,12 +3446,12 @@ def _sync_one(key, max_batches=200, batch=BATCH, log=print): def _replace_scope(con, key, plan): """SQL predicate for the mirror rows a replace owns. `_replace_stage` holds the fresh read.""" if plan.get("scope") != "internal": - return "TRUE" + return _connection_where() terms = ["id IN (SELECT id FROM _replace_stage)", "location_id IN (SELECT location_id FROM _replace_stage WHERE location_id IS NOT NULL)"] if {"id", "usage"} <= set(columns_of("stock_location", cur=con) or ()): terms.append("location_id IN (SELECT id FROM stock_location WHERE usage = 'internal')") - return "(" + " OR ".join(terms) + ")" + return "(" + " OR ".join(terms) + ") AND " + _connection_where() def _replace_entity(key, log=print): @@ -3233,7 +3473,7 @@ def _replace_entity(key, log=print): if any(field not in rec for rec in recs for field in fields): raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.") rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (), - named_m2m=spec.get("named_m2m")) + named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec) # Stop on an EMPTY page: a server-side page cap returns short pages before the end. if not recs: break @@ -3271,10 +3511,10 @@ def _replace_entity(key, log=print): raise con.execute("COMMIT") if plan.get("scope") == "internal": - n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] - con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key]) + n = con.execute(f"SELECT count(*) FROM {key} WHERE {_connection_where()}").fetchone()[0] + con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, _connection_state_key(key)]) else: - _save_state(con, key, "live", max((r["id"] for r in rows), default=0), None) + _save_state(con, key, "live", max((r.get("_source_id", r["id"]) for r in rows), default=0), None) log(f" {key}: full read {len(rows):,} rows, {changed:,} changed") return "live", changed finally: @@ -3285,10 +3525,12 @@ def _replace_entity(key, log=print): def _already(con, key, rec): - return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone()) + ident = _connection_ids(con, ENTITIES[key]["model"], [rec["id"]])[rec["id"]] + return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [ident]).fetchone()) -def sync_all(entities=None, max_batches=200, log=print, after_entity=None): +def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch=BATCH, connection_id=None, + client_factory=None): selected = list(ENTITIES) if entities is None else list(entities) if entities is not None and not selected: return {} @@ -3301,6 +3543,11 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None): _paused_notice("sync", log) return {} rt = _resident_odoo_runtime() + entries = _odoo_entries(rt) if rt is not None else None + if entries == []: + raise RuntimeError("This tenant has no Odoo credentials. Reconnect Odoo before syncing.") + if connection_id not in (None, "default") and entries is not None and not any(e["id"] == connection_id for e in entries): + raise RuntimeError("The requested Odoo connection is no longer connected to this tenant") if rt is not None and restore_odoo_mirror(rt.key, rt, log=log)["state"] == "refused": raise RuntimeError("Odoo saved mirror was refused; repair store access and restart before syncing") out = {} @@ -3308,16 +3555,42 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None): publication = (mirror_store.sync_mirror("odoo", rt.key, rt, report=out, log=log, expected_tables=selected) if rt is not None else nullcontext()) with publication: - for key in selected: - t0 = time.time() - phase, pulled = sync_entity(key, max_batches=max_batches, log=log) - out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)} - if after_entity is not None: - # Beside this entity's copy, before the next pull. A hook failure never fails the pass. - try: - after_entity(key) - except Exception as exc: # noqa: BLE001 - log(f" {key}: after-entity hook failed ({type(exc).__name__})") + candidate = getattr(_SYNC_TLS, "candidate", None) + if candidate is not None: + selected = [key for key in selected if key in candidate["requested"]] + if entries is not None and not candidate.get("resuming"): + for key in selected: + candidate["con"].execute( + "DELETE FROM _sync_state WHERE entity=? OR " + "(starts_with(entity, ?) AND ends_with(entity, '.pass'))", + [key + ".pass", key + ".connection."]) + for entry in entries if entries is not None else [None]: + binding = _odoo_connection(candidate["con"], rt, entry, client_factory) if entry is not None else nullcontext() + with binding: + for key in selected: + t0 = time.time() + marker = _connection_state_key(key + ".pass") + previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None + if entry and candidate.get("resuming") and previous == ("done",): + phase, pulled = "live", 0 + else: + if entry: + candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker]) + phase, pulled = sync_entity(key, max_batches=max_batches, log=log, + **({"batch": batch} if batch != BATCH else {})) + if entry and phase == "live": + candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")]) + row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0}) + if phase != "live": + row["phase"] = phase + row["pulled"] += pulled + row["secs"] = round(row["secs"] + time.time() - t0, 1) + if after_entity is not None: + try: + after_entity(key) + except Exception as exc: # noqa: BLE001 + log(f" {key}: after-entity hook failed ({type(exc).__name__})") if rt is not None and out: persist_odoo_mirror(rt.key, rt, out, log=log) # A full pass that lands every entity in live mode latches process readiness in-process — so @@ -3339,7 +3612,17 @@ def reconcile_deletes(entities=None, log=print): report = {} selected = list(ENTITIES) if entities is None else list(entities) with mirror_store.sync_mirror("odoo", rt.key, rt, report=report, log=log, expected_tables=selected): - result = _reconcile_deletes_candidate(entities=selected, log=log) + candidate = _SYNC_TLS.candidate + selected = [key for key in selected if key in candidate["requested"]] + entries = _odoo_entries(rt) + if entries == []: + raise RuntimeError("This tenant has no Odoo credentials. Reconnect Odoo before syncing.") + result = {} + for entry in entries if entries is not None else [None]: + binding = _odoo_connection(candidate["con"], rt, entry) if entry is not None else nullcontext() + with binding: + for key, count in _reconcile_deletes_candidate(entities=selected, log=log).items(): + result[key] = result.get(key, 0) + count report.update({key: {"phase": "live"} for key in result}) return result @@ -3368,7 +3651,7 @@ def _reconcile_deletes_candidate(entities=None, log=print): # (1M+ ids for account_move_line) runs only for an entity whose count moved. try: source_n = O.get_odoo().search_count(spec["model"], _base_domain(spec)) - mirror_n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + mirror_n = con.execute(f"SELECT count(*) FROM {key} WHERE {_connection_where()}").fetchone()[0] except Exception: # noqa: BLE001 source_n, mirror_n = None, None if source_n is not None and int(source_n) == int(mirror_n): @@ -3380,9 +3663,9 @@ def _reconcile_deletes_candidate(entities=None, log=print): ["id"], limit=50000, order="id asc") if not recs: break - live_ids.update(r["id"] for r in recs) + live_ids.update(_connection_ids(con, spec["model"], [r["id"] for r in recs]).values()) last = recs[-1]["id"] - store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()} + store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key} WHERE {_connection_where()}").fetchall()} dead = store_ids - live_ids if dead: con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)") @@ -3390,8 +3673,8 @@ def _reconcile_deletes_candidate(entities=None, log=print): con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead]) con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)") con.execute("DELETE FROM _dead") - n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] - con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key]) + n = con.execute(f"SELECT count(*) FROM {key} WHERE {_connection_where()}").fetchone()[0] + con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, _connection_state_key(key)]) out[key] = len(dead) log(f" {key}: reconciled {len(dead):,} hard-deleted rows") if any(out.values()): diff --git a/platform/harness/evals.py b/platform/harness/evals.py index 85dce7348edc5a5b8326e10745e5377020a2c938..a19f0c052cf58be14076f5b1c60cc20645110b03 100644 --- a/platform/harness/evals.py +++ b/platform/harness/evals.py @@ -1,24 +1,264 @@ -"""harness/evals.py — the Analyst eval gate (OM-4, 2026-07-11). - -Runs the golden set (evals/analyst_golden.yml — truths from the parity-proven semantic layer over -CLOSED windows) against the Analyst and scores DETERMINISTICALLY: expected number found in the -answer (tolerance-aware, handles $/commas/% forms), required substrings, required tools in the -trace, required artifacts. No LLM judge in v1 — the validate() culture applied to AI answers. - -Gate rule (from the plan): run before any deploy that touches model/ or the Analyst; ship only on -pass-rate >= the bar (start 80%, tighten as the model/skills improve). - -Usage: - python -X utf8 -c "import ssl; ssl._create_default_https_context=ssl._create_unverified_context; - import harness.evals as E; E.run()" # live (needs OPENROUTER_API_KEY) - E.run(chat_fn=scripted) # offline harness test -""" +"""Score the real Analyst against a content-checked private snapshot, never the serving store.""" +import builtins +import copy +import datetime as dt +import hashlib +import json +import os import re +import tempfile from pathlib import Path +from types import ModuleType, SimpleNamespace import yaml GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml" +SNAPSHOT = GOLDEN.with_name("analyst_snapshot.json") + + +class SnapshotError(RuntimeError): + """The pinned evaluation data is absent, corrupt or incompatible with the golden set.""" + + +class _SnapshotTools: + """Run the shipped tools, semantic compiler and transforms with private module state.""" + + READS = frozenset(("list_topics", "describe_topic", "get_field_values", + "run_semantic_query", "transform_result", "make_chart", "make_table", + "make_kpi_card", "report_gap")) + + def __init__(self, manifest_path, golden, offline=False): + self._tmp = None + self._modules = {} + self.events = [] + try: + manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + ids = [item["id"] for item in golden["items"]] + if (manifest.get("version") != 1 or manifest.get("format") != "duckdb" + or len(set(ids)) != len(ids) or set(ids) != set(manifest["items"]) + or len(ids) != len(manifest["items"])): + raise ValueError("manifest must bind every golden item exactly once") + seed = manifest["seed"] + if any(seed[key] != golden["seed"][key] for key in + ("dataset", "revision", "filename", "retainedFilename", "contentSha", "bytes", "cutDate")): + raise ValueError("golden seed differs from the snapshot manifest") + if (not re.fullmatch(r"[0-9a-f]{40}", seed["revision"]) + or not re.fullmatch(r"[0-9a-f]{64}", seed["contentSha"]) + or not seed["filename"] or Path(seed["filename"]).is_absolute() + or ".." in Path(seed["filename"]).parts + or seed["retainedFilename"] != f"analyst_snapshots/{seed['contentSha']}.duckdb" + or type(seed["bytes"]) is not int or seed["bytes"] <= 0): + raise ValueError("an immutable revision, checksum and relative filename are required") + dt.date.fromisoformat(seed["cutDate"]) + self.today = dt.date.fromisoformat(manifest["clock"]).isoformat() + clock = dt.date.fromisoformat(self.today) + + class Date(dt.date): + @classmethod + def today(cls): + return cls(clock.year, clock.month, clock.day) + + class DateTime(dt.datetime): + @classmethod + def now(cls, tz=None): + value = cls(clock.year, clock.month, clock.day, tzinfo=dt.timezone.utc) + return value.astimezone(tz) if tz else value.replace(tzinfo=None) + + @classmethod + def utcnow(cls): + return cls.now() + + self._datetime = SimpleNamespace(**{**vars(dt), "date": Date, "datetime": DateTime}) + self.identity = {"seed": dict(seed), "clock": self.today, "items": ids} + override = os.environ.get("ANALYST_EVAL_SNAPSHOT_FILE") + if override: + source = Path(override) + self.identity["resolved_from"] = "exact-file-override" + else: + from huggingface_hub import hf_hub_download + from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError, RevisionNotFoundError + kwargs = dict(repo_type="dataset", token=os.environ.get("HF_TOKEN") or None, + local_files_only=offline) + try: + source = Path(hf_hub_download(seed["dataset"], seed["filename"], + revision=seed["revision"], **kwargs)) + self.identity["resolved_from"] = "pinned-revision" + except (EntryNotFoundError, RevisionNotFoundError) as exc: + if offline or isinstance(exc, LocalEntryNotFoundError): + raise + # Squashed history may recover only the content-addressed object at HEAD. + source = Path(hf_hub_download(seed["dataset"], seed["retainedFilename"], + revision="main", **kwargs)) + self.identity["resolved_from"] = "retained-content-at-head" + self._tmp = tempfile.TemporaryDirectory(prefix="analyst-eval-") + self.path = Path(self._tmp.name) / "snapshot.duckdb" + digest = hashlib.sha256() + with source.open("rb") as src, self.path.open("wb") as dst: + while chunk := src.read(8 * 1024 * 1024): + digest.update(chunk) + dst.write(chunk) + if digest.hexdigest() != seed["contentSha"] or self.path.stat().st_size != seed["bytes"]: + raise ValueError("snapshot checksum mismatch") + self.path.chmod(0o400) + import harness.datastore as DS + self._datastore = SimpleNamespace( + DB_PATH=self.path, ENTITIES=copy.deepcopy(DS.ENTITIES), + ready=lambda: True, ro_cursor=self._cursor, entity_live=self._entity_live, + _columns_backfilled=DS._columns_backfilled) + cur = self._cursor() + try: + for table in manifest["required_tables"]: + if not re.fullmatch(r"[a-z_]+", table): + raise ValueError("invalid required table") + cur.execute(f'SELECT 1 FROM "{table}" LIMIT 0') + if not self._entity_live(table): + raise ValueError(f"snapshot source {table} has not completed its backfill") + finally: + cur.close() + self._package = ModuleType("harness") + self._package.datastore = self._datastore + self._package.telemetry = SimpleNamespace(log=self.log) + self._core = ModuleType("core") + self._core.odoo = SimpleNamespace(_ctx=lambda: None, sale_line_domain=self._refuse) + self._core.store = SimpleNamespace(available=lambda: False) + self._semantic = self._load("semantic") + tenant = yaml.safe_load((self._semantic.MODEL_DIR / "tenant.yml").read_text(encoding="utf-8")) + self._semantic._tenant = lambda: copy.deepcopy(tenant) + self._semantic._store_con = self._cursor + query = self._semantic.store_query + + def pinned_query(*args, **kwargs): + if any(kwargs.get(key) is not None for key in ("runtime", "user", "cur")): + raise SnapshotError("evaluation queries cannot bind another data source") + kwargs["today"] = self.today + return query(*args, **kwargs) + + self._semantic.store_query = pinned_query + self._semantic._model() # Freeze source metadata before the first model turn. + self._tools = self._load("tools") + self._RESULTS = self._tools._RESULTS + except Exception as exc: + self.close() + # Hub exceptions can contain signed URLs. Report only the dependency identity. + if isinstance(exc, (ValueError, KeyError)): + why = str(exc) + else: + why = type(exc).__name__ + raise SnapshotError(f"pinned Analyst snapshot unavailable or invalid: {why}; " + "supply ANALYST_EVAL_SNAPSHOT_FILE with the manifest's exact bytes; " + "no serving-store fallback") from None + + @staticmethod + def _refuse(*args, **kwargs): + raise SnapshotError("live source access is forbidden during a pinned evaluation") + + def _entity_live(self, table): + cur = self._cursor() + try: + row = cur.execute("SELECT phase FROM _sync_state WHERE entity = ?", [table]).fetchone() + spec = self._datastore.ENTITIES.get(table) or {} + columns_ready = (not spec.get("backfill_fields") + or self._datastore._columns_backfilled(cur, table, spec)) + return bool(row and row[0] == "live" and columns_ready) + finally: + cur.close() + + def _cursor(self): + import duckdb + connection = duckdb.connect(str(self.path), read_only=True, + config={"enable_external_access": False, "threads": 1}) + clock = self.today + + class Cursor: + def execute(self, sql, params=None): + # Pin SQL clock expressions without rewriting quoted data. + parts = re.split(r"('(?:''|[^'])*')", sql) + for i in range(0, len(parts), 2): + parts[i] = re.sub(r"\bCURRENT_DATE\b(?:\s*\(\s*\))?|\btoday\s*\(\s*\)", + f"DATE '{clock}'", parts[i], flags=re.I) + parts[i] = re.sub(r"\bCURRENT_TIMESTAMP\b|\bnow\s*\(\s*\)", + f"TIMESTAMP '{clock} 00:00:00'", parts[i], flags=re.I) + connection.execute("".join(parts), params or []) + return self + + def __getattr__(self, name): + return getattr(connection, name) + + return Cursor() + + def _import(self, name, globals=None, locals=None, fromlist=(), level=0): + if name == "datetime": + return self._datetime + if name in ("harness.semantic", "harness.tools", "harness.transforms"): + module = self._load(name.split(".")[-1]) + return module if fromlist else self._package + if name == "harness.datastore": + return self._datastore if fromlist else self._package + if name == "harness.telemetry": + return self._package.telemetry if fromlist else self._package + if name == "harness": + for child in fromlist: + if child in ("semantic", "tools", "transforms"): + self._load(child) + elif child not in ("datastore", "telemetry"): + self._refuse() + return self._package + if name == "core.odoo": + return self._core.odoo if fromlist else self._core + if name == "core" and tuple(fromlist) == ("store",): + return self._core + if name in ("core", "modules", "connectors", "odoo_client"): + self._refuse() + if name.startswith(("harness.", "core.", "modules.", "connectors.")) and name not in ( + "harness.filter_sql", "core.filter_contract", "modules.finance_spine", "modules.amazon_profit"): + self._refuse() + return builtins.__import__(name, globals, locals, fromlist, level) + + def _load(self, name): + if name not in self._modules: + path = Path(__file__).with_name(name + ".py") + module = ModuleType("harness." + name) + module.__file__ = str(path) + module.__builtins__ = {**vars(builtins), "__import__": self._import} + self._modules[name] = module + setattr(self._package, name, module) + # Private globals preserve the shipped implementation without swapping sys.modules. + exec(compile(path.read_text(encoding="utf-8"), str(path), "exec"), module.__dict__) + if name == "tools": + import time + import uuid + epoch = dt.datetime.fromisoformat(self.today).replace(tzinfo=dt.timezone.utc).timestamp() + module.time = SimpleNamespace( + strftime=lambda fmt: time.strftime(fmt, time.gmtime(epoch))) + counter = iter(range(1, 10**12)) + module.uuid = SimpleNamespace(uuid4=lambda: uuid.UUID(int=next(counter) << 88)) + return self._modules[name] + + def log(self, kind, **fields): + self.events.append({"kind": kind, **fields}) + + def reset(self): + self._RESULTS.clear() + self._tools._RESULT_CALLERS.clear() + self.events.clear() + + def list_topics(self): + return self._tools.list_topics() + + def openai_tools(self): + return [tool for tool in self._tools.openai_tools() if tool["function"]["name"] in self.READS] + + def dispatch(self, name, args): + if name not in self.READS: + return {"ok": False, "error": "tool unavailable in a read-only pinned evaluation"} + return self._tools.dispatch(name, args) + + def close(self): + if self._tmp is not None: + if self.path.exists(): + self.path.chmod(0o600) + self._tmp.cleanup() + self._tmp = None def _numbers(text): @@ -97,32 +337,54 @@ def _print_cost(cost): if cost["cost_missing"] else "")) -def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True): +def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True, snapshot_path=SNAPSHOT): + """Score golden questions on their pinned snapshot, including when chat_fn is supplied.""" + import harness.analyst as A + + golden = yaml.safe_load(GOLDEN.read_text(encoding="utf-8")) + items = golden["items"] + if only: + unknown = set(only) - {item["id"] for item in items} + if unknown: + raise ValueError(f"unknown golden items: {sorted(unknown)}") + items = [item for item in items if item["id"] in only] + source = _SnapshotTools(snapshot_path, golden, offline=chat_fn is not None) + patient, call_item = os.environ.get("ANALYST_PATIENT"), A.CALL_ITEM + try: + return _run(items, source, chat_fn, model, bar, verbose) + finally: + A.CALL_ITEM = call_item + if patient is None: + os.environ.pop("ANALYST_PATIENT", None) + else: + os.environ["ANALYST_PATIENT"] = patient + source.close() + + +def _run(items, source, chat_fn, model, bar, verbose): """Run the golden set. Returns {passed, total, pass_rate, gate_ok, results, cost}. ANALYST_CALL_LOG names a JSONL cost ledger; `cost` is None when it is unset. ANALYST_EVAL_PACE_S sets the sleep before each item (default 6 s). """ import harness.analyst as A - items = yaml.safe_load(GOLDEN.read_text(encoding="utf-8"))["items"] - if only: - items = [i for i in items if i["id"] in only] results, passed = [], 0 import os as _os import time as _time _os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast infra_fails = 0 aborted = False - pace = float(_os.environ.get("ANALYST_EVAL_PACE_S") or 6) + pace = 0 if chat_fn is not None else float(_os.environ.get("ANALYST_EVAL_PACE_S") or 6) start = dict(A.CALL_TOTALS) for item in items: _time.sleep(pace) # pace the free-tier per-minute windows (Groq/Cerebras) A.CALL_ITEM = item["id"] - kw = {"chat_fn": chat_fn} if chat_fn else {} + source.reset() + kw = {"chat_fn": chat_fn} if chat_fn is not None else {} if model: kw["model"] = model try: - res = A.ask(item["ask"], telemetry_kind="analyst_eval", **kw) + res = A.ask(item["ask"], telemetry_kind="analyst_eval", data_source=source, **kw) ok, reasons = _check(item, res) infra_fails = 0 except Exception as e: @@ -138,7 +400,8 @@ def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True): A.CALL_ITEM = None passed += ok results.append({"id": item["id"], "ok": ok, "reasons": reasons, - "usage": res.get("usage"), "iterations": res.get("iterations")}) + "usage": res.get("usage"), "iterations": res.get("iterations"), + "snapshot_sha": source.identity["seed"]["contentSha"]}) if verbose: mark = "OK " if ok else "XX " print(f" {mark}{item['id']:18s}" + ("" if ok else f" — {'; '.join(reasons)[:90]}")) @@ -154,11 +417,12 @@ def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True): f"{passed}/{len(ran) or 1} of the items that actually ran passed. " "Gate is INCONCLUSIVE (not a model failure); re-run when quota recovers.") return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False, - "inconclusive": True, "results": results, "cost": cost} + "inconclusive": True, "results": results, "cost": cost, + "snapshot": source.identity} rate = passed / len(items) if items else 0.0 gate = rate >= bar if verbose: print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): " f"{'PASS' if gate else 'FAIL'}") return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate, - "results": results, "cost": cost} + "results": results, "cost": cost, "snapshot": source.identity} diff --git a/platform/harness/filter_sql.py b/platform/harness/filter_sql.py index fd74c2c682a7f7328a4d0b970e39d7ee42980b8b..5826db69a860cb1473ee171ddafb174299450fe9 100644 --- a/platform/harness/filter_sql.py +++ b/platform/harness/filter_sql.py @@ -256,6 +256,8 @@ def _value_sql(spec): return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), {places}), 0)" if t == 'date': return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')" + if t == 'checkbox': + return f"COALESCE(TRY_CAST(NULLIF(TRIM(CAST({e} AS VARCHAR)), '') AS BOOLEAN), FALSE)" return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))" @@ -454,6 +456,21 @@ def _compile_leaf(rule, columns, state): if rspec.get('aggregate'): state['aggregate'] = True + if t == 'checkbox': + if op not in {'eq', 'neq'}: + raise ValueError('checkbox filters support eq, neq, isEmpty and isNotEmpty') + operator = '=' if op == 'eq' else '<>' + if rhs_v is not None: + return used(f"({v} {operator} {rhs_v})") + raw = rule.get('value') + if isinstance(raw, bool): + flag = raw + elif str(raw).strip().lower() in {'true', '1', 'false', '0'}: + flag = str(raw).strip().lower() in {'true', '1'} + else: + raise ValueError('checkbox equality requires a Boolean value') + return used(f"({v} {operator} ?)", [flag]) + if t in NUMERIC_TYPES: if rhs_v is not None: return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, []) diff --git a/platform/harness/meta_store.py b/platform/harness/meta_store.py index 373c6510ae519e3ec7e7dde6cad31afe269d03c2..d0b73da230d1a5234536627d286fddd0b1f0bdcc 100644 --- a/platform/harness/meta_store.py +++ b/platform/harness/meta_store.py @@ -181,12 +181,9 @@ def _slices(days, size): cur = stop + timedelta(days=1) return out -#: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is -#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is -#: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it -#: is reported as a problem, never absorbed. -MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200) PAGE = int(os.environ.get("META_PAGE_SIZE") or 100) +ORIGIN_COLUMNS = ("_connection_id", "store", "_row_key") +INSERT_BATCH_ROWS = 500 class MetaError(RuntimeError): @@ -214,7 +211,7 @@ def _env_token(): return "" -def token(rt=None, *, tenant_key=None): +def token(rt=None, *, tenant_key=None, connection_id=None): """The tenant's stored Meta credential first; environment fallback only for Royal Imports. ⭐⭐ W49-T13 / D-202 — THE PRECEDENCE IS THE WHOLE DEFECT, NOT THE CALL. This module read @@ -227,6 +224,7 @@ def token(rt=None, *, tenant_key=None): With no runtime, only an explicitly supplied `royal-imports` tenant key authorizes the standalone tenant #0 fallback. Missing or unresolved non-Royal tenants get "". When both a runtime and requested tenant key are supplied, they must agree before any credential is read. + Named connections never fall back to the deployment token. ⚠ THE FIELD NAME IS RESOLVED, NOT ASSUMED. A Meta credential is conventionally stored under `access_token`, but `token` and `api_key` are both accepted by the connector that stores it, @@ -245,7 +243,8 @@ def token(rt=None, *, tenant_key=None): if tenant_key is not None and runtime_key != requested: return "" import core.keychain as _keychain - fields = _keychain.meta_creds(rt) or {} + fields = (_keychain.meta_creds(rt) if connection_id is None else + _keychain.meta_creds(rt, connection_id=connection_id)) or {} for name in _META_TOKEN_FIELDS: stored = str(fields.get(name) or "").strip() if stored: @@ -254,7 +253,7 @@ def token(rt=None, *, tenant_key=None): # The exception may quote a credential field; it must never leave this resolver. pass royal = runtime_key == "royal-imports" if rt is not None else requested == "royal-imports" - if royal: + if royal and connection_id is None: try: return _env_token() except Exception: # noqa: BLE001 @@ -309,9 +308,9 @@ def _walk(path, tok, log, **params): silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always every column per row. - -> (rows, hit_page_cap) + -> (rows, incomplete_read) """ - out, pages = [], 0 + out, cursors = [], set() limit = int(params.pop("limit", None) or PAGE) after, waited = None, 0 while True: @@ -340,16 +339,13 @@ def _walk(path, tok, log, **params): return out, True raise out.extend(body.get("data") or []) - pages += 1 after = ((body.get("paging") or {}).get("cursors") or {}).get("after") has_next = bool((body.get("paging") or {}).get("next")) and bool(after) if not has_next: return out, False - if pages >= MAX_PAGES: - log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. " - f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the " - f"window with META_INSIGHTS_DAYS.") - return out, True + if after in cursors: + raise MetaError(f"Graph repeated a paging cursor on /{path}. Retry this sync.") + cursors.add(after) def _cell(value): @@ -402,13 +398,7 @@ def _coltype(name, rows=None): def _ensure(con, table, fields, rows=None): - """Create or widen the table, with every column typed from `rows` when they are available. - - ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a - column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type - disagreement drops and recreates rather than limping on with a column that refuses every - insert. The alternative is a mirror that is permanently unwritable for one bad guess. - """ + """Create missing columns and widen conflicting types to text without discarding history.""" want = {f: _coltype(f, rows) for f in fields} # DuckDB raises CatalogException for table_info on a new tenant's absent table. # Check existence first so the first pull reaches the create path below. @@ -420,9 +410,10 @@ def _ensure(con, table, fields, rows=None): if have: clash = [f for f, ty in want.items() if f in have and have[f] != ty and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")] - if clash: - con.execute(f"DROP TABLE {table}") - have = {} + for f in clash: + # A sibling's value shape must never discard another connection's history. + con.execute(f"ALTER TABLE {table} ALTER COLUMN {f} TYPE VARCHAR") + have[f] = "VARCHAR" if not have: cols = ", ".join(f"{f} {want[f]}" for f in fields) con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})" @@ -435,20 +426,91 @@ def _ensure(con, table, fields, rows=None): def _upsert(con, table, fields, rows): - """Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so - a re-sync updates in place and can never append a second copy of the same object.""" + """Upsert by connection identity in bounded SQL batches, retaining vendor IDs and older rows.""" if not rows: return 0 cols = list(fields) - ids = [str(r.get("id") or "") for r in rows] - q = ",".join("?" for _ in ids) - con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids) - con.executemany( - f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})", - [[_cell(r.get(c)) for c in cols] for r in rows]) + identity = "_row_key" if "_row_key" in cols else "id" + rows = list({str(r.get(identity) or ""): r for r in rows}.values()) + for start in range(0, len(rows), INSERT_BATCH_ROWS): + part = rows[start:start + INSERT_BATCH_ROWS] + ids = [str(r.get(identity) or "") for r in part] + q = ",".join("?" for _ in ids) + con.execute(f"DELETE FROM {table} WHERE {identity} IN ({q})", ids) + con.executemany( + f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})", + [[_cell(r.get(c)) for c in cols] for r in part]) return len(rows) +def _connection_rows(rt, tenant_key, report): + """Resolve each business connection by its stable keychain identity.""" + from core import keychain + try: + entries = sorted((row for row in keychain.list_entries(rt) if row.get("type") == "meta_ads"), + key=lambda row: (int(row.get("slot") or 1), str(row["id"]))) + default = keychain.resolve_connection_id(rt, "meta_ads") + except Exception: + report["problems"].append("The Meta Ads connections could not be read. Retry this sync.") + return [] + connections, eligible = [], False + for row in entries: + cid = str(row.get("connection_id") or row["id"]) + if keychain.resolve_connection_id(rt, "meta_ads", cid) is None: + continue + eligible = True + value = token(rt, tenant_key=tenant_key, connection_id=cid) + if value: + connections.append((cid, str(row.get("label") or cid), value, cid == default)) + else: + report["problems"].append(f"{row.get('label') or cid}: the Meta Ads credential is unavailable. Test this connection again.") + if not eligible: + value = token(rt, tenant_key=tenant_key) + if value: + connections.append(("", "Meta Ads", value, True)) + return connections + + +def _row_key(connection_id, vendor_id): + """Keep legacy identities when unnamed and separate named connections without changing vendor IDs.""" + return (json.dumps([connection_id, str(vendor_id or "")], separators=(",", ":")) + if connection_id else str(vendor_id or "")) + + +def _origin_rows(fields, rows, connection_id, label): + """Add local identity and label columns without changing the measured vendor fields.""" + links = {"account_id": "_account_key", "campaign_id": "_campaign_key", "adset_id": "_adset_key", "ad_id": "_ad_key"} + cols = list(fields) + list(ORIGIN_COLUMNS) + [v for k, v in links.items() if k in fields] + out = [] + for raw in rows: + row = dict(raw, _connection_id=connection_id, store=label, + _row_key=_row_key(connection_id, raw.get("id"))) + row.update({target: _row_key(connection_id, raw.get(source)) + for source, target in links.items() if source in fields}) + out.append(row) + return cols, out + + +def _refresh_origin(con, table, connection_id, label, legacy=True): + """Claim legacy default rows once and refresh labels on stored history.""" + if legacy: + con.execute(f"UPDATE {table} SET _connection_id = ?, store = ? " + "WHERE _connection_id IS NULL", [connection_id, label]) + cols = {row[1] for row in con.execute(f"PRAGMA table_info('{table}')").fetchall()} + names = [name for name in ("id", "account_id", "campaign_id", "adset_id", "ad_id") if name in cols] + missing = con.execute(f"SELECT {', '.join(names)} FROM {table} " + "WHERE _row_key IS NULL AND _connection_id = ?", [connection_id]).fetchall() + for values in missing: + row = dict(zip(names, values)) + updates = {"_row_key": _row_key(connection_id, row["id"])} + updates.update({"_" + name[:-3] + "_key": _row_key(connection_id, row[name]) + for name in names if name != "id"}) + con.execute(f"UPDATE {table} SET " + ", ".join(f"{name} = ?" for name in updates) + + " WHERE id = ? AND _connection_id = ? AND _row_key IS NULL", + list(updates.values()) + [row["id"], connection_id]) + con.execute(f"UPDATE {table} SET store = ? WHERE _connection_id = ?", [label, connection_id]) + + def _runtime_for(tenant_key): """The tenant runtime for `tenant_key`, or None when this process cannot resolve one. @@ -474,7 +536,6 @@ def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insight # runtime that cannot be resolved retains the requested tenant identity: only the standalone # Royal caller may use the deployment token; an unresolved non-Royal tenant fails closed. rt = _runtime_for(tenant_key) - tok = tok or token(rt, tenant_key=tenant_key) # ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already @@ -484,12 +545,14 @@ def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insight # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT. days = int(insights_days or INSIGHTS_DAYS) - report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []} - if not tok: + report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": [], "connections": []} + if rt is not None and str(getattr(rt, "key", "") or "").strip().lower() != str(tenant_key).strip().lower(): + report["problems"].append("The runtime does not match the requested tenant. No Meta rows were pulled.") + return report + connections = [("", "Meta Ads", tok, True)] if tok else _connection_rows(rt, tenant_key, report) + if not connections: report["problems"].append( - "No Meta Ads credential could be resolved for this tenant, so nothing was pulled. " - "Add the tenant's credential under Settings > Keychains. The deployment token is " - "available only to Royal Imports.") + "No Meta Ads credential could be resolved for this tenant. Open Meta Ads Setup to add a connection.") return report if rt is None: @@ -499,7 +562,22 @@ def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insight expected = list(SPECS) + ([INSIGHTS_TABLE] if insights else []) with mirror_store.sync_mirror("meta", tenant_key, rt, report=report, log=log, expected_tables=expected) as con: - return _pull_into(con, tok, log, insights, days, report) + for cid, label, value, legacy in connections: + report["connections"].append({"id": cid, "label": label}) + if tok: + _pull_into(con, value, log, insights, days, report) + continue + try: + _pull_into(con, value, log, insights, days, report, cid, label, legacy) + except MetaError as exc: + reason = str(exc).replace(value, "") + report["problems"].append(f"{label}: {reason[:200]}. Retry the sync.") + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + ["meta", "partial" if report["problems"] else "done", 0, + f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}", + sum(t["in_mirror"] for t in report["tables"].values()), + time.strftime("%Y-%m-%d %H:%M:%S")]) + return report def _record_table_state(con, table, total, capped, cursor=""): @@ -509,18 +587,20 @@ def _record_table_state(con, table, total, capped, cursor=""): time.strftime("%Y-%m-%d %H:%M:%S")]) -def _pull_into(con, tok, log, insights, days, report): +def _pull_into(con, tok, log, insights, days, report, connection_id="", label="Meta Ads", legacy=True): """Fetch Meta rows into the tenant connection owned by the mirror sync context.""" - accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE) - report["accounts"] = [a.get("id") for a in accts] + accts, accounts_partial = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE) + report["accounts"].extend(a.get("id") for a in accts) + if accounts_partial: + report["problems"].append(f"{label}: the account listing is incomplete. Retry the sync.") if not accts: - report["problems"].append("the token reaches no ad accounts") + report["problems"].append(f"{label}: the token reaches no ad accounts") return report # ── the five entity levels ──────────────────────────────────────────────────────────────── for table, (edge, fields, _parent) in SPECS.items(): - rows, capped = [], False + rows, capped = [], accounts_partial for acct in accts: aid = acct["id"] if edge is None: @@ -529,19 +609,23 @@ def _pull_into(con, tok, log, insights, days, report): got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE) rows.extend(got) capped = capped or hit + fields, rows = _origin_rows(fields, rows, connection_id, label) _ensure(con, table, fields, rows) + _refresh_origin(con, table, connection_id, label, legacy) n = _upsert(con, table, fields, rows) total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0] - report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped} + previous = report["tables"].get(table, {}) + capped = capped or previous.get("capped", False) + report["tables"][table] = {"pulled": previous.get("pulled", 0) + n, "in_mirror": total, "capped": capped} _record_table_state(con, table, total, capped) log(f" {table:<20} pulled {n:>6} mirror total {total:>6}" - + (" !! PAGE CAP HIT" if capped else "")) + + (" incomplete read" if capped else "")) if capped: - report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}") + report["problems"].append(f"{label}: {table} has an incomplete read. Retry the sync.") # ── the daily Insights grain ────────────────────────────────────────────────────────────── if insights: - rows, capped = [], False + rows, capped = [], accounts_partial for acct in accts: for since, until in _slices(days, INSIGHTS_SLICE): got, hit = _walk(f"{acct['id']}/insights", tok, log, @@ -556,22 +640,20 @@ def _pull_into(con, tok, log, insights, days, report): r["id"] = ":".join(str(r.get(k) or "") for k in (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop")) fields = ["id"] + INSIGHT_FIELDS + fields, rows = _origin_rows(fields, rows, connection_id, label) _ensure(con, INSIGHTS_TABLE, fields, rows) + _refresh_origin(con, INSIGHTS_TABLE, connection_id, label, legacy) n = _upsert(con, INSIGHTS_TABLE, fields, rows) total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0] - report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped} + previous = report["tables"].get(INSIGHTS_TABLE, {}) + capped = capped or previous.get("capped", False) + report["tables"][INSIGHTS_TABLE] = {"pulled": previous.get("pulled", 0) + n, "in_mirror": total, "capped": capped} _record_table_state(con, INSIGHTS_TABLE, total, capped, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}") if capped: - report["problems"].append(f"{INSIGHTS_TABLE}: stopped at MAX_PAGES={MAX_PAGES}") + report["problems"].append(f"{label}: {INSIGHTS_TABLE} has an incomplete read. Retry the sync.") log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}" - + (" !! PAGE CAP HIT" if capped else "")) - - con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - ["meta", "partial" if report["problems"] else "done", 0, - f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}", - sum(t["in_mirror"] for t in report["tables"].values()), - time.strftime("%Y-%m-%d %H:%M:%S")]) + + (" incomplete read" if capped else "")) return report diff --git a/platform/harness/mirror_store.py b/platform/harness/mirror_store.py index b60a1131c62feef15b2fe0a31560a67f7f06fe9a..0951045775f580991a895781d32ab041306ea73b 100644 --- a/platform/harness/mirror_store.py +++ b/platform/harness/mirror_store.py @@ -323,6 +323,8 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa raise completed, current = _sync_result(connector, report) current = current and _catch_up_complete and Path(rt.datastore_path).exists() + if current and connector == "odoo": + current = pending_sync_entities(connector, rt.key, rt) is None newest = None # An Amazon pass that published part of the source still moves the measured record dates. if current or (completed and connector == "amazon" and Path(rt.datastore_path).exists()): @@ -1286,6 +1288,32 @@ def _open_candidate(store, tenant_key, rt, published): raise +def pending_sync_entities(connector, tenant_key, rt): + """Return a matching private cohort, or None when no unpublished candidate remains.""" + store = for_connector(connector) + store._mirror_cache_key(tenant_key, rt) + path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb") + with store.lock: + if not path.exists(): + return None + published, _ = store._connect(tenant_key, rt) + try: + candidate = datastore.apply_limits(datastore.duckdb.connect(str(path))) + try: + base = candidate.execute("SELECT identity FROM _candidate_base").fetchone() + if not base or base[0] != datastore._publication(published, store.connector).get("snapshot_id", "unknown"): + return None + pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()} + if store.connector == "odoo": + pending.update(row[0].split(".", 1)[0] for row in store._state_rows(candidate) + if row[1] not in {"live", "done"}) + return sorted(pending) + finally: + candidate.close() + finally: + published.close() + + def _hold_previous(store, candidate, published, tables): """Give each held table its published rows and data markers, inside the candidate's open transaction.""" prior = store._tables(published) @@ -1365,11 +1393,15 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub note("candidate open", opened) pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()} requested = set(store.spec.tables if expected_tables is None else expected_tables) + if odoo and pending: + # Finish the admitted cohort before refreshing entities that already completed. + requested &= pending if requested: candidate.executemany("INSERT OR IGNORE INTO _candidate_pending VALUES (?)", [[name] for name in sorted(requested)]) datastore._SYNC_TLS.candidate = {"tenant": tenant_key, "connector": store.connector, - "path": Path(rt.datastore_path).resolve(), "con": candidate} + "path": Path(rt.datastore_path).resolve(), "con": candidate, "requested": requested, + "resuming": bool(pending)} try: yield candidate finally: diff --git a/platform/harness/mirror_tokens.py b/platform/harness/mirror_tokens.py index 09c7547f1e2c6315e1c4524060edf5e0f9946d9b..b84aae3d33c90a819857e0f9d39af25a8949ae97 100644 --- a/platform/harness/mirror_tokens.py +++ b/platform/harness/mirror_tokens.py @@ -20,10 +20,10 @@ import time from core import sheet_binding from core.sheet_binding import fingerprint -# D-2670: database -> (key, table rows). The key holds the mirror file's size and mtime (and its -# WAL's), the install generation, the receipts and the schema, so any committed write rescans. +# Cache source fingerprints only. Permission decisions remain with each request's scoped reader. _TABLE_ROWS = {} _TABLE_ROWS_LOCK = threading.Lock() +_TABLE_ROWS_MAX = 128 def _file_marks(cur): @@ -32,6 +32,25 @@ def _file_marks(cur): return datastore.mirror_file_marks(cur) +def _store_identity(st): + """Separate tenant namespaces and physical stores even when they share a mirror file.""" + while hasattr(st, '_runtime'): + st = st._runtime + document_store = getattr(st, '_document_store', None) + reader = (document_store() if callable(document_store) else + st._store() if callable(getattr(st, '_store', None)) else st) + namespace = getattr(st, 'store_key', str)('__mirror_token__') + return (getattr(st, 'key', None), getattr(reader, 'tenant_slug', None), namespace, id(reader)) + + +def _revision(cur, marks, generation, sources): + """Decline reuse during writes or transactions whose snapshot can outlive file marks.""" + from harness import datastore + if marks is None or getattr(cur, '_in_transaction', False) or datastore._ACTIVE_WRITES[0]: + return None + return fingerprint([marks, generation, datastore._MIRROR_WRITES[0], sources]) + + def mirror_change_token(st, database): """One SQL token read over source tables, not N binding queries or Python row copies. @@ -62,6 +81,15 @@ def mirror_change_token(st, database): cur = mirror_cursor(st) # An in-memory mirror cannot show an in-place write, so it is never cached. marks = _file_marks(cur) + slot = (_store_identity(st), database) + revision = _revision(cur, marks, generation, [sources, sorted(optional)]) + with _TABLE_ROWS_LOCK: + cached = _TABLE_ROWS.get(slot) + if (revision is not None and cached is not None and len(cached) == 4 + and cached[2] == revision + and _revision(cur, _file_marks(cur), datastore.mirror_generation(), + [sources, sorted(optional)]) == revision): + return cached[3] placeholders = ",".join("?" for _ in sources) # An ENTITIES table or a link target may be absent from a seed. A connector's own source may not. present = set() @@ -87,19 +115,17 @@ def mirror_change_token(st, database): sizes = ("SELECT 'size', table_name, CAST(estimated_size AS VARCHAR), '' FROM duckdb_tables() " f"WHERE table_name IN ({','.join('?' for _ in hashed)})") key = None if marks is None else fingerprint([ - marks, list(generation), hashed, sorted(optional - present), + marks, list(generation), datastore._MIRROR_WRITES[0], hashed, sorted(optional - present), cur.execute(" UNION ALL ".join([*meta, sizes]) + " ORDER BY 1,2,3,4", [*meta_params, *hashed]).fetchall()]) with _TABLE_ROWS_LOCK: - cached = _TABLE_ROWS.get(database) - if key is not None and cached is not None and cached[0] == key: + cached = _TABLE_ROWS.get(slot) + if revision is not None and key is not None and cached is not None and cached[0] == key: table_rows = cached[1] else: started = time.perf_counter() table_rows = [tuple(row) for row in cur.execute(" UNION ALL ".join(clauses)).fetchall()] if key is not None: - with _TABLE_ROWS_LOCK: - _TABLE_ROWS[database] = (key, table_rows) # One line per scan: an opening GET that prints none read no mirror table. print(f"[sheet-token] scan db={database} tables={len(hashed)} " f"ms={round((time.perf_counter() - started) * 1000)}", flush=True) @@ -112,6 +138,13 @@ def mirror_change_token(st, database): raise RuntimeError("The connector has no sync receipt.") absent = sorted(optional - present) # Absent names stay in the token, so a table that appears later changes it. - return fingerprint([rows, {"absent": absent}]) if absent else fingerprint(rows) + token = fingerprint([rows, {"absent": absent}]) if absent else fingerprint(rows) + after = _revision(cur, _file_marks(cur), datastore.mirror_generation(), [sources, sorted(optional)]) + if revision is not None and after == revision: + with _TABLE_ROWS_LOCK: + if slot not in _TABLE_ROWS and len(_TABLE_ROWS) >= _TABLE_ROWS_MAX: + _TABLE_ROWS.pop(next(iter(_TABLE_ROWS))) + _TABLE_ROWS[slot] = (key, table_rows, revision, token) + return token sheet_binding.set_mirror_token_provider(mirror_change_token) diff --git a/platform/harness/runtime.py b/platform/harness/runtime.py index a0d6acba7ea845dc3b98edecf6e1d90d70c13683..a4a88a74a2195716c86f892aef647d6bfcf61c83 100644 --- a/platform/harness/runtime.py +++ b/platform/harness/runtime.py @@ -297,6 +297,10 @@ class TenantRuntime: def put(self, name, data): return self._store().put(self.store_key(name), data) + def transaction(self, connection): + """Enlist this tenant's writes in an existing receipt transaction.""" + return self._document_store().transaction(connection) + def update(self, name, fn, flush="sync", expect_rev=None, audit_row=None): """Read-modify-write this tenant's bucket, optionally rev-guarded and audited (C3 / R9). @@ -334,6 +338,9 @@ class TenantRuntime: stays in the ring. The second bottleneck is `core/store.py::update`, in nobody's fence. """ st = self._store() + slug = getattr(st, 'tenant_slug', None) + if slug is not None and slug != self.key: + raise DatastoreMismatch('The store handle belongs to another tenant.') key = self.store_key(name) if expect_rev is None and audit_row is None: return st.update(key, fn, flush=flush) diff --git a/platform/modules/account_health.py b/platform/modules/account_health.py index 180e659548f57d82fe08cadcca3d3630e69673a9..ca089d8518e90c4048d59b10c8ee3cee3efc8fbf 100644 --- a/platform/modules/account_health.py +++ b/platform/modules/account_health.py @@ -33,6 +33,7 @@ DEFAULTS = {"alert_distance_pct": 20.0} DUE_DAYS = 7 EMPTY = "No account health data" INBOUND_DAYS = 30 +PAGE_SIZE = 50 KEY_SETTINGS = "account_health_settings" KEY_ALERTS = "account_health_alerts" @@ -151,27 +152,38 @@ def _num(value): return None -def _snapshot_rows(cur): - """(snapshot date, {marketplace: {column: value}}) for the newest snapshot; rows of one marketplace merge.""" +def _snapshot_sql(cur): + """Merge the newest snapshot in report order inside SQL.""" cols = _columns(cur, AH_TABLE) if cur is not None else set() if not cols: - return None, {} + return None, "", [] date_col = "_snapshot_date" in cols newest = None if date_col: newest = cur.execute(f'SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {AH_TABLE}').fetchone()[0] where = 'WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date_col and newest is not None else "" order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else "" - res = cur.execute(f"SELECT * FROM {AH_TABLE} {where}{order}", [newest] if where else []) + market = 'COALESCE(CAST("marketplace_id" AS VARCHAR), \'\')' if "marketplace_id" in cols else "''" + fields = [f'{market} AS marketplace_id'] + for col in sorted(cols - {"marketplace_id"}): + quoted = '"' + col.replace('"', '""') + '"' + fields.append(f'FIRST({quoted}{order}) FILTER (WHERE {quoted} IS NOT NULL) AS {quoted}') + sql = f"SELECT {', '.join(fields)} FROM {AH_TABLE} {where} GROUP BY 1" + return (str(newest) if newest is not None else None), sql, [newest] if where else [] + + +def _snapshot_rows(cur): + """Read the complete merged snapshot for background sync and independent validation.""" + newest, sql, args = _snapshot_sql(cur) + if not sql: + return newest, {} + res = cur.execute(sql, args) names = [d[0] for d in res.description] out = {} for row in res.fetchall(): rec = dict(zip(names, row)) - merged = out.setdefault(str(rec.get("marketplace_id") or ""), {}) - for key, value in rec.items(): - if merged.get(key) is None and value is not None: - merged[key] = value - return (str(newest) if newest is not None else None), out + out[str(rec.get("marketplace_id") or "")] = {k: v for k, v in rec.items() if v is not None} + return newest, out def _inbound_count(cur, snapshot_date): @@ -187,15 +199,10 @@ def _inbound_count(cur, snapshot_date): except (TypeError, ValueError): end = _now().date() start = end - dt.timedelta(days=INBOUND_DAYS) - count = 0 - for (value,) in cur.execute(f'SELECT CAST("{col}" AS VARCHAR) FROM {INBOUND_TABLE}').fetchall(): - try: - day = dt.date.fromisoformat(str(value)[:10]) - except (TypeError, ValueError): - continue - if start < day <= end: - count += 1 - return count + return int(cur.execute( + f'SELECT COUNT(*) FROM {INBOUND_TABLE} ' + f'WHERE TRY_CAST(SUBSTR(CAST("{col}" AS VARCHAR), 1, 10) AS DATE) > ? ' + f'AND TRY_CAST(SUBSTR(CAST("{col}" AS VARCHAR), 1, 10) AS DATE) <= ?', [start, end]).fetchone()[0]) # --------------------------------------------------------------------------- metric status @@ -320,6 +327,134 @@ def summary(tenant_key, *, st=None): "statuses": list(METRIC_STATUSES)} +def _metric_sql(cur, merged, snapshot_date, distance): + """Project searchable metrics in SQL before selecting a page.""" + cols = _columns(cur, AH_TABLE) + + def field(name, numeric=False): + quoted = '"' + name.replace('"', '""') + '"' + if name not in cols: + return "NULL" + return f"TRY_CAST(REPLACE(TRIM(CAST({quoted} AS VARCHAR)), ',', '') AS DOUBLE)" if numeric else quoted + + specs = [(p, label, "percent", p + "_rate", denominator) for p, label, denominator in RATE_METRICS] + specs += [(p, label, "count", p + "_defects_count", None) for p, label in COUNT_METRICS] + specs.append((WARNINGS[0], WARNINGS[1], "count", WARNINGS[2], None)) + selects = [] + for rank, (key, label, unit, value_col, denominator) in enumerate(specs): + target_col = key + "_target_value" + if value_col not in cols and target_col not in cols: + continue + label_sql = label.replace("'", "''") + window = key + "_reporting_date_range_reporting_date_" + selects.append( + f"SELECT '{key}' AS key, '{label_sql}' AS label, '{unit}' AS unit, marketplace_id AS market, " + f"{rank} AS metric_order, {field(value_col, True)} AS raw, {field(target_col, True)} AS raw_target, " + f"{field(key + '_target_condition')} AS condition, {field(key + '_status')} AS amazon_status, " + f"{field(denominator, True) if denominator else 'NULL'} AS orders, " + f"CAST({field(window + 'from')} AS VARCHAR) AS window_from, " + f"CAST({field(window + 'to')} AS VARCHAR) AS window_to FROM merged " + f"WHERE {field(value_col)} IS NOT NULL OR {field(target_col)} IS NOT NULL") + inbound = _inbound_count(cur, snapshot_date) + if inbound is not None: + selects.append(f"SELECT '{INBOUND[0]}', '{INBOUND[1]}', 'count', '', {len(specs)}, " + f"{int(inbound)}::DOUBLE, NULL::DOUBLE, NULL::VARCHAR, NULL::VARCHAR, " + "NULL::DOUBLE, NULL::VARCHAR, NULL::VARCHAR") + if not selects: + selects.append("SELECT '' AS key, '' AS label, '' AS unit, '' AS market, 0 AS metric_order, " + "NULL::DOUBLE AS raw, NULL::DOUBLE AS raw_target, NULL::VARCHAR AS condition, " + "NULL::VARCHAR AS amazon_status, NULL::DOUBLE AS orders, " + "NULL::VARCHAR AS window_from, NULL::VARCHAR AS window_to WHERE FALSE") + return (f"WITH merged AS ({merged}), raw_metrics AS ({' UNION ALL '.join(selects)}), " + "scaled AS (SELECT *, CASE WHEN unit = 'percent' AND " + "GREATEST(COALESCE(ABS(raw), 0), COALESCE(ABS(raw_target), 0)) <= 1 THEN 100.0 ELSE 1.0 END AS scale " + "FROM raw_metrics), values_ AS (SELECT *, " + "CASE WHEN unit = 'percent' THEN ROUND(raw * scale, 6) ELSE raw END AS value, " + "CASE WHEN unit = 'percent' THEN ROUND(raw_target * scale, 6) ELSE raw_target END AS target, " + "key || CASE WHEN market = '' THEN '' ELSE '@' || market END AS id " + "FROM scaled), distance_ AS (SELECT *, " + "CASE WHEN CONTAINS(UPPER(condition), 'GREATER') THEN " + "(CASE WHEN unit = 'percent' THEN 100.0 ELSE GREATEST(value, target) END) - value ELSE value END AS v, " + "CASE WHEN CONTAINS(UPPER(condition), 'GREATER') THEN " + "(CASE WHEN unit = 'percent' THEN 100.0 ELSE GREATEST(value, target) END) - target ELSE target END AS t " + "FROM values_), metrics AS (SELECT *, CASE " + "WHEN unit = 'percent' AND orders = 0 THEN 'No orders' " + "WHEN value IS NULL THEN 'No data' WHEN target IS NULL THEN 'No target' " + "WHEN CASE WHEN COALESCE(CONTAINS(UPPER(condition), 'EQUAL'), FALSE) OR t = 0 " + "THEN v > t ELSE v >= t END THEN 'At risk' " + f"WHEN t > 0 AND v >= t * (1 - {float(distance)} / 100.0) THEN 'Near target' ELSE 'Good' END AS status " + "FROM distance_)") + + +def _page_bounds(offset, limit, total): + size = _limit(limit) + start = max(0, int(offset or 0)) + if start >= total: + start = max(0, (total - 1) // size * size) + return start, size + + +def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=None, st=None): + """Read one metric page or one exact metric without loading the complete snapshot.""" + st = _rt(tenant_key, st) + distance = settings(tenant_key, st=st)["alertDistancePct"] + out = {"state": "empty", "message": EMPTY, "rating": None, "ratingStatus": None, + "snapshotDate": None, "marketplaceId": None, "marketCount": 0, "metrics": [], + "alertDistancePct": distance, "statuses": list(METRIC_STATUSES), "metricTotal": 0} + start, size = _page_bounds(offset, limit, 0) + with _reader(tenant_key, st) as cur: + date, merged, args = _snapshot_sql(cur) + if merged: + market_count = cur.execute(f"SELECT COUNT(*) FROM ({merged})", args).fetchone()[0] + if market_count: + cols = _columns(cur, AH_TABLE) + rating = 'TRY_CAST("account_health_rating_ahr_score" AS DOUBLE)' \ + if "account_health_rating_ahr_score" in cols else "NULL::DOUBLE" + rating_status = '"account_health_rating_ahr_status"' \ + if "account_health_rating_ahr_status" in cols else "NULL::VARCHAR" + first = cur.execute(f"SELECT marketplace_id, {rating} AS rating, {rating_status} " + f"FROM ({merged}) ORDER BY rating IS NULL, marketplace_id LIMIT 1", args).fetchone() + sql = _metric_sql(cur, merged, date, distance) + where, filters = ("id = ?", [str(metric)]) if metric is not None else ( + "CONTAINS(LOWER(CONCAT_WS(' ', id, label, market, status, amazon_status, value, target, " + "CASE WHEN unit = 'percent' THEN printf('%.2f%%', value) ELSE CAST(value AS VARCHAR) END, " + "CASE WHEN unit = 'percent' THEN printf('%.2f%%', target) ELSE CAST(target AS VARCHAR) END, " + "CASE WHEN target IS NULL THEN 'No target' ELSE (CASE WHEN CONTAINS(UPPER(condition), 'GREATER') " + "THEN CASE WHEN CONTAINS(UPPER(condition), 'EQUAL') THEN 'At least' ELSE 'Above' END " + "ELSE CASE WHEN CONTAINS(UPPER(condition), 'EQUAL') OR target = 0 THEN 'At most' ELSE 'Below' END END) " + "|| ' ' || CASE WHEN unit = 'percent' THEN printf('%.2f%%', target) ELSE CAST(target AS VARCHAR) END END, " + "condition, window_from, window_to)), ?)", [str(search or "").strip().lower()]) + total = cur.execute(sql + f" SELECT COUNT(*) FROM metrics WHERE {where}", [*args, *filters]).fetchone()[0] + start, size = _page_bounds(offset, limit, total) + res = cur.execute(sql + f" SELECT * FROM metrics WHERE {where} " + "ORDER BY CASE WHEN key = 'inbound_noncompliance' THEN 1 ELSE 0 END, " + "market, metric_order LIMIT ? OFFSET ?", [*args, *filters, 1 if metric else size, start]) + names = [d[0] for d in res.description] + rows = [] + for values in res.fetchall(): + rec = dict(zip(names, values)) + scale = _rate_scale(rec["raw"], rec["raw_target"]) if rec["unit"] == "percent" else 1.0 + value = rec["raw"] + target = rec["raw_target"] + if rec["unit"] == "percent": + value = None if value is None else round(value * scale, 6) + target = None if target is None else round(target * scale, 6) + row = _row(rec["id"], rec["key"], rec["market"], rec["label"], rec["unit"], + None if rec["orders"] == 0 else value, target, rec["condition"], + rec["amazon_status"], distance, no_orders=rec["orders"] == 0) + row.update(windowFrom=_day(rec["window_from"]), windowTo=_day(rec["window_to"])) + if rec["unit"] == "percent": + row.update(raw=rec["raw"], scale=scale, orders=rec["orders"]) + rows.append(row) + out.update(state="ready", message=None, rating=first[1], ratingStatus=_plain(first[2]), + snapshotDate=date, marketplaceId=first[0] or None, marketCount=market_count, + metrics=rows, metricTotal=total) + if metric is not None and not out["metrics"]: + raise KeyError(metric) + return {**out, "metricOffset": start, "metricLimit": size, + "cursor": str(start + size) if start + size < out["metricTotal"] else None} + + # --------------------------------------------------------------------------- settings and owner #: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit). @@ -508,9 +643,55 @@ def raise_alerts(tenant_key, view=None, *, st=None, now=None): return created +def _pg_document(st, tenant_key): + """Resolve a checked tenant store for bounded authored-document queries.""" + from core import store, store_pg + resolve = getattr(st, "_document_store", None) + backend = resolve() if callable(resolve) else st + if backend is store and store.backend() == "pg": + backend = store._d() + if not isinstance(backend, store_pg.PgStore): + return None + if backend.tenant_slug != str(tenant_key).strip().lower(): + raise PermissionError("The account health store belongs to another tenant.") + return backend + + +def _document_key(st, name): + namespace = getattr(st, "store_key", None) + return namespace(name) if callable(namespace) else name + + +def _history_pg(st, backend, metric, cursor, limit): + from core import store_pg + name = _document_key(st, KEY_HISTORY) + with store_pg._pool().connection() as con: + entry = con.execute(store_pg._q( + "SELECT e.key, e.value ->> 'label', e.value ->> 'unit' FROM {tbl} kv " + "CROSS JOIN LATERAL jsonb_each(COALESCE(kv.value -> 'metrics', '{{}}'::jsonb)) e " + "WHERE kv.key = %s AND (e.key = %s OR split_part(e.key, '@', 1) = %s) " + "ORDER BY (e.key = %s) DESC, e.key LIMIT 1", "store_kv", backend.tenant_slug), + (name, str(metric), str(metric), str(metric))).fetchone() + if not entry: + raise KeyError(metric) + sql = ("FROM {tbl} kv CROSS JOIN LATERAL jsonb_each(COALESCE(" + "kv.value #> %s::text[], '{{}}'::jsonb)) p WHERE kv.key = %s AND (%s::text IS NULL OR p.key < %s)") + args = (["metrics", entry[0], "points"], name, cursor or None, cursor or None) + size = _limit(limit) + total = con.execute(store_pg._q("SELECT COUNT(*) " + sql, "store_kv", backend.tenant_slug), args).fetchone()[0] + rows = con.execute(store_pg._q("SELECT p.key, p.value " + sql + " ORDER BY p.key DESC LIMIT %s", + "store_kv", backend.tenant_slug), (*args, size)).fetchall() + return {"metric": entry[0], "label": entry[1], "unit": entry[2], + "points": [{"day": day, **point} for day, point in rows], + "cursor": rows[-1][0] if len(rows) < total and rows else None} + + def history(tenant_key, metric, *, cursor=None, limit=400, st=None): """One metric's daily points, newest first, paged by a date cursor. No cap: every day is reachable.""" st = _rt(tenant_key, st) + backend = _pg_document(st, tenant_key) if limit is not None else None + if backend is not None: + return _history_pg(st, backend, metric, cursor, limit) metrics = _doc(st, KEY_HISTORY).get("metrics") or {} mid = str(metric or "") if mid not in metrics: @@ -580,20 +761,50 @@ def _public(alert): "note": alert.get("note"), "createdAt": alert.get("createdAt"), "updatedAt": alert.get("updatedAt")} -def items(tenant_key, user=None, *, status=None, cursor=None, limit=200, st=None): +def _items_pg(st, backend, status, search, cursor, offset, limit, metric): + from core import store_pg + where = ("FROM {tbl} kv CROSS JOIN LATERAL jsonb_each(COALESCE(kv.value -> 'items', '{{}}'::jsonb)) e " + "WHERE kv.key = %s AND (%s::text IS NULL OR e.value ->> 'status' = %s) " + "AND (%s::text IS NULL OR e.value ->> 'metricId' = %s) " + "AND strpos(lower(concat_ws(' ', e.value ->> 'title', e.value ->> 'label', e.value ->> 'reason', " + "e.value ->> 'owner', e.value ->> 'status', e.value ->> 'due', e.value ->> 'marketplaceId')), %s) > 0 " + "AND (%s::text IS NULL OR CASE WHEN e.key ~ '^h[0-9]+$' THEN substring(e.key from 2)::bigint ELSE 0 END < %s)") + args = (_document_key(st, KEY_ALERTS), status, status, metric, metric, str(search or "").strip().lower(), + cursor or None, _seq(cursor) if cursor else None) + with store_pg._pool().connection() as con: + total = con.execute(store_pg._q("SELECT COUNT(*) " + where, "store_kv", backend.tenant_slug), args).fetchone()[0] + start, size = _page_bounds(offset, limit, total) + rows = con.execute(store_pg._q( + "SELECT e.value - 'history' " + where + " ORDER BY " + "CASE WHEN %s::text IS NOT NULL AND e.value ->> 'status' IN ('Open', 'Investigating') THEN 0 ELSE 1 END, " + "CASE WHEN e.key ~ '^h[0-9]+$' THEN substring(e.key from 2)::bigint ELSE 0 END DESC, e.key " + "LIMIT %s OFFSET %s", "store_kv", backend.tenant_slug), (*args, metric, size, start)).fetchall() + page = [_public(row[0]) for row in rows] + return {"items": page, "cursor": page[-1]["id"] if start + size < total and page else None, + "statuses": list(STATUSES), "total": total, "offset": start, "limit": size} + + +def items(tenant_key, user=None, *, status=None, search="", cursor=None, offset=0, limit=200, metric=None, st=None): """C4: every alert, newest first, paged by cursor. No cap: the cursor reaches every row.""" if status is not None and status not in STATUSES: raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") st = _rt(tenant_key, st) + backend = _pg_document(st, tenant_key) if limit is not None else None + if backend is not None: + return _items_pg(st, backend, status, search, cursor, offset, limit, metric) + needle = str(search or "").strip().lower() rows = [a for a in (_doc(st, KEY_ALERTS).get("items") or {}).values() - if status is None or a.get("status") == status] - rows.sort(key=lambda a: _seq(a.get("id")), reverse=True) + if (status is None or a.get("status") == status) and (metric is None or a.get("metricId") == metric) + and needle in " ".join(str(a.get(k) or "") for k in ( + "title", "label", "reason", "owner", "status", "due", "marketplaceId")).lower()] + rows.sort(key=lambda a: (0 if metric is not None and a.get("status") in OPEN_STATUSES else 1, -_seq(a.get("id")))) if cursor: rows = [a for a in rows if _seq(a.get("id")) < _seq(cursor)] - page = rows if limit is None else rows[:_limit(limit)] - more = len(page) < len(rows) + start, size = _page_bounds(offset, limit, len(rows)) if limit is not None else (0, len(rows)) + page = rows[start:start + size] + more = start + len(page) < len(rows) return {"items": [_public(a) for a in page], "cursor": page[-1]["id"] if more and page else None, - "statuses": list(STATUSES)} + "statuses": list(STATUSES), "total": len(rows), "offset": start, "limit": size} def update_item(tenant_key, user, item_id, *, owner=None, status=None, due=None, note=None, st=None, @@ -691,7 +902,9 @@ def validate(tenant_key, *, st=None, **period): if col not in cols: continue where = ' WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date is not None else "" - q = (f'SELECT CAST("marketplace_id" AS VARCHAR), MAX(TRY_CAST("{col}" AS DOUBLE)) ' + order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else "" + q = (f'SELECT CAST("marketplace_id" AS VARCHAR), FIRST(TRY_CAST("{col}" AS DOUBLE){order}) ' + f'FILTER (WHERE "{col}" IS NOT NULL) ' f'FROM {AH_TABLE}{where} GROUP BY 1') for market, value in cur.execute(q, [date] if date is not None else []).fetchall(): independent[(key, str(market or ""))] = value diff --git a/platform/modules/aged_stock.py b/platform/modules/aged_stock.py index b9c01b935ef279ef5207c57d86cd202db109f83f..2c27f107fb0dde72e5a5939a62291cc99ed056fb 100644 --- a/platform/modules/aged_stock.py +++ b/platform/modules/aged_stock.py @@ -25,6 +25,7 @@ import datetime as dt import hashlib import json import re +import time from pathlib import Path APP_LABEL = "Aged stock" @@ -44,10 +45,12 @@ LIMIT = ("Each aged stock document is limited to 128 MB by the store. About 40,0 "of monthly fees fit in the items document, and history splits by month. Split the items " "document by action when it nears that size.") UNKNOWN_CURRENCY = "UNKNOWN" +MAX_PAGE_SIZE = 200 KEY_SETTINGS = "aged_stock_settings" KEY_ITEMS = "aged_stock_items" KEY_HISTORY = "aged_stock_history" +KEY_REFRESH = "aged_stock_refresh" HISTORY_PREFIX = "aged_stock_history_" STORAGE = "amzn_storage_fees" @@ -96,6 +99,38 @@ def _doc(st, key): return value if isinstance(value, dict) else {} +def _sql_scope(st): + """Resolve the bound Postgres store and its tenant namespace without a default tenant.""" + from core import store_pg + backend = st._document_store() if callable(getattr(st, "_document_store", None)) else st + if not isinstance(backend, store_pg.PgStore): + raise RuntimeError("The aged stock screen requires a store with server paging.") + tenant = backend.tenant_slug + if getattr(st, "key", tenant) != tenant: + raise PermissionError("The aged stock store belongs to another tenant.") + return store_pg, tenant, getattr(st, "store_key", str) + + +def _select(st, sql, params): + """Read one bounded SQL result through the tenant store's checked pool.""" + from core import request_timing + pg, tenant, key = _sql_scope(st) + started = time.perf_counter() + with pg._pool().connection() as con: + row = con.execute(pg._q(sql, "store_kv", tenant), params(key)).fetchone() + value = row[0] if row else None + value = json.loads(value) if isinstance(value, str) else value + request_timing.record_store("aged_stock_page", key(KEY_ITEMS), bytes_read=len(json.dumps(value).encode()), + milliseconds=(time.perf_counter() - started) * 1000) + return value + + +def _paths(st, name, paths): + """Read selected nested values without transferring their document.""" + pg, tenant, key = _sql_scope(st) + return pg._get_json_paths(key(name), paths, tenant_slug=tenant) + + def item_id_for(key): """A stable item id from the SKU (or the FNSKU when no SKU is known), so a refresh never duplicates.""" return "as_" + hashlib.sha1(str(key).encode("utf-8")).hexdigest()[:10] @@ -191,6 +226,16 @@ def _columns(cur, table): return set() +def _read_rows(cur, sql, params=(), size=200): + """Consume background mirror reads in bounded batches.""" + result = cur.execute(sql, params) + while True: + rows = result.fetchmany(size) + if not rows: + return + yield from rows + + def _newest(cols, table): """A WHERE clause that keeps the newest `_snapshot_date` only, or every row without one.""" if "_snapshot_date" not in cols: @@ -239,9 +284,9 @@ def _fnsku_map(cur): cols = _columns(cur, table) if not {sku_col, fn_col} <= cols: continue - rows = cur.execute(f"SELECT {_col(cols, fn_col)}, MIN({_col(cols, sku_col)}) FROM {table} " + rows = _read_rows(cur, f"SELECT {_col(cols, fn_col)}, MIN({_col(cols, sku_col)}) FROM {table} " f"WHERE {_col(cols, fn_col)} IS NOT NULL AND {_col(cols, sku_col)} IS NOT NULL " - f"GROUP BY 1").fetchall() + f"GROUP BY 1") for fnsku, sku in rows: out.setdefault(fnsku, sku) return out @@ -255,7 +300,7 @@ def _storage(cur): sql = (f"SELECT {storage_month_sql(cols)}, {_col(cols, 'fnsku')}, {_col(cols, 'asin')}, " f"MAX({_col(cols, 'product_name')}), {_currency_sql(cols)}, " f"SUM({_dbl(cols, 'estimated_monthly_storage_fee')}), COUNT(*) FROM {STORAGE} GROUP BY 1, 2, 3, 5") - return cur.execute(sql).fetchall() + return _read_rows(cur, sql) def _long_term(cur): @@ -267,7 +312,7 @@ def _long_term(cur): f"{_col(cols, 'asin')}, MAX({_col(cols, 'product_name')}), {_currency_sql(cols)}, " f"{_col(cols, 'surcharge_age_tier')}, SUM({_dbl(cols, 'amount_charged')}), " f"SUM({_dbl(cols, 'qty_charged')}), COUNT(*) FROM {LONG_TERM} GROUP BY 1, 2, 3, 4, 6, 7") - return cur.execute(sql).fetchall() + return _read_rows(cur, sql) def _removals(cur): @@ -279,7 +324,7 @@ def _removals(cur): f"{_col(cols, 'fnsku')}, {_col(cols, 'asin')}, MAX({_col(cols, 'product_name')}), " f"SUM({_dbl(cols, 'sellable_quantity')}), SUM({_dbl(cols, 'sellable_removal_quantity')}), " f"SUM({_dbl(cols, 'unsellable_quantity')}) FROM {REMOVALS} GROUP BY 1, 2, 3, 4") - return cur.execute(sql).fetchall() + return _read_rows(cur, sql) STRANDED_COLS = ("sku", "fnsku", "asin", "product_name", "stranded_reason", "primary_action", "fulfillable_qty", @@ -294,7 +339,7 @@ def _stranded(cur): return None sql = (f"SELECT {', '.join(_col(cols, c) for c in STRANDED_COLS)} FROM {STRANDED} " f"WHERE {_newest(cols, STRANDED)}") - return [dict(zip(STRANDED_COLS, r)) for r in cur.execute(sql).fetchall()] + return (dict(zip(STRANDED_COLS, r)) for r in _read_rows(cur, sql)) PLANNING_COLS = (("sku", "fnsku", "asin", "product_name", "currency", "recommended_action") @@ -313,7 +358,7 @@ def _planning(cur): sql = (f"SELECT {', '.join(_col(cols, c) for c in PLANNING_COLS)} FROM {PLANNING} " f"WHERE {_newest(cols, PLANNING)} AND {_col(cols, 'sku')} IS NOT NULL") out = {} - for raw in cur.execute(sql).fetchall(): + for raw in _read_rows(cur, sql): r = dict(zip(PLANNING_COLS, raw)) p = out.setdefault(r["sku"], {"bands": {b: None for b, _l, _d in BANDS}, "tiers": {}, "nums": {}, "text": {}}) @@ -760,9 +805,9 @@ def history(tenant_key, key, *, st=None): st = _rt(tenant_key, st) out = {"removals": [], "stranded": []} for month in _doc(st, KEY_HISTORY).get("months") or (): - doc = _doc(st, _month_key(month)) - for kind, field in (("removals", "snapshot"), ("stranded", "date")): - for at, point in ((doc.get(kind) or {}).get(key) or {}).items(): + selected = _paths(st, _month_key(month), [(kind, key) for kind in out]) + for (kind, field), points in zip((("removals", "snapshot"), ("stranded", "date")), selected): + for at, point in (points or {}).items(): out[kind].append({field: at, **point}) out["removals"].sort(key=lambda p: p["snapshot"]) out["stranded"].sort(key=lambda p: p["date"]) @@ -771,43 +816,70 @@ def history(tenant_key, key, *, st=None): # --------------------------------------------------------------------------- summary -def summary(tenant_key, *, st=None): - """Month trend and totals per currency, the next tier list by money at stake, and counts.""" +SUMMARY_SQL = """ +WITH doc AS MATERIALIZED (SELECT value FROM {tbl} WHERE key = %s), +live AS MATERIALIZED ( + SELECT e.value AS v FROM doc, LATERAL jsonb_each(COALESCE(doc.value->'items', '{{}}'::jsonb)) e + WHERE COALESCE(e.value->>'inReport', 'true') <> 'false' +), trend AS MATERIALIZED ( + SELECT p.value->>'currency' AS currency, p.value->>'month' AS month, + SUM(CAST(p.value->>'storage' AS DOUBLE PRECISION)) AS storage, + SUM(CAST(p.value->>'long_term' AS DOUBLE PRECISION)) AS long_term + FROM live, LATERAL jsonb_array_elements(COALESCE(v->'fee_history', '[]'::jsonb)) p + GROUP BY 1, 2 +), stakes AS ( + SELECT s.key AS currency, SUM(CAST(s.value AS DOUBLE PRECISION)) AS at_stake + FROM live, LATERAL jsonb_each_text(COALESCE(v->'at_stake', '{{}}'::jsonb)) s GROUP BY 1 +), currencies AS (SELECT currency FROM trend UNION SELECT currency FROM stakes), +totals AS ( + SELECT c.currency, COALESCE(SUM(t.storage), 0) AS storage, COALESCE(SUM(t.long_term), 0) AS long_term, + COALESCE(MAX(s.at_stake), 0) AS at_stake FROM currencies c + LEFT JOIN trend t ON t.currency = c.currency LEFT JOIN stakes s ON s.currency = c.currency GROUP BY 1 +), months AS MATERIALIZED (SELECT DISTINCT month FROM trend), bounds AS ( + SELECT count(*) AS total, CASE WHEN count(*) = 0 THEN 0 + ELSE LEAST(%s, ((count(*) - 1) / %s)::bigint * %s) END AS start FROM months +), month_page AS (SELECT month FROM months ORDER BY month LIMIT %s OFFSET (SELECT start FROM bounds)) +SELECT jsonb_build_object( + 'meta', (SELECT value - 'items' FROM doc), 'empty', NOT EXISTS(SELECT 1 FROM live), + 'nextTierCount', (SELECT count(*) FROM live WHERE v->>'next_tier' IS NOT NULL), + 'statusCounts', (SELECT jsonb_object_agg(status, n) FROM + (SELECT v->>'status' AS status, count(*) AS n FROM live WHERE v->>'status' IS NOT NULL GROUP BY 1) c), + 'actionCounts', (SELECT jsonb_object_agg(action, n) FROM + (SELECT v->>'action' AS action, count(*) AS n FROM live WHERE v->>'action' IS NOT NULL GROUP BY 1) c), + 'totals', (SELECT jsonb_object_agg(currency, jsonb_build_object('storage', storage, 'long_term', long_term, + 'total', storage + long_term, 'at_stake', at_stake)) FROM totals), + 'trend', (SELECT jsonb_object_agg(currency, points) FROM + (SELECT currency, jsonb_agg(jsonb_build_object('month', month, 'storage', storage, 'long_term', long_term, + 'total', storage + long_term) ORDER BY month) AS points + FROM trend WHERE month IN (SELECT month FROM month_page) GROUP BY currency) c), + 'firstMonth', (SELECT MIN(month) FROM months), 'lastMonth', (SELECT MAX(month) FROM months), + 'totalMonths', (SELECT total FROM bounds), 'offset', (SELECT start FROM bounds)) +""" + + +def _page_size(value): + """Bound one page without limiting the number of reachable records.""" + try: + size = int(value) + except (TypeError, ValueError): + raise ValueError("The page size must be a whole number.") from None + if size < 1: + raise ValueError("The page size must be at least 1.") + return min(size, MAX_PAGE_SIZE) + + +def summary(tenant_key, *, offset=0, limit=24, st=None): + """Aggregate fees and counts in SQL and return one month page without SKU rows.""" st = _rt(tenant_key, st) - doc = _doc(st, KEY_ITEMS) + offset, limit = max(0, int(offset)), _page_size(limit) + out = _select(st, SUMMARY_SQL, lambda key: (key(KEY_ITEMS), offset // limit * limit, limit, limit, limit)) + doc = out.pop("meta") or {} ready = doc.get("state") == "ready" - live = [i for i in (doc.get("items") or {}).values() if i.get("inReport", True)] - trend, totals = {}, {} - for item in live: - for point in item.get("fee_history") or (): - month = trend.setdefault(point["currency"], {}).setdefault( - point["month"], {"storage": 0.0, "long_term": 0.0, "total": 0.0}) - for key in ("storage", "long_term"): - month[key] += point[key] - month["total"] += point["storage"] + point["long_term"] - for cur, amount in (item.get("at_stake") or {}).items(): - totals.setdefault(cur, {"storage": 0.0, "long_term": 0.0, "total": 0.0, "at_stake": 0.0}) - totals[cur]["at_stake"] += amount - for cur, months in trend.items(): - t = totals.setdefault(cur, {"storage": 0.0, "long_term": 0.0, "total": 0.0, "at_stake": 0.0}) - for m in months.values(): - for key in ("storage", "long_term", "total"): - t[key] += m[key] - next_tier = sorted(({"id": i["id"], "sku": i.get("sku"), "fnsku": i.get("fnsku"), "title": i.get("title"), - **i["next_tier"]} for i in live if i.get("next_tier")), - key=lambda r: (r["at_stake"] is None, -(r["at_stake"] or 0), str(r.get("sku") or ""))) - by_status = {s: 0 for s in STATUSES} - by_action = {a: 0 for a in ACTIONS} - for item in live: - if item.get("status") in by_status: - by_status[item["status"]] += 1 - if item.get("action") in by_action: - by_action[item["action"]] += 1 - return {"ready": ready, "notReady": None if ready else (doc.get("message") or NOT_READY), - "empty": not live, "notices": list(doc.get("notices") or ()), - "primaryCurrency": doc.get("primaryCurrency"), - "trend": {cur: [{"month": m, **v} for m, v in sorted(months.items())] for cur, months in trend.items()}, - "totals": totals, "nextTier": next_tier, "statusCounts": by_status, "actionCounts": by_action, + return {**out, "trend": out.get("trend") or {}, "totals": out.get("totals") or {}, "monthLimit": limit, + "ready": ready, "notReady": None if ready else (doc.get("message") or NOT_READY), + "notices": list(doc.get("notices") or ()), "primaryCurrency": doc.get("primaryCurrency"), + "statusCounts": {s: int((out.get("statusCounts") or {}).get(s, 0)) for s in STATUSES}, + "actionCounts": {a: int((out.get("actionCounts") or {}).get(a, 0)) for a in ACTIONS}, "refreshedAt": doc.get("refreshedAt"), "lastError": doc.get("last_error"), "limit": LIMIT} @@ -824,7 +896,7 @@ def _cursor_offset(cursor): if not cursor: return 0 try: - return int(json.loads(base64.urlsafe_b64decode(str(cursor).encode()).decode())["o"]) + return max(0, int(json.loads(base64.urlsafe_b64decode(str(cursor).encode()).decode())["o"])) except Exception: # noqa: BLE001 raise ValueError("The page cursor is not valid. Start from the first page.") from None @@ -834,7 +906,7 @@ def _cursor(offset): def _public(item): - out = {k: v for k, v in item.items() if k != "history"} + out = {k: v for k, v in item.items() if k not in ("history", "fee_history", "fees")} out.setdefault("title", item.get("sku") or item.get("key")) return out @@ -844,45 +916,106 @@ def _matches(item, needle): return any(needle in str(item.get(k) or "").lower() for k in ("sku", "fnsku", "title", "asin")) -def items(tenant_key, user=None, *, status=None, action=None, cursor=None, limit=200, q=None, st=None): +PAGE_SQL = """ +WITH doc AS MATERIALIZED (SELECT value FROM {tbl} WHERE key = %s), +searched AS MATERIALIZED ( + SELECT e.key AS iid, e.value->>'status' AS status, e.value->>'action' AS action, + COALESCE(e.value->>'inReport', 'true') AS report, COALESCE(e.value->>'key', '') AS sku, + e.value->>'next_tier' IS NOT NULL AS has_tier, + CAST(e.value->'next_tier'->>'at_stake' AS DOUBLE PRECISION) AS tier_stake, + (SELECT MAX(CAST(value AS DOUBLE PRECISION)) FROM + jsonb_each_text(COALESCE(e.value->'at_stake', '{{}}'::jsonb))) AS stake + FROM doc, LATERAL jsonb_each(COALESCE(doc.value->'items', '{{}}'::jsonb)) e + WHERE %s = '' OR strpos(lower(COALESCE(e.value->>'sku', '')), %s) > 0 + OR strpos(lower(COALESCE(e.value->>'fnsku', '')), %s) > 0 + OR strpos(lower(COALESCE(e.value->>'title', '')), %s) > 0 + OR strpos(lower(COALESCE(e.value->>'asin', '')), %s) > 0 +), filtered AS MATERIALIZED ( + SELECT * FROM searched WHERE (CAST(%s AS TEXT) IS NULL OR status = %s) + AND (CAST(%s AS TEXT) IS NULL OR action = %s) + AND (NOT %s OR (has_tier AND report <> 'false')) +), bounds AS ( + SELECT count(*) AS total, CASE WHEN count(*) = 0 THEN 0 + ELSE LEAST(%s, ((count(*) - 1) / %s)::bigint * %s) END AS start FROM filtered +), ranked AS ( + SELECT iid, row_number() OVER (ORDER BY + CASE WHEN %s THEN 'true' ELSE report END DESC, + CASE WHEN %s THEN 0 ELSE CASE action WHEN 'Remove' THEN 0 WHEN 'Price down' THEN 1 + WHEN 'Push with ads' THEN 2 ELSE 3 END END, + CASE WHEN %s THEN tier_stake ELSE COALESCE(stake, 0) END DESC NULLS LAST, sku COLLATE "C", iid) AS n + FROM filtered +), page AS (SELECT iid, n FROM ranked ORDER BY n LIMIT %s OFFSET (SELECT start FROM bounds)), +projected AS (SELECT (doc.value->'items')->page.iid AS v, page.n FROM page CROSS JOIN doc) +SELECT jsonb_build_object('meta', (SELECT value - 'items' FROM doc), + 'total', (SELECT total FROM bounds), 'offset', (SELECT start FROM bounds), + 'items', (SELECT jsonb_agg(v - 'history' - 'fee_history' - 'fees' ORDER BY n) FROM projected)) +""" + + +def items(tenant_key, user=None, *, status=None, action=None, next_tier=False, cursor=None, limit=200, q=None, st=None): """C4: every item matching q, filtered by status and action, paged by an opaque cursor with no total cap.""" if status is not None and status not in STATUSES: raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") if action is not None and action not in ACTIONS: raise ValueError("Action must be one of: " + ", ".join(ACTIONS) + ".") - try: - limit = int(limit) - except (TypeError, ValueError): - raise ValueError("The page size must be a whole number.") from None - if limit < 1: - raise ValueError("The page size must be at least 1.") + limit = _page_size(limit) st = _rt(tenant_key, st) - doc = _doc(st, KEY_ITEMS) - every = list((doc.get("items") or {}).values()) needle = str(q or "").strip().lower() - if needle: - every = [i for i in every if _matches(i, needle)] - rows = sorted((i for i in every if (status is None or i.get("status") == status) - and (action is None or i.get("action") == action)), key=_order) - start = _cursor_offset(cursor) - page = rows[start:start + limit] + start = _cursor_offset(cursor) // limit * limit + result = _select(st, PAGE_SQL, lambda key: (key(KEY_ITEMS), needle, needle, needle, needle, needle, + status, status, action, action, next_tier, start, limit, limit, + next_tier, next_tier, next_tier, limit)) + doc = result.get("meta") or {} + start, total = int(result["offset"]), int(result["total"]) ready = doc.get("state") == "ready" - return {"items": [_public(i) for i in page], - "cursor": _cursor(start + limit) if start + limit < len(rows) else None, + return {"items": result.get("items") or [], "total": total, "offset": start, "limit": limit, + "cursor": _cursor(start + limit) if start + limit < total else None, + "previousCursor": _cursor(max(0, start - limit)) if start else None, "ready": ready, "message": None if ready else (doc.get("message") or NOT_READY), "notices": list(doc.get("notices") or ()), "refreshedAt": doc.get("refreshedAt"), "lastError": doc.get("last_error"), "statuses": list(STATUSES), "actions": list(ACTIONS)} -def item_detail(tenant_key, item_id, *, st=None): - """One SKU with its fee history, age bands, removal and stranded units, events and stored history.""" +def item_detail(tenant_key, item_id, *, offset=0, limit=50, st=None): + """Read one SKU and a page of its fees, events, removal and stranded history.""" st = _rt(tenant_key, st) - item = (_doc(st, KEY_ITEMS).get("items") or {}).get(item_id) + offset, limit = max(0, int(offset)), _page_size(limit) + base = "WITH item AS (SELECT value #> %s::text[] AS v FROM {tbl} WHERE key=%s) " + item = _select(st, base + "SELECT v - 'history' - 'fee_history' - 'fees' FROM item", + lambda key: (["items", item_id], key(KEY_ITEMS))) if not item: raise KeyError(item_id) - past = history(tenant_key, item.get("key"), st=st) - return {**_public(item), "events": list(item.get("history") or ()), - "removal_history": past["removals"], "stranded_history": past["stranded"]} + totals, children = {}, {} + for field, public in (("fee_history", "fee_history"), ("history", "events")): + result = _select(st, base + "SELECT jsonb_build_object('total', " + "jsonb_array_length(COALESCE(v->%s, '[]'::jsonb)), 'rows', " + "(SELECT jsonb_agg(value ORDER BY n) FROM (SELECT value, n FROM jsonb_array_elements(" + "COALESCE(v->%s, '[]'::jsonb)) WITH ORDINALITY a(value, n) ORDER BY n DESC " + "LIMIT %s OFFSET %s) p)) FROM item", + lambda key: (["items", item_id], key(KEY_ITEMS), field, field, limit, offset)) + totals[public] = int(result.get("total") or 0) + children[public] = result.get("rows") or [] + for kind, public, stamp in (("removals", "removal_history", "snapshot"), + ("stranded", "stranded_history", "date")): + sql = """WITH points AS MATERIALIZED ( + SELECT e.key AS day, e.value AS point FROM {tbl} kv, + LATERAL jsonb_each(COALESCE(kv.value #> %s::text[], '{{}}'::jsonb)) e + WHERE kv.key LIKE %s + ), page AS (SELECT day, point FROM points ORDER BY day DESC LIMIT %s OFFSET %s) + SELECT jsonb_build_object('total', (SELECT count(*) FROM points), + 'rows', (SELECT jsonb_agg(point || jsonb_build_object(%s::text, day) ORDER BY day) FROM page))""" + # The fixed prefix contains underscores, so escape LIKE metacharacters before the suffix wildcard. + key_prefix = _sql_scope(st)[2](HISTORY_PREFIX) + pattern = key_prefix.replace("\\", "\\\\").replace("_", "\\_").replace("%", "\\%") + "%" + result = _select(st, sql, lambda key: ([kind, item.get("key")], pattern, limit, offset, stamp)) + totals[public] = int(result.get("total") or 0) + children[public] = result.get("rows") or [] + total = max(totals.values()) + start = min(offset // limit * limit, max(0, (total - 1) // limit * limit)) + if start != offset: + return item_detail(tenant_key, item_id, offset=start, limit=limit, st=st) + return {**item, **children, "detailOffset": offset, "detailLimit": limit, + "detailTotal": total, "detailTotals": totals} def _uname(user): @@ -923,11 +1056,7 @@ def update_item(tenant_key, item_id, *, owner=None, status=None, due=None, actio name = _uname(user) or "unknown" out = {} - def _apply(doc): - doc = doc if isinstance(doc, dict) else {} - item = (doc.get("items") or {}).get(item_id) - if not item: - return doc + def _apply(item): at = _stamp(_now()) event = {"at": at, "by": name} if owner is not None: @@ -944,13 +1073,42 @@ def update_item(tenant_key, item_id, *, owner=None, status=None, due=None, actio item["updatedAt"], item["updatedBy"] = at, name item.setdefault("history", []).append(event) out["item"] = _public(item) - return doc - st.update(KEY_ITEMS, _apply) + return item + _update_one(st, item_id, _apply) if "item" not in out: raise KeyError(item_id) return out["item"] +def _update_one(st, item_id, apply): + """Lock the document but transfer and patch only the selected item's scalar fields.""" + pg, tenant, key = _sql_scope(st) + name, path = key(KEY_ITEMS), ["items", item_id] + pg.check_write(tenant, "update aged stock item") + try: + with pg._pool().connection() as con, con.transaction(): + con.execute("SELECT set_config('lock_timeout', %s, true)", (f"{int(pg._lock_timeout_s() * 1000)}ms",)) + row = con.execute(pg._q("SELECT (value #> %s::text[]) - 'history' - 'fee_history' - 'fees', " + "octet_length(value::text) FROM {tbl} WHERE key=%s FOR UPDATE", + "store_kv", tenant), (path, name)).fetchone() + item = row[0] if row else None + if not item: + raise KeyError(item_id) + item = json.loads(item) if isinstance(item, str) else item + changed = apply(item) + events = json.dumps(changed.pop("history", []), ensure_ascii=False, allow_nan=False) + payload = json.dumps(changed, ensure_ascii=False, allow_nan=False) + saved = con.execute(pg._q("UPDATE {tbl} SET value=jsonb_set(value, %s::text[], " + "(value #> %s::text[]) || %s::jsonb || jsonb_build_object('history', " + "COALESCE(value #> %s::text[], '[]'::jsonb) || %s::jsonb)), " + "rev=rev+1, updated_at=now() WHERE key=%s RETURNING octet_length(value::text)", + "store_kv", tenant), (path, path, payload, path + ["history"], events, name)).fetchone() + if saved[0] > pg.DOCUMENT_CEILING_BYTES and saved[0] > row[1]: + raise pg.StoreDocumentTooLarge(name, saved[0]) + finally: + pg._bump_write_generation(name, tenant) + + def set_status(tenant_key, user, item_id, status, *, note=None, st=None): """C4: move one item's status with an optional note.""" return update_item(tenant_key, item_id, status=status, note=note, user=user, st=st) @@ -1067,7 +1225,7 @@ def validate(tenant_key, *, start=None, end=None, st=None, **period): def app_keys(st): """Every store document this App owns.""" months = _doc(st, KEY_HISTORY).get("months") or () - return [KEY_SETTINGS, KEY_ITEMS, KEY_HISTORY, *(_month_key(m) for m in months)] + return [KEY_SETTINGS, KEY_ITEMS, KEY_HISTORY, KEY_REFRESH, *(_month_key(m) for m in months)] def remove_app_data(tenant_key, *, st=None): diff --git a/platform/modules/amazon_operator.py b/platform/modules/amazon_operator.py index e99cb28407fda6b2a0f85fdc8e39bf3a07997237..e81029571090210cb674bd961160804d83255176 100644 --- a/platform/modules/amazon_operator.py +++ b/platform/modules/amazon_operator.py @@ -817,21 +817,147 @@ def proposals(tenant_key, *, st=None): "action": p.get("action"), "reason": p.get("reason")} for p in rows] -def today(tenant_key, *, st=None, now=None): +def today(tenant_key, *, st=None, now=None, cursor=None, app_cursor=None, held_cursor=None, limit=50, q="", refresh=True): """The latest built day as the App shows it: every proposal, held notes, settings and the App inbox.""" st = _rt(tenant_key, st) - _safe_pull(tenant_key, st=st, now=now) - rec = day_record(tenant_key, st=st) or {} + if refresh: + _safe_pull(tenant_key, st=st, now=now) + return screen_today(tenant_key, st=st, now=now, cursor=cursor, app_cursor=app_cursor, + held_cursor=held_cursor, limit=limit, q=q) + + +def _screen_backend(st, tenant_key): + from core import store, store_pg + resolve = getattr(st, "_document_store", None) + backend = resolve() if callable(resolve) else st + if backend is store and store.backend() == "pg": + backend = store._d() + if not isinstance(backend, store_pg.PgStore): + return None + if backend.tenant_slug != str(tenant_key).strip().lower(): + raise PermissionError("The Amazon operator store belongs to another tenant.") + return backend + + +@contextlib.contextmanager +def _screen_store(tenant_key, st): + """Expose authored JSON to SQL without reading complete production documents into Python.""" + backend = _screen_backend(st, tenant_key) + key = getattr(st, "store_key", lambda name: name) + names = [key(KEY_DAYS), key(KEY_INBOX)] + if backend is not None: + from core import store_pg + with store_pg._pool().connection() as con: + def execute(sql, args=()): + return con.execute(store_pg._q(sql.replace("'{}'", "'{{}}'"), "store_kv", backend.tenant_slug), [*names, *args]) + yield execute, True, names + else: + import duckdb + con = duckdb.connect() + try: + con.execute("CREATE TABLE docs (name VARCHAR, value JSON)") + for name, document in zip(names, (KEY_DAYS, KEY_INBOX)): + con.execute("INSERT INTO docs VALUES (?, ?)", [name, json.dumps(_doc(st, document))]) + yield lambda sql, args=(): con.execute(sql, [*names, *args]), False, names + finally: + con.close() + + +def _stored_sql(pg): + each = "jsonb_each" if pg else "json_each" + array = "jsonb_array_elements" if pg else "json_each" + placeholder = "%s" if pg else "?" + docs = f"SELECT key AS name, value FROM {{tbl}} WHERE key IN ({placeholder}, {placeholder})" if pg else ( + f"SELECT * FROM docs WHERE name IN ({placeholder}, {placeholder})") + return f"""WITH documents AS ({docs}), + day_doc AS (SELECT value FROM documents ORDER BY (value ->> 'latest') IS NULL LIMIT 1), + ad AS (SELECT p.value AS r, d.key AS day FROM day_doc, + LATERAL {each}(COALESCE(value -> 'days', '{{}}')) d, + LATERAL {each}(COALESCE(d.value -> 'items', '{{}}')) p), + apps AS (SELECT p.value AS r, left(COALESCE(p.value ->> 'decided_at', ''), 10) AS day FROM documents, + LATERAL {each}(COALESCE(value -> 'items', '{{}}')) p), + held AS (SELECT h.value AS r, d.key AS day FROM day_doc, + LATERAL {each}(COALESCE(value -> 'days', '{{}}')) d, + LATERAL {array}(COALESCE(d.value -> 'held', '[]')) h) + """ + + +_SCREEN_FIELDS = ("id", "status", "owner", "due", "reason", "sku", "asin", "kind", "action", "change", "gate", + "ad_type", "campaign", "campaign_id", "day", "data_date", "spend", "decision_id", "decided_by", + "decided_at", "note", "cause", "outcome_at", "app", "source_kind", "source_module", "item_id", + "first_seen", "last_seen") + + +def _stored_page(execute, pg, sql, section, *, q="", cursor=None, limit=50, status=None, now=None): + try: + start, size = int(cursor or 0), int(limit) + except (TypeError, ValueError): + raise ValueError("The page cursor and size must be whole numbers.") from None + if start < 0 or size < 1: + raise ValueError("The page cursor must be nonnegative and the size at least 1.") + p = "%s" if pg else "?" + today_day = (now or _now()).date().isoformat() + args = [] + if section == "history": + selected = "SELECT * FROM ad UNION ALL SELECT * FROM apps WHERE (r ->> 'status') IN ('approved', 'dismissed')" + order = "day DESC, (r ->> 'id')" + elif section == "apps": + selected = f"SELECT * FROM apps WHERE (r ->> 'status') IN ('proposed', 'stale') OR day = {p}" + args.append(today_day) + order = ("CASE (r ->> 'status') WHEN 'proposed' THEN 0 WHEN 'stale' THEN 1 ELSE 2 END, " + "(r ->> 'app'), (r ->> 'sku'), (r ->> 'id')") + else: + selected = f"SELECT * FROM {section} WHERE day = (SELECT value ->> 'latest' FROM day_doc)" + order = ("CASE WHEN (r ->> 'status') = 'proposed' THEN 0 ELSE 1 END, " + "(r ->> 'ad_type'), (r ->> 'campaign'), (r ->> 'id')") if section == "ad" else "CAST(r AS TEXT)" + search_fields = ("id", "campaign", "campaign_id", "sku", "asin", "reason", "change", "gate", "app", "action", "status") + text = "concat_ws(' ', " + ", ".join(f"(r ->> '{f}')" for f in search_fields) + ")" + where = f"strpos(lower({text}), {p}) > 0" + args.append(str(q or "").strip().lower()) + if status: + where += f" AND (r ->> 'status') = {p}" + args.append(status) + sql += f", selected AS ({selected}), matched AS (SELECT * FROM selected WHERE {where}) " + total, opened = execute(sql + "SELECT count(*), count(*) FILTER (WHERE (r ->> 'status') = 'proposed') FROM matched", args).fetchone() + if start >= total: + start = max(0, (total - 1) // size * size) + builder = "jsonb_build_object" if pg else "json_object" + fields = _SCREEN_FIELDS if section != "held" else ("ad_type", "campaign", "campaign_id", "sku", "asin", "reason") + projection = builder + "(" + ", ".join(f"'{f}', r -> '{f}'" for f in fields) + ")" + rows = execute(sql + f"SELECT {projection} FROM matched ORDER BY {order} LIMIT {p} OFFSET {p}", [*args, size, start]).fetchall() + values = [r[0] if isinstance(r[0], dict) else json.loads(r[0]) for r in rows] + return values, {"total": total, "open": opened, "offset": start, "limit": size, + "cursor": str(start + size) if start + size < total else None} + + +def screen_today(tenant_key, *, st=None, now=None, cursor=None, app_cursor=None, held_cursor=None, limit=50, q=""): + """Read bounded Ads, source App and held pages without generating proposals or writing documents.""" + st = _rt(tenant_key, st) conf = settings(tenant_key, st=st) - rows = sorted(((rec.get("items") or {}).values()), - key=lambda p: (p.get("status") not in OPEN, p.get("ad_type") or "", p.get("campaign") or "", - p["id"])) - return {"day": rec.get("day") or _doc(st, KEY_DAYS).get("latest"), "dataDate": rec.get("data_date"), - "builtAt": rec.get("built_at"), "items": [_item(p) for p in rows], "held": list(rec.get("held") or []), - "spend": dict(rec.get("spend") or {}), "appItems": app_items_today(st, now=now), + with _screen_store(tenant_key, st) as (execute, pg, _names): + sql = _stored_sql(pg) + latest = "(SELECT value ->> 'latest' FROM day_doc)" + meta = execute(sql + f"SELECT day, r ->> 'data_date', r ->> 'built_at' FROM " + f"(SELECT d.key AS day, d.value AS r FROM day_doc, LATERAL " + f"{'jsonb_each' if pg else 'json_each'}(COALESCE(value -> 'days', '{{}}')) d) t " + f"WHERE day = {latest} LIMIT 1").fetchone() + ads, ad_page = _stored_page(execute, pg, sql, "ad", q=q, cursor=cursor, limit=limit, now=now) + apps, app_page = _stored_page(execute, pg, sql, "apps", q=q, cursor=app_cursor, limit=limit, now=now) + held, held_page = _stored_page(execute, pg, sql, "held", q=q, cursor=held_cursor, limit=limit, now=now) + return {"day": meta[0] if meta else None, "dataDate": meta[1] if meta else None, + "builtAt": meta[2] if meta else None, "items": [_item(p) for p in ads], "held": held, + "appItems": [_app_item(p) for p in apps], "pages": {"items": ad_page, "apps": app_page, "held": held_page}, "settings": {k: conf[k] for k in DEFAULTS} | {"defaultOwner": conf["defaultOwner"]}} +def screen_history(tenant_key, *, st=None, status=None, cursor=None, limit=50, q=""): + """Read and count one history page in the tenant store.""" + st = _rt(tenant_key, st) + with _screen_store(tenant_key, st) as (execute, pg, _names): + rows, page = _stored_page(execute, pg, _stored_sql(pg), "history", q=q, cursor=cursor, limit=limit, status=status) + return {**page, "items": [_app_item(p) if p.get("kind") == APP_KIND else _item(p) for p in rows]} + + # --------------------------------------------------------------------------- decisions and named writes def _ads_client(): @@ -962,6 +1088,7 @@ def set_status(tenant_key, user, item_id, status, *, note=None, st=None, now=Non if status == "dismissed" and not note: raise ValueError("Give a reason to dismiss.") at = _stamp(now or _now()) + _safe_pull(tenant_key, st=st, now=now) inbox = _inbox(st) app_ids = [pid for pid in ids if pid in inbox] app_rows, app_decision = (_decide_apps(st, tenant_key, app_ids, inbox, status, note, who, at) diff --git a/platform/modules/competitor_watch.py b/platform/modules/competitor_watch.py index 2eee5b70d5e05dd5777ca62fa76c73c54a36558b..eebcc3926119b7e95a6c51e09402e8ac7620b7ae 100644 --- a/platform/modules/competitor_watch.py +++ b/platform/modules/competitor_watch.py @@ -3,20 +3,23 @@ Store documents (tenant runtime store, App owned, never the mirror and never Amazon): competitor_watch_settings {undercut_pct, rank_surge_pct, default_owner} competitor_watch_terms {seeded: [...], added: [...], removed: [...], seeded_at} - competitor_watch_alerts {seq, items: {id: alert}, checks: [check, ...]} + competitor_watch_alerts legacy source, retained for explicit resumable migration + competitor_watch_alert/ one alert per store row + competitor_watch_check/ one check per store row + competitor_watch_alerts_index migration checkpoint and serialized alert sequence competitor_watch_weeks {last_compared, comparison: {...}, metrics: {period_start: {asin: metric}}} + competitor_watch_terms_index {version: 1}, prepared by sync + competitor_watch_term/ {term, active}, one indexed store key per term Top 3 history is read from the connector table `amzn_brand_analytics`, which the store's weekly -retention keeps for watched terms. Deleting this module and its four documents leaves the -connector intact (rule 13). +retention keeps for watched terms. App documents never change the connector (rule 13). -Limit (rule 1): each document is one JSON value, and the Postgres store refuses a document over -`core.store_pg.DOCUMENT_CEILING_BYTES` (128 MB). At about 600 bytes an alert, the alerts document -reaches it near 200,000 alerts. Nothing here caps terms or alerts. The fix when it nears: split -`competitor_watch_alerts` by year. +Alerts and checks page in the tenant's existing Postgres store. No collection document or row cap +limits their growth. GET never prepares rows or reads the legacy collection. """ import contextlib import datetime as dt +import json from pathlib import Path TRIGGERS = { @@ -32,15 +35,23 @@ DEFAULTS = {"undercut_pct": 5.0, "rank_surge_pct": 50.0} DUE_DAYS = 7 NOT_READY = "Comparison starts after the next weekly pull." #: The mirror holds the tables but this read failed or was busy. `{cause}` is the error class. -READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so no search term was compared. " - "The next Amazon sync reads it again.") -LIMIT = ("Each competitor watch document is limited to 128 MB by the store. About 200,000 alerts " - "fit in one. Split the alerts document by year when it nears that size.") +READ_FAILED = ("The Amazon mirror could not be read ({cause}), so no search term was compared " + "and the next Amazon sync will retry.") +BA_DATES_MISSING = "Brand Analytics report dates are missing" +BA_WEEKLY_MISSING = "Brand Analytics has no dated weekly reports" +LIMIT = "Alerts and checks use separate store rows and server pages, with no row cap." KEY_SETTINGS = "competitor_watch_settings" KEY_TERMS = "competitor_watch_terms" KEY_ALERTS = "competitor_watch_alerts" KEY_WEEKS = "competitor_watch_weeks" +KEY_TERM_INDEX = "competitor_watch_terms_index" +TERM_PREFIX = "competitor_watch_term/" +TERMS_NOT_READY = "Search terms are not ready. The next Amazon sync prepares term paging." +KEY_ALERT_INDEX = "competitor_watch_alerts_index" +ALERT_PREFIX = "competitor_watch_alert/" +CHECK_PREFIX = "competitor_watch_check/" +ALERTS_NOT_READY = "Alert paging is not ready. The next Amazon sync or an administrator must finish alert migration." BA_TABLE = "amzn_brand_analytics" SQP_TABLE = "amzn_search_query_performance" @@ -75,6 +86,118 @@ def _doc(st, key): return value if isinstance(value, dict) else {} +class TermsNotReady(RuntimeError): + """Term paging has not been prepared by a completed sync.""" + + def __init__(self, message, *, problems=()): + super().__init__(message) + self.problems = list(problems) + + +class AlertsNotReady(RuntimeError): + """Alert rows are not ready for reads or edits.""" + + +def _bound_store(tenant_key, st): + backend = st._document_store() if hasattr(st, "_document_store") else st + slug = getattr(backend, "tenant_slug", None) + if slug is not None and slug != tenant_key: + raise RuntimeError("The store handle belongs to another tenant.") + return backend + + +def _paths(tenant_key, st, name, paths): + backend = _bound_store(tenant_key, st) + reader = getattr(backend, "get_json_paths", None) + if callable(reader): + key = st.store_key(name) if hasattr(st, "store_key") else name + return reader(key, paths) + doc = _doc(st, name) + out = [] + for path in paths: + value = doc + for part in path: + value = value.get(part) if isinstance(value, dict) else None + out.append(value) + return out + + +class _TermIndex: + """Tenant-bound term rows use the existing store primary key, with no new schema.""" + + def __init__(self, tenant_key, st): + from core import store_pg + self.pg, self.slug, self.st = store_pg, tenant_key, st + self.key = st.store_key if hasattr(st, "store_key") else str + self.prefix = self.key(TERM_PREFIX) + + def connection(self): + return self.pg._pool().connection() + + def sql(self, statement): + return self.pg._q(statement, "store_kv", self.slug) + + def ready(self): + with self.connection() as con: + row = con.execute(self.sql("SELECT value ->> 'version' FROM {tbl} WHERE key = %s"), + [self.key(KEY_TERM_INDEX)]).fetchone() + return bool(row and row[0] == "1") + + def seed(self, found, legacy): + self.pg.check_write(self.slug, "prepare competitor watch term paging") + migrate = not self.ready() + removed = {_norm(t) for t in legacy.get("removed") or ()} if migrate else set() + terms = set(found) + if migrate: + terms |= {_norm(t) for t in [*(legacy.get("seeded") or ()), + *(legacy.get("added") or ()), *removed]} + statement = self.sql("INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) " + "ON CONFLICT (key) DO NOTHING") + rows = [(self.prefix + t, json.dumps({"term": t, "active": t not in removed})) + for t in sorted(terms) if t] + with self.connection() as con: + if rows: + with con.cursor() as cur: + cur.executemany(statement, rows) + con.execute(statement, [self.key(KEY_TERM_INDEX), json.dumps({"version": 1})]) + + def page(self, search, offset, limit, selected): + if not self.ready(): + raise TermsNotReady(TERMS_NOT_READY) + # Count, selection and page share one snapshot, including an empty final page. + statement = self.sql("WITH matches AS (SELECT key, value ->> 'term' AS term FROM {tbl} " + "WHERE starts_with(key, %s) AND (value ->> 'active') = 'true' " + "AND strpos(value ->> 'term', %s) > 0), " + "totals AS (SELECT count(*) AS total FROM matches), " + "bounds AS (SELECT total, CASE WHEN %s >= total THEN " + "greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) " + "SELECT total, skip, EXISTS(SELECT 1 FROM {tbl} WHERE key = %s " + "AND (value ->> 'active') = 'true'), " + "ARRAY(SELECT term FROM matches ORDER BY key COLLATE \"C\" LIMIT %s OFFSET (SELECT skip FROM bounds)) " + "FROM bounds") + with self.connection() as con: + row = con.execute(statement, [self.prefix, search, offset, limit, limit, offset, + self.prefix + selected, limit]).fetchone() + total, start, held, terms = row + return {"terms": list(terms), "term": selected if held else next(iter(terms), None), + "termTotal": total, "termOffset": start, "termLimit": limit} + + def all(self): + with self.connection() as con: + return [r[0] for r in con.execute(self.sql("SELECT value ->> 'term' FROM {tbl} " + "WHERE starts_with(key, %s) AND (value ->> 'active') = 'true' ORDER BY key COLLATE \"C\""), + [self.prefix]).fetchall()] + + +def _term_index(tenant_key, st): + from core.store_pg import PgStore + return _TermIndex(tenant_key, st) if isinstance(_bound_store(tenant_key, st), PgStore) else None + + +def _term_sql(column): + return f"lower(trim(regexp_replace(coalesce({column}, ''), '\\s+', ' ', 'g')))" + + # --------------------------------------------------------------------------- mirror reads def _mirror_path(tenant_key, st): @@ -87,6 +210,14 @@ class _ReadFailed(Exception): """A mirror read failed, was locked or busy. Never read as an absent table.""" +class SourceNotReady(_ReadFailed): + """A populated source lacks known weekly report dates required for comparison.""" + + def __init__(self, message, *, page=None): + super().__init__(message) + self.page = page + + @contextlib.contextmanager def _reader(tenant_key, st): """A read-only cursor on the tenant mirror, None when the mirror file does not exist. @@ -125,7 +256,37 @@ def _columns(cur, table): raise _ReadFailed(type(exc).__name__) from exc -def _sqp(cur): +def _weekly_period_sql(): + """Only finite dates or timestamps with an inclusive seven-day span define a report week.""" + start = 'TRY_CAST("_period_start" AS TIMESTAMP)' + end = 'TRY_CAST("_period_end" AS TIMESTAMP)' + return (f"isfinite({start}) AND isfinite({end}) AND " + f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) = 6") + + +def _ba_columns(cur): + """Check weekly readiness with metadata and scalar existence checks, without loading source rows.""" + cols = _columns(cur, BA_TABLE) if cur is not None else set() + if cols and not {"_period_start", "_period_end"} <= cols: + try: + populated = cur.execute(f"SELECT EXISTS(SELECT 1 FROM {BA_TABLE})").fetchone()[0] + except Exception as exc: # noqa: BLE001 + raise _ReadFailed(type(exc).__name__) from exc + if populated: + raise SourceNotReady(BA_DATES_MISSING) + elif {"_period_start", "_period_end"} <= cols: + try: + populated, weekly = cur.execute( + f"SELECT EXISTS(SELECT 1 FROM {BA_TABLE}), " + f"EXISTS(SELECT 1 FROM {BA_TABLE} WHERE {_weekly_period_sql()})").fetchone() + except Exception as exc: # noqa: BLE001 + raise _ReadFailed(type(exc).__name__) from exc + if populated and not weekly: + raise SourceNotReady(BA_WEEKLY_MISSING) + return cols + + +def _sqp(cur, term=None): """SQP rows as (term, asin, impressions, end_date). SQP reports our ASINs only.""" need = {"search_query_data_search_query", "asin"} cols = _columns(cur, SQP_TABLE) if cur is not None else set() @@ -134,16 +295,26 @@ def _sqp(cur): imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)' if "impression_data_asin_impression_count" in cols else "0") end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''" + where = f' WHERE {_term_sql("search_query_data_search_query")} = ?' if term is not None else "" rows = cur.execute( - f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}').fetchall() + f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}', + [term] if term is not None else []).fetchall() return [(_norm(t), str(a or "").strip(), float(i or 0), str(e or "")) for t, a, i, e in rows if _norm(t)] -def _our_asins(cur, sqp_rows): +def _our_asins(cur, sqp_rows, candidates=None): ours = {a for _t, a, _i, _e in sqp_rows if a} - if cur is not None and "asin1" in _columns(cur, LISTINGS_TABLE): + if candidates and cur is not None and "asin" in _columns(cur, SQP_TABLE): + wanted = sorted(candidates) ours |= {str(r[0]).strip() for r in cur.execute( - f'SELECT DISTINCT "asin1" FROM {LISTINGS_TABLE} WHERE "asin1" IS NOT NULL').fetchall()} + f'SELECT DISTINCT "asin" FROM {SQP_TABLE} WHERE trim("asin") IN (' + + ','.join('?' for _ in wanted) + ')', wanted).fetchall()} + if cur is not None and candidates != set() and "asin1" in _columns(cur, LISTINGS_TABLE): + wanted = sorted(candidates) if candidates is not None else None + where = ' AND trim("asin1") IN (' + ','.join('?' for _ in wanted) + ')' if wanted is not None else '' + ours |= {str(r[0]).strip() for r in cur.execute( + f'SELECT DISTINCT "asin1" FROM {LISTINGS_TABLE} WHERE "asin1" IS NOT NULL{where}', + wanted or []).fetchall()} return {a for a in ours if a} @@ -162,17 +333,22 @@ def _our_asin_for(term, sqp_rows): def _top3(cur, terms=None): """{term: {period_start: {"end": str, "rows": {asin: (rank, click_share, conversion_share)}}}}.""" - cols = _columns(cur, BA_TABLE) if cur is not None else set() - if not {"search_term", "clicked_asin", "click_share_rank", "_period_start"} <= cols: + cols = _ba_columns(cur) + if not {"search_term", "clicked_asin", "click_share_rank", "_period_start", "_period_end"} <= cols: return {} cs = 'TRY_CAST("click_share" AS DOUBLE)' if "click_share" in cols else "NULL" cv = 'TRY_CAST("conversion_share" AS DOUBLE)' if "conversion_share" in cols else "NULL" pe = 'CAST("_period_end" AS VARCHAR)' if "_period_end" in cols else "''" + wanted = None if terms is None else sorted({_norm(t) for t in terms}) + if wanted == []: + return {} + where = (f' AND {_term_sql("search_term")} IN (' + ','.join('?' for _ in wanted) + ')' + if wanted is not None else '') rows = cur.execute( f'SELECT "search_term", CAST("_period_start" AS VARCHAR), {pe}, "clicked_asin", ' f'TRY_CAST("click_share_rank" AS INTEGER), {cs}, {cv} FROM {BA_TABLE} ' - f'WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND 3').fetchall() - wanted = None if terms is None else set(terms) + f'WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND 3 ' + f'AND ({_weekly_period_sql()}){where}', wanted or []).fetchall() out = {} for term, start, end, asin, rank, share, conv in rows: key = _norm(term) @@ -184,9 +360,10 @@ def _top3(cur, terms=None): def _periods(cur): - if cur is None or "_period_start" not in _columns(cur, BA_TABLE): + if not {"_period_start", "_period_end"} <= _ba_columns(cur): return [] - rows = cur.execute(f'SELECT DISTINCT CAST("_period_start" AS VARCHAR) FROM {BA_TABLE}').fetchall() + rows = cur.execute(f'SELECT DISTINCT CAST("_period_start" AS VARCHAR) FROM {BA_TABLE} ' + f'WHERE {_weekly_period_sql()}').fetchall() return sorted((str(r[0]) for r in rows if r[0] is not None), reverse=True) @@ -194,7 +371,11 @@ def _periods(cur): def watched_terms(tenant_key, *, st=None): """Seeded SQP terms plus terms a person added, minus terms a person removed (store only).""" - doc = _doc(_rt(tenant_key, st), KEY_TERMS) + st = _rt(tenant_key, st) + index = _term_index(tenant_key, st) + if index is not None and index.ready(): + return index.all() + doc = _doc(st, KEY_TERMS) removed = {_norm(t) for t in doc.get("removed") or ()} terms = {_norm(t) for t in [*(doc.get("seeded") or ()), *(doc.get("added") or ())]} return sorted(t for t in terms if t and t not in removed) @@ -203,11 +384,20 @@ def watched_terms(tenant_key, *, st=None): def seed_terms(tenant_key, *, st=None, cur=None): """Add every distinct SQP search query to the seeded set. A removed term stays removed.""" st = _rt(tenant_key, st) + def distinct(reader): + if reader is None or "search_query_data_search_query" not in _columns(reader, SQP_TABLE): + raise TermsNotReady(TERMS_NOT_READY) + return {r[0] for r in reader.execute(f'SELECT DISTINCT {_term_sql("search_query_data_search_query")} ' + f'AS term FROM {SQP_TABLE} WHERE {_term_sql("search_query_data_search_query")} <> \'\'').fetchall()} + if cur is None: with _reader(tenant_key, st) as reader: - found = {t for t, _a, _i, _e in _sqp(reader)} if reader is not None else set() + found = distinct(reader) else: - found = {t for t, _a, _i, _e in _sqp(cur)} + found = distinct(cur) + index = _term_index(tenant_key, st) + if index is not None: + index.seed(found, _doc(st, KEY_TERMS)) def _seed(doc): doc = doc if isinstance(doc, dict) else {} @@ -221,11 +411,20 @@ def seed_terms(tenant_key, *, st=None, cur=None): return watched_terms(tenant_key, st=st) -def _edit_terms(tenant_key, terms, add, st): +def _edit_terms(tenant_key, terms, add, st, page_only=False): clean = sorted({_norm(t) for t in terms or () if _norm(t)}) if not clean: raise ValueError("Name at least one search term.") st = _rt(tenant_key, st) + index = _term_index(tenant_key, st) + if index is not None: + if not index.ready(): + raise TermsNotReady(TERMS_NOT_READY) + for term in clean: + st.update(TERM_PREFIX + term, lambda _doc, t=term: {"term": t, "active": add}) + return clean if page_only else index.all() + if page_only: + raise TermsNotReady(TERMS_NOT_READY) def _apply(doc): doc = doc if isinstance(doc, dict) else {} @@ -243,14 +442,14 @@ def _edit_terms(tenant_key, terms, add, st): return watched_terms(tenant_key, st=st) -def add_terms(tenant_key, terms, *, st=None): +def add_terms(tenant_key, terms, *, st=None, page_only=False): """Watch these terms. A term a person removed before comes back.""" - return _edit_terms(tenant_key, terms, True, st) + return _edit_terms(tenant_key, terms, True, st, page_only) -def remove_terms(tenant_key, terms, *, st=None): +def remove_terms(tenant_key, terms, *, st=None, page_only=False): """Stop watching these terms. A later SQP seed does not bring them back.""" - return _edit_terms(tenant_key, terms, False, st) + return _edit_terms(tenant_key, terms, False, st, page_only) def terms_summary(tenant_key, *, st=None): @@ -265,10 +464,7 @@ def terms_summary(tenant_key, *, st=None): terms = watched_terms(tenant_key, st=st) with _reader(tenant_key, st) as cur: held = _top3(cur, terms) - open_counts = {} - for alert in (_doc(st, KEY_ALERTS).get("items") or {}).values(): - if alert.get("status") == "Open": - open_counts[alert.get("term")] = open_counts.get(alert.get("term"), 0) + 1 + open_counts = _alert_index(tenant_key, st).open_counts() return {"terms": [{"term": t, "source": "seeded" if t in seeded else "added", "newestPeriod": max(held.get(t, {}), default=None), "openAlerts": open_counts.get(t, 0)} for t in terms]} @@ -277,22 +473,37 @@ def terms_summary(tenant_key, *, st=None): def term_page(tenant_key, *, st=None, search="", offset=0, limit=50, selected=None): """Page the complete watched vocabulary without comparing every term on a screen read.""" st = _rt(tenant_key, st) - if not _doc(st, KEY_TERMS).get("seeded_at"): - seed_terms(tenant_key, st=st) - terms = watched_terms(tenant_key, st=st) - picked = _norm(selected or "") - picked = picked if picked in terms else None - needle = _norm(search or "") - matches = [term for term in terms if needle in term] if needle else terms + error, = _paths(tenant_key, st, KEY_WEEKS, [("last_error",)]) + problems = [str(error)] if error else [] size = max(1, min(int(limit), 100)) start = max(0, int(offset)) - if start >= len(matches): - start = max(0, (len(matches) - 1) // size * size) - page = matches[start:start + size] - error = _doc(st, KEY_WEEKS).get("last_error") - return {"terms": page, "term": picked or next(iter(page), None), - "termTotal": len(matches), "termOffset": start, "termLimit": size, - "problems": [str(error)] if error else []} + page_error = None + try: + index = _term_index(tenant_key, st) + if index is None: + raise TermsNotReady("Search term paging needs the tenant store migration. Ask an administrator to finish it.") + page = index.page(_norm(search), start, size, _norm(selected)) + except TermsNotReady as exc: + page_error = exc + source_error = None + try: + with _reader(tenant_key, st) as cur: + _ba_columns(cur) + except SourceNotReady as exc: + source_error = exc + cause = READ_FAILED.format(cause=exc) + if cause not in problems: + problems.append(cause) + except _ReadFailed as exc: + cause = READ_FAILED.format(cause=exc) + if cause not in problems: + problems.append(cause) + if page_error is not None: + raise TermsNotReady(str(page_error), problems=problems) from page_error + page = {**page, "problems": problems} + if source_error is not None: + raise SourceNotReady(str(source_error), page=page) from source_error + return page # --------------------------------------------------------------------------- settings and owner @@ -457,10 +668,14 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None """Compare the newest held week with the week before and raise owned alerts. Returns a summary.""" st = _rt(tenant_key, st) now = now or _now() + while not migrate_alerts(tenant_key, st=st): + pass synced = set(synced_tables or ()) with _reader(tenant_key, st) as cur: if cur is None: return {"state": "no_mirror"} + if "brand_analytics" in synced: + _ba_columns(cur) sqp = _sqp(cur) seed_terms(tenant_key, st=st, cur=cur) if "brand_analytics" not in synced: @@ -551,36 +766,7 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None found(asin, "rank_surge", {"rank": m["rank"], "previousRank": old["rank"]}, asin_cause(asin)) - owner = conf["defaultOwner"] - created = [] - - def _alerts(doc): - doc = doc if isinstance(doc, dict) else {} - doc.setdefault("seq", 0) - items = doc.setdefault("items", {}) - latest = {} - for alert in sorted(items.values(), key=lambda a: int(str(a.get("id", "a0"))[1:] or 0)): - latest[(alert.get("term"), alert.get("asin"), alert.get("triggerKey"))] = alert - for hit in raised: - key = (hit["term"], hit["asin"], hit["trigger"]) - if key in latest and latest[key].get("status") in HOLDING: - continue - doc["seq"] += 1 - aid = f"a{doc['seq']}" - alert = {"id": aid, "term": hit["term"], "asin": hit["asin"], "trigger": TRIGGERS[hit["trigger"]], - "triggerKey": hit["trigger"], "periodStart": hit["periodStart"], - "periodEnd": hit["periodEnd"], "previousPeriodStart": hit["previousPeriodStart"], - "detail": hit["detail"], "cause": hit["cause"], "owner": owner, "status": "Open", - "due": (now.date() + dt.timedelta(days=DUE_DAYS)).isoformat(), - "createdAt": _stamp(now), "updatedAt": _stamp(now), "updatedBy": "system", - "history": [{"at": _stamp(now), "by": "system", "event": "created"}]} - items[aid] = alert - latest[key] = alert - created.append(aid) - doc["checks"] = [c for c in doc.get("checks") or () if c.get("periodStart") != newest] + checks - return doc - if raised or checks: - st.update(KEY_ALERTS, _alerts) + created = _alert_index(tenant_key, st).record(raised, checks, newest, conf["defaultOwner"], now) ready = previous is not None @@ -601,13 +787,168 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None # --------------------------------------------------------------------------- alerts -def alerts(tenant_key, *, status=None, st=None): - """Every alert, newest first, plus the newest week's check rows. No cap.""" - st = _rt(tenant_key, st) - doc = _doc(st, KEY_ALERTS) - rows = [a for a in (doc.get("items") or {}).values() if status is None or a.get("status") == status] - rows.sort(key=lambda a: int(str(a.get("id", "a0"))[1:] or 0), reverse=True) - return {"alerts": rows, "checks": list(doc.get("checks") or ()), "statuses": list(STATUSES)} +class _AlertIndex(_TermIndex): + """Separate alert and check rows share the tenant store and one transactional sequence lock.""" + + @staticmethod + def decode(value): + return json.loads(value) if isinstance(value, str) else value + + def ready(self): + with self.connection() as con: + row = con.execute(self.sql("SELECT value ->> 'version' FROM {tbl} WHERE key = %s"), + [self.key(KEY_ALERT_INDEX)]).fetchone() + return bool(row and row[0] == "1") + + def open_counts(self): + if not self.ready(): + raise AlertsNotReady(ALERTS_NOT_READY) + with self.connection() as con: + return dict(con.execute(self.sql("SELECT value ->> 'term', count(*) FROM {tbl} " + "WHERE starts_with(key, %s) AND (value ->> 'status') = 'Open' GROUP BY value ->> 'term'"), + [self.key(ALERT_PREFIX)]).fetchall()) + + def _put(self, con, key, value, *, keep=False): + conflict = "DO NOTHING" if keep else "DO UPDATE SET value = excluded.value" + con.execute(self.sql("INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) " + "ON CONFLICT (key) " + conflict), [self.key(key), json.dumps(value)]) + + @contextlib.contextmanager + def _locked(self, *, prepare=False): + self.pg.check_write(self.slug, "write competitor watch alert rows") + with self.connection() as con: + con.execute("SET LOCAL lock_timeout = '10s'") + con.execute("SET LOCAL statement_timeout = '30s'") + if prepare: + self._put(con, KEY_ALERT_INDEX, {"seq": 0, "itemCursor": "", "checkOffset": 0}, keep=True) + row = con.execute(self.sql("SELECT value FROM {tbl} WHERE key = %s FOR UPDATE"), + [self.key(KEY_ALERT_INDEX)]).fetchone() + meta = self.decode(row[0]) if row else {} + if not prepare and meta.get("version") != 1: + raise AlertsNotReady(ALERTS_NOT_READY) + yield con, meta + self._put(con, KEY_ALERT_INDEX, meta) + + def _legacy_items(self, con, cursor, size): + return con.execute(self.sql("SELECT item.key, item.value FROM {tbl}, " + "LATERAL jsonb_each(value -> 'items') AS item WHERE {tbl}.key = %s " + "AND item.key COLLATE \"C\" > %s ORDER BY item.key COLLATE \"C\" LIMIT %s"), + [self.key(KEY_ALERTS), cursor, size]).fetchall() + + def _legacy_checks(self, con, offset, size): + return con.execute(self.sql("SELECT item.ordinality, item.value FROM {tbl}, " + "LATERAL jsonb_array_elements(value -> 'checks') WITH ORDINALITY AS item(value, ordinality) " + "WHERE key = %s AND item.ordinality > %s ORDER BY item.ordinality LIMIT %s"), + [self.key(KEY_ALERTS), offset, size]).fetchall() + + def migrate(self, size): + with self._locked(prepare=True) as (con, meta): + if meta.get("version") == 1: + return True + old_seq = con.execute(self.sql("SELECT value ->> 'seq' FROM {tbl} WHERE key = %s"), + [self.key(KEY_ALERTS)]).fetchone() + meta["seq"] = max(meta["seq"], int(old_seq[0] or 0) if old_seq else 0) + items = self._legacy_items(con, meta["itemCursor"], size) + for key, value in items: + alert = self.decode(value) + sequence = int(str(alert.get("id") or key)[1:]) + # Missing createdAt stays missing. Copy every other legacy field unchanged. + self._put(con, ALERT_PREFIX + key, {**alert, "sequence": sequence}, keep=True) + meta["seq"] = max(meta["seq"], sequence) + meta["itemCursor"] = key + checks = self._legacy_checks(con, meta["checkOffset"], size) + for ordinal, value in checks: + self._put(con, CHECK_PREFIX + f"legacy/{ordinal}", + {**self.decode(value), "sequence": ordinal}, keep=True) + meta["checkOffset"] = ordinal + if len(items) < size and len(checks) < size: + meta["version"] = 1 + return meta.get("version") == 1 + + def _page(self, con, prefix, search, status, offset, limit, *, checks=False): + order = "period DESC, sequence, key COLLATE \"C\"" if checks else "sequence DESC" + final_order = "page.period DESC, page.sequence, page.key COLLATE \"C\"" if checks else "page.sequence DESC" + # Pick page keys before projecting alert bodies and their histories. + statement = self.sql("WITH matches AS (SELECT key, value ->> 'periodStart' AS period, " + "coalesce(value ->> 'sequence', '0')::numeric AS sequence FROM {tbl} WHERE starts_with(key, %s) " + "AND (%s::text IS NULL OR (value ->> 'status') = %s) " + "AND strpos(lower(concat_ws(' ', value ->> 'term', value ->> 'asin', value ->> 'trigger', " + "value ->> 'owner', value ->> 'cause')), %s) > 0), " + "totals AS (SELECT count(*) AS total FROM matches), bounds AS (SELECT total, " + "CASE WHEN %s >= total THEN greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) " + "ELSE %s END AS skip FROM totals) SELECT total, skip, " + "ARRAY(SELECT stored.value::text FROM (SELECT key, sequence, period FROM matches ORDER BY " + order + + " LIMIT %s OFFSET (SELECT skip FROM bounds)) AS page JOIN {tbl} AS stored ON stored.key = page.key " + "ORDER BY " + final_order + ") FROM bounds") + row = con.execute(statement, [self.key(prefix), status, status, search, offset, limit, limit, offset, limit]).fetchone() + return {"total": row[0], "offset": row[1], "limit": limit, + "rows": [self.decode(value) for value in row[2]]} + + def page(self, search, status, offset, limit, check_offset, check_limit): + if not self.ready(): + raise AlertsNotReady(ALERTS_NOT_READY) + with self.connection() as con: + alerts = self._page(con, ALERT_PREFIX, search, status, offset, limit) + checks = self._page(con, CHECK_PREFIX, search, None, check_offset, check_limit, checks=True) + return {"alerts": alerts.pop("rows"), "checks": checks.pop("rows"), + "alertPage": alerts, "checkPage": checks, "statuses": list(STATUSES)} + + def record(self, raised, checks, newest, owner, now): + created = [] + with self._locked() as (con, meta): + for hit in raised: + latest = con.execute(self.sql("SELECT value ->> 'status' FROM {tbl} " + "WHERE starts_with(key, %s) AND (value ->> 'term') = %s AND (value ->> 'asin') = %s " + "AND (value ->> 'triggerKey') = %s ORDER BY (value ->> 'sequence')::numeric DESC LIMIT 1"), + [self.key(ALERT_PREFIX), hit["term"], hit["asin"], hit["trigger"]]).fetchone() + if latest and latest[0] in HOLDING: + continue + meta["seq"] += 1 + aid = f"a{meta['seq']}" + alert = {**hit, "id": aid, "sequence": meta["seq"], "triggerKey": hit["trigger"], + "trigger": TRIGGERS[hit["trigger"]], "owner": owner, "status": "Open", + "due": (now.date() + dt.timedelta(days=DUE_DAYS)).isoformat(), + "createdAt": _stamp(now), "updatedAt": _stamp(now), "updatedBy": "system", + "history": [{"at": _stamp(now), "by": "system", "event": "created"}]} + self._put(con, ALERT_PREFIX + aid, alert) + created.append(aid) + con.execute(self.sql("DELETE FROM {tbl} WHERE starts_with(key, %s) AND (value ->> 'periodStart') = %s"), + [self.key(CHECK_PREFIX), newest]) + for i, check in enumerate(checks): + self._put(con, CHECK_PREFIX + f"sync/{newest}/{i}", {**check, "sequence": i}) + return created + + def update(self, alert_id, change): + with self._locked() as (con, _meta): + row = con.execute(self.sql("SELECT value FROM {tbl} WHERE key = %s"), + [self.key(ALERT_PREFIX + alert_id)]).fetchone() + if row is None: + raise KeyError(alert_id) + alert = self.decode(row[0]) + change(alert) + self._put(con, ALERT_PREFIX + alert_id, alert) + return alert + + +def _alert_index(tenant_key, st): + from core.store_pg import PgStore + if not isinstance(_bound_store(tenant_key, st), PgStore): + raise AlertsNotReady("Alert paging needs the tenant Postgres store. Ask an administrator to finish the store migration.") + return _AlertIndex(tenant_key, st) + + +def migrate_alerts(tenant_key, *, st=None, batch_size=500): + """Copy one restartable legacy batch and return whether alert paging is ready, without schema changes.""" + return _alert_index(tenant_key, _rt(tenant_key, st)).migrate(max(1, int(batch_size))) + + +def alerts(tenant_key, *, status=None, st=None, search="", offset=0, limit=50, check_offset=0, check_limit=50): + """Read server-filtered pages of alerts and checks without touching the legacy collection.""" + if status is not None and status not in STATUSES: + raise ValueError("Unknown alert status.") + return _alert_index(tenant_key, _rt(tenant_key, st)).page( + _norm(search), status, max(0, int(offset)), max(1, min(100, int(limit))), + max(0, int(check_offset)), max(1, min(100, int(check_limit)))) def update_alert(tenant_key, alert_id, *, owner=None, status=None, due=None, user="system", st=None): @@ -622,13 +963,8 @@ def update_alert(tenant_key, alert_id, *, owner=None, status=None, due=None, use if owner is not None and not str(owner).strip(): raise ValueError("Owner cannot be empty.") st = _rt(tenant_key, st) - out = {} - def _apply(doc): - doc = doc if isinstance(doc, dict) else {} - alert = (doc.get("items") or {}).get(alert_id) - if not alert: - return doc + def _apply(alert): at = _stamp(_now()) event = {"at": at, "by": user} if owner is not None: @@ -639,12 +975,7 @@ def update_alert(tenant_key, alert_id, *, owner=None, status=None, due=None, use alert["due"] = event["due"] = due alert["updatedAt"], alert["updatedBy"] = at, user alert.setdefault("history", []).append(event) - out["alert"] = dict(alert) - return doc - st.update(KEY_ALERTS, _apply) - if "alert" not in out: - raise KeyError(alert_id) - return out["alert"] + return _alert_index(tenant_key, st).update(alert_id, _apply) # --------------------------------------------------------------------------- term detail @@ -654,16 +985,19 @@ def term_detail(tenant_key, term, *, st=None): st = _rt(tenant_key, st) key = _norm(term) with _reader(tenant_key, st) as cur: - sqp = _sqp(cur) if cur is not None else [] - ours = _our_asins(cur, sqp) if cur is not None else set() + sqp = _sqp(cur, key) if cur is not None else [] weeks_held = _top3(cur, [key]).get(key, {}) + candidates = {a for week in weeks_held.values() for a in week["rows"]} + ours = _our_asins(cur, sqp, candidates) if cur is not None else set() periods = sorted(weeks_held, reverse=True) - weeks = _doc(st, KEY_WEEKS) + metric_asins = sorted(weeks_held[periods[0]]["rows"]) if periods else [] + state, *values = _paths(tenant_key, st, KEY_WEEKS, [("comparison",)] + + [("metrics", periods[0], asin) for asin in metric_asins]) + metric = dict(zip(metric_asins, values)) top3 = [] if periods: newest = weeks_held[periods[0]]["rows"] before = weeks_held[periods[1]]["rows"] if len(periods) > 1 else None - metric = (weeks.get("metrics") or {}).get(periods[0]) or {} for asin, (rank, share, conv) in sorted(newest.items(), key=lambda kv: kv[1][0]): change = None if before is not None: @@ -685,7 +1019,7 @@ def term_detail(tenant_key, term, *, st=None): trend = [{"asin": a, "isOurs": a in ours, "clickShare": [(weeks_held[p]["rows"].get(a) or (None, None, None))[1] if a in weeks_held[p]["rows"] else None for p in periods]} for a in every] - state = weeks.get("comparison") or {} + state = state or {} ready = len(periods) > 1 and bool(state.get("ready", True)) return {"term": key, "periods": periods, "top3": top3, "trend": trend, "comparison": {"ready": ready, "message": None if ready else NOT_READY}} diff --git a/platform/modules/customer_data.py b/platform/modules/customer_data.py index fc58c5d7feb38fd263d5448a11cef0624c3c2f53..c208de92082fb1612bb237c84d633a9c0c124ee7 100644 --- a/platform/modules/customer_data.py +++ b/platform/modules/customer_data.py @@ -673,8 +673,9 @@ def identity_pids(agent_name=None, team_id=None, t=None): def source_batches(cur, *, st, agent_name=None, team_id=None, ids=None, needed=None, ledger_batches=()): """Stream the canonical customer identity union and project only one batch at a time.""" - from harness.canonical_source import bounded, BATCH - with bounded(cur, 'identity' if ids is None else 'customer', ids or [], st) as identity: + from harness.canonical_source import bounded, mirrored, BATCH + with mirrored(cur, st, kind='customer'), \ + bounded(cur, 'identity' if ids is None else 'customer', ids or [], st) as identity: live = identity.snapshot(_book_records(team_id)) where = '' if agent_name and agent_name not in {'All', 'All agents'}: diff --git a/platform/modules/opportunity_finder.py b/platform/modules/opportunity_finder.py index c5a6a11081be35b9b30e4c4f5ad47e94e35e4515..81037fee0df719a8d60e31ce9483b97d16a004a3 100644 --- a/platform/modules/opportunity_finder.py +++ b/platform/modules/opportunity_finder.py @@ -284,12 +284,19 @@ def _columns(cur, table): def iso_period(value): - """True when a `_period_start` text starts with an ISO date (YYYY-MM-DD), as `stamp_week` writes it.""" + """Accept an ISO calendar date or timestamp with a valid ISO calendar date prefix.""" + text = _text(value) + if len(text) < 10 or text[4:5] != "-" or text[7:8] != "-": + return False try: - dt.date.fromisoformat(_text(value)[:10]) + dt.date.fromisoformat(text[:10]) + if len(text) > 10: + if text[10] not in ("T", " "): + return False + dt.datetime.fromisoformat(text.replace("Z", "+00:00")) except ValueError: return False - return len(_text(value)) >= 10 + return True def _our_asins(cur): @@ -311,12 +318,13 @@ def _period(cur, kind="VARCHAR"): newest = ('MAX("_period_start")' if str(kind).upper().startswith("VARCHAR") else 'CAST(MAX("_period_start") AS VARCHAR)') rows = _bounded(cur, f'SELECT {newest} FROM {BA_TABLE}') - return str(rows[0][0]) if rows and rows[0][0] is not None else None + value = str(rows[0][0]) if rows and rows[0][0] is not None else None + return value[:10] if value is not None and iso_period(value) else value _TERM = 'lower(trim(CAST("search_term" AS VARCHAR)))' _DEPT = 'trim(CAST("department_name" AS VARCHAR))' -_IN_PERIOD = 'CAST("_period_start" AS VARCHAR) = ?' +_IN_PERIOD = 'substr(CAST("_period_start" AS VARCHAR), 1, 10) = ?' _IN_DEPTS = f"{_DEPT} IN (SELECT unnest(?::VARCHAR[]))" diff --git a/platform/modules/payouts.py b/platform/modules/payouts.py index b642cf6f32fabf43be95e7c465209b38d2e205b4..1431ef860d48ea91969d8d7c22f62b8cf4344909 100644 --- a/platform/modules/payouts.py +++ b/platform/modules/payouts.py @@ -317,7 +317,7 @@ def _load(cur): return headers, groups -def _economics(cur): +def _economics(cur, month=None): """`(currency, {month: net proceeds}, basis, first day, last day)` from the economics report; empty when absent.""" cols = _columns(cur, ECONOMICS) if not cols or "start_date" not in cols: @@ -340,10 +340,20 @@ def _economics(cur): sales = "COALESCE(sales_net_product_sales_amount, 0)" if "sales_net_product_sales_amount" in cols else "0" fees = "COALESCE(fees_total_amount, 0)" if "fees_total_amount" in cols else "0" expr, basis = f"{sales} - {fees} - ({ads})", "net sales minus fees and ads" - months = {m: float(v or 0.0) for m, v in cur.execute( - f"SELECT left(CAST(start_date AS VARCHAR), 7), SUM({expr}) FROM {ECONOMICS} GROUP BY 1").fetchall() if m} first, last = cur.execute(f"SELECT min(left(CAST(start_date AS VARCHAR), 10)), " f"max(left(CAST(start_date AS VARCHAR), 10)) FROM {ECONOMICS}").fetchone() + where, args = "", [] + if month: + where = (" WHERE left(CAST(start_date AS VARCHAR), 7) = ? OR left(CAST(start_date AS VARCHAR), 7) = " + f"(SELECT max(left(CAST(e.start_date AS VARCHAR), 7)) FROM {ECONOMICS} e " + "WHERE left(CAST(e.start_date AS VARCHAR), 7) < ? " + "AND CAST(left(CAST(e.start_date AS VARCHAR), 7) || '-01' AS DATE) >= TRY_CAST(? AS DATE) " + "AND last_day(TRY_CAST(left(CAST(e.start_date AS VARCHAR), 10) AS DATE)) <= TRY_CAST(? AS DATE) " + f"AND EXISTS (SELECT 1 FROM {SETTLEMENTS} s WHERE " + f"strftime({_POSTED}, '%Y-%m') = left(CAST(e.start_date AS VARCHAR), 7)))") + args = [month, month, first, last] + months = {m: float(v or 0.0) for m, v in cur.execute( + f"SELECT left(CAST(start_date AS VARCHAR), 7), SUM({expr}) FROM {ECONOMICS}{where} GROUP BY 1", args).fetchall() if m} return (str(currency).strip() if currency else None), months, basis, parse_date(first), parse_date(last) @@ -671,6 +681,233 @@ def overview(tenant_key, *, st=None, con=None, now=None): } +# --------------------------------------------------------------------------- bounded screen reads + +def _moment_sql(name): + text = f"replace(trim(CAST({name} AS VARCHAR)), ' UTC', '')" + return (f"COALESCE(TRY_CAST({text} AS TIMESTAMPTZ) AT TIME ZONE 'UTC', " + f"try_strptime({text}, ['%d.%m.%Y %H:%M:%S', '%d.%m.%Y %H:%M', '%d.%m.%Y']))") + + +def _settlement_sql(cur): + """Aggregate in SQL before any settlement crosses into Python.""" + _require(cur) + return f"""WITH heads AS ( + SELECT CAST(settlement_id AS VARCHAR) AS id, + any_value(CAST(settlement_start_date AS VARCHAR)) FILTER (WHERE total_amount IS NOT NULL) AS start_text, + any_value(CAST(settlement_end_date AS VARCHAR)) FILTER (WHERE total_amount IS NOT NULL) AS end_text, + any_value(CAST(deposit_date AS VARCHAR)) FILTER (WHERE total_amount IS NOT NULL) AS deposit_text, + any_value(TRY_CAST(total_amount AS DOUBLE)) AS total, + trim(COALESCE(any_value(CAST(currency AS VARCHAR)) FILTER (WHERE total_amount IS NOT NULL), '')) AS currency, + COALESCE(sum(TRY_CAST(amount AS DOUBLE)), 0) AS lines, + count(*) FILTER (WHERE total_amount IS NOT NULL) AS headers, + count(*) FILTER (WHERE amount IS NOT NULL) AS line_count + FROM {SETTLEMENTS} WHERE settlement_id IS NOT NULL GROUP BY 1), + dated AS (SELECT *, {_moment_sql('start_text')} AS start_at, + {_moment_sql('end_text')} AS end_at, {_moment_sql('deposit_text')} AS deposit_at FROM heads), + settlements AS (SELECT *, total IS NOT NULL AND abs(lines - total) < {CENT} AS reconciled FROM dated)""" + + +def _bounds_page(cursor, limit, total): + _page([], cursor, 1) + size = max(1, int(limit)) + start = int(cursor or 0) + if start >= total: + start = max(0, (total - 1) // size * size) + return start, size + + +def _summary_row(values): + sid, currency, start, end, deposit, total, lines, reconciled = values + start, end, deposit = (parse_moment(v) for v in (start, end, deposit)) + return {"id": sid, "currency": currency, "start": _iso(start), "end": _iso(end), + "depositDate": _iso(deposit), "amazonTotal": None if total is None else _money(total), + "linesTotal": _money(lines), "gap": None if total is None else _money(lines - total), + "reconciled": reconciled, + "title": f"Settlement {_day(start)} to {_day(end)}" + (f", {currency}" if currency else "")} + + +_SUMMARY_FIELDS = "id, currency, start_at, end_at, deposit_at, total, lines, reconciled" + + +def _screen_kpis(cur, sql, currency, now): + """Read fixed-size deposit and stream projections with SQL aggregate scans.""" + args = [currency, currency] + where = "(? IS NULL OR currency = ?)" + paid = cur.execute(sql + f" SELECT {_SUMMARY_FIELDS} FROM settlements WHERE {where} " + "AND deposit_at <= ? AND total IS NOT NULL ORDER BY deposit_at DESC, id DESC LIMIT 1", + [*args, now.replace(tzinfo=None)]).fetchone() + last = None + if paid: + previous = cur.execute(sql + " SELECT total, deposit_at FROM settlements WHERE currency = ? " + "AND total IS NOT NULL AND (deposit_at, id) < (?, ?) " + "ORDER BY deposit_at DESC, id DESC LIMIT 1", [paid[1], paid[4], paid[0]]).fetchone() + last = {"settlementId": paid[0], "date": _iso(parse_moment(paid[4])), "amount": _money(paid[5]), + "currency": paid[1], "previousAmount": None if previous is None else _money(previous[0]), + "previousDate": None if previous is None else _iso(parse_moment(previous[1])), + "change": None if previous is None else _money(paid[5] - previous[0])} + pending = cur.execute(sql + f" SELECT {_SUMMARY_FIELDS} FROM settlements WHERE {where} " + "AND deposit_at > ? ORDER BY deposit_at, id LIMIT 1", + [*args, now.replace(tzinfo=None)]).fetchone() + # Chain statistics stay in SQL, including medians over the complete history. + chains = sql + """, ordered AS ( + SELECT *, lag(end_at) OVER (PARTITION BY currency ORDER BY start_at, id) AS previous_end + FROM settlements WHERE start_at IS NOT NULL AND end_at IS NOT NULL), + numbered AS (SELECT *, sum(CASE WHEN previous_end = start_at THEN 0 ELSE 1 END) + OVER (PARTITION BY currency ORDER BY start_at, id) AS chain FROM ordered), + streams AS (SELECT currency, chain, max(end_at) AS newest, + first(id ORDER BY end_at DESC, start_at, id) AS newest_id, + COALESCE(median(epoch(end_at - start_at)) FILTER (WHERE end_at > start_at), 1209600) AS length, + COALESCE(median(epoch(deposit_at - end_at)) FILTER (WHERE deposit_at > end_at), 172800) AS lag, + COALESCE(sum(total) FILTER (WHERE deposit_at BETWEEN ? - INTERVAL 90 DAY AND ?), 0) AS recent + FROM numbered GROUP BY currency, chain), + ranked AS (SELECT *, row_number() OVER (ORDER BY recent DESC, currency, newest_id) AS stream_rank FROM streams), + active AS (SELECT *, newest + length * INTERVAL 1 SECOND AS close_at, + newest + (length + lag) * INTERVAL 1 SECOND AS expected_at FROM ranked + WHERE epoch(? - newest) <= 2 * length + 604800)""" + at = now.replace(tzinfo=None) + opened = cur.execute(chains + f" SELECT currency, newest, close_at, expected_at FROM active WHERE {where} " + "ORDER BY stream_rank LIMIT 1", [at, at, at, *args]).fetchone() + expected = cur.execute(chains + f" SELECT currency, expected_at, stream_rank FROM active WHERE {where} " + "ORDER BY expected_at <= ?, CASE WHEN expected_at > ? THEN expected_at END, " + "expected_at DESC LIMIT 1", [at, at, at, *args, at, at]).fetchone() + nxt = ({"expected": False, "date": _iso(parse_moment(pending[4])), "amount": pending[5], + "currency": pending[1], "settlementId": pending[0]} if pending else + {"expected": True, "date": _iso(parse_moment(expected[1])), "amount": None, + "currency": expected[0], "stream": f"{expected[0] or 'unknown'}-{expected[2]}", + "overdue": expected[1] <= at} if expected else None) + return {"lastDeposit": last, "nextDeposit": nxt, + "openSettlement": None if opened is None else {"currency": opened[0], + "start": _iso(parse_moment(opened[1])), "expectedClose": _iso(parse_moment(opened[2])), + "expectedDeposit": _iso(parse_moment(opened[3])), + "title": f"Open settlement from {_day(parse_moment(opened[1]))}, {opened[0]}".rstrip(", ")}} + + +def screen(tenant_key, *, st=None, con=None, currency=None, view="all", q="", cursor=None, limit=50, now=None): + """Return one searchable settlement page and fixed-size KPI projections.""" + now = parse_moment(now) if now is not None else _now() + currency = str(currency).strip().upper() if currency else None + if view not in ("all", "unreconciled"): + raise ValueError("Choose all or unreconciled settlements.") + with _source(tenant_key, st, con) as (cur, _source_): + sql = _settlement_sql(cur) + where = "(? IS NULL OR currency = ?) AND contains(lower(concat_ws(' ', id, currency, start_text, end_text, deposit_text)), ?)" + args = [currency, currency, str(q or "").strip().lower()] + total, bad = cur.execute(sql + f" SELECT count(*), count(*) FILTER (WHERE NOT reconciled) " + f"FROM settlements WHERE {where}", args).fetchone() + count = total if view == "all" else bad + start, size = _bounds_page(cursor, limit, count) + clause = "" if view == "all" else " AND NOT reconciled" + rows = cur.execute(sql + f" SELECT {_SUMMARY_FIELDS} FROM settlements WHERE {where}{clause} " + "ORDER BY end_at DESC NULLS LAST, id DESC LIMIT ? OFFSET ?", [*args, size, start]).fetchall() + currencies = [r[0] for r in cur.execute(sql + " SELECT DISTINCT currency FROM settlements " + "WHERE currency <> '' ORDER BY currency").fetchall()] + source_count = cur.execute(sql + " SELECT count(*) FROM settlements").fetchone()[0] + kpis = _screen_kpis(cur, sql, currency, now) + return {"empty": source_count == 0, "asOf": _stamp(now), "currencies": currencies, + "counts": {"all": total, "unreconciled": bad}, "settlements": [_summary_row(r) for r in rows], + "cursor": str(start + size) if start + size < count else None, "offset": start, "limit": size, + "kpis": {**kpis, "settlements": total, "unreconciled": bad}} + + +def _bucket_sql(): + tt = "lower(trim(COALESCE(CAST(transaction_type AS VARCHAR), '')))" + at = "lower(trim(COALESCE(CAST(amount_type AS VARCHAR), '')))" + ad = "lower(trim(COALESCE(CAST(amount_description AS VARCHAR), '')))" + quoted = lambda values: ", ".join("'" + v.replace("'", "''") + "'" for v in sorted(values)) + return f"""CASE WHEN {tt} = 'automated loan repayment' OR {at} = 'automated loan repayment' THEN 'loan_repayment' + WHEN {at} = 'itemwithheldtax' OR contains({ad}, 'withholding') THEN 'withheld_tax' + WHEN contains({ad}, 'reserve') THEN 'reserve' + WHEN {at} = 'cost of advertising' OR {ad} = 'cost of advertising' THEN 'ad_cost' + WHEN starts_with({tt}, 'refund') THEN 'refunds' + WHEN {at} IN ('itemprice', 'promotion') THEN 'sales' + WHEN {at} IN ('itemfees', 'itemfee') OR {tt} IN ({quoted(FEE_TYPES)}) + OR (starts_with({tt}, 'other-transaction') AND ({ad} IN ({quoted(FEE_DESCRIPTIONS)}) + OR {at} IN ({quoted(FEE_DESCRIPTIONS)}))) THEN 'fees' ELSE 'other' END""" + + +def screen_settlement(tenant_key, settlement_id, *, st=None, con=None, bucket_key=None, cursor=None, limit=50, q=""): + """Read one settlement, aggregate its tie, and page each waterfall bucket's parts in SQL.""" + with _source(tenant_key, st, con) as (cur, source): + sql = _settlement_sql(cur) + values = cur.execute(sql + f" SELECT {_SUMMARY_FIELDS}, start_text, end_text, deposit_text, headers, line_count " + "FROM settlements WHERE id = ?", [str(settlement_id)]).fetchone() + if values is None: + raise KeyError(settlement_id) + row = _summary_row(values[:8]) + row.update(startText=values[8], endText=values[9], depositText=values[10], headerRows=values[11], lineCount=values[12]) + row["undatedLines"] = cur.execute(f"SELECT count(*) FROM {SETTLEMENTS} WHERE " + f"CAST(settlement_id AS VARCHAR) = ? AND amount IS NOT NULL AND {_POSTED} IS NULL", + [str(settlement_id)]).fetchone()[0] + parts_sql = (f"WITH parts AS (SELECT {_bucket_sql()} AS bucket, " + "COALESCE(CAST(transaction_type AS VARCHAR), '') AS tt, " + "COALESCE(CAST(amount_type AS VARCHAR), '') AS atyp, " + "COALESCE(CAST(amount_description AS VARCHAR), '') AS ad, " + f"sum(TRY_CAST(amount AS DOUBLE)) AS amount, count(*) AS n FROM {SETTLEMENTS} " + "WHERE CAST(settlement_id AS VARCHAR) = ? AND amount IS NOT NULL GROUP BY 1, 2, 3, 4)") + sums = {k: (amount, n, count) for k, amount, n, count in cur.execute( + parts_sql + " SELECT bucket, sum(amount), sum(n), count(*) FROM parts GROUP BY 1", [str(settlement_id)]).fetchall()} + waterfall = [] + if bucket_key is not None and bucket_key not in LABELS: + raise ValueError("That waterfall bucket does not exist.") + for key, label in BUCKETS: + amount, n, count = sums.get(key, (0, 0, 0)) + filters = [str(settlement_id), key, str(q or "").strip().lower()] + where = "bucket = ? AND contains(lower(concat_ws(' ', tt, atyp, ad)), ?)" + total = cur.execute(parts_sql + f" SELECT count(*) FROM parts WHERE {where}", filters).fetchone()[0] + start, size = _bounds_page(cursor if bucket_key == key else None, limit, total) + chosen = cur.execute(parts_sql + f" SELECT tt, atyp, ad, amount, n FROM parts WHERE {where} " + "ORDER BY abs(amount) DESC, tt, atyp, ad LIMIT ? OFFSET ?", [*filters, size, start]).fetchall() \ + if bucket_key in (None, key) else [] + waterfall.append({"key": key, "label": label, "amount": _money(amount), "lines": n, + "parts": [{"transactionType": tt, "amountType": at, "description": ad, + "amount": _money(a), "lines": n} for tt, at, ad, a, n in chosen], + "total": total, "offset": start, + "cursor": str(start + size) if start + size < total else None}) + waterfall.append({"key": "deposit", "label": "Deposit", "amount": row["amazonTotal"], "lines": 0, "parts": []}) + month = values[3].strftime("%Y-%m") if values[3] else None + economics = _economics(cur, month=month or "0001-01") + months = list(economics[1]) + totals = {} + if months: + grouped = cur.execute(sql + f" SELECT strftime({_POSTED}, '%Y-%m'), {_bucket_sql()}, " + f"sum(TRY_CAST(amount AS DOUBLE)) FROM {SETTLEMENTS} s " + "JOIN settlements h ON CAST(s.settlement_id AS VARCHAR) = h.id " + f"WHERE h.currency = ? AND strftime({_POSTED}, '%Y-%m') IN " + "(" + ",".join("?" for _ in months) + ") GROUP BY 1, 2", [row["currency"], *months]).fetchall() + for m, key, amount in grouped: + totals.setdefault(m, {})[key] = float(amount or 0) + row.update(waterfall=waterfall, _end=parse_moment(values[3])) + tie = _tie(row, [{"currency": row["currency"], "_months": totals}], economics) + if not row["reconciled"]: + row["reason"] = _reason(row) + row.pop("_end") + return {**row, "tie": tie, "source": source} + + +def screen_item(tenant_key, row, *, st=None): + """Read one owned override or project the same default item without writing during a GET.""" + st = _rt(tenant_key, st) + from core import store, store_pg + resolve = getattr(st, "_document_store", None) + backend = resolve() if callable(resolve) else st + if backend is store and store.backend() == "pg": + backend = store._d() + if isinstance(backend, store_pg.PgStore): + if backend.tenant_slug != str(tenant_key).strip().lower(): + raise PermissionError("The Payouts store belongs to another tenant.") + namespace = getattr(st, "store_key", lambda name: name) + with store_pg._pool().connection() as con: + result = con.execute(store_pg._q( + "SELECT (value #> %s::text[]) - 'history' FROM {tbl} WHERE key = %s", + "store_kv", backend.tenant_slug), (["items", row["id"]], namespace(KEY_ITEMS))).fetchone() + item = result[0] if result else None + else: + item = (_doc(st, KEY_ITEMS).get("items") or {}).get(row["id"]) + return item or {"id": row["id"], "status": "Open", "owner": settings(tenant_key, st=st)["defaultOwner"], + "due": (_now() + dt.timedelta(days=DUE_DAYS)).date().isoformat()} + + # --------------------------------------------------------------------------- settings and owner #: Accounts that sign in for a process, never a person who can own an item. @@ -774,6 +1011,14 @@ def items(tenant_key, user, *, status=None, cursor=None, limit=PAGE, st=None, co return {"items": chunk, "cursor": nxt} +def refresh_items(tenant_key, *, st=None, now=None): + """Generate owned settlement items in the worker, so screen reads preserve their stored due dates.""" + st = _rt(tenant_key, st) + rows, _economics_, _source_ = _all(tenant_key, st) + doc = _refresh_items(tenant_key, st, rows, now or _now()) + return len(doc.get("items") or {}) + + def _who(user): """The acting name, or raise PermissionError for no user, a viewer, or a user who may not edit.""" if not user: @@ -793,6 +1038,11 @@ def set_status(tenant_key, user, item_id, status, *, note=None, st=None): if status not in STATUSES: raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") st = _rt(tenant_key, st) + if str(item_id) not in (_doc(st, KEY_ITEMS).get("items") or {}): + detail = screen_settlement(tenant_key, item_id, st=st, limit=1) + if detail["reconciled"]: + raise KeyError(item_id) + _refresh_items(tenant_key, st, [detail], _now()) out = {} def _apply(doc): diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 6d38602fbb4f7e535d1d9cb9623de3a6c9fa5a67..02d2ec84f81ffa08fe768fbafa7cc58d04c24903 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -782,6 +782,8 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None): tiers, _tier_report = products.tier_prices_by_code(_prods) if wants( [f'price_{n}' for n in range(1, 4)] + [f'unit_{n}' for n in range(1, 4)] + [key for price, unit, _name, _n in products.PRICELIST_TIER_COLUMNS for key in (price, unit)]) else ({}, {}) + # Scope even an unscoped reader or cached projection before pooled cells and overflow receipts. + tiers, _tier_report = products.scope_tier_prices(tiers, _tier_report, team_id=team_id) pre_book, open_backlog, _backlog_report = backorders.sku_backlog(t=t, team_id=team_id) \ if wants(['pre_book_qty', 'open_backlog_qty']) else ({}, {}, {}) cat = _catalogue_by_code() @@ -899,7 +901,7 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None): # # ⚠ NO SECOND ODOO READ. `tiers` already carries the pricelist name on every entry; # `per_list_tier_cells` is a group-by over the list this loop is already holding. - row.update(products.per_list_tier_cells(_tiers)) + row.update(products.per_list_tier_cells(_tiers, team_id=team_id)) # ⭐ The cap's own receipt, and it is SET ONLY WHEN NON-EMPTY. `rows_from_pool` projects # strictly by the declared contract, so this rides the pool row for `validate()` and can # never reach a grid cell; writing `{}` on all 5,873 rows to say "nothing dropped" would @@ -1550,7 +1552,7 @@ def validate(team_id=None, t=None): checks.extend(validate_order_counts(rows, t=t, team_id=team_id)) checks.extend(validate_measures(t=t, team_id=team_id, pool_codes={r["code"] for r in rows})) - checks.extend(validate_price_and_unit_cells(rows)) + checks.extend(validate_price_and_unit_cells(rows, team_id=team_id)) checks.extend(validate_stock_measures(t=t)) checks.extend(validate_fba(rows)) checks.extend(validate_available_forecasted(rows)) @@ -2125,7 +2127,7 @@ def validate_stock_measures(t=None, days=90, sample=4): return checks -def validate_price_and_unit_cells(rows, sample=6): +def validate_price_and_unit_cells(rows, sample=6, *, team_id=None): """Reconcile typed Price 1..3 / Unit 1..3 cells to a fresh, per-SKU Odoo read. The pool groups price rules in bulk. This asks Odoo per SKU so a wrong grouping, a lost @@ -2141,34 +2143,41 @@ def validate_price_and_unit_cells(rows, sample=6): "ours": {"with_price_1": len(priced), "with_first_pair": len(with_first_pair), "rows": len(rows)}, "theirs": {"price_1": ">0", "typed_first_pair": len(rows)}, - "ok": bool(priced) and len(with_first_pair) == len(rows) + "ok": (bool(priced) or team_id is not None) and len(with_first_pair) == len(rows) and not any("tier_prices" in r or "units" in r for r in rows), "detail": {"two_or_more_prices": sum(1 for r in rows if r.get("price_2") is not None), "three_prices": sum(1 for r in rows if r.get("price_3") is not None)}, }) # ── W40-T09 / R6 — the PER-LIST cells, which is what the pooled three above are not ────── - _tier_keys = [k for t in products.PRICELIST_TIER_COLUMNS for k in t[:2]] + _allowed_lists = products.pricelist_tier_lists(team_id) + _tier_columns = [t for t in products.PRICELIST_TIER_COLUMNS if t[2] in _allowed_lists] + _tier_keys = [k for t in _tier_columns for k in t[:2]] + _denied_keys = {k for t in products.PRICELIST_TIER_COLUMNS if t[2] not in _allowed_lists + for k in t[:2]} _typed = [r for r in rows if all(k in r for k in _tier_keys)] + _leaked = [r.get('code') for r in rows if _denied_keys.intersection(r)] _per_list = {} - for _p, _u, _name, _n in products.PRICELIST_TIER_COLUMNS: + for _p, _u, _name, _n in _tier_columns: _per_list.setdefault(_name, {})[f"t{_n}"] = sum(1 for r in rows if r.get(_p) is not None) checks.append({ - "check": "the product grid serves all 18 per-list tier cells (price and unit 1..3 for " - "each declared pricelist), typed and on every row", - "ours": {"rows_with_all_18": len(_typed), "rows": len(rows), "cells": len(_tier_keys)}, - "theirs": {"rows_with_all_18": len(rows), "cells": 18}, - "ok": len(_typed) == len(rows) and len(_tier_keys) == 18, + "check": f"the product grid serves all {len(_tier_keys)} authorized per-list tier cells " + "typed on every row, with denied columns absent", + "ours": {f"rows_with_all_{len(_tier_keys)}": len(_typed), "rows": len(rows), + "cells": len(_tier_keys), "denied_columns_on_rows": len(_leaked)}, + "theirs": {f"rows_with_all_{len(_tier_keys)}": len(rows), "cells": 6 * len(_allowed_lists), + "denied_columns_on_rows": 0}, + "ok": len(_typed) == len(rows) and len(_tier_keys) == 6 * len(_allowed_lists) and not _leaked, "detail": {"skus_priced_per_list": _per_list}, }) # ⛔ CHEAPEST-FIRST PACKS FROM THE TOP, so a hole is the signature of a misaligned write: # tier 2 present with tier 1 empty would mean the cells no longer mean "the nth cheapest". _price_key = {(_name, _n): _p - for _p, _u, _name, _n in products.PRICELIST_TIER_COLUMNS} + for _p, _u, _name, _n in _tier_columns} _unit_key = {(_name, _n): _u - for _p, _u, _name, _n in products.PRICELIST_TIER_COLUMNS} + for _p, _u, _name, _n in _tier_columns} _declared_names = list(dict.fromkeys(_n2 for _p, _u, _n2, _n in - products.PRICELIST_TIER_COLUMNS)) + _tier_columns)) _holes = [r.get("code") for r in rows for (_name, _n), _p in _price_key.items() if _n > 1 and r.get(_p) is not None @@ -2211,6 +2220,8 @@ def validate_price_and_unit_cells(rows, sample=6): # the DATA was right and only the check was wrong. _plname = {p["id"]: str(p.get("name") or "").strip() for p in o.search_read("product.pricelist", [], ["id", "name"])} + if team_id is not None: + _plname = {pid: name for pid, name in _plname.items() if name in _allowed_lists} # Prefer SKUs that carry more than one price, the ticket's own subject. cand = sorted(priced, key=lambda r: -sum(r.get(f"price_{n}") is not None for n in range(1, 4)))[:sample] @@ -2230,6 +2241,8 @@ def validate_price_and_unit_cells(rows, sample=6): ("applied_on", "in", ["0_product_variant", "1_product"]), ("fixed_price", ">", 0), "|", ("product_id", "=", pid), ("product_tmpl_id", "=", tmpl)] + if team_id is not None: + dom.append(("pricelist_id", "in", list(_plname))) live = o.search_read("product.pricelist.item", dom, ["pricelist_id", "product_id", "product_tmpl_id", "fixed_price", "packaging_price", "packaging_id", "applied_on"]) diff --git a/platform/modules/products.py b/platform/modules/products.py index 330df17afd62c5c0f13adc0bdeeab63d1fae63c5..89303bab9acc1268dbcb54a63724dd1312e7c30b 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -582,6 +582,13 @@ PRICELIST_TIER_DEPTH = 3 PRICELIST_TIER_LISTS = ("Fisch", "Royal 1", "Royal 2") +def pricelist_tier_lists(team_id=None): + """Return authorized tier lists, with no lists for an unknown scoped team.""" + if team_id is None: + return PRICELIST_TIER_LISTS + return {5: ("Fisch",), 6: ("Royal 1", "Royal 2")}.get(team_id, ()) + + def _pricelist_slug(name): """`Royal 1` -> `royal1`, the stable token used by a numbered tier key.""" return "".join(ch.lower() for ch in str(name) if ch.isalnum()) @@ -597,8 +604,8 @@ PRICELIST_TIER_COLUMNS = tuple( ) -def per_list_tier_cells(tiers): - """One SKU's `tier_prices_by_code` entry -> `{column_key: value}` for all 18 R6 cells. +def per_list_tier_cells(tiers, team_id=None): + """Project authorized tier cells and keep unpriced authorized lists blank. ⭐⭐ THE WHOLE OF W40-T09, and it is a GROUP-BY, not a new read. `tier_prices_by_code` already carries the pricelist NAME on every entry, so attributing prices to the list that @@ -623,15 +630,19 @@ def per_list_tier_cells(tiers): Public Pricelist, so they carry all eighteen cells empty. `tier_prices_by_code` still sees those prices; nothing is lost from the honest set, only from these named columns. """ + allowed = pricelist_tier_lists(team_id) grouped = {} for tier in tiers or (): - grouped.setdefault(tier.get("pricelist"), []).append(tier) + if tier.get("pricelist") in allowed: + grouped.setdefault(tier.get("pricelist"), []).append(tier) for got in grouped.values(): got.sort(key=lambda t: (t.get("unit_price") or 0.0, str(t.get("unit") or "").lower(), str(t.get("pricelist") or "").lower())) cells = {} for price_key, unit_key, name, n in PRICELIST_TIER_COLUMNS: + if name not in allowed: + continue got = grouped.get(name) or [] tier = got[n - 1] if len(got) >= n else None cells[price_key] = tier.get("unit_price") if tier else None @@ -639,6 +650,28 @@ def per_list_tier_cells(tiers): return cells +def scope_tier_prices(by_code, report, team_id=None): + """Detach scoped tiers and overflow receipts without exposing global source counts or errors.""" + if team_id is None: + return by_code, report + from harness.canonical_source import TierProjection + allowed = pricelist_tier_lists(team_id) + scoped = {} + for code, tiers in by_code.items(): + kept = TierProjection() + kept.extend(dict(t) for t in tiers if t.get("pricelist") in allowed) + kept.overflow = {name: count for name, count in tier_cap_overflow(tiers).items() + if name in allowed} + if kept: + scoped[code] = kept + scoped_report = {"lists": list(allowed)} + if "bounded" in report: + scoped_report["bounded"] = report["bounded"] + if report.get("error"): + scoped_report["error"] = "tier price source unavailable" + return scoped, scoped_report + + def tier_cap_overflow(tiers): """`{pricelist_name: how many tiers PRICELIST_TIER_DEPTH hides}` for one SKU, `{}` when none. @@ -682,7 +715,7 @@ def active_products(limit=50000, extra_fields=()): return prods -def tier_prices_by_code(prods=None): +def tier_prices_by_code(prods=None, *, team_id=None): """`({code: [{pricelist, unit_price, unit}]}, report)` — EVERY live price a SKU really has. ⭐⭐ W37-T14 (owner: multiple prices per SKU). This is the honest set — one entry per PRICE @@ -690,9 +723,8 @@ def tier_prices_by_code(prods=None): rows for the same SKU; collapsing those rows loses the unit-specific prices the catalogue needs to show. - ⛔ IT READS EVERY LIVE PRICELIST. `per_list_tier_cells` later selects the three list names the - product contract declares; that selection must never narrow this source reader, or a future - tenant's additional list will quietly disappear from the honest price set. + Unscoped callers retain every live pricelist. Scoped callers receive only authorized tiers + and metadata, including when the active source returns a cached mirror projection. ⛔ ROWS ARE TEMPLATE-SCOPED (`proto/P2-pricing-uom.md`): joining on `product_id` drops 99.9% of price rows, because `applied_on` is `1_product` on 10,454 of 10,468 items and `1_product` means @@ -710,8 +742,10 @@ def tier_prices_by_code(prods=None): catalogue-true and commercially misleading. The entry keeps the list NAME for provenance. """ from harness.canonical_source import active + if team_id is not None and not pricelist_tier_lists(team_id): + return {}, {"lists": []} if active() is not None: - return active().tiers(prods) + return scope_tier_prices(*active().tiers(prods), team_id=team_id) report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0, "rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0, "rules_with_packaging": 0} @@ -753,7 +787,7 @@ def tier_prices_by_code(prods=None): prods = active_products() if prods is None else prods except Exception as e: # noqa: BLE001 report["error"] = f"{type(e).__name__}: {str(e)[:200]}" - return {}, report + return scope_tier_prices({}, report, team_id=team_id) out = {} for p in prods: @@ -790,7 +824,7 @@ def tier_prices_by_code(prods=None): report["skus_with_no_price"] += 1 for tiers in out.values(): tiers.sort(key=lambda t: (t["unit_price"], t["unit"].lower(), t["pricelist"].lower())) - return out, report + return scope_tier_prices(out, report, team_id=team_id) def packagings_by_code(prods=None): diff --git a/platform/modules/ranking_diagnostic.py b/platform/modules/ranking_diagnostic.py index 701f9b9f7ef23a3c185712a5ebfcb068933b59e0..01b6db0571cbf9c0ff5c8b72d75cec52942c6bf9 100644 --- a/platform/modules/ranking_diagnostic.py +++ b/platform/modules/ranking_diagnostic.py @@ -22,9 +22,11 @@ A re-sync in the same week replaces that week's point and never adds a second on Limit (rule 1): one document per week, so no document grows with the history. At about 400 bytes an ASIN, a week document reaches the store's 128 MB ceiling near 300,000 ASINs. """ +import base64 import contextlib import datetime as dt import hashlib +import heapq import json import re from pathlib import Path @@ -748,30 +750,111 @@ def on_amazon_sync(tenant_key, synced_tables, *, st=None, now=None): # --------------------------------------------------------------------------- C4 -def _all_items(st): - out = [] - for week in reversed(_index(st)): - for iid, row in (_doc(st, ITEMS_PREFIX + week).get("items") or {}).items(): - out.append({"id": iid, **{k: v for k, v in row.items() if k != "history"}}) - out.sort(key=lambda i: (-int(i["week"].replace("-", "")), CAUSE_ORDER.get(i.get("causeKey"), 99), i["asin"])) - return out +def _document_reader(st): + return st._document_store() if callable(getattr(st, "_document_store", None)) else st + + +def _store_key(st, name): + return st.store_key(name) if callable(getattr(st, "store_key", None)) else name + + +def _paths(st, name, paths): + """Read selected fields without copying the surrounding weekly population.""" + return _document_reader(st).get_json_paths(_store_key(st, name), paths) + +def _entries(st, name, population): + """Stream stored members in bounded batches for reconciliation and non-SQL stores.""" + yield from _document_reader(st).iter_json_items(_store_key(st, name), (population,), batch_size=200) -def _page(rows, cursor, limit): + +def _page_args(cursor, limit, population): try: - start = int(cursor or 0) size = int(limit) - except (TypeError, ValueError): - raise ValueError("The page cursor is not valid.") from None - if start < 0 or size < 1: - raise ValueError("The page cursor is not valid.") - end = start + size - return rows[start:end], (str(end) if end < len(rows) else None) + after = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode()) if cursor else None + if size < 1 or size > DEFAULT_LIMIT: + raise ValueError + if after is not None: + types = (int, int, str, str) if population == "items" else (float, str) + if not isinstance(after, list) or len(after) != len(types): + raise ValueError + after = tuple(kind(value) for kind, value in zip(types, after)) + except (ValueError, TypeError, UnicodeError): + raise ValueError("The page cursor or size is not valid.") from None + return after, size + + +def _row_order(iid, row, population): + if population == "items": + return (-int(row["week"].replace("-", "")), CAUSE_ORDER.get(row.get("causeKey"), 99), row["asin"], iid) + return (-(row.get("sales") or 0.0), iid) + + +def _list_page(st, names, population, *, cursor=None, limit=DEFAULT_LIMIT, status=None, asin=None, q=""): + """Search, count and select one page before stored members reach the App response.""" + after, size = _page_args(cursor, limit, population) + query = _text(q).lower() + backend = _document_reader(st) + from core import store_pg + if isinstance(backend, store_pg.PgStore): + cases = " ".join(f"WHEN '{key}' THEN {value}" for key, value in CAUSE_ORDER.items()) + order = (["-CAST(replace(v->>'week', '-', '') AS bigint)", f"CASE v->>'causeKey' {cases} ELSE 99 END", + "v->>'asin'", "id"] if population == "items" else + ["-COALESCE((v->>'sales')::double precision, 0)", "id"]) + fields = "concat_ws(' ', id, v->>'asin', v->>'title', v->>'productTitle', v->>'sku', (v->'skus')::text)" + params = [population, [_store_key(st, name) for name in names], query, query, asin, asin, status, status] + seek = "TRUE" + if after is not None: + seek = f"({', '.join(order)}) > ({', '.join('%s' for _ in order)})" + params.extend(after) + params.append(size + 1) + sql = ( + "WITH documents AS MATERIALIZED (SELECT value -> %s AS members FROM {tbl} WHERE key=ANY(%s::text[])), " + "source AS MATERIALIZED (SELECT e.key AS id, e.value AS v FROM documents " + "CROSS JOIN LATERAL jsonb_each(COALESCE(members, '{{}}'::jsonb)) e), " + f"filtered AS MATERIALIZED (SELECT * FROM source WHERE (%s='' OR position(%s in lower({fields}))>0) " + "AND (%s::text IS NULL OR v->>'asin'=%s)), " + "wanted AS MATERIALIZED (SELECT * FROM filtered WHERE (%s::text IS NULL OR v->>'status'=%s)), " + f"page AS (SELECT id, v - 'history' AS v FROM wanted WHERE {seek} ORDER BY {', '.join(order)} LIMIT %s) " + "SELECT (SELECT COALESCE(jsonb_object_agg(status, n), '{{}}'::jsonb)::text FROM " + "(SELECT v->>'status' AS status, count(*) AS n FROM filtered WHERE v ? 'status' GROUP BY 1) counts), " + "(SELECT count(*) FROM wanted), (SELECT COALESCE(jsonb_agg(jsonb_build_array(id, v) " + f"ORDER BY {', '.join(order)}), '[]'::jsonb)::text FROM page)") + with store_pg._pool().connection() as con: + counts, total, raw = con.execute(store_pg._q(sql, "store_kv", backend.tenant_slug), params).fetchone() + counts, chosen = json.loads(counts), json.loads(raw) + else: + counts, total = {}, 0 + + def matching(): + nonlocal total + for name in names: + for batch in _entries(st, name, population): + for iid, row in batch: + fields = [iid, row.get("asin"), row.get("title"), row.get("productTitle"), + row.get("sku"), json.dumps(row.get("skus") or [])] + if (asin is not None and row.get("asin") != asin) or (query and query not in + " ".join(str(v) for v in fields if v is not None).lower()): + continue + state = row.get("status") + if state: + counts[state] = counts.get(state, 0) + 1 + if status is not None and state != status: + continue + total += 1 + if after is None or _row_order(iid, row, population) > after: + yield iid, row + chosen = heapq.nsmallest(size + 1, matching(), key=lambda entry: _row_order(*entry, population)) + more = len(chosen) > size + chosen = chosen[:size] + nxt = (base64.urlsafe_b64encode(json.dumps(_row_order(*chosen[-1], population)).encode()).decode() + if more else None) + return chosen, nxt, counts, int(total) def _problems(st): held = _index(st) - out = list(_doc(st, WEEK_PREFIX + held[-1]).get("problems") or ()) if held else [ + out = list(_paths(st, WEEK_PREFIX + held[-1], [("problems",)])[0] or ()) if held else [ "No weekly history yet. The first point is written at the next Amazon sync."] if len(held) == 1: out.append(NOT_READY) @@ -781,19 +864,17 @@ def _problems(st): return out -def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, week=None, asin=None, st=None): +def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, week=None, asin=None, q="", st=None): """C4: owned drop items, newest week first, root causes first, one page. Counts cover every item.""" if status is not None and status not in STATUSES: raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") st = _rt(tenant_key, st) - every = [i for i in _all_items(st) if (week is None or i["week"] == week) and (asin is None or i["asin"] == asin)] - counts = {s: 0 for s in STATUSES} - for item in every: - counts[item["status"]] = counts.get(item["status"], 0) + 1 - wanted = [i for i in every if status is None or i["status"] == status] - page, nxt = _page(wanted, cursor, limit) held = _index(st) - return {"items": page, "cursor": nxt, "counts": counts, "problems": _problems(st), + names = [ITEMS_PREFIX + w for w in held if week is None or week == w] + chosen, nxt, counts, total = _list_page(st, names, "items", cursor=cursor, limit=limit, status=status, asin=asin, q=q) + page = [{"id": iid, **{k: v for k, v in row.items() if k != "history"}} for iid, row in chosen] + return {"items": page, "cursor": nxt, "counts": {s: counts.get(s, 0) for s in STATUSES}, "total": total, + "problems": _problems(st), "week": week or (held[-1] if held else None), "weeks": held} @@ -801,13 +882,13 @@ def item_detail(tenant_key, user, iid, *, st=None): """One item with its history, the two points it compares, and its cause card.""" st = _rt(tenant_key, st) week = _week_of_id(iid) - row = (_doc(st, ITEMS_PREFIX + week).get("items") or {}).get(iid) if week else None + row = _paths(st, ITEMS_PREFIX + week, [("items", iid)])[0] if week else None if not isinstance(row, dict): raise KeyError(iid) held = _index(st) before = [w for w in held if w < week] - prev = (_doc(st, WEEK_PREFIX + before[-1]).get("asins") or {}).get(row["asin"]) if before else None - cur = (_doc(st, WEEK_PREFIX + week).get("asins") or {}).get(row["asin"]) + prev = _paths(st, WEEK_PREFIX + before[-1], [("asins", row["asin"])])[0] if before else None + cur = _paths(st, WEEK_PREFIX + week, [("asins", row["asin"])])[0] return {"item": {"id": iid, **{k: v for k, v in row.items() if k != "history"}}, "history": list(row.get("history") or ()), "previous": prev, "current": cur, "previousWeek": before[-1] if before else None, @@ -862,18 +943,30 @@ def on_decision(tenant_key, iid, decision, reason, *, st=None): return {"id": iid, "decision": decision, "applied": False} -def asins(tenant_key, user=None, *, cursor=None, limit=DEFAULT_LIMIT, week=None, st=None): +def asins(tenant_key, user=None, *, cursor=None, limit=DEFAULT_LIMIT, week=None, q="", st=None): """The ASIN picker: every ASIN in the week's history (newest by default), most sales first. No cap.""" st = _rt(tenant_key, st) held = _index(st) week = week or (held[-1] if held else None) - points = (_doc(st, WEEK_PREFIX + week).get("asins") or {}) if week else {} - flagged = {i["asin"]: i["causeKey"] for i in _all_items(st) if i["week"] == week} + chosen, nxt, _counts, total = _list_page(st, [WEEK_PREFIX + week] if week else [], "asins", + cursor=cursor, limit=limit, q=q) + flags = _paths(st, ITEMS_PREFIX + week, [("items", item_id(week, a), "causeKey") for a, _p in chosen]) if chosen else [] + flagged = dict(zip((a for a, _p in chosen), flags)) rows = [{"asin": a, "title": p.get("title"), "skus": p.get("skus") or [], "units": p.get("units"), - "sales": p.get("sales"), "cause": flagged.get(a)} for a, p in points.items()] - rows.sort(key=lambda r: (-(r["sales"] or 0.0), r["asin"])) - page, nxt = _page(rows, cursor, limit) - return {"asins": page, "cursor": nxt, "total": len(rows), "week": week} + "sales": p.get("sales"), "cause": flagged.get(a)} for a, p in chosen] + return {"asins": rows, "cursor": nxt, "total": total, "week": week} + + +def screen_totals(tenant_key, week=None, *, st=None): + """The selected week's source totals, using independent SQL for the newest week.""" + st = _rt(tenant_key, st) + held = _index(st) + if not held: + return None + if week and week != held[-1]: + return _paths(st, WEEK_PREFIX + week, [("sourceTotals",)])[0] + with _reader(tenant_key, st) as cur: + return source_totals(cur) if cur is not None else None SIGNALS = ("sessions", "buyBox", "units", "salesRank", "searchShare", "adSpend") @@ -884,7 +977,7 @@ def trend(tenant_key, asin, *, st=None): st = _rt(tenant_key, st) points = [] for week in _index(st): - point = (_doc(st, WEEK_PREFIX + week).get("asins") or {}).get(asin) + point = _paths(st, WEEK_PREFIX + week, [("asins", asin)])[0] if point is not None: points.append({"week": week, **{s: point.get(s) for s in SIGNALS}}) if not points: @@ -916,12 +1009,15 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None): app = {"sessions": 0.0, "units": 0.0} ind = {"sessions": 0.0, "units": 0.0} for w in chosen: - doc = _doc(st, WEEK_PREFIX + w) - points = doc.get("asins") or {} - mine = {"sessions": round(sum(p.get("sessions") or 0.0 for p in points.values()), 4), - "units": round(sum(p.get("units") or 0.0 for p in points.values()), 4), - "asinsWithUnits": sum(1 for p in points.values() if (p.get("units") or 0) > 0)} - theirs = live if w == held[-1] else doc.get("sourceTotals") + mine = {"sessions": 0.0, "units": 0.0, "asinsWithUnits": 0} + for batch in _entries(st, WEEK_PREFIX + w, "asins"): + for _asin, point in batch: + for key in ("sessions", "units"): + mine[key] += point.get(key) or 0.0 + mine["asinsWithUnits"] += int((point.get("units") or 0) > 0) + for key in ("sessions", "units"): + mine[key] = round(mine[key], 4) + theirs = live if w == held[-1] else _paths(st, WEEK_PREFIX + w, [("sourceTotals",)])[0] tied = bool(theirs) and all(abs(mine[k] - float(theirs.get(k) or 0)) < 0.005 for k in ("sessions", "units")) \ and mine["asinsWithUnits"] == int(theirs.get("asinsWithUnits") or 0) ok = ok and tied diff --git a/platform/modules/reimbursements.py b/platform/modules/reimbursements.py index 49685bddff29c42eef015580569aada7fb156006..832521130f2f849c4990c531c9b2f7ce52b983d4 100644 --- a/platform/modules/reimbursements.py +++ b/platform/modules/reimbursements.py @@ -22,6 +22,7 @@ Limit (rule 1): every list is uncapped and paged by cursor. The items document h import contextlib import datetime as dt import hashlib +import json import re from pathlib import Path @@ -149,13 +150,15 @@ def _columns(cur, table): return set() -def _rows(cur, table, wanted, where=""): - """Every row as {column: text or None}. A missing column reads None. No cap (rule 1).""" +def _rows(cur, table, wanted, where="", *, args=(), limit=None, offset=0): + """Project source rows, with a SQL page on interactive reads.""" cols = _columns(cur, table) if not cols: return [] exprs = [f'CAST("{c}" AS VARCHAR)' if c in cols else "NULL" for c in wanted] - got = cur.execute(f'SELECT {", ".join(exprs)} FROM {table} {where}').fetchall() + paging = " LIMIT ? OFFSET ?" if limit is not None else "" + params = [*args, limit, offset] if limit is not None else list(args) + got = cur.execute(f'SELECT {", ".join(exprs)} FROM {table} {where}{paging}', params).fetchall() return [dict(zip(wanted, row)) for row in got] @@ -460,18 +463,175 @@ def _page(rows, cursor, limit): # --------------------------------------------------------------------------- C4 -def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, st=None): - """C4: claim items, soonest claim-by first, one page. Counts cover every item. No cap (rule 1).""" +def _bounds(cursor, limit, total): + """Validate and snap an offset to the last reachable page.""" + try: + start, size = int(cursor or 0), int(limit) + except (TypeError, ValueError): + raise ValueError("The page cursor is not valid.") from None + if start < 0 or size < 1: + raise ValueError("The page cursor is not valid.") + if start >= total: + start = max(0, (total - 1) // size * size) + return start, size + + +def _day_sql(column): + raw = f"SUBSTR(TRIM(CAST({column} AS VARCHAR)), 1, 10)" + return (f"COALESCE(TRY_CAST({raw} AS DATE), CAST(TRY_STRPTIME({raw}, '%d.%m.%Y') AS DATE), " + f"CAST(TRY_STRPTIME({raw}, '%m/%d/%Y') AS DATE))") + + +def _source_sql(cur, table, wanted, *, empty=False): + cols = _columns(cur, table) + exprs = [f'COALESCE(TRIM(CAST("{c}" AS VARCHAR)), \'\') AS "{c}"' if c in cols + else f'CAST(\'\' AS VARCHAR) AS "{c}"' for c in wanted] + empty_exprs = [f"CAST('' AS VARCHAR) AS {c}" for c in wanted] + return (f'SELECT rowid AS _row, {", ".join(exprs)} FROM {table}' if cols and not empty + else f'SELECT 0::BIGINT AS _row, {", ".join(empty_exprs)} WHERE FALSE') + + +def _claims_sql(cur, st): + """Calculate claim identities and offsets in SQL without materializing sources in Python.""" + problems = [] + quoted = _ledger_quoted(cur) + if quoted: + problems.append(D2682) + for table, label in ((LEDGER_TABLE, "inventory ledger"), (REIMBURSEMENTS_TABLE, "reimbursements report"), + (RETURNS_TABLE, "customer returns report"), (SETTLEMENTS_TABLE, "settlement report")): + if not _columns(cur, table): + problems.append(f"The Amazon {label} has no rows in the mirror yet.") + overrides = _doc(st, KEY_ITEMS).get("items") or {} + states = {item_id: {"status": row.get("status"), "caseId": _text(row.get("caseId"))} + for item_id, row in overrides.items() if isinstance(row, dict)} + sql = f"""WITH + ledger AS ({_source_sql(cur, LEDGER_TABLE, LEDGER_COLS, empty=quoted)}), + reimb AS ({_source_sql(cur, REIMBURSEMENTS_TABLE, REIMBURSEMENT_COLS)}), + returned AS ({_source_sql(cur, RETURNS_TABLE, RETURN_COLS)}), + settlements AS ({_source_sql(cur, SETTLEMENTS_TABLE, SETTLEMENT_COLS)}), + overrides AS (SELECT key AS id, json_extract_string(value, '$.status') AS status, + TRIM(COALESCE(json_extract_string(value, '$.caseId'), '')) AS case_id FROM json_each(?)), + filed AS (SELECT DISTINCT case_id FROM overrides WHERE case_id <> ''), + unpaid AS (SELECT * FROM reimb WHERE case_id = '' OR case_id NOT IN (SELECT case_id FROM filed)), + paid_units AS (SELECT CASE reason WHEN '{LOST_REIMBURSEMENT}' THEN 'lost' ELSE 'damaged' END AS kind, + fnsku, SUM(COALESCE(TRY_CAST(quantity_reimbursed_total AS DOUBLE), 0)) AS units + FROM unpaid WHERE reason IN ('{LOST_REIMBURSEMENT}', '{DAMAGED_REIMBURSEMENT}') GROUP BY kind, fnsku), + found AS (SELECT fnsku, SUM(TRY_CAST(quantity AS DOUBLE)) AS units FROM ledger + WHERE event_type = 'Adjustments' AND reason = 'F' AND TRY_CAST(quantity AS DOUBLE) > 0 GROUP BY fnsku), + losses AS (SELECT *, CASE WHEN reason IN ('M', '5') THEN 'lost' ELSE 'damaged' END AS kind, + -TRY_CAST(quantity AS DOUBLE) AS lost_units, {_day_sql('date')} AS event_day + FROM ledger WHERE event_type = 'Adjustments' AND reason IN ('M', '5', 'E', '6', '7', 'H') + AND TRY_CAST(quantity AS DOUBLE) < 0), + loss_keys AS (SELECT *, CONCAT_WS('|', kind, date, fnsku, reference_id, reason, fulfillment_center, quantity) + AS id_parts FROM losses), + ordered_losses AS (SELECT *, + COALESCE(SUM(lost_units) OVER (PARTITION BY kind, fnsku + ORDER BY event_day NULLS FIRST, date_and_time, reference_id, _row + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0) AS prior_units, + ROW_NUMBER() OVER (PARTITION BY SHA1(id_parts) + ORDER BY kind, fnsku, event_day NULLS FIRST, date_and_time, reference_id, _row) AS occurrence FROM loss_keys), + offsets AS (SELECT l.*, COALESCE(p.units, 0) + CASE WHEN l.kind = 'lost' THEN COALESCE(f.units, 0) + ELSE 0 END AS offset_units FROM ordered_losses l LEFT JOIN paid_units p USING (kind, fnsku) + LEFT JOIN found f USING (fnsku)), + ledger_claims AS (SELECT 'r' || SUBSTR(SHA1(CASE WHEN occurrence = 1 THEN id_parts + ELSE id_parts || '|' || CAST(occurrence AS VARCHAR) END), 1, 12) AS id, + kind, lost_units - LEAST(lost_units, CASE WHEN prior_units = 0 THEN offset_units + ELSE GREATEST(0, offset_units - prior_units) END) AS units, + lost_units, event_day, event_day + {WAREHOUSE_CLAIM_DAYS} AS claim_by, + msku AS sku, asin, fnsku, title, fulfillment_center AS fc, reference_id AS reference, + '' AS order_id FROM offsets), + refunds AS (SELECT COALESCE(NULLIF(order_id, ''), amazon_order_id) AS order_id, sku, + MIN(COALESCE({_day_sql('posted_date')}, {_day_sql('posted_date_time')})) AS event_day, + SUM(COALESCE(NULLIF(ABS(TRY_CAST(quantity_purchased AS DOUBLE)), 0), 1)) AS units + FROM settlements WHERE transaction_type = 'Refund' AND amount_type = 'ItemPrice' + AND amount_description = 'Principal' GROUP BY ALL), + refund_claims AS (SELECT 'r' || SUBSTR(SHA1(CONCAT_WS('|', 'refund', r.order_id, r.sku)), 1, 12) AS id, + 'refund' AS kind, units, units AS lost_units, event_day, + event_day + {REFUND_CLAIM_DAYS} AS claim_by, r.sku, '' AS asin, '' AS fnsku, '' AS title, + '' AS fc, r.order_id AS reference, r.order_id FROM refunds r + WHERE r.order_id <> '' AND r.sku <> '' AND event_day <= CAST(? AS DATE) - {REFUND_CLAIM_OPENS_DAYS} + AND NOT EXISTS (SELECT 1 FROM returned t WHERE t.order_id = r.order_id AND t.sku = r.sku) + AND NOT EXISTS (SELECT 1 FROM unpaid p WHERE p.amazon_order_id = r.order_id AND p.sku = r.sku)), + claims AS (SELECT * FROM ledger_claims WHERE units > 0 UNION ALL SELECT * FROM refund_claims), + identified AS (SELECT c.*, CASE WHEN o.status = 'Filed' AND o.case_id <> '' + AND EXISTS (SELECT 1 FROM reimb p WHERE p.case_id = o.case_id) THEN 'Reimbursed' + WHEN o.status IN ('To file', 'Filed', 'Reimbursed', 'Denied') THEN o.status ELSE 'To file' END AS status, + COALESCE(o.case_id, '') AS case_id FROM claims c LEFT JOIN overrides o USING (id)) + """ + return sql, [json.dumps(states), _today()], problems, overrides + + +def _project_sql(): + return f"""SELECT c.*, unit.amount_per_unit AS unit_amount, unit.currency_unit AS unit_currency + FROM identified c JOIN page USING (id) LEFT JOIN LATERAL ( + SELECT amount_per_unit, currency_unit FROM reimb + WHERE TRY_CAST(amount_per_unit AS DOUBLE) IS NOT NULL + AND ((fnsku <> '' AND fnsku = c.fnsku) OR (sku <> '' AND sku = c.sku)) + ORDER BY CASE WHEN fnsku <> '' AND fnsku = c.fnsku THEN 0 ELSE 1 END, + {_day_sql('approval_date')} DESC NULLS LAST, _row DESC LIMIT 1 + ) unit ON TRUE ORDER BY c.claim_by NULLS LAST, c.id LIMIT ?""" + + +def _project_claim(row, overrides, owner): + """Project one selected claim with its newest per-unit value and authored state.""" + claim = {"id": row["id"], "kind": row["kind"], "units": row["units"], "lostUnits": row["lost_units"], + "eventDate": row["event_day"], "claimBy": row["claim_by"], "order": row["order_id"], + **{k: row[k] or None for k in ("sku", "asin", "fnsku", "title", "fc", "reference")}} + unit = (_num(row["unit_amount"]), row["unit_currency"] or None) if row["unit_amount"] is not None else None + value = round(claim["units"] * unit[0], 2) if unit else None + override = overrides.get(claim["id"]) if isinstance(overrides.get(claim["id"]), dict) else {} + label = claim["sku"] or claim["fnsku"] or claim["reference"] or "Unknown SKU" + due = claim["claimBy"].isoformat() if claim["claimBy"] else None + return {"id": claim["id"], "title": f"{_plural(claim['units'])} {KINDS[claim['kind']].lower()}: {label}", + "status": row["status"], "owner": override.get("owner") or owner, + "due": due, "reason": _reason(claim, value), + **{k: claim[k] for k in ("sku", "asin", "fnsku", "kind")}, + "units": int(claim["units"]) if float(claim["units"]).is_integer() else claim["units"], + "value": value, "currency": unit[1] if unit else None, "claimBy": due, + "expired": bool(claim["claimBy"] and claim["claimBy"] < _today()), "caseId": row["case_id"] or None, + "eventDate": claim["eventDate"].isoformat() if claim["eventDate"] else None, + "dismissed": override.get("dismissed") is True, "_claim": claim} + + +def _dict_rows(result): + names = [c[0] for c in result.description] + return [dict(zip(names, row)) for row in result.fetchall()] + + +def items(tenant_key, user=None, *, status=None, search="", cursor=None, limit=DEFAULT_LIMIT, st=None): + """Read one server-filtered claim page with full status counts and no source row cap.""" if status is not None and status not in STATUSES: raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") st = _rt(tenant_key, st) - every, problems, _src = _compute(tenant_key, st) counts = {s: 0 for s in STATUSES} - for item in every: - counts[item["status"]] += 1 - wanted = [i for i in every if status is None or i["status"] == status] - page, nxt = _page(wanted, cursor, limit) - return {"items": [_public(i) for i in page], "cursor": nxt, "counts": counts, "problems": problems} + with _reader(tenant_key, st) as cur: + if cur is None: + start, size = _bounds(cursor, limit, 0) + return {"items": [], "cursor": None, "counts": counts, "total": 0, "offset": start, "limit": size, + "paidTotal": 0, + "problems": ["The Amazon mirror is not on this server yet. It appears after the first sync."]} + sql, args, problems, overrides = _claims_sql(cur, st) + needle = _text(search).lower() + filtered = "CONTAINS(LOWER(CONCAT_WS(' ', sku, fnsku, asin, title, reference, case_id, kind)), ?)" + summary = cur.execute(sql + f""" SELECT + COUNT(*) FILTER (WHERE status = 'To file'), COUNT(*) FILTER (WHERE status = 'Filed'), + COUNT(*) FILTER (WHERE status = 'Reimbursed'), COUNT(*) FILTER (WHERE status = 'Denied'), + COUNT(*) FILTER (WHERE (? IS NULL OR status = ?) AND {filtered}), + (SELECT COUNT(*) FROM reimb) FROM identified""", + [*args, status, status, needle]).fetchone() + counts = dict(zip(STATUSES, summary[:4])) + total = summary[4] + start, size = _bounds(cursor, limit, total) + selected = _dict_rows(cur.execute(sql + f""", page AS ( + SELECT id FROM identified WHERE (? IS NULL OR status = ?) AND {filtered} + ORDER BY claim_by NULLS LAST, id LIMIT ? OFFSET ?) + """ + _project_sql(), + [*args, status, status, needle, size, start, size])) + owner = _default_owner(st, tenant_key) + page = [_public(_project_claim(row, overrides, owner)) for row in selected] + return {"items": page, "cursor": str(start + size) if start + size < total else None, + "counts": counts, "problems": problems, "total": total, "offset": start, "limit": size, + "paidTotal": summary[5]} def _case_text(item, claim): @@ -491,31 +651,67 @@ def _case_text(item, claim): f"Please reimburse {units} under the FBA inventory reimbursement policy.") -def item_detail(tenant_key, user, item_id, *, st=None): - """One item, its source ledger events (or refund and return rows), and case text ready to copy.""" - st = _rt(tenant_key, st) - every, problems, src = _compute(tenant_key, st) - item = next((i for i in every if i["id"] == item_id), None) - if item is None: +def _one_claim(cur, tenant_key, st, item_id): + if cur is None: raise KeyError(item_id) - claim = item["_claim"] - if item["kind"] == "refund": - events = list(claim["source"]) - returns = [r for r in src["returns"] - if _text(r.get("order_id")) == claim["order"] and _text(r.get("sku")) == claim["sku"]] - paid = [r for r in src["reimbursements"] - if _text(r.get("amazon_order_id")) == claim["order"] and _text(r.get("sku")) == claim["sku"]] - else: - reasons = (LOST_REASONS + FOUND_REASONS) if item["kind"] == "lost" else DAMAGED_REASONS - events = [r for r in src["ledger"] - if _text(r.get("fnsku")) == (item["fnsku"] or "") and _text(r.get("reason")) in reasons] - returns = [] - want = LOST_REIMBURSEMENT if item["kind"] == "lost" else DAMAGED_REIMBURSEMENT - paid = [r for r in src["reimbursements"] - if _text(r.get("fnsku")) == (item["fnsku"] or "") and _text(r.get("reason")) == want] + sql, args, problems, overrides = _claims_sql(cur, st) + selected = _dict_rows(cur.execute(sql + ", page AS (SELECT id FROM identified WHERE id = ? LIMIT 1) " + + _project_sql(), [*args, item_id, 1])) + if not selected: + raise KeyError(item_id) + return _project_claim(selected[0], overrides, _default_owner(st, tenant_key)), problems + + +def _evidence_page(cur, table, wanted, where, args, cursor, limit): + cols = _columns(cur, table) + if not cols: + start, size = _bounds(cursor, limit, 0) + return [], {"cursor": None, "offset": start, "limit": size, "total": 0} + # Normalize only predicate operands, while returning the unchanged source text. + source = _source_sql(cur, table, wanted) + total = cur.execute(f"WITH source AS ({source}) SELECT COUNT(*) FROM source WHERE {where}", args).fetchone()[0] + start, size = _bounds(cursor, limit, total) + selected = f"rowid IN (SELECT _row FROM ({source}) source WHERE {where} ORDER BY _row LIMIT ? OFFSET ?)" + page = _rows(cur, table, wanted, f"WHERE {selected} ORDER BY rowid", args=[*args, size, start], limit=size) + return page, {"cursor": str(start + size) if start + size < total else None, + "offset": start, "limit": size, "total": total} + + +def item_detail(tenant_key, user, item_id, *, events_cursor=None, returns_cursor=None, + reimbursements_cursor=None, history_cursor=None, limit=DEFAULT_LIMIT, st=None): + """Read one claim and one reachable page per evidence section.""" + st = _rt(tenant_key, st) + with _reader(tenant_key, st) as cur: + item, problems = _one_claim(cur, tenant_key, st, item_id) + claim = item["_claim"] + if item["kind"] == "refund": + event_table, event_cols = SETTLEMENTS_TABLE, SETTLEMENT_COLS + event_where = ("transaction_type = 'Refund' AND amount_type = 'ItemPrice' " + "AND amount_description = 'Principal' " + "AND COALESCE(NULLIF(order_id, ''), amazon_order_id) = ? AND sku = ?") + event_args = [claim["order"], item["sku"]] + return_where, return_args = "order_id = ? AND sku = ?", event_args + paid_where, paid_args = "amazon_order_id = ? AND sku = ?", event_args + else: + event_table, event_cols = LEDGER_TABLE, LEDGER_COLS + reasons = (LOST_REASONS + FOUND_REASONS) if item["kind"] == "lost" else DAMAGED_REASONS + event_where = f"event_type = 'Adjustments' AND fnsku = ? AND reason IN ({','.join('?' for _ in reasons)})" + event_args = [item["fnsku"] or "", *reasons] + return_where, return_args = "FALSE", [] + want = LOST_REIMBURSEMENT if item["kind"] == "lost" else DAMAGED_REIMBURSEMENT + paid_where, paid_args = "fnsku = ? AND reason = ?", [item["fnsku"] or "", want] + events, event_page = _evidence_page(cur, event_table, event_cols, event_where, event_args, events_cursor, limit) + returns, return_page = _evidence_page(cur, RETURNS_TABLE, RETURN_COLS, return_where, return_args, returns_cursor, limit) + paid, paid_page = _evidence_page(cur, REIMBURSEMENTS_TABLE, REIMBURSEMENT_COLS, paid_where, paid_args, + reimbursements_cursor, limit) override = (_doc(st, KEY_ITEMS).get("items") or {}).get(item_id) or {} + history = override.get("history") or [] + start, size = _bounds(history_cursor, limit, len(history)) return {"item": _public(item), "events": events, "returns": returns, "reimbursements": paid, - "history": list(override.get("history") or ()), "note": override.get("note"), + "history": history[start:start + size], "note": override.get("note"), + "pages": {"events": event_page, "returns": return_page, "reimbursements": paid_page, + "history": {"offset": start, "limit": size, "total": len(history), + "cursor": str(start + size) if start + size < len(history) else None}}, "caseText": _case_text(item, claim), "problems": problems} @@ -528,9 +724,8 @@ def set_status(tenant_key, user, item_id, status, *, note=None, case_id=None, st who = _editor(st, user) if status not in STATUSES: raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") - every, _problems, _src = _compute(tenant_key, st) - if not any(i["id"] == item_id for i in every): - raise KeyError(item_id) + with _reader(tenant_key, st) as cur: + _one_claim(cur, tenant_key, st, item_id) case = _text(case_id) if not case and note: found = _CASE_RE.search(str(note)) @@ -553,18 +748,36 @@ def set_status(tenant_key, user, item_id, status, *, note=None, case_id=None, st row.setdefault("history", []).append(event) return doc st.update(KEY_ITEMS, _apply) - return next(_public(i) for i in _compute(tenant_key, st)[0] if i["id"] == item_id) + with _reader(tenant_key, st) as cur: + return _public(_one_claim(cur, tenant_key, st, item_id)[0]) + +def _paid_sql(cur): + """The complete paid projection shared by the list and its SQL reconciliation.""" + return _source_sql(cur, REIMBURSEMENTS_TABLE, REIMBURSEMENT_COLS) -def paid(tenant_key, user=None, *, cursor=None, limit=DEFAULT_LIMIT, st=None): + +def paid(tenant_key, user=None, *, search="", cursor=None, limit=DEFAULT_LIMIT, st=None): """Every reimbursement row Amazon paid, newest approval first, one page. Rows are never deduped (F30).""" st = _rt(tenant_key, st) with _reader(tenant_key, st) as cur: - rows = _rows(cur, REIMBURSEMENTS_TABLE, REIMBURSEMENT_COLS) if cur is not None else [] - rows.sort(key=lambda r: (_date(r.get("approval_date")) or dt.date.min, _text(r.get("approval_date"))), - reverse=True) - page, nxt = _page(rows, cursor, limit) - return {"rows": page, "cursor": nxt, "total": len(rows)} + if not _columns(cur, REIMBURSEMENTS_TABLE): + start, size = _bounds(cursor, limit, 0) + return {"rows": [], "cursor": None, "total": 0, "offset": start, "limit": size} + source = _paid_sql(cur) + needle = _text(search).lower() + where = "CONTAINS(LOWER(CONCAT_WS(' ', reimbursement_id, case_id, amazon_order_id, sku, fnsku, asin, product_name, reason)), ?)" + total = cur.execute(f"WITH source AS ({source}) SELECT COUNT(*) FROM source WHERE {where}", [needle]).fetchone()[0] + start, size = _bounds(cursor, limit, total) + identities = (f"SELECT _row FROM ({source}) source WHERE {where} ORDER BY {_day_sql('approval_date')} " + "DESC NULLS LAST, approval_date DESC, _row LIMIT ? OFFSET ?") + cols = _columns(cur, REIMBURSEMENTS_TABLE) + approval = '"approval_date"' if "approval_date" in cols else "NULL" + page = _rows(cur, REIMBURSEMENTS_TABLE, REIMBURSEMENT_COLS, + f"WHERE rowid IN ({identities}) ORDER BY {_day_sql(approval)} DESC NULLS LAST, " + f"TRIM(CAST({approval} AS VARCHAR)) DESC, rowid", args=[needle, size, start], limit=size) + return {"rows": page, "cursor": str(start + size) if start + size < total else None, + "total": total, "offset": start, "limit": size} def proposals(tenant_key, *, st=None): @@ -585,8 +798,8 @@ def on_decision(tenant_key, item_id, decision, reason, *, st=None): if decision not in DECISIONS: raise ValueError("Decision must be approved or dismissed.") st = _rt(tenant_key, st) - if not any(i["id"] == item_id for i in _compute(tenant_key, st)[0]): - raise KeyError(item_id) + with _reader(tenant_key, st) as cur: + _one_claim(cur, tenant_key, st, item_id) at = _stamp() def _apply(doc): @@ -612,23 +825,22 @@ def validate(tenant_key, *, start=None, end=None, st=None): if (start and lo is None) or (end and hi is None): raise ValueError("Start and end must be dates such as 2026-10-01.") st = _rt(tenant_key, st) - app_total, count, cursor = 0.0, 0, None - while True: - page = paid(tenant_key, cursor=cursor, st=st) - for row in page["rows"]: - when = _date(row.get("approval_date")) - if when is None or (lo and when < lo) or (hi and when >= hi): - continue - app_total += _num(row.get("amount_total")) or 0.0 - count += 1 - cursor = page["cursor"] - if cursor is None: - break with _reader(tenant_key, st) as cur: cols = _columns(cur, REIMBURSEMENTS_TABLE) if not {"amount_total", "approval_date"} <= cols: - return {"ok": False, "app_total": round(app_total, 2), "independent_total": None, + return {"ok": False, "app_total": 0.0, "independent_total": None, "detail": "The reimbursements report is not in the mirror yet, so nothing can be tied."} + app_day = _day_sql('approval_date') + app_where, app_args = [f"{app_day} IS NOT NULL"], [] + if lo: + app_where.append(f"{app_day} >= ?") + app_args.append(lo) + if hi: + app_where.append(f"{app_day} < ?") + app_args.append(hi) + app_total, count = cur.execute( + f"WITH paid AS ({_paid_sql(cur)}) SELECT COALESCE(SUM(TRY_CAST(amount_total AS DOUBLE)), 0), " + f"COUNT(*) FROM paid WHERE {' AND '.join(app_where)}", app_args).fetchone() day = 'TRY_CAST(SUBSTR(CAST("approval_date" AS VARCHAR), 1, 10) AS DATE)' where, args = [f"{day} IS NOT NULL"], [] if lo: diff --git a/platform/modules/restock_planner.py b/platform/modules/restock_planner.py index 657a2379901f82fc5d1a75a8f2ed7cd8b25f5590..3830847464a4e66a66cccaceb11ca5feaeeb2586 100644 --- a/platform/modules/restock_planner.py +++ b/platform/modules/restock_planner.py @@ -20,6 +20,7 @@ import contextlib import datetime as dt import hashlib import json +import time from pathlib import Path APP_LABEL = "Restock planner" @@ -43,10 +44,12 @@ READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so the r LIMIT = ("Each restock planner document is limited to 128 MB by the store. About 60,000 SKUs fit in " "the items document, and history splits by month. Split the items document by band when " "it nears that size.") +MAX_PAGE_SIZE = 200 KEY_SETTINGS = "restock_planner_settings" KEY_ITEMS = "restock_planner_items" KEY_HISTORY = "restock_planner_history" +KEY_REFRESH = "restock_planner_refresh" HISTORY_PREFIX = "restock_planner_history_" PLANNING = "amzn_inventory_planning" @@ -84,6 +87,38 @@ def _doc(st, key): return value if isinstance(value, dict) else {} +def _sql_scope(st): + """Resolve the bound Postgres store and its document namespace without a default tenant.""" + from core import store_pg + backend = st._document_store() if callable(getattr(st, "_document_store", None)) else st + if not isinstance(backend, store_pg.PgStore): + raise RuntimeError("The restock screen requires a store with server paging.") + tenant = backend.tenant_slug + if getattr(st, "key", tenant) != tenant: + raise PermissionError("The restock store belongs to another tenant.") + return store_pg, tenant, getattr(st, "store_key", str) + + +def _select(st, sql, params): + """Read one bounded SQL result through the tenant store's checked pool.""" + from core import request_timing + pg, tenant, key = _sql_scope(st) + started = time.perf_counter() + with pg._pool().connection() as con: + row = con.execute(pg._q(sql, "store_kv", tenant), params(key)).fetchone() + value = row[0] if row else None + value = json.loads(value) if isinstance(value, str) else value + request_timing.record_store("restock_page", key(KEY_ITEMS), bytes_read=len(json.dumps(value).encode()), + milliseconds=(time.perf_counter() - started) * 1000) + return value + + +def _paths(st, name, paths): + """Read selected nested values without transferring their document.""" + pg, tenant, key = _sql_scope(st) + return pg._get_json_paths(key(name), paths, tenant_slug=tenant) + + def item_id_for(sku): """A stable item id from the SKU, so a refresh never duplicates an item.""" return "rp_" + hashlib.sha1(str(sku).encode("utf-8")).hexdigest()[:10] @@ -169,7 +204,7 @@ def _expr(cols, name, agg, kind="DOUBLE"): return f'{agg}(TRY_CAST("{name}" AS {kind}))' -def _planning(cur): +def _planning(cur, skus): """{sku: planning values} from the newest snapshot, summed per SKU; None when the table is absent.""" cols = _columns(cur, PLANNING) if "sku" not in cols: @@ -181,9 +216,9 @@ def _planning(cur): f'{_expr(cols, "recommended_ship_in_quantity", "SUM")}, ' f'{_expr(cols, "recommended_ship_in_date", "MIN", "VARCHAR")}, COUNT(*) ' f'FROM {PLANNING} WHERE {_newest(cols, PLANNING)} AND "sku" IS NOT NULL ' - f'AND TRIM(CAST("sku" AS VARCHAR)) <> \'\' GROUP BY 1') + f'AND TRIM(CAST("sku" AS VARCHAR)) IN (SELECT unnest(?)) GROUP BY 1') out = {} - for sku, asin, fnsku, name, avail, dos, units, inbound, qty, date, n in cur.execute(sql).fetchall(): + for sku, asin, fnsku, name, avail, dos, units, inbound, qty, date, n in cur.execute(sql, [skus]).fetchall(): # Several rows for one SKU: days of supply comes from the summed quantities, not one row. if n > 1: dos = None @@ -192,7 +227,7 @@ def _planning(cur): return out -def _restock(cur): +def _restock(cur, skus): """{sku: restock values} from the newest snapshot; None when the table is absent.""" cols = _columns(cur, RESTOCK) if "merchant_sku" not in cols: @@ -205,28 +240,28 @@ def _restock(cur): f'{_expr(cols, "recommended_replenishment_qty", "SUM")}, ' f'{_expr(cols, "recommended_ship_date", "MIN", "VARCHAR")}, COUNT(*) ' f'FROM {RESTOCK} WHERE {_newest(cols, RESTOCK)} AND "merchant_sku" IS NOT NULL ' - f'AND TRIM(CAST("merchant_sku" AS VARCHAR)) <> \'\' GROUP BY 1') + f'AND TRIM(CAST("merchant_sku" AS VARCHAR)) IN (SELECT unnest(?)) GROUP BY 1') out = {} - for sku, asin, fnsku, name, avail, dos, units, qty, date, n in cur.execute(sql).fetchall(): + for sku, asin, fnsku, name, avail, dos, units, qty, date, n in cur.execute(sql, [skus]).fetchall(): out[sku] = {"asin": asin, "fnsku": fnsku, "title": name, "available": avail, "days": None if n > 1 else dos, "units": units, "qty": qty, "date": date} return out -def _codes(cur): +def _codes(cur, codes): """{default_code: [product id, ...]} from the Odoo product mirror; None when it is absent.""" if not {"id", "default_code"} <= _columns(cur, "product_product"): return None out = {} for pid, code in cur.execute( "SELECT TRY_CAST(id AS BIGINT), TRIM(CAST(default_code AS VARCHAR)) FROM product_product " - "WHERE default_code IS NOT NULL").fetchall(): + "WHERE TRIM(CAST(default_code AS VARCHAR)) IN (SELECT unnest(?))", [codes]).fetchall(): if code: out.setdefault(code, []).append(pid) return out -def _odoo_stock(cur): +def _odoo_stock(cur, ids): """({product id: [{warehouse, onHand, fba}]}, fba_known), or (None, False) when quants are absent.""" quant, loc = _columns(cur, "stock_quant"), _columns(cur, "stock_location") if not {"product_id", "location_id", "quantity"} <= quant or not {"id", "usage"} <= loc: @@ -239,7 +274,8 @@ def _odoo_stock(cur): f"SELECT TRY_CAST(sq.product_id AS BIGINT), TRIM(split_part(CAST({name} AS VARCHAR), '/', 1)), " f"{fba}, SUM(TRY_CAST(sq.quantity AS DOUBLE)) FROM stock_quant sq " f"JOIN stock_location sl ON TRY_CAST(sl.id AS BIGINT) = TRY_CAST(sq.location_id AS BIGINT) " - f"WHERE sl.usage = 'internal' GROUP BY 1, 2, 3").fetchall() + f"WHERE sl.usage = 'internal' AND TRY_CAST(sq.product_id AS BIGINT) IN (SELECT unnest(?)) " + f"GROUP BY 1, 2, 3", [ids]).fetchall() out = {} for pid, warehouse, is_fba, qty in rows: out.setdefault(pid, []).append({"warehouse": warehouse or "Unnamed", "onHand": float(qty or 0), @@ -247,7 +283,7 @@ def _odoo_stock(cur): return out, fba_known -def _open_pos(cur): +def _open_pos(cur, ids): """{product id: [{order, vendor, openQty, expected}]}, or None when purchase orders are absent.""" po, pol = _columns(cur, "purchase_order"), _columns(cur, "purchase_order_line") if not {"id", "state"} <= po or not {"order_id", "product_id", "product_qty"} <= pol: @@ -261,7 +297,8 @@ def _open_pos(cur): f"TRY_CAST(pol.product_qty AS DOUBLE) - {received}, {planned} FROM purchase_order_line pol " f"JOIN purchase_order po ON TRY_CAST(po.id AS BIGINT) = TRY_CAST(pol.order_id AS BIGINT) " f"WHERE po.state IN ('purchase', 'done') " - f"AND TRY_CAST(pol.product_qty AS DOUBLE) - {received} > 0").fetchall() + f"AND TRY_CAST(pol.product_qty AS DOUBLE) - {received} > 0 " + f"AND TRY_CAST(pol.product_id AS BIGINT) IN (SELECT unnest(?))", [ids]).fetchall() out = {} for pid, name, partner, qty, expected in rows: out.setdefault(pid, []).append({"order": name, "vendor": partner, "openQty": float(qty), @@ -389,17 +426,53 @@ def _band(row, conf): return HEALTHY, f"{head}, inside the {_fmt(send)} to {_fmt(over)} day range." +def _source_pages(cur, size=200): + """Select SKU identities in SQL before reading their report or Odoo values.""" + sources = [] + for table, field in ((PLANNING, "sku"), (RESTOCK, "merchant_sku")): + cols = _columns(cur, table) + if field in cols: + sources.append(f'SELECT TRIM(CAST("{field}" AS VARCHAR)) AS sku FROM {table} ' + f'WHERE {_newest(cols, table)} AND "{field}" IS NOT NULL ' + f'AND TRIM(CAST("{field}" AS VARCHAR)) <> \'\'') + if not sources: + yield [] + return + after = "" + while True: + rows = cur.execute("SELECT DISTINCT sku FROM (" + " UNION ALL ".join(sources) + + ") s WHERE sku > ? ORDER BY sku LIMIT ?", [after, size]).fetchall() + if not rows: + if not after: + yield [] + return + yield [row[0] for row in rows] + after = rows[-1][0] + + def build_rows(cur, links, conf): + """Build all refresh rows in bounded source pages outside the HTTP request.""" + rows, notices = [], [] + for skus in _source_pages(cur): + page, notices = _build_page(cur, links, conf, skus) + if page is None: + return None, notices + rows.extend(page) + return rows, notices + + +def _build_page(cur, links, conf, skus): """(rows, notices): one row per SKU in either Amazon report, or (None, notices) when neither synced.""" - planning, restock = _planning(cur), _restock(cur) + planning, restock = _planning(cur, skus), _restock(cur, skus) notices = [f"{SOURCE_NAMES[t]} has not synced yet." for t, got in ((PLANNING, planning), (RESTOCK, restock)) if got is None] if planning is None and restock is None: return None, notices planning, restock = planning or {}, restock or {} - codes = _codes(cur) - stock, fba_known = _odoo_stock(cur) - pos = _open_pos(cur) + codes = _codes(cur, list(set(skus) | {links[s][0] for s in skus if s in links})) + ids = sorted({pid for values in (codes or {}).values() for pid in values if pid is not None}) + stock, fba_known = _odoo_stock(cur, ids) + pos = _open_pos(cur, ids) if codes is None or stock is None: notices.append("Odoo stock has not synced yet.") elif not fba_known: @@ -582,7 +655,7 @@ def history(tenant_key, sku, *, st=None): st = _rt(tenant_key, st) points = [] for month in _doc(st, KEY_HISTORY).get("months") or (): - for day, point in ((_doc(st, _month_key(month)).get("skus") or {}).get(sku) or {}).items(): + for day, point in (_paths(st, _month_key(month), [("skus", sku)])[0] or {}).items(): points.append({"date": day, **point}) return sorted(points, key=lambda p: p["date"]) @@ -601,7 +674,7 @@ def _cursor_offset(cursor): if not cursor: return 0 try: - return int(json.loads(base64.urlsafe_b64decode(str(cursor).encode()).decode())["o"]) + return max(0, int(json.loads(base64.urlsafe_b64decode(str(cursor).encode()).decode())["o"])) except Exception: # noqa: BLE001 raise ValueError("The page cursor is not valid. Start from the first page.") from None @@ -611,7 +684,7 @@ def _cursor(offset): def _public(item): - out = {k: v for k, v in item.items() if k != "history"} + out = {k: v for k, v in item.items() if k not in ("history", "warehouses", "openPurchaseOrders")} out.setdefault("title", item.get("sku")) return out @@ -621,49 +694,114 @@ def _matches(item, needle): return any(needle in str(item.get(k) or "").lower() for k in ("sku", "title", "asin")) +PAGE_SQL = """ +WITH doc AS MATERIALIZED (SELECT value FROM {tbl} WHERE key = %s), +searched AS MATERIALIZED ( + SELECT e.key AS iid, e.value->>'status' AS status, e.value->>'band' AS band, + e.value->>'sku' AS sku, COALESCE(e.value->>'inReport', 'true') AS report, + CAST(e.value->>'daysOfSupply' AS DOUBLE PRECISION) AS days FROM doc, + LATERAL jsonb_each(COALESCE(doc.value->'items', '{{}}'::jsonb)) e + WHERE %s = '' OR strpos(lower(COALESCE(e.value->>'sku', '')), %s) > 0 + OR strpos(lower(COALESCE(e.value->>'title', '')), %s) > 0 + OR strpos(lower(COALESCE(e.value->>'asin', '')), %s) > 0 +), filtered AS MATERIALIZED ( + SELECT * FROM searched WHERE (CAST(%s AS TEXT) IS NULL OR status = %s) + AND (CAST(%s AS TEXT) IS NULL OR band = %s) +), bounds AS ( + SELECT count(*) AS total, CASE WHEN count(*) = 0 THEN 0 + ELSE LEAST(%s, ((count(*) - 1) / %s)::bigint * %s) END AS start FROM filtered +), page AS ( + SELECT iid FROM filtered ORDER BY report DESC, + CASE band WHEN 'Send to FBA' THEN 0 WHEN 'Reorder from vendor' THEN 1 + WHEN 'Overstock' THEN 2 WHEN 'Healthy' THEN 3 ELSE 4 END, + days NULLS LAST, sku COLLATE "C" + LIMIT %s OFFSET (SELECT start FROM bounds) +), projected AS ( + SELECT (doc.value->'items')->page.iid AS v FROM page CROSS JOIN doc +) +SELECT jsonb_build_object( + 'meta', (SELECT value - 'items' FROM doc), + 'counts', (SELECT jsonb_object_agg(band, n) FROM + (SELECT band, count(*) AS n FROM searched WHERE band IS NOT NULL GROUP BY 1) c), + 'total', (SELECT total FROM bounds), 'offset', (SELECT start FROM bounds), + 'items', (SELECT jsonb_agg(v - 'history' - 'warehouses' - 'openPurchaseOrders') FROM projected)) +""" + + +def _page_size(value): + """Bound one page without limiting the number of reachable records.""" + try: + size = int(value) + except (TypeError, ValueError): + raise ValueError("The page size must be a whole number.") from None + if size < 1: + raise ValueError("The page size must be at least 1.") + return min(size, MAX_PAGE_SIZE) + + def items(tenant_key, user=None, *, status=None, cursor=None, limit=200, band=None, q=None, st=None): """C4: every item matching q, filtered by status and band, paged by an opaque cursor with no total cap.""" if status is not None and status not in STATUSES: raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".") if band is not None and band not in BANDS: raise ValueError("Band must be one of: " + ", ".join(BANDS) + ".") - try: - limit = int(limit) - except (TypeError, ValueError): - raise ValueError("The page size must be a whole number.") from None - if limit < 1: - raise ValueError("The page size must be at least 1.") + limit = _page_size(limit) st = _rt(tenant_key, st) - doc = _doc(st, KEY_ITEMS) - every = list((doc.get("items") or {}).values()) needle = str(q or "").strip().lower() - if needle: - every = [i for i in every if _matches(i, needle)] - counts = {b: 0 for b in BANDS} - for item in every: - if item.get("band") in counts: - counts[item["band"]] += 1 - rows = sorted((i for i in every if (status is None or i.get("status") == status) - and (band is None or i.get("band") == band)), key=_order) - start = _cursor_offset(cursor) - page = rows[start:start + limit] + start = _cursor_offset(cursor) // limit * limit + result = _select(st, PAGE_SQL, lambda key: (key(KEY_ITEMS), needle, needle, needle, needle, + status, status, band, band, start, limit, limit, limit)) + doc = result.get("meta") or {} + start, total = int(result["offset"]), int(result["total"]) + counts = {b: int((result.get("counts") or {}).get(b, 0)) for b in BANDS} ready = doc.get("state") == "ready" - return {"items": [_public(i) for i in page], - "cursor": _cursor(start + limit) if start + limit < len(rows) else None, + return {"items": result.get("items") or [], "total": total, "offset": start, "limit": limit, + "cursor": _cursor(start + limit) if start + limit < total else None, + "previousCursor": _cursor(max(0, start - limit)) if start else None, "counts": counts, "ready": ready, "message": None if ready else (doc.get("message") or NOT_READY), "notices": list(doc.get("notices") or ()), "refreshedAt": doc.get("refreshedAt"), "lastError": doc.get("last_error"), "statuses": list(STATUSES), "bands": list(BANDS)} -def item_detail(tenant_key, item_id, *, st=None): +def item_detail(tenant_key, item_id, *, offset=0, limit=50, st=None): """One SKU with its warehouses, open purchase orders, item history and daily points.""" st = _rt(tenant_key, st) - item = (_doc(st, KEY_ITEMS).get("items") or {}).get(item_id) + offset, limit = max(0, int(offset)), _page_size(limit) + base = "WITH item AS (SELECT value #> %s::text[] AS v FROM {tbl} WHERE key=%s) " + item = _select(st, base + "SELECT v - 'history' - 'warehouses' - 'openPurchaseOrders' FROM item", + lambda key: (["items", item_id], key(KEY_ITEMS))) if not item: raise KeyError(item_id) - return {**_public(item), "events": list(item.get("history") or ()), - "points": history(tenant_key, item.get("sku"), st=st)} + totals = {} + children = {} + for field, public in (("warehouses", "warehouses"), ("openPurchaseOrders", "openPurchaseOrders"), + ("history", "events")): + result = _select(st, base + "SELECT jsonb_build_object('total', " + "jsonb_array_length(COALESCE(v->%s, '[]'::jsonb)), 'rows', " + "(SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(" + "COALESCE(v->%s, '[]'::jsonb)) WITH ORDINALITY a(value, n) ORDER BY n " + "LIMIT %s OFFSET %s) p)) FROM item", + lambda key: (["items", item_id], key(KEY_ITEMS), field, field, limit, offset)) + totals[public] = int(result.get("total") or 0) + children[public] = result.get("rows") or [] + months = _doc(st, KEY_HISTORY).get("months") or [] + point_sql = """WITH points AS MATERIALIZED ( + SELECT e.key AS day, e.value AS point FROM {tbl} kv, + LATERAL jsonb_each(COALESCE(kv.value #> %s::text[], '{{}}'::jsonb)) e + WHERE kv.key = ANY(%s::text[]) + ), page AS (SELECT day, point FROM points ORDER BY day DESC LIMIT %s OFFSET %s) + SELECT jsonb_build_object('total', (SELECT count(*) FROM points), + 'rows', (SELECT jsonb_agg(point || jsonb_build_object('date', day)) FROM page))""" + points = _select(st, point_sql, lambda key: (["skus", item.get("sku")], + [key(_month_key(m)) for m in months], limit, offset)) + totals["points"] = int(points.get("total") or 0) + total = max(totals.values()) + start = min(offset // limit * limit, max(0, (total - 1) // limit * limit)) + if start != offset: + return item_detail(tenant_key, item_id, offset=start, limit=limit, st=st) + return {**item, **children, "points": points.get("rows") or [], "detailOffset": offset, + "detailLimit": limit, "detailTotal": total, "detailTotals": totals} def _may_change(st, user): @@ -694,11 +832,7 @@ def update_item(tenant_key, item_id, *, owner=None, status=None, due=None, note= raise PermissionError("Your account can view this App but not change it.") out = {} - def _apply(doc): - doc = doc if isinstance(doc, dict) else {} - item = (doc.get("items") or {}).get(item_id) - if not item: - return doc + def _apply(item): at = _stamp(_now()) event = {"at": at, "by": user} if owner is not None: @@ -712,13 +846,43 @@ def update_item(tenant_key, item_id, *, owner=None, status=None, due=None, note= item["updatedAt"], item["updatedBy"] = at, user item.setdefault("history", []).append(event) out["item"] = _public(item) - return doc - st.update(KEY_ITEMS, _apply) + return item + _update_one(st, item_id, _apply) if "item" not in out: raise KeyError(item_id) return out["item"] +def _update_one(st, item_id, apply): + """Lock the document but transfer and update only the selected item.""" + pg, tenant, key = _sql_scope(st) + name, path = key(KEY_ITEMS), ["items", item_id] + pg.check_write(tenant, "update restock item") + try: + with pg._pool().connection() as con, con.transaction(): + con.execute("SELECT set_config('lock_timeout', %s, true)", (f"{int(pg._lock_timeout_s() * 1000)}ms",)) + row = con.execute(pg._q("SELECT (value #> %s::text[]) - 'history' - 'warehouses' " + "- 'openPurchaseOrders', octet_length(value::text) " + "FROM {tbl} WHERE key=%s FOR UPDATE", + "store_kv", tenant), (path, name)).fetchone() + item = row[0] if row else None + if not item: + raise KeyError(item_id) + item = json.loads(item) if isinstance(item, str) else item + changed = apply(item) + events = json.dumps(changed.pop("history", []), ensure_ascii=False, allow_nan=False) + payload = json.dumps(changed, ensure_ascii=False, allow_nan=False) + saved = con.execute(pg._q("UPDATE {tbl} SET value=jsonb_set(value, %s::text[], " + "(value #> %s::text[]) || %s::jsonb || jsonb_build_object('history', " + "COALESCE(value #> %s::text[], '[]'::jsonb) || %s::jsonb)), " + "rev=rev+1, updated_at=now() WHERE key=%s RETURNING octet_length(value::text)", + "store_kv", tenant), (path, path, payload, path + ["history"], events, name)).fetchone() + if saved[0] > pg.DOCUMENT_CEILING_BYTES and saved[0] > row[1]: + raise pg.StoreDocumentTooLarge(name, saved[0]) + finally: + pg._bump_write_generation(name, tenant) + + def set_status(tenant_key, user, item_id, status, *, note=None, st=None): """C4: move one item's status with an optional note.""" return update_item(tenant_key, item_id, status=status, note=note, user=user, st=st) @@ -753,13 +917,11 @@ def on_decision(tenant_key, item_id, decision, reason, *, st=None): def validate(tenant_key, *, st=None, **period): """Rule 8: FBA available over the planner's rows ties to the planning table's own total.""" st = _rt(tenant_key, st) - conf = settings(tenant_key, st=st) try: with _reader(tenant_key, st) as cur: if cur is None or "available" not in _columns(cur, PLANNING): return {"ok": False, "app_total": None, "independent_total": None, "detail": f"{SOURCE_NAMES[PLANNING]} has not synced yet."} - rows, _notices = build_rows(cur, _links(st), conf) cols = _columns(cur, PLANNING) independent = cur.execute( f'SELECT SUM(TRY_CAST("available" AS DOUBLE)), COUNT(DISTINCT "sku") FROM {PLANNING} ' @@ -768,19 +930,24 @@ def validate(tenant_key, *, st=None, **period): return {"ok": False, "app_total": None, "independent_total": None, "detail": f"The Amazon mirror could not be read just now ({exc}), so nothing can be tied. " "Try again in a minute."} - planned = [r for r in rows or () if r["availableSource"] == "planning"] - app_total = float(sum(r["fbaAvailable"] or 0 for r in planned)) + held = _select(st, """SELECT jsonb_build_object('total', + COALESCE(sum(CAST(e.value->>'fbaAvailable' AS DOUBLE PRECISION)), 0), 'skus', count(*)) + FROM {tbl} kv, LATERAL jsonb_each(COALESCE(kv.value->'items', '{{}}'::jsonb)) e + WHERE kv.key=%s AND (e.value->>'availableSource')='planning' + AND COALESCE(e.value->>'inReport', 'true')='true'""", lambda key: (key(KEY_ITEMS),)) + app_total = float(held["total"]) + planned_count = int(held["skus"]) independent_total = float(independent[0] or 0) - ok = abs(app_total - independent_total) < 1e-6 and len(planned) == int(independent[1] or 0) + ok = abs(app_total - independent_total) < 1e-6 and planned_count == int(independent[1] or 0) return {"ok": ok, "app_total": app_total, "independent_total": independent_total, - "detail": (f"{len(planned)} planner SKUs against {int(independent[1] or 0)} planning SKUs " + "detail": (f"{planned_count} planner SKUs against {int(independent[1] or 0)} planning SKUs " f"in the newest snapshot.")} def app_keys(st): """Every store document this App owns.""" months = _doc(st, KEY_HISTORY).get("months") or () - return [KEY_SETTINGS, KEY_ITEMS, KEY_HISTORY, *(_month_key(m) for m in months)] + return [KEY_SETTINGS, KEY_ITEMS, KEY_HISTORY, KEY_REFRESH, *(_month_key(m) for m in months)] def remove_app_data(tenant_key, *, st=None): diff --git a/platform/modules/returns_repeat.py b/platform/modules/returns_repeat.py index 4f7966dc33c14ed1ff154d099d0bab8e7b0befe2..a53dcc836ea91a74e0f008fbc7dd7f0987f1c7cd 100644 --- a/platform/modules/returns_repeat.py +++ b/platform/modules/returns_repeat.py @@ -18,7 +18,9 @@ document nears the ceiling: split it by year. """ import contextlib import datetime as dt +import json import logging +import time from pathlib import Path LOG = logging.getLogger(__name__) @@ -53,6 +55,7 @@ REASON_LABELS = {"DAMAGED_BY_FC": "Damaged at the fulfillment center", "UNDELIVERABLE_UNKNOWN": "Undeliverable", "SWITCHEROO": "Different item returned"} TREND_DAYS = 7 +MAX_PAGE_SIZE = 100 def reason_label(code): @@ -546,109 +549,366 @@ def _previous(data, start, end, conf): "repeatShare": _share(_repeat_by_asin(data, p0, p1))} -def summary(tenant_key, *, start=None, end=None, st=None, now=None): - """The period's return rate and repeat purchase share, with the empty state and segment counts.""" +def _page_size(value): + """Bound transport without capping the source.""" + try: + size = int(value) + except (TypeError, ValueError, OverflowError): + raise ValueError("The page size must be a whole number.") from None + if size < 1: + raise ValueError("The page size must be at least 1.") + return min(size, MAX_PAGE_SIZE) + + +def _offset(cursor): + try: + offset = int(cursor) if cursor not in (None, "") else 0 + except (TypeError, ValueError, OverflowError): + raise ValueError("The cursor is not valid. Start again from the first page.") from None + if offset < 0: + raise ValueError("The cursor is not valid. Start again from the first page.") + return offset + + +def _navigation(offset, limit, total): + return {"offset": offset, "limit": limit, "total": total, + "cursor": str(offset + limit) if offset + limit < total else None, + "previousCursor": str(max(0, offset - limit)) if offset else None} + + +def _sql_scope(st): + """Resolve the tenant-bound Postgres document namespace.""" + from core import store_pg + backend = st._document_store() if callable(getattr(st, "_document_store", None)) else st + if not isinstance(backend, store_pg.PgStore): + raise RuntimeError("The returns screen requires a store with server paging.") + tenant = backend.tenant_slug + if getattr(st, "key", tenant) != tenant: + raise PermissionError("The returns store belongs to another tenant.") + return store_pg, tenant, getattr(st, "store_key", str) + + +def _store_select(st, sql, params): + """Transfer one aggregate or bounded page from the tenant document store.""" + from core import request_timing + pg, tenant, key = _sql_scope(st) + started = time.perf_counter() + with pg._pool().connection() as con: + row = con.execute(pg._q(sql, "store_kv", tenant), params(key)).fetchone() + value = row[0] if row else None + value = json.loads(value) if isinstance(value, str) else value + request_timing.record_store("returns_repeat_page", key(KEY_ITEMS), bytes_read=len(json.dumps(value).encode()), + milliseconds=(time.perf_counter() - started) * 1000) + return value + + +ITEM_PAGE_SQL = """ +WITH doc AS (SELECT value FROM {tbl} WHERE key=%s), picked AS MATERIALIZED ( + SELECT e.value AS v FROM doc, LATERAL jsonb_each(COALESCE(value->'items', '{{}}'::jsonb)) e + WHERE (%s::text IS NULL OR (e.value->>'status')=%s) AND (%s::text IS NULL OR (e.value->>'sku')=%s) +), page AS (SELECT v FROM picked ORDER BY CAST(substring(v->>'id' FROM 2) AS bigint) DESC + LIMIT %s OFFSET %s) +SELECT jsonb_build_object('total', (SELECT count(*) FROM picked), + 'items', (SELECT jsonb_agg(v - 'history' ORDER BY CAST(substring(v->>'id' FROM 2) AS bigint) DESC) FROM page)) +""" + + +def _item_page(st, *, status=None, sku=None, offset=0, limit=100): + out = _store_select(st, ITEM_PAGE_SQL, lambda key: (key(KEY_ITEMS), status, status, sku, sku, limit, offset)) + return {"items": [_public(item) for item in out.get("items") or []], + **_navigation(offset, limit, int(out["total"]))} + + +HISTORY_SQL = """ +WITH points AS MATERIALIZED ( + SELECT e.key AS day, e.value->%s::text AS point FROM {tbl} kv, + LATERAL jsonb_each(COALESCE(kv.value->'snapshots', '{{}}'::jsonb)) e + WHERE kv.key=%s AND jsonb_typeof(e.value->%s::text)='array' AND jsonb_array_length(e.value->%s::text)>=2 +), page AS (SELECT day, point FROM points ORDER BY day LIMIT %s OFFSET %s) +SELECT jsonb_build_object('total', (SELECT count(*) FROM points), 'rows', + (SELECT jsonb_agg(jsonb_build_object('date', day, 'rate30', point->0, 'rate90', point->1) ORDER BY day) FROM page)) +""" +EVENTS_SQL = """ +WITH latest AS ( + SELECT e.value AS v FROM {tbl} kv, LATERAL jsonb_each(COALESCE(kv.value->'items', '{{}}'::jsonb)) e + WHERE kv.key=%s AND (e.value->>'sku')=%s + ORDER BY CAST(substring(e.value->>'id' FROM 2) AS bigint) DESC LIMIT 1 +), points AS MATERIALIZED ( + SELECT a.value AS point, a.n FROM latest, + LATERAL jsonb_array_elements(COALESCE(v->'history', '[]'::jsonb)) WITH ORDINALITY a(value,n) +), page AS (SELECT point,n FROM points ORDER BY n LIMIT %s OFFSET %s) +SELECT jsonb_build_object('total', (SELECT count(*) FROM points), + 'rows', (SELECT jsonb_agg(point ORDER BY n) FROM page)) +""" +ITEM_LOOKUP_SQL = """ +WITH ranked AS ( + SELECT e.value AS v, row_number() OVER (PARTITION BY e.value->>'sku' + ORDER BY CAST(substring(e.value->>'id' FROM 2) AS bigint) DESC) AS n + FROM {tbl} kv, LATERAL jsonb_each(COALESCE(kv.value->'items', '{{}}'::jsonb)) e + WHERE kv.key=%s AND (e.value->>'sku')=ANY(%s::text[]) +) +SELECT COALESCE(jsonb_object_agg(v->>'sku', v - 'history'), '{{}}'::jsonb) FROM ranked WHERE n=1 +""" + + +def _sql_day(expr): + return (f"COALESCE(TRY_CAST(SUBSTR({expr}, 1, 10) AS DATE), " + f"CAST(TRY_STRPTIME(split_part({expr}, ' ', 1), ['%m/%d/%Y', '%Y/%m/%d']) AS DATE), " + f"CAST(TRY_STRPTIME({expr}, ['%m/%d/%Y', '%Y/%m/%d', '%d-%b-%Y', '%b %d, %Y']) AS DATE))") + + +def _relations(cur, start, end, conf): + """SQL relations retain source rows and grouped metrics inside the mirror.""" + def source(table, fields): + cols = _columns(cur, table) + values = [] + for alias, names, kind in fields: + name = _first(cols, *names) + expr = _num(cols, name) if kind == "num" else _text(cols, name) + if kind == "day": + expr = _sql_day(expr) + elif kind == "text": + expr = f"NULLIF(trim({expr}), '')" + if alias == "units": + expr = f"COALESCE({expr}, 0)" if name else "CAST(1 AS DOUBLE)" + values.append(f"{expr} AS {alias}") + invalid_identity = table in (RETURNS_TABLE, SHIPPED_TABLE) and not cols & {"sku", "asin"} + return "SELECT " + ", ".join(values) + (f" FROM {table}" if cols else "") + ( + " WHERE false" if not cols or invalid_identity else "") + + identity = [("sku", ("sku",), "text"), ("asin", ("asin",), "text")] + r = source(RETURNS_TABLE, identity + [("day", ("return_date",), "day"), + ("reason", ("reason",), "text"), ("units", ("quantity",), "num")]) + sh = source(SHIPPED_TABLE, identity + [("day", ("shipment_date",), "day"), ("units", ("quantity",), "num")]) + p = source(REPEAT_TABLE, [("asin", ("asin",), "text"), ("start", ("start_date", "_period_start"), "day"), + ("finish", ("end_date", "_period_end"), "day"), ("orders", ("orders",), "num"), + ("customers", ("unique_customers",), "num"), ("pct", ("repeat_customers_pct_total",), "num"), + ("revenue", ("repeat_purchase_revenue_amount",), "num"), + ("currency", ("repeat_purchase_revenue_currency_code",), "text"), + ("rev_pct", ("repeat_purchase_revenue_pct_total",), "num")]) + b = source(BASKET_TABLE, [("asin", ("asin",), "text"), ("partner", ("purchased_with_asin",), "text"), + ("rank", ("purchased_with_rank",), "num"), ("pct", ("combination_pct",), "num"), + ("start", ("start_date", "_period_start"), "day")]) + s, e = start.isoformat(), end.isoformat() + w30, w90 = (end - dt.timedelta(days=29)).isoformat(), (end - dt.timedelta(days=89)).isoformat() + ratio, floor = float(conf["risingRatio"]), float(conf["risingMinPct"]) + return f"""WITH r0 AS ({r}), s0 AS ({sh}), p_raw AS ({p}), b_raw AS ({b}), + p0 AS (SELECT * FROM p_raw WHERE asin IS NOT NULL), + b0 AS (SELECT * FROM b_raw WHERE asin IS NOT NULL AND partner IS NOT NULL), + events AS ( + SELECT COALESCE(sku, asin, '{NO_SKU}') AS key, sku, asin, day, units AS ret, 0.0 AS ship FROM r0 + UNION ALL SELECT COALESCE(sku, asin, '{NO_SKU}'), sku, asin, day, 0.0, units FROM s0 + ), links AS (SELECT DISTINCT key, asin FROM events WHERE asin IS NOT NULL), + sums AS (SELECT key, MIN(sku) AS sku, MIN(asin) AS asin, + COALESCE(SUM(ret) FILTER (WHERE day BETWEEN DATE '{s}' AND DATE '{e}'), 0) AS returned, + COALESCE(SUM(ship) FILTER (WHERE day BETWEEN DATE '{s}' AND DATE '{e}'), 0) AS shipped, + COALESCE(SUM(ret) FILTER (WHERE day BETWEEN DATE '{w30}' AND DATE '{e}'), 0) AS r30, + COALESCE(SUM(ship) FILTER (WHERE day BETWEEN DATE '{w30}' AND DATE '{e}'), 0) AS s30, + COALESCE(SUM(ret) FILTER (WHERE day BETWEEN DATE '{w90}' AND DATE '{e}'), 0) AS r90, + COALESCE(SUM(ship) FILTER (WHERE day BETWEEN DATE '{w90}' AND DATE '{e}'), 0) AS s90 + FROM events GROUP BY key), + rates AS (SELECT *, round(100.0*returned/NULLIF(shipped,0),4) AS return_rate, + round(100.0*r30/NULLIF(s30,0),4) AS rate30, round(100.0*r90/NULLIF(s90,0),4) AS rate90 FROM sums), + model AS (SELECT *, COALESCE(rate30>={ratio}*rate90 AND rate30>={floor},false) AS rising FROM rates), + scales AS (SELECT CASE WHEN MAX(abs(pct))<=1 THEN 100.0 ELSE 1.0 END AS pct_scale, + CASE WHEN MAX(abs(rev_pct))<=1 THEN 100.0 ELSE 1.0 END AS rev_scale FROM p0), + used_repeat AS (SELECT p0.* FROM p0 WHERE asin IS NOT NULL + AND start<=DATE '{e}' AND COALESCE(finish,start)>=DATE '{s}'), + rep_sums AS (SELECT asin, SUM(COALESCE(orders,0)) AS orders, + SUM(CASE WHEN pct IS NOT NULL THEN COALESCE(customers,0) ELSE 0 END) AS customers, + SUM(COALESCE(customers*pct*pct_scale,0)) AS weighted, AVG(pct*pct_scale) AS plain, + SUM(revenue) AS revenue, AVG(rev_pct*rev_scale) AS revenue_share, + first(currency ORDER BY start, finish, currency) FILTER (WHERE currency IS NOT NULL) AS currency + FROM used_repeat CROSS JOIN scales GROUP BY asin), + repeats AS (SELECT *, round(CASE WHEN customers<>0 THEN weighted/customers ELSE plain END,4) AS share + FROM rep_sums), + reason_totals AS (SELECT COALESCE(sku,asin,'{NO_SKU}') AS key, COALESCE(reason,'No reason given') AS reason, + SUM(units) AS units, count(*) AS n FROM r0 WHERE day BETWEEN DATE '{s}' AND DATE '{e}' GROUP BY 1,2), + reason_rank AS (SELECT *, row_number() OVER (PARTITION BY key ORDER BY units DESC,n DESC,reason) AS pos + FROM reason_totals), + sku_facts AS (SELECT model.*, reason_rank.reason AS top_reason, repeats.share AS repeat_share, + round(repeats.revenue,2) AS repeat_revenue FROM model + LEFT JOIN reason_rank ON model.key=reason_rank.key AND reason_rank.pos=1 + LEFT JOIN repeats ON model.asin=repeats.asin) + """ + + +@contextlib.contextmanager +def _screen_reader(tenant_key, st): + with _reader(tenant_key, st) as cur: + if cur is not None: + yield cur, True + else: + import duckdb + con = duckdb.connect() + try: + yield con, False + finally: + con.close() + + +def _mirror_one(cur, sql, params=()): + result = cur.execute(sql, params) + names = [field[0] for field in result.description] + row = result.fetchone() + return dict(zip(names, row)) if row else None + + +def _mirror_page(cur, sql, params, offset, limit): + result = cur.execute(sql + " LIMIT ? OFFSET ?", [*params, limit, offset]) + names = [field[0] for field in result.description] + return [dict(zip(names, row)) for row in result.fetchmany(limit)] + + +def _collection(cur, base, query, params, offset, limit): + total = int(_mirror_one(cur, base + "SELECT count(*) AS n FROM (" + query + ") counted", params)["n"]) + return _mirror_page(cur, base + query, params, offset, limit), total + + +def _sku_row(row, item=None): + reason = row["top_reason"] + return {"key": row["key"], "sku": row["sku"], "asin": row["asin"], + "returnedUnits": row["returned"], "shippedUnits": row["shipped"], "returnRate": row["return_rate"], + "rate30": row["rate30"], "rate90": row["rate90"], "rising": row["rising"], + "topReason": reason, "topReasonLabel": reason_label(reason) if reason else None, + "repeatShare": row["repeat_share"], "repeatRevenue": row["repeat_revenue"], + "itemId": (item or {}).get("id"), "itemStatus": (item or {}).get("status")} + + +def _aggregate(cur, base): + metrics = _mirror_one(cur, base + "SELECT COALESCE(SUM(returned),0) AS returned, " + "COALESCE(SUM(shipped),0) AS shipped, count(*) AS highest, count(*) FILTER (WHERE rising) AS rising FROM model") + rep = _mirror_one(cur, base + "SELECT count(*) AS n, round(CASE WHEN SUM(customers)<>0 " + "THEN SUM(weighted)/SUM(customers) ELSE AVG(share) END,4) AS share, round(SUM(revenue),2) AS revenue FROM repeats") + return {"returnedUnits": metrics["returned"], "shippedUnits": metrics["shipped"], + "returnRate": _rate(metrics["returned"], metrics["shipped"]), "repeatShare": rep["share"], + "repeatRevenue": rep["revenue"], "counts": {"highest": metrics["highest"], + "rising": metrics["rising"], "repeat": rep["n"]}} + + +def summary(tenant_key, *, start=None, end=None, cursor=None, limit=24, st=None, now=None): + """Return complete SQL totals and one bounded page of weekly buckets and repeat months.""" st = _rt(tenant_key, st) s, e = _period(start, end, now) + offset, limit = _offset(cursor), _page_size(limit) conf = settings(tenant_key, st=st) - data = _load(tenant_key, st) - model = _model(data, s, e, conf) - repeat = _repeat_by_asin(data, s, e) - returned = sum(x["returned"] for x in model.values()) - shipped = sum(x["shipped"] for x in model.values()) - share = _share(repeat) - revenue = [f["repeatRevenue"] for f in repeat.values() if f["repeatRevenue"] is not None] - months = sorted({m for f in repeat.values() for m in f["months"]}) - open_items = sum(1 for i in (_doc(st, KEY_ITEMS).get("items") or {}).values() if i.get("status") == "Open") - return {"period": {"start": s.isoformat(), "end": e.isoformat()}, - "returnedUnits": returned, "shippedUnits": shipped, "returnRate": _rate(returned, shipped), - "repeatShare": share, "repeatRevenue": round(sum(revenue), 2) if revenue else None, - "repeatMonths": months, "empty": returned == 0, - "emptyMessage": EMPTY if returned == 0 else None, "mirror": data["mirror"], - "openItems": open_items, - "counts": {"highest": len(model), "rising": sum(1 for x in model.values() if x["rising"]), - "repeat": len(repeat)}, - "segments": dict(SEGMENTS), - "previous": _previous(data, s, e, conf), "trend": _trend(data, s, e)} - - -def skus(tenant_key, *, segment="highest", start=None, end=None, cursor=None, limit=200, q=None, st=None, now=None): - """One segment's rows, paged by cursor. `highest` holds every SKU; no SKU is dropped (rule 1). - - `q` keeps the rows whose SKU or ASIN contains it, case blind, before counting and paging.""" + with _screen_reader(tenant_key, st) as (cur, present): + base = _relations(cur, s, e, conf) + out = _aggregate(cur, base) + p1 = s - dt.timedelta(days=1) + p0 = p1 - (e - s) + previous = _aggregate(cur, _relations(cur, p0, p1, conf)) + months, month_total = _collection(cur, base, "SELECT DISTINCT start FROM used_repeat ORDER BY start", [], offset, limit) + trend_total = (e - s).days // TREND_DAYS + 1 + buckets = ("SELECT CAST(? AS DATE) + CAST(i*7 AS INTEGER) AS start, " + "LEAST(CAST(? AS DATE), CAST(? AS DATE) + CAST(i*7+6 AS INTEGER)) AS finish " + "FROM range(?) t(i) ORDER BY i LIMIT ? OFFSET ?") + trend = _mirror_page(cur, base + "SELECT start, finish AS end, " + "round(100.0*(SELECT COALESCE(SUM(units),0) FROM r0 WHERE day BETWEEN buckets.start AND buckets.finish)/" + "NULLIF((SELECT SUM(units) FROM s0 WHERE day BETWEEN buckets.start AND buckets.finish),0),4) AS rate " + "FROM (" + buckets + ") buckets ORDER BY start", [s, e, s, trend_total, limit, offset], 0, limit) + out.update({"repeatMonths": [m["start"].isoformat() for m in months], + "trend": [{"start": p["start"].isoformat(), "end": p["end"].isoformat(), + "returnRate": p["rate"]} for p in trend]}) + return {**out, **_navigation(offset, limit, max(trend_total, month_total)), + "collectionTotals": {"trend": trend_total, "repeatMonths": month_total}, + "period": {"start": s.isoformat(), "end": e.isoformat()}, "mirror": present, + "empty": out["returnedUnits"] == 0, "emptyMessage": EMPTY if out["returnedUnits"] == 0 else None, + "openItems": _item_page(st, status="Open", limit=1)["total"], "segments": dict(SEGMENTS), + "previous": {"start": p0.isoformat(), "end": p1.isoformat(), + "returnRate": previous["returnRate"], "repeatShare": previous["repeatShare"]}} + + +def skus(tenant_key, *, segment="highest", start=None, end=None, cursor=None, limit=100, q=None, st=None, now=None): + """Search and select one page in SQL without loading source rows or child arrays.""" if segment not in SEGMENTS: raise ValueError("Segment must be one of: " + ", ".join(SEGMENTS) + ".") st = _rt(tenant_key, st) s, e = _period(start, end, now) - conf = settings(tenant_key, st=st) - data = _load(tenant_key, st) - model = _model(data, s, e, conf) - repeat = _repeat_by_asin(data, s, e) - items = _items_by_sku(st) - if segment == "repeat": - by_asin = {} - for x in model.values(): - for asin in x["asins"]: - by_asin.setdefault(asin, []).append(x["key"]) - rows = [] - for asin, f in repeat.items(): - rows.append({"key": asin, "asin": asin, "skus": sorted(by_asin.get(asin, [])), - "repeatShare": f["repeatShare"], "repeatRevenue": f["repeatRevenue"], - "currency": f["currency"], "orders": f["orders"], - "uniqueCustomers": f["uniqueCustomers"], - "boughtWith": [b["asin"] for b in _bought_with(data, asin)]}) - rows.sort(key=lambda r: (r["repeatShare"] is None, -(r["repeatShare"] or 0), r["asin"])) - else: - picked = [x for x in model.values() if segment == "highest" or x["rising"]] - if segment == "rising": - picked.sort(key=lambda x: (-(x["rate30"] or 0), x["key"])) + offset, limit = _offset(cursor), _page_size(limit) + needle = " ".join(str(q or "").split()).lower() + with _screen_reader(tenant_key, st) as (cur, _present): + base = _relations(cur, s, e, settings(tenant_key, st=st)) + if segment == "repeat": + query = ("SELECT asin AS key, asin, share AS \"repeatShare\", round(revenue,2) AS \"repeatRevenue\", " + "currency, orders, NULLIF(customers,0) AS \"uniqueCustomers\", " + "(SELECT count(*) FROM links WHERE links.asin=repeats.asin) AS \"skuCount\" FROM repeats " + "WHERE (?='' OR contains(lower(asin),?) OR EXISTS " + "(SELECT 1 FROM links WHERE links.asin=repeats.asin AND contains(lower(key),?))) " + "ORDER BY share DESC NULLS LAST, asin") else: - picked.sort(key=lambda x: (x["returnRate"] is None, -(x["returnRate"] or 0), -x["returned"], x["key"])) - rows = [_row(x, repeat, items.get(x["key"])) for x in picked] - needle = " ".join(str(q or "").split()).casefold() - if needle: - rows = [r for r in rows if any(needle in str(v or "").casefold() for v in ( - r.get("key"), r.get("sku"), r.get("asin"), *(r.get("asins") or ()), *(r.get("skus") or ())))] - page, nxt = _page(rows, cursor, limit) + query = ("SELECT * FROM sku_facts WHERE (?='' OR contains(lower(key),?) OR EXISTS " + "(SELECT 1 FROM links WHERE links.key=sku_facts.key AND contains(lower(asin),?))) " + + ("AND rising " if segment == "rising" else "") + + ("ORDER BY rate30 DESC, key" if segment == "rising" + else "ORDER BY return_rate DESC NULLS LAST, returned DESC, key")) + rows, total = _collection(cur, base, query, [needle, needle, needle], offset, limit) + if segment != "repeat": + keys = [row["key"] for row in rows] + owned = _store_select(st, ITEM_LOOKUP_SQL, lambda key: (key(KEY_ITEMS), keys)) if keys else {} + rows = [_sku_row(row, owned.get(row["key"])) for row in rows] return {"segment": segment, "label": SEGMENTS[segment], "period": {"start": s.isoformat(), "end": e.isoformat()}, - "q": needle or None, "total": len(rows), "rows": page, "cursor": nxt} + "q": needle or None, "rows": rows, **_navigation(offset, limit, total)} -def detail(tenant_key, key, *, start=None, end=None, st=None, now=None): - """One SKU or ASIN: ranked reasons with counts, repeat share and revenue, frequently-bought-with ASINs.""" +def detail(tenant_key, key, *, start=None, end=None, cursor=None, limit=25, st=None, now=None): + """Read scalar SKU facts and bounded pages of every nested collection.""" want = _clean(key) if not want: raise ValueError("Name a SKU or an ASIN.") st = _rt(tenant_key, st) s, e = _period(start, end, now) - conf = settings(tenant_key, st=st) - data = _load(tenant_key, st) - model = _model(data, s, e, conf) - repeat = _repeat_by_asin(data, s, e) - found = model.get(want) or next((x for x in model.values() if want in x["asins"]), None) - asin = want if found is None or want in (found["asins"] or ()) else found["asin"] - if found is None and want not in repeat and not any(b["asin"] == want for b in data["basket"]): - raise KeyError(want) - history = _doc(st, KEY_HISTORY).get("snapshots") or {} - sku_key = found["key"] if found else None - series = [{"date": d, "rate30": v[0], "rate90": v[1]} for d, snap in sorted(history.items()) - for k, v in snap.items() if k == sku_key and isinstance(v, list) and len(v) >= 2] - rep = repeat.get(asin) if asin else None - item = _items_by_sku(st).get(sku_key) if sku_key else None - return {"key": want, "sku": found["sku"] if found else None, "asin": asin, + offset, limit = _offset(cursor), _page_size(limit) + with _screen_reader(tenant_key, st) as (cur, _present): + base = _relations(cur, s, e, settings(tenant_key, st=st)) + found = _mirror_one(cur, base + "SELECT * FROM model WHERE key=? OR EXISTS " + "(SELECT 1 FROM links WHERE links.key=model.key AND asin=?) ORDER BY (key=?) DESC, key LIMIT 1", + [want, want, want]) + sku = found["key"] if found else None + direct = _mirror_one(cur, base + "SELECT 1 AS n FROM links WHERE key=? AND asin=? LIMIT 1", [sku, want]) + asin = want if not found or direct else found["asin"] + rep = _mirror_one(cur, base + 'SELECT asin, share AS "repeatShare", round(revenue,2) AS "repeatRevenue", ' + 'round(revenue_share,4) AS "repeatRevenueShare", currency, orders, NULLIF(customers,0) AS "uniqueCustomers" ' + "FROM repeats WHERE asin=?", [asin]) + basket = _mirror_one(cur, base + "SELECT 1 AS n FROM b0 WHERE asin=? LIMIT 1", [want]) + if not found and not rep and not basket: + raise KeyError(want) + out, totals = {}, {} + queries = { + "reasons": ("SELECT COALESCE(reason,'No reason given') AS reason, SUM(units) AS units, count(*) AS count " + "FROM r0 WHERE COALESCE(sku,asin,?)=? AND day BETWEEN ? AND ? GROUP BY reason " + "ORDER BY units DESC, count DESC, reason", [NO_SKU, sku, s, e]), + "skus": ("SELECT key FROM links WHERE asin=? ORDER BY key", [asin]), + "asins": ("SELECT asin FROM links WHERE key=? ORDER BY asin", [sku]), + "repeatMonths": ("SELECT DISTINCT start FROM used_repeat WHERE asin=? ORDER BY start", [asin]), + "boughtWith": ("SELECT partner AS asin, rank, round(pct*(SELECT CASE WHEN MAX(abs(pct))<=1 " + "THEN 100.0 ELSE 1.0 END FROM b0),4) AS \"combinationPct\" FROM b0 WHERE asin=? " + "AND partner IS NOT NULL AND start IS NOT DISTINCT FROM (SELECT MAX(start) FROM b0 WHERE asin=?) " + "ORDER BY rank NULLS LAST, partner", [asin, asin]), + } + for name, (query, params) in queries.items(): + out[name], totals[name] = _collection(cur, base, query, params, offset, limit) + out["reasons"] = [{**r, "label": reason_label(r["reason"])} for r in out["reasons"]] + out["skus"] = [r["key"] for r in out["skus"]] + out["asins"] = [r["asin"] for r in out["asins"]] + out["repeatMonths"] = [r["start"].isoformat() for r in out["repeatMonths"]] + for name, sql, args in ( + ("history", HISTORY_SQL, lambda key: (sku, key(KEY_HISTORY), sku, sku, limit, offset)), + ("itemHistory", EVENTS_SQL, lambda key: (key(KEY_ITEMS), sku, limit, offset))): + page = _store_select(st, sql, args) + out[name], totals[name] = page.get("rows") or [], int(page["total"]) + item = _item_page(st, sku=sku, limit=1)["items"] if sku else [] + return {**out, **_navigation(offset, limit, max(totals.values())), "collectionTotals": totals, + "key": want, "sku": found["sku"] if found else None, "asin": asin, "period": {"start": s.isoformat(), "end": e.isoformat()}, "returnedUnits": found["returned"] if found else 0.0, "shippedUnits": found["shipped"] if found else 0.0, - "returnRate": found["returnRate"] if found else None, + "returnRate": found["return_rate"] if found else None, "rate30": found["rate30"] if found else None, "rate90": found["rate90"] if found else None, - "rising": bool(found and found["rising"]), - "reasons": found["reasonList"] if found else [], - "repeat": None if rep is None else {k: v for k, v in rep.items() if not k.startswith("_")}, + "rising": bool(found and found["rising"]), "repeat": rep, "repeatShare": rep["repeatShare"] if rep else None, "repeatRevenue": rep["repeatRevenue"] if rep else None, - "boughtWith": _bought_with(data, asin) if asin else [], - "history": series, "item": dict(item) if item else None, - "emptyMessage": EMPTY if not found or not found["returned"] else None} + "item": item[0] if item else None, "emptyMessage": EMPTY if not found or not found["returned"] else None} # --------------------------------------------------------------------------- the after-sync recompute @@ -674,59 +934,91 @@ def on_amazon_sync(tenant_key, synced_tables): def recompute(tenant_key, *, st=None, now=None): - """Append today's per-SKU rate snapshot and raise one owned item per newly rising SKU.""" + """Refresh snapshots and rising items from bounded SQL batches after a background sync.""" st = _rt(tenant_key, st) now = now or _now() conf = settings(tenant_key, st=st) - data = _load(tenant_key, st) - if not data["mirror"]: - return {"state": "no_mirror", "created": []} today = now.date() - model = _model(data, today - dt.timedelta(days=PERIOD_DAYS - 1), today, conf) - snapshot = {x["key"]: [x["rate30"], x["rate90"], x["r30"], x["s30"], x["r90"], x["s90"]] - for x in model.values()} - - def _history(doc): - doc = doc if isinstance(doc, dict) else {} - doc.setdefault("snapshots", {})[today.isoformat()] = snapshot - doc["last_run"] = _stamp(now) - return doc - st.update(KEY_HISTORY, _history) - - rising = sorted((x for x in model.values() if x["rising"]), key=lambda x: x["key"]) - owner = conf["defaultOwner"] - created = [] - - def _items(doc): - doc = doc if isinstance(doc, dict) else {} - doc.setdefault("seq", 0) - items = doc.setdefault("items", {}) - latest = {} - for item in sorted(items.values(), key=_seq): - latest[item.get("sku")] = item - for x in rising: - if x["key"] in latest and latest[x["key"]].get("status") in HOLDING: - continue - doc["seq"] += 1 - iid = f"r{doc['seq']}" - times = x["rate30"] / x["rate90"] if x["rate90"] else None - reason = (f"30 day return rate {x['rate30']:.1f}% is {times:.1f} times the 90 day rate " - f"{x['rate90']:.1f}%." if times else f"30 day return rate {x['rate30']:.1f}%.") - item = {"id": iid, "title": f"Return rate rising on {x['key']}", "status": "Open", - "owner": owner, "due": (today + dt.timedelta(days=DUE_DAYS)).isoformat(), - "reason": reason, "sku": x["key"], "asin": x["asin"], - "rate30": x["rate30"], "rate90": x["rate90"], "topReason": - x["reasonList"][0]["reason"] if x["reasonList"] else None, - "createdAt": _stamp(now), "updatedAt": _stamp(now), "updatedBy": "system", + created, rising = [], [] + count = created_count = rising_count = 0 + with _screen_reader(tenant_key, st) as (cur, present): + if not present: + return {"state": "no_mirror", "created": []} + base = _relations(cur, today - dt.timedelta(days=PERIOD_DAYS - 1), today, conf) + while True: + rows = _mirror_page(cur, base + "SELECT * FROM sku_facts ORDER BY key", [], count, MAX_PAGE_SIZE) + ids = _refresh_batch(st, rows, now, conf["defaultOwner"], reset=count == 0) + created_count += len(ids) + created.extend(ids[:MAX_PAGE_SIZE - len(created)]) + keys = [row["key"] for row in rows if row["rising"]] + rising_count += len(keys) + rising.extend(keys[:MAX_PAGE_SIZE - len(rising)]) + count += len(rows) + if len(rows) < MAX_PAGE_SIZE: + break + return {"state": "computed", "date": today.isoformat(), "skus": count, "rising": rising, + "created": created, "risingCount": rising_count, "createdCount": created_count, + "previewLimit": MAX_PAGE_SIZE} + + +def _refresh_batch(st, rows, now, owner, *, reset=False): + """Merge a bounded snapshot and new items without transferring either containing document.""" + pg, tenant, key = _sql_scope(st) + names = [key(KEY_ITEMS), key(KEY_HISTORY)] + for name in names: + pg.check_write(tenant, f"refresh {name}") + created = {} + try: + with pg._pool().connection() as con, con.transaction(): + con.execute("SELECT set_config('lock_timeout', %s, true)", (f"{int(pg._lock_timeout_s()*1000)}ms",)) + for name, initial in zip(names, ({"seq": 0, "items": {}}, {"snapshots": {}})): + con.execute(pg._q("INSERT INTO {tbl} (key,value) VALUES (%s,%s::jsonb) ON CONFLICT (key) DO NOTHING", + "store_kv", tenant), (name, json.dumps(initial))) + lengths = {} + for name in sorted(names): + row = con.execute(pg._q("SELECT COALESCE((value->>'seq')::bigint,0), octet_length(value::text) " + "FROM {tbl} WHERE key=%s FOR UPDATE", "store_kv", tenant), (name,)).fetchone() + lengths[name] = row[1] + if name == names[0]: + seq = int(row[0]) + selected = [row["key"] for row in rows if row["rising"]] + latest = con.execute(pg._q(ITEM_LOOKUP_SQL, "store_kv", tenant), (names[0], selected)).fetchone()[0] + latest = json.loads(latest) if isinstance(latest, str) else latest + for row in rows: + if not row["rising"] or (latest.get(row["key"]) or {}).get("status") in HOLDING: + continue + seq += 1 + iid = f"r{seq}" + times = row["rate30"] / row["rate90"] if row["rate90"] else None + reason = (f"30 day return rate {row['rate30']:.1f}% is {times:.1f} times the 90 day rate " + f"{row['rate90']:.1f}%." if times else f"30 day return rate {row['rate30']:.1f}%.") + created[iid] = {"id": iid, "title": f"Return rate rising on {row['key']}", "status": "Open", + "owner": owner, "due": (now.date() + dt.timedelta(days=DUE_DAYS)).isoformat(), + "reason": reason, "sku": row["key"], "asin": row["asin"], "rate30": row["rate30"], + "rate90": row["rate90"], "topReason": row["top_reason"], "createdAt": _stamp(now), + "updatedAt": _stamp(now), "updatedBy": "system", "history": [{"at": _stamp(now), "by": "system", "event": "created"}]} - items[iid] = item - latest[x["key"]] = item - created.append(iid) - return doc - if rising: - st.update(KEY_ITEMS, _items) - return {"state": "computed", "date": today.isoformat(), "skus": len(snapshot), - "rising": [x["key"] for x in rising], "created": created} + snapshot = {row["key"]: [row[name] for name in ("rate30", "rate90", "r30", "s30", "r90", "s90")] + for row in rows} + day = now.date().isoformat() + saved = con.execute(pg._q("UPDATE {tbl} SET value=value || jsonb_build_object('snapshots', " + "COALESCE(value->'snapshots', '{{}}'::jsonb) || jsonb_build_object(%s::text, " + "CASE WHEN %s THEN '{{}}'::jsonb ELSE COALESCE(value #> %s::text[], '{{}}'::jsonb) END || %s::jsonb), " + "'last_run', %s::text), rev=rev+1, updated_at=now() WHERE key=%s RETURNING octet_length(value::text)", + "store_kv", tenant), (day, reset, ["snapshots", day], json.dumps(snapshot), _stamp(now), names[1])).fetchone() + if saved[0] > pg.DOCUMENT_CEILING_BYTES and saved[0] > lengths[names[1]]: + raise pg.StoreDocumentTooLarge(names[1], saved[0]) + if created: + saved = con.execute(pg._q("UPDATE {tbl} SET value=value || jsonb_build_object('seq', %s::bigint, " + "'items', COALESCE(value->'items', '{{}}'::jsonb) || %s::jsonb), rev=rev+1, updated_at=now() " + "WHERE key=%s RETURNING octet_length(value::text)", "store_kv", tenant), + (seq, json.dumps(created), names[0])).fetchone() + if saved[0] > pg.DOCUMENT_CEILING_BYTES and saved[0] > lengths[names[0]]: + raise pg.StoreDocumentTooLarge(names[0], saved[0]) + finally: + for name in names: + pg._bump_write_generation(name, tenant) + return list(created) # --------------------------------------------------------------------------- C4 item protocol @@ -768,18 +1060,14 @@ def may_edit(st, user): return _may_read_source(st, rec) -def items(tenant_key, user, *, status=None, cursor=None, limit=200, st=None): +def items(tenant_key, user, *, status=None, cursor=None, limit=100, st=None): """Every item, newest first, paged by cursor. No cap on the total (rule 1).""" st = _rt(tenant_key, st) if user is not None: rec = _user_record(st, user) if not rec or not _may_read_source(st, rec): raise PermissionError("Your account cannot read Amazon FBA returns.") - rows = [i for i in (_doc(st, KEY_ITEMS).get("items") or {}).values() - if status in (None, "") or i.get("status") == status] - rows.sort(key=_seq, reverse=True) - page, nxt = _page(rows, cursor, limit) - return {"items": [_public(i) for i in page], "cursor": nxt} + return _item_page(st, status=status or None, offset=_offset(cursor), limit=_page_size(limit)) def _status(value): @@ -810,31 +1098,34 @@ def update_item(tenant_key, user, item_id, *, owner=None, status=None, due=None, def _apply_item(st, item_id, by, *, owner=None, status=None, due=None, note=None, event=None): - out = {} - - def _apply(doc): - doc = doc if isinstance(doc, dict) else {} - item = (doc.get("items") or {}).get(str(item_id)) - if not item: - return doc - at = _stamp(_now()) - change = {"at": at, "by": by, **(event or {})} - if owner is not None: - item["owner"] = change["owner"] = str(owner).strip() - if status is not None: - item["status"] = change["status"] = status - if due is not None: - item["due"] = change["due"] = due - if note is not None and str(note).strip(): - item["note"] = change["note"] = str(note).strip() - item["updatedAt"], item["updatedBy"] = at, by - item.setdefault("history", []).append(change) - out["item"] = dict(item) - return doc - st.update(KEY_ITEMS, _apply) - if "item" not in out: - raise KeyError(item_id) - return _public(out["item"]) + pg, tenant, key = _sql_scope(st) + name, path = key(KEY_ITEMS), ["items", str(item_id)] + pg.check_write(tenant, "update returns item") + at = _stamp(_now()) + change = {"at": at, "by": by, **(event or {})} + payload = {"updatedAt": at, "updatedBy": by} + for field, value in (("owner", owner), ("status", status), ("due", due), ("note", note)): + if value is not None and (field != "note" or str(value).strip()): + payload[field] = change[field] = str(value).strip() + try: + with pg._pool().connection() as con, con.transaction(): + con.execute("SELECT set_config('lock_timeout', %s, true)", (f"{int(pg._lock_timeout_s()*1000)}ms",)) + row = con.execute(pg._q("SELECT (value #> %s::text[]) - 'history', octet_length(value::text) " + "FROM {tbl} WHERE key=%s FOR UPDATE", "store_kv", tenant), (path, name)).fetchone() + item = row[0] if row else None + if not item: + raise KeyError(item_id) + item = json.loads(item) if isinstance(item, str) else item + saved = con.execute(pg._q("UPDATE {tbl} SET value=jsonb_set(value,%s::text[], " + "(value #> %s::text[]) || %s::jsonb || jsonb_build_object('history', " + "COALESCE(value #> %s::text[], '[]'::jsonb) || %s::jsonb)), rev=rev+1, updated_at=now() " + "WHERE key=%s RETURNING octet_length(value::text)", "store_kv", tenant), + (path, path, json.dumps(payload), path + ["history"], json.dumps([change]), name)).fetchone() + if saved[0] > pg.DOCUMENT_CEILING_BYTES and saved[0] > row[1]: + raise pg.StoreDocumentTooLarge(name, saved[0]) + finally: + pg._bump_write_generation(name, tenant) + return _public({**item, **payload}) def set_status(tenant_key, user, item_id, status, *, note=None, st=None): @@ -876,7 +1167,8 @@ def validate(tenant_key, *, start=None, end=None, st=None, now=None, **_period_k st = _rt(tenant_key, st) s, e = _period(start or _period_kw.get("period_start"), end or _period_kw.get("period_end"), now) conf = settings(tenant_key, st=st) - app_total = sum(x["returned"] for x in _model(_load(tenant_key, st), s, e, conf).values()) + with _screen_reader(tenant_key, st) as (cur, _present): + app_total = _aggregate(cur, _relations(cur, s, e, conf))["returnedUnits"] independent = None with _reader(tenant_key, st) as cur: cols = _columns(cur, RETURNS_TABLE) if cur is not None else set() diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index 049f3b0258839c234db73c782f1713dae5ce943a..dc9fddd6f6429fc21a9c9b2ee93e01d1271a774e 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -8d3b583f2e7aeeca049adf1a1feb3cd28c940dcbf26c20a2081d4338fe03bc26 +8f847d7f14aa19b71294c415fa13ed211bc7973f7fea36b653982a34154749ad diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json index 24b557d279db3dd2de64eab3de532503f47e9228..ccfcaf023527e0192b98023b6ecd4b20473a75eb 100644 --- a/web/public/sample_customers.json +++ b/web/public/sample_customers.json @@ -195,18 +195,25 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + "description": "Receivable not yet due on Odoo's Aged Receivable basis: due today or later, no grace." }, { "key": "ar_overdue", - "label": "AR overdue $", + "label": "Overdue", "type": "currency", "source": "odoo", "default": false, - "description": "Invoiced money past due, on the same basis as the Collections page." + "description": "Odoo open receivables past due, no grace; New Customer Hold due now. Credits and unapplied payments net per customer, tying to Odoo to the cent." }, { "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", + "key": "ar_overdue_past_grace", + "label": "Overdue past grace", + "type": "currency", + "source": "odoo", + "description": "Invoice balances more than 5 days past due (the 5-day grace collections uses). Not Odoo's overdue." + }, + { "key": "ar_outstanding", "label": "AR outstanding $", "type": "currency", @@ -228,7 +235,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + "description": "Overdue between 1 and 30 days. The four aging buckets sum to Overdue." }, { "key": "ar_aged_31_60", @@ -236,7 +243,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + "description": "Overdue between 31 and 60 days. The four aging buckets sum to Overdue." }, { "key": "ar_aged_61_90", @@ -244,7 +251,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + "description": "Overdue between 61 and 90 days. The four aging buckets sum to Overdue." }, { "key": "ar_aged_90_plus", @@ -252,7 +259,7 @@ "type": "currency", "source": "odoo", "default": false, - "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + "description": "Overdue by more than 90 days. The four aging buckets sum to Overdue." }, { "key": "days_to_pay", diff --git a/web/s71_frontend_contracts.py b/web/s71_frontend_contracts.py index 1e4ca0b90fafff0ab886e1ff3773039819d88533..322166f7020cf8c0a8b9f3ff82813cadd82f8691 100644 --- a/web/s71_frontend_contracts.py +++ b/web/s71_frontend_contracts.py @@ -16,18 +16,20 @@ def catalog_host_faults(grid: str, loader: str, map_view: str, catalog: str) -> "Catalog has no route event listener": bool(re.search(r'useEffect\(\(\) => \{\s*if \(!routeApplicationSurface\) return;\s*const openRoute', grid)), "cold App lookup precedes row admission": grid.find('const addressedDefinition') < grid.find('} = useCustomerData(scope,') # S78-T17 (C4): a Competitor watch App also reads no source rows. - and bool(re.search(r'const legacyRowsEnabled = !isCatalogApplication && ' + and bool(re.search(r'const legacyRowsEnabled = !isPagedOdooApplication && ' r'(?:!isCompetitorWatchApplication &&\s+)?' r'\(!isApplicationSurface \|\| !!addressedDefinition\)', grid)) and 'rowsEnabled: legacyRowsEnabled' in grid, "Catalog disables legacy rows and subscriptions": 'options.rowsEnabled === false ? EMPTY_APPLICATION_PAYLOAD' in loader and 'const suppliedPayload = embeddedPayload !== undefined' in loader and 'if (suppliedPayload || STREAMLIT_COMPONENT) return; // the caller owns refresh' in loader, - "Catalog mount independent of a legacy field": '(isCatalogApplication || fieldByKey.get(lockedKey))' in grid + "Catalog mount independent of a legacy field": '(isPagedOdooApplication || fieldByKey.get(lockedKey))' in grid and 'if (!props.field)' in map_view, - "typed complete pool supplies the real drawer": 'onCatalogData={acceptCatalogData}' in grid - and 'if (isCatalogApplication) return catalogData?.rows ?? []' in grid + "typed page supplies the real drawer": 'onCatalogData={acceptCatalogData}' in grid + and 'if (isPagedOdooApplication) return catalogData?.rows ?? []' in grid and 'onData={props.onCatalogData}' in map_view and 'onData?.(data.fields, data.allRows)' in catalog, + "Routing shares the paged source admission": 'const isPagedOdooApplication = isCatalogApplication || isRoutingApplication' in grid + and '(isRoutingApplication || !serverWindowed)' in grid, } return [name for name, ok in checks.items() if not ok] @@ -44,10 +46,12 @@ def catalog_host_scan() -> int: (0, 'resolvedApplicationMatches && applicationTemplateSurface(resolvedApplication) === "route"', 'true'), (0, 'if (!routeApplicationSurface) return;\n const openRoute', 'const openRoute'), (0, 'rowsEnabled: legacyRowsEnabled', 'rowsEnabled: true'), - (0, '(isCatalogApplication || fieldByKey.get(lockedKey))', 'fieldByKey.get(lockedKey)'), + (0, '(isPagedOdooApplication || fieldByKey.get(lockedKey))', 'fieldByKey.get(lockedKey)'), (0, 'onCatalogData={acceptCatalogData}', ''), (1, 'options.rowsEnabled === false ? EMPTY_APPLICATION_PAYLOAD', 'false ? EMPTY_APPLICATION_PAYLOAD'), (3, 'onData?.(data.fields, data.allRows)', 'onData?.(data.fields, data.rows)'), + (0, 'const isPagedOdooApplication = isCatalogApplication || isRoutingApplication', + 'const isPagedOdooApplication = isCatalogApplication'), ] for index, old, new in controls: mutant = sources.copy() @@ -58,7 +62,7 @@ def catalog_host_scan() -> int: else: print('ok NC S71 Catalog host: ' + old) if not faults: - print('PASS S71 Catalog host: 8 contracts, 8 removal controls') + print('PASS S71 Catalog host: 9 contracts, 9 removal controls') return int(bool(faults)) @@ -75,6 +79,7 @@ def mounted_catalog_host(base_url: str, out_dir: Path, negative_controls: bool = self.catalog = s76_t09_fixture_record() self.catalog["pages"][3]["bindings"]["products"] = ["WF-1", "CS-2"] self.typed_cursors = [] + self.named_reads = [] self.fields.extend([{"key": key, "label": key, "type": kind, "source": "odoo"} for key, kind in (("stock", "text"), ("list_price", "int"), ("category", "text"), ("description", "text"))]) @@ -91,16 +96,26 @@ def mounted_catalog_host(base_url: str, out_dir: Path, negative_controls: bool = path = urlsplit(request.url).path.removeprefix("/api/v1/") result = None if path == "typed/openapi.json": - properties = {field["key"]: {"type": "number" if field["type"] == "int" else "string"} + properties = {field["key"]: {"type": "number" if field["type"] in ("int", "number") else "string"} for field in self.fields} properties["pid"] = {"type": "integer"} result = {"components": {"schemas": {"Record_customer_data": {"properties": properties}}}} elif path == "typed/databases/customer_data/query": body = request.post_data_json self.typed_cursors.append(body.get("cursor")) - if not self.rows: - result = {"database": self.database, "fields": self.fields, "records": [], - "page": {"returned": 0, "has_more": False, "next_cursor": None}} + filters = (body.get("filters") or {}).get("nodes", []) + named = [node for node in filters if node.get("op") == "eq" and node.get("colId") in ("code", "pid")] + if named: + self.named_reads.extend(node["value"] for node in named) + rows = [row for row in self.rows if all(str(row.get(node["colId"])) == str(node["value"]) for node in named)] + more = False + else: + tail = body.get("cursor") == "fixture-next" + rows = self.rows[1:2] if tail else self.rows[:1] + more = bool(self.rows) and not tail + result = {"database": self.database, "fields": self.fields, "records": rows, + "page": {"returned": len(rows), "has_more": more, + "next_cursor": "fixture-next" if more else None}} elif request.method == "GET" and path == "apps/app-fixture/catalogs": result = {"catalogs": [self.catalog] if self.catalog else [], "nextCursor": None} elif request.method == "GET" and path == "catalogs/catalog-six": @@ -212,9 +227,10 @@ def mounted_catalog_host(base_url: str, out_dir: Path, negative_controls: bool = check('six editor page kinds remain available', page.locator('.cg-catalog-page-thumb').evaluate_all("nodes => nodes.map(node => node.dataset.pageKind)") == ['cover', 'toc', 'product', 'grid', 'section', 'closing']) - check('typed pool walks both cursor pages', None in fixture.typed_cursors - and 'fixture-next' in fixture.typed_cursors) - check('all cursor pages painted', sorted(page.locator('.cg-catalog-canvas .cg-catalog-product-open').evaluate_all( + check('typed first page never walks the next cursor', None in fixture.typed_cursors + and 'fixture-next' not in fixture.typed_cursors) + check('saved products resolve by named reads', {'WF-1', 'CS-2'}.issubset(set(fixture.named_reads))) + check('saved products from both source pages painted', sorted(page.locator('.cg-catalog-canvas .cg-catalog-product-open').evaluate_all( "nodes => nodes.map(node => node.getAttribute('aria-label').slice(5))")) == ['Cedar Supply', 'Willow Floral']) before = page.locator('.cg-catalog-canvas').inner_text() @@ -252,7 +268,7 @@ def mounted_catalog_host(base_url: str, out_dir: Path, negative_controls: bool = finally: context.close() for surface in ('data-map', 'route', 'legacy-catalog'): - fixture = AppRoutingFixture() + fixture = CatalogHostFixture() if surface == 'route' else AppRoutingFixture() fixture.config['display'] = {'mode': 'map'} fixture.workspace['views'][0]['config']['display'] = {'mode': 'map'} if surface == 'legacy-catalog': @@ -262,6 +278,7 @@ def mounted_catalog_host(base_url: str, out_dir: Path, negative_controls: bool = fixture.workspace['views'][0]['config']['display'] = display fixture.original_workspace = __import__('copy').deepcopy(fixture.workspace) if surface == 'route': + fixture.legacy_unavailable = True fixture.fields.extend([{'key': 'lat', 'label': 'Latitude', 'type': 'number', 'source': 'odoo'}, {'key': 'lon', 'label': 'Longitude', 'type': 'number', 'source': 'odoo'}]) for index, row in enumerate(fixture.rows): @@ -293,6 +310,8 @@ def mounted_catalog_host(base_url: str, out_dir: Path, negative_controls: bool = page.locator('.cg-mapview.cg-app-route-surface').wait_for(timeout=30000) page.wait_for_timeout(600) check('Routing keeps Routes rail', page.get_by_text('Routes', exact=True).count() > 0) + check('cold Routing has no legacy customer request', not any(request['path'] == 'customers' for request in fixture.requests)) + check('cold Routing reads one typed page', fixture.typed_cursors == [None]) page.evaluate("window.dispatchEvent(new CustomEvent('aios:route-open', {detail:{routeId:'fixture-route'}}))") page.wait_for_timeout(200) check('Routing route event keeps its Map surface', page.locator('.cg-map-settings').count() == 1) diff --git a/web/src/apps/AppReason.tsx b/web/src/apps/AppReason.tsx new file mode 100644 index 0000000000000000000000000000000000000000..0669814c1131a1a634f7a53fb9ba3dec0af37444 --- /dev/null +++ b/web/src/apps/AppReason.tsx @@ -0,0 +1,94 @@ +import { Component, useEffect, useState } from "react"; +import type { ReactNode } from "react"; +import { API_V1, CREDENTIALS, UNAUTHORIZED_EVENT, checkTenant, signal } from "../apiContract"; + +export const APP_OPEN_TIMEOUT_MS = 12_000; + +const MESSAGES = { + loading: "Loading this App.", + router_unavailable: "This App cannot open because its routes are unavailable.", + no_grant: "Your access limit prevents this App from reading its source.", + not_connected: "Connect the App's connector to open this App.", + no_rows: "This App has no source rows yet.", + source_unavailable: "This App cannot read its source right now.", + held: "This App is not available yet.", + chunk_unavailable: "This App could not load its screen.", + unavailable: "This App is unavailable or is not shared with you.", +}; +export type AppReasonState = keyof typeof MESSAGES; +export interface AppAvailability { + state: AppReasonState | "ready"; + message?: string | null; + lock?: string | null; +} + +/** One sentence in the existing missing-view style. */ +export function AppReason({ state, connector }: { state: AppReasonState; connector?: string }) { + const label = ({ amazon: "Amazon", amazon_ads: "Amazon Ads", odoo: "Odoo" } as Record)[connector || ""]; + const message = state === "not_connected" && label ? `Connect ${label} to open this App.` : MESSAGES[state]; + return

{message}

; +} + +/** The browser accepts reason codes, never arbitrary server exception text. */ +export function availabilityState(value: unknown): AppAvailability["state"] { + const state = value && typeof value === "object" ? (value as { state?: unknown }).state : null; + return state === "ready" || (typeof state === "string" && state !== "loading" + && Object.prototype.hasOwnProperty.call(MESSAGES, state)) ? state as AppAvailability["state"] : "unavailable"; +} + +export function useAppOpening(id: string, initial?: AppAvailability): { state: AppAvailability["state"]; application?: unknown } { + const [result, setResult] = useState<{ id: string; state: AppAvailability["state"]; application?: unknown } | null>(null); + useEffect(() => { + const controller = new AbortController(); + let active = true; + const settle = (state: AppAvailability["state"], application?: unknown) => { + if (active) setResult({ id, state, application }); + }; + const timeout = setTimeout(() => { + settle("source_unavailable"); + active = false; + controller.abort(); + }, APP_OPEN_TIMEOUT_MS); + void fetch(`${API_V1}/applications/${encodeURIComponent(id)}/availability`, { + credentials: CREDENTIALS, signal: controller.signal, + }).then(async (response) => { + if (checkTenant(response)) return; + if (response.status === 401) signal(UNAUTHORIZED_EVENT); + if (!response.ok) { + settle(response.status >= 500 ? "source_unavailable" : "unavailable"); + return; + } + const body = await response.json() as { availability?: unknown; application?: unknown }; + settle(availabilityState(body.availability), body.application); + }).catch(() => settle("source_unavailable")).finally(() => clearTimeout(timeout)); + return () => { + active = false; + clearTimeout(timeout); + controller.abort(); + }; + }, [id]); + return result?.id === id ? result : { state: initial ? availabilityState(initial) : "loading" }; +} + +export function useAppAvailability(id: string, initial?: AppAvailability): AppAvailability["state"] { + return useAppOpening(id, initial).state; +} + +/** A lost chunk reaches an error boundary even when its request never settles. */ +export async function boundedAppChunk(load: () => Promise, timeoutMs = APP_OPEN_TIMEOUT_MS): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([load(), new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error("App screen unavailable")), timeoutMs); + })]); + } finally { + clearTimeout(timer); + } +} + +export class AppViewErrorBoundary extends Component<{ children: ReactNode; fallback: ReactNode }, { failed: boolean }> { + state = { failed: false }; + static getDerivedStateFromError() { return { failed: true }; } + render() { return this.state.failed ? this.props.fallback : this.props.children; } +} diff --git a/web/src/apps/account-health/AppView.tsx b/web/src/apps/account-health/AppView.tsx index ba921428ccd25d8284aec753c8438c2dd5f40457..0386a5555ef71611aea219471a0764b844266cb5 100644 --- a/web/src/apps/account-health/AppView.tsx +++ b/web/src/apps/account-health/AppView.tsx @@ -43,6 +43,9 @@ interface Summary { ratingStatus: string; snapshotDate: string; markets: number; + total: number; + offset: number; + limit: number; metrics: Metric[]; canEdit: boolean; } @@ -60,20 +63,23 @@ interface Alert { note: string; } -interface AlertPage { items: Alert[]; cursor: string | null; statuses: string[]; owners: string[]; canEdit: boolean } +interface AlertPage { + items: Alert[]; cursor: string | null; statuses: string[]; owners: string[]; canEdit: boolean; + total: number; offset: number; limit: number; +} interface Point { day: string; value: number | null; target: number | null; status: string } interface HistoryPage { points: Point[]; cursor: string | null; label: string; unit: string } -type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; -type Picked = { kind: "metric" | "alert"; id: string }; +type Load = { phase: "loading"; value?: T } | { phase: "error"; message: string; value?: T } + | { phase: "ready"; value: T }; +type Picked = { kind: "metric" | "alert"; id: string; metric?: Metric; alert?: Alert }; type Tab = "metrics" | "alerts"; const EMPTY = "No account health data"; const ALERT_STATUSES = ["Open", "Investigating", "Resolved", "Dismissed"]; -const OPEN = new Set(["Open", "Investigating"]); -const PAGE = 200; -const HISTORY_PAGE = 400; +const PAGE = 50; +const HISTORY_PAGE = 50; const text = (value: unknown) => (typeof value === "string" ? value : ""); const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); @@ -109,7 +115,8 @@ function parseSummary(raw: unknown): Summary { return { state: body.state === "ready" ? "ready" : "empty", message: text(body.message), rating: num(body.rating), ratingStatus: text(body.ratingStatus), snapshotDate: text(body.snapshotDate), - markets: list(body.marketplaces).length, + markets: num(body.marketCount) ?? 0, + total: num(body.metricTotal) ?? 0, offset: num(body.metricOffset) ?? 0, limit: num(body.metricLimit) ?? PAGE, metrics: list(body.metrics).map(parseMetric).filter((row): row is Metric => row !== null), canEdit: body.canEdit === true, }; @@ -133,6 +140,7 @@ function parseAlerts(raw: unknown): AlertPage { items: list(body.items).map(parseAlert).filter((row): row is Alert => row !== null), cursor: text(body.cursor) || null, statuses: statuses.length ? statuses : ALERT_STATUSES, owners: words(body.owners), canEdit: body.canEdit === true, + total: num(body.total) ?? 0, offset: num(body.offset) ?? 0, limit: num(body.limit) ?? PAGE, }; } @@ -193,6 +201,21 @@ function Failed({ message, onRetry }: { message: string; onRetry: () => void })

; } +// Busy buttons stay focusable so Enter keeps its place without a focus restore. +function Pager({ page, previous, next, busy, onPrevious, onNext }: { + page: number; previous: boolean; next: boolean; busy: boolean; onPrevious: () => void; onNext: () => void; +}) { + const buttonStyle = (unavailable: boolean) => unavailable + ? { opacity: "var(--lp-home-disabled-opacity)", cursor: "not-allowed" } : undefined; + return ; +} + function chartColors() { const style = getComputedStyle(document.documentElement); const read = (token: string) => style.getPropertyValue(token).trim(); @@ -259,63 +282,55 @@ function History({ metricId, label, url }: { }) { const [page, setPage] = useState>({ phase: "loading" }); const [attempt, setAttempt] = useState(0); - const [loadingMore, setLoadingMore] = useState(false); - const [moreError, setMoreError] = useState(""); + const [cursors, setCursors] = useState([""]); + const [index, setIndex] = useState(0); + const cursor = cursors[index] ?? ""; useEffect(() => { const controller = new AbortController(); - setPage({ phase: "loading" }); - setMoreError(""); - void fetch(url("/history", { metric: metricId, limit: String(HISTORY_PAGE) }), + setPage((current) => ({ phase: "loading", value: current.value })); + void fetch(url("/history", { metric: metricId, limit: String(HISTORY_PAGE), cursor }), { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "The history of this metric could not be read.")) .then((body) => { if (!controller.signal.aborted) setPage({ phase: "ready", value: parseHistory(body) }); }) .catch((reason: unknown) => { if (!controller.signal.aborted) { - setPage({ phase: "error", message: failure(reason, "The history of this metric could not be read.") }); + setPage((current) => ({ phase: "error", value: current.value, + message: failure(reason, "The history of this metric could not be read.") })); } }); return () => controller.abort(); - }, [url, metricId, attempt]); + }, [url, metricId, cursor, attempt]); - const loadMore = async () => { - if (page.phase !== "ready" || !page.value.cursor) return; - setLoadingMore(true); - setMoreError(""); - try { - const response = await fetch(url("/history", { metric: metricId, limit: String(HISTORY_PAGE), - cursor: page.value.cursor }), { credentials: CREDENTIALS }); - const next = parseHistory(await readJson(response, "More days could not be read.")); - setPage((current) => current.phase !== "ready" ? current : { phase: "ready", value: { - ...current.value, points: [...current.value.points, ...next.points], cursor: next.cursor } }); - } catch (reason) { - setMoreError(failure(reason, "More days could not be read.")); - } finally { - setLoadingMore(false); - } - }; - - if (page.phase === "loading") return ; - if (page.phase === "error") return setAttempt((n) => n + 1)} />; - const { points, unit, cursor } = page.value; - if (points.length === 0) return

No history yet

; + const held = page.value; + const points = held?.points ?? []; return

Daily history

- + {page.phase === "loading" && } + {page.phase === "error" && setAttempt((n) => n + 1)} />} + {page.phase === "ready" && points.length === 0 &&

No history yet

} + {page.phase === "ready" && points.length > 0 && <> +
{points.map((point) => - + )}
DayValue Status
{day(point.day)}{amount(point.value, unit, point.status)}{amount(point.value, held?.unit ?? "", point.status)}
- {moreError &&

{moreError}

} - {cursor &&
- -
} + } + 0} next={Boolean(held?.cursor)} busy={page.phase !== "ready"} + onPrevious={() => { + setPage((current) => ({ phase: "loading", value: current.value })); + setIndex(index - 1); + }} onNext={() => { + if (!held?.cursor) return; + setPage((current) => ({ phase: "loading", value: current.value })); + setCursors((current) => [...current.slice(0, index + 1), held.cursor ?? ""]); + setIndex(index + 1); + }} /> ; } @@ -387,15 +402,17 @@ interface PanelProps { url: (path: string, params?: Record) => string; onSaved: (alert: Alert) => void; onClose: () => void; + loading: boolean; + error: string; + onRetry: () => void; } /** The detail panel on the right: a picked metric with its daily history, or a picked alert. */ -function Panel({ metric, alert, editable, owners, statuses, url, onSaved, onClose }: PanelProps) { +function Panel({ metric, alert, editable, owners, statuses, url, onSaved, onClose, loading, error, onRetry }: PanelProps) { const panel = useRef(null); const close = useRef(null); useEscapeClose(true, onClose, { container: () => panel.current }); - const focusKey = `${metric?.id ?? ""}:${alert?.id ?? ""}`; - useEffect(() => { close.current?.focus(); }, [focusKey]); + useEffect(() => { close.current?.focus(); }, []); const title = alert && !metric ? alert.title : metric?.label ?? alert?.title ?? ""; const metricId = metric?.id ?? alert?.metricId ?? ""; @@ -405,6 +422,8 @@ function Panel({ metric, alert, editable, owners, statuses, url, onSaved, onClos

{title}

+ {loading && } + {error && } {metric &&
{metricText(metric)} @@ -495,64 +514,100 @@ export default function AccountHealthView({ application, canEdit }: AccountHealt const [alerts, setAlerts] = useState>({ phase: "loading" }); const [alertsAttempt, setAlertsAttempt] = useState(0); const [statusFilter, setStatusFilter] = useState(""); - const [loadingMore, setLoadingMore] = useState(false); - const [moreError, setMoreError] = useState(""); + const [search, setSearch] = useState(""); + const [query, setQuery] = useState(""); + const [metricOffset, setMetricOffset] = useState(0); + const [alertOffset, setAlertOffset] = useState(0); const [tab, setTab] = useState("metrics"); const [picked, setPicked] = useState(null); const opener = useRef(null); - const alive = useRef(true); - useEffect(() => () => { alive.current = false; }, []); + const [detailMetric, setDetailMetric] = useState(null); + const [detailAlert, setDetailAlert] = useState(null); + const [detailLoading, setDetailLoading] = useState(false); + const [detailError, setDetailError] = useState(""); + const [detailAttempt, setDetailAttempt] = useState(0); useEffect(() => { - const controller = new AbortController(); setSummary({ phase: "loading" }); - void fetch(url("/summary"), { credentials: CREDENTIALS, signal: controller.signal }) + setAlerts({ phase: "loading" }); + setPicked(null); + setSearch(""); + setQuery(""); + setMetricOffset(0); + setAlertOffset(0); + }, [appId]); + + useEffect(() => { + if (search === query) return; + const timer = window.setTimeout(() => { + setQuery(search); + setMetricOffset(0); + setAlertOffset(0); + }, 250); + return () => window.clearTimeout(timer); + }, [search, query]); + + useEffect(() => { + const controller = new AbortController(); + setSummary((current) => ({ phase: "loading", value: current.value })); + void fetch(url("/summary", { search: query, offset: String(metricOffset), limit: String(PAGE) }), + { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "Account health could not be read.")) .then((body) => { if (!controller.signal.aborted) setSummary({ phase: "ready", value: parseSummary(body) }); }) .catch((reason: unknown) => { - if (!controller.signal.aborted) setSummary({ phase: "error", message: failure(reason, "Account health could not be read.") }); + if (!controller.signal.aborted) setSummary((current) => ({ phase: "error", value: current.value, + message: failure(reason, "Account health could not be read.") })); }); return () => controller.abort(); - }, [url, summaryAttempt]); + }, [url, summaryAttempt, query, metricOffset]); - const alertsUrl = useCallback((cursor?: string) => { - const params: Record = { limit: String(PAGE) }; + const alertsUrl = useCallback(() => { + const params: Record = { limit: String(PAGE), offset: String(alertOffset), search: query }; if (statusFilter) params.status = statusFilter; - if (cursor) params.cursor = cursor; return url("/items", params); - }, [url, statusFilter]); + }, [url, statusFilter, alertOffset, query]); useEffect(() => { const controller = new AbortController(); - setAlerts({ phase: "loading" }); - setMoreError(""); + setAlerts((current) => ({ phase: "loading", value: current.value })); void fetch(alertsUrl(), { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "The alerts could not be read.")) .then((body) => { if (!controller.signal.aborted) setAlerts({ phase: "ready", value: parseAlerts(body) }); }) .catch((reason: unknown) => { - if (!controller.signal.aborted) setAlerts({ phase: "error", message: failure(reason, "The alerts could not be read.") }); + if (!controller.signal.aborted) setAlerts((current) => ({ phase: "error", value: current.value, + message: failure(reason, "The alerts could not be read.") })); }); return () => controller.abort(); }, [alertsUrl, alertsAttempt]); - const loadMore = async () => { - if (alerts.phase !== "ready" || !alerts.value.cursor) return; - setLoadingMore(true); - setMoreError(""); - try { - const response = await fetch(alertsUrl(alerts.value.cursor), { credentials: CREDENTIALS }); - const next = parseAlerts(await readJson(response, "More alerts could not be read.")); - if (!alive.current) return; - setAlerts((current) => current.phase !== "ready" ? current : { phase: "ready", value: { - ...current.value, items: [...current.value.items, ...next.items], cursor: next.cursor } }); - } catch (reason) { - if (alive.current) setMoreError(failure(reason, "More alerts could not be read.")); - } finally { - if (alive.current) setLoadingMore(false); - } - }; + useEffect(() => { + if (!picked) return; + const controller = new AbortController(); + const mid = picked.metric?.id ?? picked.alert?.metricId ?? picked.id; + setDetailMetric(picked.metric ?? null); + setDetailAlert(picked.alert ?? null); + setDetailLoading(true); + setDetailError(""); + void Promise.all([ + fetch(url("/summary", { metric: mid, limit: "1" }), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "This metric could not be read.")) + .then((body) => parseSummary(body).metrics[0] ?? null), + picked.alert ? Promise.resolve(picked.alert) + : fetch(url("/items", { metric: mid, limit: "1" }), { credentials: CREDENTIALS, signal: controller.signal }) + .then((response) => readJson(response, "This alert could not be read.")) + .then((body) => parseAlerts(body).items[0] ?? null), + ]).then(([metric, alert]) => { + if (!controller.signal.aborted) { setDetailMetric(metric); setDetailAlert(alert); } + }).catch((reason: unknown) => { + if (!controller.signal.aborted) setDetailError(failure(reason, "The detail could not be read.")); + }).finally(() => { if (!controller.signal.aborted) setDetailLoading(false); }); + return () => controller.abort(); + }, [url, picked, detailAttempt]); const pick = (next: Picked) => { + setDetailMetric(next.metric ?? null); + setDetailAlert(next.alert ?? null); + setDetailError(""); opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; setPicked(next); }; @@ -561,27 +616,24 @@ export default function AccountHealthView({ application, canEdit }: AccountHealt opener.current?.focus(); }, []); const saved = useCallback((alert: Alert) => { + setDetailAlert(alert); setAlerts((current) => current.phase !== "ready" ? current : { phase: "ready", value: { ...current.value, items: current.value.items.map((row) => (row.id === alert.id ? { ...row, ...alert } : row)), } }); + setAlertsAttempt((value) => value + 1); }, []); - const view = summary.phase === "ready" ? summary.value : null; - const page = alerts.phase === "ready" ? alerts.value : null; + const view = summary.value ?? null; + const page = alerts.value ?? null; const editable = canEdit === true && page?.canEdit === true; const items = page?.items ?? []; - const metric = picked && view ? (picked.kind === "metric" - ? view.metrics.find((row) => row.id === picked.id) ?? null - : view.metrics.find((row) => row.id === items.find((a) => a.id === picked.id)?.metricId) ?? null) : null; - const alert = picked ? (picked.kind === "alert" ? items.find((row) => row.id === picked.id) ?? null - : items.find((row) => row.metricId === picked.id && OPEN.has(row.status)) - ?? items.find((row) => row.metricId === picked.id) ?? null) : null; - const alertCount = `${items.length}${page?.cursor ? "+" : ""}`; - - if (summary.phase === "loading") { + const metric = detailMetric; + const alert = detailAlert; + + if (summary.phase === "loading" && !view) { return
; } - if (summary.phase === "error") { + if (summary.phase === "error" && !view) { return
setSummaryAttempt((n) => n + 1)} />
; @@ -592,7 +644,10 @@ export default function AccountHealthView({ application, canEdit }: AccountHealt
; } - const open = Boolean(picked && (metric || alert)); + const open = Boolean(picked); + const busy = tab === "metrics" ? summary.phase !== "ready" : alerts.phase !== "ready"; + const shownOffset = tab === "metrics" ? view.offset : page?.offset ?? 0; + const shownTotal = tab === "metrics" ? view.total : page?.total ?? 0; return
@@ -608,27 +663,39 @@ export default function AccountHealthView({ application, canEdit }: AccountHealt
+
+ +
+ {tab === "metrics" &&
- {view.metrics.length === 0 - ?

{EMPTY}

+ {summary.phase === "loading" && } + {summary.phase === "error" && setSummaryAttempt((n) => n + 1)} />} + {summary.phase === "ready" && (view.metrics.length === 0 + ?

No matching metrics

: pick({ kind: "metric", id: row.id })} />} + onPick={(row) => pick({ kind: "metric", id: row.id, metric: row })} />)}
} {tab === "alerts" &&
{alerts.phase === "loading" && } {alerts.phase === "error" && setAlertsAttempt((n) => n + 1)} />} - {page && items.length === 0 &&

No alerts

} - {page && items.length > 0 && pick({ kind: "alert", id: row.id })} />} - {moreError &&

{moreError}

} - {page?.cursor &&
- -
} + {alerts.phase === "ready" && items.length === 0 &&

No alerts

} + {alerts.phase === "ready" && items.length > 0 && pick({ kind: "alert", id: row.id, alert: row })} />}
} + 0} + next={shownOffset + PAGE < shownTotal} busy={busy || search !== query} + onPrevious={() => tab === "metrics" ? setMetricOffset(Math.max(0, shownOffset - PAGE)) + : setAlertOffset(Math.max(0, shownOffset - PAGE))} + onNext={() => tab === "metrics" ? setMetricOffset(shownOffset + PAGE) : setAlertOffset(shownOffset + PAGE)} />
{open && } + onClose={closePanel} loading={detailLoading} error={detailError} onRetry={() => setDetailAttempt((n) => n + 1)} />}
; } diff --git a/web/src/apps/aged-stock/AppView.tsx b/web/src/apps/aged-stock/AppView.tsx index e65124f8eea0d9240f216df0373f20d8c33bafc4..b6862d5c4a9310da4289363418c7be313365608e 100644 --- a/web/src/apps/aged-stock/AppView.tsx +++ b/web/src/apps/aged-stock/AppView.tsx @@ -48,7 +48,7 @@ interface Removal { snapshot: string; removal: number | null; sellable: number | interface Stranded { reasons: string[]; units: number | null; autoRemoval: string; since: string } interface Detail { row: Row; reason: string; fees: FeePoint[]; bands: [string, number | null][]; removal: Removal | null; - stranded: Stranded | null; owners: string[]; + stranded: Stranded | null; owners: string[]; offset: number; limit: number; total: number; } interface Summary { @@ -60,7 +60,12 @@ interface Summary { trend: Month[]; totals: Totals | null; others: [string, number][]; - nextTier: Row[]; + nextTierCount: number; + monthOffset: number; + monthLimit: number; + totalMonths: number; + firstMonth: string; + lastMonth: string; statusCounts: Record; actionCounts: Record; refreshedAt: string; @@ -68,7 +73,9 @@ interface Summary { canEdit: boolean; } -interface ListPage { items: Row[]; cursor: string | null; owners: string[] } +interface ListPage { + items: Row[]; cursor: string | null; previousCursor: string | null; owners: string[]; offset: number; total: number; +} type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; @@ -92,6 +99,8 @@ const text = (value: unknown) => (typeof value === "string" ? value : ""); const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); const list = (value: unknown) => (Array.isArray(value) ? value : []); const words = (value: unknown) => list(value).map(text).filter(Boolean); +const pageButtonStyle = (disabled: boolean) => disabled + ? { opacity: "var(--lp-home-disabled-opacity)", cursor: "not-allowed" } : undefined; const record = (value: unknown) => (value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}); @@ -161,19 +170,6 @@ function parseItem(raw: unknown): Row | null { }; } -function parseNextTier(raw: unknown): Row | null { - const row = record(raw); - const id = text(row.id); - if (!id) return null; - const item = record(row.item); - return { - id, sku: text(row.sku), fnsku: text(row.fnsku), asin: "", title: text(row.title) || text(row.sku), - action: text(item.action), recommended: text(item.recommended_action), tier: text(row.tier), - units: num(row.units), atStake: num(row.at_stake), currency: text(row.currency), owner: text(item.owner), - status: text(item.status), due: text(item.due), inReport: true, - }; -} - function parseTotals(raw: unknown): Totals { const t = record(raw); return { storage: num(t.storage) ?? 0, longTerm: num(t.long_term) ?? 0, total: num(t.total) ?? 0, @@ -204,16 +200,23 @@ function parseSummary(raw: unknown): Summary { // Other currencies are listed, never added to the primary one. others: Object.keys(totals).filter((cur) => cur !== primary).sort() .map((cur): [string, number] => [cur, parseTotals(totals[cur]).total]), - nextTier: list(body.nextTier).map(parseNextTier).filter((row): row is Row => row !== null), + nextTierCount: num(body.nextTierCount) ?? 0, monthOffset: num(body.offset) ?? 0, + monthLimit: num(body.monthLimit) ?? 24, totalMonths: num(body.totalMonths) ?? 0, + firstMonth: text(body.firstMonth), lastMonth: text(body.lastMonth), statusCounts: counts(body.statusCounts, STATUSES), actionCounts: counts(body.actionCounts, ACTIONS), refreshedAt: text(body.refreshedAt), lastError: text(body.lastError), canEdit: body.canEdit === true, }; } -function parsePage(raw: unknown): ListPage { +function parsePage(raw: unknown, nextTier: boolean): ListPage { const body = record(raw); - return { items: list(body.items).map(parseItem).filter((row): row is Row => row !== null), - cursor: text(body.cursor) || null, owners: words(body.owners) }; + return { items: list(body.items).map((rawItem) => { + const item = parseItem(rawItem); + const tier = record(record(rawItem).next_tier); + return item && nextTier ? { ...item, units: num(tier.units), atStake: num(tier.at_stake) } : item; + }).filter((row): row is Row => row !== null), cursor: text(body.cursor) || null, + previousCursor: text(body.previousCursor) || null, offset: num(body.offset) ?? 0, + total: num(body.total) ?? 0, owners: words(body.owners) }; } function parseDetail(raw: unknown, fallback: Row): Detail { @@ -222,6 +225,7 @@ function parseDetail(raw: unknown, fallback: Row): Detail { const removal = body.removal ? record(body.removal) : null; const stranded = body.stranded ? record(body.stranded) : null; return { + offset: num(body.detailOffset) ?? 0, limit: num(body.detailLimit) ?? 50, total: num(body.detailTotal) ?? 0, row: parseItem(body) ?? fallback, reason: text(body.reason), fees: list(body.fee_history).map((entry) => { @@ -306,9 +310,9 @@ function Kpi({ label, value, period }: { label: string; value: string; period: s function KpiStrip({ summary }: { summary: Summary }) { const cur = summary.primary; const t = summary.totals ?? { storage: 0, longTerm: 0, total: 0, atStake: 0 }; - const span = summary.trend.length === 0 ? "No months yet" - : summary.trend.length === 1 ? monthWords(summary.trend[0].month) - : `${monthWords(summary.trend[0].month)} to ${monthWords(summary.trend[summary.trend.length - 1].month)}`; + const span = !summary.firstMonth ? "No months yet" + : summary.firstMonth === summary.lastMonth ? monthWords(summary.firstMonth) + : `${monthWords(summary.firstMonth)} to ${monthWords(summary.lastMonth)}`; const open = summary.statusCounts.Open ?? 0; return
@@ -337,7 +341,7 @@ function MonthTable({ months, totals, currency }: { months: Month[]; totals: Tot {monthWords(m.month)}{money(m.storage, currency)} {money(m.longTerm, currency)}{money(m.total, currency)} )} - Total{money(totals.storage, currency)} + All months{money(totals.storage, currency)} {money(totals.longTerm, currency)} {money(totals.total, currency)} @@ -385,9 +389,13 @@ function Drawer({ row, editable, owners: listOwners, url, onSaved, onClose }: Dr const close = useRef(null); const [detail, setDetail] = useState>({ phase: "loading" }); const [attempt, setAttempt] = useState(0); + const [offset, setOffset] = useState(0); + const detailUrl = url(`/items/${encodeURIComponent(row.id)}`, { offset: String(offset) }); + const latestRequest = useRef(detailUrl); + latestRequest.current = detailUrl; const [saving, setSaving] = useState(false); const [saveError, setSaveError] = useState(""); - // The row object changes after a save. Only a different SKU reloads the detail. + // Keep saved row fields while drawer pages change. const latest = useRef(row); latest.current = row; useEscapeClose(true, onClose, { container: () => panel.current }); @@ -396,14 +404,21 @@ function Drawer({ row, editable, owners: listOwners, url, onSaved, onClose }: Dr useEffect(() => { const controller = new AbortController(); setDetail({ phase: "loading" }); - void fetch(url(`/items/${encodeURIComponent(row.id)}`), { credentials: CREDENTIALS, signal: controller.signal }) + void fetch(detailUrl, { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "This SKU could not be loaded.")) - .then((body) => { if (!controller.signal.aborted) setDetail({ phase: "ready", value: parseDetail(body, latest.current) }); }) + .then((body) => { + if (controller.signal.aborted || latestRequest.current !== detailUrl) return; + const value = parseDetail(body, latest.current); + setOffset(value.offset); + setDetail({ phase: "ready", value }); + }) .catch((reason: unknown) => { - if (!controller.signal.aborted) setDetail({ phase: "error", message: failure(reason, "This SKU could not be loaded.") }); + if (!controller.signal.aborted && latestRequest.current === detailUrl) { + setDetail({ phase: "error", message: failure(reason, "This SKU could not be loaded.") }); + } }); return () => controller.abort(); - }, [url, row.id, attempt]); + }, [detailUrl, attempt]); /** Store one change, then show the value the server answered. */ const save = async (change: Change) => { @@ -545,6 +560,16 @@ function Drawer({ row, editable, owners: listOwners, url, onSaved, onClose }: Dr :

No stranded units

}
} + ; } @@ -599,77 +624,100 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView const [search, setSearch] = useState(""); const [q, setQ] = useState(""); const [page, setPage] = useState>({ phase: "loading" }); - const [loadingMore, setLoadingMore] = useState(false); - const [moreError, setMoreError] = useState(""); + const [loadedScope, setLoadedScope] = useState(""); + const [loadingPage, setLoadingPage] = useState(true); + const [pageError, setPageError] = useState(""); + const [navigation, setNavigation] = useState<{ scope: string; cursor: string | null }>({ scope: "", cursor: null }); + const [monthOffset, setMonthOffset] = useState(0); + const [loadingSummary, setLoadingSummary] = useState(true); + const [summaryApp, setSummaryApp] = useState(""); const [refreshing, setRefreshing] = useState(false); const [refreshError, setRefreshError] = useState(""); const [openedId, setOpenedId] = useState(null); const opener = useRef(null); const alive = useRef(true); - useEffect(() => () => { alive.current = false; }, []); + useEffect(() => { alive.current = true; return () => { alive.current = false; }; }, []); // Rule 1: the server filters every item by q before paging, so the search never runs over a loaded page. + const trimmedSearch = search.trim(); useEffect(() => { - const timer = window.setTimeout(() => setQ(search.trim()), SEARCH_DELAY_MS); + if (trimmedSearch === q) return; + const timer = window.setTimeout(() => { + setQ(trimmedSearch); + setNavigation({ scope: "", cursor: null }); + setOpenedId(null); + }, SEARCH_DELAY_MS); return () => window.clearTimeout(timer); - }, [search]); + }, [trimmedSearch, q]); + const summaryUrl = url("/summary", { offset: String(monthOffset) }); + const latestSummary = useRef(summaryUrl); + latestSummary.current = summaryUrl; useEffect(() => { const controller = new AbortController(); - setSummary({ phase: "loading" }); - void fetch(url("/summary"), { credentials: CREDENTIALS, signal: controller.signal }) + setLoadingSummary(true); + void fetch(summaryUrl, { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "The aged stock summary could not be loaded.")) - .then((body) => { if (!controller.signal.aborted) setSummary({ phase: "ready", value: parseSummary(body) }); }) + .then((body) => { + if (controller.signal.aborted || latestSummary.current !== summaryUrl) return; + const value = parseSummary(body); + setMonthOffset(value.monthOffset); + setSummaryApp(appId); + setSummary({ phase: "ready", value }); + }) .catch((reason: unknown) => { - if (!controller.signal.aborted) { + if (!controller.signal.aborted && latestSummary.current === summaryUrl) { setSummary({ phase: "error", message: failure(reason, "The aged stock summary could not be loaded.") }); } + }).finally(() => { + if (!controller.signal.aborted && latestSummary.current === summaryUrl) setLoadingSummary(false); }); return () => controller.abort(); - }, [url, attempt]); + }, [summaryUrl, appId, attempt]); const listQuery = useCallback((cursor?: string) => { const params: Record = { limit: String(PAGE) }; + if (tab === NEXT_TIER) params.nextTier = "true"; if (ACTIONS.includes(tab)) params.action = tab; if (q) params.q = q; if (cursor) params.cursor = cursor; return url("/items", params); }, [url, tab, q]); - const itemsTab = tab !== NEXT_TIER; + const scope = listQuery(); + const cursor = navigation.scope === scope ? navigation.cursor : null; + const requestUrl = listQuery(cursor ?? undefined); + const latestRequest = useRef(requestUrl); + latestRequest.current = requestUrl; useEffect(() => { - if (!itemsTab) return; const controller = new AbortController(); - setPage({ phase: "loading" }); - setMoreError(""); - void fetch(listQuery(), { credentials: CREDENTIALS, signal: controller.signal }) + setLoadingPage(true); + setPageError(""); + setOpenedId(null); + void fetch(requestUrl, { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "The SKU list could not be loaded.")) - .then((body) => { if (!controller.signal.aborted) setPage({ phase: "ready", value: parsePage(body) }); }) + .then((body) => { + if (controller.signal.aborted || latestRequest.current !== requestUrl) return; + const value = parsePage(body, tab === NEXT_TIER); + setPage({ phase: "ready", value }); + setLoadedScope(scope); + }) .catch((reason: unknown) => { - if (!controller.signal.aborted) setPage({ phase: "error", message: failure(reason, "The SKU list could not be loaded.") }); + if (!controller.signal.aborted && latestRequest.current === requestUrl) { + setPageError(failure(reason, "The SKU list could not be loaded.")); + } + }).finally(() => { + if (!controller.signal.aborted && latestRequest.current === requestUrl) setLoadingPage(false); }); return () => controller.abort(); - }, [itemsTab, listQuery, attempt]); + }, [requestUrl, scope, tab, attempt]); - const screen = summary.phase === "ready" ? summary.value : null; + const screen = summary.phase === "ready" && summaryApp === appId ? summary.value : null; + const listed = page.phase === "ready" && loadedScope === scope ? page.value : null; const editable = canEdit === true && screen?.canEdit === true; - const loadMore = async () => { - if (page.phase !== "ready" || !page.value.cursor) return; - setLoadingMore(true); - setMoreError(""); - try { - const response = await fetch(listQuery(page.value.cursor), { credentials: CREDENTIALS }); - const next = parsePage(await readJson(response, "More SKUs could not be loaded.")); - if (!alive.current) return; - setPage((current) => current.phase !== "ready" ? current : { phase: "ready", value: { - ...current.value, items: [...current.value.items, ...next.items], cursor: next.cursor, - } }); - } catch (reason) { - if (alive.current) setMoreError(failure(reason, "More SKUs could not be loaded.")); - } finally { - if (alive.current) setLoadingMore(false); - } + const turnPage = (next: string | null) => { + if (!loadingPage) setNavigation({ scope, cursor: next }); }; const refresh = async () => { @@ -677,8 +725,21 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView setRefreshError(""); try { const response = await fetch(url("/refresh"), { method: "POST", credentials: CREDENTIALS }); - await readJson(response, "The aged stock view could not be refreshed."); - if (alive.current) setAttempt((count) => count + 1); + const queued = await readJson(response, "The aged stock view could not be refreshed.") as { requestId?: string }; + const deadline = Date.now() + 120000; + while (alive.current) { + if (Date.now() >= deadline) throw new Error("The refresh is still running. Try again shortly."); + await new Promise((resolve) => window.setTimeout(resolve, 1000)); + if (!alive.current) return; + const state = await readJson(await fetch(url("/refresh"), { credentials: CREDENTIALS }), + "The refresh status could not be read.") as { state?: string; requestId?: string; message?: string }; + if (state.requestId !== queued.requestId) throw new Error("Another refresh started. Reload the view."); + if (state.state === "failed") throw new Error(state.message || "The aged stock view could not be refreshed."); + if (state.state === "done") { + setAttempt((count) => count + 1); + break; + } + } } catch (reason) { if (alive.current) setRefreshError(failure(reason, "The aged stock view could not be refreshed.")); } finally { @@ -694,39 +755,33 @@ export default function AgedStockView({ application, canEdit }: AgedStockAppView setOpenedId(null); opener.current?.focus(); }, []); - /** A stored change lands on the row in both lists, from the server's answer. */ + /** A stored change updates only the displayed page. */ const saved = useCallback((stored: Row) => { const patch = (row: Row) => (row.id === stored.id ? { ...row, action: stored.action, owner: stored.owner, status: stored.status, due: stored.due } : row); - setSummary((current) => current.phase !== "ready" ? current - : { phase: "ready", value: { ...current.value, nextTier: current.value.nextTier.map(patch) } }); setPage((current) => current.phase !== "ready" ? current : { phase: "ready", value: { ...current.value, items: current.value.items.map(patch) } }); }, []); const pickTab = (next: string) => { setTab(next); + setNavigation({ scope: "", cursor: null }); setOpenedId(null); }; - const needle = q.toLowerCase(); - // The Next tier list is the server's whole list, never a page, so filtering it here filters every row. - const nextRows = (screen?.nextTier ?? []).filter((row) => !needle - || [row.sku, row.fnsku, row.title].some((value) => value.toLowerCase().includes(needle))); - const rows: Row[] = itemsTab ? (page.phase === "ready" ? page.value.items : []) : nextRows; + const rows = listed?.items ?? []; const tabCount = (name: string) => { if (!screen) return null; - if (name === NEXT_TIER) return screen.nextTier.length; + if (name === NEXT_TIER) return screen.nextTierCount; if (name === ALL) return Object.values(screen.statusCounts).reduce((sum, n) => sum + n, 0); return screen.actionCounts[name] ?? 0; }; const openedRow = openedId ? rows.find((row) => row.id === openedId) ?? null : null; - const owners = page.phase === "ready" ? page.value.owners : []; - const listLoading = itemsTab && page.phase === "loading"; + const owners = listed?.owners ?? []; - return
+ return
- {summary.phase === "loading" && <> + {!screen && loadingSummary && <> - {listLoading && } - {itemsTab && page.phase === "error" - && setAttempt((count) => count + 1)} />} - {!listLoading && !(itemsTab && page.phase === "error") && (rows.length === 0 - ?

{EMPTY_TAB}

- : )} + {loadingPage && !listed && } + {pageError && setAttempt((count) => count + 1)} />} +
+ {listed && (rows.length === 0 + ?

{EMPTY_TAB}

+ : )} +
- {moreError &&

{moreError}

} - {itemsTab && page.phase === "ready" && page.value.cursor &&
- -
} + } } diff --git a/web/src/apps/amazon-operator/AppView.tsx b/web/src/apps/amazon-operator/AppView.tsx index 17b911bd61e483642895e2fb4670f81fbe48fded..8e49589057e1f1792f4c563d1e096afe6731ae6f 100644 --- a/web/src/apps/amazon-operator/AppView.tsx +++ b/web/src/apps/amazon-operator/AppView.tsx @@ -40,7 +40,9 @@ interface Proposal { interface Held { adType: string; campaign: string; sku: string; asin: string; reason: string } -interface TodayScreen { dataDate: string; items: Proposal[]; appItems: Proposal[]; held: Held[]; canEdit: boolean } +interface Page { total: number; open: number; offset: number; cursor: string | null } +interface TodayScreen { dataDate: string; items: Proposal[]; appItems: Proposal[]; held: Held[]; canEdit: boolean; + pages: { items: Page; apps: Page; held: Page } } interface HistoryPage { items: Proposal[]; cursor: string | null; total: number | null } @@ -83,6 +85,12 @@ const proposals = (raw: unknown) => (Array.isArray(raw) ? raw : []) function parseToday(raw: unknown): TodayScreen { const body = (raw ?? {}) as Record; + const pages = (body.pages ?? {}) as Record; + const page = (key: string): Page => { + const row = (pages[key] ?? {}) as Record; + return { total: num(row.total) ?? 0, open: num(row.open) ?? 0, offset: num(row.offset) ?? 0, + cursor: text(row.cursor) || null }; + }; return { dataDate: text(body.dataDate), items: proposals(body.items), @@ -93,6 +101,7 @@ function parseToday(raw: unknown): TodayScreen { asin: text(row.asin), reason: text(row.reason) }; }).filter((row) => row.reason), canEdit: body.canEdit === true, + pages: { items: page("items"), apps: page("apps"), held: page("held") }, }; } @@ -190,6 +199,8 @@ function Drawer({ row, onClose }: { row: Proposal; onClose: () => void }) { interface TodayProps { state: Load; + busy: boolean; + onPage: (section: "items" | "apps" | "held", offset: number) => void; editable: boolean; excluded: ReadonlySet; settled: ReadonlySet; @@ -358,18 +369,32 @@ function GroupHeading({ title, count }: { title: string; count: number }) { return

{title} {count}

; } +function Pager({ label, page, count, busy, onPage }: { + label: string; page: Page; count: number; busy: boolean; onPage: (offset: number) => void; +}) { + return ; +} + function TodayPanel(props: TodayProps) { const { state, editable, excluded, settled, sending, decideError, dismissing, onApprove, onDismissStart, onDismissReason, onDismissCancel, onDismiss, onRetry } = props; if (state.phase === "loading") return ; if (state.phase === "error") return ; - const { items, appItems, held, dataDate } = state.value; + const { items, appItems, held, dataDate, pages } = state.value; const adOpen = items.filter(isOpen); const appOpen = appItems.filter(isOpen); const open = [...adOpen, ...appOpen]; const batch = open.filter((row) => !excluded.has(row.id)); const empty = open.length === 0 && settled.size === 0 && !appItems.some((row) => row.status === "stale"); - return
+ return
{dataDate && Ads data for {dayWords(dataDate)}} {editable && open.length > 0 &&
@@ -386,15 +411,19 @@ function TodayPanel(props: TodayProps) {
{decideError &&

{decideError}

} {empty &&

Nothing to change today

} - {items.length > 0 &&
- +
+ -
} - {appItems.length > 0 &&
- + props.onPage("items", offset)} /> +
+
+ -
} - {held.length > 0 &&
+ props.onPage("apps", offset)} /> +
+

Held

    {held.map((row, index) =>
  • @@ -403,7 +432,9 @@ function TodayPanel(props: TodayProps) { {row.reason}
  • )}
-
} + props.onPage("held", offset)} /> +
; } @@ -413,12 +444,13 @@ interface HistoryProps { loadingMore: boolean; moreError: string; onFilter: (status: string) => void; - onMore: () => void; + offset: number; + onPage: (offset: number) => void; onOpen: (row: Proposal) => void; onRetry: () => void; } -function HistoryPanel({ state, filter, loadingMore, moreError, onFilter, onMore, onOpen, onRetry }: HistoryProps) { +function HistoryPanel({ state, filter, loadingMore, moreError, onFilter, offset, onPage, onOpen, onRetry }: HistoryProps) { return
; } -export default function AmazonOperatorView({ application, canEdit }: { +function OperatorBody({ application, canEdit }: { application: OperatorApplication; canEdit?: boolean; }) { const appId = application.id; @@ -476,6 +507,12 @@ export default function AmazonOperatorView({ application, canEdit }: { const [tab, setTab] = useState<"today" | "history">("today"); const [today, setToday] = useState>({ phase: "loading" }); const [todayAttempt, setTodayAttempt] = useState(0); + const [todayBusy, setTodayBusy] = useState(false); + const [todayError, setTodayError] = useState(""); + const [offsets, setOffsets] = useState({ items: 0, apps: 0, held: 0 }); + const [historyOffset, setHistoryOffset] = useState(0); + const [search, setSearch] = useState(""); + const [query, setQuery] = useState(""); const [history, setHistory] = useState>({ phase: "loading" }); const [historyAttempt, setHistoryAttempt] = useState(0); const [filter, setFilter] = useState(""); @@ -491,40 +528,55 @@ export default function AmazonOperatorView({ application, canEdit }: { const alive = useRef(true); useEffect(() => () => { alive.current = false; }, []); + useEffect(() => { + if (search === query) return; + const timer = window.setTimeout(() => { + setQuery(search); setOffsets({ items: 0, apps: 0, held: 0 }); setHistoryOffset(0); + }, 250); + return () => window.clearTimeout(timer); + }, [search, query]); + + useEffect(() => { + if (sending || dismissing || tab !== "today") return; + const timer = window.setInterval(() => setTodayAttempt((n) => n + 1), 30000); + return () => window.clearInterval(timer); + }, [sending, dismissing, tab]); + + useEffect(() => { setSettled(new Set()); setExcluded(new Set()); }, [offsets, query]); + useEffect(() => { const controller = new AbortController(); - setToday({ phase: "loading" }); - setSettled(new Set()); - void fetch(url("/today"), { credentials: CREDENTIALS, signal: controller.signal }) + setTodayBusy(true); + setTodayError(""); + void fetch(url("/today", { cursor: String(offsets.items), app_cursor: String(offsets.apps), + held_cursor: String(offsets.held), limit: String(HISTORY_PAGE), q: query }), { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "Today's proposals could not be loaded.")) .then((body) => { if (!controller.signal.aborted) setToday({ phase: "ready", value: parseToday(body) }); }) .catch((reason: unknown) => { - if (!controller.signal.aborted) setToday({ phase: "error", - message: reason instanceof Error ? reason.message : "Today's proposals could not be loaded." }); - }); + if (!controller.signal.aborted) setTodayError(reason instanceof Error ? reason.message : "Today's proposals could not be loaded."); + }).finally(() => { if (!controller.signal.aborted) setTodayBusy(false); }); return () => controller.abort(); - }, [url, todayAttempt]); + }, [url, todayAttempt, offsets, query]); useEffect(() => { const controller = new AbortController(); - setHistory({ phase: "loading" }); + setLoadingMore(true); setMoreError(""); - const params: Record = { limit: String(HISTORY_PAGE) }; + const params: Record = { limit: String(HISTORY_PAGE), cursor: String(historyOffset), q: query }; if (filter) params.status = filter; void fetch(url("/history", params), { credentials: CREDENTIALS, signal: controller.signal }) .then((response) => readJson(response, "History could not be loaded.")) .then((body) => { if (!controller.signal.aborted) setHistory({ phase: "ready", value: parseHistory(body) }); }) .catch((reason: unknown) => { - if (!controller.signal.aborted) setHistory({ phase: "error", - message: reason instanceof Error ? reason.message : "History could not be loaded." }); - }); + if (!controller.signal.aborted) setMoreError(reason instanceof Error ? reason.message : "History could not be loaded."); + }).finally(() => { if (!controller.signal.aborted) setLoadingMore(false); }); return () => controller.abort(); - }, [url, filter, historyAttempt]); + }, [url, filter, historyAttempt, historyOffset, query]); const screen = today.phase === "ready" ? today.value : null; const editable = canEdit === true && screen?.canEdit === true; const openCount = useMemo(() => (screen - ? screen.items.filter(isOpen).length + screen.appItems.filter(isOpen).length : null), [screen]); + ? screen.pages.items.open + screen.pages.apps.open : null), [screen]); const historyCount = history.phase === "ready" ? history.value.total ?? history.value.items.length : null; @@ -568,9 +620,19 @@ export default function AmazonOperatorView({ application, canEdit }: { if (status === "dismissed") return { ...item, status, note: note.trim() }; return { ...item, status: text(row.current) || item.status, notice: text(row.cause) || undefined }; }; - setToday((current) => current.phase !== "ready" ? current : { phase: "ready", value: { - ...current.value, items: current.value.items.map(settle), appItems: current.value.appItems.map(settle), - } }); + setToday((current) => { + if (current.phase !== "ready") return current; + const items = current.value.items.map(settle); + const appItems = current.value.appItems.map(settle); + const pages = current.value.pages; + return { phase: "ready", value: { ...current.value, items, appItems, pages: { + ...pages, + items: { ...pages.items, open: Math.max(0, pages.items.open + - current.value.items.filter(isOpen).length + items.filter(isOpen).length) }, + apps: { ...pages.apps, open: Math.max(0, pages.apps.open + - current.value.appItems.filter(isOpen).length + appItems.filter(isOpen).length) }, + } } }; + }); setSettled((current) => new Set([...current, ...ids])); setDismissing(null); setHistoryAttempt((count) => count + 1); @@ -581,26 +643,6 @@ export default function AmazonOperatorView({ application, canEdit }: { } }; - const loadMore = async () => { - if (history.phase !== "ready" || !history.value.cursor) return; - setLoadingMore(true); - setMoreError(""); - const params: Record = { limit: String(HISTORY_PAGE), cursor: history.value.cursor }; - if (filter) params.status = filter; - try { - const response = await fetch(url("/history", params), { credentials: CREDENTIALS }); - const page = parseHistory(await readJson(response, "More history could not be loaded.")); - if (!alive.current) return; - setHistory((current) => current.phase !== "ready" ? current : { phase: "ready", value: { - items: [...current.value.items, ...page.items], cursor: page.cursor, total: current.value.total, - } }); - } catch (reason) { - if (alive.current) setMoreError(reason instanceof Error ? reason.message : "More history could not be loaded."); - } finally { - if (alive.current) setLoadingMore(false); - } - }; - const openRow = (row: Proposal, from: "today" | "history") => { opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; setOpened({ row, from }); @@ -623,9 +665,16 @@ export default function AmazonOperatorView({ application, canEdit }: { {label}{count !== null && {count}} )}
+
+ {todayError && tab === "today" && setTodayAttempt((n) => n + 1)} />} {tab === "today" - ? { + setOffsets((current) => ({ ...current, [section]: offset })); setDismissing(null); + }} editable={editable} excluded={excluded} settled={settled} sending={sending || todayBusy} decideError={decideError} dismissing={dismissing} onToggle={toggle} onToggleAll={toggleAll} onApprove={(batch) => void decide("approve", batch.map((row) => row.id))} onDismissStart={(id) => setDismissing({ id, reason: "" })} @@ -635,10 +684,17 @@ export default function AmazonOperatorView({ application, canEdit }: { onOpen={(row) => openRow(row, "today")} onRetry={() => setTodayAttempt((count) => count + 1)} /> : void loadMore()} onOpen={(row) => openRow(row, "history")} + offset={historyOffset} onFilter={(status) => { setFilter(status); setHistoryOffset(0); }} + onPage={setHistoryOffset} onOpen={(row) => openRow(row, "history")} onRetry={() => setHistoryAttempt((count) => count + 1)} />}
{openedRow && }
; } + +export default function AmazonOperatorView({ application, canEdit }: { + application: OperatorApplication; canEdit?: boolean; +}) { + return ; +} diff --git a/web/src/apps/catalog/CatalogApp.tsx b/web/src/apps/catalog/CatalogApp.tsx index 677f940512365335f71eb80c6ad4f03d92d2f4a6..4002664e568e471a1201a689fcdad46d243acaf0 100644 --- a/web/src/apps/catalog/CatalogApp.tsx +++ b/web/src/apps/catalog/CatalogApp.tsx @@ -1,4 +1,4 @@ -import { useEffect } from "react"; +import { useEffect, useState } from "react"; import { CatalogApplicationView } from "../../customer-grid/CatalogView"; import type { CatalogApplicationDefinition, CatalogViewSpec } from "../../customer-grid/CatalogView"; import type { CatalogSpec, Field, Row } from "../../customer-grid/types"; @@ -9,7 +9,7 @@ interface CatalogAppProps { definition: CatalogApplicationDefinition; database: string; scope: CatalogReadScope; - onOpen: (pid: number) => void; + onOpen: (pid: number, row?: Row) => void; onData?: (fields: Field[], rows: Row[]) => void; editor: { catalogs: CatalogSpec[]; @@ -20,35 +20,36 @@ interface CatalogAppProps { } export function CatalogApp({ definition, database, scope, onOpen, onData, editor }: CatalogAppProps) { - const data = useCatalogData(database, scope); + const data = useCatalogData(database, scope, [definition.fieldMap.key, definition.fieldMap.title]); + const [search, setSearch] = useState(""); useEffect(() => { if (!data.loading && !data.error) onData?.(data.fields, data.allRows); }, [data.loading, data.error, data.fields, data.allRows, onData]); - // The Catalogs list reads its own records, so it paints while rows load. Only the editor waits. - if (data.loading) { - return
- -
; - } - if (data.error) { - return
- {data.error} - -
; - } const fieldByKey = new Map(data.fields.map((field) => [field.key, field])); const identityField = fieldByKey.get(definition.fieldMap.key); - if (!identityField || !fieldByKey.has(definition.fieldMap.title) || - (definition.fieldMap.image && !fieldByKey.has(definition.fieldMap.image))) { - return
- The Catalog field mapping is unavailable. - -
; - } + const mapped = identityField && fieldByKey.has(definition.fieldMap.title) && + (!definition.fieldMap.image || fieldByKey.has(definition.fieldMap.image)); + const error = data.error || (!data.loading && !mapped ? "The Catalog field mapping is unavailable." : ""); const filteredCodes = data.rows.map((row) => String(row[definition.fieldMap.key] ?? "").trim()).filter(Boolean); - return
- + return
+
{ + event.preventDefault(); data.setSearch(search.trim()); + }}> + setSearch(event.target.value)} /> + + + {data.loading ? "Loading products" : `Product page ${data.pageNumber}`} + +
+ {error &&
{error} +
} +
+ +
; } diff --git a/web/src/apps/catalog/useCatalogData.ts b/web/src/apps/catalog/useCatalogData.ts index e94d106323a74709cd91ee03b18fbded8d5c5148..e3cd0bfb7532ceca508409c64f0177df4e3c2d8c 100644 --- a/web/src/apps/catalog/useCatalogData.ts +++ b/web/src/apps/catalog/useCatalogData.ts @@ -12,6 +12,8 @@ export interface CatalogData { rows: Row[]; allRows: Row[]; returned: number; + nextCursor: string | null; + limit: number; } async function documentProperties(document: TypedOpenApi, database: string): Promise { @@ -51,17 +53,17 @@ function projectRow(record: QueryResponse["records"][number], properties: Proper return row; } -const FALLBACK_PAGE_SIZE = 500; -// The server's last reported page cap. Its cursor binds the limit, so one walk keeps one size. +const FALLBACK_PAGE_SIZE = 50; +// A cursor keeps its negotiated request size across page changes. let serverPageSize = FALLBACK_PAGE_SIZE; function pageCap(value: unknown): number | null { return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : null; } -/** Reads the first page at the server's own cap, restarting once if that cap differs. */ +/** Negotiate a bounded request size without changing an existing cursor. */ async function firstPage(database: string, body: QueryRequest, signal: AbortSignal) { - let limit = serverPageSize; + let limit = Math.min(body.limit, serverPageSize); for (let attempt = 0; ; attempt += 1) { signal.throwIfAborted(); let response: QueryResponse; @@ -78,51 +80,13 @@ async function firstPage(database: string, body: QueryRequest, signal: AbortSign } const cap = pageCap(response.caps?.max_page_size); if (cap !== null) serverPageSize = cap; - if (attempt || cap === null || cap <= limit || !response.page?.has_more) return { response, limit }; - limit = cap; + return { response, limit }; } } -async function walkPages(database: string, body: QueryRequest, properties: Properties, signal: AbortSignal, - onPage?: (read: number) => void) { - const rows: Row[] = []; - const seen = new Set(); - let fields: Field[] | undefined; - let cursor: string | null = null; - let returned = 0; - let limit = 0; - do { - signal.throwIfAborted(); - let response: QueryResponse; - if (cursor === null) ({ response, limit } = await firstPage(database, body, signal)); - else response = await typedQuery(database, { ...body, limit, cursor }, signal); - signal.throwIfAborted(); - if (response.database !== database || !Array.isArray(response.records) || - response.page?.returned !== response.records.length) { - throw new Error("The Catalog page could not be confirmed."); - } - if (!fields) { - fields = response.fields.filter((field) => Object.hasOwn(properties, field.key)).map((field) => ({ - ...field, - label: field.label ?? field.key, - type: field.type as Field["type"], - source: field.source === "odoo" ? "odoo" : "overlay", - })); - } - rows.push(...response.records.map((record) => projectRow(record, properties))); - returned += response.page.returned; - onPage?.(rows.length); - if (!response.page.has_more) break; - cursor = response.page.next_cursor; - if (!cursor || seen.has(cursor) || response.page.returned === 0) { - throw new Error("The Catalog cursor did not advance. Retry the request."); - } - seen.add(cursor); - } while (true); - return { fields: fields ?? [], rows, returned }; -} +type FilterTree = Extract, { conj: "and" | "or" }>; -function typedFilters(scope: CatalogReadScope): QueryRequest["filters"] { +function typedFilters(scope: CatalogReadScope): FilterTree | null { if (!scope.filters.length) return null; function node(value: FilterNode, depth: number): unknown { if (isFilterGroup(value)) { @@ -135,53 +99,127 @@ function typedFilters(scope: CatalogReadScope): QueryRequest["filters"] { const { colId, op, id, rhs, value: operand, value2, values } = value; return { colId, op, id, rhs, value: operand, value2, values }; } - return { conj: scope.filterConj ?? "and", nodes: scope.filters.map((value) => node(value, 1)) } as QueryRequest["filters"]; + return { conj: scope.filterConj ?? "and", nodes: scope.filters.map((value) => node(value, 1)) } as FilterTree; +} + +export interface CatalogPageRequest { + cursor?: string | null; + limit?: number; + search?: string; + searchFields?: string[]; + priorCursors?: (string | null)[]; } export async function loadCatalogData(database: string, scope: CatalogReadScope, signal: AbortSignal, - onProgress?: (read: number) => void): Promise { + page: CatalogPageRequest = {}): Promise { const properties = await documentProperties(await typedOpenApi(signal), database); signal.throwIfAborted(); - // The limit sizes each page and firstPage replaces it with the server cap. The walk reads every row. + const filters = typedFilters(scope); + const search = page.search?.trim(); + const searchNodes = search ? [...new Set(page.searchFields)].filter(Boolean).map((colId) => + ({ colId, op: "contains", value: search })) : []; + if (search && !searchNodes.length) throw new Error("Choose a search field."); const body: QueryRequest = { - limit: FALLBACK_PAGE_SIZE, - filters: typedFilters(scope), + limit: page.limit ?? FALLBACK_PAGE_SIZE, + filters: searchNodes.length ? { conj: "and", nodes: [ + ...(filters ? [{ conj: filters.conj, children: filters.nodes }] : []), + { conj: "or", children: searchNodes }, + ] } as QueryRequest["filters"] : filters, sorts: scope.sorts.map((sort) => ({ field: sort.colId, direction: sort.dir })), }; - const filtered = await walkPages(database, body, properties, signal, onProgress); - // Saved print placements join the complete permitted pool, before the display filter. - const pool = scope.filters.length - ? await walkPages(database, { ...body, filters: null }, properties, signal, - onProgress && ((read) => onProgress(filtered.rows.length + read))) - : filtered; - return { ...filtered, allRows: pool.rows }; + const cursor = page.cursor ?? null; + const { response, limit } = cursor === null ? await firstPage(database, body, signal) + : { response: await typedQuery(database, { ...body, cursor }, signal), limit: body.limit }; + signal.throwIfAborted(); + if (response.database !== database || !Array.isArray(response.records) || !Array.isArray(response.fields) || + response.page?.returned !== response.records.length || response.records.length > limit || + typeof response.page.has_more !== "boolean") { + throw new Error("The App page could not be confirmed."); + } + const nextCursor = response.page.has_more ? response.page.next_cursor : null; + if (response.page.has_more && (typeof nextCursor !== "string" || !nextCursor || nextCursor === cursor || + page.priorCursors?.includes(nextCursor) || !response.page.returned)) { + throw new Error("The App cursor did not advance. Retry the request."); + } + const fields = response.fields.filter((field) => Object.hasOwn(properties, field.key)).map((field) => ({ + ...field, label: field.label ?? field.key, type: field.type as Field["type"], + source: field.source === "odoo" ? "odoo" : "overlay", + })); + const rows = response.records.map((record) => projectRow(record, properties)); + return { fields, rows, allRows: rows, returned: response.page.returned, nextCursor, limit }; } -// progress counts rows read so far, so a loading state can show the walk moving. type CatalogState = CatalogData & { key: string; loading: boolean; error: string; progress: number }; -const EMPTY: CatalogData = { fields: [], rows: [], allRows: [], returned: 0 }; +const EMPTY: CatalogData = { fields: [], rows: [], allRows: [], returned: 0, nextCursor: null, limit: FALLBACK_PAGE_SIZE }; -export function useCatalogData(database: string, scope: CatalogReadScope) { +export function useCatalogData(database: string, scope: CatalogReadScope, searchFields: string[] = []) { + const [search, setSearch] = useState(""); const scopeJson = JSON.stringify(scope); - const key = `${database}::${scopeJson}`; + const searchJson = JSON.stringify(searchFields); + const queryKey = `${database}::${scopeJson}::${searchJson}::${search}`; + const [navigation, setNavigation] = useState({ key: queryKey, index: 0, cursors: [null] as (string | null)[], limit: FALLBACK_PAGE_SIZE }); + const page = navigation.key === queryKey ? navigation + : { key: queryKey, index: 0, cursors: [null], limit: FALLBACK_PAGE_SIZE }; + const cursor = page.cursors[page.index]; + const key = `${queryKey}::${page.index}::${cursor}`; const [attempt, setAttempt] = useState(0); const [state, setState] = useState(() => ({ ...EMPTY, key, loading: true, error: "", progress: 0 })); - const retry = useCallback(() => setAttempt((value) => value + 1), []); + const retry = useCallback(() => { + setNavigation({ key: queryKey, index: 0, cursors: [null], limit: FALLBACK_PAGE_SIZE }); + setAttempt((value) => value + 1); + }, [queryKey]); useEffect(() => { const controller = new AbortController(); - setState({ ...EMPTY, key, loading: true, error: "", progress: 0 }); - const progress = (read: number) => { - if (!controller.signal.aborted) setState((live) => live.key === key && live.loading ? { ...live, progress: read } : live); - }; - void loadCatalogData(database, JSON.parse(scopeJson) as CatalogReadScope, controller.signal, progress).then((data) => { + setState((live) => ({ ...EMPTY, fields: live.fields, key, loading: true, error: "", progress: 0 })); + void loadCatalogData(database, JSON.parse(scopeJson) as CatalogReadScope, controller.signal, { + cursor, limit: page.limit, search, searchFields: JSON.parse(searchJson), priorCursors: page.cursors.slice(0, page.index + 1), + }).then((data) => { if (!controller.signal.aborted) setState({ ...data, key, loading: false, error: "", progress: data.allRows.length }); }).catch((error: unknown) => { - if (!controller.signal.aborted) setState({ ...EMPTY, key, loading: false, progress: 0, - error: error instanceof Error ? error.message : "The Catalog could not be loaded." }); + if (!controller.signal.aborted) setState((live) => ({ ...EMPTY, fields: live.fields, key, loading: false, progress: 0, + error: error instanceof Error ? error.message : "The App could not be loaded." })); }); return () => controller.abort(); - }, [database, scopeJson, key, attempt]); + }, [database, scopeJson, searchJson, search, key, cursor, page.limit, attempt]); - return { ...(state.key === key ? state : { ...EMPTY, key, loading: true, error: "", progress: 0 }), retry }; + const live = state.key === key ? state : { ...EMPTY, fields: state.fields, key, loading: true, error: "", progress: 0 }; + return { ...live, retry, search, setSearch, pageNumber: page.index + 1, hasPrevious: page.index > 0, + previous: () => { if (!live.loading && page.index > 0) setNavigation({ ...page, index: page.index - 1 }); }, + next: () => { if (!live.loading && !live.error && live.nextCursor) setNavigation({ key: queryKey, + index: page.index + 1, cursors: [...page.cursors.slice(0, page.index + 1), live.nextCursor], limit: live.limit }); }, + }; +} + +/** Resolve only saved product codes, independently of the visible search and filters. */ +export async function loadCatalogPlacements(database: string, key: string, codes: string[], signal: AbortSignal): Promise { + const rows: Row[] = []; + for (const code of new Set(codes)) { + const scope: CatalogReadScope = { filters: [{ id: "placement", colId: key, op: "eq", value: code }], filterConj: "and", sorts: [] }; + const page = await loadCatalogData(database, scope, signal); + if (page.nextCursor || page.rows.length !== 1 || String(page.rows[0][key]) !== code) throw new Error(key === "pid" + ? "A selected customer could not be confirmed." : `The saved product ${code} could not be confirmed.`); + rows.push(page.rows[0]); + } + return rows; +} + +export function useCatalogPlacements(database: string, key: string, codes: string[]) { + const codesJson = JSON.stringify([...new Set(codes)].sort()); + const requestKey = `${database}::${key}::${codesJson}`; + const [attempt, setAttempt] = useState(0); + const [state, setState] = useState({ key: "", rows: [] as Row[], loading: true, error: "" }); + useEffect(() => { + const controller = new AbortController(); + setState({ key: requestKey, rows: [], loading: true, error: "" }); + void loadCatalogPlacements(database, key, JSON.parse(codesJson), controller.signal).then((rows) => { + if (!controller.signal.aborted) setState({ key: requestKey, rows, loading: false, error: "" }); + }).catch((error: unknown) => { + if (!controller.signal.aborted) setState({ key: requestKey, rows: [], loading: false, + error: error instanceof Error ? error.message : "Saved products could not be loaded." }); + }); + return () => controller.abort(); + }, [database, key, codesJson, requestKey, attempt]); + return { ...(state.key === requestKey ? state : { rows: [], loading: true, error: "" }), + retry: () => setAttempt((value) => value + 1) }; } diff --git a/web/src/apps/competitor-watch/CompetitorWatchView.tsx b/web/src/apps/competitor-watch/CompetitorWatchView.tsx index 4593837ebfcbf9005c11dfa3a29c14e17367130c..0334195a74bb1d9ccd1a9590d079e7ba429fd330 100644 --- a/web/src/apps/competitor-watch/CompetitorWatchView.tsx +++ b/web/src/apps/competitor-watch/CompetitorWatchView.tsx @@ -46,6 +46,7 @@ interface TrendSeries { interface TermScreen { canEdit: boolean; + termsReady: boolean; terms: string[]; termTotal: number; termOffset: number; @@ -68,23 +69,32 @@ interface Alert { owner: string; status: string; due: string; + createdAt: string; cause?: string; } interface Check { term: string; asin: string; trigger: string; cause: string } -interface AlertScreen { canEdit: boolean; alerts: Alert[]; checks: Check[] } +interface RowPage { total: number; offset: number; limit: number } +interface AlertScreen { canEdit: boolean; alerts: Alert[]; checks: Check[]; alertPage: RowPage; checkPage: RowPage } type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; value: T }; const text = (value: unknown) => (typeof value === "string" ? value : ""); const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null); +class ScreenReadError extends Error { + readonly screen: unknown; + constructor(message: string, screen: unknown) { super(message); this.screen = screen; } +} + async function readJson(response: Response, fallback: string): Promise { const body: unknown = await response.json().catch(() => null); if (response.ok) return body; - const error = (body as { detail?: { error?: { message?: unknown } } } | null)?.detail?.error; - throw new Error(text(error?.message) || fallback); + type Refusal = { message?: unknown; details?: { screen?: unknown } }; + const envelope = body as { error?: Refusal; detail?: { error?: Refusal } } | null; + const error = envelope?.error ?? envelope?.detail?.error; + throw new ScreenReadError(text(error?.message) || fallback, error?.details?.screen); } function parseWeek(raw: unknown): Week | null { @@ -98,6 +108,7 @@ function parseTermScreen(raw: unknown): TermScreen { const trend = Array.isArray(body.trend) ? body.trend : []; return { canEdit: body.canEdit === true, + termsReady: body.termsReady !== false, terms: Array.isArray(body.terms) ? body.terms.map(text).filter(Boolean) : [], termTotal: num(body.termTotal) ?? 0, termOffset: num(body.termOffset) ?? 0, @@ -142,9 +153,14 @@ function parseAlerts(raw: unknown): AlertScreen { const body = (raw ?? {}) as Record; const rows = Array.isArray(body.alerts) ? body.alerts : []; const checks = Array.isArray(body.checks) ? body.checks : []; + const page = (raw: unknown): RowPage => { + const row = (raw ?? {}) as Record; + return { total: num(row.total) ?? 0, offset: num(row.offset) ?? 0, limit: num(row.limit) ?? 50 }; + }; return { canEdit: body.canEdit === true, alerts: rows.map((item) => parseAlert(item)).filter((row): row is Alert => row !== null), + alertPage: page(body.alertPage), checkPage: page(body.checkPage), checks: checks.map((item) => { const row = (item ?? {}) as Record; return { term: text(row.term), asin: text(row.asin), trigger: text(row.trigger), cause: text(row.cause) }; @@ -159,6 +175,7 @@ function parseAlert(item: unknown): Alert | null { return { id, term: text(row.term), asin: text(row.asin), trigger: text(row.trigger), owner: text(row.owner), status: text(row.status), due: text(row.due), + createdAt: text(row.createdAt), cause: text(row.cause) || undefined, }; } @@ -293,15 +310,17 @@ function TermsView({ appId, canEditApp }: { appId: string; canEditApp: boolean } const [query, setQuery] = useState(""); const [offset, setOffset] = useState(0); const [state, setState] = useState>({ phase: "loading" }); + const [screen, setScreen] = useState(null); const [attempt, setAttempt] = useState(0); const [draft, setDraft] = useState(""); const [termError, setTermError] = useState(""); const [busy, setBusy] = useState(false); useEffect(() => { + if (search === query) return; const timer = setTimeout(() => { setQuery(search); setOffset(0); }, 200); return () => clearTimeout(timer); - }, [search]); + }, [search, query]); useEffect(() => { const controller = new AbortController(); @@ -314,18 +333,32 @@ function TermsView({ appId, canEditApp }: { appId: string; canEditApp: boolean } if (controller.signal.aborted) return; const value = parseTermScreen(body); displayedTerm.current = value.term ?? ""; + setScreen(value); setState({ phase: "ready", value }); }) .catch((reason: unknown) => { - if (!controller.signal.aborted) setState({ phase: "error", + if (controller.signal.aborted) return; + const diagnostic = reason instanceof ScreenReadError && reason.screen ? parseTermScreen(reason.screen) : null; + if (diagnostic) displayedTerm.current = diagnostic.term ?? ""; + setScreen(diagnostic); + setState({ phase: "error", message: reason instanceof Error ? reason.message : "The competitor watch could not be loaded." }); }); return () => controller.abort(); }, [url, term, query, offset, attempt]); - const screen = state.phase === "ready" ? state.value : null; const canEdit = canEditApp && screen?.canEdit === true; - const selected = screen?.term ?? ""; + const loading = state.phase === "loading"; + const selected = loading ? term || screen?.term || "" : screen?.term ?? ""; + const previousUnavailable = !screen?.termsReady || screen.termOffset === 0; + const nextUnavailable = !screen?.termsReady || screen.termOffset + screen.terms.length >= screen.termTotal; + const diagnosticError = state.phase === "error" && screen?.problems.includes(state.message); + + const movePage = (direction: -1 | 1) => { + if (!screen || loading || (direction < 0 ? previousUnavailable : nextUnavailable)) return; + setOffset(Math.max(0, screen.termOffset + direction * screen.termLimit)); + setState({ phase: "loading" }); + }; const editTerms = async (action: "add" | "remove", value: string) => { setBusy(true); @@ -338,7 +371,7 @@ function TermsView({ appId, canEditApp }: { appId: string; canEditApp: boolean } await readJson(response, action === "add" ? "The term was not added." : "The term was not removed."); setDraft(""); if (action === "add") setTerm(value.trim().toLowerCase()); - else setTerm(""); + else { setTerm(""); displayedTerm.current = ""; } setAttempt((count) => count + 1); } catch (reason) { setTermError(reason instanceof Error ? reason.message : "The term was not saved."); @@ -353,27 +386,28 @@ function TermsView({ appId, canEditApp }: { appId: string; canEditApp: boolean } return
-
+ {kpis && <> + setSummaryNavigation({ scope: summaryScope, cursor: next })} /> + {(kpis.cursor || kpis.previousCursor) && kpis.repeatMonths.length > 0 + &&

Repeat months: {kpis.repeatMonths.join(", ")}

} + } {summary.phase === "error" && setSummaryAttempt((count) => count + 1)} />} {kpis?.empty &&

{kpis.emptyMessage || EMPTY_PERIOD}

} @@ -676,17 +768,14 @@ export default function ReturnsRepeatView({ application, canEdit }: ReturnsAppVi {screen && rowsShown === 0 && !q && !(kpis?.empty && segment !== "repeat") &&

{EMPTY_SEGMENT[segment]}

} {screen && rowsShown > 0 && - {rowsShown < screen.total ? `${qty(rowsShown)} of ${qty(screen.total)}` : `${qty(screen.total)}`} + {`${qty(screen.offset + 1)} to ${qty(screen.offset + rowsShown)} of ${qty(screen.total)}`} {screen.total === 1 ? (segment === "repeat" ? " ASIN" : " SKU") : (segment === "repeat" ? " ASINs" : " SKUs")} } {screen && rowsShown > 0 && segment !== "repeat" && } {screen && rowsShown > 0 && segment === "repeat" && } - {moreError &&

{moreError}

} - {screen?.cursor &&
- -
} + {screen && { setNavigation({ scope, cursor: next }); setOpened(null); }} />}
{opened && letter.toUpperCase()); } +function automationFailure(error: unknown, fallback: string): string { + return error instanceof AutomationError ? error.message : fallback; +} + type AutomationOpenRequest = AutomationOpenDetail & { requestId: number }; let pendingOpen: AutomationOpenRequest | null = null; let openSequence = 0; @@ -214,10 +218,9 @@ function useRefusalToast() { useEffect(() => { if (!messages.length) return; const through = messages[messages.length - 1].id; - for (const message of messages) { - if (message.id <= delivered.current) continue; - delivered.current = message.id; - const text = message.text; + for (const { id, text } of messages) { + if (id <= delivered.current) continue; + delivered.current = id; window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } })); } setMessages((current) => current.filter((message) => message.id > through)); @@ -381,11 +384,7 @@ export default function AutomationSurface() { } catch (e) { if ((e as Error)?.name === "AbortError") return null; if (mine === gen.current) - setError( - e instanceof AutomationError - ? e.message - : "The automation service did not answer." - ); + setError(automationFailure(e, "The automation service did not answer.")); return null; } }, []); @@ -426,11 +425,7 @@ export default function AutomationSurface() { }) .catch((reason) => { if ((reason as Error)?.name === "AbortError") return; - setVoiceLoadError( - reason instanceof AutomationError - ? reason.message - : "Voice configuration could not be loaded." - ); + setVoiceLoadError(automationFailure(reason, "Voice configuration could not be loaded.")); }); return () => ac.abort(); }, []); @@ -468,7 +463,7 @@ export default function AutomationSurface() { if (e instanceof AutomationError && [401, 403, 404].includes(e.status)) { refuse(e.message); } else { - setError(e instanceof AutomationError ? e.message : "The automation was not created."); + setError(automationFailure(e, "The automation was not created.")); } } finally { creatingRef.current = false; @@ -622,8 +617,7 @@ export default function AutomationSurface() { setSelectedDetail({ id, automation: result.automation, error: "" }); }).catch((reason) => { if (!isCurrent() || (reason as Error)?.name === "AbortError") return; - setSelectedDetail({ id, automation: null, error: reason instanceof AutomationError - ? reason.message : "The automation service did not answer." }); + setSelectedDetail({ id, automation: null, error: automationFailure(reason, "The automation service did not answer.") }); }); return () => { controller.abort(); @@ -639,6 +633,7 @@ export default function AutomationSurface() { !railNeedle || label.toLowerCase().includes(railNeedle); const activeVoice = voiceAgents.find((agent) => agent.automationId === activeVoiceId) || null; const menuAutomation = items.find((a) => a.id === rowMenu?.id) || null; + const railToggleLabel = railShut ? "Expand automations" : "Minimize automations"; const onRailKey = (event: ReactKeyboardEvent) => { if (event.target instanceof HTMLInputElement) return; @@ -649,9 +644,8 @@ export default function AutomationSurface() { if (!rows.length) return; event.preventDefault(); const at = rows.indexOf(document.activeElement as HTMLButtonElement); - const next = event.key === "ArrowDown" ? (at + 1) % rows.length - : (at - 1 + rows.length) % rows.length; - rows[next < 0 ? 0 : next]?.focus(); + const next = (at + (event.key === "ArrowDown" ? 1 : rows.length - 1)) % rows.length; + rows[next]?.focus(); }; return ( @@ -669,9 +663,9 @@ export default function AutomationSurface() { }> - ({ text })), { text:
+ Connect account
setValues((current) => ({ ...current, [field.key]: event.target.value }))} /> {spec.fieldHints[field.key] ?

{spec.fieldHints[field.key]}

: null}
)} - {storedId || uncertainSave ? : null} -
}, - ...(hasSelection ? [{ text:
+ {hasSelection ?
Choose databases {spec.initialDatabases.map((database) => )} -
}] : []), - ...(error || connected ? [{ text: <> +
: null} + {storedId || uncertainSave ? : null} + }, + ]} /> + {error || connected ? <> {error ?

{error}

{storedId || uncertainSave ? : null}
: null} {connected ?

{spec.type === "stripe" || spec.type === "shopify" ? "Key stored" : spec.type === "odoo" ? "Server reachable" : "Connected"}

: null} - }] : []), - ]} /> + : null} ; } diff --git a/web/src/connectors/ConnectorModal.tsx b/web/src/connectors/ConnectorModal.tsx index b71159218094337e349acec82b76e44353e426f2..8a7b1a93625c64d59590553899682b76502d308c 100644 --- a/web/src/connectors/ConnectorModal.tsx +++ b/web/src/connectors/ConnectorModal.tsx @@ -6,7 +6,7 @@ // INTO it and Settings > Keychains becomes a link to the modal (one door). Field // enablement moves into the Databases tab."* // -// Overview and Keychain are the only tabs. Retired keys open Overview. +// Overview and Setup are the only tabs. Retired keys open Setup. // The C7 status reader stays mounted independently of the paged detail read. // // ⛔ NO REUSABLE MODAL COMPONENT EXISTS IN THIS REPO. `CustomerGrid.tsx`'s @@ -51,9 +51,14 @@ import "../styles/wave52/connectors.css"; import "../styles/wave55/connectors.css"; import "../styles/wave59/connectors.css"; import "../styles/sdlc74/connectors.css"; +import "../styles/sdlc80/connector-setup.css"; -/** Retired tab keys remain aliases for existing deep links to Overview. */ -export type ConnectorTab = "overview" | "keychain" | "databases" | "relationships" | "cadence"; +/** Retired tab keys remain aliases for existing deep links to Setup. */ +export type ConnectorTab = "overview" | "setup" | "keychain" | "databases" | "relationships" | "cadence"; + +export function connectorTab(tab: ConnectorTab): "overview" | "setup" { + return tab === "overview" ? "overview" : "setup"; +} /** Every connector keeps the same two tabs. */ export function connectorTabs( @@ -61,7 +66,7 @@ export function connectorTabs( ): ReadonlyArray<{ key: ConnectorTab; label: string }> { return [ { key: "overview", label: "Overview" }, - { key: "keychain", label: "Keychain" }, + { key: "setup", label: "Setup" }, ]; } @@ -72,7 +77,7 @@ interface ConnectorSyncCapability { } /** Discover sync support without reading database pages or duplicating the provider registry. */ -function useConnectorSyncCapability(connectorKey: string, refreshKey: number) { +function useConnectorSyncCapability(connectorKey: string, refreshKey: number, directoryRow: ConnectorRow) { const identity = JSON.stringify([connectorKey, refreshKey]); const [reading, setReading] = useState<{ identity: string; capability: ConnectorSyncCapability | null; problem: string; @@ -101,7 +106,12 @@ function useConnectorSyncCapability(connectorKey: string, refreshKey: number) { return () => controller.abort(); }, [connectorKey, identity, refreshKey]); const current = reading?.identity === identity ? reading : null; - return { capability: current?.capability ?? null, problem: current?.problem ?? "" }; + const declaredSupport = obj(directoryRow).syncSupported; + // Directory support starts a fresh read. Only this open's capability response grants Sync now. + const declaredCapability: ConnectorSyncCapability | null = directoryRow.key === connectorKey + && typeof declaredSupport === "boolean" + ? { provider: connectorKey, supported: declaredSupport, syncNow: null } : null; + return { capability: current ? current.capability : declaredCapability, problem: current?.problem ?? "" }; } /** Only a server-declared sync provider mounts the C7 reader. */ @@ -130,6 +140,10 @@ export interface ConnectorOverview { export interface ConnectorKeychainEntry { id: string; + connectionId?: string; + type?: string; + slot?: number; + status?: string; label: string; /** The masked preview, or "" when this reader is not allowed one (R4). */ masked: string; @@ -164,12 +178,14 @@ export interface ConnectorConnectSpec { /** The scopes THIS reader may choose, already narrowed by R4 and by `TENANT_WIDE_TYPES`. */ scopes: string[]; tenantWide: boolean; + defaultLabel?: string; } export interface ConnectorKeychainHalf { entries: ConnectorKeychainEntry[]; canConnect: boolean; canDisconnect: boolean; + migrationPending?: boolean; /** `null` when this connector has no credential door at all (Webhooks, the Scraper, OAuth). */ connect: ConnectorConnectSpec | null; } @@ -360,6 +376,10 @@ function parseKeychainHalf(raw: unknown): ConnectorKeychainHalf { if (!id) continue; entries.push({ id, + connectionId: str(entry.connectionId).trim() || id, + type: str(entry.type).trim(), + slot: typeof entry.slot === "number" ? entry.slot : undefined, + status: str(entry.status).trim(), label: str(entry.label).trim() || id, masked: str(entry.masked), scope: str(entry.scope).trim(), @@ -372,6 +392,7 @@ function parseKeychainHalf(raw: unknown): ConnectorKeychainHalf { entries, canConnect: source.canConnect === true, canDisconnect: source.canDisconnect === true, + migrationPending: source.migrationPending === true, connect: parseConnectSpec(source.connect), }; } @@ -406,6 +427,7 @@ export function parseConnectSpec(raw: unknown): ConnectorConnectSpec | null { // stored before this wave actually is, which is the same read default the scope bucket uses. scopes: scopes.length ? scopes : ["business"], tenantWide: source.tenantWide === true, + defaultLabel: str(source.defaultLabel).trim() || undefined, }; } @@ -905,7 +927,7 @@ export default function ConnectorModal({ }: { row: ConnectorRow; onClose: () => void; - /** W49-T13 opens this modal on the Keychain tab from Settings; every other door opens Overview. */ + /** Credential links land on Setup, including the retired tab aliases. */ initialTab?: ConnectorTab; /** Retained for callers that still pass the retired Databases-tab callback. */ onOpenDatabases?: () => void; @@ -914,7 +936,7 @@ export default function ConnectorModal({ const connectorKey = row.key; const panelRef = useRef(null); const closeRef = useRef(null); - const [tab, setTab] = useState(initialTab === "keychain" ? "keychain" : "overview"); + const [tab, setTab] = useState<"overview" | "setup">(() => connectorTab(initialTab)); const [loadKey, setLoadKey] = useState(connectorKey); const [reading, setLoad] = useState(() => { const cached = connectorDetailCache.get(connectorKey); @@ -925,6 +947,7 @@ export default function ConnectorModal({ const [databasePendingCount, setDatabasePendingCount] = useState(0); const [databaseTabBusy, setDatabaseTabBusy] = useState(false); const [databaseSaving, setDatabaseSaving] = useState(false); + const [setupNotice, setSetupNotice] = useState<{ key: string; message: string } | null>(null); const syncWasActive = useRef(false); const [pendingLeave, setPendingLeave] = useState(null); /** @@ -934,7 +957,7 @@ export default function ConnectorModal({ * so the effect below keys on this counter as well and every write bumps it. */ const [reads, setReads] = useState(0); - const syncSource = useConnectorSyncCapability(connectorKey, reads); + const syncSource = useConnectorSyncCapability(connectorKey, reads, row); const [syncReady, setSyncReady] = useState<{ connectorKey: string; refreshKey: number } | null>(null); const markSyncReady = useCallback((key: string, refreshKey: number) => { setSyncReady((current) => current?.connectorKey === key && current.refreshKey === refreshKey @@ -994,7 +1017,7 @@ export default function ConnectorModal({ setPendingLeave({ kind: "tab", tab: next }); return; } - setTab(next); + setTab(connectorTab(next)); }, [databasePendingCount, databaseSaving, databaseTabBusy, tab]); useEffect(() => { @@ -1202,6 +1225,9 @@ export default function ConnectorModal({ ? amazonRotationNotice(rotationEntry.client_secret_rotation_deadline, rotationNow) : null; const renderTab = useCallback(() => { + if (tab === "setup" && !isAdmin) { + return

Only an admin can manage connector keys.

; + } // ⛔ THE KEYCHAIN TAB IS BELOW THE LOAD GUARD, and the placeholder it replaces was above it. // C4 serves the credential rows on the SAME payload as the overview, so a keychain tab that // rendered before the read landed could only render an empty list — i.e. "nobody has @@ -1223,16 +1249,23 @@ export default function ConnectorModal({ ); } - if (tab === "keychain") { - return database.identity?.connectionId)?.identity?.connectionId} />; + if (tab === "setup") { + return <> + {setupNotice?.key === connectorKey ?

{setupNotice.message}

: null} + { + setSetupNotice(message ? { key: connectorKey, message } : null); + reread(); + }} + isAdmin={isAdmin} + connectionId={load.detail.databases.find((database) => database.identity?.connectionId)?.identity?.connectionId} /> + ; } if (tab === "overview") { return ( <> {rotationNotice ? setTab("keychain") : undefined} /> : null} + onRotate={isAdmin && rotationEntry?.replaceable && load.detail.keychain.connect + ? () => setTab("setup") : undefined} /> : null} {row.spawns_tables === true || row.key === "scraper" ? <> { if (!databasePendingCount || !databasesRef.current) return; @@ -1274,7 +1307,7 @@ export default function ConnectorModal({ discardDatabaseChanges(); setPendingLeave(null); if (destination.kind === "close") onClose(); - else setTab(destination.tab); + else setTab(connectorTab(destination.tab)); }, [discardDatabaseChanges, onClose, pendingLeave]); const expired = rotationNotice?.tone === "expired"; @@ -1359,7 +1392,7 @@ export default function ConnectorModal({ id="conn-tab-panel" role="tabpanel" aria-labelledby={`conn-tab-${tab}`} - aria-busy={tab === "keychain" && load.phase === "loading"} + aria-busy={tab === "setup" && isAdmin && load.phase === "loading"} >