"""amazon_store.py — the Reports-API loader for the tenant mirror, and the only place Amazon rows are written. Reports remain the bulk spine; Pricing and AWD use their exact read-only operations. python platform/harness/amazon_store.py --sync pull into tenant #0's mirror python platform/harness/amazon_store.py --sync --tenant gtmlab python platform/harness/amazon_store.py --status what is in the mirror now python platform/harness/amazon_store.py --purge --tenant gtmlab --confirm-tenant gtmlab python platform/harness/amazon_store.py --selftest parsers, typing, naming, validate ⭐ THE SHAPE, and it is deliberately the third instance of one pattern rather than a new one: `Odoo XML-RPC -> mirror -> odoo_relational -> ut_odoo_*`, `Meta Graph -> mirror -> meta_relational -> ut_meta_*`, and now `SP-API Reports -> mirror -> amazon_relational -> ut_amzn_*`. Same middle, same end, one different first hop. The mirror is ONE store with three loaders, never three stores. ⛔⛔ **NOT ONE COLUMN IN THIS FILE WAS TRANSCRIBED FROM A DOCUMENT, AND THAT IS THE WHOLE DESIGN.** `meta_store.py` measured Meta's 285 columns by asking the API what it accepted. SP-API cannot be asked: it has no introspection endpoint at all. So the columns come from one of two places, per `amazon_schema.PROVENANCE`: · **`schema_artifact`** — Amazon's own published JSON Schema, flattened mechanically and pinned with its blob sha (`pricing`, `brand_analytics`). · **`measured_at_sync`** — the report document's OWN HEADER ROW, read at sync time by `_measure` below. Amazon publishes no schema for any flat-file report, and line 1 of the document is the column list straight from the source. ⭐ The consequence worth stating: **a table with no artifact ships with NO columns until it first syncs.** That is not a limitation to apologise for, it is the alternative to inventing them. ⛔⛔ **ENTITY KEYS ARE `amzn_`-PREFIXED, AND THIS IS A NAMING RULE STANDING IN FOR A SCHEMA CHANGE.** `datastore._sync_state`'s primary key is `entity VARCHAR` with NO source column, so two sources syncing "orders" would overwrite each other's progress marker. Amazon orders is exactly that collision. Meta already shares the table and avoids it the same way. **Never write a bare entity name here.** ⛔ **STANDING RULE 1: NO CAP ON A CONNECTOR'S RECORDS.** `records_mutable('ut_amzn_*')` is False, so there is no `MAX_ROWS` on this path and there must not be. Where Amazon imposes a ceiling we cannot remove, it is REPORTED with its cause and a recommended fix rather than silently enforced: see `pricing`'s Buy Box PRICE split in `connectors_amazon.REPORT_TYPES`. ⛔ **NO DASHBOARD AND NO MODULE (R14).** This file ends at the mirror. `amazon_relational` spawns the grids, Data lists them, and Sheet binds them on the `ut_` prefix with no further work. """ from __future__ import annotations import argparse import csv import functools import hashlib import io import json import logging import re import os import shutil import sys import tempfile import time import threading import uuid from contextlib import closing, contextmanager from datetime import datetime, timedelta, timezone from decimal import Decimal, InvalidOperation from pathlib import Path _HERE = Path(__file__).resolve().parent if str(_HERE.parent) not in sys.path: sys.path.insert(0, str(_HERE.parent)) from harness import amazon_schema # noqa: E402 from harness import datastore # noqa: E402 from core import connector_spec # noqa: E402 #: ⛔ THE ENTITY-KEY PREFIX THAT KEEPS `_sync_state` COLLISION-FREE. See the header. PREFIX = "amzn_" #: The seven tables of owner ruling R4, in sync order. ⚠ `order_items` is derived from the SAME #: report document as `orders` rather than costing a second report: the flat-file order report is #: already one row per order ITEM, so orders is the deduplicated head and order_items the lines. TABLES = ("orders", "order_items", "listings", "settlements", "inventory", "pricing", "brand_analytics", "economics", "transactions", "fba_fee_estimates", "referral_fees", "storage_fees", "search_query_performance") MARGIN_TABLES = ("economics", "transactions", "fba_fee_estimates", "referral_fees", "storage_fees") SQP_TABLE = "search_query_performance" SQP_LIMIT_KEY = "amzn_limit_search_query_performance" #: A scheduled pass spends at most Amazon's createReport burst on SQP documents. SQP_PASS_REPORTS = 15 #: An SQP backlog resumes this long after its pass stops: 15 reports per 15 minutes is Amazon's sustained rate. SQP_RESUME_SECONDS = 900 GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES) TABLES += GROUP_A_TABLES #: Customer Feedback entity snapshots: weekly current state, no history window. FEEDBACK_TABLES = ("review_topics", "review_trends", "browse_node_review_topics") DERIVED_TABLES = ("sku_keywords", "keyword_ranks") #: Snapshot reports whose changes are kept as versions in `amzn__history`. HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES) HISTORY_TABLES = tuple(t + "_history" for t in HISTORY_SOURCES) DATABASE_TABLES = TABLES + DERIVED_TABLES + HISTORY_TABLES + ("competitor_snapshots", "competitor_images") PERIOD_TABLES = MARGIN_TABLES + GROUP_A_TABLES LEGACY_SNAPSHOT_TABLES = ("listings", "inventory") #: Head -> lines: both tables read the same document; a second report buys no new rows. SHARED_REPORT_SIBLINGS = {"orders": "order_items"} #: Contracts C2 and C5: called as `fn(tenant, synced_tables)` after a pass publishes. Each import is #: lazy and a failure (a missing App module too) is one log line, never a failed sync. AFTER_SYNC = ("modules.amazon_profit:on_amazon_sync", "modules.competitor_watch:on_amazon_sync", "modules.ranking_diagnostic:on_amazon_sync", "modules.restock_planner:on_amazon_sync", "modules.account_health:on_amazon_sync", "modules.aged_stock:on_amazon_sync", "modules.returns_refunds:on_amazon_sync", "modules.repeat_buyers:on_amazon_sync", "modules.opportunity_finder:on_amazon_sync") #: S82-T11: a publication queues its synced tables here and `run_after_sync_loop` drains them per App. AFTER_SYNC_QUEUE_KEY = "amazon_after_sync_queue" #: Minutes one App callback may run before the runner logs it over budget and starts the next App. AFTER_SYNC_BUDGET_MINUTES = 30 AFTER_SYNC_BUDGETS = {"modules.opportunity_finder:on_amazon_sync": 120} #: R18: `fn(tenant) -> iterable of str`. Older brand analytics weeks keep these search terms only. #: A missing module is an empty set; a reader that raises prunes nothing that pass. WATCHED_TERMS = "modules.competitor_watch:watched_terms" #: Standing rule 1 copy for the brand analytics limit (o7-brand-analytics-week.md). It reaches a screen. BRAND_ANALYTICS_LIMIT = ( "Brand analytics keeps the latest full week of search terms. Older weeks keep only your watched " "search terms, because one week is about 9.6 million rows and 3.5 GB. " "To keep a term's history, add it to the competitor watch.") # C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity. ENABLEMENT_KEY = "amazon_connector_enablement" MIRROR_PREFIX = "amazon_mirror/" _MIRROR_CACHE = {} _MIRROR_LOCK = threading.RLock() _ACCOUNT = threading.local() def _account_scope(): return getattr(_ACCOUNT, "scope", None) def _ordered_account_entries(entries): """Order Amazon metadata by keychain slot and entry ID, including saved revocation snapshots.""" return sorted((dict(row) for row in entries if row.get("type") == "amazon_sp"), key=lambda row: (int(row.get("slot") or 1), str(row["id"]))) def account_entries(tenant_key, rt): """List every tenant Amazon connection without decrypting credentials.""" _tenant_mirror_path(tenant_key, rt) if rt is None: raise RuntimeError("Amazon sync requires a tenant runtime and stored credentials") from core import keychain rows = _ordered_account_entries(keychain.list_entries(rt)) identities = [str(row.get("connection_id") or "") for row in rows] if not rows or any(not cid for cid in identities) or len(set(identities)) != len(rows): raise RuntimeError("Amazon sync requires distinct stored amazon_sp connections") return rows def _account_prefix(connection_id): return "amzn_account_" + str(connection_id).encode("utf-8").hex() + "__" def _sp_objects(con): return [(name, kind) for name, kind in _amazon_objects(con) if name.startswith(PREFIX) and not name.startswith("amzn_ads_") and name != "amzn_apify_rank_observations"] def legacy_identity_owner(con, table): """Return the immutable owner of a table's pre-connection numeric identity namespace.""" return (_read_state(con, "amzn_account_legacy_identity_" + table) or {}).get("detail") def _adopt_legacy_account(con, entry): """Bind untagged single-account rows and markers once to the designated connection.""" if _read_state(con, "amzn_account_binding"): return set() con.execute("BEGIN TRANSACTION") try: changed = _bind_legacy_account(con, entry) con.execute("COMMIT") return changed except BaseException: con.execute("ROLLBACK") raise def _bind_legacy_account(con, entry): cid, label = str(entry["connection_id"]), str(entry["label"]) changed = set() if _read_state(con, "amzn_account_binding"): return changed for name, kind in _sp_objects(con): if kind != "BASE TABLE": continue columns = _columns_of(con, name) if amazon_schema.CONNECTION_ID not in columns: if con.execute(f'SELECT 1 FROM {name} LIMIT 1').fetchone(): _record_state(con, "amzn_account_legacy_identity_" + name.removeprefix(PREFIX), "done", detail=cid) con.execute(f'ALTER TABLE {name} ADD COLUMN "_connection_id" VARCHAR') if amazon_schema.STORE not in columns: con.execute(f'ALTER TABLE {name} ADD COLUMN store VARCHAR') if con.execute(f'SELECT 1 FROM {name} WHERE "_connection_id" IS NULL LIMIT 1').fetchone(): con.execute(f'UPDATE {name} SET "_connection_id"=?, store=? WHERE "_connection_id" IS NULL', [cid, label]) changed.add(name) prefix = _account_prefix(cid) con.execute("INSERT OR REPLACE INTO _sync_state SELECT ? || entity, phase, last_id, cursor_wd, rows, updated_at " "FROM _sync_state WHERE starts_with(entity, 'amzn_') " "AND NOT starts_with(entity, 'amzn_account_') AND NOT starts_with(entity, 'amzn_ads_')", [prefix]) con.execute("DELETE FROM _sync_state WHERE starts_with(entity, 'amzn_') " "AND NOT starts_with(entity, 'amzn_account_') AND NOT starts_with(entity, 'amzn_ads_')") _record_state(con, "amzn_account_binding", "done", detail=cid) return changed def _refresh_account_status(con, entries, rt): """Project account progress for existing status readers without lending it to another account.""" enabled = read_database_enablement(rt)["databases"] tables = [t for t in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + t)] for table in TABLES: for family in ("", "sync_tick_", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "refused_", "waiting_"): key = entity_key(family + table) states = [_read_state(con, _account_prefix(e["connection_id"]) + key) for e in entries] present = [s for s in states if s] con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) if not present: continue if family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX) and len(present) != len(entries): continue state = next((s for s in present if s["state"] != "done"), present[0]) phase = state["state"] if len(present) == len(entries) else "pending" detail = (min(s["detail"] for s in present if s.get("detail")) if family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX) and all(s.get("detail") for s in present) else state.get("detail")) _record_state(con, key, phase, sum(s.get("rows", 0) for s in present), detail) updated = min(s["updated_at"] for s in present) con.execute("UPDATE _sync_state SET updated_at=? WHERE entity=?", [updated, key]) # H2: tenant-level readers (needs_first_load, status) see the oldest account floor once every account has one. floor_keys = [(entity_key(SELLER_FIRST_MONTH_KEY), "month"), (entity_key(HISTORY_FLOOR_CAUSE_KEY), None), *((_history_limit_key(t), "floor") for t in TABLES)] for key, field in floor_keys: present = [s for s in (_read_state(con, _account_prefix(e["connection_id"]) + key) for e in entries) if s] con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) if not present or (field and len(present) != len(entries)): continue try: details = [json.loads(s["detail"]) for s in present if s.get("detail")] except (ValueError, TypeError): continue if not details: continue dated = [d for d in details if field and d.get(field)] # A measured account with no orders never hides another account's older month. chosen = min(dated, key=lambda d: str(d[field])) if dated else details[0] _record_state(con, key, present[0]["state"], detail=json.dumps(chosen)) loads, remaining, eta, waits = [], 0, 0, [] for entry in entries: prefix = _account_prefix(entry["connection_id"]) state = _read_state(con, prefix + entity_key(FIRST_LOAD_END_KEY)) progress = _read_state(con, prefix + entity_key(FIRST_LOAD_STATUS_KEY)) if state and progress: values = json.loads(progress["detail"]) loads.append(state["state"]) remaining += int(values["remaining"]) eta += int(values["etaSeconds"]) else: owed = sum(not _read_state(con, prefix + entity_key(WATERMARK_PREFIX + t)) and (_read_state(con, prefix + entity_key(t)) or {}).get("state") != "done" for t in tables) loads.append("partial" if owed else "done") remaining += owed if loads[-1] != "done": due = _read_state(con, prefix + entity_key(FIRST_LOAD_DUE_KEY)) waits.append(due.get("detail") if due and due.get("state") == "pending" else None) phase = "failed" if "failed" in loads else "partial" if any(s != "done" for s in loads) else "done" _record_state(con, entity_key(FIRST_LOAD_END_KEY), phase, detail=_iso(_utc())) _record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), phase, detail=json.dumps({"remaining": remaining, "etaSeconds": eta})) due = min(waits) if waits and all(waits) else None _record_state(con, entity_key(FIRST_LOAD_DUE_KEY), "pending" if due else "done", detail=due) @contextmanager def _account_workspace(con, tenant_key, rt, entry): """Stage one connection with SQL transfers, retaining all rows and its durable resume markers.""" import duckdb path = _tenant_mirror_path(tenant_key, rt) cid, label = str(entry["connection_id"]), str(entry["label"]) prefix = _account_prefix(cid) with tempfile.TemporaryDirectory(prefix="amazon-account-", dir=path.parent) as directory: workspace = Path(directory) / "account.duckdb" alias = "amazon_account_" + uuid.uuid4().hex con.execute(f"ATTACH '{str(workspace).replace(chr(39), chr(39)*2)}' AS {alias}") try: con.execute(f"CREATE TABLE {alias}._sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") con.execute(f"INSERT INTO {alias}._sync_state SELECT substr(entity, ?), phase, last_id, " "cursor_wd, rows, updated_at FROM _sync_state WHERE starts_with(entity, ?)", [len(prefix) + 1, prefix]) for name, kind in _sp_objects(con): if kind != "BASE TABLE": continue columns = _columns_of(con, name) if amazon_schema.CONNECTION_ID not in columns: continue con.execute(f'CREATE TABLE {alias}.{name} AS SELECT * FROM main.{name} ' 'WHERE "_connection_id"=?', [cid]) con.execute(f'UPDATE {alias}.{name} SET store=?', [label]) # Public keyword captures belong to Apify, and each seller derives its own SKU rows. if _columns_of(con, "amzn_apify_rank_observations"): con.execute(f'CREATE TABLE {alias}.amzn_apify_rank_observations AS ' 'SELECT * FROM main.amzn_apify_rank_observations') finally: con.execute(f"DETACH {alias}") root = duckdb.connect(str(workspace)) scope = {"tenant": tenant_key, "rt": rt, "entry": entry, "root": root, "path": path, "workspace": workspace, "changed": set()} previous = _account_scope() _ACCOUNT.scope = scope try: datastore.apply_limits(root) refresh_sku_keywords(root) yield scope finally: _ACCOUNT.scope = previous root.close() def _merge_account(con, scope): """Replace only this account's population and state, including empty snapshots.""" entry = scope["entry"] cid, label, prefix = str(entry["connection_id"]), str(entry["label"]), _account_prefix(entry["connection_id"]) alias = "amazon_account_" + uuid.uuid4().hex path = str(scope["workspace"]).replace("'", "''") # Close the workspace writer before attaching it to the combined candidate. scope["root"].close() con.execute(f"ATTACH '{path}' AS {alias} (READ_ONLY)") changed = set() try: objects = con.execute("SELECT table_name FROM information_schema.tables " "WHERE table_catalog=? AND table_schema='main' AND table_type='BASE TABLE' " "AND starts_with(table_name, 'amzn_') " "AND table_name != 'amzn_apify_rank_observations'", [alias]).fetchall() con.execute("BEGIN TRANSACTION") try: for (name,) in objects: columns = {row[0]: row[1] for row in con.execute(f'DESCRIBE {alias}.{name}').fetchall()} # Sync loaders stamp metadata before writes. Empty inherited tables retain it too. if amazon_schema.CONNECTION_ID not in columns: raise RuntimeError("Amazon connection attribution is missing") target_columns = {r[0]: r[1] for r in con.execute("SELECT column_name, data_type FROM information_schema.columns " "WHERE table_catalog=current_database() AND table_schema='main' AND table_name=?", [name]).fetchall()} if not target_columns: con.execute(f'CREATE TABLE main.{name} AS SELECT * FROM {alias}.{name} WHERE FALSE') else: for column, kind in columns.items(): if column not in target_columns: con.execute(f'ALTER TABLE main.{name} ADD COLUMN {_quoted_identifier(column)} {kind}') elif target_columns[column] != kind: numeric = {"BIGINT", "INTEGER", "DOUBLE", "FLOAT", "HUGEINT"} widened = "DOUBLE" if {kind, target_columns[column]} <= numeric else "VARCHAR" if target_columns[column] != widened: con.execute(f'ALTER TABLE main.{name} ALTER COLUMN {_quoted_identifier(column)} TYPE {widened}') con.execute(f'DELETE FROM main.{name} WHERE "_connection_id"=?', [cid]) con.execute(f'INSERT INTO main.{name} BY NAME SELECT * FROM {alias}.{name}') con.execute(f'UPDATE main.{name} SET store=? WHERE "_connection_id"=?', [label, cid]) changed.add(name) con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [prefix]) con.execute(f"INSERT INTO _sync_state SELECT ? || entity, phase, last_id, cursor_wd, rows, updated_at " f"FROM {alias}._sync_state WHERE starts_with(entity, 'amzn_')", [prefix]) con.execute("COMMIT") except Exception: con.execute("ROLLBACK") raise finally: con.execute(f"DETACH {alias}") return changed def _purge_connection_rows(con, connection_id): """Delete one immutable connection while preserving sibling rows and resume markers.""" con.execute("BEGIN TRANSACTION") try: result = _delete_connection_rows(con, connection_id) con.execute("COMMIT") return result except BaseException: con.execute("ROLLBACK") raise def _delete_connection_rows(con, connection_id): changed, deleted = set(), 0 for name, kind in _sp_objects(con): if kind != "BASE TABLE": continue if amazon_schema.CONNECTION_ID not in _columns_of(con, name): if con.execute(f"SELECT 1 FROM {name} LIMIT 1").fetchone(): raise RuntimeError("Amazon legacy ownership must be bound before scoped deletion") continue deleted += con.execute(f'SELECT count(*) FROM {name} WHERE "_connection_id"=?', [connection_id]).fetchone()[0] con.execute(f'DELETE FROM {name} WHERE "_connection_id"=?', [connection_id]) changed.add(name) con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [_account_prefix(connection_id)]) refresh_sku_keywords(con) return changed, deleted def purge_connection(tenant_key, connection_id, *, rt, entries_before): """Revoke one account's serving, saved and private observations without deleting combined tables.""" previous = getattr(_ACCOUNT, "purging", False) _ACCOUNT.purging = True try: return _purge_connection(tenant_key, connection_id, rt=rt, entries_before=entries_before) finally: _ACCOUNT.purging = previous def _purge_connection(tenant_key, connection_id, *, rt, entries_before): key, cid = _tenant_key(tenant_key), str(connection_id or "") _tenant_mirror_path(key, rt) entries = _ordered_account_entries(entries_before) if not cid or not any(e.get("connection_id") == cid for e in entries): raise ValueError("Amazon scoped deletion requires the original connection metadata") if _account_scope() or getattr(datastore._SYNC_TLS, "candidate", None) is not None: raise RuntimeError("Amazon scoped deletion requires the resident writer outside a sync pass") obligation = retention_check(key, rt=rt, raise_overdue=True) if obligation["earliestTriggerAt"]: raise RuntimeError("Amazon tenant deletion is outstanding; complete its tenant-wide lifecycle purge") remaining = [e for e in entries if e["connection_id"] != cid] with _MIRROR_LOCK: restored = restore_mirror(key, rt, log=lambda *_: None) if restored["state"] == "refused": raise RuntimeError("Amazon scoped deletion could not restore its saved observations") con, _path = _connect_tenant_mirror(key, rt) try: changed = _adopt_legacy_account(con, entries[0]) removed, count = _purge_connection_rows(con, cid) changed.update(removed) _refresh_account_status(con, remaining, rt) report = {"problems": []} manifest = _MIRROR_STORE.persist_mirror(key, rt, con, changed, report=report, log=lambda *_: None, _publication={"snapshot_id": uuid.uuid4().hex, "started_at": _control_stamp(), "state": "complete"}) if not manifest or report["problems"]: raise RuntimeError("Amazon scoped deletion could not save its retained observations") datastore._write_publication(con, "amazon", manifest) private_path = Path(rt.datastore_path).with_suffix(".amazon.candidate.duckdb") private_deleted = 0 if private_path.exists(): with closing(datastore.apply_limits(datastore.duckdb.connect(str(private_path)))) as private: _adopt_legacy_account(private, entries[0]) _unused, private_deleted = _purge_connection_rows(private, cid) _refresh_account_status(private, remaining, rt) if _columns_of(private, "_candidate_base"): private.execute("DELETE FROM _candidate_base") private.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], _control_stamp()]) current = set(_manifest_keys(manifest, strict=False)) ledger = _MIRROR_STORE._read_key_ledger(rt, key) candidates = set(ledger["keys"]) | {name + ".parquet" for name in changed} | {"sync_state.json"} removed_keys = set() for blob in sorted(candidates - current): if blob.startswith("sync_state.") or any(blob.startswith(name + ".") for name in changed): path = MIRROR_PREFIX + blob if rt.delete_path(path) is False or rt.download_bytes(path) is not None: raise RuntimeError("Amazon scoped deletion left superseded saved bytes") removed_keys.add(blob) ledger["keys"] = {k: v for k, v in ledger["keys"].items() if k not in removed_keys} rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger)) _purge_runtime_caches(rt, key, key) return {"rows": count, "privateRows": private_deleted, "tables": sorted(changed), "connectionId": cid, "snapshotId": manifest["snapshot_id"]} finally: con.close() def read_database_enablement(rt): """Read C1 without importing the web layer; malformed/unreadable state fails closed.""" keys = tuple(amazon_schema.DATABASE_PREFIX + table for table in DATABASE_TABLES) unavailable = {"available": False, "databases": dict.fromkeys(keys, False)} try: current = rt.get(ENABLEMENT_KEY) if rt is not None else None except Exception: return unavailable if current is None or current == {}: return {"available": True, "databases": dict.fromkeys(keys, True)} if not isinstance(current, dict) or not isinstance(current.get("databases"), dict): return unavailable stored = current["databases"] if any(type(value) is not bool for value in stored.values()): return unavailable if set(stored) - set(keys): logging.getLogger(__name__).warning("Ignoring unknown Amazon database enablement keys") return {"available": True, "databases": {key: stored.get(key, True) for key in keys}} # Day length for per-table cadence. Manual sync keeps the report's own refresh floor. SCHEDULED_FLOOR_SECONDS = 86400 SCHEDULED_FLOOR_GRACE_SECONDS = 0 # The resident cadence checks once a minute. SCHEDULE_PHASE_GRACE_SECONDS = 60 #: `_sync_state` marker: the scheduled first-load guard deferred this table; never a tick, so no card reads it. PHASE_HOLD_PREFIX = "phase_hold_" _SCHEDULE_OFFSET_READER = None def register_schedule_offset(reader): """Bind the application-owned slot offset without importing the API layer.""" if not callable(reader): raise TypeError("Amazon schedule offset requires a callable") global _SCHEDULE_OFFSET_READER _SCHEDULE_OFFSET_READER = reader def _scheduled_phase_due(table, state, now, earliest): """Keep scheduled pulls on deployment phases, including stale or absent boot ticks.""" shift = int(_SCHEDULE_OFFSET_READER()) interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"] phase = ((now.timestamp() - shift) // interval) * interval + shift stamp = _utc(state["updated_at"]).timestamp() if state and state.get("updated_at") else None window = interval / 2 fastest = SCHEDULED_FLOOR_SECONDS // max(cadence["perDay"] for cadence in _client().CADENCE.values()) if shift % fastest: # A shifted deployment's late run ends 30 minutes before the next live slot. window = min(window, fastest - phase % fastest - 1800) # A tick that missed its phase, in any state or absent, runs late inside the first half of the interval. # The second half is left alone: a staging copy of a live tick stays off the live phase. if (stamp is None or stamp < phase) and earliest.timestamp() - phase < window: return earliest if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS or (stamp is not None and stamp >= phase)): phase += interval while earliest.timestamp() > phase + SCHEDULE_PHASE_GRACE_SECONDS: phase += interval return max(earliest, datetime.fromtimestamp(phase, timezone.utc)) def _cadence_seconds(table, state=None): """Return the table's scheduled interval, using calendar months for monthly reports.""" cadence = _client().CADENCE[table] if cadence["period"] == "week": if state: stamp = _utc(state["updated_at"]) return int((_week_start(stamp) + timedelta(days=7) - stamp).total_seconds()) return 7 * SCHEDULED_FLOOR_SECONDS if cadence["period"] == "month": stamp = _utc(state["updated_at"]) if state else None if stamp is None: return 30 * SCHEDULED_FLOOR_SECONDS year, month = stamp.year + (stamp.month == 12), stamp.month % 12 + 1 due = stamp.replace(year=year, month=month, day=1, hour=0, minute=0, second=0, microsecond=0) return int((due - stamp).total_seconds()) return SCHEDULED_FLOOR_SECONDS // cadence["perDay"] def sync_schedule(states, now=None): """Return every SP-API table's last sync, next sync and configured passes per day.""" now = _utc(now) schedule = [] for table in TABLES: state = states.get(table) seconds = _cadence_seconds(table, state) _wait, due, _seconds, _scheduled = _interval_wait(table, state, now, floor_s=seconds) if due is None and state and state.get("state") == "done": vendor = _client().REPORT_TYPES[table].get("min_interval_s", 0) due = _utc(state["updated_at"]) + timedelta(seconds=max(vendor, seconds)) # A due time already past is due now: the schedule never shows a past next sync. schedule.append({"table": table, "lastSyncAt": state.get("updated_at") if state else None, "nextSyncAt": _iso(due if due is not None and due > now else now), "perDay": _client().CADENCE[table]["perDay"]}) return schedule def sync_plan(states, tables=None, *, now=None, slot_pass=False): """Plan every due table, oldest first, with the same per-table cadence inside and outside the daily slot.""" from harness.amazon_client import REPORT_STAGGER_SECONDS now = _utc(now) want = list(TABLES if tables is None else tables) stamps = {table: _utc(state["updated_at"]) for table, state in states.items() if state and table != "settlements"} deferred = {} due = [] for table in want: floor = _cadence_seconds(table, states.get(table)) if (reason := _interval_reason(table, states.get(table), now, floor_s=floor)): deferred[table] = reason else: due.append(table) # Boot and the resident first pass can be adjacent. Report the delay instead of sleeping. recent = max(stamps.values()) if stamps else None ready = recent is None or (now - recent).total_seconds() >= REPORT_STAGGER_SECONDS oldest = datetime.min.replace(tzinfo=timezone.utc) selected = sorted(due, key=lambda table: (stamps.get(table, oldest), want.index(table))) if ready else [] if not ready: for table in due: deferred[table] = ("table stagger: waiting for a later cadence tick; " f"minimum spacing {REPORT_STAGGER_SECONDS:g}s") return {"tables": selected, "deferred": deferred} def scheduled_sync_plan(tenant_key, *, rt, now=None): """Combine enabled tables' due plans from each connection's own resident cadence ticks.""" path = _tenant_mirror_path(tenant_key, rt) if path.resolve() != Path(datastore.DB_PATH).resolve(): raise RuntimeError("Amazon scheduling requires the tenant's resident mirror") enabled = read_database_enablement(rt) tables = [table for table in TABLES if enabled["databases"].get(amazon_schema.DATABASE_PREFIX + table)] if not tables: return {"tables": [], "deferred": {}} now = _utc(now) con, _path = _connect_tenant_mirror(tenant_key, rt) try: due, deferred = {}, {} for entry in account_entries(tenant_key, rt): prefix = _account_prefix(entry["connection_id"]) + entity_key("sync_tick_") ticks = {entity[len(prefix):]: {"state": phase, "rows": rows, "updated_at": at, "detail": detail} for entity, phase, rows, at, detail in con.execute( "SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state " "WHERE starts_with(entity, ?)", [prefix]).fetchall()} plan = sync_plan(ticks, tables, now=now) hold = _account_prefix(entry["connection_id"]) + entity_key(PHASE_HOLD_PREFIX) # The planner reads cadence rows through this one query shape only, never source data. holds = {entity[len(hold):]: {"state": phase, "rows": 0, "updated_at": at, "detail": None} for entity, phase, _rows, at, _detail in con.execute( "SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state " "WHERE starts_with(entity, ?)", [hold]).fetchall()} for table in list(plan["tables"]): held, tick = holds.get(table), ticks.get(table) # A first-load deferral newer than the tick waits for the next phase, as the sync defers it. if held and (tick is None or _utc(held["updated_at"]) > _utc(tick["updated_at"])): reason = _interval_reason(table, held, now, floor_s=_cadence_seconds(table, tick)) if reason: plan["tables"].remove(table) plan["deferred"][table] = reason due.update(dict.fromkeys(plan["tables"])) for table, reason in plan["deferred"].items(): deferred.setdefault(table, []).append(reason) return {"tables": list(due), "deferred": { table: "; ".join(dict.fromkeys(reasons)) for table, reasons in deferred.items() if table not in due}} finally: con.close() def _readable_duration(seconds): """`86400` -> "24 hours", `2592000` -> "30 days". Screen copy, never raw seconds (S66-T42).""" seconds = int(seconds) for size, unit in ((86400, "day"), (3600, "hour"), (60, "minute")): if seconds >= size and seconds % size == 0: count = seconds // size if unit == "day" and count <= 2: count, unit = count * 24, "hour" return f"{count} {unit}" + ("s" if count != 1 else "") return f"{seconds} seconds" def _interval_wait(table, state, now, floor_s=None): """Return the wait and deciding refresh floor, preserving the vendor floor on manual sync.""" now = _utc(now) vendor = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s") or 0 seconds = max(int(vendor), int(floor_s or 0)) phased = bool(floor_s and _SCHEDULE_OFFSET_READER is not None) earliest = now if state and state.get("state") == "throttled" and state.get("detail"): due = _utc(state["detail"]) wait = (due - now).total_seconds() if table == SQP_TABLE: # An SQP backlog resumes at its own deadline, never on the daily phase. return (wait, due, int(wait), False) if wait > 0 else (0, None, seconds, False) if wait > 0: if not phased: return wait, due, int(wait), False earliest = due if seconds and state and state["state"] == "done": # Daily cadence follows slots. Longer report periods and vendor floors keep their bounds. minimum = (int(vendor) if phased and _client().CADENCE[table]["period"] == "day" else seconds) due = _utc(state["updated_at"]) + timedelta(seconds=minimum) earliest = max(earliest, due) wait = (due - now).total_seconds() if wait > 0 and not phased: return wait, due, seconds, seconds > int(vendor) if phased: due = _scheduled_phase_due(table, state, now, earliest) wait = (due - now).total_seconds() if wait > 0: return wait, due, seconds, True return 0, None, seconds, False def _interval_reason(table, state, now, floor_s=None): wait, due, seconds, scheduled = _interval_wait(table, state, now, floor_s=floor_s) if wait <= 0: return None if state and state.get("state") == "throttled": return f"Amazon is limiting requests; next retry {_iso(due)}" if scheduled: return (f"This Amazon table refreshes every {_readable_duration(seconds)}; " f"next due {_iso(due)}") return (f"Amazon allows this report at most once every {_readable_duration(seconds)}; " f"next due {_iso(due)}") #: Durable proof that the two non-report role operations actually ran. A report-backed table can #: contain rows while one of these role grants is still missing, so table presence alone is not #: evidence that Pricing or AWD was authorized. ENTITY_STATE_KEYS = { "pricing": "amzn_pricing_competitive_summary", "inventory": "amzn_inventory_awd", } #: Mirror table names. The `ut_amzn_*` grids are spawned from these by `amazon_relational`. def table_name(table): """`orders` -> `amzn_orders`. One place, so a rename cannot half-land.""" return PREFIX + str(table) def entity_key(table): """The `_sync_state` key. ⛔ ALWAYS prefixed: see the header's collision note.""" return PREFIX + str(table) #: FALLBACK ONLY: the history floor while the seller's first order month is unmeasured (R30 revoked #: by the owner 2026-11-14). `history_floor` is the rule; a backfill is sliced (`_windows`). DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730) #: `_sync_state` marker: the seller's first month with orders, from getOrderMetrics. SELLER_FIRST_MONTH_KEY = "seller_first_month" #: `_sync_state` marker: why the seller floor is unmeasured; a failed measurement retries the next UTC day. HISTORY_FLOOR_CAUSE_KEY = "history_floor_cause" #: Measurement walks back no further than this. SELLER_FLOOR_EARLIEST = datetime(2005, 1, 1, tzinfo=timezone.utc) #: A brand-level table (`seller_floor: False`) never plans before this. BRAND_FLOOR_EARLIEST = datetime(2015, 1, 1, tzinfo=timezone.utc) #: A brand-level table stops after this many landed-empty windows older than its oldest window with rows. BRAND_EMPTY_STOP = 12 #: A rolling Amazon floor is read one day late, so a window planned now is still valid when requested. ROLLING_FLOOR_MARGIN = timedelta(days=1) #: Per-table marker whose detail is the ISO UTC end of the last successfully landed window. WATERMARK_PREFIX = "sync_mark_" #: D-17409: the end of a table's last refresh while no first-load job serves its tenant. Never a watermark. CATCH_UP_PREFIX = "catchup_" #: A first load whose job lease, chain record and own record have not moved for this long serves nobody. FIRST_LOAD_IDLE_SECONDS = 24 * 3600 #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one. MAX_WINDOW_DAYS = 30 #: R18: the most recent month lands before any history, so a newly connected account has usable #: rows in minutes instead of hours. One window, because Amazon's own ceiling is 30 days. FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS #: F4: Amazon paces createReport (burst 15, one per 60 s, 5 s gap), so 7 tables x 25 windows is #: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident #: thread for hours and the Odoo resync behind it would never run. FIRST_LOAD_REPORTS_PER_PASS = 15 #: A generation's repeated other-tables pass: about an hour of reports at Amazon's rate, then it publishes. FIRST_LOAD_DELEGATE_REPORTS = 60 #: Weekly history weeks (a 3 GB search terms report each) one generation takes before its other-tables pass. FIRST_LOAD_WEEKLY_HISTORY_PER_GENERATION = 2 #: Older weeks of a weekly table are not planned: the weekly writer defers a history week by its cadence, and a #: generation that wrote one never finished it (staging 2026-11-14). The latest week stays first, as R18. WEEKLY_HISTORY_PLANNED = False #: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap: #: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25 #: minutes each) never hold the sync desk for hours (D-7081). FIRST_LOAD_ROWS_PER_PASS = 3_000_000 BACKFILL_FLOOR_PREFIX = "sync_floor_" #: Report windows whose stored rows cannot be healed in place; the next pass downloads them again. RESYNC_PREFIX = "resync_" #: Queued resync windows one pass downloads at most: Amazon's createReport burst, less the pass's spend. RESYNC_REPORTS_PER_PASS = FIRST_LOAD_REPORTS_PER_PASS FIRST_LOAD_END_KEY = "sync_first_load_end" FIRST_LOAD_STATUS_KEY = "sync_first_load_status" FIRST_LOAD_DUE_KEY = "sync_first_load_next_due" # Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores; # JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata. _ORIGIN_TYPES = {"_report_document_id": "VARCHAR", "_report_row_number": "BIGINT", "_data_origin": "VARCHAR", "_report_type": "VARCHAR"} DELETION_DAYS = 30 # Allow re-add for 25 days, leaving five days inside Amazon's 30-day deadline. # Even twice-daily resident resync checks then have ample time to complete deletion. PURGE_AFTER_DAYS = 25 DELETION_TRIGGERS = ("revocation", "authorization_loss", "termination", "notice") _RETENTION_CHECKED = {} _RETENTION_LOCK = threading.RLock() def _origin(row, document_id, row_number, report_type): if not document_id or not report_type: raise ValueError("Amazon row origin requires a document ID and report type") row.update({"_report_document_id": str(document_id), "_report_row_number": row_number, "_data_origin": "Amazon Services API", "_report_type": str(report_type)}) return row #: Columns that must be text whatever they look like. An Amazon order id is `111-1234567-1234567` #: and an ASIN is `B0…`; a settlement id is a long decimal that exceeds 2^53. Same ruling as Meta's #: ids and `ig_id` before it, for the same reason: a float path corrupts them silently. _FORCE_TEXT = {"amazon_order_id", "merchant_order_id", "order_id", "asin", "parent_asin", "child_asin", "sku", "seller_sku", "settlement_id", "shipment_id", "order_item_id", "merchant_order_item_id", "fnsku", "product_id", "marketplace_id", "seller_id"} _FORCE_TEXT.update({"msku", "transaction_id", "currency_code", "month", "fulfillment_center", "estimate_id"}) _FORCE_TEXT.update({"seller_fulfillment_order_id", "seller_fulfillment_order_item_id"}) _FORCE_TEXT.add("search_query_data_search_query") _FORCE_TEXT.update({"ean", "upc", "isbn", "part_number", "model_number", "promotion_id", "coupon_id", "reimbursement_id", "replacement_amazon_order_id", "original_amazon_order_id"}) def _snake(name): """A report's own header cell -> a safe DuckDB column name. ⚠ MEASURED HEADERS ARE HOSTILE. Amazon flat files carry `purchase-date`, `Units Ordered`, `item-price`, a UTF-8 BOM on cell one, and occasionally a blank trailing column. Every one of those has to become a legal identifier without colliding with its neighbour. ⭐ THE BOM NEEDS NO SPECIAL CASE, AND PROVING THAT DELETED A LINE. A first version opened with `.replace("\\ufeff", "")`; perturbing it changed nothing, because the sanitizer below already maps every non-alphanumeric character to `_` and the trailing `strip("_")` removes it. The explicit replace was dead code that made NC-AZ2 a TAUTOLOGY: the check could not fail whatever the code did. The guarantee is kept and now asserted against the real mechanism. """ s = str(name or "").strip().lower() out = [] for ch in s: out.append(ch if (ch.isalnum() or ch == "_") else "_") col = "".join(out) while "__" in col: col = col.replace("__", "_") col = col.strip("_") if not col: return "" return ("c_" + col) if col[0].isdigit() else col def _dedupe(cols): """Make a measured header list unique, preserving order. A duplicate becomes `name_2`.""" seen, out = {}, [] for c in cols: if not c: continue if c in seen: seen[c] += 1 out.append(f"{c}_{seen[c]}") else: seen[c] = 1 out.append(c) return out def buyer_column(name): """Deny buyer identity and destination address columns, regardless of spelling.""" return _buyer_name(str(name)) @functools.lru_cache(maxsize=1 << 16) def _buyer_name(name): # D-9924: a report repeats the same keys on every row, so each name is judged once. compact = re.sub(r"[^a-z0-9]", "", name.lower()) return (bool(re.search(r"(?:buyer|customer|recipient)[a-z0-9]*(?:name|email|phone|telephone)", compact)) or any(word in compact for word in ("buyername", "buyeremail", "buyerphone", "customername", "customeremail", "customerphone", "recipientname", "recipientphone", "recipientemail", "rateremail", "shipaddress", "shippingaddress", "shiptoaddress", "destinationaddress", "removaladdress", "addressline", "deliveryaddress", "destinationname", "destinationphone", "destinationemail", "contactname", "contactphone", "contactemail", "notificationemail", "streetaddress")) or compact in {"email", "emailaddress", "phone", "phonenumber", "address", "address1", "address2", "address3", "shiptoname", "shiptophone", "shiptoemail", "street", "street1", "street2", "telephone", "mobilephone"}) def _without_buyer_data(value): # Lists can be serialized into one cell, so sanitize recursively before flattening. if isinstance(value, dict): return {k: _without_buyer_data(v) for k, v in value.items() if not buyer_column(k)} if isinstance(value, list): return [_without_buyer_data(v) for v in value] return value #: Rows per INSERT statement. A batch, not a cap: every row of a document is written (standing rule 1). INSERT_BATCH_ROWS = 500 #: Bound parameters per INSERT statement, so a wide table takes fewer rows per statement. INSERT_BATCH_VALUES = 30_000 #: Characters read per step of a spooled document. DOCUMENT_READ_CHARS = 1 << 20 def _spooled(document): """True for a `ReportDocument` the client spooled to disk; False for in-memory text.""" return hasattr(document, "open_text") def _document_text(document): """The whole text of a document, for a consumer whose documents are small.""" return document.read_text() if _spooled(document) else document def _blank(document): """`not str(text or "").strip()`, read in pieces when the document is spooled.""" if not _spooled(document): return not str(document or "").strip() with document.open_text() as handle: while True: piece = handle.read(DOCUMENT_READ_CHARS) if not piece: return True if piece.strip(): return False def _value_kind(v): """One non-empty value's sniffed kind: `int`, `float` or `str`.""" if isinstance(v, bool): return "str" if isinstance(v, int): return "int" if isinstance(v, float): return "float" if isinstance(v, (list, dict)): return "str" s = str(v).strip() try: int(s) return "int" except ValueError: try: float(s) return "float" except ValueError: return "str" def _kinds_type(kinds): if not kinds or "str" in kinds: return "VARCHAR" return "DOUBLE" if "float" in kinds else "BIGINT" class _DocumentRows: """Rows parsed again from a spooled report document on every pass, so no pass holds them all. ⛔ D-7081: one list of every row of the Brand Analytics month took the Space to 16 GB. A caller iterates, takes `len()` or tests truth; `map` and `pipe` replay a row change on every pass. """ def __init__(self, produce, steps=()): self._produce, self._steps = produce, tuple(steps) self._count, self._sniff, self._unattributed = None, None, None def __iter__(self): rows = self._produce() for step in self._steps: rows = step(rows) return iter(rows) def pipe(self, step): return _DocumentRows(self._produce, self._steps + (step,)) def map(self, change): return self.pipe(lambda rows: (change(row) for row in rows)) @staticmethod def chain(*parts): return _DocumentRows(lambda: (row for part in parts for row in part)) def with_values(self, values): """Set `values` on every row. A profiled parent's profile carries over, so no pass reads them again.""" def stamped(row): row.update(values) return row out = self.map(stamped) if self._sniff is not None: filled = {k: v for k, v in values.items() if v is not None and v != ""} out._count, out._unattributed = self._count, self._unattributed out._sniff = {**self._sniff, **{k: [1 if k in filled else 0, {_value_kind(v)} if k in filled else set()] for k, v in values.items()}} return out def unattributed(self): """True when a row lacks complete source attribution, from the one profile pass.""" if self._unattributed is None: self._profile() return self._unattributed def _profile(self): """One pass: the row count, every key, each key's first 500 non-empty kinds, and attribution.""" count, sniff, bad = 0, {}, False for row in self: count += 1 if not bad: number = row.get("_report_row_number") bad = (type(number) is not int or number < 1 or any(row.get(c) in (None, "") for c in _ORIGIN_TYPES)) for key, value in row.items(): entry = sniff.get(key) if entry is None: entry = sniff[key] = [0, set()] if entry[0] >= 500 or value is None or value == "": continue entry[1].add(_value_kind(value)) entry[0] += 1 self._count, self._sniff, self._unattributed = count, sniff, bad def keys(self): if self._sniff is None: self._profile() return set(self._sniff) def sniffed(self, name): """`_coltype(name, rows)` for these rows, from the one profile pass.""" if self._sniff is None: self._profile() return _kinds_type(self._sniff.get(name, (0, ()))[1]) def __len__(self): if self._count is None: self._profile() return self._count def __bool__(self): if self._count is not None: return self._count > 0 rows = iter(self) try: return next(rows, None) is not None finally: getattr(rows, "close", lambda: None)() #: Rows per spooled batch: one `marshal` call per batch, never one per row. SPOOL_BATCH_ROWS = 1024 class _RowSpool: """D-9924: rows parsed from a report document once, then replayed from a local spool. The first complete pass reads the document and writes each row with `marshal` before yielding it. Every later pass (the profile after a stamp, the insert) replays the spool, so a 3.5 GB Brand Analytics week is parsed once with bounded memory. A pass that stops early keeps no spool. """ def __init__(self, produce): self._produce, self._path = produce, None self._finalizer = None def __call__(self): if self._path is not None: return self._replay() return self._first() def _first(self): import marshal fd, path = tempfile.mkstemp(prefix="amazon-rows-", dir=os.environ.get("AIOS_DUCKDB_TEMP_DIR") or None) complete = False try: with os.fdopen(fd, "wb") as handle: shapes, batch = {}, [] for row in self._produce(): # Taken before it is yielded, so a later step's change to the row is not spooled. keys = tuple(row) batch.append((shapes.setdefault(keys, len(shapes)), tuple(row.values()))) if len(batch) >= SPOOL_BATCH_ROWS: marshal.dump((list(shapes), batch), handle) shapes, batch = {}, [] yield row if batch: marshal.dump((list(shapes), batch), handle) complete = True finally: if complete and self._path is None: self._path = path import weakref self._finalizer = weakref.finalize(self, _unlink_quiet, path) else: _unlink_quiet(path) def _replay(self): import marshal with open(self._path, "rb") as handle: while True: try: shapes, batch = marshal.load(handle) except EOFError: return for shape, values in batch: yield dict(zip(shapes[shape], values)) def _unlink_quiet(path): try: os.unlink(path) except OSError: pass def _rss_gb(): """This process's resident memory in GB, or None where /proc is absent.""" try: with open("/proc/self/status", encoding="ascii") as handle: for line in handle: if line.startswith("VmRSS:"): return int(line.split()[1]) / 1048576 except Exception: # noqa: BLE001 return None return None class _DocumentProbe: """One server-log line per report document: table, bytes, rows, seconds and the memory peak. ⚠ `print`, not `log`: the sync desk passes a silent log, so its first load logged nothing until a pass ended and the document in flight at an out-of-memory kill was unknown (D-7081). """ _lock = threading.Lock() _active = None def __init__(self, table, report_type): self.table, self.report_type = table, report_type self.started, self.size, self.peak = time.monotonic(), None, _rss_gb() self._halt = threading.Event() with _DocumentProbe._lock: previous, _DocumentProbe._active = _DocumentProbe._active, self if previous is not None: previous._halt.set() threading.Thread(target=self._sample, name="amazon-document-probe", daemon=True).start() def _sample(self): deadline = time.monotonic() + 6 * 3600 while not self._halt.wait(0.5) and time.monotonic() < deadline: rss = _rss_gb() if rss is not None and (self.peak is None or rss > self.peak): self.peak = rss @classmethod def downloaded(cls, document): probe = cls._active if probe is not None and not probe._halt.is_set(): probe.size = (document.size if _spooled(document) else len(str(document or "").encode("utf-8", "replace"))) probe._say("downloaded", None) def finish(self, rows): self._halt.set() self._say("written", rows) def _say(self, phase, rows): rss = _rss_gb() peak = max(x for x in (self.peak, rss, 0.0) if x is not None) size = "unknown" if self.size is None else f"{self.size:,}" print(f"[aios-api] amazon document {phase}: table {self.table}, {self.report_type}, " f"{size} bytes, {'-' if rows is None else f'{rows:,}'} rows, " f"{time.monotonic() - self.started:.0f}s, rss {rss or 0:.2f} GB, peak {peak:.2f} GB", flush=True) #: D-2682: unquote a document whose header cells are all wrapped in `"`. Off only in a negative control. UNQUOTE_QUOTED_DOCUMENTS = True def _quote_closed(cell): """True when `cell`, which opens with `"`, also ends its quoted value (an odd closing run).""" run = len(cell[1:]) - len(cell[1:].rstrip('"')) return run % 2 == 1 def _quoted_header(header): """True when every non-empty header cell is wrapped in one pair of `"` (the ledger detail).""" cells = [cell for cell in header if cell != ""] return bool(cells) and all(cell.startswith('"') and len(cell) >= 2 and _quote_closed(cell) for cell in cells) def _unquote_cells(raw, width=None): """One quoted document's split line -> its values: wrapping quotes off, `""` -> `"`. A tab inside a quoted value split it in two; the pieces join again up to the closing quote. Measured-width rows keep their boundaries, including Amazon's unescaped terminal inch marks. A supplied width refuses incomplete or ambiguous rows instead of padding or truncating them. """ if width is not None and len(raw) == width: out = [] for cell in raw: if cell.startswith('"'): if len(cell) < 2 or not cell.endswith('"'): raise ValueError("Amazon quoted report has an unclosed cell; no rows were replaced") cell = cell[1:-1].replace('""', '"') elif cell: raise ValueError("Amazon quoted report has an unwrapped cell; no rows were replaced") out.append(cell) return out out, open_cells = [], None for cell in raw: if open_cells is not None: open_cells.append(cell) if _quote_closed("\t".join(open_cells)): out.append("\t".join(open_cells)[1:-1].replace('""', '"')) open_cells = None continue if cell.startswith('"') and len(cell) >= 2 and _quote_closed(cell): out.append(cell[1:-1].replace('""', '"')) elif cell.startswith('"'): open_cells = [cell] else: if width is not None and cell: raise ValueError("Amazon quoted report has an unwrapped cell; no rows were replaced") out.append(cell) if width is not None and (open_cells is not None or len(out) != width): raise ValueError("Amazon quoted report row does not match its header width; no rows were replaced") return out + (open_cells or []) def _flat_rows(lines, document_id, report_type): """`(columns, row generator)` over one flat file's lines, or `(None, ())` with no header line.""" reader = csv.reader(lines, delimiter="\t", quoting=csv.QUOTE_NONE) try: header = next(reader) except StopIteration: return None, () quoted = UNQUOTE_QUOTED_DOCUMENTS and _quoted_header(header) if quoted: header = _unquote_cells(header) cols = _dedupe([_snake(h) for h in header]) width = len(cols) ledger = UNQUOTE_QUOTED_DOCUMENTS and (report_type == "GET_LEDGER_DETAIL_VIEW_DATA" or {"event_type", "quantity", "fulfillment_center"} <= set(cols)) # The read-only connectors_amazon report probe measures headers without capture metadata. # Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies # BOTH keywords, and a missing document ID with a report type is an error, never a fallback. attributed = document_id is not None or report_type is not None def rows(): for row_number, raw in enumerate(reader, 1): if not raw or (len(raw) == 1 and not str(raw[0]).strip()): continue if quoted: raw = _unquote_cells(raw, width) if ledger and len(raw) != width: raise ValueError("Amazon ledger row does not match its header width; no rows were replaced") vals = list(raw[:width]) + [None] * max(0, width - len(raw)) row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals) if not buyer_column(c)} if ledger and any(row.get(c) is not None and not re.fullmatch( r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?", str(row[c]).strip()) for c in ("quantity", "reconciled_quantity", "unreconciled_quantity")): raise ValueError("Amazon ledger has a nonnumeric quantity; no rows were replaced") yield _origin(row, document_id, row_number, report_type) if attributed else row return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows() def _parse_flat_document(document, *, document_id=None, report_type=None): """`parse_flat_file` over a spooled document: the header now, the rows again on every pass.""" if _blank(document): return [], [] with document.open_text() as handle: cols, _rows = _flat_rows(handle, document_id, report_type) if cols is None: return [], [] def produce(): with document.open_text() as handle: yield from _flat_rows(handle, document_id, report_type)[1] return cols, _DocumentRows(_RowSpool(produce)) def parse_flat_file(text, *, document_id=None, report_type=None): """One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header. ⭐ THIS FUNCTION IS WHY NO ORDER, LISTING OR SETTLEMENT COLUMN IS TRANSCRIBED ANYWHERE. Line 1 of the document is Amazon's own column list; anything else would be a guess that looks like a measurement on the disclosure screen. ⚠ TAB-SEPARATED, AND `QUOTE_NONE` IS LOAD-BEARING. Amazon flat files are TSV in which a plain `"` inside a product title is a literal character, not a quote. Python's default dialect would swallow the rest of the row and silently merge it with the next one, so a catalogue with one inch-mark in a title would lose rows with nothing raising. """ if _spooled(text): return _parse_flat_document(text, document_id=document_id, report_type=report_type) if not str(text or "").strip(): return [], [] buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n")) cols, rows = _flat_rows(buf, document_id, report_type) if cols is None: return [], [] return cols, list(rows) def parse_json_report(text, root, *, document_id=None, report_type=None): """One JSON report document -> `(columns, rows)` flattened onto the pinned artifact's names. ⚠ THE COLUMN LIST IS THE PINNED SCHEMA'S, NOT THE PAYLOAD'S. Amazon omits null fields from a JSON report exactly as Graph does, so building the list from what came back would drop columns with nothing going red — the defect `meta_store.py`'s header describes in as many words. The artifact is the authority; a field the payload omits is NULL. """ if _spooled(text): return _parse_json_document(text, root, document_id=document_id, report_type=report_type) try: doc = json.loads(str(text or "") or "{}") except (ValueError, TypeError) as exc: raise ValueError("Amazon returned malformed JSON; no rows were replaced") from exc if not isinstance(doc, dict) or root not in doc: if (isinstance(doc, dict) and root == "recommendations" and report_type in { "GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU", "GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON"}): return [], [] raise ValueError(f"Amazon JSON report omitted its {root} row root; no rows were replaced") body = doc[root] if isinstance(body, dict): body = [body] if not isinstance(body, list) or any(not isinstance(item, dict) for item in body): raise ValueError(f"Amazon JSON report has an invalid {root} row list") rows = list(_json_rows(body, document_id, report_type)) return sorted({k for r in rows for k in r}), rows def _json_rows(body, document_id, report_type): """Flatten and attribute each row object, numbering from one.""" attributed = document_id is not None or report_type is not None for row_number, item in enumerate(body, 1): row = _flatten_row(_without_buyer_data(item)) yield _origin(row, document_id, row_number, report_type) if attributed else row class _JsonFallback(Exception): """A shape only the whole-text parser answers exactly: malformed, not an object, a repeated root.""" _JSON_WS = re.compile(r"[ \t\n\r]*") _JSON_DECODER = json.JSONDecoder() #: A single value may span this many characters before the stream hands it to `json.loads`. _JSON_VALUE_CHARS = 256 << 20 class _JsonStream: """JSON read from a text stream one value at a time, with `json.loads` value semantics.""" def __init__(self, handle): self.handle, self.buf, self.pos = handle, "", 0 def _fill(self): piece = self.handle.read(DOCUMENT_READ_CHARS) if not piece: return False self.buf = self.buf[self.pos:] + piece self.pos = 0 return True def peek(self): while True: self.pos = _JSON_WS.match(self.buf, self.pos).end() if self.pos < len(self.buf) or not self._fill(): return self.buf[self.pos:self.pos + 1] def take(self, char): if self.peek() != char: raise _JsonFallback() self.pos += 1 def value(self): self.peek() while True: try: obj, end = _JSON_DECODER.raw_decode(self.buf, self.pos) except ValueError: if len(self.buf) - self.pos < _JSON_VALUE_CHARS and self._fill(): continue raise _JsonFallback() from None # A number ending at the buffer's edge may continue in the next piece. if end == len(self.buf) and self._fill(): continue self.pos = end return obj def skip(self): if self.peek() != "[": self.value() return self.pos += 1 if self.peek() == "]": self.pos += 1 return while True: self.skip() char = self.peek() self.pos += 1 if char == "]": return if char != ",": raise _JsonFallback() def items(self): if self.peek() == "{": yield self.value() return self.take("[") if self.peek() == "]": self.pos += 1 return while True: item = self.value() if not isinstance(item, dict): raise _JsonFallback() yield item char = self.peek() self.pos += 1 if char == "]": return if char != ",": raise _JsonFallback() def _json_root_items(document, root): """Each row object under the top-level `root` key of a spooled document, decoded one at a time. Raises `_JsonFallback` for any shape `json.loads` must judge: malformed text, a document that is not an object, a missing or repeated root, or a root that is not a list of objects. """ with document.open_text() as handle: stream, found = _JsonStream(handle), False stream.take("{") if stream.peek() == "}": stream.pos += 1 else: while True: key = stream.value() if not isinstance(key, str): raise _JsonFallback() stream.take(":") if key == root: if found: raise _JsonFallback() found = True yield from stream.items() else: stream.skip() char = stream.peek() stream.pos += 1 if char == "}": break if char != ",": raise _JsonFallback() if stream.peek() or not found: raise _JsonFallback() def _parse_json_document(document, root, *, document_id=None, report_type=None): """`parse_json_report` over a spooled document; any shape it cannot stream reads the whole text.""" rows = _DocumentRows(_RowSpool(lambda: _json_rows(_json_root_items(document, root), document_id, report_type))) try: keys = rows.keys() except _JsonFallback: return parse_json_report(document.read_text(), root, document_id=document_id, report_type=report_type) if not len(rows): return [], [] return sorted(keys), rows #: D-9924: each row repeats the same field paths, so each path is spelled once. _snake_path = functools.lru_cache(maxsize=1 << 16)(amazon_schema._snake) def _flatten_row(obj, prefix=""): """`{"salesByAsin": {"unitsOrdered": 3}}` -> `{"sales_by_asin_units_ordered": 3}`. Uses `amazon_schema._snake` so a measured row and a pinned column agree on spelling. A nested LIST becomes compact JSON text rather than being dropped or exploded, the same call `meta_store._cell` makes. """ out = {} for k, v in (obj or {}).items(): path = f"{prefix}.{k}" if prefix else k if buyer_column(path): continue if isinstance(v, dict): out.update(_flatten_row(v, path)) else: name = _snake_path(path) if name in _ORIGIN_TYPES: raise ValueError("Amazon JSON field collides with reserved origin metadata") out[name] = json.dumps(_without_buyer_data(v), separators=(",", ":"), ensure_ascii=False) if isinstance( v, list) else v return out def _money(value): """The two Amazon money shapes, with missing values kept distinct from zero.""" if not isinstance(value, dict): raise ValueError("Amazon monetary summary is missing") amount = value.get("amount", value.get("currencyAmount")) currency = value.get("currencyCode") if amount is None or not currency: raise ValueError("Amazon monetary summary needs an amount and currency") number = float(amount) if not -float("inf") < number < float("inf"): raise ValueError("Amazon monetary summary must be finite") return number, str(currency) def economics_records(text): """Data Kiosk JSONL records, failing loudly on malformed or error payloads.""" records = [] for line in str(text or "").splitlines(): if not line.strip(): continue record = json.loads(line) if not isinstance(record, dict) or "startDate" not in record: raise ValueError("Economics JSONL record has no startDate") records.append(record) return records def economics_rows(records, document_id): """Pivot all fee/ads types into numeric columns; costs and sales flatten normally.""" rows = [] for position, record in enumerate(records, 1): row = _flatten_row({k: v for k, v in record.items() if k not in ("fees", "ads")}) currencies = set() fee_total = 0.0 charge_counts = {} for family, label in (("fees", "fee"), ("ads", "ads")): names = {} for entry in record.get(family) or []: raw_name = entry.get("feeTypeName" if family == "fees" else "adTypeName") name = _snake(amazon_schema._snake(raw_name or "")) if not name or (name in names and names[name] != raw_name): raise ValueError("Amazon fee names cannot be mapped to distinct columns") names[name] = raw_name prefix = f"{label}_{name}" row[prefix + "_type_name"] = raw_name charges = ((entry.get("charges") or ([entry["charge"]] if entry.get("charge") else [])) if family == "fees" else [entry.get("charge")]) for charge in charges: if not charge: continue detail = charge.get("aggregatedDetail", charge) amount, currency = _money(detail.get("totalAmount")) currencies.add(currency) if family == "fees": fee_total += amount row[prefix + "_amount"] = row.get(prefix + "_amount", 0.0) + amount # Rate-card changes can split charges within a day. Preserve each charge's # per-unit amounts separately; adding those would invent a per-unit rate. charge_counts[prefix] = charge_counts.get(prefix, 0) + 1 charge_prefix = f"{prefix}_charge_{charge_counts[prefix]}" row.update(_flatten_row(charge, charge_prefix)) for source in (record.get("sales", {}).get("orderedProductSales"), record.get("netProceeds", {}).get("total")): if source: currencies.add(_money(source)[1]) if len(currencies) != 1: raise ValueError("Economics row must carry one currency") row["currency_code"] = next(iter(currencies)) row["fees_total_amount"] = fee_total rows.append(_origin(row, document_id, position, "analytics_economics_2024_03_15")) return rows def economics_summary(records): """Sum the separate coarser vendor response, independently of the detail flattener.""" totals = {} for record in records: sales, currency = _money(record["sales"]["orderedProductSales"]) net, net_currency = _money(record["netProceeds"]["total"]) if net_currency != currency: raise ValueError("Economics summary mixes currencies") fee_total = 0.0 for fee in record.get("fees") or []: for charge in fee.get("charges") or ([fee["charge"]] if fee.get("charge") else []): fee_amount, fee_currency = _money(charge["aggregatedDetail"]["totalAmount"]) if fee_currency != currency: raise ValueError("Economics summary fee currency differs") fee_total += fee_amount group = totals.setdefault(currency, {"sales": 0.0, "fees": 0.0, "net": 0.0}) for key, value in (("sales", sales), ("fees", fee_total), ("net", net)): group[key] += value return totals def _breakdowns(value): # Amazon's published example wraps this list; its model declares a direct array. return (value.get("breakdowns") if isinstance(value, dict) else value) or [] def _pivot_breakdowns(value, row, prefix): for part in _breakdowns(value): name = _snake(amazon_schema._snake(part["breakdownType"])) amount, currency = _money(part["breakdownAmount"]) column = f"{prefix}_breakdown_{name}_amount" row[column] = row.get(column, 0.0) + amount row[column + "_currency_code"] = currency for child in _breakdowns(part.get("breakdowns")): child_name = _snake(amazon_schema._snake(child["breakdownType"])) child_amount, child_currency = _money(child["breakdownAmount"]) column = f"{prefix}_sub_breakdown_{name}__{child_name}_amount" row[column] = row.get(column, 0.0) + child_amount row[column + "_currency_code"] = child_currency def transaction_rows(transactions, operation_id): """One row per item. Transaction totals/breakdowns live only on item position zero.""" rows, ids = [], set() for transaction in transactions: transaction_id = transaction.get("transactionId") if not transaction_id or transaction_id in ids: raise ValueError("Finances returned a missing or duplicate transactionId") ids.add(transaction_id) amount, currency = _money(transaction.get("totalAmount")) order_id = next((x.get("relatedIdentifierValue") for x in transaction.get("relatedIdentifiers") or [] if x.get("relatedIdentifierName") == "ORDER_ID"), None) for position, item in enumerate(transaction.get("items") or [{}]): row = _flatten_row({k: v for k, v in transaction.items() if k not in ("items", "breakdowns", "totalAmount")}) row.update(transaction_id=str(transaction_id), item_position=position, amazon_order_id=order_id, total_amount=amount if position == 0 else None, currency_code=currency, transaction_item_count=len(transaction.get("items") or [{}]) if position == 0 else None) row["posted_date"] = _utc(transaction["postedDate"]).isoformat().replace("+00:00", "Z") row.update(_flatten_row({k: v for k, v in item.items() if k != "breakdowns"}, "item")) context = next((c for c in item.get("contexts") or [] if c.get("contextType") == "ProductContext" or c.get("sku")), {}) row.update(sku=context.get("sku"), asin=context.get("asin")) if position == 0: _pivot_breakdowns(transaction.get("breakdowns"), row, "transaction") _pivot_breakdowns(item.get("breakdowns"), row, "item") rows.append(_origin(row, operation_id, len(rows) + 1, "listTransactions")) return rows def transaction_summary(transactions): totals = {} for transaction in transactions: amount, currency = _money(transaction.get("totalAmount")) group = totals.setdefault(currency, {"amount": 0.0, "count": 0}) group["amount"] += amount group["count"] += 1 return totals def _competitive_rows(responses): """Amazon batch responses -> complete flattened enrichments keyed by returned ASIN.""" out = [] for response in responses or (): body = (response or {}).get("body") or {} asin = str(body.get("asin") or "").strip() if not asin: continue row = _flatten_row(response or {}, "competitive") row["competitive_body_asin"] = asin out.append(row) return out def _awd_rows(inventory, operation_id=None): """AWD inventory summaries -> flattened rows that remain distinguishable from FBA ledger.""" rows = [] # AWD is a paginated operation, not a report. Retain its full response-list position and # an explicitly operation-prefixed capture ID instead of inventing an Amazon document ID. operation_id = operation_id or "operation:" + uuid.uuid4().hex for row_number, item in enumerate(inventory or (), 1): row = _flatten_row(item or {}) row["inventory_source"] = "AWD" rows.append(_origin(row, operation_id, row_number, "listInventory")) return rows def _tag_fba_inventory(rows): """Mark report rows so one combined inventory table never conflates FBA with AWD.""" if isinstance(rows, _DocumentRows): return _stamp_rows(rows, {"inventory_source": "FBA"}) for row in rows or (): row["inventory_source"] = "FBA" return rows def _replace_awd_snapshot(con, rows): """Atomically replace this account workspace's AWD snapshot, including an empty response.""" mirror = table_name("inventory") columns = sorted({"inventory_source"} | {column for row in rows for column in row}) con.execute("BEGIN TRANSACTION") try: types = _ensure(con, mirror, columns, rows, preserve_rows=True) con.execute(f"DELETE FROM {mirror} WHERE inventory_source='AWD'") count = _replace(con, mirror, columns, types, rows) con.execute("COMMIT") return count except Exception: con.execute("ROLLBACK") raise def _pricing_asins(rows): """Distinct child ASINs from Sales and Traffic rows, preserving source order.""" seen, out = set(), [] for row in rows or (): asin = str(row.get("child_asin") or row.get("parent_asin") or "").strip() if asin and asin not in seen: seen.add(asin) out.append(asin) return out def _apply_competitive(con, table, responses): """Replace the mirror's competitive fields with this pass's responses on matching ASIN rows. ⛔ A row whose ASIN did not answer this pass is cleared, never left holding an older price shown as current; `validate()` counts competitive ASINs against this pass's responses. """ have = _columns_of(con, table) personal = set(personal_columns(table)) cipher = None if personal: from harness.pii_cipher import PiiCipher cipher = PiiCipher() stale = sorted(c for c in have if c.startswith("competitive_")) if stale: con.execute(f"UPDATE {table} SET " + ", ".join(f'"{c}"=NULL' for c in stale)) rows = _competitive_rows(responses) if not rows: return 0, 0 cols = sorted({key for row in rows for key in row}) match_columns = [c for c in ("child_asin", "parent_asin") if c in _columns_of(con, table)] if not match_columns: raise ValueError("Pricing mirror has no ASIN column; sync Sales and Traffic before price enrichment") types = _ensure(con, table, cols, rows) assignments = ", ".join(f'"{c}"=?' for c in cols) # A parent ASIN's summary prices only a row with no child ASIN, never that parent's SKU rows. where = ('"child_asin"=? OR ("child_asin" IS NULL AND "parent_asin"=?)' if len(match_columns) == 2 else f'"{match_columns[0]}"=?') updated = 0 for row in rows: asin = row["competitive_body_asin"] values = [cipher.encrypt(row.get(c), table=table.removeprefix(PREFIX), column=c) if c in personal else _cell(row.get(c), types.get(c, "VARCHAR")) for c in cols] result = con.execute(f'UPDATE {table} SET {assignments} WHERE {where}', values + [asin] * len(match_columns)) try: updated += int(result.fetchone()[0]) except Exception: # noqa: BLE001 pass return len(rows), updated def merge_columns(pinned, parsed): """The column set for a `schema_artifact` table: the pin, plus anything the payload really had. ⛔⛔ AMAZON'S OWN ARTIFACT CONTRADICTS ITSELF, WHICH IS WHY THIS IS A UNION AND NOT THE PIN. `sellerSalesAndTrafficReport.json` declares `mobileAppPageViews` in `properties` while its own `examples` carry `mobilePageViews`, four columns' worth. `properties` is the right authority for a COLUMN LIST; the payload is the only authority on what actually arrived. Taking the pin alone would DROP a field Amazon really returned, silently, which is the one thing R2's *"do not drop any column"* forbids and which nothing would go red for. [[a-published-fixture-corpus-can-answer-wrongly]] ⭐ PINNED ORDER FIRST, so the grid still follows the schema and the surprises land at the end where a reader notices them. ⚠ IT IS A NAMED FUNCTION RATHER THAN THREE LINES INSIDE `sync` BECAUSE THE GATE HAS TO DRIVE THE REAL ONE. A first version inlined it, so `verify_amazon`'s pipeline check computed the same union itself and stayed green when the production line was broken -- the check's ruler was its own copy of the logic. One implementation, two callers. """ have = set(pinned or ()) return list(pinned or ()) + [c for c in (parsed or ()) if c not in have] def _coltype(name, rows=None): """The DuckDB type for one column, decided by the DATA when there is data. ⛔ A FLAT FILE HAS NO TYPES AT ALL: every cell is text, so an order report's `quantity` arrives as `"3"` and its `item-price` as `"27.30"`. Sniffing them is what makes the grid sortable and the `validate()` sums possible. But the sniff is CONSERVATIVE and text wins ties, because a column that is numeric for 500 rows and then carries `N/A` must not make the insert fail after five tables are already written — the exact failure `meta_store._coltype` was rewritten for. """ if name in _ORIGIN_TYPES: return _ORIGIN_TYPES[name] if name in _FORCE_TEXT: return "VARCHAR" if not rows: return "VARCHAR" if isinstance(rows, _DocumentRows): return rows.sniffed(name) seen, kinds = 0, set() for r in rows: v = r.get(name) if v is None or v == "": continue kinds.add(_value_kind(v)) seen += 1 if seen >= 500: break return _kinds_type(kinds) def _cell(value, kind): """One parsed value -> one DuckDB cell of the declared type. Never raises on bad input.""" if value is None or value == "": return None if kind == "VARCHAR": return value if isinstance(value, str) else json.dumps( value, separators=(",", ":"), ensure_ascii=False) if isinstance( value, (list, dict)) else str(value) try: return int(str(value).strip()) if kind == "BIGINT" else float(str(value).strip()) except (TypeError, ValueError): return None def _ensure(con, table, cols, rows=None, declared=None, *, preserve_rows=False): """Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched. ⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data that can be re-pulled. A type disagreement drops and recreates rather than leaving a mirror permanently unwritable over one bad guess. """ # ⛔⛔ A DECLARED TYPE BEATS A SNIFFED ONE, AND THE PIPELINE CHECK PROVED WHY WITH ONE ROW. # Amazon's own example has `buyBoxPercentage: 100` -- a whole number -- so sniffing typed the # column BIGINT, and the very next real value (91.2) would have been truncated to 91 or refused # outright. The schema DECLARES it `number`. **Sniffing is the right answer only where there is # no schema to ask**, which is exactly the `measured_at_sync` tables; where Amazon publishes a # type, that type is the authority and one sample cannot overrule it. # ⚠ A column the schema does not declare still sniffs -- that is how the union's extra fields # (the vendor's own examples/properties disagreement) get a usable type at all. scope = _account_scope() if scope is not None: for column in (amazon_schema.CONNECTION_ID, amazon_schema.STORE): if column not in cols: cols.append(column) declared = {**(declared or {}), amazon_schema.CONNECTION_ID: "VARCHAR", amazon_schema.STORE: "VARCHAR"} personal = set(personal_columns(table)).intersection(cols) if personal: from harness.pii_cipher import PiiCipher PiiCipher() declared = dict(declared or {}) declared.update(dict.fromkeys(personal, "VARCHAR")) want = {c: (declared.get(c) or _coltype(c, rows)) for c in cols} # ⛔⛔ GUARDED, AND IT WAS UNGUARDED FIRST, WHICH WOULD HAVE BROKEN THE VERY FIRST SYNC. # DuckDB RAISES `CatalogException` from `PRAGMA table_info` on a table that does not exist yet # -- it does not return an empty list -- and "does not exist yet" is exactly the state every # table is in on its first ever run. The end-to-end pipeline check found this the first time it # drove a real document through; no unit-level check could, because each one worked on a table # some earlier step had already created. try: have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()} except Exception: # noqa: BLE001 have = {} if have: clash = [c for c, ty in want.items() if c in have and have[c] != ty and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")] if clash and preserve_rows: # Settlement document markers preclude replay: widening must retain prior lines. # The measured types are VARCHAR, BIGINT and DOUBLE; never narrow historical data. for c in clash: widened = "VARCHAR" if "VARCHAR" in (have[c], want[c]) else "DOUBLE" if have[c] != widened: con.execute(f'ALTER TABLE "{table}" ALTER COLUMN "{c}" TYPE {widened}') have[c] = want[c] = widened elif clash: con.execute(f"DROP TABLE {table}") have = {} if not have: body = ", ".join(f'"{c}" {want[c]}' for c in cols) con.execute(f"CREATE TABLE IF NOT EXISTS {table} ({body})") return want for c in cols: if c not in have: con.execute(f'ALTER TABLE {table} ADD COLUMN "{c}" {want[c]}') return want def personal_columns(table): """Return only columns the Amazon schema explicitly marks personal.""" name = str(table).removeprefix("ut_amzn_").removeprefix(PREFIX) spec = amazon_schema.TABLE_SPECS.get(name) or {} return tuple(column["name"] for column in spec.get("columns", []) if column.get("personal") is True) def decrypt_personal_rows(table, rows): """Decrypt marked cells in rows already admitted by the caller's permission walls.""" from harness.pii_cipher import PiiCipher name = str(table).removeprefix("ut_amzn_").removeprefix(PREFIX) marked = personal_columns(name) cipher = None result = [] for row in rows: admitted = dict(row) for column in marked: if column in admitted and admitted[column] is not None: cipher = cipher or PiiCipher() admitted[column] = cipher.decrypt(admitted[column], table=name, column=column) result.append(admitted) return result def _decode_scoped_rows(table_key, rows, *, user, st): """Audit each projected personal column before returning decrypted, permitted rows.""" rows = list(rows) columns = tuple(column for column in personal_columns(table_key) if any(column in row for row in rows)) if not columns: return rows tenant = getattr(st, "key", None) principal = user.get("username") if isinstance(user, dict) else None if (not isinstance(tenant, str) or not tenant.strip() or not isinstance(principal, str) or not principal.strip()): raise AmazonAuditUnavailable("Amazon personal reads require a trusted user and tenant runtime") tenant = _tenant_key(tenant) admitted = decrypt_personal_rows(table_key, rows) stamp = _control_stamp() for column in columns: _c2_event("amazon.personal_read", tenant, principal, {"table": table_key, "column": column, "read_at": stamp}, st) return admitted connector_spec.register_scoped_row_decoder("amazon", _decode_scoped_rows) def _assert_personal_storage(con, table, *, source=None): """Refuse plaintext personal cells before snapshot publication or installation.""" from harness.pii_cipher import PiiCiphertextError, is_envelope columns = personal_columns(table) if not columns: return relation = source or _quoted_identifier(table) present = {item[0] for item in con.execute(f"SELECT * FROM {relation} LIMIT 0").description} for column in set(columns).intersection(present): cursor = con.execute(f"SELECT {_quoted_identifier(column)} FROM {relation} " f"WHERE {_quoted_identifier(column)} IS NOT NULL") while batch := cursor.fetchmany(1000): if any(not is_envelope(value) for (value,) in batch): raise PiiCiphertextError(f"Amazon personal column {table}.{column} requires AES-256-GCM ciphertext") def _replace(con, table, cols, types, rows, *, source_table=None): """Insert one window. The sync caller clears the table ONCE before its first window.""" if not rows: return 0 name = str(source_table or table).removeprefix(PREFIX) personal = set(personal_columns(name)).intersection(cols) cipher = None if personal: from harness.pii_cipher import PiiCipher cipher = PiiCipher() if not set(_ORIGIN_TYPES).issubset(cols) or (rows.unattributed() if isinstance(rows, _DocumentRows) else any( any(r.get(c) in (None, "") for c in _ORIGIN_TYPES) or type(r.get("_report_row_number")) is not int or r["_report_row_number"] < 1 for r in rows)): raise ValueError("refusing Amazon rows without complete source attribution") ph = "(" + ", ".join("?" for _ in cols) + ")" insert = f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES ' per = max(1, min(INSERT_BATCH_ROWS, INSERT_BATCH_VALUES // len(cols))) full = insert + ", ".join([ph] * per) # D-9924: a spooled report document inserts through typed frames, one DuckDB scan per batch. frames = _frame_writer(con, table, cols, types) if isinstance(rows, _DocumentRows) and not personal else None per_batch = max(1, FRAME_BATCH_VALUES // len(cols)) if frames is not None else per def write(batch): if frames is not None and frames(batch): return for i in range(0, len(batch), per): part = batch[i:i + per] con.execute(full if len(part) == per else insert + ", ".join([ph] * len(part)), [value for row in part for value in row]) # Multi-row statements in batches: no copy of every row exists at once, the caller's transaction # keeps the window all-or-nothing, and values bind and cast exactly as `executemany` did. count, batch = 0, [] for r in rows: scope = _account_scope() if scope is not None: r = {**r, amazon_schema.CONNECTION_ID: str(scope["entry"]["connection_id"]), amazon_schema.STORE: str(scope["entry"]["label"])} batch.append([cipher.encrypt(r.get(c), table=name, column=c) if c in personal else _cell(r.get(c), types.get(c, "VARCHAR")) for c in cols]) if len(batch) == per_batch: write(batch) count, batch = count + len(batch), [] if batch: write(batch) count += len(batch) return count #: Values per typed frame on the spooled-document insert path; D-7081 bounds the Python peak by it. FRAME_BATCH_VALUES = 100_000 _FRAME_DTYPES = {"BIGINT": "Int64", "DOUBLE": "Float64"} def _frame_writer(con, table, cols, types): """A batch writer that appends typed pandas frames, or None where pandas, `from_df` or the column order differs. The writer returns False for a batch pandas cannot type, and the caller binds that batch as values. """ if not callable(getattr(con, "from_df", None)): return None try: import pandas as pd except Exception: # noqa: BLE001 return None # D-7081: a registered view outlives `unregister` until COMMIT and pins a window's frames. A relation # append releases each batch but is positional, so another column set or order binds values instead. if [d[0] for d in con.execute(f"SELECT * FROM {table} LIMIT 0").description] != list(cols): return None dtypes = [_FRAME_DTYPES.get(types.get(c, "VARCHAR"), "string") for c in cols] def write(batch): try: columns = list(zip(*batch)) frame = pd.DataFrame({f"c{i}": pd.array(list(columns[i]), dtype=dtypes[i]) for i in range(len(cols))}) except (TypeError, ValueError, OverflowError): return False con.from_df(frame).insert_into(table) return True return write def replace_keys_for(table): """Replace on declared identity excluding lineage, which changes on every fetch.""" return tuple(c for c in (amazon_schema.TABLE_SPECS.get(table) or {}).get("row_identity", ()) if c not in _ORIGIN_TYPES) @contextmanager def _report_transaction(con): con.execute("BEGIN TRANSACTION") try: yield con.execute("COMMIT") except Exception: con.execute("ROLLBACK") raise def replace_window(con, table, cols, types, rows, keys, after=None, before=None): """Replace incoming key groups atomically, or report why only insertion is possible. `after(con)` runs inside the same transaction, after the insert and before the commit. """ missing = [c for c in keys if c not in cols] if missing or not keys or not rows: note = (f"{table}: window replacement unavailable; missing identity columns: " f"{', '.join(missing)}; rows inserted; use a report carrying these columns") if missing else None if before is None and after is None: return _replace(con, table, cols, types, rows), note with _report_transaction(con): if before is not None: before(con) count = _replace(con, table, cols, types, rows) if after is not None: after(con) return count, note quote = lambda name: '"' + str(name).replace('"', '""') + '"' target = quote(table) # The window's rows are staged in batches inside the transaction, typed as the mirror and # converted by the insert's own cells, so NULL and numeric keys agree and no key list exists. temporary = quote("amazon_window_rows_" + uuid.uuid4().hex) columns = ", ".join(quote(c) for c in cols) con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE") try: predicate = " AND ".join( f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys) with _report_transaction(con): if before is not None: before(con) count = _replace(con, temporary, cols, types, rows, source_table=table) inserted = cols if table == table_name("brand_analytics") and {"_period_start", "_period_end"} <= set(keys): _normalise_week_periods(con, target) _normalise_week_periods(con, temporary) inserted = _preserve_brand_week_identity(con, target, temporary, cols, keys) con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})') inserted_columns = ", ".join(quote(c) for c in inserted) con.execute(f"INSERT INTO {target} ({inserted_columns}) SELECT {inserted_columns} FROM {temporary}") if after is not None: after(con) return count, None finally: con.execute(f"DROP TABLE IF EXISTS {temporary}") def _weekly_date_sql(column): """Read an ISO period's calendar date without interpreting unknown legacy text as a date.""" text = f"CAST({column} AS VARCHAR)" return (f"CASE WHEN regexp_full_match({text}, '\\d{{4}}-\\d{{2}}-\\d{{2}}([T ].+)?') " f"AND TRY_CAST({column} AS TIMESTAMP) IS NOT NULL " f"THEN TRY_CAST(substr({text}, 1, 10) AS DATE) END") def _normalise_week_periods(con, relation): """Canonicalize known ISO week dates without filling unknown periods or changing saved PIDs.""" for name in ("_period_start", "_period_end"): col = _quoted_identifier(name) day = _weekly_date_sql(col) con.execute(f"UPDATE {relation} SET {col} = CAST({day} AS VARCHAR) " f"WHERE {day} IS NOT NULL AND {col} IS DISTINCT FROM CAST({day} AS VARCHAR)") def _preserve_brand_week_identity(con, target, incoming, cols, keys): """Carry prepared PIDs across calendar-equivalent report refreshes with one set-based join.""" pid = amazon_schema.IDENTITY_PID if pid not in {row[0] for row in con.execute(f"DESCRIBE {target}").fetchall()}: return cols qpid = _quoted_identifier(pid) if pid not in cols: con.execute(f"ALTER TABLE {incoming} ADD COLUMN {qpid} BIGINT") con.execute(f"UPDATE {incoming} SET {qpid} = NULL") predicate = " AND ".join(f'"old".{_quoted_identifier(c)} IS NOT DISTINCT FROM "k".{_quoted_identifier(c)}' for c in keys) complete = " AND ".join(f'NULLIF(CAST("k".{_quoted_identifier(c)} AS VARCHAR), \'\') IS NOT NULL' for c in keys) con.execute(f'UPDATE {incoming} SET {qpid} = saved.pid FROM ' f'(SELECT "k".rowid AS rid, min("old".{qpid}) AS pid FROM {incoming} AS "k" ' f'JOIN {target} AS "old" ON {predicate} WHERE {complete} GROUP BY "k".rowid) saved ' f'WHERE {incoming}.rowid = saved.rid') return [*cols, *(() if pid in cols else (pid,))] def _windows(start, end, size=MAX_WINDOW_DAYS): """Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden.""" out, cur = [], start while cur < end: nxt = min(cur + timedelta(days=size), end) out.append((cur, nxt)) cur = nxt return out def _month_windows(start, end): """Completed calendar months only; never a sliding 30-day storage-fee period.""" current = _utc(start).replace(day=1, hour=0, minute=0, second=0, microsecond=0) stop = _utc(end).replace(day=1, hour=0, minute=0, second=0, microsecond=0) out = [] while current < stop: following = (current.replace(day=28) + timedelta(days=4)).replace(day=1) out.append((current, following)) current = following return out def _week_start(value): """The Sunday 00:00 UTC on or before `value`: Amazon's WEEK report runs Sunday to Saturday.""" day = _utc(value).replace(hour=0, minute=0, second=0, microsecond=0) return day - timedelta(days=(day.weekday() + 1) % 7) def _week_windows(start, end): """Completed Sunday to Saturday weeks only, as half-open UTC windows.""" current, stop, out = _week_start(start), _week_start(end), [] while current < stop: out.append((current, current + timedelta(days=7))) current += timedelta(days=7) return out def _periodic(spec): """A report that lands one completed calendar period, a month or a week, per window.""" return bool((spec or {}).get("monthly") or (spec or {}).get("weekly")) def _period_windows(spec, start, end): return _week_windows(start, end) if spec.get("weekly") else _month_windows(start, end) def stamp_week(cols, rows, w0, w1): """Stamp a window's report period on every row, for a table whose identity names the period.""" period = {"_period_start": _utc(w0).date().isoformat(), "_period_end": _utc(w1).date().isoformat()} return [*cols, *(c for c in period if c not in cols)], _stamp_rows(rows, period) def prepare_brand_analytics(con): """Add nullable period fields and precompute stable numeric identities on the writer connection.""" table = table_name("brand_analytics") names = set(_columns_of(con, table)) if not names: return False ordinal = amazon_schema.IDENTITY_OCCURRENCE changed = False for name in ("_period_start", "_period_end"): if name not in names: con.execute(f'ALTER TABLE {table} ADD COLUMN "{name}" VARCHAR') names.add(name) changed = True if ordinal not in names: con.execute(f'ALTER TABLE {table} ADD COLUMN "{ordinal}" BIGINT DEFAULT 0') changed = True pid = amazon_schema.IDENTITY_PID if pid not in names: con.execute(f'ALTER TABLE {table} ADD COLUMN "{pid}" BIGINT') changed = True if not changed and not con.execute(f'SELECT 1 FROM {table} WHERE "{pid}" IS NULL LIMIT 1').fetchone(): return False identity = amazon_schema.row_key_columns_for("brand_analytics", names) fallback = sorted(set((*identity, *amazon_schema.row_fallback_columns_for("brand_analytics")))) def value(name): return f'NULLIF(CAST("{name}" AS VARCHAR), \'\')' if name in names else "NULL::VARCHAR" complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity) partition = ", ".join(value(name) for name in fallback) if con.execute(f'SELECT 1 FROM {table} WHERE "{pid}" IS NULL AND NOT ({complete}) LIMIT 1').fetchone(): # Saved ordinals survive reordered Parquet parts. New equal rows follow their largest saved ordinal. con.execute(f'UPDATE {table} SET "{ordinal}" = ranked.occurrence FROM ' f'(SELECT rowid AS rid, coalesce(max(CASE WHEN "{pid}" IS NOT NULL THEN "{ordinal}" END) ' f'OVER (PARTITION BY {partition}), -1) + row_number() OVER ' f'(PARTITION BY {partition}, "{pid}" IS NULL ORDER BY rowid) AS occurrence ' f'FROM {table} WHERE NOT ({complete})) ranked ' f'WHERE {table}.rowid = ranked.rid AND {table}."{pid}" IS NULL') def digest(values): return f"sha256(to_json(list_value({', '.join(values)})))" fallback_digest = digest(["'__amazon_missing_identity__'", *(value(name) for name in fallback), f'CAST("{ordinal}" AS VARCHAR)']) digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} " f"ELSE {fallback_digest} END") con.execute(f'UPDATE {table} SET "{pid}" = ' f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT)) WHERE \"{pid}\" IS NULL") return True def _weekly_period_repair(con, table, end): """Choose an evidenced completed week or the preceding completed week with a full week of processing time.""" saved = _read_state(con, entity_key("reconcile_" + table)) try: proof = json.loads(saved["detail"]) if saved and saved.get("state") == "done" else {} start, stop = _utc(proof["start"]), _utc(proof["end"]) if (proof.get("count", 0) > 0 and start == _week_start(start) and stop == start + timedelta(days=7) and stop <= _week_start(end)): return start, stop except (KeyError, TypeError, ValueError, AttributeError, OverflowError): pass stop = _week_start(end) - timedelta(days=7) return stop - timedelta(days=7), stop def watched_terms(tenant_key): """R18's watched set -> `(terms, None)`, or `(None, problem)` when the reader fails.""" module_name, _, name = WATCHED_TERMS.partition(":") try: import importlib module = importlib.import_module(module_name) except ModuleNotFoundError as exc: if exc.name == module_name or module_name.startswith(f"{exc.name}."): return set(), None return None, _watched_problem(exc) except Exception as exc: # noqa: BLE001 return None, _watched_problem(exc) try: return {str(term).strip().lower() for term in getattr(module, name)(tenant_key) or () if str(term or "").strip()}, None except Exception as exc: # noqa: BLE001 return None, _watched_problem(exc) def _watched_problem(exc): return (f"brand_analytics: watched search terms unreadable ({type(exc).__name__}); older weeks " "are kept in full this pass and pruned when the list reads again.") def _retain_weekly(con, mirror, watched): """R18: the newest week stays in full; every older period keeps watched search terms only.""" quote = lambda name: '"' + str(name).replace('"', '""') + '"' terms = quote("amazon_watched_terms_" + uuid.uuid4().hex) target = quote(mirror) period = _weekly_date_sql('"_period_start"') con.execute(f"CREATE TEMP TABLE {terms} (term VARCHAR)") try: if watched: con.executemany(f"INSERT INTO {terms} VALUES (?)", [[term] for term in sorted(watched)]) row = con.execute( f'DELETE FROM {target} WHERE {period} IS NOT NULL AND {period} IS DISTINCT FROM ' f'(SELECT MAX({period}) FROM {target}) ' f"AND (search_term IS NULL OR lower(trim(search_term)) NOT IN (SELECT term FROM {terms}))").fetchone() return int(row[0]) if row else 0 finally: con.execute(f"DROP TABLE IF EXISTS {terms}") def write_week(con, table, cols, types, rows, tenant_key, report, log=print): """Land one week by its period identity and apply R18's retention in the same transaction.""" watched, problem = watched_terms(tenant_key) if problem: report.setdefault("problems", []).append(problem) log(problem) mirror = table_name(table) if table == "brand_analytics": prepare_brand_analytics(con) after = None if watched is None else (lambda c: _retain_weekly(c, mirror, watched)) result = replace_window(con, mirror, cols, types, rows, replace_keys_for(table), after=after) if table == "brand_analytics": prepare_brand_analytics(con) return result def _record_brand_analytics_limit(con): _record_state(con, entity_key("limit_brand_analytics"), "done", detail=json.dumps({"reason": BRAND_ANALYTICS_LIMIT})) def _week_not_ready(exc): """Amazon's FATAL for a week it has not finished processing, which is a wait and never a refusal.""" return (getattr(exc, "status", None) == "FATAL" and bool(re.search(r"\bnot available yet\b", str(getattr(exc, "vendor_reason", "") or exc), re.I))) def _vendor_error_details(exc): """Amazon's `errorDetails` text from a FATAL error document, else its whole reason.""" text = str(getattr(exc, "vendor_reason", "") or exc) try: parsed = json.loads(text) except (ValueError, TypeError): return text[:500] return str(parsed.get("errorDetails") or text)[:500] if isinstance(parsed, dict) else text[:500] def _record_week_wait(con, table, w0, w1, exc): """B2 contract: `waiting_` names the unfinished week; `weekEnd` is its Saturday, as Amazon names it.""" _record_state(con, entity_key("waiting_" + table), "waiting", detail=json.dumps({ "weekStart": _utc(w0).date().isoformat(), "weekEnd": (_utc(w1) - timedelta(days=1)).date().isoformat(), "error": _vendor_error_details(exc)})) def _clear_week_wait(con, table, w0): """A landed week at or after the waited one deletes its wait: B2's readers do not filter on phase.""" key = entity_key("waiting_" + table) saved = _read_state(con, key) if not saved or saved.get("state") != "waiting": return try: week = json.loads(saved.get("detail") or "{}").get("weekStart") or "" except (ValueError, TypeError, AttributeError): week = "" if not week or week <= _utc(w0).date().isoformat(): con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) def _newest_week(con, mirror): """The newest ISO `_period_start` date the mirror holds, or None.""" if "_period_start" not in set(_columns_of(con, mirror)): return None day = _weekly_date_sql('"_period_start"') row = con.execute(f"SELECT MAX({day}) FROM {mirror}").fetchone() return row[0] if row else None def sqp_batches(asins, max_chars=200): """Stable, lossless space-joined options within Amazon's character ceiling.""" if type(max_chars) is not int or max_chars < 1: raise ValueError("Search Query Performance needs a positive ASIN option limit") unique = sorted(set(asins)) batches, current = [], "" for asin in unique: if not isinstance(asin, str) or not asin or any(c.isspace() for c in asin): raise ValueError(f"Invalid catalogue ASIN for Search Query Performance: {asin!r}") if len(asin) > max_chars: raise ValueError(f"ASIN {asin} exceeds Amazon's {max_chars}-character option limit") candidate = f"{current} {asin}" if current else asin if len(candidate) > max_chars: batches.append(current) current = asin else: current = candidate return batches + ([current] if current else []) def _sqp_catalogue(con): """Our mirror only: prefer the listings catalogue, fall back to pricing if empty.""" for table, column in (("amzn_listings", "asin1"), ("amzn_pricing", "child_asin")): columns = {r[0] for r in con.execute( "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()} if column in columns: asins = sorted({str(r[0]).strip() for r in con.execute( f'SELECT DISTINCT "{column}" FROM "{table}" WHERE "{column}" IS NOT NULL').fetchall() if str(r[0]).strip()}) if asins: return asins return [] _ASIN_RE = re.compile(r"[A-Z0-9]{10}") def _columns_of(con, table): return {r[0] for r in con.execute( "SELECT column_name FROM information_schema.columns WHERE table_catalog=current_database() " "AND table_schema=current_schema() AND table_name=?", [table]).fetchall()} def _catalog_walk_asins(con): """Walk the listed catalogue plus every sales child ASIN without widening SQP's catalogue.""" asins = set(_sqp_catalogue(con)) if "child_asin" in _columns_of(con, "amzn_economics"): asins.update(str(row[0]).strip() for row in con.execute( "SELECT DISTINCT child_asin FROM amzn_economics WHERE child_asin IS NOT NULL").fetchall() if str(row[0]).strip()) return sorted(asins) _NEWEST_ORDER = ("start_date", "_period_start", "_snapshot_date", "_report_row_number") def _newest_child_parent_sql(columns, table, only=None): """SELECT child_asin, parent_asin: one row per child, the newest period's parent; None if absent. `only` is an optional SQL SELECT of the child ASINs wanted. It filters BEFORE the window, which keeps or drops whole `child_asin` partitions, so every kept child's answer is unchanged. Once the catalog carries parents (its first sync after S67-T29), `asin_parents_sql` passes only the few own ASINs the catalog leaves without one, so S67-T30's per-page lookup stops ranking all of economics (572,320 rows on staging, about 100 ms a query) to answer them. """ if not {"child_asin", "parent_asin"} <= columns: return None order = ", ".join(f'"{c}" DESC NULLS LAST' for c in _NEWEST_ORDER if c in columns) or "1" wanted = f" AND TRIM(CAST(child_asin AS VARCHAR)) IN ({only})" if only else "" return (f'SELECT child_asin, parent_asin FROM (SELECT TRIM(CAST(child_asin AS VARCHAR)) AS child_asin, ' f'TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, ROW_NUMBER() OVER (' f'PARTITION BY TRIM(CAST(child_asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM "{table}" ' f"WHERE child_asin IS NOT NULL AND parent_asin IS NOT NULL " f"AND TRIM(CAST(parent_asin AS VARCHAR)) <> ''{wanted}) WHERE _rn = 1") def _child_parent_map(con, table): """child ASIN -> parent ASIN from the newest rows of `table`; {} when it has no child_asin.""" sql = _newest_child_parent_sql(_columns_of(con, table), table) if sql is None: return {} return {str(c): str(p) for c, p in con.execute(sql).fetchall() if _ASIN_RE.fullmatch(str(c)) and _ASIN_RE.fullmatch(str(p))} def _pricing_parent_map(con): """O23 fallback. Used only when pricing HAS child_asin: the live Sales and Traffic mirror is stored at parent granularity (parent_asin only), so this is {} there and economics carries on.""" return _child_parent_map(con, "amzn_pricing") def _economics_parent_map(con): """Data Kiosk economics child_asin -> parent_asin, newest start_date wins.""" return _child_parent_map(con, "amzn_economics") def _catalog_parent(item, asin, pricing_map, marketplace, economics_map=None): """One item's (parent_asin, source); never raises on a missing or malformed `relationships`.""" parents, is_parent = [], False groups = item.get("relationships") for group in groups if isinstance(groups, list) else (): if not isinstance(group, dict): continue if marketplace and group.get("marketplaceId") not in (None, marketplace): continue entries = group.get("relationships") for rel in entries if isinstance(entries, list) else (): # Only VARIATION defines a parent; PACKAGE_HIERARCHY is a pack, not a family. if not isinstance(rel, dict) or rel.get("type") != "VARIATION": continue found = rel.get("parentAsins") parents += [str(a).strip() for a in (found if isinstance(found, list) else ()) if _ASIN_RE.fullmatch(str(a).strip())] if isinstance(rel.get("childAsins"), list) and rel["childAsins"]: is_parent = True if parents: return parents[0], "relationships" if is_parent: return asin, "relationships" for source, mapping in (("pricing", pricing_map), ("economics", economics_map)): fallback = str((mapping or {}).get(asin) or "").strip() if _ASIN_RE.fullmatch(fallback): return fallback, source return asin, "self" def catalog_parent_map(items, pricing_map=None, marketplace=None, economics_map=None): """S67-T29 (O23): own ASIN -> (parent ASIN, source) for every catalog item. Source order: the catalog VARIATION `parentAsins` for the requested marketplace; an item that is itself a variation parent maps to itself; else pricing's `parent_asin` for that ASIN as `child_asin` (O23); else economics child_asin -> parent_asin; else itself. A standalone ASIN therefore maps to itself, never an error. """ out = {} for item in items or (): if not isinstance(item, dict): continue asin = str(item.get("asin") or "").strip() if asin: out[asin] = _catalog_parent(item, asin, pricing_map, marketplace, economics_map) return out #: S67 lane D (S67-T29 finding): Sales and Traffic defaults `asinGranularity` to PARENT, so the #: saved `amzn_pricing` mirror carried `parent_asin` with no `child_asin` and no `sku`, and #: `connector_spec`'s pricing identity `(child_asin, parent_asin)` and its two listings links never #: formed. SKU grain returns parentAsin, childAsin and sku per row: the SAME report type and role #: (the PRD's Amazon compliance line: no new endpoint or role). REPORT_OPTIONS = {"pricing": {"asinGranularity": "SKU"}} def _pricing_regrain(con, mirror, rows, log=print): """Clear a PARENT-grain pricing mirror before the first SKU-grain rows land; True if cleared. `amzn_pricing` keeps history across incremental passes, so without this the old parent-grain days would sit beside SKU-grain days and any sum across days would count twice. A mirror that already holds any `child_asin`, an empty or absent mirror, or rows without `child_asin` are left alone. """ if not any((row or {}).get("child_asin") for row in rows): return False have = _columns_of(con, mirror) # empty for an absent table (first load), never raises if not have: return False if "child_asin" in have and con.execute( f'SELECT COUNT(*) FROM {mirror} WHERE "child_asin" IS NOT NULL').fetchone()[0]: return False stale = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] if not stale: return False con.execute(f"DELETE FROM {mirror} WHERE 1=1") log(f" pricing: replaced {stale} parent-grain rows with SKU-grain rows (one grain per table)") return True def _latest_account_snapshot_sql(columns, table): """Select each connection and marketplace's latest snapshot, using available partition columns.""" sql = f'SELECT * FROM "{table}" AS snapshot' if "_snapshot_date" not in columns: return sql partitions = [name for name in ("_connection_id", "marketplace_id") if name in columns] match = " AND ".join(f'latest."{name}" IS NOT DISTINCT FROM snapshot."{name}"' for name in partitions) return (sql + f' WHERE snapshot."_snapshot_date" = (SELECT MAX(latest."_snapshot_date") ' f'FROM "{table}" AS latest' + (" WHERE " + match if match else "") + ")") def asin_parents_sql(con): """S67-T29: ONE read-time resolver, one row per own ASIN: `(asin, parent_asin, parent_asin_source)`. Own ASINs = DISTINCT listings `asin1` UNION each account/marketplace's latest catalog `asin`. Legacy catalogs without account columns use their latest global date. Parent = that latest catalog row's `parent_asin` (source as stamped at sync), else pricing child_asin -> parent_asin when the pricing mirror carries child_asin (source `pricing`, O23), else economics child_asin -> parent_asin (newest start_date, source `economics`), else the ASIN itself (`self`). Covers own ASINs Amazon's catalog search returns no item for, without stub catalog rows. Degrades with absent tables or columns; None only when no own-ASIN source exists. Reusable verbatim by a SQL join (S67-T30): wrap it as a subquery. """ listings, catalog = _columns_of(con, "amzn_listings"), _columns_of(con, "amzn_catalog_items") current_catalog = _latest_account_snapshot_sql(catalog, "amzn_catalog_items") own = [] if "asin1" in listings: own.append("SELECT TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM amzn_listings WHERE asin1 IS NOT NULL") has_catalog = {"asin", "_snapshot_date"} <= catalog if has_catalog: own.append("SELECT TRIM(CAST(asin AS VARCHAR)) AS asin " f"FROM ({current_catalog}) AS current_catalog WHERE asin IS NOT NULL") if not own: return None ctes = ["own AS (SELECT DISTINCT asin FROM (" + " UNION ".join(own) + ") WHERE asin <> '')"] joins, parents, sources = [], [], [] if has_catalog and "parent_asin" in catalog: source = ("CAST(parent_asin_source AS VARCHAR)" if "parent_asin_source" in catalog else "'relationships'") order = '"_report_row_number"' if "_report_row_number" in catalog else "1" ctes.append( "cat AS (SELECT asin, parent_asin, parent_asin_source FROM (SELECT TRIM(CAST(asin AS VARCHAR)) AS asin, " "TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, " f"COALESCE({source}, 'relationships') AS parent_asin_source, ROW_NUMBER() OVER (" f"PARTITION BY TRIM(CAST(asin AS VARCHAR)) ORDER BY {order}) AS _rn " f"FROM ({current_catalog}) AS current_catalog WHERE parent_asin IS NOT NULL " "AND TRIM(CAST(parent_asin AS VARCHAR)) <> '') WHERE _rn = 1)") joins.append("LEFT JOIN cat ON cat.asin = own.asin") parents.append("cat.parent_asin") sources.append("WHEN cat.parent_asin IS NOT NULL THEN cat.parent_asin_source") # Economics is only the fallback, so rank only the own ASINs the catalog left without a parent. only = "SELECT asin FROM own" + (" WHERE asin NOT IN (SELECT asin FROM cat)" if joins else "") # O23: pricing is the fallback BEFORE economics. None (no step) while the pricing mirror has no # `child_asin`, which is the live Sales and Traffic mirror today (S67-T29 finding). pricing = _newest_child_parent_sql(_columns_of(con, "amzn_pricing"), "amzn_pricing", only=only) if pricing: ctes.append(f"pri AS ({pricing})") joins.append("LEFT JOIN pri ON pri.child_asin = own.asin") parents.append("pri.parent_asin") sources.append("WHEN pri.parent_asin IS NOT NULL THEN 'pricing'") economics = _newest_child_parent_sql(_columns_of(con, "amzn_economics"), "amzn_economics", only=only) if economics: ctes.append(f"eco AS ({economics})") joins.append("LEFT JOIN eco ON eco.child_asin = own.asin") parents.append("eco.parent_asin") sources.append("WHEN eco.parent_asin IS NOT NULL THEN 'economics'") return ("WITH " + ", ".join(ctes) + " SELECT own.asin AS asin, " f"COALESCE({', '.join([*parents, 'own.asin'])}) AS parent_asin, " + (f"CASE {' '.join(sources)} ELSE 'self' END" if sources else "'self'") + " AS parent_asin_source " "FROM own " + " ".join(joins) + " ORDER BY own.asin") def asin_parents(con): """`asin_parents_sql` as `{asin: (parent_asin, source)}`; {} when no own-ASIN source exists.""" sql = asin_parents_sql(con) return {} if sql is None else {a: (p, s) for a, p, s in con.execute(sql).fetchall()} def sqp_limit_sentence(marker): """Pure projection of saved sync facts; never asks Amazon for a catalogue.""" numbers = [marker[k] for k in ("maxChars", "maxPerBatch", "asinCount", "batchCount")] if any(type(n) is not int or n < 0 for n in numbers) or not all(numbers[:2]): raise ValueError("Invalid saved Search Query Performance limits") chars, maximum, count, batches = numbers sentence = (f"Amazon accepts at most {chars} characters of ASINs per Search Query Performance " f"report, about {maximum} ASINs, so {count} ASINs are requested in {batches} reports per month.") progress = sqp_progress_sentence(marker) return f"{sentence} {progress}" if progress else sentence def sqp_progress_sentence(detail): """Batches done of total and the time left, from saved SQP progress; "" without progress.""" progress = detail.get("progress") if not progress: return "" steps = [progress.get(k) for k in ("monthsDone", "monthsTotal", "batchesDone", "batchesTotal", "etaSeconds")] if any(type(n) is not int or n < 0 for n in steps): raise ValueError("Invalid saved Search Query Performance progress") months_done, months, done, total, eta = steps weeks_done, weeks = progress.get("weeksDone", 0), progress.get("weeksTotal", 0) if any(type(n) is not int or n < 0 for n in (weeks_done, weeks)): raise ValueError("Invalid saved Search Query Performance weekly progress") weekly = f", {weeks_done:,} of {weeks:,} weeks complete" if weeks else "" sentence = f"{months_done:,} of {months:,} months complete{weekly}, {done:,} of {total:,} batches." if eta: hours, minutes = round(eta / 3600), max(1, round(eta / 60)) left = (f"{hours:,} hours" if hours >= 2 else "1 minute" if minutes == 1 else f"{minutes} minutes") sentence += f" About {left} left at Amazon's rate of one report a minute." return sentence def sqp_refusal_message(asin, reason): return f"Amazon refused ASIN {asin} for Search Query Performance: {reason}" def _sqp_period(start, end): return "WEEK" if _utc(end) - _utc(start) == timedelta(days=7) else "MONTH" def _sqp_month_key(start, period="MONTH"): suffix = _utc(start).strftime("%Y_%m_%d_WEEK" if period == "WEEK" else "%Y_%m") return entity_key("reconcile_search_query_performance_" + suffix) def _sqp_month(con, start, end): period = _sqp_period(start, end) state = _read_state(con, _sqp_month_key(start, period)) return json.loads(state["detail"]) if state and state.get("detail") else { "period": period, "start": _utc(start).date().isoformat(), "end": (_utc(end) - timedelta(days=1)).date().isoformat(), "batches": {}, "asins": {}, "refusedAsins": {}, "retries": {}} def _sqp_save_month(con, start, marker): _record_state(con, _sqp_month_key(start, marker.get("period", "MONTH")), "done", detail=json.dumps(marker)) _record_reconciliation(con, SQP_TABLE, marker) def _sqp_done(con, start, end, batch): return not _sqp_missing_asins(_sqp_month(con, start, end), batch.split()) def _sqp_missing_asins(marker, asins): """Successful ASIN summaries and durable named exclusions complete a period independently of batching.""" done = set(marker["asins"]) | set(marker.get("refusedAsins", {})) for batch, retry in marker["retries"].items(): done.update(set(batch.split()) - set(retry["asins"])) return [asin for asin in asins if asin not in done] def _sqp_owed_tasks(con, tasks): """Batch only each period's missing ASINs, retaining unfinished retry identities across catalogue changes.""" windows = {} for task in tasks: if task[0] == SQP_TABLE: windows.setdefault(task[1:3], []).extend(task[3].split()) result = [] for task in tasks: if task[0] != SQP_TABLE: result.append(task) continue asins = windows.pop(task[1:3], None) if asins is None: continue marker = _sqp_month(con, *task[1:3]) missing = _sqp_missing_asins(marker, list(dict.fromkeys(asins))) remaining, batches = set(missing), [] for batch, retry in marker["retries"].items(): requested = set(retry["asins"]) if requested and requested <= remaining: batches.append(batch) remaining.difference_update(requested) batches.extend(sqp_batches([asin for asin in missing if asin in remaining], _client().REPORT_TYPES[SQP_TABLE]["asin_option_max_chars"])) result.extend((SQP_TABLE, *task[1:3], batch) for batch in batches) return result def _sqp_week_tasks(start, end, batches): """Schedule every completed week newest first through the same ASIN batch queue.""" if "WEEK" not in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()): return [] return [(SQP_TABLE, w0, w1, batch) for w0, w1 in reversed(_week_windows(start, end)) for batch in batches] def _sqp_history_done(con, end): """Every admitted period and current catalogue ASIN needs its own durable completion evidence.""" asins = _sqp_catalogue(con) tasks = first_load_plan([SQP_TABLE], history_floor(con, SQP_TABLE, end), end, sqp_asins=asins, starts=history_starts(con, [SQP_TABLE], end)) return bool(asins) and all(_sqp_done(con, task[1], task[2], task[3]) for task in tasks) def _sqp_progress(con, months, weeks, asins, batches, end): """Count durable per-ASIN completion separately for admitted MONTH and WEEK windows.""" from harness.amazon_client import REPORT_SUSTAINED_SECONDS progress = {} for period, windows, unit, label in (("MONTH", months, "month", "%Y-%m"), ("WEEK", weeks, "week", "%Y-%m-%d")): floor = history_floor(con, SQP_TABLE, end, period=period) rows = [] for w0, w1 in windows: if w0 < floor: continue missing = set(_sqp_missing_asins(_sqp_month(con, w0, w1), asins)) rows.append({unit: w0.strftime(label), "total": len(batches), "done": sum(not missing.intersection(batch.split()) for batch in batches)}) progress[unit + "s"] = rows progress[unit + "sDone"] = sum(r["done"] >= r["total"] for r in rows) progress[unit + "sTotal"] = len(rows) rows = progress["months"] + progress["weeks"] done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows) progress.update(batchesDone=done, batchesTotal=total, etaSeconds=int((total - done) * REPORT_SUSTAINED_SECONDS)) return progress if rows and batches else None def _sqp_first_load_status(con, cohort): """Project current period floors and per-ASIN evidence through the shared SQP progress shape.""" asins = _sqp_catalogue(con) spec = _client().REPORT_TYPES[SQP_TABLE] months = _month_windows(history_floor(con, SQP_TABLE, cohort), max(cohort, datetime.now(timezone.utc))) weeks = (_week_windows(history_floor(con, SQP_TABLE, cohort, period="WEEK"), cohort) if "WEEK" in spec.get("periods", ()) else []) progress = _sqp_progress(con, months, weeks, asins, sqp_batches(asins, spec["asin_option_max_chars"]), cohort) return {"progress": progress, "sentence": sqp_progress_sentence({"progress": progress})} if progress else None class SQPBudgetDeferred(RuntimeError): """A batch or its single retry remains durable for the next bounded pass.""" class SQPPassDeferred(SQPBudgetDeferred): """A scheduled pass spent `SQP_PASS_REPORTS`; only SQP stops, other tables keep running.""" def _sqp_document_rows(text, spec, requested, marker, meta): """Validate the source document before flattening rows and computing per-ASIN totals.""" text = _document_text(text) doc = json.loads(text) if not isinstance(doc, dict): raise ValueError("Search Query Performance document must be a JSON object") raw = doc.get(spec["json_root"]) if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw): raise ValueError("Search Query Performance document has no dataByAsin row array") totals = {asin: {"count": 0, "purchases": 0} for asin in requested} for row in raw: asin = row.get("asin") if not isinstance(asin, str): raise ValueError("Search Query Performance ASIN must be a string") if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]: raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch") purchase_data = row.get("purchaseData") if not isinstance(purchase_data, dict): raise ValueError("Search Query Performance purchaseData must be a JSON object") purchase = purchase_data.get("asinPurchaseCount") if type(purchase) is not int or purchase < 0: raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count") totals[asin]["count"] += 1 totals[asin]["purchases"] += purchase columns, rows = parse_json_report(text, spec["json_root"], document_id=meta.get("reportDocumentId"), report_type=spec["report"]) seen = set() for row in rows: key = amazon_schema.row_identity_for(SQP_TABLE, row) if key in seen: raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query") seen.add(key) row["_report_period"] = marker.get("period", "MONTH") return columns, rows, totals, len(raw) def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem): """One completed period and batch using the shared report transport and pass budget. A brand refusal is durable before the retry, so a budget boundary or process restart resumes without the named ASINs. Only a successful document can replace mirror rows. """ cx = _client() from harness.amazon_client import AmazonReportPending spec = cx.REPORT_TYPES[SQP_TABLE] marker = _sqp_month(con, start, end) retry = marker["retries"].get(batch) if retry and retry.get("failed"): raise cx.AmazonError(retry["failed"]) requested = retry["asins"] if retry else batch.split() while requested: try: text, meta = request(fields, spec["report"], start=_iso(start), end=_iso(end - timedelta(seconds=1)), options={"reportPeriod": marker.get("period", "MONTH"), "asin": " ".join(requested)}) break except (SQPBudgetDeferred, AmazonReportPending): raise except cx.AmazonError as exc: # A period outside Amazon's data range is a history floor, never an ASIN refusal. out_of_range = bool(_AMAZON_FLOOR_TEXT.search(str(getattr(exc, "vendor_reason", "") or exc))) refused = [] if out_of_range else cx.sqp_refused_asins(exc, requested) reason = getattr(exc, "vendor_reason", str(exc)) reason = reason.replace("\u2014", "; ").replace("\u2013", " to ") for asin in refused: limit["refusedAsins"][asin] = reason marker.setdefault("refusedAsins", {})[asin] = reason problem(sqp_refusal_message(asin, reason)) if not refused: limit["errors"] = [reason] _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(limit)) if retry: retry["failed"] = str(exc) _sqp_save_month(con, start, marker) raise if not refused: raise requested = [asin for asin in requested if asin not in refused] retry = marker["retries"][batch] = {"asins": requested} _sqp_save_month(con, start, marker) else: # Amazon refused every ASIN. Record the exclusion, without inventing a source summary. marker["batches"][batch] = {"asins": [], "count": 0, "purchases": 0, "refused": True} _sqp_save_month(con, start, marker) return 0 columns, rows, totals, source_count = _sqp_document_rows(text, spec, requested, marker, meta) identity = amazon_schema.TABLE_SPECS[SQP_TABLE]["row_identity"] columns = list(dict.fromkeys([*columns, *identity, "_report_period", "purchase_data_asin_purchase_count", *_ORIGIN_TYPES])) mirror = table_name(SQP_TABLE) types = _ensure(con, mirror, columns, rows, preserve_rows=True, declared={"purchase_data_asin_purchase_count": "BIGINT"}) marker["asins"].update(totals) marker["batches"][batch] = {"asins": requested, "count": source_count, "purchases": sum(t["purchases"] for t in totals.values()), "reportDocumentId": meta["reportDocumentId"]} con.execute("BEGIN TRANSACTION") try: # Empty documents remove stale rows too; other batches and months remain untouched. con.execute(f"DELETE FROM {mirror} WHERE start_date=? AND end_date=? " f"AND asin IN ({','.join('?' for _ in requested)})", [marker["start"], marker["end"], *requested]) count = _replace(con, mirror, columns, types, rows) _sqp_save_month(con, start, marker) refresh_sku_keywords(con) con.execute("COMMIT") except Exception: con.execute("ROLLBACK") raise return count def _validate_sqp(con): state = _read_state(con, entity_key("reconcile_" + SQP_TABLE)) if not state or not state.get("detail"): return {"state": "unproven", "reason": "Amazon Search Query Performance summary is absent"} try: marker = json.loads(state["detail"]) expected = {key: sum(t[key] for t in marker["asins"].values()) for key in ("count", "purchases")} actual_rows = con.execute( f"SELECT asin, COUNT(*), SUM(purchase_data_asin_purchase_count), " f"COUNT(purchase_data_asin_purchase_count) FROM {table_name(SQP_TABLE)} " "WHERE start_date=? AND end_date=? GROUP BY asin", [marker["start"], marker["end"]]).fetchall() actual = {"count": sum(r[1] for r in actual_rows), "purchases": sum(r[2] or 0 for r in actual_rows)} if not actual_rows or not marker["asins"]: return {"state": "unproven", "reason": "latest Search Query Performance month has no proven rows", "expected": expected, "actual": actual} per_asin = {r[0]: {"count": r[1], "purchases": r[2]} for r in actual_rows} matches = (expected == actual and all(r[1] == r[3] for r in actual_rows) and all(per_asin.get(a, {"count": 0, "purchases": 0}) == v for a, v in marker["asins"].items()) and set(per_asin) <= set(marker["asins"])) return {"state": "ok" if matches else "refused", "expected": expected, "actual": actual, "reason": f"Amazon summary {expected}; stored detail {actual}"} except Exception as exc: return {"state": "refused", "reason": f"Search Query Performance reconciliation failed: {exc}"} def _sku_keyword_source_fields(columns): from harness.amazon_sqp_reads import report_period_sql fields = {"start_date": "start_date", "end_date": "end_date", "asin": "asin", "search_query": "search_query_data_search_query", "impressions": "impression_data_asin_impression_count", "clicks": "click_data_asin_click_count", "cart_adds": "cart_add_data_asin_cart_add_count", "purchases": "purchase_data_asin_purchase_count"} expressions = [] for name, source in fields.items(): numeric = name in ("impressions", "clicks", "cart_adds", "purchases") kind, cast = ("BIGINT", "TRY_CAST") if numeric else ("VARCHAR", "CAST") expressions.append(f'{cast}("{source}" AS {kind}) AS {name}' if source in columns else f'NULL::{kind} AS {name}') period = report_period_sql(columns) if {"start_date", "end_date"} <= columns else "NULL::VARCHAR" return ", ".join([*expressions, f"{period} AS _report_period"]) def refresh_sku_keywords(con): """Bind a local derived view to listing summaries and raw SQP without another source copy.""" listing_columns = _columns_of(con, "amzn_listing_items") if "sku" not in listing_columns: con.execute("DROP VIEW IF EXISTS amzn_keyword_ranks") con.execute("DROP VIEW IF EXISTS amzn_sku_keywords") return False summaries = "TRY_CAST(summaries AS JSON)" if "summaries" in listing_columns else "NULL::JSON" keywords = "CAST(backend_keywords AS VARCHAR)" if "backend_keywords" in listing_columns else "NULL::VARCHAR" raw_columns = _columns_of(con, table_name(SQP_TABLE)) scoped = amazon_schema.CONNECTION_ID in listing_columns listing_meta = ', "_connection_id", store' if scoped else "" mapping_meta = ', l."_connection_id"' if scoped else "" account_group = ', "_connection_id"' if scoped else "" raw_meta = (', "_connection_id"' if amazon_schema.CONNECTION_ID in raw_columns else ', NULL::VARCHAR AS "_connection_id"') if scoped else "" mapping_join = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" # An SQP row saved before account tagging has no account; it belongs to a single-account tenant's listings. query_join = (' AND (q."_connection_id" IS NOT DISTINCT FROM l."_connection_id" OR (q."_connection_id" IS NULL' ' AND (SELECT COUNT(DISTINCT "_connection_id") FROM listings) = 1))') if scoped else "" count_join = ' AND c."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" grain_meta = ', "_connection_id"' if scoped else "" matched_meta = ', l."_connection_id", l.store' if scoped else "" query_columns = 'q.* EXCLUDE ("_connection_id")' if scoped else "q.*" output_meta = ', "_connection_id", store' if scoped else "" backend_meta = ', l."_connection_id", l.store' if scoped else "" backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" select = _sku_keyword_source_fields(raw_columns) raw_from = " FROM amzn_search_query_performance" if raw_columns else " WHERE FALSE" con.execute(f""" CREATE OR REPLACE VIEW amzn_sku_keywords AS WITH listings AS ( SELECT CAST(sku AS VARCHAR) AS sku, COALESCE({keywords}, '') AS backend_keywords, {summaries} AS summaries{listing_meta} FROM amzn_listing_items WHERE sku IS NOT NULL AND CAST(sku AS VARCHAR) <> '' ), mapping AS ( SELECT DISTINCT l.sku, json_extract_string(j.value, '$.asin') AS asin{mapping_meta} FROM listings l, json_each(l.summaries) j WHERE NULLIF(json_extract_string(j.value, '$.asin'), '') IS NOT NULL ), mapped_counts AS ( SELECT asin{account_group}, COUNT(DISTINCT sku) AS mapped_sku_count FROM mapping GROUP BY asin{account_group} ), raw AS (SELECT {select}{raw_meta}{raw_from}), queries AS ( SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain FROM raw WHERE _report_period IS NOT NULL ), matched AS ( SELECT l.sku, {query_columns}, l.backend_keywords, c.mapped_sku_count{matched_meta} FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join} JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join} ) SELECT md5(CAST(to_json([sku, source_grain]) AS VARCHAR)) AS keyword_id, sku, asin, CASE WHEN _report_period='MONTH' THEN substr(start_date, 1, 7) || '-01' END AS month, start_date, end_date, search_query, impressions, clicks, cart_adds, purchases, backend_keywords, source_grain, mapped_sku_count, 'Amazon ASIN/query metrics shared across ' || CAST(mapped_sku_count AS VARCHAR) || ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis, _report_period{output_meta} FROM matched UNION ALL SELECT md5(CAST(to_json([l.sku, 'backend_keywords'{', l."_connection_id"' if scoped else ''}]) AS VARCHAR)), l.sku, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, l.backend_keywords, NULL, NULL, 'Listing backend keywords only. No matching Search Query Performance rows.', NULL::VARCHAR{backend_meta} FROM listings l WHERE NOT EXISTS (SELECT 1 FROM matched m WHERE m.sku=l.sku{backend_match}) """) refresh_keyword_ranks(con) return True def refresh_keyword_ranks(con): """Join uncapped observations to each SKU's latest exact query or backend-only keyword row.""" if not _columns_of(con, "amzn_apify_rank_observations") or not _columns_of(con, "amzn_sku_keywords"): con.execute("DROP VIEW IF EXISTS amzn_keyword_ranks") return False summaries = ("TRY_CAST(l.summaries AS JSON)" if "summaries" in _columns_of(con, "amzn_listing_items") else "NULL::JSON") scoped = amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_sku_keywords") partition_meta = ', k."_connection_id"' if scoped else "" matched_meta = ', k."_connection_id" AS _rank_connection_id, k.store AS _rank_store' if scoped else "" observation_match = (' AND r."_connection_id" IS NOT DISTINCT FROM k."_connection_id"' if scoped and amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_apify_rank_observations") else "") listing_match = ' AND l."_connection_id" IS NOT DISTINCT FROM k."_connection_id"' if scoped else "" rank_meta = ', _rank_connection_id' if scoped else "" output_meta = ', _rank_connection_id AS "_connection_id", _rank_store AS store' if scoped else "" con.execute(f""" CREATE OR REPLACE VIEW amzn_keyword_ranks AS WITH matched AS ( SELECT r.*, k.keyword_id, k.sku{matched_meta}, row_number() OVER (PARTITION BY r.observation_id, k.sku{partition_meta} ORDER BY k.start_date DESC, k.keyword_id) AS recency FROM amzn_apify_rank_observations r JOIN amzn_sku_keywords k ON ((r.asin=k.asin AND lower(trim(r.keyword))=lower(trim(k.search_query))) OR (k.search_query IS NULL AND EXISTS ( SELECT 1 FROM amzn_listing_items l, json_each({summaries}) j WHERE l.sku=k.sku AND json_extract_string(j.value, '$.asin')=r.asin{listing_match}) AND EXISTS (SELECT 1 FROM unnest(string_split(k.backend_keywords, ';')) t(term) WHERE lower(trim(term))=lower(trim(r.keyword))))) AND (k.start_date <= r.date OR k.start_date IS NULL){observation_match} ) SELECT sha256(CAST(to_json([observation_id, keyword_id{rank_meta}]) AS VARCHAR)) AS rank_id, observation_id, keyword_id, sku, keyword, asin, date, organic_rank, sponsored_rank, global_position, is_sponsored, sponsored_rank_basis, dataset_id{output_meta} FROM matched WHERE recency=1 """) return True def import_keyword_ranks(con, rows, dataset_id): """Replace one dataset atomically while retaining every raw observation and its vendor JSON.""" from core.connector_spec import AMAZON_FIELDS fields = {field["name"]: field["type"] for field in AMAZON_FIELDS["ut_amzn_keyword_ranks"] if field["name"] not in {"rank_id", "keyword_id", "sku", amazon_schema.STORE}} fields["source_json"] = "VARCHAR" if not dataset_id or any(row.get("dataset_id") != dataset_id for row in rows): raise ValueError("Amazon rank dataset attribution is missing or inconsistent") columns = list(fields) con.execute("BEGIN TRANSACTION") try: _ensure(con, "amzn_apify_rank_observations", columns, rows, declared=fields, preserve_rows=True) con.execute("DELETE FROM amzn_apify_rank_observations WHERE dataset_id=?", [dataset_id]) if rows: con.executemany('INSERT INTO amzn_apify_rank_observations (' + ', '.join('"' + column + '"' for column in columns) + ') VALUES (' + ', '.join('?' for _ in columns) + ')', [[row.get(column) for column in columns] for row in rows]) refresh_keyword_ranks(con) _record_state(con, entity_key("apify_keyword_ranks"), "done", rows=len(rows), detail={"dataset_id": dataset_id}) con.execute("COMMIT") except Exception: con.execute("ROLLBACK") raise linked = con.execute("SELECT count(*) FROM amzn_keyword_ranks WHERE dataset_id=?", [dataset_id]).fetchone()[0] if _columns_of(con, "amzn_keyword_ranks") else 0 return {"state": "ok", "observations": len(rows), "linked_rows": linked, "reason": amazon_schema.PROVENANCE_COPY["apify_dataset"], "changed_tables": ("amzn_apify_rank_observations",)} def _validate_keyword_ranks(con): """Count raw observations independently from the SKU projection and expose unmatched captures.""" if not _columns_of(con, "amzn_apify_rank_observations"): return {"state": "unproven", "reason": "No Apify keyword rank dataset has been imported"} observations = con.execute("SELECT count(*) FROM amzn_apify_rank_observations").fetchone()[0] if not _columns_of(con, "amzn_keyword_ranks"): return {"state": "unproven", "observations": observations, "reason": "SKU keyword source is absent. Raw Apify observations are retained."} linked, distinct_observations, invalid = con.execute(""" SELECT count(*), count(DISTINCT observation_id), count(*) FILTER ( WHERE (is_sponsored AND (organic_rank IS NOT NULL OR sponsored_rank IS NULL)) OR (NOT is_sponsored AND sponsored_rank IS NOT NULL)) FROM amzn_keyword_ranks """).fetchone() return {"state": "refused" if invalid else "ok", "observations": observations, "linked_rows": linked, "unmatched_observations": observations - distinct_observations, "reason": amazon_schema.PROVENANCE_COPY["apify_dataset"]} def _validate_sku_keywords(con): """Tie distinct derived grains and purchases to raw rows separately for every MONTH and WEEK.""" from harness.amazon_sqp_reads import _independent_period basis = amazon_schema.PROVENANCE_COPY["derived"] columns = _columns_of(con, table_name(SQP_TABLE)) if not columns: return {"state": "unproven", "reason": "Search Query Performance source is absent", "source_basis": basis} try: declared = "_report_period" if "_report_period" in columns else "NULL::VARCHAR" account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in columns else "" derived_account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_sku_keywords") else "" raw = con.execute(f""" SELECT CAST(start_date AS VARCHAR), CAST(end_date AS VARCHAR), list(DISTINCT {declared}), COUNT(*), COUNT(DISTINCT (asin, search_query_data_search_query{account})), SUM(purchase_data_asin_purchase_count), COUNT(purchase_data_asin_purchase_count), COUNT(*) FILTER (WHERE purchase_data_asin_purchase_count < 0 OR start_date IS NULL OR end_date IS NULL OR asin IS NULL OR search_query_data_search_query IS NULL) FROM amzn_search_query_performance GROUP BY ALL """).fetchall() derived = con.execute(f""" SELECT start_date, end_date, _report_period, COUNT(*), SUM(purchases), SUM(invalid) FROM ( SELECT start_date, end_date, _report_period, asin, search_query{derived_account}, MIN(purchases) AS purchases, CASE WHEN COUNT(purchases) <> COUNT(*) OR MIN(purchases) <> MAX(purchases) OR COUNT(*) <> COUNT(DISTINCT sku) OR COUNT(*) FILTER (WHERE month IS DISTINCT FROM CASE WHEN _report_period='MONTH' THEN substr(start_date, 1, 7) || '-01' END) > 0 THEN 1 ELSE 0 END AS invalid FROM amzn_sku_keywords WHERE source_grain IS NOT NULL GROUP BY ALL ) GROUP BY ALL """).fetchall() actual = {(start, end, period): {"grains": n, "purchases": purchases, "invalid": invalid} for start, end, period, n, purchases, invalid in derived} held, months = {}, {} for start, end, declarations, count, distinct, purchases, present, invalid in raw: labels = {_independent_period(start, end, declared) for declared in declarations} period = labels.pop() if len(labels) == 1 else None key = (start, end, period) observed = actual.get(key, {"grains": 0, "purchases": 0, "invalid": 0}) good = (period is not None and count == distinct == present and not invalid and not observed["invalid"] and observed["grains"] == distinct and observed["purchases"] == purchases) held[key] = {"state": "ok" if good else "refused", "start": start, "end": end, "period": period, "expected": {"grains": distinct, "purchases": purchases}, "actual": observed} if period == "MONTH": months[start[:7]] = held[key] extra = set(actual) - set(held) good = bool(held) and not extra and all(row["state"] == "ok" for row in held.values()) return {"state": "ok" if good else "refused" if held or extra else "unproven", "months": months, "periods": list(held.values()), "source_basis": basis, "reason": ("Distinct ASIN/query grains reconcile to raw SQP for every period" if good else "SQP coverage or purchases differ. Unmapped ASINs, duplicate grains and missing counts cannot reconcile")} except Exception as exc: return {"state": "refused", "source_basis": basis, "reason": f"SKU keyword reconciliation failed: {exc}"} def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS, sqp_asins=(), starts=None): """R18's request order: every recent month, then each table's history, oldest last. Settlements have one discovery slot (None, None), never a createReport window. This planner is pure; durable floors filter its fixed windows when a pass resumes. SQP adds a fourth tuple member, its ASIN option, for each month x batch task. `starts` gives a table its own oldest bound (`history_starts`); a table it omits uses `start`. With `starts`, a weekly table also plans its older weeks, after every other table's history. """ start, end = _utc(start), _utc(end) if starts is not None: starts = {table: _utc(value) for table, value in starts.items() if value is not None} elif start >= end: return [] size = min(float(size), MAX_WINDOW_DAYS) recent_days = FIRST_LOAD_RECENT_DAYS if recent_days is None else recent_days recent_days = min(float(recent_days), size) if size <= 0 or recent_days <= 0: raise ValueError("Amazon report windows must have a positive size.") tables = list(dict.fromkeys(tables)) # Hotfix 2026-09-24: the settlements slot goes LAST among the recent entries. It downloads # every new settlement document until the pass budget is spent and then ends the pass, so in # table order it starved inventory, pricing and brand_analytics of their recent month for # about four passes on live (52 fortnightly documents over 730 days). recent, history, weekly_history = [], [], [] for table in tables: spec = _client().REPORT_TYPES[table] table_start = starts.get(table, start) if starts is not None else start if table == SQP_TABLE: rule = _history_rule(spec, "WEEK") week_start = _date_utc(rule["from"]) if rule.get("from") else table_start week_start = starts.get((table, "WEEK"), week_start) if starts is not None else week_start weekly_history.extend(_sqp_week_tasks(week_start, end, sqp_batches( sqp_asins, spec["asin_option_max_chars"]))) if table == "settlements": continue if spec.get("snapshot"): recent.append((table, None, None)) elif table_start >= end: continue elif spec.get("weekly"): # R18: the latest completed week lands first. Older weeks keep watched terms only, so # with per-table floors they plan after every other table's history, newest first. weeks = _week_windows(table_start, end) recent.extend((table, w0, w1) for w0, w1 in weeks[-1:]) if starts is not None and table in starts: weekly_history.extend((table, w0, w1) for w0, w1 in reversed(weeks[:-1])) elif spec.get("monthly"): months = list(reversed(_month_windows(table_start, end))) if not months: continue tasks = ([(table, w0, w1, batch) for w0, w1 in months for batch in sqp_batches(sqp_asins, spec["asin_option_max_chars"])] if table == SQP_TABLE else [(table, w0, w1) for w0, w1 in months]) recent.extend(task for task in tasks if task[1:3] == months[0]) history.extend(task for task in tasks if task[1:3] != months[0]) else: table_size = spec.get("max_window_days", size) table_recent = max(table_start, end - timedelta(days=min(recent_days, table_size))) recent.append((table, table_recent, end)) history.extend((table, w0, w1) for w0, w1 in reversed(_windows(table_start, table_recent, table_size))) recent.extend((table, None, None) for table in tables if table == "settlements") return recent + history + weekly_history def _sqp_after_cohort(tasks, end, asins): """Insert each monthly table's months closed after the frozen cohort end, newest first, before its older tasks. SQP takes one task per month and ASIN batch; every other monthly table takes one (table, w0, w1) per month. """ cx = _client() later = list(reversed(_month_windows(end, datetime.now(timezone.utc)))) out, seen = [], set() for task in tasks: table = task[0] if later and table not in seen: seen.add(table) spec = cx.REPORT_TYPES.get(table) or {} if table == SQP_TABLE: out.extend((table, w0, w1, batch) for w0, w1 in later for batch in sqp_batches(asins, spec["asin_option_max_chars"])) elif spec.get("monthly"): out.extend((table, w0, w1) for w0, w1 in later) out.append(task) return out #: A periodic table's pass that landed no window says so on its done markers. NOTHING_NEW_DETAIL = "No new completed period landed in this pass." def _after_cohort(table, w0, end): """A non-SQP monthly window that starts in or after the frozen cohort end's month.""" if w0 is None or end is None or table == SQP_TABLE: return False if not (_client().REPORT_TYPES.get(table) or {}).get("monthly"): return False return _utc(w0) >= _utc(end).replace(day=1, hour=0, minute=0, second=0, microsecond=0) def _cohort_month_key(table, w0): return entity_key(f"after_cohort_{table}_{_utc(w0).strftime('%Y-%m')}") def _landed_nothing(table, landed_window, landed): """A periodic table, not SQP, whose block landed no window in this pass.""" return (table != SQP_TABLE and _periodic(_client().REPORT_TYPES.get(table)) and not landed_window and not any(task[0] == table and task[2] is not None for task in landed)) def _cohort_month_done(con, table, w0): """The durable marker of one landed month after the cohort end; the backfill floor never moves for it.""" return (_read_state(con, _cohort_month_key(table, w0)) or {}).get("state") == "done" def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None): """Amazon createReport pacing floor in seconds; polling and other work can add time.""" from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS if report_count <= 0: return 0 stagger = REPORT_STAGGER_SECONDS if stagger_seconds is None else stagger_seconds tokens = REPORT_BURST if burst is None else burst rate = 60 / REPORT_SUSTAINED_SECONDS if per_minute is None else per_minute seconds = max(0, (report_count - 1) * stagger, (report_count - tokens) * 60 / rate) return int(seconds) + (seconds > int(seconds)) def first_load_eta_cause(report_count): """Screen-safe explanation of the vendor ceiling and the automatic continuation.""" from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS return (f"Amazon's report rate allows a burst of {REPORT_BURST}, then one report every " f"{REPORT_SUSTAINED_SECONDS:g} seconds for {report_count} remaining tasks. " f"The report pacing alone needs at least {first_load_eta(report_count, burst=0)} seconds. " "Keep the connection enabled. Per-report refresh floors, API pages and report processing add time.") def _iso(dt): return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def _runtime_for(tenant_key): """The tenant runtime, or None when this process cannot resolve one. Lazy and guarded, so `--status` works with no store reachable.""" try: from harness import runtime as _rt return _rt.get_runtime(str(tenant_key or "")) except Exception: # noqa: BLE001 return None def credential(tenant_key="royal-imports", rt=None, connection_id=None): """The tenant's stored Amazon credential, or None. ⛔ THE KEYCHAIN IS THE ONLY RUNTIME SOURCE, AND THERE IS NO ENVIRONMENT FALLBACK. `AMAZON_SP_*` exists for `connectors_amazon.py --check`, which is an operator proving a freshly minted token before pasting it. Serving it here would hand one tenant another's Amazon seller account, which is precisely the elevation `keychain.TENANT_WIDE_TYPES` was written to stop. """ key = _tenant_key(tenant_key) runtime = rt if rt is not None else _runtime_for(key) # Keep identity failures OUTSIDE the optional-keychain try: fail closed with the diagnosis, # not a silent "no credential". A resolved runtime must match too, not only an explicit one. if runtime is not None: runtime_key = str(getattr(runtime, "key", "") or "").strip().lower() if runtime_key != key: raise RuntimeError( f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}") try: import core.keychain as kc scope = _account_scope() connection_id = connection_id or (scope["entry"]["connection_id"] if scope else None) fields = kc.amazon_creds(runtime, connection_id) if connection_id else kc.amazon_creds(runtime) if runtime is not None: # Delivery clocks keep running while credentials remain stored or cannot decrypt. # Claim the interval before work, including failures; no recursion via credential(). with _RETENTION_LOCK: tick = time.monotonic() previous = _RETENTION_CHECKED.get(key) if previous is None or tick - previous >= 3600: _RETENTION_CHECKED[key] = tick try: enforce_retention(key, runtime) except Exception as exc: print(f"Amazon retention check failed ({type(exc).__name__}); retry on the next check.") return fields except Exception: # noqa: BLE001 return None def _tenant_key(value): """A non-empty canonical tenant slug for a destructive or store-binding operation.""" key = str(value or "").strip().lower() if not key: raise ValueError("an explicit tenant slug is required") return key def _tenant_mirror_path(tenant_key, rt=None): """Resolve the exact tenant mirror path without opening or creating it.""" key = _tenant_key(tenant_key) if rt is not None: runtime_key = str(getattr(rt, "key", "") or "").strip().lower() if runtime_key != key: raise RuntimeError( f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}") return Path(rt.datastore_path) return Path(datastore.path_for(key)) def _connect_tenant_mirror(tenant_key, rt=None): """Bind this single-tenant process to the requested mirror, then open a writer cursor. `datastore.connect()` deliberately takes no path argument. The safe path switch is `datastore.use_path()`, which closes the cached writer and invalidates every read cursor before changing the process-wide file. Calling `connect(path)` looks plausible but raises before a credentialed Amazon sync can write its first row. """ path = _tenant_mirror_path(tenant_key, rt) scope = _account_scope() if scope is not None: if tenant_key != scope["tenant"] or path != scope["path"]: raise RuntimeError("Amazon account workspace does not match the tenant") return scope["root"].cursor(), path candidate = datastore._candidate_connection(tenant_key, path) if candidate is not None: try: return datastore.apply_limits(candidate), path except BaseException: candidate.close() raise if Path(datastore.DB_PATH).resolve() != path.resolve(): datastore.use_path(path) if rt is not None: rt.assert_datastore_matches() return datastore.connect(), path def _require_purge_confirmation(tenant_key, confirmation): """Require the operator to repeat the exact canonical tenant slug.""" key = _tenant_key(tenant_key) if str(confirmation or "").strip() != key: raise RuntimeError( f"refusing permanent deletion: --confirm-tenant must be exactly {key!r}") return key def _assert_amazon_credential_revoked(rt): """Refuse mirror deletion while any Amazon credential metadata still exists. This checks metadata, not decrypted fields. It therefore still catches a credential when the keychain encryption key is unavailable, and an unreadable tenant store raises rather than masquerading as an empty keychain. """ from core import keychain as kc if kc.first_entry_of_type(rt, "amazon_sp") is not None: raise RuntimeError( "refusing permanent deletion: this tenant still has a stored Amazon credential") return True def _quoted_identifier(value): """Quote one database-discovered identifier for a DROP statement.""" return '"' + str(value).replace('"', '""') + '"' class AmazonRetentionOverdue(RuntimeError): """A dated deletion obligation is overdue, not a throttle or operation refusal.""" class AmazonAuditUnavailable(RuntimeError): """The mandatory C2 event failed; its failure is durable in the lifecycle ledger.""" def _utc(value=None): value = value or datetime.now(timezone.utc) if isinstance(value, str): value = datetime.fromisoformat(value.replace("Z", "+00:00")) if value.tzinfo is None or value.utcoffset() is None: raise ValueError("a timezone-aware deletion timestamp is required") return value.astimezone(timezone.utc) def _control_stamp(value=None): # Preserve subsecond ordering: a notice arriving AFTER a purge in the same second is new # work. Report-request windows keep their existing second-precision _iso contract. return _utc(value).isoformat(timespec="microseconds").replace("+00:00", "Z") LIFECYCLE_KEY = "amazon_lifecycle" DELETION_OWNER_SCOPES = ("workbooks", "service_retained_exports", "other_process_and_unattributed_caches") def register_deletion_owner(rt, scope, *, inventory, delete, fixture=False): """Register tenant-bound callbacks that enumerate opaque object IDs and delete one object.""" _ledger_runtime(rt.key, rt) if scope not in DELETION_OWNER_SCOPES or not callable(inventory) or not callable(delete): raise ValueError("A deletion owner needs a known scope and inventory/delete callbacks") if type(fixture) is not bool: raise ValueError("A deletion owner must label fixture evidence explicitly") owners = dict(getattr(rt, "_amazon_deletion_owners", {})) owners[scope] = {"inventory": inventory, "delete": delete, "fixture": fixture} rt._amazon_deletion_owners = owners # Registration after a restart must recover obligations from the durable event ledger. _mark_deletion_owners(rt.key, rt, _utc()) def _owner_inventory(rt, owner): values = owner["inventory"](rt) if not isinstance(values, (list, tuple)) or any( not isinstance(value, str) or not value or len(value) > 512 or any(ord(char) < 32 for char in value) for value in values): raise ValueError("Deletion inventory must contain opaque object IDs without cell values") if len(set(values)) != len(values): raise ValueError("Deletion inventory contains duplicate object IDs") return sorted(values) def _save_owner_obligation(key, rt, row): from copy import deepcopy def save(doc): rows = doc.setdefault("owner_obligations", []) identity = (row["triggerKind"], row["triggerAt"], row["scope"], row["fixture"]) for index, old in enumerate(rows): if (old["triggerKind"], old["triggerAt"], old["scope"], old["fixture"]) == identity: rows[index] = deepcopy(row) return rows.append(deepcopy(row)) rt.update(LIFECYCLE_KEY, save, flush="sync") def _mark_deletion_owners(key, rt, clock): """Persist object-level obligations without changing the event's deletion deadline.""" from copy import deepcopy ledger = _ledger(key, rt) for trigger in _active_deletion_triggers(key, ledger, clock): for scope, owner in getattr(rt, "_amazon_deletion_owners", {}).items(): row = next((deepcopy(old) for old in ledger.get("owner_obligations", []) if old["triggerKind"] == trigger["kind"] and old["triggerAt"] == trigger["occurred_at"] and old["scope"] == scope and old["fixture"] == owner["fixture"]), None) row = row or {"triggerKind": trigger["kind"], "triggerAt": trigger["occurred_at"], "scope": scope, "fixture": owner["fixture"], "objects": []} try: current = _owner_inventory(rt, owner) objects = {item["object"]: item for item in row["objects"]} for object_id in current: item = objects.setdefault(object_id, {"object": object_id, "markedAt": _control_stamp(clock), "deletedAt": None}) # Resurrected content needs fresh deletion evidence. item["deletedAt"] = None if current: row["verifiedAt"] = None row.update(objects=list(objects.values()), inventoriedAt=_control_stamp(clock), cause=None) except Exception: row["cause"] = "inventory_failed" _save_owner_obligation(key, rt, row) def _purge_deletion_owners(key, rt, clock): """Delete marked objects and persist a timestamp only after independent owner readback.""" _assert_amazon_credential_revoked(rt) _mark_deletion_owners(key, rt, clock) ledger = _ledger(key, rt) active = {(row["kind"], row["occurred_at"]) for row in _active_deletion_triggers(key, ledger, clock)} for row in ledger.get("owner_obligations", []): if (row["triggerKind"], row["triggerAt"]) not in active: continue owner = getattr(rt, "_amazon_deletion_owners", {}).get(row["scope"]) if owner is None or owner["fixture"] != row["fixture"]: continue try: if row.get("cause"): continue for item in row["objects"]: if item["deletedAt"]: continue # A disappeared object without this owner's readback is still unproven. owner["delete"](rt, item["object"], checked_at=clock) if item["object"] in _owner_inventory(rt, owner): raise RuntimeError("Deletion owner readback found residue") item["deletedAt"] = _control_stamp(clock) _save_owner_obligation(key, rt, row) if _owner_inventory(rt, owner): raise RuntimeError("Deletion owner discovered new residue") row.update(verifiedAt=row.get("verifiedAt") or _control_stamp(clock), cause=None) except Exception: row["cause"] = "delete_or_readback_failed" _save_owner_obligation(key, rt, row) def _backup_deletion_policy(rt, trigger_at, clock, *, fixture): """Read expiry and expiry verification from the store owner, never from a project plan.""" unknown = {"state": "UNPROVEN", "cause": "store_backup_policy_unavailable"} reader = getattr(rt, "amazon_backup_policy", None) if not callable(reader): return unknown try: policy = reader(trigger_at=trigger_at, checked_at=clock) if (not isinstance(policy, dict) or policy.get("fixture") is not fixture or not all(isinstance(policy.get(field), str) and policy[field].strip() for field in ("policyId", "source", "method", "expiresAt"))): return unknown expiry = _utc(policy["expiresAt"]) verified = _utc(policy["verifiedExpiredAt"]) if policy.get("verifiedExpiredAt") else None if verified and (not expiry <= verified <= clock or verified < _utc(trigger_at)): return unknown return {"state": "VERIFIED" if verified else "DUE", "policyId": policy["policyId"], "source": policy["source"], "method": policy["method"], "fixture": fixture, "expiresAt": _control_stamp(expiry), "verifiedExpiredAt": _control_stamp(verified) if verified else None} except Exception: return unknown def _deletion_owner_evidence(key, rt, ledger, trigger, clock, *, fixture): """Require every registered scope and policy expiry proof for this exact event.""" missing, deleted = [], [] for scope in DELETION_OWNER_SCOPES: owner = getattr(rt, "_amazon_deletion_owners", {}).get(scope) row = next((item for item in ledger.get("owner_obligations", []) if item["scope"] == scope and item["fixture"] is fixture and item["triggerKind"] == trigger["kind"] and item["triggerAt"] == trigger["occurred_at"]), None) try: valid = (owner is not None and owner["fixture"] is fixture and row is not None and not row.get("cause") and row.get("verifiedAt") and _utc(trigger["occurred_at"]) <= _utc(row["verifiedAt"]) <= clock and not _owner_inventory(rt, owner) and all(item.get("deletedAt") and _utc(trigger["occurred_at"]) <= _utc(item["deletedAt"]) <= clock for item in row["objects"])) except Exception: valid = False if valid: deleted.extend(item["deletedAt"] for item in row["objects"]) deleted.append(row["verifiedAt"]) else: missing.append(scope) backup = _backup_deletion_policy(rt, trigger["occurred_at"], clock, fixture=fixture) if backup["state"] != "VERIFIED": missing.append("backups") else: deleted.append(backup["verifiedExpiredAt"]) return {"unverifiedScopes": missing, "deletedAt": max(deleted) if deleted else None, "backup": backup} def deletion_drill_receipt(tenant_key, *, rt, checked_at=None, fixture=False): """Return a value-free receipt for durable triggers, object deletions and store backup expiry.""" from copy import deepcopy key, clock = _tenant_key(tenant_key), _utc(checked_at) rt = _ledger_runtime(key, rt) ledger = _ledger(key, rt) triggers = _active_deletion_triggers(key, ledger, clock) evidence = [dict(triggerAt=trigger["occurred_at"], trigger=trigger["kind"], deadlineAt=_control_stamp(_utc(trigger["occurred_at"]) + timedelta(days=DELETION_DAYS)), **_deletion_owner_evidence(key, rt, ledger, trigger, clock, fixture=fixture)) for trigger in triggers] return {"tenant": key, "evidence": "fixture" if fixture else "service", "checkedAt": _control_stamp(clock), "triggers": evidence, "mirrorObjects": [deepcopy(item) for row in ledger.get("checks", []) if row["kind"] == "purge" and row["success"] and _utc(row["checked_at"]) <= clock for item in row.get("objects", [])], "owners": deepcopy([row for row in ledger.get("owner_obligations", []) if row["fixture"] is fixture and any(row["triggerAt"] == t["occurred_at"] and row["triggerKind"] == t["kind"] for t in triggers)]), "retention_check": retention_check(key, rt=rt, checked_at=clock, scope="fixture" if fixture else "all")} def _ledger_runtime(tenant_key, rt=None): """Resolve the ONE tenant store; an unavailable obligation ledger must fail loudly.""" key = _tenant_key(tenant_key) rt = rt if rt is not None else _runtime_for(key) if rt is None: raise RuntimeError("Amazon lifecycle ledger requires a resolved tenant runtime") runtime_key = str(getattr(rt, "key", "") or "").strip().lower() if runtime_key != key: raise RuntimeError( f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}") return rt def _ledger(tenant_key, rt=None): # The tenant-store document SURVIVES purge and mirror restore: neither touches this key. # Receipts contain opaque local object IDs and UTC dates, never cell values or credentials. # Provider report IDs stay out. It is not an amzn_ object or a _sync_state row, so the mirror's # collateral snapshot stays exact. Fresh reads cannot hide another process's obligation. rt = _ledger_runtime(tenant_key, rt) return rt.get(LIFECYCLE_KEY, fresh=True) or { "triggers": [], "checks": [], "audit_failures": []} def _append_ledger(tenant_key, rt, section, row): rt = _ledger_runtime(tenant_key, rt) def append(doc): for part in ("triggers", "checks", "audit_failures"): doc.setdefault(part, []) doc.setdefault(section, []) # Preserve INSERT OR IGNORE's exact-event idempotency for triggers only. if section != "triggers" or row not in doc[section]: doc[section].append(row) # Commit through the tenant's bound backend before returning; never buffer an obligation. rt.update(LIFECYCLE_KEY, append, flush="sync") def _c2_event(kind, tenant, principal, detail, rt=None): try: # C2 is declared by lane F. Missing or failing audit is NEVER a successful emission. from importlib import import_module audit = import_module("core.audit") if audit.record(kind, tenant, principal, detail) is None: raise RuntimeError("mandatory audit event could not be saved") except Exception as exc: _append_ledger(tenant, rt, "audit_failures", {"tenant": tenant, "kind": kind, "checked_at": _control_stamp()}) raise AmazonAuditUnavailable( f"mandatory C2 event {kind} failed; recorded in the lifecycle ledger") from exc def record_deletion_trigger(tenant_key, trigger, *, occurred_at=None, rt=None, principal="system"): """Record the real event time; calling this schedules an obligation, never deletes data. Credential-revocation, termination and notice owners must call this at their event boundary. Authorization-loss owners use the same door, which also emits C2. Current keychain absence alone is NOT evidence of when revocation occurred. """ key = _tenant_key(tenant_key) if trigger not in DELETION_TRIGGERS: raise ValueError("unknown Amazon deletion trigger") when = _utc(occurred_at) if when > _utc(): raise ValueError("a deletion trigger cannot be dated in the future") rt = _ledger_runtime(key, rt) _append_ledger(key, rt, "triggers", {"tenant": key, "kind": trigger, "occurred_at": _control_stamp(when)}) _mark_deletion_owners(key, rt, _utc()) if trigger == "authorization_loss": _c2_event("amazon.authorization_lost", key, principal, {"trigger": trigger, "occurred_at": _control_stamp(when)}, rt) return retention_check(key, rt=rt) def _active_deletion_triggers(key, ledger, now): """Historical read: a cancellation names one exact event and takes effect only later.""" return sorted((row for row in ledger.get("triggers", []) if row["tenant"] == key and _utc(row["occurred_at"]) <= now and not any(c.get("tenant") == key and c.get("kind") == "cancellation" and c.get("trigger_kind") == row["kind"] and c.get("trigger_occurred_at") == row["occurred_at"] and _utc(row["occurred_at"]) < _utc(c["occurred_at"]) <= now for c in ledger.get("cancellations", []))), key=lambda row: _utc(row["occurred_at"])) def cancel_deletion_trigger(tenant_key, *, rt=None, occurred_at=None, principal="system"): """Reconnection cancels pending credential loss during grace, never an Amazon notice. The original trigger stays intact. A notice or termination needs its own explicit resolution; storing a credential cannot overrule that independent deletion obligation. """ key, now = _tenant_key(tenant_key), _utc(occurred_at) rt = _ledger_runtime(key, rt) from core import keychain as kc if kc.first_entry_of_type(rt, "amazon_sp") is None: return [] cancelled = [] def cancel(doc): cancelled.clear() checks = [r for r in doc.get("checks", []) if r["tenant"] == key and r["kind"] == "purge" and r["success"] and _utc(r["checked_at"]) <= now] for trigger in _active_deletion_triggers(key, doc, now): when = _utc(trigger["occurred_at"]) if (trigger["kind"] not in ("revocation", "authorization_loss") or not when < now < when + timedelta(days=PURGE_AFTER_DAYS) or any(_utc(r["checked_at"]) >= when for r in checks)): continue row = {"tenant": key, "kind": "cancellation", "occurred_at": _control_stamp(now), "principal": str(principal), "trigger_kind": trigger["kind"], "trigger_occurred_at": trigger["occurred_at"]} doc.setdefault("cancellations", []).append(row) cancelled.append(row) # Tenant-store update serializes against concurrent triggers and preserves append-only history. rt.update(LIFECYCLE_KEY, cancel, flush="sync") return cancelled def scheduled_purge(tenant_key, rt, *, now=None, ignore_credential_loss=False): """Read-only policy schedule for the earliest uncancelled, unpurged mirror obligation.""" key, clock = _tenant_key(tenant_key), _utc(now) ledger = _ledger(key, rt) checks = [_utc(r["checked_at"]) for r in ledger.get("checks", []) if r["tenant"] == key and r["kind"] == "purge" and r["success"] and _utc(r["checked_at"]) <= clock] for trigger in _active_deletion_triggers(key, ledger, clock): if ignore_credential_loss and trigger["kind"] in ("revocation", "authorization_loss"): continue when = _utc(trigger["occurred_at"]) if not any(stamp >= when for stamp in checks): return {"purgeAt": _control_stamp(when + timedelta(days=PURGE_AFTER_DAYS)), "deadlineAt": _control_stamp(when + timedelta(days=DELETION_DAYS)), "trigger": dict(trigger)} return None def deletion_schedule_message(plan): trigger = plan["trigger"] date, purge_date = trigger["occurred_at"][:10], plan["purgeAt"][:10] if trigger["kind"] in ("revocation", "authorization_loss"): event = "The Amazon credential was removed" if trigger["kind"] == "revocation" else "Amazon authorization was lost" return (f"{event} on {date}. Amazon data in this workspace will be deleted on {purge_date} " "unless the credential is added again before then.") return (f"Amazon data deletion was requested on {date}. " f"Amazon data in this workspace is scheduled for deletion on {purge_date}.") def _personal_retention_specs(): """Select personal columns only from the schema's explicit boolean markers.""" return {table_name(name): spec for name, spec in amazon_schema.TABLE_SPECS.items() if any(column.get("personal") is True for column in spec.get("columns", []))} def _personal_retention_plan(con, specs, clock): """Group delivery clocks without reading or returning any personal cell values.""" tables = [] present = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"} for name, spec in specs.items(): if name not in present: continue columns = _columns_of(con, name) personal = sorted({column["name"] for column in spec["columns"] if column.get("personal") is True and column["name"] in columns}) if not personal: continue delivery = next((column for column in ( spec.get("delivery_date_column"), "delivery_date", "delivered_at", "actual_delivery_date") if column and column in columns), None) row = {"table": name, "personalColumns": personal, "deliveryColumn": delivery, "rows": con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0], "dueRows": 0, "missingDeliveryRows": 0, "clocks": [], "cause": None} if not delivery: row["missingDeliveryRows"] = row["rows"] if row["rows"]: row["cause"] = "Personal rows have no delivery date column. The delivery clock cannot be proved." else: expression = f"TRY_CAST({_quoted_identifier(delivery)} AS TIMESTAMPTZ)" for delivered, count in con.execute( f"SELECT {expression}, count(*) FROM {_quoted_identifier(name)} GROUP BY 1 ORDER BY 1").fetchall(): if delivered is None: row["missingDeliveryRows"] += count row["cause"] = "Personal rows have missing or invalid delivery dates. The delivery clock cannot be proved." continue deadline = _utc(delivered) + timedelta(days=DELETION_DAYS) due = deadline <= clock row["dueRows"] += count if due else 0 row["clocks"].append({"deliveredAt": _control_stamp(delivered), "deadlineAt": _control_stamp(deadline), "rows": count, "state": "DUE" if due else "WITHIN_WINDOW"}) tables.append(row) return {"clock": "delivery date for each row", "days": DELETION_DAYS, "tables": tables, "state": ("UNPROVEN" if any(row["cause"] for row in tables) else "DUE" if any(row["dueRows"] for row in tables) else "WITHIN_WINDOW" if any(row["rows"] for row in tables) else "NO_PERSONAL_DATA")} def _personal_retention_check(key, rt, clock): specs = _personal_retention_specs() if not specs or not _tenant_mirror_path(key, rt).exists(): return {"clock": "delivery date for each row", "days": DELETION_DAYS, "state": "NO_PERSONAL_DATA", "tables": []} con, _path = _connect_tenant_mirror(key, rt) try: return _personal_retention_plan(con, specs, clock) finally: con.close() def _sweep_personal_rows(key, rt, clock): """Delete expired rows and superseded saved bytes, with one count-only event per sweep.""" specs = _personal_retention_specs() if not specs: return None if getattr(datastore._SYNC_TLS, "candidate", None) is not None: raise RuntimeError("Personal retention requires the resident writer before a source candidate opens") with _MIRROR_LOCK: restored = _MIRROR_STORE.restore_mirror(key, rt, log=lambda *_: None) if restored["state"] == "refused": raise RuntimeError("Personal retention could not restore the saved Amazon mirror") con, _path = _connect_tenant_mirror(key, rt) deleted, private_deleted, success = 0, 0, False plan = None try: plan = _personal_retention_plan(con, specs, clock) if not plan["tables"]: return None if plan["state"] == "UNPROVEN": raise ValueError(next(row["cause"] for row in plan["tables"] if row["cause"])) changed = {row["table"] for row in plan["tables"] if row["dueRows"]} if changed: con.execute("BEGIN TRANSACTION") try: for row in plan["tables"]: if row["table"] not in changed: continue con.execute(f"DELETE FROM {_quoted_identifier(row['table'])} " f"WHERE TRY_CAST({_quoted_identifier(row['deliveryColumn'])} AS TIMESTAMPTZ) <= ?", [clock - timedelta(days=DELETION_DAYS)]) con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [row["rows"] - row["dueRows"], row["table"]]) con.execute("COMMIT") deleted = sum(row["dueRows"] for row in plan["tables"]) except BaseException: con.execute("ROLLBACK") raise restored["dirty"].update(changed) if restored["dirty"]: report = {"problems": []} manifest = _MIRROR_STORE.persist_mirror(key, rt, con, restored["dirty"], report=report, log=lambda *_: None, _publication={"snapshot_id": uuid.uuid4().hex, "started_at": _control_stamp(), "state": "complete"}) if not manifest or report["problems"]: raise RuntimeError("Personal retention could not save the deleted rows. Retry the sweep.") datastore._write_publication(con, "amazon", manifest) # Retention cannot leave personal bytes behind during the normal orphan grace period. manifest = restored["manifest"] current = set(_manifest_keys(manifest, strict=False)) ledger = _MIRROR_STORE._read_key_ledger(rt, key) candidates = set(ledger["keys"]) | {name + ".parquet" for name in specs} removed = set() for blob in sorted(candidates - current): if any(blob.startswith(row["table"] + ".") for row in plan["tables"]): path = MIRROR_PREFIX + blob if rt.delete_path(path) is False or rt.download_bytes(path) is not None: raise RuntimeError("Personal retention left superseded saved bytes. Retry the sweep.") removed.add(blob) if removed.intersection(ledger["keys"]): ledger = _MIRROR_STORE._read_key_ledger(rt, key) ledger["keys"] = {blob: stamp for blob, stamp in ledger["keys"].items() if blob not in removed} rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger)) _purge_runtime_caches(rt, key, key) # Preserve private source progress while removing its expired personal rows too. candidate = Path(rt.datastore_path).with_suffix(".amazon.candidate.duckdb") if candidate.exists(): with closing(datastore.duckdb.connect(str(candidate))) as private: private_plan = _personal_retention_plan(private, specs, clock) if private_plan["state"] == "UNPROVEN": raise ValueError("Private Amazon rows have missing or invalid delivery dates. Retry after correcting the delivery clock.") private.execute("BEGIN TRANSACTION") try: for row in private_plan["tables"]: if row["dueRows"]: private.execute(f"DELETE FROM {_quoted_identifier(row['table'])} " f"WHERE TRY_CAST({_quoted_identifier(row['deliveryColumn'])} AS TIMESTAMPTZ) <= ?", [clock - timedelta(days=DELETION_DAYS)]) if _columns_of(private, "_sync_state"): private.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [row["rows"] - row["dueRows"], row["table"]]) private.execute("COMMIT") private_deleted = sum(row["dueRows"] for row in private_plan["tables"]) except BaseException: private.execute("ROLLBACK") raise success = True return {"personalRowsDeleted": deleted, "privateRowsDeleted": private_deleted, "personalRetention": _personal_retention_plan(con, specs, clock)} finally: con.close() if plan and plan["tables"]: _c2_event("data.purge", key, "retention policy", {"scope": "personal_rows", "deleted_rows": deleted, "days": DELETION_DAYS, "private_deleted_rows": private_deleted, "tables": len(plan["tables"]), "success": success}, rt) def enforce_retention(tenant_key, rt=None, *, now=None): """The resident writer's policy check; a completed purge is never performed twice.""" key, clock = _tenant_key(tenant_key), _utc(now) rt = _ledger_runtime(key, rt) with _RETENTION_LOCK: plan = scheduled_purge(key, rt, now=clock) # Mirror completion cannot strand a failed owner when the next hourly check retries. if plan is None and getattr(rt, "_amazon_deletion_owners", {}): ledger = _ledger(key, rt) due = any(_utc(row["occurred_at"]) + timedelta(days=PURGE_AFTER_DAYS) <= clock for row in _active_deletion_triggers(key, ledger, clock)) if due: from core import keychain as kc if kc.first_entry_of_type(rt, "amazon_sp") is None: _purge_deletion_owners(key, rt, clock) if plan is None and not _personal_retention_specs(): return None success = False try: from core import keychain as kc if plan and _utc(plan["purgeAt"]) <= clock and kc.first_entry_of_type(rt, "amazon_sp") is None: result = purge(key, key, rt=rt, principal="retention policy", now=clock) else: result = _sweep_personal_rows(key, rt, clock) or plan success = True return result finally: _record_deletion_check(key, rt, "retention", success, checked_at=clock) def retention_check(tenant_key, *, rt=None, checked_at=None, scope="all", raise_overdue=False): """Check the original deadline against mirror readback and each required deletion owner.""" key, now = _tenant_key(tenant_key), _utc(checked_at) if scope not in ("all", "mirror", "fixture"): raise ValueError("deletion scope must be all, mirror or fixture") rt = _ledger_runtime(key, rt) ledger = _ledger(key, rt) active = _active_deletion_triggers(key, ledger, now) triggers = [r["occurred_at"] for r in active] checks = sorted(r["checked_at"] for r in ledger["checks"] if r["tenant"] == key and r["kind"] == "purge" and r["success"]) # A check in the future cannot prove deletion at the requested historical observation time. checks = [v for v in checks if _utc(v) <= now] triggers = [v for v in triggers if _utc(v) <= now] completion, missing = {}, set() for trigger in active: stamp = trigger["occurred_at"] identity = (trigger["kind"], stamp) mirror = next((c for c in checks if _utc(c) >= _utc(stamp)), None) if scope == "mirror": if mirror: completion[identity] = mirror continue evidence = _deletion_owner_evidence(key, rt, ledger, trigger, now, fixture=scope == "fixture") missing.update(evidence["unverifiedScopes"]) if mirror and not evidence["unverifiedScopes"]: completion[identity] = max(mirror, evidence["deletedAt"]) outstanding = [row["occurred_at"] for row in active if (row["kind"], row["occurred_at"]) not in completion] earliest = outstanding[0] if outstanding else None deadline = _utc(earliest) + timedelta(days=DELETION_DAYS) if earliest else None state = ("OVERDUE" if now > deadline else "DUE") if deadline else "NO_TRIGGER" if not outstanding and triggers and checks: # Historical late deletion stays late, even after a later successful retry. state = "COMPLETED_LATE" if any( _utc(completion[(row["kind"], row["occurred_at"])]) > _utc(row["occurred_at"]) + timedelta(days=DELETION_DAYS) for row in active) else "COMPLETED_ON_TIME" elif state == "DUE" and missing and checks: state = "UNPROVEN" result = {"tenant": key, "scope": scope, "state": state, "checkedAt": _control_stamp(now), "earliestTriggerAt": earliest, "deadlineAt": _control_stamp(deadline) if deadline else None, "mirrorDeletedAt": checks[-1] if checks else None} if scope != "mirror": result.update(evidence="fixture" if scope == "fixture" else "service", unverifiedScopes=sorted(missing)) result["personalRetention"] = _personal_retention_check(key, rt, now) personal = result["personalRetention"] if personal["state"] == "UNPROVEN" and raise_overdue: raise ValueError(next(row["cause"] for row in personal["tables"] if row["cause"])) if (state == "OVERDUE" or personal["state"] == "DUE") and raise_overdue: raise AmazonRetentionOverdue("Amazon deletion deadline has passed; see the dated check") return result def _record_deletion_check(key, rt, kind, success, *, checked_at=None, deleted_objects=None): stamp = _control_stamp(checked_at) row = {"tenant": key, "kind": kind, "checked_at": stamp, "success": bool(success)} if success and deleted_objects is not None: row["objects"] = [{"object": item, "deletedAt": stamp} for item in sorted(set(deleted_objects))] _append_ledger(key, rt, "checks", row) return stamp def _is_amazon_object(name): return str(name).startswith((PREFIX, "ut_amzn_")) def _amazon_objects(con): """Every live mirror object reserved to Amazon by the `amzn_` naming contract.""" rows = con.execute( "SELECT table_name, table_type FROM information_schema.tables " "WHERE table_catalog=current_database() AND table_schema='main'").fetchall() out = [] for name, kind in rows: name = str(name) if not name.startswith(PREFIX) and not name.startswith("ut_amzn_"): continue out.append((name, str(kind or "BASE TABLE").upper())) return sorted(out, key=lambda row: ("VIEW" not in row[1], row[0])) def _sync_state_rows(con): """The complete state table, used to prove non-Amazon markers survive a purge unchanged.""" return [tuple(row) for row in con.execute( "SELECT entity, phase, last_id, cursor_wd, rows, updated_at " "FROM _sync_state ORDER BY entity").fetchall()] # The durable algorithm is shared; these hooks retain Amazon's retention and # marker semantics, including legacy snapshots and its existing fixture seams. from harness import mirror_store as _mirrors from core.connector_spec import mirror_for as _mirror_spec MANIFEST_VERSIONS = _mirrors.MANIFEST_VERSIONS MANIFEST_VERSION = _mirrors.MANIFEST_VERSION MIRROR_PART_MAX_BYTES = min(_mirrors.MIRROR_PART_MAX_BYTES, 8_000_000) _MANIFEST_KEY = "manifest.json" _KEY_LEDGER = "keys.json" _TABLE_BLOB_RE = _mirrors._TABLE_BLOB_RE ORPHAN_GRACE = _mirrors.ORPHAN_GRACE _mirror_json = _mirrors._mirror_json _blob_metadata = _mirrors._blob_metadata _parquet_source = _mirrors._parquet_source #: D-11860: the hourly Amazon Marketing Stream publishes through `amazon_ads_store._stream_store`, never here. STREAM_TABLES = frozenset({"amzn_ads_sp_hourly_events", "amzn_ads_sp_hourly"}) STREAM_STATE_ENTITIES = ("amzn_sync_mark_ads_sp_hourly", "amzn_ads_sp_hourly") STREAM_STATE_PREFIX = "amzn_ads_stream_" def _stream_name(name): """Whether a table or state entity belongs to the hourly stream store.""" return name in STREAM_TABLES or name in STREAM_STATE_ENTITIES or name.startswith(STREAM_STATE_PREFIX) def _without_stream_tables(raw): """A manifest without hourly stream tables; anything unreadable passes through for the strict check.""" try: manifest = json.loads(raw) except (ValueError, TypeError): return raw if not isinstance(manifest, dict) or not isinstance(manifest.get("tables"), dict) or not any( isinstance(name, str) and _stream_name(name) for name in manifest["tables"]): return raw manifest = {**manifest, "tables": {name: entry for name, entry in manifest["tables"].items() if not (isinstance(name, str) and _stream_name(name))}} return json.dumps(manifest) def _account_table_complete(row): """One account's row for a table: complete, current or disabled.""" return isinstance(row, dict) and (row.get("state") == "disabled" or _mirrors._table_sync_complete("amazon", row)) def _named_tables(problem): """The Amazon tables a problem names, as `amzn_
` or the bare table name.""" text = str(problem) return {table for table in TABLES if re.search(r"(? 1 scoped = {} for name in sorted(unfinished) if scoped_accounts else (): per_account = (rows.get(name) or {}).get("accounts") if isinstance(rows.get(name), dict) else None if not isinstance(per_account, dict): continue missing = {cid for cid in accounts if not _account_table_complete(per_account.get(cid))} if missing and len(missing) < len(accounts): unfinished.discard(name) for cid in missing: scoped.setdefault(cid, set()).add(name) # Head and lines come from one document, so neither publishes without the other. pairs = {**SHARED_REPORT_SIBLINGS, **{line: head for head, line in SHARED_REPORT_SIBLINGS.items()}} partner = {pairs[name]: name for name in unfinished if name in pairs and pairs[name] not in unfinished} unfinished |= set(partner) for names in scoped.values(): names |= {pairs[name] for name in list(names) if name in pairs and pairs[name] not in unfinished} if not unfinished and not scoped and not problems and report.get("state") not in {"partial", "failed"}: return None owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))} account_owned = {str(problem) for cid, result in accounts.items() if cid in scoped or (result.get("state") == "failed" and unfinished) for problem in result.get("problems") or ()} named, noted = {}, [] for problem in problems: if problem.split(":", 1)[0].strip() in owners or problem in account_owned: continue tables = _named_tables(problem) for name in tables: named.setdefault(name, problem) if not tables: noted.append(problem) held = {} for name in sorted(unfinished): if name in partner: held[name] = f"held with {partner[name]}, which reads the same report document" continue row = rows.get(name) if isinstance(rows.get(name), dict) else {} state = " ".join(str(part) for part in (row.get("state") or "not reached in this pass", row.get("amazonStatus")) if part) held[name] = state + (f": {row['reason']}" if row.get("reason") else "") for name, problem in sorted(named.items()): if name not in held: held[name] = "a problem names this table: " + problem[:200] for names in scoped.values(): names -= set(held) account_holds = {} for cid, names in sorted(scoped.items()): if not names: continue result = accounts[cid] reason = next(iter(result.get("problems") or ()), "") or "this account did not complete these tables" account_holds[cid] = {"tables": {table_name(t) for name in names for t in (name, name + "_history") if t == name or name in HISTORY_SOURCES}, "label": result.get("store") or cid, "reason": str(reason)} standing = {name for name in held if isinstance(rows.get(name), dict) and _standing_refusal(rows[name])} unsettled = [problem for problem in problems if problem.split(":", 1)[0].strip() not in standing or problem in noted] # A held source keeps its history at the previous rows too: both are written in one transaction. return {"held": held, "tables": {table_name(t) for name in held for t in (name, name + "_history") if t == name or name in HISTORY_SOURCES}, "blocked": [], "accounts": account_holds, "noted": noted, "standing": standing, "unsettled": unsettled} def _hold_scoped(self, candidate, published, holds): """Give a held account its published rows and data markers in each table its siblings publish.""" tables = set(holds.get("tables") or ()) prior = self._tables(published) with tempfile.TemporaryDirectory(prefix="mirror-held-account-") as tmp: for cid, hold in sorted((holds.get("accounts") or {}).items()): for name in sorted(set(hold["tables"]) - tables): columns = _columns_of(candidate, name) if amazon_schema.CONNECTION_ID not in columns: continue candidate.execute(f'DELETE FROM {_quoted_identifier(name)} WHERE "_connection_id"=?', [cid]) kept = _columns_of(published, name) if name in prior else set() if amazon_schema.CONNECTION_ID not in kept: continue shared = ", ".join(_quoted_identifier(column) for column in sorted(kept & columns)) path = Path(tmp) / f"{uuid.uuid4().hex}.parquet" # DuckDB binds no parameter inside COPY: a bound one silently writes no file. literal = "'" + str(cid).replace("'", "''") + "'" published.execute(f'COPY (SELECT {shared} FROM {_quoted_identifier(name)} ' f'WHERE "_connection_id"={literal}) TO ' f"'{str(path).replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET, COMPRESSION ZSTD)") if not path.exists(): raise RuntimeError("The held Amazon account rows could not be copied") candidate.execute(f"INSERT INTO {_quoted_identifier(name)} ({shared}) " f"SELECT {shared} FROM {_parquet_source([path])}") prefix = _account_prefix(cid) private = [row for row in self._held_state_rows(self._state_rows(candidate), hold["tables"]) if str(row[0]).startswith(prefix)] if private: candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in private]) previous = [row for row in self._held_state_rows(self._state_rows(published), hold["tables"]) if str(row[0]).startswith(prefix)] if previous: candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous) def _held_state_rows(self, rows, tables): """A held table's data markers: watermark, floor, proofs, limits, role results and documents. Its status, tick, refusal and pending-report markers describe the pass, so they publish with it. """ tables = set(tables) names = sorted({table_name(t) for t in TABLES} | tables, key=lambda name: (-len(name), name)) def owner(entity): if entity.startswith("amzn_account_") and "__" in entity: entity = entity.split("__", 1)[1] if entity.startswith(entity_key("settlement_doc_")): return table_name("settlements") for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_", RESYNC_PREFIX): for name in names: marker = entity_key(family + name[len(PREFIX):]) if entity == marker or entity.startswith(marker + "_"): return name if entity in names: return None return next((name for name in names if entity.startswith(name + "_")), None) return [row for row in rows if owner(str(row[0])) in tables] def _before_restore(self, tenant_key, rt): obligation = retention_check(tenant_key, rt=rt, raise_overdue=True) if obligation["earliestTriggerAt"]: raise ValueError("Amazon deletion is outstanding; saved data cannot be restored") def _after_restore(self, con, result, log): _heal_snapshots(con, result, log) _heal_quoted_values(con, result, log) _heal_identity_duplicates(con, result, log) if prepare_brand_analytics(con): result["dirty"].add(table_name("brand_analytics")) refresh_sku_keywords(con) def _heal_restored(self, con, result, log): # Heal the served rows without marking them dirty: a dirty set makes the retention sweep publish. healed = {"dirty": set(), "problems": result["problems"]} for heal in (_heal_snapshots, _heal_quoted_values, _heal_identity_duplicates): heal(con, healed, log) prepare_brand_analytics(con) def _restored_state_rows(self, rows, tables, refused): return _restored_state_rows(rows, tables, refused) def _first_load_complete(self, con): load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) return bool(load and load["state"] == "done") def _state_complete(self, rows): return bool(rows) def _log_restored(self, paths, total, rows, log): marks = {row[0][len(PREFIX + WATERMARK_PREFIX):]: row[3] for row in rows if row[0].startswith(PREFIX + WATERMARK_PREFIX)} resume = ", ".join(f"{table} from {marks.get(table) or 'no watermark'}" for table in sorted(set(TABLES) | set(marks))) docs = [row for row in rows if row[0].startswith("amzn_settlement_doc_") and row[1] == "done"] last = max(docs, key=lambda row: (row[3] or "", row[5] or "", row[0])) if docs else None document = last[0][len("amzn_settlement_doc_"):] if last else "none" # State is vendor-derived; keep the log to one line and enforce rule 2 here too. line = (f"Amazon mirror restored from the store: {len(paths)} tables, {total:,} rows; " f"resuming {resume}; settlements after document {document}") log(line.replace("\u2014", "-").replace("\u2013", "-").replace("\n", " ").replace("\r", " ")) _MIRROR_STORE = _AmazonMirrorStore() # Compatibility for the deletion path and existing controls. There is one # implementation of each operation, in mirror_store.MirrorStore. _mirror_cache_key = _MIRROR_STORE._mirror_cache_key _manifest_keys = _MIRROR_STORE._manifest_keys _is_mirror_blob_key = _MIRROR_STORE._is_mirror_blob_key _validated_snapshot = _MIRROR_STORE._validated_snapshot def restore_mirror(tenant_key, rt, log=None): if _account_scope() is not None: return {"state": "scoped", "dirty": set(), "problems": []} result = _MIRROR_STORE.restore_mirror(tenant_key, rt, log=log) if result["state"] != "refused": con, _path = _connect_tenant_mirror(tenant_key, rt) try: if not _columns_of(con, "amzn_sku_keywords"): refresh_sku_keywords(con) finally: con.close() return result def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print): changed_tables = tuple(changed_tables) # A snapshot source's history is written in its transaction, so it publishes with it. existing = {name for name, _kind in _amazon_objects(con)} changed_tables += tuple(dict.fromkeys( history_table(t) for t in HISTORY_SOURCES if table_name(t) in changed_tables and history_table(t) in existing and history_table(t) not in changed_tables)) scope = _account_scope() if scope is not None: scope["changed"].update(changed_tables) prepare_brand_analytics(con) return None return _MIRROR_STORE.persist_mirror( tenant_key, rt, con, changed_tables, report=report, log=log) def _restored_state_rows(rows, tables, refused): """Remove refused tables' markers and shared first-load state without altering good markers.""" if not refused: return rows refused = set(refused) names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name)) families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "resume_", "reconcile_", "limit_", "refused_", RESYNC_PREFIX) shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)} def keep(row): entity = row[0] if entity.startswith("amzn_account_") and "__" in entity: return keep((entity.split("__", 1)[1], *row[1:])) if entity in shared or ("amzn_settlements" in refused and entity.startswith("amzn_settlement_doc_")): return False if entity.startswith("amzn_pending_report_"): try: window = json.loads(row[3])["window"] except (ValueError, TypeError, KeyError): return True for name in refused: table = name[len(PREFIX):] spec = _client().REPORT_TYPES.get(table, {}) for report_type in {spec.get(key) for key in ("report", "report_incremental", "report_discover")} - {None}: identity = json.dumps([table, report_type, window], sort_keys=True) marker = entity_key("pending_report_" + hashlib.sha256(identity.encode()).hexdigest()) if entity == marker: return False for family in families: for name in names: marker = entity_key(family + name[len(PREFIX):]) if entity == marker or entity.startswith(marker + "_"): return name not in refused return True return [row for row in rows if keep(row)] def _snapshot_tables(): """Current-state tables: every report or API the client declares `snapshot`.""" cx = _client() return tuple(t for t in TABLES if (cx.REPORT_TYPES.get(t) or {}).get("snapshot")) def _collapse_snapshots(con, log=None): """D-3031 heal: a snapshot table holding more than one pull keeps only the latest. -> {mirror: (rows before, rows after, copies)}. The ledger keeps its latest document's dates (a DAILY document spans several), every other table its newest `_snapshot_date`; AWD rows are another leg and never touched. Idempotent: one copy is left alone. """ existing = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"} collapsed = {} for table in _snapshot_tables(): mirror = table_name(table) if mirror not in existing: continue cols = _columns_of(con, mirror) if "_snapshot_date" not in cols: continue scope = ("COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" and "inventory_source" in cols else "TRUE") if amazon_schema.CONNECTION_ID in cols: before = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] for (cid,) in con.execute(f'SELECT DISTINCT "_connection_id" FROM {mirror}').fetchall(): value = "NULL" if cid is None else "'" + str(cid).replace("'", "''") + "'" account_scope = f'{scope} AND "_connection_id" IS NOT DISTINCT FROM {value}' dates = [r[0] for r in con.execute(f'SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) ' f'FROM {mirror} WHERE {account_scope}').fetchall()] present = sorted(d for d in dates if d is not None) if len(dates) <= 1 or not present: continue keep = present[-1:] if table == "inventory": key = entity_key("reconcile_" + table) saved = _read_state(con, (_account_prefix(cid) if not _account_scope() and cid else "") + key) try: marked = json.loads(saved["detail"])["dates"] if saved and saved.get("detail") else {} except (ValueError, KeyError, TypeError): marked = {} keep = [d for d in present if d in marked] or keep con.execute(f'DELETE FROM {mirror} WHERE {account_scope} AND (_snapshot_date IS NULL ' f'OR CAST(_snapshot_date AS VARCHAR) NOT IN ({",".join("?" for _ in keep)}))', keep) after = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] if before != after: collapsed[mirror] = (before, after, 0) continue dates = [row[0] for row in con.execute( f"SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) FROM {mirror} WHERE {scope}").fetchall()] present = sorted(d for d in dates if d is not None) if len(dates) <= 1 or not present: continue keep = [] if table == "inventory": saved = _read_state(con, entity_key("reconcile_" + table)) try: marked = json.loads(saved["detail"])["dates"] if saved and saved.get("detail") else {} except (ValueError, KeyError, TypeError): marked = {} keep = [d for d in present if d in marked] keep = keep or present[-1:] stale = (f"{scope} AND (_snapshot_date IS NULL OR " f"CAST(_snapshot_date AS VARCHAR) NOT IN ({','.join('?' for _ in keep)}))") if not con.execute(f"SELECT count(*) FROM {mirror} WHERE {stale}", keep).fetchone()[0]: continue before = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] con.execute("BEGIN TRANSACTION") try: con.execute(f"DELETE FROM {mirror} WHERE {stale}", keep) con.execute("COMMIT") except Exception: con.execute("ROLLBACK") raise after = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] collapsed[mirror] = (before, after, len(dates)) if log: log(f"Amazon snapshot {mirror}: {len(dates)} copies collapsed to the latest, " f"{before:,} rows to {after:,}") return collapsed def _heal_snapshots(con, result, log): """Collapse after a restore; the healed tables are dirty so the next checkpoint saves one copy.""" try: result["dirty"].update(_collapse_snapshots(con, log)) except Exception as exc: # noqa: BLE001 message = (f"Amazon snapshot collapse failed ({type(exc).__name__}); the next resync of each " "snapshot table replaces its earlier copies.") result["problems"].append(message) log(message) _QUOTE_IDENTITY = ("msku", "fnsku", "asin", "sku") def _app_date_sql(col): """SQL twin of `app_date` over one VARCHAR column: US date or stamp to ISO, else unchanged.""" q = _quoted_identifier(col) parts = (f"regexp_extract({q}, '^\\s*(\\d{{1,2}}/\\d{{1,2}}/\\d{{4}}[ T]\\d{{1,2}}:\\d{{2}}" f"(?::\\d{{2}})?)\\s*(.*?)\\s*$', ['stamp', 'zone'])") stamp = f"replace(({parts}).stamp, 'T', ' ')" ts = (f"coalesce(try_strptime({stamp}, '%m/%d/%Y %H:%M:%S'), " f"try_strptime({stamp}, '%m/%d/%Y %H:%M'))") tail = f"({parts}).zone" named = " ".join(f"WHEN '{name}' THEN '{offset}'" for name, offset in _ZONES.items()) zone = (f"CASE upper({tail}) WHEN '' THEN '' {named} ELSE CASE " f"WHEN regexp_full_match({tail}, '[+-]\\d{{2}}:\\d{{2}}') THEN {tail} " f"WHEN regexp_full_match({tail}, '[+-]\\d{{4}}') " f"THEN substr({tail}, 1, 3) || ':' || substr({tail}, 4) END END") return (f"CASE WHEN regexp_full_match(trim({q}), '\\d{{1,2}}/\\d{{1,2}}/\\d{{4}}') " f"AND try_strptime(trim({q}), '%m/%d/%Y') IS NOT NULL " f"THEN strftime(try_strptime(trim({q}), '%m/%d/%Y'), '%Y-%m-%d') " f"WHEN {q} IS NOT NULL AND {ts} IS NOT NULL AND ({zone}) IS NOT NULL " f"THEN strftime({ts}, '%Y-%m-%dT%H:%M:%S') || ({zone}) ELSE {q} END") def _unquote_mirror_values(con, log=None): """D-2682 heal: mirror rows saved with quoted values get them stripped, in SQL. -> {mirror: rows healed}. A base table row qualifies when an identity value is wrapped in `"`. Its VARCHAR values wrapped in `"` lose one pair, `""` becomes `"`, and the date-named columns of an App table become ISO. Rows a fixed parse wrote are left alone, so a table holding both kinds still heals. Idempotent: a healed row no longer qualifies. """ healed = {} for mirror, kind in _amazon_objects(con): if kind != "BASE TABLE" or not mirror.startswith(PREFIX): continue types = dict(con.execute( "SELECT column_name, data_type FROM information_schema.columns " "WHERE table_schema='main' AND table_name=?", [mirror]).fetchall()) text = [c for c, t in types.items() if str(t).upper() == "VARCHAR" and c not in (amazon_schema.CONNECTION_ID, amazon_schema.STORE)] identity = [c for c in _QUOTE_IDENTITY if c in text] if not identity: continue table = _quoted_identifier(mirror) stale = " OR ".join(f"{_quoted_identifier(c)} LIKE '\"%\"'" for c in identity) if not con.execute(f"SELECT 1 FROM {table} WHERE {stale} LIMIT 1").fetchone(): continue lost = _unhealable_sql(types, text) sets = [] dated = mirror[len(PREFIX):] in APP_DATE_TABLES for col in text: q = _quoted_identifier(col) plain = (f"CASE WHEN {q} LIKE '\"%\"' THEN replace(substr({q}, 2, length({q}) - 2), " f"'\"\"', '\"') ELSE {q} END") if dated and is_app_date_column(col): plain = _app_date_sql(col).replace(q, f"({plain})") sets.append(f"{q} = {plain}") con.execute("BEGIN TRANSACTION") try: healed[mirror] = con.execute(f"SELECT count(*) FROM {table} WHERE {stale}").fetchone()[0] resync = _queue_resync(con, mirror, types, f"({stale}) AND ({lost})") if lost else None con.execute(f"UPDATE {table} SET {', '.join(sets)} WHERE {stale}") con.execute("COMMIT") except Exception: healed.pop(mirror, None) con.execute("ROLLBACK") raise if log: log(f"Amazon quoted values {mirror}: {healed[mirror]:,} rows unquoted") if resync: log(f"Amazon resync {mirror}: {resync[0]:,} rows lost cells to the old parse; " f"{len(resync[1])} report windows from {resync[1][0][0]} download again on the next passes") if _queue_invalid_ledger_rows(con, log): healed.setdefault(table_name("inventory_ledger_detail"), 0) return healed _INVALID_LEDGER_QUANTITY = "quantity IS NOT NULL AND TRY_CAST(quantity AS DOUBLE) IS NULL" def ledger_parse_diagnosis(con): """Read invalid ledger counts and their evidenced report windows without exposing row values or writing.""" mirror = table_name("inventory_ledger_detail") cols = _columns_of(con, mirror) if "quantity" not in cols: return {"invalid_rows": 0, "undated_rows": 0, "windows": []} invalid = con.execute(f"SELECT count(*) FROM {mirror} WHERE {_INVALID_LEDGER_QUANTITY}").fetchone()[0] if not {"_period_start", "_period_end"} <= cols: return {"invalid_rows": invalid, "undated_rows": invalid, "windows": []} found = con.execute( f"SELECT CAST(_period_start AS VARCHAR), CAST(_period_end AS VARCHAR), count(*) FROM {mirror} " f"WHERE {_INVALID_LEDGER_QUANTITY} GROUP BY 1, 2 ORDER BY 1, 2").fetchall() return {"invalid_rows": invalid, "undated_rows": sum(n for start, end, n in found if start is None or end is None), "windows": [(start, end) for start, end, _n in found if start is not None and end is not None]} def _queue_invalid_ledger_rows(con, log=None): """Queue only evidenced windows of already-unquoted shifted ledger rows, once per queue change.""" diagnosis = ledger_parse_diagnosis(con) if not diagnosis["windows"]: return False mirror = table_name("inventory_ledger_detail") key = entity_key(RESYNC_PREFIX + "inventory_ledger_detail") def markers(): return [row for row in _sync_state_rows(con) if row[0] == key or row[0].endswith("__" + key)] before = markers() queued = _queue_resync(con, mirror, _columns_of(con, mirror), _INVALID_LEDGER_QUANTITY) changed = before != markers() if changed and queued and log: log(f"Amazon ledger parse repair: {diagnosis['invalid_rows']:,} invalid rows; " f"{len(diagnosis['windows'])} evidenced report windows queued for resync") return changed def _unhealable_sql(types, text): """SQL true for a quoted row the unquote cannot repair, or "" when no such test applies. A cell opening an unclosed quote is a value the old parse split at a tab, so every later cell shifted. A row whose every numeric measure is NULL lost them to the failed numeric cast. """ split = [f"({q} LIKE '\"%' AND length(regexp_extract(substr({q}, 2), '\"*$')) % 2 = 0)" for q in map(_quoted_identifier, text)] measures = [_quoted_identifier(c) for c, t in types.items() if str(t).upper() != "VARCHAR" and not str(c).startswith("_")] if measures: split.append("(" + " AND ".join(f"{q} IS NULL" for q in measures) + ")") return " OR ".join(split) def _queue_resync(con, mirror, types, where): """Record the report windows of rows matching `where` for download again -> (rows, windows) or None. Only a dated window table qualifies: its next pass replaces each queued `_period_start` whole. """ table = mirror[len(PREFIX):] if (table not in PERIOD_TABLES or not {"_period_start", "_period_end"} <= set(types) or (_client().REPORT_TYPES.get(table) or {}).get("snapshot")): return None if amazon_schema.CONNECTION_ID in types and _account_scope() is None: results = [] for (cid,) in con.execute(f'SELECT DISTINCT "_connection_id" FROM {_quoted_identifier(mirror)} ' f'WHERE ({where})').fetchall(): if cid is None: continue literal = "'" + str(cid).replace("'", "''") + "'" unscoped = ({k: v for k, v in types.items() if k != amazon_schema.CONNECTION_ID} if hasattr(types, "items") else set(types) - {amazon_schema.CONNECTION_ID}) result = _queue_resync(con, mirror, unscoped, f'({where}) AND "_connection_id"={literal}') if result: key = entity_key(RESYNC_PREFIX + table) row = _read_state(con, key) scoped_key = _account_prefix(cid) + key saved = _read_state(con, scoped_key) windows = {tuple(w) for w in json.loads(row["detail"])["windows"]} if saved: windows |= {tuple(w) for w in json.loads(saved["detail"])["windows"]} detail = json.dumps({"windows": sorted(windows)}) if not saved or (saved["state"], saved["rows"], saved["detail"]) != ("pending", result[0], detail): _record_state(con, scoped_key, "pending", result[0], detail) con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) results.append((result[0], windows)) return (sum(n for n, _w in results), sorted(set().union(*(w for _n, w in results)))) if results else None found = con.execute( f"SELECT CAST(_period_start AS VARCHAR), CAST(_period_end AS VARCHAR), count(*) " f"FROM {_quoted_identifier(mirror)} WHERE ({where}) AND _period_start IS NOT NULL " f"AND _period_end IS NOT NULL GROUP BY 1, 2").fetchall() if not found: return None key = entity_key(RESYNC_PREFIX + table) saved = _read_state(con, key) try: queued = {tuple(w) for w in json.loads(saved["detail"])["windows"]} if saved else set() except (ValueError, KeyError, TypeError): queued = set() windows = sorted(queued | {(start, end) for start, end, _n in found}) rows = sum(n for _s, _e, n in found) detail = json.dumps({"windows": [list(w) for w in windows]}) if not saved or (saved["state"], saved["rows"], saved["detail"]) != ("pending", rows, detail): _record_state(con, key, "pending", rows, detail) return rows, windows def queued_resyncs(con, table): """Every queued resync window of `table`, oldest first, as `(start, end)` datetimes.""" saved = _read_state(con, entity_key(RESYNC_PREFIX + table)) try: return [(_utc(start), _utc(end)) for start, end in json.loads(saved["detail"])["windows"]] except (TypeError, ValueError, KeyError, IndexError, AttributeError): return [] def queued_resync(con, table): """The oldest queued resync window of `table` as `(start, end)` datetimes, or None.""" windows = queued_resyncs(con, table) return windows[0] if windows else None def resync_landed(con, table, window): """Drop one landed window from the queue; the marker goes when the queue is empty.""" key = entity_key(RESYNC_PREFIX + table) saved = _read_state(con, key) try: windows = [w for w in json.loads(saved["detail"])["windows"] if (_utc(w[0]), _utc(w[1])) != tuple(window)] except (TypeError, ValueError, KeyError, AttributeError): windows = [] if windows: _record_state(con, key, "pending", saved["rows"], json.dumps({"windows": windows})) else: con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) #: Tables whose declared identity can hold rows from two documents after an identity change. _IDENTITY_COLLAPSE_TABLES = ("pricing",) def _collapse_identity(con, log=None): """Rows sharing a declared identity keep the newest landed document only -> {mirror: (before, after)}. Newest is the document holding the highest `rowid` (landing order, the ranking diagnostic's rule), then the larger document ID. A row with no `_period_start` goes once the same child and SKU hold a dated row. Idempotent: one probe per table when nothing is doubled. """ existing = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"} collapsed = {} for table in _IDENTITY_COLLAPSE_TABLES: mirror = table_name(table) if mirror not in existing: continue cols = _columns_of(con, mirror) keys = [c for c in replace_keys_for(table) if c in cols] if amazon_schema.CONNECTION_ID in cols: keys.insert(0, amazon_schema.CONNECTION_ID) if not keys or "_report_document_id" not in cols: continue q, target = _quoted_identifier, _quoted_identifier(mirror) part = ", ".join(q(c) for c in keys) doomed = (f"SELECT rid FROM (SELECT rid, d, first_value(d) OVER (PARTITION BY {part} " f"ORDER BY landed DESC, d DESC) AS keep FROM (SELECT rowid AS rid, {part}, " f"_report_document_id AS d, max(rowid) OVER (PARTITION BY " + ('"_connection_id", ' if amazon_schema.CONNECTION_ID in cols else "") + "_report_document_id) AS landed " f"FROM {target})) WHERE d IS DISTINCT FROM keep") if "_period_start" in keys: same = " AND ".join(f"o.{q(c)} IS NOT DISTINCT FROM t.{q(c)}" for c in keys if c not in ("_period_start", "_period_end")) or "TRUE" doomed += (f" UNION SELECT t.rowid FROM {target} AS t WHERE t._period_start IS NULL AND EXISTS " f"(SELECT 1 FROM {target} AS o WHERE o._period_start IS NOT NULL AND {same})") if not con.execute(f"SELECT 1 FROM ({doomed}) LIMIT 1").fetchone(): continue before = con.execute(f"SELECT count(*) FROM {target}").fetchone()[0] con.execute("BEGIN TRANSACTION") try: con.execute(f"DELETE FROM {target} WHERE rowid IN ({doomed})") con.execute("COMMIT") except Exception: con.execute("ROLLBACK") raise after = con.execute(f"SELECT count(*) FROM {target}").fetchone()[0] collapsed[mirror] = (before, after) if log: log(f"Amazon identity {mirror}: {before:,} rows to {after:,}, one document per identity") return collapsed def _heal_identity_duplicates(con, result, log): """Collapse doubled identities after a restore; healed tables are dirty for the next checkpoint.""" try: result["dirty"].update(_collapse_identity(con, log)) except Exception as exc: # noqa: BLE001 message = (f"Amazon identity collapse failed ({type(exc).__name__}); the next resync of each " "doubled table replaces its earlier document's rows.") result["problems"].append(message) log(message) def _heal_quoted_values(con, result, log): """Unquote after a restore; healed tables are dirty so the next checkpoint saves them clean.""" try: result["dirty"].update(_unquote_mirror_values(con, log)) except Exception as exc: # noqa: BLE001 message = (f"Amazon quoted value heal failed ({type(exc).__name__}); the next resync of each " "quoted table writes clean values.") result["problems"].append(message) log(message) def _purge_json(rt, name): """A saved mirror JSON document for a purge: the parsed dict, or {} when absent or unreadable.""" try: raw = rt.download_bytes(MIRROR_PREFIX + name) doc = json.loads(raw) if raw else {} except (TypeError, ValueError, UnicodeError, RecursionError): return {} return doc if isinstance(doc, dict) else {} def _purge_saved_mirror(tenant_key, rt, local_tables): """Delete data before its inventory so a failed deletion remains retryable after restart. Every version goes: each legacy in-place key, each key any manifest names, and each key the write-ahead ledger lists, which includes blobs a killed checkpoint uploaded but never published. Then the manifest, then the ledger itself, each read back. """ with _MIRROR_LOCK: key = _mirror_cache_key(tenant_key, rt) cached = _MIRROR_CACHE.get(key, {}) # Deletion never refuses on unreadable inventory: a corrupt or foreign manifest or ledger # still yields every blob name it holds, and every path below goes through THIS tenant's # runtime, so a ledger labelled with another tenant cannot point a delete outside it. manifest = _purge_json(rt, _MANIFEST_KEY) ledger_keys = {str(k) for k in (_purge_json(rt, _KEY_LEDGER).get("keys") or {}) if _is_mirror_blob_key(str(k))} names = ({table_name(t) for t in TABLES} | set(local_tables) | set(manifest.get("tables", {})) | set(cached.get("manifest", {}).get("tables", {})) | set(cached.get("dirty", ()))) blobs = {name + ".parquet" for name in names if re.fullmatch(r"amzn_[a-z0-9_]+", name)} blobs.add("sync_state.json") blobs |= set(_manifest_keys(manifest, strict=False)) blobs |= set(_manifest_keys(cached.get("manifest", {}), strict=False)) blobs |= ledger_keys paths = [MIRROR_PREFIX + blob for blob in sorted(blobs)] paths.extend((MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER)) deleted = [] for path in paths: existed = rt.download_bytes(path) is not None if rt.delete_path(path) is False or rt.download_bytes(path) is not None: raise RuntimeError("Amazon saved mirror deletion failed or left residue") if existed: deleted.append("blob:" + path) candidate = Path(rt.datastore_path).with_suffix(".amazon.candidate.duckdb") if candidate.exists(): deleted.append("local:amazon.candidate.duckdb") candidate.unlink(missing_ok=True) if Path(str(candidate) + ".wal").exists(): deleted.append("local:amazon.candidate.duckdb.wal") Path(str(candidate) + ".wal").unlink(missing_ok=True) _MIRROR_CACHE.pop(key, None) return deleted def _collateral_rows(con): """Compare complete non-Amazon row multisets, including duplicate counts, not just names.""" from collections import Counter names = [str(r[0]) for r in con.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() if not _is_amazon_object(r[0]) and r[0] != "_sync_state"] return {name: Counter(repr(tuple(row)) for row in con.execute( f"SELECT * FROM {_quoted_identifier(name)}" + (" WHERE source != 'amazon'" if name == "_mirror_publications" else "")).fetchall()) for name in names} def _purge_mirror(con, tenant_key, confirmation): """Transactionally delete every `amzn_` mirror object and state marker, then read it back. The prefix is the connector's reserved namespace, so this also catches a future Amazon table that has not yet been added to `TABLES`. Non-Amazon tables and sync markers are snapshotted and compared inside the transaction and again after commit. No caller supplies a table name. """ _require_purge_confirmation(tenant_key, confirmation) objects = _amazon_objects(con) before_tables = {str(row[0]) for row in con.execute( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='main'").fetchall()} before_other_tables = sorted(name for name in before_tables if not _is_amazon_object(name)) before_other_rows = _collateral_rows(con) before_state = _sync_state_rows(con) amazon_state = [str(row[0]) for row in before_state if str(row[0]).startswith(PREFIX)] other_state = [row for row in before_state if not str(row[0]).startswith(PREFIX)] row_counts = {} for name, _kind in objects: row_counts[name] = int(con.execute( f"SELECT COUNT(*) FROM {_quoted_identifier(name)}").fetchone()[0]) try: con.execute("BEGIN TRANSACTION") for name, kind in objects: noun = "VIEW" if "VIEW" in kind else "TABLE" con.execute(f"DROP {noun} {_quoted_identifier(name)}") for entity in amazon_state: con.execute("DELETE FROM _sync_state WHERE entity=?", [entity]) if "_mirror_publications" in before_tables: con.execute("DELETE FROM _mirror_publications WHERE source='amazon'") if _amazon_objects(con): raise RuntimeError("Amazon mirror objects remain inside the purge transaction") inside_other_tables = sorted(str(row[0]) for row in con.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() if not _is_amazon_object(row[0])) if inside_other_tables != before_other_tables: raise RuntimeError("a non-Amazon mirror object changed during the purge") inside_state = _sync_state_rows(con) if inside_state != other_state: raise RuntimeError("a non-Amazon sync marker changed during the purge") if _collateral_rows(con) != before_other_rows: raise RuntimeError("non-Amazon rows changed during the purge") con.execute("COMMIT") except Exception: try: con.execute("ROLLBACK") except Exception: # noqa: BLE001 pass raise residue_objects = _amazon_objects(con) residue_state = [str(row[0]) for row in _sync_state_rows(con) if str(row[0]).startswith(PREFIX)] after_other_tables = sorted(str(row[0]) for row in con.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() if not _is_amazon_object(row[0])) after_other_state = [row for row in _sync_state_rows(con) if not str(row[0]).startswith(PREFIX)] if residue_objects or residue_state: raise RuntimeError( "Amazon mirror purge committed but independent readback found residue") if (after_other_tables != before_other_tables or after_other_state != other_state or _collateral_rows(con) != before_other_rows): raise RuntimeError( "Amazon mirror purge committed but independent readback found collateral change") return { "tables": [name for name, _kind in objects], "rows": sum(row_counts.values()), "syncMarkers": len(amazon_state), "residueTables": [], "residueSyncMarkers": [], } def _purge_materialized(rt, tenant_key, confirmation): """Extend the operator purge through the existing tenant-bound authorship store seam.""" _require_purge_confirmation(tenant_key, confirmation) _tenant_mirror_path(tenant_key, rt) if not callable(getattr(rt, "update", None)): return {"materializedVerified": False, "materializedTables": 0, "materializedRows": 0} from copy import deepcopy removed, kept = {}, {} def remove(doc): removed.clear() kept.clear() for name, value in doc.items(): if str(name).startswith("ut_amzn_"): removed[name] = len((value or {}).get("rows") or []) else: kept[name] = deepcopy(value) for name in removed: del doc[name] rt.update("user_tables", remove, flush="sync") readback = rt.get("user_tables", fresh=True) or {} if any(str(name).startswith("ut_amzn_") for name in readback) or readback != kept: raise RuntimeError("materialized Amazon purge readback found residue or collateral change") return {"materializedVerified": True, "materializedTables": len(removed), "materializedRows": sum(removed.values()), "materializedObjects": ["materialized:" + name for name in removed]} def _amazon_cache_key(value): if isinstance(value, (tuple, list)): return any(_amazon_cache_key(item) for item in value) return isinstance(value, str) and _is_amazon_object(value) def _purge_runtime_caches(rt, tenant_key, confirmation, *, deleted_objects=None): _require_purge_confirmation(tenant_key, confirmation) _tenant_mirror_path(tenant_key, rt) count = 0 for attr in ("pool_cache", "measure_memo", "mset_memo", "series_memo"): cache = getattr(rt, attr, {}) for key in list(cache): if _amazon_cache_key(key): if deleted_objects is not None: digest = hashlib.sha256(repr(key).encode()).hexdigest() deleted_objects.append(f"runtime-cache:{attr}:{digest}") del cache[key] count += 1 if any(_amazon_cache_key(key) for key in cache): raise RuntimeError("Amazon runtime cache residue remains") return count def purge(tenant_key, confirmation, rt=None, log=print, *, principal="operator", now=None): """Permanently remove one tenant's live Amazon mirror after credential revocation. Used by the operator and the retention policy. Both require the exact tenant and absent credential metadata. Policy deletion also covers saved blobs after a container loses its local mirror, without creating a new file as purported evidence of deletion. """ key = _require_purge_confirmation(tenant_key, confirmation) rt = rt if rt is not None else _runtime_for(key) if rt is None: raise RuntimeError( f"tenant {key!r} could not be resolved; refusing to guess a database target") path = _tenant_mirror_path(key, rt) if not path.is_file() and principal != "retention policy": raise RuntimeError( f"tenant {key!r} has no existing mirror file; refusing to create one as purge proof") _assert_amazon_credential_revoked(rt) success, result, deleted = False, {}, [] collect_objects = bool(getattr(rt, "_amazon_deletion_owners", {})) bound_path = path try: if path.is_file(): con, bound_path = _connect_tenant_mirror(key, rt) try: objects = _amazon_objects(con) if collect_objects: deleted.extend("mirror:" + name for name, _kind in objects) deleted.extend("sync-marker:" + str(row[0]) for row in _sync_state_rows(con) if str(row[0]).startswith(PREFIX)) saved = _purge_saved_mirror(key, rt, [name for name, _kind in objects]) if collect_objects: deleted.extend(saved) result = _purge_mirror(con, key, confirmation) finally: con.close() else: saved = _purge_saved_mirror(key, rt, []) if collect_objects: deleted.extend(saved) result = {"tables": [], "rows": 0, "syncMarkers": 0, "residueTables": [], "residueSyncMarkers": []} result.update(_purge_materialized(rt, key, confirmation)) if collect_objects: deleted.extend(result.get("materializedObjects", [])) result["runtimeCacheEntries"] = _purge_runtime_caches( rt, key, confirmation, deleted_objects=deleted if collect_objects else None) # Keep the existing post-commit query and add a fresh, independently opened cursor. import duckdb if bound_path.is_file(): with closing(duckdb.connect(str(bound_path))) as readback: if _amazon_objects(readback) or any( str(r[0]).startswith(PREFIX) for r in _sync_state_rows(readback)): raise RuntimeError("independent mirror file readback found Amazon residue") _purge_deletion_owners(key, rt, _utc(now)) success = True finally: stamp = _record_deletion_check(key, rt, "purge", success, checked_at=now, deleted_objects=deleted if collect_objects else None) # Counts only. Even database-discovered table names could contain customer data, so # never send them (or exception strings) to C2's flat, secret-refusing detail object. _c2_event("data.purge", key, principal, {"success": success, "checked_at": stamp, "scope": "local", "rows": result.get("rows", 0), "tables": len(result.get("tables", [])), "known_tables": ",".join(table_name(t) for t in TABLES if table_name(t) in result.get("tables", [])), "materialized_tables": result.get("materializedTables", 0), "materialized_rows": result.get("materializedRows", 0)}, rt) report = {"tenant": key, "database": bound_path.name, **result, "checkedAt": stamp, "complete": False, "unverifiedScopes": ["workbooks", "service_retained_exports", "backups", "other_process_and_unattributed_caches"], "retention": retention_check(key, rt=rt, checked_at=now)} report["unverifiedScopes"] = report["retention"]["unverifiedScopes"] report["complete"] = report["retention"]["state"] == "COMPLETED_ON_TIME" if not result.get("materializedVerified"): report["unverifiedScopes"].append("materialized_store") log(f" tenant {key}: permanently deleted {result['rows']:,} Amazon mirror rows from " f"{len(result['tables']):,} tables and {result['syncMarkers']:,} sync markers") return report def _restore_readback(path): import duckdb with closing(duckdb.connect(str(path), read_only=True)) as con: objects = _amazon_objects(con) state = [r for r in _sync_state_rows(con) if str(r[0]).startswith(PREFIX)] if objects or state: raise RuntimeError("restored Amazon mirror contains purged residue") return _collateral_rows(con) def restore(tenant_key, confirmation, backup_path, destination, *, rt=None, principal="operator"): """Restore a real backup to a NEW local copy, read it back before publishing that copy. This never activates a service store or rewrites the live mirror. A dirty pre-purge backup is refused, not silently sanitized by a second purge implementation. The authoritative lifecycle ledger must stay outside the backup being restored. """ key = _require_purge_confirmation(tenant_key, confirmation) rt = rt if rt is not None else _runtime_for(key) if rt is None: raise RuntimeError("restore requires a resolved tenant runtime") live_path = _tenant_mirror_path(key, rt).resolve() _assert_amazon_credential_revoked(rt) source, target = Path(backup_path).resolve(), Path(destination).resolve() if target.parent != live_path.parent or target == live_path or target.exists(): raise RuntimeError("restore requires a new copy beside this tenant's mirror") if not source.is_file(): raise RuntimeError("restore requires an existing backup file") proof = any(r["tenant"] == key and r["kind"] == "purge" and r["success"] for r in _ledger(key, rt)["checks"]) if not proof: raise RuntimeError("restore requires a recorded real purge readback") success, temporary = False, None try: fd, name = tempfile.mkstemp(prefix=".amazon-restore-", suffix=".duckdb", dir=target.parent) os.close(fd) temporary = Path(name) shutil.copyfile(source, temporary) before = _restore_readback(source) if _restore_readback(temporary) != before: raise RuntimeError("restored non-Amazon rows differ from the backup") # Hard-link publication refuses an existing target, including a concurrent creator. os.link(temporary, target) if _restore_readback(target) != before: raise RuntimeError("published restore readback differs from the backup") success = True finally: if temporary is not None: temporary.unlink(missing_ok=True) stamp = _record_deletion_check(key, rt, "restore", success) _c2_event("data.restore", key, principal, {"success": success, "checked_at": stamp, "scope": "mirror_copy"}, rt) return {"tenant": key, "checkedAt": stamp, "residueTables": [], "residueSyncMarkers": [], "scope": "mirror_copy", "success": success} def _client(): """The SP-API transport, from the DATA LAYER. ⛔⛔ THIS USED TO REACH UP INTO `aios-web/api/connectors_amazon.py`, AND `verify_meta`'S ARCHITECTURE LEG CAUGHT IT: `amazon_store.py -> connectors_amazon`. Nothing under `platform/core` or `platform/harness` may import an API-layer module, because layers point one way. The transport moved DOWN to `harness/amazon_client.py` rather than being duplicated here, since the probe and this loader need the same LWA exchange, the same read-only guard and the same report loop -- and two copies of that is two answers to one question. ⚠ Still a function rather than a module-level import, so `--status` and `--selftest` work in a process that has no network stack configured and no credential at all. """ from harness import amazon_client return amazon_client def _record_state(con, key, phase, rows=0, detail=None): """Write `_sync_state` in its declared column order, never by a misleading positional guess.""" payload_key = str(key).split("__", 1)[-1] if str(key).startswith("amzn_account_") else str(key) con.execute( "INSERT OR REPLACE INTO _sync_state " "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", [str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_", "amzn_resume_", "amzn_resync_", "amzn_waiting_", "amzn_listing_content_")) else str(detail)[:300]) if detail else None, int(rows or 0), _iso(datetime.now(timezone.utc))]) def _read_state(con, key): """One durable operation marker, or None when that operation has never completed.""" try: row = con.execute( "SELECT phase, rows, updated_at, cursor_wd FROM _sync_state WHERE entity=?", [str(key)]).fetchone() except Exception: # noqa: BLE001 return None if not row: return None return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]} def settlement_is_new(con, document_id): """A completed document is durable even when it contained no mirror lines.""" state = _read_state(con, entity_key("settlement_doc_" + str(document_id))) return not state or state["state"] != "done" def read_watermark(con, table): """Missing or malformed detail means no successful window, never the current time.""" state = _read_state(con, entity_key(WATERMARK_PREFIX + table)) detail = (state or {}).get("detail") if not detail: return None try: return _utc(detail) except (ValueError, TypeError, AttributeError, OverflowError): return None def record_watermark(con, table, when, rows=0): """Persist only after this table's pass succeeds.""" _record_state(con, entity_key(WATERMARK_PREFIX + table), "done", rows, _iso(_utc(when))) def read_backfill_floor(con, table): """Oldest contiguous landed window; missing or corrupt markers restart safely.""" state = _read_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table)) try: return _utc(state["detail"]) if state and state.get("detail") else None except (ValueError, TypeError, AttributeError, OverflowError): return None def record_backfill_floor(con, table, when): _record_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table), "done", detail=_iso(_utc(when))) def _undocumented_month(con, table): """Repair a legacy completion that never parsed a monthly or weekly document.""" if table == SQP_TABLE or not _periodic(_client().REPORT_TYPES.get(table)): return False if not read_watermark(con, table) or _read_state(con, entity_key("reconcile_" + table)): return False if not _columns_of(con, table_name(table)): return True return con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] == 0 def _first_load_due(con): state = _read_state(con, entity_key(FIRST_LOAD_DUE_KEY)) return _utc(state["detail"]) if state and state["state"] == "pending" else None def _restore_first_load_wait(tenant_key, con): """Keep the history floor in the producer and release legacy connector-wide desk holds.""" from core import connector_sync due = _first_load_due(con) connector_sync.clear_sync_defer("amazon_sp", tenant_key) return due def _loaded_refresh_hold(con, table, now, *, slot_pass=False, history=False): """Return a loaded table's cadence hold while its first load continues, or None when due.""" if history and table == "brand_analytics" and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") == "1": return None now = _utc(now) tick = _read_state(con, entity_key("sync_tick_" + table)) # A queued resync is a repair: only Amazon's own floor holds it. repair = queued_resync(con, table) is not None floor = 0 if repair else _cadence_seconds(table, tick) wait, due, _seconds, _scheduled = _interval_wait(table, tick, now, floor_s=floor) reason = None if wait > 0: # The card parses "next due " out of a reason, so every hold keeps that phrase. reason = _interval_reason(table, tick, now, floor_s=floor) mark = read_watermark(con, table) if _client().REPORT_TYPES.get(table, {}).get("weekly") and mark and not repair: week = _week_start(mark) + timedelta(days=7) if week > now and (due is None or week > due): due, reason = week, f"This report covers completed weeks only. It is next due {_iso(week)}" return (due, reason) if reason and due and due > now else None def _refused_reports(con): """C4: one safe reason per declared table; empty documents are successful loads.""" return refused_reports({key: state for table in TABLES for key in (entity_key(table), entity_key("refused_" + table)) if (state := _read_state(con, key))}) def plain_refusal_reason(reason, *, status="", table="", operation=""): """Project refusals to cause and action copy, retaining the fee API's vendor reason.""" text = str(reason or "") # A reconciliation hold is our own check: never call it a load failure. reconciliation = re.search(r"\breconciliation refused:\s*([\s\S]*)$", text, re.I) if reconciliation: detail = re.sub(r"[.\s]+$", "", reconciliation[1].strip()).replace("\u2014", ". ").replace("\u2013", " to ") if re.match(r"the check could not run", detail, re.I): return "The reconciliation check could not run. The last good rows are kept until it runs." return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail else "Reconciliation did not tie. The last good rows are kept.") feedback = table in FEEDBACK_TABLES subject = {"review_trends": "review trends", "browse_node_review_topics": "category review topics"}.get(table, "review topics") if feedback and (str(status) in ("403", "NOT_OFFERED") or re.search(r"answered\s+403\b|\(403\)", text)): vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text) detail = (vendor[1] if vendor and vendor[1] else "Amazon denied this seller or marketplace").strip() detail = detail.replace("\u2014", ". ").replace("\u2013", " to ") return (f"Amazon refused Customer Feedback {subject} (403): {detail}. " "Ask an administrator to check seller eligibility, marketplace support, and either the Brand Analytics or Selling Partner Insights role, then reconnect Amazon.") if feedback: refusal = re.search(r"answered\s+(\d{3})(?:: (.*?))?(?:; fix:|$)", text) if refusal and refusal[1] != "429": detail = (refusal[2] or "Amazon refused this child ASIN or marketplace").strip() detail = detail.replace("\u2014", ". ").replace("\u2013", " to ") return (f"Amazon refused Customer Feedback {subject} ({refusal[1]}): {detail}. " "Check the child ASIN and marketplace, then retry sync. Customer Feedback supports US, UK, France, Italy, Germany, Spain and Japan.") if table == "replenishment": if text.startswith("Amazon refused replenishment metrics:"): return text.replace("\u2014", ". ").replace("\u2013", " to ") if "Amazon reason:" in text: vendor = text.split("Amazon reason:", 1)[1].split("; fix:", 1)[0].strip() vendor = vendor.replace("\u2014", ". ").replace("\u2013", " to ") return (f"Amazon refused replenishment metrics: {vendor}. " "Ask an administrator to review Brand Analytics access for this seller and reconnect Amazon.") if "listOfferMetrics limits offsets to 9000" in text: return ("Amazon listOfferMetrics limits offsets to 9000. No partial snapshot was stored. " "Ask Amazon for a higher pagination limit or a bulk replenishment export.") if table in ("outbound_fulfillment", "inbound_shipments"): subject = "inbound shipments" if table == "inbound_shipments" else "outbound fulfillment" if text.startswith(f"Amazon refused {subject}:"): return text.replace("\u2014", ". ").replace("\u2013", " to ") refusal = re.search(r"answered 403: (.+?)(?:; fix:|$)", text) if refusal: vendor = refusal[1].replace("\u2014", ". ").replace("\u2013", " to ") return (f"Amazon refused {subject}: {vendor}. " "Ask an administrator to review Amazon Fulfillment access for this seller and reconnect Amazon.") if table == "fee_estimates_api" and text.startswith("Amazon refused the fee estimate request:"): return text.replace("\u2014", ". ").replace("\u2013", " to ") if table == "fee_estimates_api" and "Amazon reason:" in text: vendor = text.split("Amazon reason:", 1)[1].split("; fix:", 1)[0].strip() vendor = vendor.replace("\u2014", ". ").replace("\u2013", " to ") return (f"Amazon refused the fee estimate request: {vendor}. " "Ask an administrator to review the Pricing role and this price basis, then reconnect Amazon and retry sync.") lower = text.lower() status = str(status or "").upper() if "within two years" in lower or "within the past two years" in lower: return "Amazon refused this date range. Choose a start date within the past two years and retry sync." if "reportperiod" in lower or "reporting period" in lower: return "Amazon needs a reporting period for this report. Check the date range and retry sync." if status == "CANCELLED" or "cancelled" in lower: return "Amazon cancelled this report. Retry sync to request it again." if status == "FATAL" or "fatal" in lower or "could not prepare" in lower: return ("Amazon could not prepare this report. " "Check its date range and seller eligibility, then retry sync.") if (status == "429" or re.search(r"\b(?:http(?: status)?|status(?: code)?|answered)\s*[:=]?\s*429\b", lower) or "throttl" in lower or "limiting requests" in lower): return "Amazon is limiting requests. Sync resumes after Amazon's waiting period." if "read-only" in lower or "operationrefused" in lower or "cannot request that report" in lower: return "This connector cannot request that report. Ask an administrator to review the Amazon connection." if (status in ("NOT_OFFERED", "403") or re.search(r"\b(?:http(?: status)?|status(?: code)?|answered)\s*[:=]?\s*403\b|\(403\)", lower) or "not allowed" in lower or "required report role" in lower or "needs the" in lower or "not granted" in lower or "does not offer" in lower): cx = _client() role = (cx.REPORT_TYPES.get(table) or {}).get("role") role = role or " / ".join((cx.OPERATIONS.get(operation) or {}).get("roles", ())) held = bool(role) and all(r.strip() in cx.ROLES_TAKEN for r in str(role).replace(" / ", " + ").split(" + ") if r.strip()) missing_access = bool(re.search(r"missing.*role|not granted|has not granted", lower)) if not missing_access and (held or "already holds" in lower or "does not offer" in lower): role_copy = f" despite the {role} role" if role else "" return (f"Amazon does not offer this report to this seller account{role_copy}. " "Check seller eligibility in Seller Central.") role_copy = f"grant the {role} role" if role else "check the required Amazon roles" return ("Amazon has not granted access to this report. " f"Ask an administrator to {role_copy} and reconnect Amazon.") return ("Amazon could not load this report. Retry sync; if it fails again, " "ask an administrator to review the Amazon connection.") def is_refusal_reason(reason): """Recognize refusal facts in legacy status rows without first-load metadata.""" return bool(re.search( r"REFUSED:|\b(?:FATAL|CANCELLED|NOT_OFFERED)\b|Amazon (?:refused|cancelled|could not (?:prepare|load)|" r"has not granted access|does not offer|is limiting requests|needs a reporting period)|This connector cannot request", str(reason or ""), re.I)) def refused_reports(states): """Project already-read mirror markers without opening a writer connection.""" rows = [] for table in TABLES: state = states.get(entity_key(table)) or {} if state.get("state") != "refused": continue saved = states.get(entity_key("refused_" + table)) or {} try: detail = json.loads(saved["detail"]) if saved.get("detail") else {} if not isinstance(detail, dict): detail = {} except (ValueError, TypeError): detail = {} reason = str(detail.get("reason") or state.get("detail") or "") legacy_status = re.search(r"\b(CANCELLED|FATAL)\b", reason) status = (detail.get("amazonStatus") or (legacy_status.group(1) if legacy_status else "NOT_OFFERED" if re.search(r"\bHTTP\s*403\b|not allowed", reason, re.I) else "REFUSED")) spec = _client().REPORT_TYPES.get(table) or {} diagnostic = str(detail.get("diagnostic") or reason)[:500] evidence = " ".join(str(detail.get(key) or "").split("; fix:", 1)[0] for key in ("diagnostic", "error")) # Only report or seller wide vendor evidence closes history, never a plan refusal or our suggested fix. unsupported = bool(re.search( r"(?:\breport type\b[^.;\n]*\bnot allowed at this time|" r"\bdoes not offer (?:this|the requested) report to (?:this|the) seller(?: account)?|" r"\b(?:this|the requested) report is not (?:offered|supported) (?:to|for) (?:this|the) seller(?: account)?|" r"\b(?:this|the) seller(?: account)? is not eligible for this report)\s*(?:[.;\"'}]|$)", evidence, re.I)) try: attempts = max(0, int(detail.get("attempts", 0))) except (ValueError, TypeError, OverflowError): attempts = 0 disposition = "not_offered" if unsupported else "failed" if attempts >= 3 else "retrying" rows.append({"table": table, "label": spec.get("label") or "Amazon " + table.replace("_", " "), "reason": plain_refusal_reason(reason, status=status, table=table), "diagnostic": diagnostic, "amazonStatus": str(status), "state": disposition, "attempts": attempts, "error": str(detail.get("error") or "")[:500]}) return rows def first_load_payload(states, remaining, eta_seconds, *, state="partial", tables=None): """Project C1 from stored markers for both the writer and the read-only API.""" refused = refused_reports(states) by_table = {row["table"]: row for row in refused} details = [] for table in (TABLES if tables is None else tables): refusal = by_table.get(table) marker = states.get(entity_key(table)) or {} spec = _client().REPORT_TYPES.get(table) or {} detail_state = refusal["state"] if refusal else "loaded" if marker.get("state") == "done" else "loading" reason = "" if refusal: diagnostic = refusal["diagnostic"] if "omitted its recommendations row root" in diagnostic: reason = "Amazon returned a report without its recommendations section." elif "has no price or fulfillment method" in diagnostic: reason = "An Amazon listing has no price or fulfillment method for this marketplace." elif "not supported for Amazon Warehousing and Distribution" in diagnostic: reason = "Amazon does not support this operation for Warehousing and Distribution inbound plans." elif re.search(r"not granted|access.*denied|role.*not granted", diagnostic, re.I): reason = "Amazon has not granted this connection access to the requested data." elif detail_state == "not_offered": reason = "Amazon does not offer this report to this seller account." elif "No error document was supplied by Amazon" in diagnostic: reason = "Amazon could not prepare this report and supplied no error document." else: reason = refusal["error"] or "Amazon could not prepare this report." try: document = json.loads(reason) if isinstance(document, dict): errors = document.get("errors") error = errors[0] if isinstance(errors, list) and errors else document reason = error.get("message") if isinstance(error, dict) else "" except (ValueError, TypeError): pass if not isinstance(reason, str) or re.search( r"https?://|GET_[A-Z_]+|[{}]|\breportId\b|\b\w+(?:Error|Exception):", reason): reason = "Amazon could not prepare this report." reason = reason or "Amazon could not prepare this report." reason = " ".join(reason.replace("\u2014", ". ").replace("\u2013", " to ").split())[:500] details.append({"table": table, "label": spec.get("label") or "Amazon " + table.replace("_", " "), "state": detail_state, "reason": reason}) retrying = any(row["state"] == "retrying" for row in details) failed = any(row["state"] == "failed" for row in details) line = ("Amazon data is loading and temporary failures retry automatically." if retrying else "Amazon data is loading automatically." if remaining and state != "failed" else "The Amazon first load finished with some data unavailable." if failed or state == "failed" else "The available Amazon data has finished loading.") eta_days = (int((eta_seconds + 86399) // 86400) if remaining and state != "failed" else None if failed or state == "failed" else 0) return {"remaining": remaining, "etaSeconds": eta_seconds, "cause": first_load_eta_cause(remaining), "refused": refused, "line": line, "etaDays": eta_days, "details": details} def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_tables=None, invalid_proofs=None, reopened_tables=None): """A fresh account or an unfinished durable load uses bounded passes, even after one table finishes.""" rt = rt if rt is not None else _runtime_for(tenant_key) restored = restore_mirror(tenant_key, rt) con, _path = _connect_tenant_mirror(tenant_key, rt) try: load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) _restore_first_load_wait(tenant_key, con) enabled = read_database_enablement(rt)["databases"] added = [table for table in GROUP_A_TABLES if table in TABLES and enabled.get(amazon_schema.DATABASE_PREFIX + table)] # S66-T45 (lead, observed live): a table Amazon refuses outright (a 400 "not allowed", # a 403 for a report it does not offer this seller, a document with no row root) never # earns a watermark, so it kept this answer True and re-queued a first-load pass on every # boot. Its refusal is on screen; the scheduled rotation retries it. invalid = set() def documented(table): proof = _read_state(con, entity_key("reconcile_" + table)) if not proof: return False invalid.add(table) if proof.get("state") != "done" or not proof.get("detail"): return False try: parsed = json.loads(proof["detail"]) if not isinstance(parsed, dict) or not parsed.get("start") or not parsed.get("end"): return False bounds = [] for value in (parsed["start"], parsed["end"]): bound = datetime.fromisoformat(value + "-01" if len(value) == 7 else value.replace("Z", "+00:00")) bounds.append(bound.replace(tzinfo=timezone.utc) if bound.tzinfo is None else bound.astimezone(timezone.utc)) if bounds[0] <= bounds[1]: invalid.discard(table) return True return False except (ValueError, TypeError, AttributeError, OverflowError): return False documented_tables = {table for table in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + table) and documented(table)} repairs = set() refusals = {row["table"]: row for row in _refused_reports(con)} try: cohort = _utc(load["detail"]) if load and load.get("detail") else datetime.now(timezone.utc) except (ValueError, TypeError, AttributeError, OverflowError): cohort = None def terminal(table): state = _read_state(con, entity_key(table)) or {} if state.get("state") == "refused": return (refusals.get(table, {}).get("state") in ("not_offered", "failed") or read_watermark(con, table) is not None) spec = _client().REPORT_TYPES.get(table) or {} proven = not (_periodic(spec) or spec.get("snapshot")) or table in documented_tables if not proven and _periodic(spec) and table != SQP_TABLE and read_watermark(con, table): mirror = table_name(table) if not _columns_of(con, mirror) or con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] == 0: repairs.add(table) if state.get("state") != "done" or not proven: return False if table == "settlements": return state.get("detail") == "No issued settlement reports found." if spec.get("snapshot"): return True floor = read_backfill_floor(con, table) if cohort is None: return False if table == SQP_TABLE: return _sqp_history_done(con, cohort) # The same per-table floor first_load_plan uses, so a load that lands every window completes. return floor is not None and floor <= history_floor(con, table, cohort) # ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left # "failed" only because Amazon refused a report (a 403 for a report it does not offer this # seller) stayed "not done" forever, so every restart queued it again AND the scheduled # producer deferred every table to it: live's Amazon tables stopped refreshing. A stopped # load is still owed only while an enabled table lacks a watermark and completion evidence. # A continuing load remains owed while its tables still lack completion evidence. # Refusal closes history debt even when a legacy monthly watermark has no document proof. completed = {table for table in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + table) and terminal(table)} if completed_tables is not None: completed_tables.update(completed) if repair_tables is not None: repair_tables.update(repairs) if invalid_proofs is not None: invalid_proofs.update(invalid) # H2: a loaded table whose history floor moved earlier owes only its history; recent data keeps landing. reopened = {table for table in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + table) and cohort is not None and table not in completed and table not in repairs and table not in refusals and _history_reopened(con, table, cohort)} if reopened_tables is not None: reopened_tables.update(reopened) missing = [table for table in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + table) and not read_watermark(con, table) and table not in completed] owed = bool(load and load["state"] != "done" and missing) partially_refused = {name[len(PREFIX):] for name in restored.get("refused_tables", ())} return (bool(partially_refused.intersection(missing)) or owed or bool(repairs) or bool(reopened) or (bool(missing) and not any(read_watermark(con, table) for table in TABLES)) or any(table in missing for table in added)) finally: con.close() def first_load_status(tenant_key, rt=None): """Durable numeric progress for the API's safe-status projection; no vendor payloads.""" con, _path = _connect_tenant_mirror(tenant_key, rt) try: state = _read_state(con, entity_key(FIRST_LOAD_STATUS_KEY)) if not state: return None values = json.loads(state["detail"]) markers = {row[0]: {"state": row[1], "detail": row[3]} for row in _sync_state_rows(con)} enabled = read_database_enablement(rt)["databases"] remaining = int(values["remaining"]) # S82-T14: one pass sees only its own tables. The job's store record counts the whole load. job = first_load_job_summary(rt) if _store_documents(rt) else None if job: job["tables"]["inventory"] = inventory_table_status(con) if job and isinstance(job.get("remaining"), int) and state["state"] != "done": remaining = job["remaining"] enabled_tables = [table for table in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + table)] load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) try: cohort = _utc(load["detail"]) if load and load.get("detail") else datetime.now(timezone.utc) except (ValueError, TypeError, AttributeError, OverflowError): cohort = datetime.now(timezone.utc) out = {"state": state["state"], **first_load_payload(markers, remaining, int(values["etaSeconds"]), state=state["state"], tables=enabled_tables), "backfillDays": DEFAULT_BACKFILL_DAYS, "historyFloors": history_status(con, enabled_tables, cohort), "nextDue": _iso(_first_load_due(con)) if _first_load_due(con) else None} if job: out["job"] = job if SQP_TABLE in enabled_tables: out["sqp"] = _sqp_first_load_status(con, cohort) return out finally: con.close() # ----------------------------------------------------------------------------- S82-T14 the first-load job #: The job's cursor, one store document per tenant: finished windows, open reportIds, throttled resumes. FIRST_LOAD_CURSOR_KEY = "amazon_first_load_cursor" #: One record per chain generation: start, end, remaining before and after, tables published. FIRST_LOAD_GENERATIONS_KEY = "amazon_first_load_generations" #: The one per-seller createReport ledger. The job and the daily pass draw from it. REPORT_LEDGER_KEY = "amazon_report_ledger" #: C-pub: the writer lease is per connector, so the job never holds the lease the Odoo resync needs. AMAZON_WRITER_LEASE_DOC = "sync_writer_lease:amazon" FIRST_LOAD_LEASE_SECONDS = 900 #: A saved reportId is polled again for this long, the same rule `_report_resume` keeps. REPORT_REUSE_SECONDS = 86400 FIRST_LOAD_POLL_SECONDS = 30 #: A generation stops requesting this long before its budget, so its open reports can land. FIRST_LOAD_RESERVE_SECONDS = 1800 #: Generation records the document keeps. The job's history, never connector data. FIRST_LOAD_GENERATION_HISTORY = 200 #: D-2643: the codes an empty or refused Amazon table names. Inventory alone also keeps #: S85-T107's `request_unknown` for missing request metadata. EMPTY_REASON_CODES = ("not_requested", "refused_by_amazon", "first_load_pending", "no_source_rows") #: D-2643: every declared database returns a row count or names why it is empty while the load runs. #: Declared datasets and saved refusals join at run time through `empty_reason_tables`. EMPTY_REASON_TABLES = DATABASE_TABLES #: Retain the latest request after pending markers are cleared, within the inventory proof family. INVENTORY_REQUEST_KEY = "reconcile_inventory_request" #: D-2642 and D-2649: these name why they are behind while the load runs. BEHIND_TABLES = ("orders", "order_items") #: D-2642: the dated tables whose newest row is held against the table's cadence on every read. LAG_DATE_COLUMNS = dict.fromkeys(BEHIND_TABLES, "purchase_date") #: The keys `table_empty_status(lag=True)` adds. `lagCode` and `lag` appear only when a table trails. LAG_KEYS = ("newestPurchaseDate", "cadence", "cadenceSeconds", "lastPullAt", "lagCode", "lag") #: S82-T02's C-status document (lane A). `connector_sync.STATUS_KEY` wins once it exists. SYNC_STATUS_DOC = "connector_sync_status" def _epoch_iso(stamp): return _iso(datetime.fromtimestamp(float(stamp), timezone.utc)) def _store_documents(rt): """True when `rt` reads and writes store documents (`get` and `update`).""" return (rt is not None and callable(getattr(rt, "get", None)) and callable(getattr(rt, "update", None)) and type(rt).__module__ != "unittest.mock") class _ProcessDocuments: """Store documents held in this process only, for a dry run. A killed process keeps none of them.""" def __init__(self): self.docs = {} def get(self, name, fresh=False, **_kwargs): return json.loads(json.dumps(self.docs[name])) if name in self.docs else None def update(self, name, fn, flush="sync", **_kwargs): doc = json.loads(json.dumps(self.docs.get(name) or {})) out = fn(doc) self.docs[name] = doc if out is None else out return self.get(name) def _doc(docs, name): try: value = docs.get(name, fresh=True) except TypeError: value = docs.get(name) return value if isinstance(value, dict) else {} #: D-17410: live's daily Amazon pass runs from 00:00Z to about 02:00Z, and staging's chain shares its seller quota. LIVE_DAILY_HOLD_START_HOUR = 0 LIVE_DAILY_HOLD_END_HOUR = 3 #: UTC hour buckets each seller's ledger row keeps, so both stores' ledgers show spend per hour. REPORT_LEDGER_HOURS_KEPT = 72 _LIVE_HOLD_NOTICED = set() def live_daily_hold_seconds(now=None): """Seconds staging's report requests wait for live's daily Amazon pass, else 0.""" if os.environ.get("AIOS_FIRST_LOAD_TARGET") != "staging": return 0.0 at = datetime.fromtimestamp(time.time() if now is None else float(now), timezone.utc) day = at.replace(hour=0, minute=0, second=0, microsecond=0) start = day + timedelta(hours=LIVE_DAILY_HOLD_START_HOUR) stop = day + timedelta(hours=LIVE_DAILY_HOLD_END_HOUR) return (stop - at).total_seconds() if start <= at < stop else 0.0 def _note_live_hold(stamp): window = datetime.fromtimestamp(float(stamp), timezone.utc).strftime("%Y-%m-%d") if window in _LIVE_HOLD_NOTICED: return _LIVE_HOLD_NOTICED.add(window) print(f"[amazon-report-ledger] staging holds its report requests until {LIVE_DAILY_HOLD_END_HOUR:02d}:00Z " "while live's daily Amazon pass runs (D-17410)", flush=True) def take_report_token(docs, seller, *, now=None, burst=None, refill_seconds=None): """Draw one createReport token for `seller` from the store ledger. -> `(granted, wait seconds)`. Amazon's plan is a burst of 15, then one token per 60 s. The store update is the compare-and-set. """ from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS stamp = time.time() if now is None else float(now) hold = live_daily_hold_seconds(stamp) if hold > 0: # A held draw spends no token and leaves the ledger document unchanged. _note_live_hold(stamp) return False, hold burst = float(REPORT_BURST if burst is None else burst) refill = float(REPORT_SUSTAINED_SECONDS if refill_seconds is None else refill_seconds) out = {} def draw(doc): sellers = doc.setdefault("sellers", {}) row = dict(sellers.get(seller) or {"tokens": burst, "updated": stamp}) elapsed = max(0.0, stamp - float(row.get("updated") or stamp)) tokens = min(burst, float(row.get("tokens", burst)) + elapsed / refill) if tokens >= 1.0: tokens -= 1.0 row.update(spent=int(row.get("spent") or 0) + 1, lastCreate=stamp) hours = dict(row.get("hours") or {}) hour = datetime.fromtimestamp(stamp, timezone.utc).strftime("%Y-%m-%dT%H") hours[hour] = int(hours.get(hour) or 0) + 1 row["hours"] = {key: hours[key] for key in sorted(hours)[-REPORT_LEDGER_HOURS_KEPT:]} out.update(granted=True, wait=0.0) else: out.update(granted=False, wait=(1.0 - tokens) * refill) row.update(tokens=tokens, updated=stamp) sellers[seller] = row return doc docs.update(REPORT_LEDGER_KEY, draw, flush="sync") return out["granted"], out["wait"] def _lease_live(doc, now, ttl=FIRST_LOAD_LEASE_SECONDS): return bool(doc.get("holder")) and float(now) - float(doc.get("renewedAt") or 0) <= ttl def first_load_job_live(docs, *, now=None): """True while a first-load generation holds the Amazon writer lease.""" try: return _lease_live(_doc(docs, AMAZON_WRITER_LEASE_DOC), time.time() if now is None else now) except Exception: # noqa: BLE001 return False #: run_first_load_generation's default holder prefix. A resident worker's own pass holds the lease as `:amazon:`. FIRST_LOAD_HOLDER_PREFIX = "amazon-first-load-" def first_load_job_holds(docs, *, now=None): """True while a first-load generation, never a resident worker, holds a live Amazon writer lease.""" try: doc = _doc(docs, AMAZON_WRITER_LEASE_DOC) return (_lease_live(doc, time.time() if now is None else now) and str(doc.get("holder") or "").startswith(FIRST_LOAD_HOLDER_PREFIX)) except Exception: # noqa: BLE001 return False def acquire_amazon_lease(docs, holder, *, now=None, ttl=FIRST_LOAD_LEASE_SECONDS): """Take or renew the Amazon-scoped writer lease. True when `holder` holds it. The Odoo lease is untouched.""" stamp = time.time() if now is None else float(now) def take(doc): if doc.get("holder") in (None, "", holder) or not _lease_live(doc, stamp, ttl): if doc.get("holder") != holder: doc.update(acquiredAt=stamp, previous=doc.get("holder")) doc.update(holder=holder, renewedAt=stamp, connector="amazon") return doc doc = docs.update(AMAZON_WRITER_LEASE_DOC, take, flush="sync") if not isinstance(doc, dict): doc = _doc(docs, AMAZON_WRITER_LEASE_DOC) return doc.get("holder") == holder def release_amazon_lease(docs, holder): def drop(doc): if doc.get("holder") == holder: doc.update(holder=None, renewedAt=0, previous=holder) return doc docs.update(AMAZON_WRITER_LEASE_DOC, drop, flush="sync") def _ledger_draw(docs): """The daily pass's draw: spend a ledger token, and defer to a live job when the seller has none left.""" cx = _client() def take(fields): try: granted, wait = take_report_token(docs, cx.seller_key(fields)) except Exception: # noqa: BLE001 return # An unreadable ledger leaves the in-process pacer in charge. if not granted and live_daily_hold_seconds() > 0: raise cx.AmazonThrottled( f"Staging holds its Amazon report requests until {LIVE_DAILY_HOLD_END_HOUR:02d}:00 UTC while " "live's daily Amazon pass runs. This report resumes after the hold.", retry_after=wait, operation="createReport") if not granted and first_load_job_live(docs): raise cx.AmazonThrottled( "The first-load job is using this seller's Amazon report quota. " "This report resumes after the quota refills.", retry_after=wait, operation="createReport") return take @contextmanager def _report_ledger(rt): if not _store_documents(rt): yield return with _client().report_ledger_scope(_ledger_draw(rt)): yield def first_load_request(table, w0, w1): """The createReport arguments `_sync_candidate` sends for one first-load window, so a prefetch matches.""" cx = _client() spec = cx.REPORT_TYPES[table] options = ({**REPORT_OPTIONS.get(table, {}), "reportPeriod": spec["period"]} if spec.get("period") else dict(REPORT_OPTIONS[table]) if table in REPORT_OPTIONS else None) return {"report_type": spec["report"], "start": _iso(w0), "end": _iso(w1 - timedelta(seconds=1)) if _periodic(spec) else _iso(w1), "options": options or None} def job_prefetchable(table): """Tables whose windows go through the one `report_document` call the job can prefetch.""" spec = _client().REPORT_TYPES.get(table) or {} return bool(spec.get("report") and not spec.get("snapshot") and table not in PERIOD_TABLES and table not in LEGACY_SNAPSHOT_TABLES and table not in ("settlements", SQP_TABLE)) def _pace_weekly_history(tasks, owed, limit=None): """Keep the newest `limit` owed weeks of each weekly table in this generation; later generations take the rest. The job runs its own reports before the other-tables pass, so unpaced weekly history held SQP back for hours. """ limit = FIRST_LOAD_WEEKLY_HISTORY_PER_GENERATION if limit is None else max(1, int(limit)) weekly = {} for task in tasks: if owed(task) and _client().REPORT_TYPES.get(task["table"], {}).get("weekly"): weekly.setdefault(task["table"], []).append(task) deferred = set() for rows in weekly.values(): rows.sort(key=lambda t: str(t.get("start") or ""), reverse=True) deferred.update(t["id"] for t in rows[limit:]) return [t for t in tasks if t["id"] not in deferred] def first_load_task(account, fields, table, w0, w1, batch=None, *, landed=False): """One owed first-load task. `key` is the report identity, shared by tables that share a report.""" ident = json.dumps([str(account), table, _iso(w0) if w0 else None, _iso(w1) if w1 else None, list(batch) if batch else None]) task = {"id": ident, "account": str(account), "table": table, "start": _iso(w0) if w0 else None, "end": _iso(w1) if w1 else None, "landed": bool(landed), "prefetch": bool(w0 is not None and w1 is not None and job_prefetchable(table))} if task["prefetch"]: task["request"] = first_load_request(table, w0, w1) req = task["request"] task["key"] = _client().prefetch_key(fields, req["report_type"], req["start"], req["end"], req["options"]) task["seller"] = _client().seller_key(fields) return task def _task_landed(con, table, w0, w1, batch, end=None, *, reopened=False): """The mirror's own evidence that an earlier pass landed this window: watermark, floor or month marker. A reopened table's watermark proves only its recent data, so its floor decides. """ if table == SQP_TABLE and w0 is not None: return _sqp_done(con, w0, w1, batch) if _after_cohort(table, w0, end): return _cohort_month_done(con, table, w0) if (not reopened and read_watermark(con, table) and (_read_state(con, entity_key(table)) or {}).get("state") == "done"): return True floor = read_backfill_floor(con, table) return w1 is not None and floor is not None and w1 > floor def first_load_job_plan(tenant_key, rt, *, log=print): """Every first-load task this tenant still owes, from `first_load_plan` and the mirror's floors. -> `(end, tasks, fields_by_account)`. `end` is the cohort end `_sync_candidate` froze. `end` is None before a first pass froze one: the job then runs one bounded pass first. """ entries = account_entries(tenant_key, rt) completed, repairs, reopened = set(), set(), set() owed = needs_first_load(tenant_key, rt=rt, completed_tables=completed, repair_tables=repairs, reopened_tables=reopened) con, _path = _connect_tenant_mirror(tenant_key, rt) try: load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) if not owed: return None, [], {} if not load or not load.get("detail"): return None, None, {} end = _utc(load["detail"]) enabled = read_database_enablement(rt)["databases"] tables = [t for t in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + t)] single = len(entries) == 1 wanted = [t for t in tables if t not in completed or t in repairs] if single else tables sqp_asins = _sqp_catalogue(con) if SQP_TABLE in wanted else () tasks, fields_by = [], {} starts = history_starts(con, wanted, end) for entry in entries: cid = entry["connection_id"] fields = fields_by[cid] = _sync_credential(tenant_key, rt, connection_id=cid) items = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins, starts=starts) for item in _sqp_after_cohort(items, end, sqp_asins): table, w0, w1 = item[:3] batch = item[3] if len(item) == 4 else None if single and table != SQP_TABLE and not _after_cohort(table, w0, end): # The same clip sync applies, so a window straddling the landed floor is fetched below it only. owed_window = _owed_window(table, w0, w1, None if table in repairs else read_backfill_floor(con, table)) w0, w1 = owed_window or (w0, w1) # Account-scoped floors live in each account's workspace; only one account reads them here. landed = single and _task_landed(con, table, w0, w1, batch, end, reopened=table in reopened) task = first_load_task(cid, fields, table, w0, w1, batch, landed=landed) # One sync writes every account, so a prefetch serves a single-account tenant only. task["prefetch"] = task["prefetch"] and single tasks.append(task) return end, tasks, fields_by finally: con.close() class _JobClient: """The job's Amazon calls, each inside `rate_limit_scope` for its tenant and account.""" def __init__(self, tenant_key, fields_by_account, log=print): self.tenant, self.fields, self.log = tenant_key, fields_by_account, log def create(self, task): cx = _client() req = task["request"] with cx.rate_limit_scope(self.tenant, task["account"]): return cx.create_report_id(self.fields[task["account"]], req["report_type"], req["start"], req["end"], req["options"]) def poll(self, task, report_id): cx = _client() with cx.rate_limit_scope(self.tenant, task["account"]): return cx.poll_report(self.fields[task["account"]], task["request"]["report_type"], report_id, log=lambda message: self.log(f"{task['table']}: {message}")) def _sync_writer(tenant_key, rt, log): """Write prefetched windows through `sync`, which parses, validates and publishes per table (C-pub).""" def write(tables, documents): cx = _client() _PUBLISHED.flag = False with cx.prefetch_scope(documents, only=True) as served: report = sync(tenant_key, tables=list(tables), log=log, rt=rt, first_load=True, report_budget=len(documents)) return {"served": set(served), "published": bool(getattr(_PUBLISHED, "flag", False)), "tables": {t: dict((report.get("tables") or {}).get(t) or {}) for t in tables}} return write def _sync_delegate(tenant_key, rt, log): """Every table but the job's own takes one bounded first-load pass: settlements, period, Group A, SQP. The same pass refreshes each loaded table whose cadence is due, so loaded data keeps moving. """ def delegate(exclude, budget): tables = None if exclude is None else [t for t in TABLES if t not in set(exclude)] return sync(tenant_key, tables=tables, log=log, rt=rt, first_load=True, report_budget=budget) return delegate def _mirror_row_counts(tenant_key, rt): def counts(tables): con, _path = _connect_tenant_mirror(tenant_key, rt) try: out = {} for table in tables: out[table] = (con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] if _columns_of(con, table_name(table)) else 0) return out finally: con.close() return counts def read_connector_status(docs): """C-status: lane A's store document (last good sync, pass times, resync state). One read, no wait.""" key = SYNC_STATUS_DOC try: from core import connector_sync key = getattr(connector_sync, "STATUS_KEY", key) except Exception: # noqa: BLE001 pass try: doc = _doc(docs, key) if key != SYNC_STATUS_DOC and (doc is None or isinstance(doc, dict) and not doc): doc = _doc(docs, SYNC_STATUS_DOC) except Exception: # noqa: BLE001 return {} connectors = doc.get("connectors") if isinstance(doc.get("connectors"), dict) else doc return {name: row for name, row in connectors.items() if isinstance(row, dict)} def _status_stamp(status, connector): """Newest stored success across this provider's connections, retaining legacy flat entries.""" row = status.get(connector) or {} if not isinstance(row, dict): return None rows = [row] if "lastGoodSyncAt" in row else list(row.values()) stamps = [] for entry in rows: stamp = entry.get("lastGoodSyncAt") if isinstance(entry, dict) else None if not isinstance(stamp, str) or not stamp.strip(): continue try: parsed = datetime.fromisoformat(stamp.replace("Z", "+00:00")) parsed = parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc) except (ValueError, OverflowError): continue stamps.append((parsed, stamp)) return max(stamps, key=lambda item: item[0])[1] if stamps else None def inventory_table_status(con): """Read inventory counts and account-scoped source proof without changing the mirror or requesting reports.""" rows = (int(con.execute("SELECT COUNT(*) FROM amzn_inventory").fetchone()[0]) if _columns_of(con, "amzn_inventory") else None) out = {"rows": rows, "empty": rows == 0 if rows is not None else None} keys = ("amzn_inventory", "amzn_reconcile_inventory", "amzn_refused_inventory", "amzn_inventory_awd", entity_key(INVENTORY_REQUEST_KEY)) clauses = " OR ".join("entity=? OR ends_with(entity, ?)" for _ in keys) saved = con.execute( "SELECT entity, phase, cursor_wd, rows, updated_at FROM _sync_state WHERE " + clauses + " OR contains(entity, 'amzn_pending_report_')", [value for key in keys for value in (key, "__" + key)]).fetchall() accounts = {} def payload(state): try: value = json.loads(state.get("detail") or "{}") except (ValueError, TypeError): value = {} return value if isinstance(value, dict) else {} def request_status(state): detail = payload(state) window = detail.get("window") if isinstance(detail.get("window"), dict) else {} return {"type": detail.get("reportType") or _client().REPORT_TYPES["inventory"]["report"], "state": state.get("state"), "reportId": detail.get("reportId"), "processingStatus": detail.get("processingStatus"), "documentId": detail.get("reportDocumentId") or detail.get("document"), "start": detail.get("start") or window.get("start"), "end": detail.get("end") or window.get("end"), "updatedAt": state.get("updated_at")} def refusal_sentence(reason): cause, separator, action = str(reason).partition(". ") return f"{cause}, so {action[:1].lower()}{action[1:]}" if separator else cause for entity, phase, detail, count, stamp in saved: prefix, key = (str(entity).split("__", 1) if str(entity).startswith("amzn_account_") and "__" in str(entity) else ("", str(entity))) state = {"state": phase, "detail": detail, "rows": count, "updated_at": stamp} if key.startswith("amzn_pending_report_") and payload(state).get("table") not in (None, "inventory"): continue accounts.setdefault(prefix, {})[key] = state # Account markers own their proof once account scoping exists. scoped = {key: state for key, state in accounts.items() if key} accounts = scoped or accounts reports, pending, unknown_request = [], [], False for states in accounts.values(): own_pending = [] for key, state in states.items(): if not key.startswith("amzn_pending_report_"): continue request = payload(state) if request.get("table") == "inventory": own_pending.append(request_status(state)) elif not request.get("table"): unknown_request = True latest = states.get(entity_key(INVENTORY_REQUEST_KEY)) or states.get("amzn_reconcile_inventory") if latest: own_pending = [row for row in own_pending if str(row.get("updatedAt") or "") >= str(latest.get("updated_at") or "")] pending.extend(own_pending) if own_pending and (not latest or max(row.get("updatedAt") or "" for row in own_pending) >= str(latest.get("updated_at") or "")): reports.extend(own_pending) elif latest: reports.append(request_status(latest)) if latest and latest.get("state") in ("pending", "throttled") and not own_pending: pending.append(request_status(latest)) out["reports"] = reports if rows: return out if pending: throttled = any(report["state"] == "throttled" for report in pending) return {**out, "code": "first_load_pending", "reason": "Amazon limited the inventory request, so keep sync enabled to resume after its waiting period." if throttled else "Amazon is preparing the inventory report, so keep sync enabled and retry after it completes."} periods = set() for states in accounts.values(): marker = states.get("amzn_inventory") or {} refused = refused_reports(states) if refused: return {**out, "code": "refused_by_amazon", "reason": refusal_sentence(refused[0]["reason"])} request = states.get(entity_key(INVENTORY_REQUEST_KEY)) or {} if request.get("state") in ("refused", "failed"): return {**out, "code": "refused_by_amazon", "reason": refusal_sentence( payload(request).get("reason") or plain_refusal_reason("", table="inventory"))} proof = states.get("amzn_reconcile_inventory") or {} if not marker and not proof and not request: if unknown_request: return {**out, "code": "request_unknown", "reason": "A saved Amazon request has no table identity, so retry sync to refresh the inventory report status."} return {**out, "code": "request_unknown", "reason": "Inventory request metadata is unavailable, so retry Amazon sync to record the FBA ledger request."} detail = payload(proof) dates = detail.get("dates") valid_dates = (isinstance(dates, dict) and bool(dates) and all(isinstance(day, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", day) and type(count) is int and count == 0 for day, count in dates.items())) awd = states.get("amzn_inventory_awd") or {} cancelled_empty = (detail.get("cancelled") is True and detail.get("processingStatus") == "CANCELLED" and isinstance(detail.get("reportId"), str) and bool(detail["reportId"].strip()) and not detail.get("document")) source_document = isinstance(detail.get("document"), str) and bool(detail["document"].strip()) if (marker.get("state") != "done" or marker.get("rows") != 0 or proof.get("state") != "done" or type(detail.get("count")) is not int or detail["count"] != 0 or not (source_document or cancelled_empty) or detail.get("column") != "_snapshot_date" or not valid_dates or detail.get("start") != min(dates) or detail.get("end") != max(dates) or awd.get("state") != "done" or awd.get("rows") != 0): return {**out, "code": "first_load_pending", "reason": "Inventory source checks are incomplete, so retry Amazon sync to finish the FBA and AWD inventory checks."} periods.add((detail["start"], detail["end"])) if not periods: return {**out, "code": "request_unknown", "reason": "Inventory request metadata is unavailable, so retry Amazon sync to record the FBA ledger request."} start, end = next(iter(periods)) period = (start if start == end else f"{start} to {end}") if len(periods) == 1 else "the loaded dates" return {**out, "code": "no_source_rows", "reason": f"Amazon returned no FBA ledger rows for {period} and no AWD inventory, so check seller stock and retry Amazon sync.", "reports": reports} _UNCOUNTED = object() _INVENTORY_UNKNOWN = ("Inventory request metadata is unavailable, " "so retry Amazon sync to record the FBA ledger request.") def _reason_line(text): """One screen line with no line break and no em or en dash.""" return " ".join(str(text or "").replace("\u2014", ". ").replace("\u2013", " to ").split()) def _marker_detail(state): try: value = json.loads((state or {}).get("detail") or "{}") except (ValueError, TypeError): value = {} return value if isinstance(value, dict) else {} def _reason_markers(con, table=None): """Saved sync markers per account prefix (`""` for unscoped). Read only. With `table`, only that table's own, refusal and reconciliation markers are read. """ if not _columns_of(con, "_sync_state"): return {} sql, params = "SELECT entity, phase, cursor_wd, rows, updated_at FROM _sync_state", [] if table is not None: dated = str(table) in LAG_DATE_COLUMNS keys = [entity_key(name + str(table)) for name in ("", "refused_", "reconcile_", *((WATERMARK_PREFIX,) if dated else ()))] keys += [entity_key(FIRST_LOAD_END_KEY)] if dated else [] sql += " WHERE " + " OR ".join("entity=? OR ends_with(entity, ?)" for _ in keys) params = [value for key in keys for value in (key, "__" + key)] accounts = {} for entity, phase, detail, count, stamp in con.execute(sql, params).fetchall(): entity = str(entity) prefix, key = (entity.split("__", 1) if entity.startswith("amzn_account_") and "__" in entity else ("", entity)) accounts.setdefault(prefix, {})[key] = {"state": phase, "detail": detail, "rows": count, "updated_at": stamp} return accounts def _table_markers(accounts, table): """Each account's markers for `table`. Account markers own a table once any account holds one.""" keys = {entity_key(name + str(table)) for name in ("", "refused_", "reconcile_")} scoped = [states for prefix, states in accounts.items() if prefix and keys & set(states)] return scoped or ([accounts[""]] if "" in accounts else []) def empty_reason_tables(con=None, *, accounts=None): """D-2643: declared databases and datasets read now, plus every dataset with a saved refusal.""" names = [*DATABASE_TABLES, *_client().DECLARED_DATASETS] if con is not None or accounts is not None: refused = entity_key("refused_") for states in (_reason_markers(con) if accounts is None else accounts).values(): names.extend(key[len(refused):] for key, state in states.items() if key.startswith(refused) and state.get("state") == "refused") return tuple(dict.fromkeys(names)) def _stamp_or_none(value): try: return _utc(value) if value else None except (ValueError, TypeError, AttributeError, OverflowError): return None def _lag_age(seconds): """`259200` -> "3 days". Screen copy for a lag, rounded down to its largest whole unit.""" seconds = max(0, int(seconds)) for size, unit in ((86400, "day"), (3600, "hour")): if seconds >= 2 * size: return f"{seconds // size} {unit}s" minutes = seconds // 60 return f"{minutes} minute" + ("" if minutes == 1 else "s") def _newest_row_at(con, table): """The newest row date of a dated table in UTC, or None when no date can be read.""" column, mirror = LAG_DATE_COLUMNS[table], table_name(table) if column not in _columns_of(con, mirror): return None value = con.execute(f'SELECT epoch(MAX(TRY_CAST("{column}" AS TIMESTAMPTZ))) FROM {mirror}').fetchone()[0] return None if value is None else datetime.fromtimestamp(float(value), timezone.utc) def _last_pull_at(markers, table): """The most behind account's newest finished pull of `table`. None when any account has none.""" pulls = [] for states in markers: stamps = [stamp for key in (entity_key(WATERMARK_PREFIX + table), entity_key(table)) if (states.get(key) or {}).get("state") == "done" and (stamp := _stamp_or_none(states[key].get("updated_at"))) is not None] if not stamps: return None pulls.append(max(stamps)) return min(pulls) if pulls else None def _lag_cause(table, markers, newest, last, now, cadence): """D-2642: the code, cause and fix of a dated table that trails its cadence. Read only.""" phases = {(states.get(entity_key(table)) or {}).get("state") for states in markers} loads = {(states.get(entity_key(FIRST_LOAD_END_KEY)) or {}).get("state") for states in markers} if "refused" in phases: return "refused_by_amazon", "Amazon refused the last request", "follow the refusal reason and retry Amazon sync" if "throttled" in phases: return "throttled", "Amazon throttled the last pass", "the next pass resumes after Amazon's waiting period" if "failed" in phases: return "pass_failed", "the last Amazon pass did not finish", "retry Amazon sync" if "pending" in phases: return "report_pending", "Amazon is still preparing the last report", "the next pass collects it" if last is None or (now - last).total_seconds() > cadence + SCHEDULE_PHASE_GRACE_SECONDS: # A pull inside its cadence makes the newest row Amazon's answer. Only a missed pull is a stop. if loads & {"running", "partial"}: return ("first_load_pending", "the first Amazon load has not finished", "newer orders arrive as it continues") if last is None: return "no_recorded_pull", "no finished Amazon pull is recorded", "run Amazon sync" return ("sync_stopped", f"no Amazon pull has finished for {_lag_age((now - last).total_seconds())}", "run Amazon sync and check its status in Connectors") return ("no_newer_orders", f"Amazon reported no orders newer than {newest.date().isoformat()} in the last pull", "check seller activity in Seller Central") def unknown_lag(table): """The lag keys of a dated table whose newest row date cannot be read. Never a fresh answer.""" noun = str(table).replace("_", " ").removesuffix("s") return {"lagCode": "unknown_timestamp", "lag": f"The purchase date of the newest {noun} cannot be read, so run Amazon sync and check again."} def _freshness(con, table, markers, now=None): """D-2642: newest row date, cadence and last pull of a dated table, plus one lag line when it trails. The table trails when its newest row is older than one cadence. An unread date never reads fresh. """ now = _utc(now) cadence = _cadence_seconds(table) last = _last_pull_at(markers, table) try: newest = _newest_row_at(con, table) except Exception: # noqa: BLE001 newest = None out = {"newestPurchaseDate": _iso(newest) if newest else None, "cadence": "every " + _readable_duration(cadence).removeprefix("1 "), "cadenceSeconds": cadence, "lastPullAt": _iso(last) if last else None} if newest is None: return {**out, **unknown_lag(table)} age = (now - newest).total_seconds() if age <= cadence: return out code, cause, fix = _lag_cause(table, markers, newest, last, now, cadence) noun = str(table).replace("_", " ").removesuffix("s") return {**out, "lagCode": code, "lag": _reason_line(f"The newest {noun} is {_lag_age(age)} old because {cause}, so {fix}.")} def table_empty_status(con, table, *, rows=_UNCOUNTED, enabled=None, accounts=None, lag=False, now=None): """D-2643: one table's row count, or the code and one-line reason it answers with. The one per-table producer every door reads (Data grid, MCP, list_workspace). A saved refusal names its reason beside any rows. Missing markers read first load pending, never a zero. `enabled` is a readable enablement map. Pass None when enablement is unknown. D-2642: with `lag`, a dated table that holds rows also answers `LAG_KEYS` from `_freshness`. """ table = str(table) declared = _client().DECLARED_DATASETS.get(table) if declared is not None: return {"table": table, "rows": None, "empty": True, "code": "not_requested", "reason": _reason_line(declared.get("reason"))} mirror = table_name(table) if rows is _UNCOUNTED: rows = (int(con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]) if _columns_of(con, mirror) else None) out = {"table": table, "rows": rows, "empty": rows == 0 if rows is not None else None} if table == "inventory": if _columns_of(con, "_sync_state"): status = inventory_table_status(con) return {**out, **{key: status[key] for key in ("code", "reason") if status.get(key)}} return out if rows else {**out, "code": "request_unknown", "reason": _INVENTORY_UNKNOWN} accounts = _reason_markers(con, table) if accounts is None else accounts markers = _table_markers(accounts, table) refusals = [row["reason"] for states in markers for row in refused_reports(states) if row["table"] == table] if not refusals and table not in TABLES: for states in markers: saved = states.get(entity_key("refused_" + table)) or {} if saved.get("state") == "refused": detail = _marker_detail(saved) refusals.append(plain_refusal_reason(detail.get("reason") or "", table=table, status=detail.get("amazonStatus") or "")) dated = (_freshness(con, table, markers, now) if lag and rows and table in LAG_DATE_COLUMNS else {}) if refusals: return {**out, "code": "refused_by_amazon", "reason": _reason_line(refusals[0]), **dated} if rows: return {**out, **dated} if enabled is not None and enabled.get(amazon_schema.DATABASE_PREFIX + table) is False: return {**out, "code": "not_requested", "reason": "This Amazon database is turned off in Amazon connection settings, so turn it on to request its rows."} if table in HISTORY_TABLES: return {**out, "code": "first_load_pending", "reason": "History starts with the next sync of the source table."} if table in DERIVED_TABLES: return {**out, "code": "no_source_rows" if rows == 0 else "first_load_pending", "reason": "This table is built from other saved Amazon data that holds no matching rows yet, " "so sync Amazon and check again."} landed = [states.get(entity_key(table)) or {} for states in markers] phases = {marker.get("state") for marker in landed} if "throttled" in phases: return {**out, "code": "first_load_pending", "reason": "Amazon is limiting requests for this table, so sync resumes after Amazon's waiting period."} if "failed" in phases: return {**out, "code": "first_load_pending", "reason": "The last Amazon sync of this table did not finish, so retry Amazon sync."} measured_zero = rows == 0 or (rows is None and all(marker.get("rows") == 0 for marker in landed)) if landed and phases == {"done"} and measured_zero: spans = [] for states in markers: proof = _marker_detail(states.get(entity_key("reconcile_" + table))) span = " to ".join(dict.fromkeys(str(proof[key])[:10] for key in ("start", "end") if proof.get(key))) if span and span not in spans: spans.append(span) period = spans[0] if len(spans) == 1 else "the loaded dates" if spans else "the last requested period" return {**out, "code": "no_source_rows", "reason": f"Amazon returned no rows for {period}, so check seller activity in Seller Central and retry Amazon sync."} return {**out, "code": "first_load_pending", "reason": loading_reason(table)} def loading_reason(table): """The generic first-load sentence. It matches `amazon_relational.missing_source`'s sync sentence.""" return (f"Amazon {str(table).replace('_', ' ')} is still loading from Amazon, " "so ask again after the next Amazon sync finishes.") def empty_reason_census(tenant_key, rt=None, *, con=None, counts=None, now=None): """D-2643: every runtime-declared table and refused dataset with its count or reason. Read only. None when this tenant's mirror is absent or never held Amazon data. Opens no writer and issues no DDL. `counts` holds row counts the caller already measured, such as a publication manifest's. D-2642: a dated table with rows also answers its newest row date, cadence and any lag line. """ enablement = read_database_enablement(rt) if rt is not None else {} enabled = enablement.get("databases") if enablement.get("available") else None measured = {str(table): rows for table, rows in (counts or {}).items() if type(rows) is int} def build(cur): accounts = _reason_markers(cur) held = {row[0] for row in cur.execute( "SELECT DISTINCT table_name FROM information_schema.columns WHERE table_catalog=current_database() " "AND table_schema=current_schema() AND starts_with(table_name, ?)", [PREFIX]).fetchall()} if not held and not any(key.startswith(PREFIX) for states in accounts.values() for key in states): return None # This tenant's mirror has never held Amazon data. return [table_empty_status(cur, table, enabled=enabled, accounts=accounts, lag=True, now=now, **({"rows": measured[table]} if table in measured else {})) for table in empty_reason_tables(accounts=accounts)] if con is not None: return build(con) path = _tenant_mirror_path(tenant_key, rt) if not path.is_file(): return None if path.resolve() == Path(datastore.DB_PATH).resolve(): cur = datastore.ro_cursor() try: return build(cur) finally: cur.close() import duckdb with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur: return build(cur) def first_load_table_status(tasks, finished, failed, counts, *, table_status=None): """D-2643, D-2642, D-2649: a row count or a named reason per table, and why orders are behind.""" out = {} for table in dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES): own = [t for t in tasks if t["table"] == table] left = [t for t in own if t["id"] not in finished and not t.get("landed")] rows = int((counts or {}).get(table) or 0) row = {"rows": rows, "remaining": len(left)} reason = next((failed[t["id"]] for t in own if t["id"] in failed), None) if not rows: row["reason"] = (reason if reason else f"The first load is still loading this table. {len(left)} report windows remain." if left else "Amazon returned no rows for this seller in the loaded period." if own else "This table holds no rows and the first load owes it no windows.") row["code"] = (("refused_by_amazon" if is_refusal_reason(reason) else "first_load_pending") if reason else "first_load_pending" if left else "no_source_rows" if own else "not_requested") if table == "inventory": row.update(empty=True, code="first_load_pending", reason= "Inventory source proof is not available in this job record, so retry Amazon sync to refresh its status.") if table in BEHIND_TABLES and left: done = [t for t in own if t["id"] in finished or t.get("landed")] oldest = min((t["start"] for t in done if t.get("start")), default=None) row["behind"] = (f"The first load is still loading this table. {len(left)} of {len(own)} report " "windows remain" + (f", and history before {oldest[:10]} is not loaded yet." if oldest else ", starting with the most recent month.")) if table_status and table in table_status: row.update(table_status[table]) if row.get("rows") is not None and not row.get("empty"): row.pop("reason", None) row.pop("code", None) out[table] = row return out def first_load_job_summary(docs, *, table_status=None): """The latest generation record and the cursor's remaining count, read from the store.""" try: record = _doc(docs, FIRST_LOAD_GENERATIONS_KEY) except Exception: # noqa: BLE001 return None generations = record.get("generations") if isinstance(record.get("generations"), list) else [] if not generations: return None last = generations[-1] return {"generation": last.get("generation"), "state": last.get("state"), "remaining": record.get("remaining", last.get("remainingAfter")), "tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"), "operation": last.get("operation"), "resume_at": last.get("resume_at"), "sqp": last.get("sqp"), "tables": {**(last.get("tables") or {}), **(table_status or {})}} def _checkpoint_first_load_sqp(tenant_key, rt, cohort, record, save): """Checkpoint derived SQP progress after publication without making it another completion authority.""" if cohort is None or not read_database_enablement(rt)["databases"].get(amazon_schema.DATABASE_PREFIX + SQP_TABLE): return con, _path = _connect_tenant_mirror(tenant_key, rt) try: record["sqp"] = _sqp_first_load_status(con, cohort) finally: con.close() save(lambda doc: doc.update(sqp=record["sqp"])) def _record_generation(docs, row): def add(doc): rows = [r for r in (doc.get("generations") or []) if r.get("generation") != row["generation"]] doc["generations"] = (rows + [row])[-FIRST_LOAD_GENERATION_HISTORY:] doc.update(remaining=row.get("remainingAfter", row.get("remainingBefore")), updatedAt=_iso(_utc())) return doc docs.update(FIRST_LOAD_GENERATIONS_KEY, add, flush="sync") def _record_brand_week_write(record, task, wall_seconds, cpu_seconds, rows): """Accumulate measured writes while retaining each week's own CPU and wall time.""" timing = record.setdefault("brandAnalytics", {"wallSeconds": 0, "cpuSeconds": 0, "periods": []}) week = {"account": task["account"], "_period_start": task["start"][:10], "_period_end": task["end"][:10], "wallSeconds": round(wall_seconds, 3), "cpuSeconds": round(cpu_seconds, 3)} timing["periods"].append(week) for field in ("wallSeconds", "cpuSeconds"): timing[field] = round(sum(item[field] for item in timing["periods"]), 3) timing["rows"] = rows def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_seconds=5 * 3600, log=print, docs=None, plan=None, client=None, write=None, delegate=None, counts=None, holder=None, now=time.time, sleep=time.sleep, poll_seconds=None, job_id=None): """S82-T14: one chain generation of the Amazon first load, resumable from the store cursor. It requests reports across tables at the ledger's pace, polls the open ones together, writes each finished window once and publishes per table. -> the generation record it stored. """ key = _tenant_key(tenant_key) rt = rt if rt is not None else _runtime_for(key) docs = docs if docs is not None else rt cursor_docs = docs holder = holder or f"amazon-first-load-{uuid.uuid4().hex[:8]}" poll_seconds = FIRST_LOAD_POLL_SECONDS if poll_seconds is None else poll_seconds started = now() deadline = started + float(budget_seconds) request_until = deadline - min(FIRST_LOAD_RESERVE_SECONDS, float(budget_seconds) / 4) record = {"generation": int(generation), "jobId": job_id, "start": _epoch_iso(started), "end": None, "remainingBefore": None, "remainingAfter": None, "tablesPublished": [], "windowsWritten": 0, "reportsRequested": 0, "downloads": 0, "throttled": [], "operation": None, "resume_at": None, "state": "running", "holder": holder} status_before = read_connector_status(docs) if not acquire_amazon_lease(docs, holder, now=started): record.update(state="waiting", end=_epoch_iso(now()), reason="Another first-load generation holds the Amazon writer lease.") return record # A long write or the other tables' pass outlasts the lease, so a real-clock run renews it each minute. beat, beater = threading.Event(), None if now is time.time: def renew(): while not beat.wait(60): try: acquire_amazon_lease(docs, holder) except Exception: # noqa: BLE001 pass beater = threading.Thread(target=renew, daemon=True, name="amazon-first-load-lease") beater.start() def save(mutate): def apply(doc): mutate(doc) doc["updatedAt"] = _epoch_iso(now()) return doc return cursor_docs.update(FIRST_LOAD_CURSOR_KEY, apply, flush="sync") try: if plan is None: end, tasks, fields_by = first_load_job_plan(key, rt, log=log) if tasks is None: # No cohort end yet: one bounded pass freezes it, and the next generation plans from it. (delegate or _sync_delegate(key, rt, log))(None, FIRST_LOAD_REPORTS_PER_PASS) end, tasks, fields_by = first_load_job_plan(key, rt, log=log) plan_fn = lambda: first_load_job_plan(key, rt, log=log)[1] or [] # noqa: E731 else: end, tasks, fields_by = plan() plan_fn = lambda: plan()[1] or [] # noqa: E731 tasks = tasks or [] client = client or _JobClient(key, fields_by, log) write = write or _sync_writer(key, rt, log) delegate = delegate or _sync_delegate(key, rt, log) counts = counts or _mirror_row_counts(key, rt) cursor = _doc(cursor_docs, FIRST_LOAD_CURSOR_KEY) if cursor.get("end") and end and cursor["end"] != _iso(end): # A new cohort end means a new first load; the old cursor's windows describe another plan. cursor = {} cohort = end or (_utc(cursor["end"]) if cursor.get("end") else None) finished = dict(cursor.get("finished") or {}) opened = dict(cursor.get("open") or {}) throttled = dict(cursor.get("throttled") or {}) failed = dict(cursor.get("failed") or {}) downloads = dict(cursor.get("downloads") or {}) save(lambda doc: doc.update(tenant=key, end=_iso(cohort) if cohort else None, backfillDays=DEFAULT_BACKFILL_DAYS, generation=int(generation), finished=finished, open=opened, throttled=throttled, failed=failed, downloads=downloads)) def owed(task): return task["id"] not in finished and not task.get("landed") and task["id"] not in failed record["remainingBefore"] = sum(owed(t) for t in tasks) mine = _pace_weekly_history([t for t in tasks if t.get("prefetch")], owed) # A write that finishes no window waits for the next generation, including unserved documents. stalled = set() own_tables = sorted({t["table"] for t in mine if owed(t)}) ready = {} next_token = started def note_throttle(exc, key_, table): row = {**exc.as_resume(), "table": table} throttled[key_] = row record["throttled"].append(row) record["operation"], record["resume_at"] = row.get("operation"), row.get("resume_at") log(f"{table}: Amazon throttled {row.get('operation') or 'a call'}; it resumes at {row.get('resume_at')}") def held(key_, stamp): row = throttled.get(key_) if not row: return False try: return _utc(row["resume_at"]).timestamp() > stamp except (KeyError, TypeError, ValueError): return False cx = _client() while True: stamp = now() if stamp >= deadline: break if not acquire_amazon_lease(docs, holder, now=stamp): record["reason"] = "Another holder took the Amazon writer lease." break pending = [t for t in mine if owed(t) and t["id"] not in stalled] # Request: one report per identity, round robin across tables, while the ledger has tokens. if stamp < request_until and stamp >= next_token: queue, seen, by_table = [], set(opened), {} blocked = set() for task in pending: if task["table"] in blocked: continue if held(task["key"], stamp): # A later window cannot be written before this one, so it is not downloaded yet. blocked.add(task["table"]) continue if task["key"] in seen or task["key"] in ready: continue seen.add(task["key"]) by_table.setdefault(task["table"], []).append(task) while any(by_table.values()): for table in list(by_table): if by_table[table]: queue.append(by_table[table].pop(0)) for task in queue: granted, wait = take_report_token(docs, task["seller"], now=stamp) if not granted: next_token = stamp + wait break try: rid = client.create(task) except cx.AmazonThrottled as exc: note_throttle(exc, task["key"], task["table"]) break except cx.AmazonError as exc: failed[task["id"]] = plain_refusal_reason(str(exc), table=task["table"]) save(lambda doc: doc.update(failed=failed)) continue throttled.pop(task["key"], None) opened[task["key"]] = {"reportId": str(rid), "createdAt": stamp, "table": task["table"], "task": task["id"], "generation": int(generation)} record["reportsRequested"] += 1 # Durable before the next request: a killed generation's successor polls this reportId. save(lambda doc: doc.update(open=opened, throttled=throttled)) # Poll every open report together. for key_, row in list(opened.items()): if key_ in ready or held(key_, stamp): continue keyed = [t for t in mine if t["key"] == key_ and owed(t)] if keyed and all(t["id"] in stalled for t in keyed): continue # Its reportId stays in the cursor for the next generation. task = next((t for t in mine if t["key"] == key_ and owed(t)), None) if task is None or stamp - float(row.get("createdAt") or 0) >= REPORT_REUSE_SECONDS: opened.pop(key_) save(lambda doc: doc.update(open=opened)) continue try: got = client.poll(task, row["reportId"]) except cx.AmazonThrottled as exc: note_throttle(exc, key_, task["table"]) save(lambda doc: doc.update(throttled=throttled)) continue except cx.AmazonError as exc: for t in mine: if t["key"] == key_: failed[t["id"]] = plain_refusal_reason(str(exc), table=t["table"]) opened.pop(key_) save(lambda doc: doc.update(open=opened, failed=failed)) continue if got is None: continue throttled.pop(key_, None) ready[key_] = got downloads[key_] = int(downloads.get(key_) or 0) + 1 record["downloads"] += 1 save(lambda doc: doc.update(downloads=downloads, throttled=throttled)) # Write: each table's owed windows in plan order, the ready prefix only, so floors stay contiguous. groups = {} for task in pending: groups.setdefault(task["account"], {}).setdefault(task["table"], []).append(task) for account, by_table in groups.items(): for table, owed_tasks in by_table.items(): if not owed(owed_tasks[0]): continue prefix = [] for task in owed_tasks: if task["key"] not in ready: break prefix.append(task) if not prefix: continue if table == "brand_analytics": prefix = prefix[:1] # Each week needs its own measured write interval. sibling = SHARED_REPORT_SIBLINGS.get(table) pair = [t for t in by_table.get(sibling, []) if owed(t)][:len(prefix)] if sibling else [] units = [table] + ([sibling] if pair and [t["key"] for t in pair] == [t["key"] for t in prefix] else []) documents = {t["key"]: ready[t["key"]] for t in prefix} t0, c0 = time.time(), time.process_time() try: result = write(units, documents) except Exception as exc: # noqa: BLE001 log(f"{table}: the first-load write failed ({type(exc).__name__}); " "its windows stay owed and the next generation downloads them again.") for t in prefix: stalled.add(t["id"]) ready.pop(t["key"], None) continue wall_seconds, cpu_seconds = time.time() - t0, time.process_time() - c0 stamp_done = _epoch_iso(now()) for unit in units: row = result["tables"].get(unit) or {} ok = result["published"] and (row.get("state", "done") not in { "failed", "refused", "throttled", "pending", "held", "deferred", "disabled"} or unit == "brand_analytics" and row.get("state") == "deferred" and row.get("current") and row.get("landedWeeks", 0) > 0) if not ok: continue for t in (prefix if unit == table else pair): if t["key"] in result["served"]: if unit == "brand_analytics": _record_brand_week_write(record, t, wall_seconds, cpu_seconds, row.get("rows")) finished[t["id"]] = {"at": stamp_done, "generation": int(generation), "reportId": (opened.get(t["key"]) or {}).get("reportId")} record["windowsWritten"] += 1 if unit not in record["tablesPublished"]: record["tablesPublished"].append(unit) for t in prefix + pair: if t["id"] not in finished: stalled.add(t["id"]) ready.pop(t["key"], None) if t["id"] in finished or t["key"] not in result["served"]: ready.pop(t["key"], None) if not any(owed(o) and o["key"] == t["key"] for o in mine): opened.pop(t["key"], None) save(lambda doc: doc.update(finished=finished, open=opened)) pending = [t for t in mine if owed(t) and t["id"] not in stalled] if not pending: break waiting_on = [t for t in pending if t["key"] in opened or t["key"] in ready] requestable = [t for t in pending if t["key"] not in opened and not held(t["key"], now())] if not waiting_on and not requestable: resumes = [_utc(throttled[t["key"]]["resume_at"]).timestamp() for t in pending if held(t["key"], now())] soon = min(resumes, default=None) if soon is None or soon >= request_until: break # Only throttled work remains this generation; the next one resumes it. sleep(max(0.0, soon - now())) continue if not waiting_on and now() >= request_until: break sleep(poll_seconds if waiting_on else max(0.0, min(poll_seconds, next_token - now()))) record["otherTablesPasses"] = 0 while now() < request_until: # After the job's own reports. Each pass publishes; another follows while request time remains, # sized to that time at Amazon's one report a minute, so SQP history moves at Amazon's rate. budget = max(FIRST_LOAD_REPORTS_PER_PASS, min(FIRST_LOAD_DELEGATE_REPORTS, int((request_until - now()) // 60))) try: result = delegate(own_tables, budget) record["otherTablesPass"] = "done" except Exception as exc: # noqa: BLE001 record["otherTablesPass"] = f"failed ({type(exc).__name__})" log(f"Amazon first-load pass for the other tables failed ({type(exc).__name__}); " "the next generation runs it again.") break record["otherTablesPasses"] += 1 result = result if isinstance(result, dict) else {} if plan is None: _checkpoint_first_load_sqp(key, rt, cohort, record, save) if result.get("state") != "partial" or not result.get("landedTasks"): break try: after = plan_fn() except Exception: # noqa: BLE001 after = tasks record["remainingAfter"] = sum(t["id"] not in finished and not t.get("landed") and t["id"] not in failed for t in after) record["brandAnalyticsPeriods"] = sorted({t["start"][:10] for t in tasks + after if t["table"] == "brand_analytics" and t.get("start") and (t.get("landed") or t["id"] in finished)}) try: row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES))) except Exception: # noqa: BLE001 row_counts = {} table_status = None if plan is None: _checkpoint_first_load_sqp(key, rt, cohort, record, save) con, _path = _connect_tenant_mirror(key, rt) try: table_status = {"inventory": inventory_table_status(con)} finally: con.close() record["tables"] = first_load_table_status(after, finished, failed, row_counts, table_status=table_status) record["state"] = "done" if not record["remainingAfter"] else "partial" status_after = read_connector_status(docs) record["odooLastGoodSyncAt"] = {"start": _status_stamp(status_before, "odoo"), "end": _status_stamp(status_after, "odoo")} return record except BaseException as exc: record.update(state="failed" if isinstance(exc, Exception) else "killed", reason=type(exc).__name__) raise finally: record["end"] = _epoch_iso(now()) try: _record_generation(docs, dict(record)) except Exception as exc: # noqa: BLE001 log(f"Amazon first-load generation record could not be saved ({type(exc).__name__})") beat.set() if beater is not None: beater.join(30) # A renewal in flight lands before the release, never after it. try: release_amazon_lease(docs, holder) except Exception: # noqa: BLE001 pass # ----------------------------------------------------------------------------- history floors #: Amazon's own words for "this period is older than I keep". SQP's text also covers an unprocessed #: recent month, so `_close_history_floor` accepts it only below newer landed history. _AMAZON_FLOOR_TEXT = re.compile( r"outside the available data range" r"|start\s*date[^.]{0,80}?(?:cannot be before|must be no more than|is too old)", re.I) #: A refused period ending inside this many days is treated as unfinished, never as a floor. _FLOOR_MIN_AGE = timedelta(days=45) def _sqp_depth_start(value, period): """Admit a complete MONTH or Sunday-based WEEK boundary.""" if period not in _client().SQP_PERIOD: raise ValueError("SQP depth supports MONTH and WEEK only") if isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): value += "T00:00:00Z" day = _utc(value) boundary = _grain_floor(day, period.lower()) if day != boundary: raise ValueError("SQP depth needs the first day of a complete period") return day def _sqp_depth_shift(start, period, count): if period == "WEEK": return start + timedelta(weeks=count) return _months_before(start, -count) def _sqp_depth_eligible(row, now): """Only Amazon's old out-of-range refusal can bound served history.""" return (row["state"] == "out_of_range" and row.get("amazonStatus") in ("FATAL", 400) and bool(_AMAZON_FLOOR_TEXT.search(row.get("amazon") or "")) and _date_utc(row["end"]) + timedelta(days=1) <= now - _FLOOR_MIN_AGE) def _sqp_depth_observation(request, asin, period, start, now): cx = _client() end = _sqp_depth_shift(start, period, 1) row = {"period": period, "start": start.date().isoformat(), "end": (end - timedelta(days=1)).date().isoformat(), "reportId": None, "state": "empty", "hasRows": False, "floorEligible": False} try: text, meta = request(cx.REPORT_TYPES[SQP_TABLE]["report"], start=_iso(start), end=_iso(end - timedelta(seconds=1)), options={"reportPeriod": period, "asin": asin}) except cx.AmazonError as exc: words, status = str(getattr(exc, "vendor_reason", "") or exc), getattr(exc, "status", None) if status not in ("FATAL", 400) or not _AMAZON_FLOOR_TEXT.search(words): raise row.update(state="out_of_range", amazonStatus=status, amazon=words, reportId=getattr(exc, "report_id", None)) row["floorEligible"] = _sqp_depth_eligible(row, now) return row _columns, _rows, _totals, count = _sqp_document_rows( text, cx.REPORT_TYPES[SQP_TABLE], [asin], row, meta) row.update(state="served" if count else "empty", hasRows=bool(count), reportId=meta.get("reportId"), reportDocumentId=meta.get("reportDocumentId"), amazonStatus="DONE") return row def _sqp_depth_linear(rows, start, period): refusals = 0 while start.date().isoformat() in rows: row = rows[start.date().isoformat()] refusals = refusals + 1 if row["floorEligible"] else 0 if refusals >= 2: return None start = _sqp_depth_shift(start, period, -1) return start def _sqp_depth_next(rows, newest, period): served = sorted(day for day, row in rows.items() if row["state"] == "served") if not served: return _sqp_depth_linear(rows, newest, period) first = _sqp_depth_start(served[0], period) refused = sorted(day for day, row in rows.items() if row["floorEligible"] and day < served[0]) if refused: low = _sqp_depth_start(refused[-1], period) distance = ((first - low).days // 7 if period == "WEEK" else (first.year - low.year) * 12 + first.month - low.month) if distance > 1: middle = _sqp_depth_shift(low, period, distance // 2) if middle.date().isoformat() not in rows: return middle return _sqp_depth_linear(rows, _sqp_depth_shift(first, period, -1), period) def _sqp_depth_result(rows, period, newest, reports, *, done=False): served = sorted(day for day, row in rows.items() if row["state"] == "served") first = _sqp_depth_start(served[0], period) if served else None bounded = bool(first) and all(rows.get(_sqp_depth_shift(first, period, -n).date().isoformat(), {}).get( "floorEligible", False) for n in (1, 2)) return {"period": period, "newest": newest.date().isoformat(), "earliestServed": served[0] if served else None, "floor": served[0] if done and bounded else None, "state": ("measured" if bounded else "unproven") if done else "running", "reportsRequested": reports, "observations": [rows[day] for day in sorted(rows, reverse=True)]} def sqp_depth_walk(request, asin, period, newest, known=(), *, now=None, checkpoint=None): """Find one ASIN's served boundary, retaining empty periods and requiring two adjacent old refusals.""" if not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin): raise ValueError("SQP depth needs one ASIN") newest, now = _sqp_depth_start(newest, period), _utc(now) if _sqp_depth_shift(newest, period, 1) > now: raise ValueError("SQP depth needs a completed newest period") rows = {} for saved in known: row = dict(saved) start = _sqp_depth_start(row["start"], period) if row.get("period") != period or start > newest or row.get("state") not in ( "served", "empty", "out_of_range"): raise ValueError("SQP depth known observations must belong to this period and window") end = (_sqp_depth_shift(start, period, 1) - timedelta(days=1)).date().isoformat() if row.get("end") != end or (row["state"] == "served" and row.get("hasRows") is not True): raise ValueError("SQP depth known observations need matching windows and row evidence") row["floorEligible"] = _sqp_depth_eligible(row, now) rows[row["start"]] = row reports = 0 while (start := _sqp_depth_next(rows, newest, period)) is not None: row = _sqp_depth_observation(request, asin, period, start, now) rows[row["start"]] = row reports += 1 if checkpoint: checkpoint(_sqp_depth_result(rows, period, newest, reports)) result = _sqp_depth_result(rows, period, newest, reports, done=True) if checkpoint: checkpoint(result) return result def _dated_table(table): """A table whose first load plans dated history windows.""" spec = _client().REPORT_TYPES.get(table) return bool(spec) and table != "settlements" and not spec.get("snapshot") def _history_grain(spec): return "month" if spec.get("monthly") else "week" if spec.get("weekly") else "day" def _grain_floor(value, grain): day = _utc(value).replace(hour=0, minute=0, second=0, microsecond=0) return day.replace(day=1) if grain == "month" else _week_start(day) if grain == "week" else day def _grain_ceil(value, grain): value = _utc(value) low = _grain_floor(value, grain) if low == value: return low if grain == "month": return (low.replace(day=28) + timedelta(days=4)).replace(day=1) return low + timedelta(days=7 if grain == "week" else 1) def _date_utc(text): return datetime.fromisoformat(str(text)[:10]).replace(tzinfo=timezone.utc) def _months_before(value, months): index = value.year * 12 + value.month - 1 - months return value.replace(year=index // 12, month=index % 12 + 1) def read_seller_first_month(con): """-> (first month with orders or None, marker detail or None). A measured seller without orders has a detail.""" state = _read_state(con, entity_key(SELLER_FIRST_MONTH_KEY)) try: detail = json.loads(state["detail"]) if state and state.get("detail") else None if not isinstance(detail, dict): return None, None return (_date_utc(detail["month"]) if detail.get("month") else None), detail except (ValueError, TypeError, KeyError): return None, None def measure_seller_first_month(con, fields, end, *, log=print): """Walk getOrderMetrics back in 24-month windows from `end`; the earliest month with orders wins. Stops after two consecutive windows without orders, or at SELLER_FLOOR_EARLIEST. Records the marker. """ cx = _client() w1, empty, found = _grain_floor(end, "month"), 0, None while w1 > SELLER_FLOOR_EARLIEST and empty < 2: w0 = max(SELLER_FLOOR_EARLIEST, _months_before(w1, cx.ORDER_MONTHS_MAX)) months = [month for month, count in cx.order_months(fields, w0, w1) if count > 0] if months: found, empty = min(months), 0 else: empty += 1 w1 = w0 detail = {"month": found, "source": "getOrderMetrics", "measured_at": _iso(datetime.now(timezone.utc))} _record_state(con, entity_key(SELLER_FIRST_MONTH_KEY), "done", detail=json.dumps(detail)) con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(HISTORY_FLOOR_CAUSE_KEY)]) log(f"Amazon seller first order month: {found or 'none found'}") return _date_utc(found) if found else None def _ensure_seller_first_month(con, fields, end, *, log=print): """Measure once per seller; a failure records its cause and retries on a later UTC day, never every pass.""" if os.environ.get("AIOS_AMAZON_SELLER_FLOOR_PROBE") == "0" or not fields: return if _read_state(con, entity_key(SELLER_FIRST_MONTH_KEY)): return cause = _read_state(con, entity_key(HISTORY_FLOOR_CAUSE_KEY)) if cause and str(cause.get("updated_at") or "")[:10] == datetime.now(timezone.utc).date().isoformat(): return if _client().prefetch_active(): return # A job writer pass serves only prefetched documents; the delegate pass measures. try: measure_seller_first_month(con, fields, end, log=log) except Exception as exc: # noqa: BLE001 reason = (f"The seller's first order month could not be read from Amazon ({type(exc).__name__}). " f"History reaches the {DEFAULT_BACKFILL_DAYS} day fallback; the reading retries tomorrow.") _record_state(con, entity_key(HISTORY_FLOOR_CAUSE_KEY), "failed", detail=json.dumps({"cause": reason, "error": str(exc)[:120]})) log(reason) def _seller_floor(con, end): """-> (floor, cause). The measured first month, else end minus DEFAULT_BACKFILL_DAYS with its cause.""" month, detail = read_seller_first_month(con) if month is not None: return month, None fallback = _utc(end) - timedelta(days=DEFAULT_BACKFILL_DAYS) if detail is not None: return fallback, (f"Amazon reports no orders for this seller, so history reaches the " f"{DEFAULT_BACKFILL_DAYS} day fallback.") failed = _read_state(con, entity_key(HISTORY_FLOOR_CAUSE_KEY)) try: reason = json.loads(failed["detail"]).get("cause") if failed and failed.get("detail") else None except (ValueError, TypeError, AttributeError): reason = None return fallback, reason or (f"The seller's first order month is not measured yet, so history reaches the " f"{DEFAULT_BACKFILL_DAYS} day fallback.") def _history_limit_key(table, period=None): """Keep the legacy MONTH key and isolate every other period's observed history floor.""" suffix = "" if period in (None, "MONTH") else "_" + period return entity_key(f"limit_{table}_history{suffix}") def _observed_history_limit(con, table, period=None): """-> (floor, reason) Amazon showed by refusing an older window, or None.""" state = _read_state(con, _history_limit_key(table, period)) try: detail = json.loads(state["detail"]) if state and state.get("detail") else {} if detail.get("floor"): return _utc(detail["floor"]), str(detail.get("reason") or "") except (ValueError, TypeError, AttributeError): pass if period not in (None, "MONTH"): return None legacy = _read_state(con, entity_key("limit_" + table)) try: reason = json.loads(legacy["detail"]).get("reason", "") if legacy and legacy.get("detail") else "" except (ValueError, TypeError, AttributeError): reason = "" floor = read_backfill_floor(con, table) if isinstance(reason, str) and reason.startswith("Amazon keeps this report back to ") and floor is not None: return floor, reason return None def _history_rule(spec, period=None): """A period-specific rule replaces the table default without inheriting another period's floor.""" rule = spec.get("history") or {} return rule.get("periods", {}).get(period, rule) def history_bound(con, table, end, *, now=None, period=None): """-> (floor, cause): the oldest history start for `table`, and why when Amazon or the fallback sets it. max(seller floor, the table's `history` rule, Amazon's observed limit), on the table's window grain. A `from` rule or `seller_floor: False` replaces the seller floor (brand-level data predates it). `end` is the frozen cohort end (seller floor); `now` is the clock a rolling Amazon rule moves with. """ spec = _client().REPORT_TYPES.get(table) or {} grain = "week" if period == "WEEK" else _history_grain(spec) now = _utc(now) if now is not None else datetime.now(timezone.utc) rule = _history_rule(spec, period) if rule.get("from") or rule.get("seller_floor") is False: # Brand-level data (Brand Analytics) predates the seller's first order, so the seller month # does not bound it: a `from` rule is the floor, else history walks back until Amazon refuses. floor = _grain_floor(BRAND_FLOOR_EARLIEST, grain) cause = ("This data is brand level, so history walks back until Amazon refuses an older period or " f"returns {BRAND_EMPTY_STOP} empty periods in a row." if not rule.get("from") else None) else: floor, cause = _seller_floor(con, end) if cause is None: floor = _grain_floor(floor, grain) bound = None if rule.get("rolling_days"): cutoff = now - timedelta(days=float(rule["rolling_days"])) + ROLLING_FLOOR_MARGIN # An end edge admits the oldest window whose end is inside the rule; a start edge its start. bound = _grain_floor(cutoff, grain) if rule.get("edge") == "end" else _grain_ceil(cutoff, grain) rule_cause = (f"Amazon serves this data for the last {int(rule['rolling_days'])} days only, " f"so history starts {bound.date().isoformat()}.") elif rule.get("from"): bound = _grain_ceil(_date_utc(rule["from"]), grain) rule_cause = f"Amazon serves this data back to {bound.date().isoformat()} only." if rule.get("provisional"): rule_cause = (f"Weekly history starts at the provisional planning bound {bound.date().isoformat()}. " "The earliest served week is unproved; an older range refusal sets the observed floor.") if bound is not None and bound > floor: floor, cause = bound, rule_cause observed = _observed_history_limit(con, table, period) if observed and observed[0] > floor: floor, cause = observed[0], observed[1] or ( f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.") if (spec.get("weekly") and not WEEKLY_HISTORY_PLANNED and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") != "1"): latest = _week_start(_utc(end)) - timedelta(days=7) if latest > floor: older = f" Amazon serves weeks back to {floor.date().isoformat()}." if bound is not None else "" floor, cause = latest, ("The latest completed week loads first; older weeks are not loaded yet, " "because weekly history requires AIOS_AMAZON_WEEKLY_HISTORY=1." + older) return floor, cause def history_floor(con, table, end, *, now=None, period=None): """The oldest history start `first_load_plan` and `needs_first_load` use for `table`.""" return history_bound(con, table, end, now=now, period=period)[0] def history_starts(con, tables, end, *, now=None): """{table: history_floor} for each dated table; the one dict every first-load planner passes.""" starts = {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)} if SQP_TABLE in starts and "WEEK" in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()): starts[SQP_TABLE, "WEEK"] = history_floor(con, SQP_TABLE, end, now=now, period="WEEK") return starts def history_status(con, tables, end): """{table: {historyFloor, cause?}} for status, beside `backfillDays`. Screen safe, no vendor text.""" out = {} for table in tables: if not _dated_table(table): continue floor, cause = history_bound(con, table, end) row = {"historyFloor": _iso(floor)} if cause: row["cause"] = " ".join(cause.replace("\u2014", ". ").replace("\u2013", " to ").split())[:300] out[table] = row return out def _history_reopened(con, table, cohort): """A loaded table whose history floor moved earlier than its landed floor: only its history is owed.""" if not _dated_table(table): return False state = _read_state(con, entity_key(table)) or {} if state.get("state") != "done" or read_watermark(con, table) is None: return False bound = history_floor(con, table, cohort) if table == SQP_TABLE: return bool(_sqp_catalogue(con)) and not _sqp_history_done(con, cohort) floor = read_backfill_floor(con, table) return floor is not None and floor > bound def _brand_level(table): return (_client().REPORT_TYPES.get(table) or {}).get("history", {}).get("seller_floor") is False def _note_history_window(con, table, w0, count): """Count a brand-level table's landed-empty windows older than its oldest window with rows. -> the stop reason once BRAND_EMPTY_STOP empty windows follow it; the floor is then that oldest window. """ if w0 is None or not _brand_level(table): return None key = entity_key(f"limit_{table}_empty") state = _read_state(con, key) try: run = json.loads(state["detail"]) if state and state.get("detail") else {} except (ValueError, TypeError): run = {} rows_at = _utc(run["oldestRows"]) if run.get("oldestRows") else None if count: if rows_at is None or w0 < rows_at: run = {"oldestRows": _iso(w0), "empty": 0} elif rows_at is not None and w0 < rows_at: run["empty"] = int(run.get("empty") or 0) + 1 _record_state(con, key, "done", detail=json.dumps(run)) if rows_at is None or count or run.get("empty", 0) < BRAND_EMPTY_STOP: return None reason = (f"Amazon keeps this report back to {rows_at.date().isoformat()}; it returned no data for " f"{BRAND_EMPTY_STOP} periods in a row before that, so older history is not requested.") _record_state(con, _history_limit_key(table), "done", detail=json.dumps({ "floor": _iso(rows_at), "reason": reason, "amazon": f"{BRAND_EMPTY_STOP} consecutive DONE reports with no rows", "window": [_iso(w0), _iso(rows_at)]})) return reason def _owed_window(table, w0, w1, floor): """A history window still owed below the landed floor: whole, clipped at the floor, or None when landed. Clipping keeps history contiguous when a moved floor shifts the day-window boundaries. """ if w1 is None or floor is None or w1 <= floor: return w0, w1 if (w0 is not None and w0 < floor and table != SQP_TABLE and not _periodic(_client().REPORT_TYPES.get(table))): return w0, floor return None def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=False, floor=None): """Pure request decision for one table: start, end, report, incremental and cause. `floor` is the table's history floor for a first full load (`history_floor`). """ end = _utc(now) days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days try: mark = _utc(mark) if mark else None except (ValueError, TypeError, AttributeError, OverflowError): mark = None if since is not None: start, incremental = _utc(since), False cause = f"caller named the window starting {_iso(start)}" elif mark is not None: start, incremental = mark, True cause = f"incremental window since watermark {_iso(mark)}" elif floor is not None: start, incremental = _utc(floor), False cause = f"first full load: history from {start.date().isoformat()}" else: start, incremental = end - timedelta(days=days), False cause = f"first full load: {days} days of backfill" spec = _client().REPORT_TYPES.get(table) or {} if incremental and not spec.get("snapshot") and not _periodic(spec): cadence = _client().CADENCE[table] lookback = cadence["intradayLookbackDays" if intraday else "lookbackDays"] start = min(start, end - timedelta(days=lookback)) cause += f"; re-read {lookback} days ({'intraday' if intraday else 'daily'})" rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report") rtype = rtype or spec.get("api") if table in ("economics", "order_metrics"): if incremental: start -= timedelta(days=3) start = start.replace(hour=0, minute=0, second=0, microsecond=0) end = end.replace(hour=0, minute=0, second=0, microsecond=0) cause += "; completed days, with a three-day restatement overlap on incremental sync" if table == "transactions": end -= timedelta(minutes=2) # Finances requires postedBefore at least two minutes ago. if spec.get("snapshot"): cause = "one current snapshot, without a data window" if spec.get("monthly"): start = start.replace(day=1, hour=0, minute=0, second=0, microsecond=0) end = end.replace(day=1, hour=0, minute=0, second=0, microsecond=0) cause += "; completed calendar months only" if spec.get("weekly"): end = _week_start(end) if incremental or since is not None: start = _week_start(start) cause += "; completed Sunday to Saturday weeks only" elif floor is not None: start = min(_week_start(floor), end - timedelta(days=7)) cause = f"first load: completed Sunday to Saturday weeks back to {start.date().isoformat()}" else: start = end - timedelta(days=7) cause = "first load: the latest completed Sunday to Saturday week only (R18)" if table == "settlements": # The watermark filters issued reports by creation time; it never requests one. rtype = spec.get("report_discover") return {"start": start, "end": end, "report": rtype, "incremental": incremental, "intraday": intraday, "cause": cause} def _lookback_intraday(con, table, now, *, slot_pass=False): """Use the daily window until this account has completed it, including after a failed attempt.""" day = _read_state(con, entity_key("lookback_day_" + table)) return bool(not slot_pass and day and day.get("state") == "done" and day.get("detail") == now.date().isoformat()) def _record_lookback_day(con, table, plan, now): """Only a completed incremental daily request earns this account's intraday window.""" if (plan["incremental"] and not plan.get("intraday") and plan["end"] >= now - timedelta(minutes=2) and not _lookback_intraday(con, table, now)): _record_state(con, entity_key("lookback_day_" + table), "done", detail=now.date().isoformat()) def _sync_event(kind, tenant, table, detail, rt, report, log): """Sync is nondestructive: preserve C2's loud failure, but continue the data pull.""" try: _c2_event(kind, tenant, "system", {"table": table, **detail}, rt=rt) except Exception as exc: # The emitter records the failure in the lifecycle ledger before raising. If that store # also fails, the report/log remains the fallback; never claim an event was delivered. message = (f"{table}: {kind} AUDIT UNAVAILABLE ({type(exc).__name__}); " "restore core.audit and the tenant lifecycle store; sync continues") report["audit_problems"].append(message) log(message) def _sync_failure_context(exc, table): """Keep unreadable-value locations without exception values, source text or frame locals.""" if not isinstance(exc, (KeyError, ValueError)): return None frame = exc.__traceback__ while frame and frame.tb_next: frame = frame.tb_next return {"classifier": "unreadable_value", "table": table, "file": os.path.basename(frame.tb_frame.f_code.co_filename) if frame else None, "line": frame.tb_lineno if frame else None, "function": frame.tb_frame.f_code.co_name if frame else None} def _sync_problem(exc, table, operation, tenant, rt, report, log): from harness.amazon_client import AmazonThrottled, AmazonOperationRefused, AmazonReportPending failure = _sync_failure_context(exc, table) if failure: location = (f"{failure['file']}:{failure['line']}:{failure['function']}" if failure["file"] else "raising frame unavailable") fix = "check the connector's expected response fields and value types, then retry sync" message = (f"{table}: {operation} REFUSED: {type(exc).__name__}: " f"unreadable_value at {location}; fix: {fix}") report["problems"].append(message) log(message) return {"state": "refused", "reason": f"Amazon could not read {table} at {location}. {fix}.", "diagnostic": message, "amazonStatus": "REFUSED", "error": "", "failure": failure} cause = f"{type(exc).__name__}: {exc}" if isinstance(exc, AmazonReportPending): message = (f"{table}: Amazon report is {exc.status}; its saved report ID will be polled on the next pass. " "A report still pending after 24 hours is requested again.") log(message) return {"state": "pending", "reason": "Amazon is preparing this report. Sync will check it again automatically.", "diagnostic": message, "amazonStatus": exc.status} if isinstance(exc, AmazonThrottled): retry = exc.retry_after state = "throttled" fix = (f"retry_after={retry if retry is not None else 'unspecified'}; " "retry after the vendor delay, widen report windows or reduce sync frequency") _sync_event("amazon.throttled", tenant, table, {"operation": operation, "retry_after": str(retry), "cause": "HTTP 429"}, rt, report, log) elif isinstance(exc, AmazonOperationRefused): state = "refused" fix = "use an operation in the approved read-only scope; check the connector operation mapping" elif re.search(r"\b(?:HTTP(?: status)?|status(?: code)?|answered)\s*[:=]?\s*403\b|\(403\)", str(exc), re.I): state = "refused" cx = _client() role = (cx.REPORT_TYPES.get(table) or {}).get("role") role = role or " / ".join((cx.OPERATIONS.get(operation) or {}).get("roles", ())) # S66-T42 (lead, observed live): Amazon also answers 403 for a report it does not offer # this seller (sold listings, RFQ bulk download) while the application HOLDS the role, so # the sentence says which case applies instead of sending an admin to re-grant a held role. held = all(r.strip() in cx.ROLES_TAKEN for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip()) held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I) fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in FEEDBACK_TABLES else "review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else "review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else f"needs the {role} role; this application already holds it, so Amazon does not " "offer this report to this seller account" if role and held else f"needs the {role} role; grant it and reauthorize the application before retrying") else: state = "refused" fix = "check the report and operation's required roles, credential and vendor response; retry" message = f"{table}: {operation} {state.upper()}: {cause}; fix: {fix}" message = message.replace("\u2014", "; ").replace("\u2013", " to ") report["problems"].append(message) log(message) status = str(getattr(exc, "status", "NOT_OFFERED" if "not allowed" in str(exc).lower() or re.search(r"\b(?:HTTP|answered)\s*403\b|\(403\)", str(exc), re.I) else "REFUSED")) return {"state": state, "reason": plain_refusal_reason(message, status=status, table=table, operation=operation), "diagnostic": message, "amazonStatus": status, "error": str(getattr(exc, "vendor_reason", ""))[:500]} def _pending_report_key(table, report_type, **kwargs): window = {key: kwargs.get(key) for key in ("start", "end", "options")} identity = json.dumps([table, report_type, window], sort_keys=True) return entity_key("pending_report_" + hashlib.sha256(identity.encode()).hexdigest()) def _save_throttled_resume(con, table, exc, plan, completed, *, first_load, since): """Keep the continuation beside the account's markers and leave the retry tick as an ISO deadline.""" continuation = {**exc.as_resume(), "plan": {**plan, "start": _iso(plan["start"]), "end": _iso(plan["end"])}, "completed": completed, "first_load": first_load, "since": _iso(_utc(since)) if since is not None else None} _record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation)) # A scheduled SQP backlog waits for Amazon's burst to refill, never less than SQP_RESUME_SECONDS. least = SQP_RESUME_SECONDS if table == SQP_TABLE and not first_load else 60.0 due = datetime.now(timezone.utc) + timedelta(seconds=max(float(least), float(exc.retry_after or 60))) _record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due)) return max(0, (due - datetime.now(timezone.utc)).total_seconds()) @contextmanager def _report_resume(con, table, report_type, **kwargs): """Resume the same report/window for up to 24 hours instead of paying createReport again.""" from harness.amazon_client import AmazonReportError, AmazonReportPending, AmazonThrottled window = {key: kwargs.get(key) for key in ("start", "end", "options")} key = _pending_report_key(table, report_type, **kwargs) state = _read_state(con, key) saved = json.loads(state["detail"]) if state and state.get("detail") else {} stamp = datetime.now(timezone.utc).timestamp() if saved and stamp - saved["createdAt"] >= 86400: con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) saved = {} if saved: kwargs["report_id"] = saved["reportId"] try: yield kwargs except AmazonReportPending as exc: _record_state(con, key, "pending", detail=json.dumps({ "reportId": exc.report_id, "createdAt": saved.get("createdAt", stamp), "window": window, "table": table, "reportType": report_type, "processingStatus": exc.status})) raise except AmazonThrottled as exc: if exc.report_id is not None: _record_state(con, key, "throttled", detail=json.dumps({ **exc.as_resume(), "reportId": str(exc.report_id), "createdAt": saved.get("createdAt", stamp), "window": window, "table": table, "reportType": report_type})) raise except AmazonReportError: con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) raise else: con.execute("DELETE FROM _sync_state WHERE entity=?", [key]) def _report_document(con, table, fields, report_type, **kwargs): from harness.amazon_client import AmazonError, AmazonReportPending, AmazonThrottled cx = _client() resume_kwargs = {} def remember(state, meta): if table == "inventory": detail = {key: meta.get(key) for key in ("reportId", "processingStatus", "reportDocumentId")} detail.update(reportType=report_type, start=kwargs.get("start"), end=kwargs.get("end")) if meta.get("reason"): detail["reason"] = meta["reason"] _record_state(con, entity_key(INVENTORY_REQUEST_KEY), state, detail=json.dumps(detail)) try: with _report_resume(con, table, report_type, **kwargs) as resume_kwargs: result = cx.report_document(fields, report_type, **resume_kwargs) except AmazonError as exc: state = ("pending" if isinstance(exc, AmazonReportPending) else "throttled" if isinstance(exc, AmazonThrottled) else "refused") status = str(exc.status) if getattr(exc, "status", None) is not None else None remember(state, {"reportId": getattr(exc, "report_id", None) or resume_kwargs.get("report_id"), "processingStatus": status, "reason": plain_refusal_reason(str(exc), status=status, table=table)}) raise remember("done", result[1]) return result def _newer_history_landed(con, table, end, period=None): """A window of `table` newer than `end` has landed, so a refusal at `end` is a floor, not an outage.""" if table == SQP_TABLE: windows = _week_windows if period == "WEEK" else _month_windows return any(any(not batch.get("refused") for batch in _sqp_month(con, w0, w1)["batches"].values()) for w0, w1 in windows(end, datetime.now(timezone.utc))) floor = read_backfill_floor(con, table) return floor is not None and end <= floor def _close_history_floor(con, table, exc, start, end): """Amazon's observed history floor: an old window it refuses below newer landed history. Counts Amazon's own words (out of range, start too old) on any dated table, and an unexplained old FATAL with no error document. Records the floor with Amazon's words; no older window is planned. """ if not _dated_table(table) or start is None or end is None: return None now = datetime.now(timezone.utc) status = getattr(exc, "status", None) period = _sqp_period(start, end) if table == SQP_TABLE else None words = str(getattr(exc, "vendor_reason", "") or exc) floor = read_backfill_floor(con, table) said = (bool(_AMAZON_FLOOR_TEXT.search(words)) and (status in ("FATAL", 400) or " answered 400" in str(exc)) and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end, period)) unexplained = (table != SQP_TABLE and status == "FATAL" and not getattr(exc, "has_error_document", True) and floor is not None and end <= floor and start < now - timedelta(days=365) and bool(_read_state(con, entity_key("reconcile_" + table)))) if not (said or unexplained): return None stop = end if said else floor reason = (f"Amazon keeps this report back to {stop.date().isoformat()}; older history is not available. " "Keep the connection enabled for newer data; ask Seller Support for older history.") amazon = " ".join(words.replace("\u2014", "; ").replace("\u2013", " to ").split())[:300] _record_state(con, _history_limit_key(table, period), "done", detail=json.dumps({ "floor": _iso(stop), "period": period, "reason": reason, "amazon": amazon, "window": [_iso(start), _iso(end)]})) if table != SQP_TABLE: # SQP's limit_ key is its progress document; its floor lives in the history marker only. _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({"reason": reason})) return reason def _history_task_period(task): return _sqp_period(task[1], task[2]) if task[0] == SQP_TABLE and task[1] is not None else None def _history_cut_tasks(tasks, table, period, end): """Only the refused period's windows at or below its floor are no longer owed.""" return (task for task in tasks if task[0] == table and task[2] is not None and task[2] <= end and _history_task_period(task) == period) def _record_reconciliation(con, table, marker): """Keep the most recent landed period, even while backfill travels backwards.""" key = entity_key("reconcile_" + table) previous = _read_state(con, key) old = json.loads(previous["detail"]) if previous and previous.get("detail") else {} if marker["end"] >= old.get("end", ""): _record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":"))) def _restatement_may_overlap(con, mirror, predicate, params, start, end, *, unsafe="TRUE"): """Treat absent or invalid legacy period metadata as unknown when checking retained rows.""" columns = set(_columns_of(con, mirror)) def period(name): return f'TRY_CAST("{name}" AS TIMESTAMPTZ)' if name in columns else "NULL::TIMESTAMPTZ" old_start, old_end = period("_period_start"), period("_period_end") return bool(con.execute( f'SELECT 1 FROM {mirror} WHERE NOT COALESCE(({predicate}), FALSE) AND ({unsafe}) ' f'AND ({old_start} IS NULL OR TRY_CAST(? AS TIMESTAMPTZ) IS NULL ' f'OR {old_start} < TRY_CAST(? AS TIMESTAMPTZ)) ' f'AND ({old_end} IS NULL OR {old_start} IS NULL OR {old_end} <= {old_start} ' f'OR {old_end} > TRY_CAST(? AS TIMESTAMPTZ)) LIMIT 1', [*params, end, end, start]).fetchone()) def _validate_report_snapshot(con, table, marker): """Reconcile one flat-file snapshot against its raw document count and source coordinates.""" total, matched, distinct = con.execute( f'SELECT COUNT(*), COUNT(*) FILTER (WHERE _snapshot_date IS NOT DISTINCT FROM ? ' 'AND _report_document_id IS NOT DISTINCT FROM ?), ' 'COUNT(DISTINCT CASE WHEN _report_row_number > 0 THEN _report_row_number END) ' f'FROM {table_name(table)}', [marker["start"], marker["document"]]).fetchone() expected = marker["count"] matches = total == matched == distinct == expected return {"state": "ok" if matches else "refused", "expected": expected, "actual": total, "source_rows": matched, "distinct": distinct, "reason": (f"Amazon document has {expected} rows; stored snapshot has {total} rows, " f"{matched} from that document and date, and {distinct} source row numbers")} #: Source columns that change on a pull without meaning. A change in them never opens a version. HISTORY_VOLATILE = frozenset(( "_report_document_id", "_report_row_number", "_data_origin", "_report_type", "_snapshot_date", amazon_schema.STORE, "_period_start", "_period_end", "snapshot_date", amazon_schema.IDENTITY_OCCURRENCE, amazon_schema.IDENTITY_PID, *connector_spec.AMAZON_HISTORY_COLUMNS)) #: Per-table additions: the ledger's `date` is its snapshot date. HISTORY_TABLE_VOLATILE = {"inventory": frozenset(("date",))} _HISTORY_KEY_EXCLUDED = frozenset(("_snapshot_date", "_report_row_number", "_report_document_id", "date")) _HISTORY_KEYS = {"inventory": ("msku", "fnsku", "asin", "disposition", "location")} _HISTORY_ACCOUNT_KEYS = (amazon_schema.CONNECTION_ID, "marketplace_id", "marketplace") _HISTORY_NUMERIC = frozenset(("BIGINT", "INTEGER", "DOUBLE", "FLOAT", "HUGEINT")) def history_table(table): """The mirror name of one snapshot source's history table.""" return table_name(table + "_history") def _typed_columns(con, table): return {r[0]: str(r[1]).upper() for r in con.execute( "SELECT column_name, data_type FROM information_schema.columns WHERE table_catalog=current_database() " "AND table_schema=current_schema() AND table_name=? ORDER BY ordinal_position", [table]).fetchall()} def _history_columns(con, table): """Create or widen the history table to carry every source column, never dropping or narrowing one.""" source, history = table_name(table), history_table(table) want, have = _typed_columns(con, source), _typed_columns(con, history) if not have: names = ", ".join(_quoted_identifier(c) for c in want) extra = ", ".join(f"NULL::VARCHAR AS {c}" for c in connector_spec.AMAZON_HISTORY_COLUMNS) con.execute(f"CREATE TABLE {history} AS SELECT {names}, {extra} FROM {source} WHERE FALSE") return _typed_columns(con, history) for column, kind in want.items(): if column not in have: con.execute(f"ALTER TABLE {history} ADD COLUMN {_quoted_identifier(column)} {kind}") elif have[column] != kind: widened = "DOUBLE" if {kind, have[column]} <= _HISTORY_NUMERIC else "VARCHAR" if have[column] != widened: con.execute(f"ALTER TABLE {history} ALTER COLUMN {_quoted_identifier(column)} TYPE {widened}") for column in connector_spec.AMAZON_HISTORY_COLUMNS: if column not in have: con.execute(f"ALTER TABLE {history} ADD COLUMN {column} VARCHAR") return _typed_columns(con, history) def _history_mark(con, history): """The newest date the history table has written, or None when it holds no version.""" first, last = con.execute(f"SELECT max(valid_from), max(valid_to) FROM {history}").fetchone() marks = [str(v) for v in (first, last) if v is not None] return max(marks) if marks else None def _history_key_columns(table, columns): """The identity columns a version is tracked by. Empty means the value hash is the identity.""" keys = _HISTORY_KEYS.get(table) or tuple(c for c in replace_keys_for(table) if c not in _HISTORY_KEY_EXCLUDED) keys = [c for c in keys if c in columns] return keys + [c for c in _HISTORY_ACCOUNT_KEYS if c in columns and c not in keys] if keys else [] def _history_hash_sql(names, alias, types, source_types=None): """md5 over `names` as text in the history column types, so a widened column hashes alike.""" def part(name): value = f"{alias}.{_quoted_identifier(name)}" if source_types is not None and source_types.get(name) != types[name]: value = f"CAST({value} AS {types[name]})" return f"COALESCE(CAST({value} AS VARCHAR), chr(30))" return f"md5(concat_ws(chr(31), {', '.join(part(n) for n in names)}))" if names else "md5('')" def _history_merge(con, table, date, where_sql, params=()): """Write one snapshot date's changes to the history table, inside the caller's transaction. Unchanged rows write nothing. A changed or absent row closes its open version at `date`. """ source, history = table_name(table), history_table(table) if not _columns_of(con, source): return date = str(date)[:10] types = _history_columns(con, table) mark = _history_mark(con, history) if mark is not None and date < mark: return # History only moves forward. source_types = _typed_columns(con, source) volatile = HISTORY_VOLATILE | HISTORY_TABLE_VOLATILE.get(table, frozenset()) values = sorted(c for c in source_types if c not in volatile) keys = _history_key_columns(table, source_types) value_new = _history_hash_sql(values, "s", types, source_types) key_new = _history_hash_sql(keys, "s", types, source_types) if keys else value_new value_old = _history_hash_sql(values, "h", types) cols = ", ".join(_quoted_identifier(c) for c in source_types) tag = uuid.uuid4().hex new, old, reopen = (f"amazon_history_{part}_{tag}" for part in ("new", "old", "reopen")) try: con.execute(f"CREATE TEMP TABLE {new} AS SELECT {cols}, " "__hk || ':' || CAST(row_number() OVER (PARTITION BY __hk ORDER BY __hv) AS VARCHAR) " "AS __history_key, __hv AS __history_hash " f"FROM (SELECT s.*, {key_new} AS __hk, {value_new} AS __hv FROM {source} s WHERE {where_sql})", list(params)) con.execute(f"CREATE TEMP TABLE {old} AS SELECT h._history_key AS __history_key, " f"{value_old} AS __history_hash FROM {history} h WHERE h.valid_to IS NULL") changed = (f"SELECT o.__history_key FROM {old} o LEFT JOIN {new} n ON n.__history_key = o.__history_key " "WHERE n.__history_key IS NULL OR n.__history_hash <> o.__history_hash") con.execute(f"UPDATE {history} SET valid_to = ? WHERE valid_to IS NULL AND valid_from < ? " f"AND _history_key IN ({changed})", [date, date]) # A same-day state is overwritten, never kept as a zero-length version. con.execute(f"DELETE FROM {history} WHERE valid_to IS NULL AND valid_from = ? " f"AND _history_key IN ({changed})", [date]) # A same-day revert reopens the version this date closed. con.execute(f"CREATE TEMP TABLE {reopen} AS SELECT n.__history_key FROM {new} n JOIN {history} h " f"ON h._history_key = n.__history_key AND h.valid_to = ? AND {value_old} = n.__history_hash " f"WHERE n.__history_key NOT IN (SELECT _history_key FROM {history} WHERE valid_to IS NULL)", [date]) con.execute(f"UPDATE {history} SET valid_to = NULL WHERE valid_to = ? " f"AND _history_key IN (SELECT __history_key FROM {reopen})", [date]) con.execute(f"INSERT INTO {history} ({cols}, valid_from, valid_to, _history_key) " f"SELECT {cols}, ?, NULL, __history_key FROM {new} " f"WHERE __history_key NOT IN (SELECT _history_key FROM {history} WHERE valid_to IS NULL)", [date]) finally: for name in (new, old, reopen): con.execute(f"DROP TABLE IF EXISTS {name}") def _history_catch_up(con, table, where_sql): """Merge every source snapshot date newer than the history, oldest first, which seeds version 1.""" source, history = table_name(table), history_table(table) columns = _columns_of(con, source) if "_snapshot_date" not in columns: return mark = _history_mark(con, history) if _columns_of(con, history) else None _history_columns(con, table) dates = [row[0] for row in con.execute( f"SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) FROM {source} " f"WHERE ({where_sql}) AND _snapshot_date IS NOT NULL ORDER BY 1").fetchall()] for day in dates: if mark is None or day > mark: _history_merge(con, table, day, f"({where_sql}) AND CAST(_snapshot_date AS VARCHAR) = ?", [day]) def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False): """Atomic range/date replacement and proof; an empty restatement removes old rows too. `snapshot=True` (S67-T32, D-3031): a current-state report is ONE copy. A successful, parsed pull replaces every earlier `_snapshot_date` of the table, not only its own day, because the grid pid hashes the date and a kept day doubles every row downstream. A failed pull raises before this runs, so it never empties the table. """ mirror = table_name(table) columns = list(dict.fromkeys([*columns, column, *_ORIGIN_TYPES])) # All JSON money columns remain fractional even when today's amounts are integers. declared = {c: "DOUBLE" for c in columns if c.endswith(("_amount", "_currency_amount"))} if table == "listing_items": declared["backend_keywords"] = "VARCHAR" if table == "outbound_fulfillment": declared.update({c: "BIGINT" for c in ("quantity", "cancelled_quantity", "unfulfillable_quantity") if c in columns}) if table == "inbound_shipments": declared["quantity"] = "BIGINT" declared.update(shipment_confirmation_id="VARCHAR", quantity_received="BIGINT") if table == "review_topics": declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns}) if table == "review_trends": declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns}) if table == "browse_node_review_topics": declared.update({name: kind for name, kind in BROWSE_NODE_TOPIC_TYPES.items() if name in columns}) types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True) predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?' if column == "posted_date" and end: predicate = ('CAST(posted_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) AND ' 'CAST(posted_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)') params = [start, end] if end else [start] if snapshot: if column != "_snapshot_date" or end: raise ValueError("a snapshot replacement is keyed by one _snapshot_date") predicate, params = "TRUE", [] source_keys = _client().REPORT_TYPES[table].get("restatement_keys", ()) staged = None if source_keys: overlap_end = (marker or {}).get("end") or end missing = set(source_keys) - set(columns) if missing: # A same-window replay remains safe without source keys; a moving overlap must refuse. if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end): raise ValueError(f"{table}: overlapping restatement omitted source keys: " + ", ".join(sorted(missing)) + "; request the complete source report") elif rows: staged = "amazon_restatement_" + uuid.uuid4().hex names = ", ".join(_quoted_identifier(c) for c in columns) con.execute(f"CREATE TEMP TABLE {staged} AS SELECT {names} FROM {mirror} WHERE FALSE") try: _replace(con, staged, columns, types, rows, source_table=mirror) required = [c for c in source_keys if c not in ("fnsku", "asin")] invalid = " OR ".join(f'NULLIF(TRIM(CAST({_quoted_identifier(c)} AS VARCHAR)), \'\') IS NULL' for c in required) if con.execute(f"SELECT 1 FROM {staged} WHERE {invalid} LIMIT 1").fetchone(): raise ValueError(f"{table}: restatement omitted a source key value; request the complete source report") if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end, unsafe=invalid): raise ValueError(f"{table}: overlapping legacy rows omitted a source key value; " "restore the complete source report before restating") except BaseException: con.execute(f"DROP TABLE {staged}") raise history = snapshot and table in HISTORY_SOURCES con.execute("BEGIN TRANSACTION") try: if history: _history_catch_up(con, table, "TRUE") con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params) if staged: matches = " AND ".join(f'{mirror}.{_quoted_identifier(c)} IS NOT DISTINCT FROM ' f'k.{_quoted_identifier(c)}' for c in source_keys) con.execute(f'DELETE FROM {mirror} WHERE EXISTS (SELECT 1 FROM {staged} k WHERE {matches})') con.execute(f"INSERT INTO {mirror} ({names}) SELECT {names} FROM {staged}") count = con.execute(f"SELECT COUNT(*) FROM {staged}").fetchone()[0] else: count = _replace(con, mirror, columns, types, rows) if snapshot and marker is not None and marker.get("document"): proof = _validate_report_snapshot(con, table, marker) if proof["state"] != "ok": raise ValueError(f"{table}: snapshot replacement refused: {proof['reason']}") if history: _history_merge(con, table, start, "TRUE") if marker is not None: _record_reconciliation(con, table, marker) if table == "listing_items": refresh_sku_keywords(con) con.execute("COMMIT") return count except Exception: con.execute("ROLLBACK") raise finally: if staged: con.execute(f"DROP TABLE {staged}") def document_row_count(text, roots=()): """Count source records before flattening, privacy filtering or mirror transformations.""" if _spooled(text): try: if roots: return sum(sum(1 for _item in _json_root_items(text, root)) for root in roots) with text.open_text() as handle: reader = csv.reader(handle, delimiter="\t", quoting=csv.QUOTE_NONE) next(reader, None) return sum(bool(raw) and not (len(raw) == 1 and not str(raw[0]).strip()) for raw in reader) except _JsonFallback: return document_row_count(text.read_text(), roots) if roots: doc = json.loads(text) return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots) reader = csv.reader(io.StringIO(str(text or "").replace("\r\n", "\n").replace("\r", "\n")), delimiter="\t", quoting=csv.QUOTE_NONE) next(reader, None) return sum(bool(raw) and not (len(raw) == 1 and not str(raw[0]).strip()) for raw in reader) def _ledger_date(value): """The DAILY ledger uses US dates; normalize dates without inventing a report date.""" value = str(value or "").strip() if re.fullmatch(r"\d{2}/\d{2}/\d{4}", value): return datetime.strptime(value, "%m/%d/%Y").date().isoformat() if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): return datetime.strptime(value, "%Y-%m-%d").date().isoformat() raise ValueError("Amazon inventory summary omitted a daily data date") #: F36: flat-file App tables whose date-named text columns are written as ISO. APP_DATE_TABLES = ("inventory_ledger_detail", "inventory_planning", "restock_recommendations", "storage_fees", "long_term_storage_fees", "fba_customer_returns", "reimbursements") _US_DATE = re.compile(r"(\d{1,2})/(\d{1,2})/(\d{4})") _US_STAMP = re.compile(r"(\d{1,2})/(\d{1,2})/(\d{4})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?" r"(?:\s*([A-Za-z]{1,5}|[+-]\d{2}:?\d{2}))?") _ZONES = {"Z": "+00:00", "UTC": "+00:00", "GMT": "+00:00", "PST": "-08:00", "PDT": "-07:00", "MST": "-07:00", "MDT": "-06:00", "CST": "-06:00", "CDT": "-05:00", "EST": "-05:00", "EDT": "-04:00"} def is_app_date_column(name): """True for `date`, `*_date`, `date_and_time` and `*_date_and_time`.""" name = str(name or "") return name == "date" or name.endswith("_date") or name == "date_and_time" \ or name.endswith("_date_and_time") def app_date(value): """`MM/DD/YYYY` -> `YYYY-MM-DD` and a US stamp -> ISO. Any other shape returns unchanged.""" if not isinstance(value, str): return value text = value.strip() try: if (m := _US_DATE.fullmatch(text)): return datetime(int(m[3]), int(m[1]), int(m[2])).date().isoformat() if (m := _US_STAMP.fullmatch(text)): zone = "" if m[7] is None else _ZONES.get(m[7].upper(), m[7] if m[7][0] in "+-" else None) if zone is None: return value if zone and ":" not in zone: zone = zone[:3] + ":" + zone[3:] stamp = datetime(int(m[3]), int(m[1]), int(m[2]), int(m[4]), int(m[5]), int(m[6] or 0)) return stamp.isoformat() + zone except ValueError: return value return value def normalise_app_dates(table, rows): """`rows` with every date-named value of an App table in ISO; other tables pass through.""" if table not in APP_DATE_TABLES: return rows def fixed(row): for key, value in row.items(): if is_app_date_column(key) and isinstance(value, str): row[key] = app_date(value) return row return rows.map(fixed) if isinstance(rows, _DocumentRows) else [fixed(row) for row in rows] def _snapshot_identity(table, row): """Mirror identity only; legacy grid IDs and AWD's replacement keys stay unchanged.""" sku = row.get("seller_sku") if table == "listings" else row.get("msku") or row.get("fnsku") if not sku or not row.get("_snapshot_date"): raise ValueError(f"Amazon {table} snapshot is missing identity date or SKU") identity = (row["_snapshot_date"], sku) if table == "inventory": if not row.get("disposition") or not row.get("location"): raise ValueError("Amazon inventory snapshot is missing identity disposition or location") identity += (row["disposition"], row["location"]) return identity def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count): """Replace the whole FBA snapshot with one parsed document, preserving the separate AWD leg. Listings is a catalogue as of the vendor's data/creation date. The ledger is a DAILY summary: its row date wins over the report creation date, including multi-date documents. Duplicate source identities refuse the whole document rather than silently dropping rows. S67-T32 (D-3031): one copy. The document's dates replace every earlier date, so the table holds exactly one pull, the row count a fresh first load would store. """ stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime") if not stamp and (table == "listings" or not rows): raise ValueError("Amazon snapshot returned no data date or report creation date") def stamped(row): period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat() row["_snapshot_date"] = period if table == "inventory": row["date"] = period row["inventory_source"] = "FBA" return row lazy = isinstance(rows, _DocumentRows) rows = rows.map(stamped) if lazy else rows counts, identities, total = {}, set(), 0 for row in rows: period = (row if lazy else stamped(row))["_snapshot_date"] identity = _snapshot_identity(table, row) if identity in identities: raise ValueError(f"Amazon {table} document has duplicate snapshot identities") identities.add(identity) counts[period] = counts.get(period, 0) + 1 total += 1 if total != source_count: raise ValueError(f"Amazon {table} snapshot row count differs from its document") if not counts: counts[_utc(stamp).date().isoformat()] = 0 columns = sorted(set(columns) | {"_snapshot_date"} | (rows.keys() if lazy else {k for row in rows for k in row})) if table == "inventory": columns = sorted(set(columns) | {"inventory_source"}) mirror = table_name(table) types = _ensure(con, mirror, columns, rows, preserve_rows=True) dates = sorted(counts) marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date", "count": source_count, "dates": counts, "document": meta.get("reportDocumentId")} marker.update({key: meta[key] for key in ("reportId", "processingStatus") if meta.get(key)}) if table == "inventory" and meta.get("cancelled") is True and meta.get("processingStatus") == "CANCELLED": marker["cancelled"] = True # Every earlier date and any undated pre-migration copy goes; AWD rows are another leg. predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE" history = table in HISTORY_SOURCES con.execute("BEGIN TRANSACTION") try: if history: _history_catch_up(con, table, predicate) con.execute(f"DELETE FROM {mirror} WHERE {predicate}") count = _replace(con, mirror, columns, types, rows) # The ledger keeps every daily snapshot as changes, oldest date first. for day in dates if history else (): _history_merge(con, table, day, f"({predicate}) AND CAST(_snapshot_date AS VARCHAR) = ?", [day]) _record_reconciliation(con, table, marker) con.execute("COMMIT") return count except Exception: con.execute("ROLLBACK") raise def history_current_sql(con, table): """`(open versions, newest source snapshot)` as two SELECTs over the same non-volatile columns.""" source, history = table_name(table), history_table(table) types = _typed_columns(con, history) source_types = _typed_columns(con, source) volatile = HISTORY_VOLATILE | HISTORY_TABLE_VOLATILE.get(table, frozenset()) names = [c for c in source_types if c not in volatile and c in types] def projected(alias, cast): return ", ".join(f"CAST({alias}.{_quoted_identifier(c)} AS {types[c]})" if cast and source_types[c] != types[c] else f"{alias}.{_quoted_identifier(c)}" for c in names) or "1" def scope(alias): return f"COALESCE({alias}.inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE" owner = ('AND p."_connection_id" IS NOT DISTINCT FROM s."_connection_id"' if amazon_schema.CONNECTION_ID in source_types else "") newest = f"SELECT max(CAST(p._snapshot_date AS VARCHAR)) FROM {source} p WHERE {scope('p')} {owner}" current = (f"SELECT {projected('s', True)} FROM {source} s " f"WHERE {scope('s')} AND CAST(s._snapshot_date AS VARCHAR) = ({newest})") return f"SELECT {projected('h', False)} FROM {history} h WHERE h.valid_to IS NULL", current def _validate_history(con, table): """The open versions equal the source's newest snapshot on every non-volatile column.""" history = history_table(table) if not _columns_of(con, history) or "_snapshot_date" not in _columns_of(con, table_name(table)): return {"state": "unproven", "reason": "History starts with the next sync of the source table."} rows = con.execute(f"SELECT COUNT(*) FROM {history}").fetchone()[0] opened, current = history_current_sql(con, table) extra = con.execute(f"SELECT COUNT(*) FROM ({opened} EXCEPT ALL {current})").fetchone()[0] missing = con.execute(f"SELECT COUNT(*) FROM ({current} EXCEPT ALL {opened})").fetchone()[0] return {"state": "ok" if not extra and not missing else "refused", "rows": rows, "reason": (f"History has {extra} open versions the newest snapshot lacks, " f"and the snapshot has {missing} rows with no open version")} def _validate_snapshot(con, table): """Reconcile raw document cardinality and each data date's distinct business identities.""" saved = _read_state(con, entity_key("reconcile_" + table)) if not saved or saved.get("state") != "done" or not saved.get("detail"): return {"state": "unproven", "reason": "Amazon snapshot reconciliation marker is absent"} try: marker = json.loads(saved["detail"]) dates = marker["dates"] if not marker["count"]: return {"state": "unproven", "reason": "Amazon snapshot document is empty"} predicate = "inventory_source='FBA'" if table == "inventory" else "TRUE" result = con.execute(f"SELECT * FROM {table_name(table)} WHERE {predicate} AND " f"_snapshot_date IN ({','.join('?' for _ in dates)})", list(dates)) rows = [dict(zip([c[0] for c in result.description], row)) for row in result.fetchall()] actual_dates = dict.fromkeys(dates, 0) identities = set() for row in rows: identities.add(_snapshot_identity(table, row)) actual_dates[row["_snapshot_date"]] += 1 matches = (len(rows) == len(identities) == marker["count"] and actual_dates == dates) return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows), "distinct": len(identities), "dates": actual_dates, "reason": (f"Amazon document has {marker['count']} data rows; stored dates contain " f"{len(rows)} rows and {len(identities)} distinct snapshot identities")} except Exception as exc: return {"state": "refused", "reason": f"Amazon snapshot reconciliation failed: {exc}"} def _fee_estimate_inputs(con, fields): """Read every listing offer's price and fulfillment method without guessing missing source values.""" cx = _client() marketplace = fields.get("marketplace_id") or cx.US_MARKETPLACE_ID columns = _columns_of(con, "amzn_listing_items") if not {"sku", "offers", "fulfillment_availability"} <= columns: raise ValueError("Fee estimates need listing offer prices and fulfillment methods. Sync Amazon listing status and issues first.") requests, unpriced = {}, 0 for sku, raw_offers, raw_fulfillment in con.execute( "SELECT sku, offers, fulfillment_availability FROM amzn_listing_items").fetchall(): offers = json.loads(raw_offers or "[]") fulfillment = json.loads(raw_fulfillment or "[]") if not isinstance(offers, list) or not isinstance(fulfillment, list): raise ValueError(f"Amazon listing SKU {sku} has malformed offers or fulfillment methods") methods = set() for method in fulfillment: channel = str(method.get("fulfillmentChannelCode") or "") if channel == "DEFAULT": methods.add(False) elif channel.startswith("AMAZON_"): methods.add(True) else: raise ValueError(f"Amazon listing SKU {sku} has no supported fulfillment method for fee estimates") priced = [offer for offer in offers if offer.get("marketplaceId") == marketplace] if not methods or not priced: # An inactive listing has no price here, so Amazon has no fee to estimate for it. unpriced += 1 continue for offer in priced: price = offer.get("price") or {} for fulfilled in sorted(methods): estimate = {"sku": sku, "price": {"Amount": price.get("amount"), "CurrencyCode": price.get("currencyCode")}, "is_amazon_fulfilled": fulfilled} identity = cx.fee_estimate_request(estimate, marketplace)["FeesEstimateRequest"]["Identifier"] requests[identity] = estimate if unpriced and not requests: raise ValueError(f"No Amazon listing has a price or fulfillment method for this marketplace " f"({unpriced} listings). No partial fee snapshot was stored.") return list(requests.values()) def listing_item_rows(records, document): """Project every listing keyword value while retaining the complete nonpersonal source row.""" rows = [] for index, record in enumerate(records, 1): row = _flatten_row(_without_buyer_data(record)) attributes = record.get("attributes") or {} keywords = attributes.get("generic_keyword") or [] row["backend_keywords"] = "; ".join( item["value"] for item in keywords if isinstance(item, dict) and isinstance(item.get("value"), str) and item["value"]) rows.append(_origin(row, document, index, "searchListingsItems")) return rows def fee_estimate_rows(records, document): """Keep one estimate per price basis with all fee lines and their nested details intact.""" rows = [] for position, record in enumerate(records, 1): echo = record["FeesEstimateIdentifier"] price = echo["PriceToEstimateFees"]["ListingPrice"] row = _flatten_row(record) by_asin = echo.get("IdType") == "ASIN" if by_asin: row["asin"] = echo["IdValue"] row.update(estimate_id=echo["SellerInputIdentifier"], sku=None if by_asin else echo["IdValue"], marketplace_id=echo["MarketplaceId"], is_amazon_fulfilled=echo["IsAmazonFulfilled"], price_amount=price["Amount"], price_currency_code=price["CurrencyCode"]) rows.append(_origin(row, document, position, "getMyFeesEstimates")) return rows _OUTBOUND_ORDER_FIELDS = ( "sellerFulfillmentOrderId", "marketplaceId", "fulfillmentOrderStatus", "shippingSpeedCategory", "receivedDate", "statusUpdatedDate", "displayableOrderDate", "fulfillmentAction", "fulfillmentPolicy", ) _OUTBOUND_ITEM_FIELDS = ( "sellerSku", "sellerFulfillmentOrderItemId", "quantity", "cancelledQuantity", "unfulfillableQuantity", "estimatedShipDate", "estimatedArrivalDate", ) _OUTBOUND_QUANTITIES = ("quantity", "cancelledQuantity", "unfulfillableQuantity") def outbound_fulfillment_rows(records, document): """Whitelist non-personal scalar order and item fields before any flattening or mirror write.""" rows, identities = [], set() def scalars(value, allowed): clean = _without_buyer_data(value) result = {} for key in allowed: if key not in clean: continue cell = clean[key] valid = type(cell) is int if key in _OUTBOUND_QUANTITIES else isinstance(cell, str) if cell is not None and not valid: raise ValueError("Amazon outbound fulfillment returned a non-scalar or invalid permitted field: " + key) if isinstance(cell, str) and cell.lstrip().startswith(("{", "[")): try: structured = isinstance(json.loads(cell), (dict, list)) except ValueError: structured = False if structured: raise ValueError("Amazon outbound fulfillment returned structured JSON in a permitted scalar field: " + key) if key in _OUTBOUND_QUANTITIES and cell is not None and cell < 0: raise ValueError("Amazon outbound fulfillment returned a negative quantity") result[amazon_schema._snake(key)] = cell return result for record in records: order = scalars(record["fulfillmentOrder"], _OUTBOUND_ORDER_FIELDS) for item in record["fulfillmentOrderItems"]: row = {**order, **scalars(item, _OUTBOUND_ITEM_FIELDS)} required = ("seller_fulfillment_order_id", "seller_fulfillment_order_item_id", "seller_sku", "quantity") if any(row.get(key) in (None, "") for key in required): raise ValueError("Amazon outbound fulfillment omitted required non-personal item fields") identity = row["seller_fulfillment_order_id"], row["seller_fulfillment_order_item_id"] if identity in identities: raise ValueError("Amazon outbound fulfillment repeated an order item identity") identities.add(identity) rows.append(_origin(row, document, len(rows) + 1, "getFulfillmentOrder")) return rows def _outbound_totals(records): """Count raw Amazon items and quantities independently of the row parser.""" totals = dict.fromkeys(_OUTBOUND_QUANTITIES, 0) count = 0 for record in records: for item in record["fulfillmentOrderItems"]: count += 1 for key in totals: value = item.get(key, 0) if type(value) is not int or value < 0: raise ValueError("Amazon outbound fulfillment returned an invalid quantity field: " + key) totals[key] += value return {"count": count, "quantities": totals} def inbound_shipment_rows(records, document): """Keep non-personal item fields with stable shipment, SKU and package identities.""" rows, occurrences = [], {} strings = ("msku", "asin", "fnsku", "expiration", "manufacturingLotCode", "labelOwner") for record in records: for item in record["items"]: clean = {} for key in strings: value = item.get(key) if value is not None and (not isinstance(value, str) or value.lstrip().startswith(("{", "["))): raise ValueError("Amazon inbound item returned an invalid scalar field: " + key) if value is not None: clean[key] = value quantity = item.get("quantity") if not clean.get("msku") or type(quantity) is not int or quantity < 1: raise ValueError("Amazon inbound item omitted its SKU or positive integer quantity") instructions = [] preparation = item.get("prepInstructions", []) if not isinstance(preparation, list): raise ValueError("Amazon inbound item returned invalid preparation instructions") for prep in preparation: if not isinstance(prep, dict): raise ValueError("Amazon inbound item returned invalid preparation instructions") instruction = {key: prep[key] for key in ("prepType", "prepOwner") if key in prep} if any(not isinstance(value, str) or value.lstrip().startswith(("{", "[")) for value in instruction.values()): raise ValueError("Amazon inbound preparation fields must be scalar strings") if "fee" in prep: fee = prep["fee"] if (not isinstance(fee, dict) or not isinstance(fee.get("code"), str) or fee["code"].lstrip().startswith(("{", "[")) or type(fee.get("amount")) not in (int, float)): raise ValueError("Amazon inbound preparation fee is invalid") instruction["fee"] = {"code": fee["code"], "amount": fee["amount"]} instructions.append(instruction) clean["prepInstructions"] = instructions identity = [record["inboundPlanId"], record["shipmentId"], *[clean.get(key) for key in strings]] # Preparation prices, quantities, status, page order and sync dates do not identify an item. natural = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) occurrence = occurrences.get(natural, 0) + 1 occurrences[natural] = occurrence item_id = hashlib.sha256(json.dumps([identity, occurrence], separators=(",", ":")).encode()).hexdigest() row = _flatten_row(clean) row.update(shipment_item_id=item_id, inbound_plan_id=record["inboundPlanId"], plan_status=record["planStatus"], shipment_id=record["shipmentId"], sku=clean["msku"], quantity=quantity, status=record["status"]) # Received units are Amazon's explicit count. An unknown receipt stays null, never 0. received, confirmation = item.get("QuantityReceived"), record.get("shipmentConfirmationId") if received is not None and (type(received) is not int or received < 0): raise ValueError("Amazon inbound item returned an invalid received quantity") if confirmation is not None and not isinstance(confirmation, str): raise ValueError("Amazon inbound shipment returned an invalid confirmation ID") row.update(shipment_confirmation_id=confirmation, quantity_received=received) rows.append(_origin(row, document, len(rows) + 1, "listShipmentItems")) return rows def _inbound_totals(records): """Count raw source items and units per shipment and seller SKU without the row transformation.""" shipments, sku_quantities = {}, {} for record in records: identity = json.dumps([record["inboundPlanId"], record["shipmentId"]], separators=(",", ":")) count, units, skus = 0, 0, {} for item in record["items"]: sku, quantity = item.get("msku"), item.get("quantity") if (not isinstance(sku, str) or not sku or sku.lstrip().startswith(("{", "[")) or type(quantity) is not int or quantity < 1): raise ValueError("Amazon inbound source returned an invalid seller SKU or item quantity") count += 1 units += quantity skus[sku] = skus.get(sku, 0) + quantity if identity in shipments: raise ValueError("Amazon inbound source repeated a shipment") shipments[identity] = {"count": count, "quantity": units} sku_quantities[identity] = skus return {"count": sum(value["count"] for value in shipments.values()), "shipments": shipments, "sku_quantities": sku_quantities} def replenishment_rows(records, document): """Keep each raw metric's source, natural identity and measured fields without buyer data.""" if not records["offerCount"]: return [] rows, identities = [], set() for scope, key, operation in (("seller", "sellingPartnerMetrics", "getSellingPartnerMetrics"), ("offer", "offerMetrics", "listOfferMetrics")): for record in records[key]: interval = record.get("timeInterval") if (not isinstance(interval, dict) or not interval.get("startDate") or not interval.get("endDate") or (scope == "offer" and any(not isinstance(record.get(key), str) or not record[key] for key in ("asin", "sku", "fulfillmentChannelType")))): raise ValueError("Amazon replenishment metric omitted its source identity; no partial snapshot was stored") identity = [operation, records["marketplaceId"], "SUBSCRIBE_AND_SAVE", "PERFORMANCE", interval["startDate"], interval["endDate"], record.get("asin"), record.get("sku"), record.get("fulfillmentChannelType")] metric_id = hashlib.sha256(json.dumps(identity, separators=(",", ":")).encode()).hexdigest() if metric_id in identities: raise ValueError("Amazon replenishment repeated a metric identity; no partial snapshot was stored") identities.add(metric_id) row = _flatten_row(_without_buyer_data(record)) row.update(metric_id=metric_id, metric_scope=scope, marketplace_id=records["marketplaceId"], program_type="SUBSCRIBE_AND_SAVE", time_period_type="PERFORMANCE") rows.append(_origin(row, document, len(rows) + 1, operation)) return rows def _name_skipped_plans(con, skipped, operations, log): """Name each inbound plan Amazon refused alone in the pass report and on the Connectors screen.""" key = entity_key("limit_inbound_shipments") if operations is not None and skipped: operations["getInboundPlan"] = {"state": "done", "skippedPlans": skipped} if not skipped: # A pass with no refused plan clears the last pass's sentence. if _read_state(con, key): _record_state(con, key, "done", detail="{}") return kinds = {"converted": "converted to Send to Amazon", "unknown": "unknown plan refusal"} named = ", ".join(f"{plan['inboundPlanId']} ({kinds.get(plan.get('refusal'), 'unknown plan refusal')})" for plan in skipped[:3]) more = f" and {len(skipped) - 3} more" if len(skipped) > 3 else "" # The status screen refuses a scalar over 512 characters, so the sentence names three plans at most. reason = (f"Amazon refused {len(skipped)} inbound plan{'s' if len(skipped) > 1 else ''} by name, so " f"{'their' if len(skipped) > 1 else 'its'} shipments are not stored: {named}{more}. " "Every other plan is stored. See these plans in Seller Central.") _record_state(con, key, "done", detail=json.dumps({"reason": reason})) log(f"inbound_shipments: skipped plans {json.dumps(skipped)}") def _catalog_listing_content(item, marketplace): """Select listing text and image links using Amazon's notification attribute identities.""" attributes = item.get("attributes") or {} def local(values): return [value for value in values if isinstance(value, dict) and value.get("marketplace_id", value.get("marketplaceId", marketplace)) == marketplace] content = {name: [{key: value[key] for key in ("value", "language_tag") if key in value} for value in local(attributes.get(name) or [])] for name in ("item_name", "bullet_point", "product_description")} if not content["item_name"]: content["item_name"] = [{"value": summary["itemName"]} for summary in local(item.get("summaries") or []) if summary.get("itemName")] for name, values in attributes.items(): if name.endswith("_image_locator") or re.fullmatch(r"other_product_image_locator_\d+", name): content[name] = sorted({value["media_location"] for value in local(values or []) if value.get("media_location")}) pictures = {} for group in local(item.get("images") or []): for picture in group.get("images") or []: variant, link = picture.get("variant"), picture.get("link") if not link: continue name = ("main_product_image_locator" if variant == "MAIN" else "swatch_product_image_locator" if variant == "SWCH" else "other_product_image_locator_" + str(int(variant[2:])) if re.fullmatch(r"PT\d+", variant or "") else "images") pictures.setdefault(name, set()).add(link) for name, links in pictures.items(): content[name] = sorted(set(content.get(name, [])) | links) return {name: value for name, value in content.items() if value} def _catalog_listing_hash(content): """Hash the selected content independently of snapshot dates and response key order.""" return hashlib.sha256(json.dumps(content, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")).hexdigest() def _catalog_listing_changes(con, records, marketplace, rt=None): """Compare every ASIN with its connection's durable content baseline and record C4 changes.""" from harness import amazon_listing_changes scope = _account_scope() rt = _ledger_runtime(scope["tenant"] if scope else getattr(rt, "key", None), rt if rt is not None else (scope or {}).get("rt")) for item in records: asin = str(item.get("asin") or "").strip() if not asin: continue identity = json.dumps([marketplace, asin], separators=(",", ":")) key = entity_key("listing_content_" + hashlib.sha256(identity.encode()).hexdigest()) content = _catalog_listing_content(item, marketplace) digest = _catalog_listing_hash(content) state = _read_state(con, key) saved = json.loads(state["detail"]) if state else None def deliver(previous): pending = previous["pending"] changed = sorted(name for name in previous["attributes"].keys() | pending["attributes"].keys() if previous["attributes"].get(name, []) != pending["attributes"].get(name, [])) amazon_listing_changes.record(rt, asin=asin, sku=None, marketplace_id=marketplace, attributesChanged=changed, source="snapshot", before_values={name: previous["attributes"].get(name, []) for name in changed}, after_values={name: pending["attributes"].get(name, []) for name in changed}, detected_at=pending["detected_at"]) baseline = {"hash": pending["hash"], "attributes": pending["attributes"]} _record_state(con, key, "done", detail=json.dumps(baseline, ensure_ascii=False)) return baseline if saved and saved.get("pending"): # A retry retains its timestamp so a recorded change cannot be inserted twice. saved = deliver(saved) if saved is None: _record_state(con, key, "done", detail=json.dumps({"hash": digest, "attributes": content}, ensure_ascii=False)) continue if saved["hash"] == digest: continue saved["pending"] = {"hash": digest, "attributes": content, "detected_at": datetime.now(timezone.utc).isoformat()} _record_state(con, key, "pending", detail=json.dumps(saved, ensure_ascii=False)) deliver(saved) REVIEW_TOPIC_METRICS = ( ("asin_metrics_number_of_mentions", ("asinMetrics", "numberOfMentions")), ("asin_metrics_occurrence_percentage", ("asinMetrics", "occurrencePercentage")), ("asin_metrics_star_rating_impact", ("asinMetrics", "starRatingImpact")), ("parent_asin_metrics_number_of_mentions", ("parentAsinMetrics", "numberOfMentions")), ("parent_asin_metrics_occurrence_percentage", ("parentAsinMetrics", "occurrencePercentage")), ("parent_asin_metrics_star_rating_impact", ("parentAsinMetrics", "starRatingImpact")), ("browse_node_metrics_occurrence_percentage_all_products", ("browseNodeMetrics", "occurrencePercentage", "allProducts")), ("browse_node_metrics_star_rating_impact_all_products", ("browseNodeMetrics", "starRatingImpact", "allProducts")), ("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")), ("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")), ) #: Amazon returns at most five subtopics per topic and three snippets per topic or subtopic. REVIEW_SUBTOPICS, REVIEW_SNIPPETS = 5, 3 _SUBTOPIC_TYPES = { **{f"subtopic_{i}": "VARCHAR" for i in range(1, REVIEW_SUBTOPICS + 1)}, **{f"subtopic_{i}_number_of_mentions": "BIGINT" for i in range(1, REVIEW_SUBTOPICS + 1)}, **{f"subtopic_{i}_occurrence_percentage": "DOUBLE" for i in range(1, REVIEW_SUBTOPICS + 1)}, **{f"subtopic_{i}_review_snippet_{j}": "VARCHAR" for i in range(1, REVIEW_SUBTOPICS + 1) for j in range(1, REVIEW_SNIPPETS + 1)}, } REVIEW_TOPIC_TYPES = { **dict.fromkeys(("asin", "marketplace_id", "sort_by", "sentiment", "topic", "review_snippets", "subtopics", "review_snippet_1", "review_snippet_2", "review_snippet_3", "browse_node_id", "browse_node_name"), "VARCHAR"), **{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE" for name, _path in REVIEW_TOPIC_METRICS}, **_SUBTOPIC_TYPES, } def _feedback_asins(con): """The tenant's active child ASINs: active listings minus every parent in the latest catalogue snapshot.""" columns = _columns_of(con, "amzn_listings") if "asin1" not in columns: return [] active = " AND LOWER(TRIM(CAST(status AS VARCHAR))) = 'active'" if "status" in columns else "" asins = [row[0] for row in con.execute( "SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), '') IS NOT NULL" + active + " ORDER BY 1").fetchall()] catalog = _columns_of(con, "amzn_catalog_items") if "parent_asin" not in catalog: return asins other = " AND TRIM(CAST(parent_asin AS VARCHAR)) <> TRIM(CAST(asin AS VARCHAR))" if "asin" in catalog else "" parents = {row[0] for row in con.execute( "SELECT DISTINCT TRIM(CAST(parent_asin AS VARCHAR)) FROM (" + _latest_account_snapshot_sql(catalog, "amzn_catalog_items") + ") AS current_catalog WHERE NULLIF(TRIM(CAST(parent_asin AS VARCHAR)), '') IS NOT NULL" + other).fetchall()} return [asin for asin in asins if asin not in parents] FEEDBACK_NO_ASINS = ("No source rows: the Amazon listings catalogue has no active child ASINs. " "Sync Amazon listings first.") def _feedback_number(value, *, count=False, percentage=False): """A Customer Feedback metric: null, or a finite number; a count is a nonnegative integer.""" import math if value is None: return None if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): raise ValueError("Amazon review topic returned a nonnumeric metric") if count and (type(value) is not int or value < 0): raise ValueError("Amazon review topic returned an invalid mention count") if percentage and not 0 <= value <= 100: raise ValueError("Amazon review topic returned an invalid occurrence percentage") return value def _feedback_snippets(snippets): if snippets is not None and (not isinstance(snippets, list) or len(snippets) > REVIEW_SNIPPETS or any(not isinstance(snippet, str) for snippet in snippets)): raise ValueError("Amazon review snippets must be null or a list of up to three strings") return snippets or [] def _subtopic_columns(topic, *, mentions=True): """Up to five subtopics as columns: name, metrics and up to three snippets each.""" subtopics = topic.get("subtopics") if subtopics is not None and (not isinstance(subtopics, list) or len(subtopics) > REVIEW_SUBTOPICS or any(not isinstance(sub, dict) for sub in subtopics)): raise ValueError("Amazon review subtopics must be null or a list of up to five objects") out = {} for i, sub in enumerate(subtopics or [], 1): metrics = sub.get("metrics") if (not isinstance(sub.get("subtopic"), str) or not sub["subtopic"].strip() or (metrics is not None and not isinstance(metrics, dict))): raise ValueError("Amazon review subtopic requires a subtopic name and a metrics object") metrics = metrics or {} out[f"subtopic_{i}"] = sub["subtopic"] if mentions: out[f"subtopic_{i}_number_of_mentions"] = _feedback_number(metrics.get("numberOfMentions"), count=True) out[f"subtopic_{i}_occurrence_percentage"] = _feedback_number(metrics.get("occurrencePercentage")) for j, snippet in enumerate(_feedback_snippets(sub.get("reviewSnippets")), 1): out[f"subtopic_{i}_review_snippet_{j}"] = snippet return out def _null_unresolved_metrics(records): """Read an unfilled Amazon placeholder (a "$" string inside a metrics object) as null; return how many.""" count = 0 def walk(node, inside): nonlocal count items = node.items() if isinstance(node, dict) else enumerate(node) if isinstance(node, list) else () for key, value in list(items): if inside and isinstance(value, str) and value.startswith("$"): node[key] = None count += 1 elif isinstance(value, (dict, list)): walk(value, inside or (isinstance(key, str) and key.endswith(("Metrics", "metrics")))) walk(records, False) return count def _drop_repeated_topics(records): """Drop a topic Amazon repeats with identical metrics in one sorted answer; return how many. A conflicting repeat stays.""" dropped = 0 for record in records: topics = record.get("topics") if not isinstance(topics, dict): continue for key in ("positiveTopics", "negativeTopics"): group = topics.get(key) if not isinstance(group, list): continue kept, first = [], {} for topic in group: name = topic.get("topic") if isinstance(topic, dict) else None metrics = (json.dumps({k: v for k, v in topic.items() if k.endswith("Metrics")}, sort_keys=True, default=str) if isinstance(topic, dict) else None) if isinstance(name, str) and name in first and first[name] == metrics: dropped += 1 continue if isinstance(name, str): first.setdefault(name, metrics) kept.append(topic) topics[key] = kept return dropped def _sorted_feedback(read): """Stamp each record with the sort it answered, in FEEDBACK_SORTS order.""" return [{**record, "_sort_by": sort} for sort in _client().FEEDBACK_SORTS for record in read.get(sort) or []] def _review_topic_evidence(records, asins, marketplace): """Count raw vendor topics and retain their metrics before the row parser runs.""" import math sorts = _client().FEEDBACK_SORTS expected = {(asin, sort) for asin in asins for sort in sorts} counts, metrics, absent, seen = {}, {}, [], set() metricless = 0 for record in records: asin, sort = record.get("asin"), record.get("_sort_by") if (asin, sort) not in expected or (asin, sort) in seen or record.get("marketplaceId") != marketplace: raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace") seen.add((asin, sort)) no_content = record.get("_no_feedback_reason") topics = record.get("topics") if not no_content and not isinstance(topics, dict): raise ValueError("Amazon getItemReviewTopics returned no valid topics object") total = 0 for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): group = None if no_content else topics.get(key) group = [] if group is None else group if not isinstance(group, list): raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list") counts[json.dumps([asin, sort, sentiment], separators=(",", ":"))] = len(group) total += len(group) for topic in group: # Amazon omits asinMetrics on some real topics: the topic stays, its metrics read null and are counted. if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip() or not isinstance(topic.get("asinMetrics", {}), (dict, type(None)))): raise ValueError("Amazon review topic requires a topic name and an asinMetrics object or none") metricless += not isinstance(topic.get("asinMetrics"), dict) identity = json.dumps([asin, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":")) if identity in metrics: raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment") values = [] for name, path in REVIEW_TOPIC_METRICS: value = topic for segment in path: if value is None: break if not isinstance(value, dict): raise ValueError("Amazon review topic returned invalid nested metrics") value = value.get(segment) if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value)): raise ValueError("Amazon review topic returned a nonnumeric metric") if name.endswith("_number_of_mentions") and value is not None and (type(value) is not int or value < 0): raise ValueError("Amazon review topic returned an invalid mention count") values.append(value) metrics[identity] = values if not total: absent.append({"asin": asin, "sort_by": sort, "code": "NO_CONTENT" if no_content else "NO_TOPICS", "reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."}) if seen != expected: raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN") return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics, "marketplace_id": marketplace, "absent_asins": absent} def review_topic_rows(records, document, browse_nodes=None): """Flatten positive and negative topics while retaining null metrics and vendor percentage units.""" rows = [] for record in records: node = (browse_nodes or {}).get(record["asin"]) or {} for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): for topic in (record.get("topics") or {}).get(key) or []: row = dict.fromkeys(REVIEW_TOPIC_TYPES) row.update(_flatten_row(topic)) row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment, sort_by=record.get("_sort_by"), browse_node_id=node.get("browse_node_id"), browse_node_name=node.get("browse_node_name")) row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record})) # All three Customer Feedback tables store the review period as a calendar date. for name in ("date_range_start_date", "date_range_end_date"): row[name] = str(row[name])[:10] if row.get(name) is not None else None for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1): row[f"review_snippet_{index}"] = snippet row.update(_subtopic_columns(topic)) rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics")) return rows def _sync_review_topics(con, fields): """Replace the complete active child ASIN topic snapshot, both sorts, with same-pass vendor reconciliation.""" asins = _feedback_asins(con) marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID read = _client().item_review_topics(fields, asins) records = _sorted_feedback(read) unresolved = _null_unresolved_metrics(records) repeated = _drop_repeated_topics(records) evidence = _review_topic_evidence(records, asins, marketplace) document = "operation:" + uuid.uuid4().hex rows = review_topic_rows(records, document, read.get("browse_nodes")) period = datetime.now(timezone.utc).date().isoformat() for row in rows: row["_snapshot_date"] = period reason = ("Amazon Customer Feedback returned no review topics for this marketplace. " "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly." if asins else FEEDBACK_NO_ASINS) marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, "unresolved_metrics": unresolved, "repeated_topics": repeated, "empty_reason": reason if not rows else ""} count = _replace_period(con, "review_topics", rows, sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}), "_snapshot_date", period, marker=marker, snapshot=True) _record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({ "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved, "repeated_topics": repeated})) return count REVIEW_TREND_TYPES = { **dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month", "date_range_start_date", "date_range_end_date"), "VARCHAR"), "asin_metrics_occurrence_percentage": "DOUBLE", "parent_asin_metrics_occurrence_percentage": "DOUBLE", "browse_node_metrics_occurrence_percentage_all_products": "DOUBLE", } _TREND_METRICS = (("asin_metrics_occurrence_percentage", ("asinMetrics", "occurrencePercentage")), ("parent_asin_metrics_occurrence_percentage", ("parentAsinMetrics", "occurrencePercentage")), ("browse_node_metrics_occurrence_percentage_all_products", ("browseNodeMetrics", "occurrencePercentage", "allProducts"))) def _trend_window(date_range): """`(start, end, month)` of one trend point: ISO dates, start before end, month the last one inside the window.""" start, end = (date_range.get(name) for name in ("startDate", "endDate")) days = [] for value in (start, end): if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}(T[0-9:.]+(Z|[+-]\d{2}:\d{2})?)?", value): raise ValueError("Amazon review trend requires ISO calendar dates") try: days.append(datetime.strptime(value[:10], "%Y-%m-%d")) if len(value) > 10: datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: raise ValueError("Amazon review trend requires ISO calendar dates") from None if days[0] >= days[1]: raise ValueError("Amazon review trend dateRange must start before it ends") return start[:10], end[:10], (days[1] - timedelta(days=1)).strftime("%Y-%m") def _trend_values(point): """The ASIN, parent and browse node occurrence percentages of one trend point.""" import math values = [] for _name, path in _TREND_METRICS: value = point for segment in path: value = value.get(segment) if isinstance(value, dict) else None if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or not 0 <= value <= 100): raise ValueError("Amazon review trend returned an invalid occurrence percentage") values.append(value) return values def _review_trend_evidence(records, asins, marketplace): """Count vendor trend points and retain source values before flattening or storing rows.""" expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, [] metricless = 0 for record in records: asin = record.get("asin") if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace: raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace") seen.add(asin) no_content = record.get("_no_feedback_reason") trends = record.get("reviewTrends") if not no_content and not isinstance(trends, dict): raise ValueError("Amazon getItemReviewTrends returned no valid reviewTrends object") total = 0 for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): topics = [] if no_content else trends.get(key) topics = [] if topics is None else topics if not isinstance(topics, list): raise ValueError("Amazon getItemReviewTrends returned an invalid sentiment topic list") group = json.dumps([asin, sentiment], separators=(",", ":")) counts[group] = 0 topic_names = set() for topic in topics: if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip() or not isinstance(topic.get("trendMetrics"), list)): raise ValueError("Amazon review trend requires a topic name and trendMetrics list") if topic["topic"] in topic_names: raise ValueError("Amazon returned duplicate trend topics for one ASIN and sentiment") topic_names.add(topic["topic"]) # Count raw points, independently of review_trend_rows. counts[group] += len(topic["trendMetrics"]) for point in topic["trendMetrics"]: if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict) or not isinstance(point.get("asinMetrics", {}), (dict, type(None)))): raise ValueError("Amazon review trend requires a dateRange object and an asinMetrics object or none") metricless += not isinstance(point.get("asinMetrics"), dict) start, end, month = _trend_window(point["dateRange"]) identity = json.dumps([asin, marketplace, sentiment, topic["topic"], month], separators=(",", ":")) if identity in metrics: raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month") metrics[identity] = [start, end, *_trend_values(point)] total += counts[group] if not total: absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS", "reason": no_content or "Amazon returned no positive or negative review trend points for this ASIN and marketplace."}) if seen != expected: raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN") return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics, "marketplace_id": marketplace, "absent_asins": absent} def review_trend_rows(records, document): """Flatten vendor occurrence percentages at ASIN, sentiment, topic and month grain.""" rows = [] for record in records: for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): for topic in (record.get("reviewTrends") or {}).get(key) or []: for point in topic["trendMetrics"]: start, end, month = _trend_window(point["dateRange"]) row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"], "sentiment": sentiment, "topic": topic["topic"], "month": month, "date_range_start_date": start, "date_range_end_date": end, **dict(zip((name for name, _path in _TREND_METRICS), _trend_values(point)))} rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends")) return rows def _sync_review_trends(con, fields): """Replace the full six-month review snapshot with independently counted source evidence.""" asins = _feedback_asins(con) marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends") unresolved = _null_unresolved_metrics(records) repeated = _drop_repeated_topics(records) evidence = _review_trend_evidence(records, asins, marketplace) rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex) period = datetime.now(timezone.utc).date().isoformat() for row in rows: row["_snapshot_date"] = period reason = ("Amazon Customer Feedback returned no review trends for this marketplace. " "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly." if asins else FEEDBACK_NO_ASINS) marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, "unresolved_metrics": unresolved, "repeated_topics": repeated, "empty_reason": reason if not rows else ""} count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES, "_snapshot_date", period, marker=marker, snapshot=True) _record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({ "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved, "repeated_topics": repeated})) return count BROWSE_NODE_TOPIC_METRICS = ( ("browse_node_metrics_occurrence_percentage_all_products", ("occurrencePercentage", "allProducts")), ("browse_node_metrics_occurrence_percentage_top_twenty_five_percent_products", ("occurrencePercentage", "topTwentyFivePercentProducts")), ("browse_node_metrics_star_rating_impact_all_products", ("starRatingImpact", "allProducts")), ("browse_node_metrics_star_rating_impact_top_twenty_five_percent_products", ("starRatingImpact", "topTwentyFivePercentProducts")), ) BROWSE_NODE_TOPIC_TYPES = { **dict.fromkeys(("browse_node_id", "browse_node_name", "marketplace_id", "sort_by", "sentiment", "topic", "date_range_start_date", "date_range_end_date", "review_snippet_1", "review_snippet_2", "review_snippet_3", "our_asins"), "VARCHAR"), **dict.fromkeys((name for name, _path in BROWSE_NODE_TOPIC_METRICS), "DOUBLE"), **{name: kind for name, kind in _SUBTOPIC_TYPES.items() if "_number_of_mentions" not in name}, "our_asin_count": "BIGINT", } def _browse_node_topic_evidence(records, nodes, marketplace): """Count raw category topics per node and sort and retain their metrics before the row parser runs.""" sorts = _client().FEEDBACK_SORTS expected = {(node, sort) for node in nodes for sort in sorts} counts, metrics, absent, seen = {}, {}, [], set() metricless = 0 for record in records: node, sort = str(record.get("browseNodeId")), record.get("_sort_by") if (node, sort) not in expected or (node, sort) in seen or record.get("marketplaceId") != marketplace: raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign browse node or marketplace") seen.add((node, sort)) no_content = record.get("_no_feedback_reason") topics = record.get("topics") if not no_content and not isinstance(topics, dict): raise ValueError("Amazon getBrowseNodeReviewTopics returned no valid topics object") total = 0 for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): group = None if no_content else topics.get(key) group = [] if group is None else group if not isinstance(group, list): raise ValueError("Amazon getBrowseNodeReviewTopics returned an invalid sentiment topic list") counts[json.dumps([node, sort, sentiment], separators=(",", ":"))] = len(group) total += len(group) for topic in group: if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip() or not isinstance(topic.get("browseNodeMetrics", {}), (dict, type(None)))): raise ValueError("Amazon category review topic requires a topic name and a browseNodeMetrics object or none") metricless += not isinstance(topic.get("browseNodeMetrics"), dict) identity = json.dumps([node, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":")) if identity in metrics: raise ValueError("Amazon returned duplicate topics for one browse node and sentiment") values = [] for _name, path in BROWSE_NODE_TOPIC_METRICS: value = topic.get("browseNodeMetrics") for segment in path: if value is not None and not isinstance(value, dict): raise ValueError("Amazon category review topic returned invalid nested metrics") value = value.get(segment) if value is not None else None values.append(_feedback_number(value)) metrics[identity] = values if not total: absent.append({"browse_node_id": node, "sort_by": sort, "code": "NO_CONTENT" if no_content else "NO_TOPICS", "reason": no_content or "Amazon returned no positive or negative category topics for this browse node and marketplace."}) if seen != expected: raise ValueError("Amazon Customer Feedback did not return a response for every browse node") return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics, "marketplace_id": marketplace, "absent_browse_nodes": absent} def browse_node_topic_rows(records, document, our_asins): """Flatten each browse node's category topics with the tenant's own ASINs in that node.""" rows = [] for record in records: node = str(record["browseNodeId"]) dates = record.get("dateRange") if isinstance(record.get("dateRange"), dict) else {} ours = sorted(our_asins.get(node, {}).get("asins", ())) for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")): for topic in (record.get("topics") or {}).get(key) or []: row = dict.fromkeys(BROWSE_NODE_TOPIC_TYPES) row.update(browse_node_id=node, marketplace_id=record["marketplaceId"], sort_by=record["_sort_by"], browse_node_name=record.get("displayName") or our_asins.get(node, {}).get("name"), sentiment=sentiment, topic=topic["topic"], date_range_start_date=str(dates.get("startDate") or "")[:10] or None, date_range_end_date=str(dates.get("endDate") or "")[:10] or None, our_asins=json.dumps(ours, separators=(",", ":")), our_asin_count=len(ours)) for name, path in BROWSE_NODE_TOPIC_METRICS: value = topic.get("browseNodeMetrics") for segment in path: value = value.get(segment) if isinstance(value, dict) else None row[name] = value for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1): row[f"review_snippet_{index}"] = snippet row.update(_subtopic_columns(topic, mentions=False)) rows.append(_origin(row, document, len(rows) + 1, "getBrowseNodeReviewTopics")) return rows def _sync_browse_node_review_topics(con, fields): """Replace the category topic snapshot of every browse node the active child ASINs sit in, both sorts.""" asins = _feedback_asins(con) marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID located = _client().item_browse_nodes(fields, asins) nodes, unplaced = {}, [] for asin in asins: node = located.get(asin) or {} if node.get("browse_node_id"): entry = nodes.setdefault(node["browse_node_id"], {"name": node.get("browse_node_name"), "asins": set()}) entry["asins"].add(asin) else: unplaced.append({"asin": asin, "code": "NO_BROWSE_NODE", "reason": node.get("_no_feedback_reason") or "Amazon returned no browse node for this ASIN."}) records = _sorted_feedback(_client().browse_node_review_topics(fields, sorted(nodes))) unresolved = _null_unresolved_metrics(records) repeated = _drop_repeated_topics(records) evidence = _browse_node_topic_evidence(records, sorted(nodes), marketplace) rows = browse_node_topic_rows(records, "operation:" + uuid.uuid4().hex, nodes) period = datetime.now(timezone.utc).date().isoformat() for row in rows: row["_snapshot_date"] = period reason = (FEEDBACK_NO_ASINS if not asins else "Amazon returned no browse node for any active child ASIN. The per-ASIN reasons are recorded in the sync evidence." if not nodes else "Amazon Customer Feedback returned no category review topics for these browse nodes. " "The per-node reasons are recorded in the sync evidence. Amazon refreshes this data weekly.") marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, "absent_asins": unplaced, "unresolved_metrics": unresolved, "repeated_topics": repeated, "empty_reason": reason if not rows else ""} count = _replace_period(con, "browse_node_review_topics", rows, sorted(set(BROWSE_NODE_TOPIC_TYPES) | {key for row in rows for key in row}), "_snapshot_date", period, marker=marker, snapshot=True) _record_state(con, entity_key("limit_browse_node_review_topics"), "done", detail=json.dumps({ "reason": marker["empty_reason"], "absent_asins": unplaced, "absent_browse_nodes": evidence["absent_browse_nodes"], "unresolved_metrics": unresolved, "repeated_topics": repeated})) return count def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations=None, rt=None): """The five money tables share the outer sync's budget, errors and completion markers.""" cx = _client() spec = cx.REPORT_TYPES[table] if table == "review_topics": return _sync_review_topics(con, fields) if table == "review_trends": return _sync_review_trends(con, fields) if table == "browse_node_review_topics": return _sync_browse_node_review_topics(con, fields) if table == "economics": start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat() if start >= end: return 0 inclusive_end = (_utc(end + "T00:00:00Z") - timedelta(days=1)).date().isoformat() text, meta = cx.economics_document(fields, start, inclusive_end, max_wait=max_wait) records = economics_records(text) rows = economics_rows(records, meta["reportDocumentId"]) if any(not start <= str(row.get("start_date", "")) < end for row in rows): raise ValueError("Economics returned a row outside the requested date window") # Every request is reconciled. This includes the newest window and ensures a resumed # first-load pass cannot leave its only proof tied to an older period. text, _summary_meta = cx.economics_document( fields, start, inclusive_end, summary=True, max_wait=max_wait) totals = economics_summary(economics_records(text)) marker = {"start": start, "end": end, "totals": totals} return _replace_period(con, table, rows, sorted({k for r in rows for k in r}), "start_date", start, end, marker) if table == "transactions": clock = datetime.now(timezone.utc) # Calendar years, plus a day: the frozen first-load cohort drifts across this vendor floor. floor = clock.replace(year=clock.year - 2, day=min(clock.day, 28) if clock.month == 2 else clock.day) + timedelta(days=1) if w0 < floor: _record_state(con, "amzn_limit_transactions", "done", detail=json.dumps({"reason": f"Amazon Finance exposes the last two years; history before {floor.date().isoformat()} is unavailable. " "Keep the connection enabled for newer transactions; request older statements from Amazon."})) if w1 <= floor: return 0 # The fully older window is closed without a vendor call. w0 = max(w0, floor) start, end = _iso(w0), _iso(w1) transactions = cx.transactions(fields, _iso(w0 - timedelta(seconds=1)), end) # Finances date endpoints can be inclusive; the mirror uses half-open UTC windows. transactions = [t for t in transactions if _utc(w0) <= _utc(t["postedDate"]) < _utc(w1)] rows = transaction_rows(transactions, "operation:" + uuid.uuid4().hex) marker = {"start": start, "end": end, "totals": transaction_summary(transactions)} return _replace_period(con, table, rows, sorted({k for r in rows for k in r}), "posted_date", start, end, marker) if spec.get("api"): asins = _catalog_walk_asins(con) if table == "catalog_items" else [] if table == "catalog_items": _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({ "reason": (f"Amazon Catalog: {len(asins)} ASINs in {(len(asins) + 19) // 20} batches. " "Limit: 20 ASINs per search and 2 requests per second. " "Every page is followed; keep the connection enabled."), })) if table == "order_metrics": w0 = _utc(w0).replace(hour=0, minute=0, second=0, microsecond=0) w1 = _utc(w1).replace(hour=0, minute=0, second=0, microsecond=0) if w0 >= w1: return 0 start, end = (_iso(w0), _iso(w1)) if w0 is not None else (None, None) records = (cx.fee_estimates(fields, _fee_estimate_inputs(con, fields)) if table == "fee_estimates_api" else cx.outbound_fulfillment_orders(fields) if table == "outbound_fulfillment" else cx.inbound_shipment_items(fields) if table == "inbound_shipments" else cx.replenishment_metrics(fields) if table == "replenishment" else cx.group_a_records(fields, spec["api"], start=start, end=end, asins=asins)) document = "operation:" + uuid.uuid4().hex rows = (fee_estimate_rows(records, document) if table == "fee_estimates_api" else outbound_fulfillment_rows(records, document) if table == "outbound_fulfillment" else inbound_shipment_rows(records, document) if table == "inbound_shipments" else replenishment_rows(records, document) if table == "replenishment" else listing_item_rows(records, document) if table == "listing_items" else [_origin(_flatten_row(_without_buyer_data(row)), document, i, spec["api"]) for i, row in enumerate(records, 1)]) if table == "catalog_items": parents = catalog_parent_map(records, _pricing_parent_map(con), fields.get("marketplace_id") or cx.US_MARKETPLACE_ID, _economics_parent_map(con)) from harness import amazon_pictures mkt = fields.get("marketplace_id") or cx.US_MARKETPLACE_ID pictures = {str(r.get("asin") or "").strip(): amazon_pictures.main_image(r, mkt) for r in records if isinstance(r, dict)} for row in rows: asin = str(row.get("asin") or "").strip() row["parent_asin"], row["parent_asin_source"] = parents.get(asin, (asin or None, "self")) # The raw `images` JSON column stays; this names the MAIN link `amazon_pictures.fill` stores. row["main_image_url"] = (pictures.get(asin) or {}).get("url") row["main_image_full_url"] = (pictures.get(asin) or {}).get("full_url") _catalog_listing_changes(con, records, mkt, rt=rt) column = "_snapshot_date" if spec.get("snapshot") else "_period_start" period = datetime.now(timezone.utc).date().isoformat() if spec.get("snapshot") else start for row in rows: row[column] = period if end: row["_period_end"] = end marker = {"start": period, "end": end or period, "column": column, "count": len(records)} if table == "replenishment": offers, seller = records["offerCount"], len(records["sellingPartnerMetrics"]) if offers != len(records["offerMetrics"]): raise ValueError("Amazon replenishment totalResults does not match its raw offer metrics") marker.update(count=seller + offers if offers else 0, seller_count=seller if offers else 0, offer_count=offers, raw_seller_count=seller, source_start=records["start"], source_end=records["end"]) if table == "outbound_fulfillment": marker.update(_outbound_totals(records)) if table == "inbound_shipments": marker.update(_inbound_totals(records)) if table == "order_metrics": marker["metrics"] = _order_metric_totals(records) marker["range"] = True for row in rows: day = _utc(str(row.get("interval") or "").split("--", 1)[0]) if not w0 <= day < w1: raise ValueError("Amazon order metrics returned an interval outside the requested period") row[column] = _iso(day) columns = {k for r in rows for k in r} if table == "listing_items": columns.add("backend_keywords") if table == "fee_estimates_api": columns.add("estimate_id") if table == "outbound_fulfillment": columns.update(("seller_fulfillment_order_id", "seller_fulfillment_order_item_id", "seller_sku", "quantity")) if table == "inbound_shipments": columns.update(("shipment_item_id", "inbound_plan_id", "shipment_id", "sku", "quantity", "status", "plan_status")) columns.update(("shipment_confirmation_id", "quantity_received")) if table == "replenishment": columns.update(("metric_id", "metric_scope", "sku")) count = _replace_period(con, table, rows, sorted(columns), column, period, end if table == "order_metrics" else None, marker=marker, snapshot=bool(spec.get("snapshot"))) if table == "replenishment": _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({ "reason": cx.REPLENISHMENT_EMPTY if marker["offer_count"] == 0 else ""})) if table == "inbound_shipments": _name_skipped_plans(con, list(getattr(records, "skipped_plans", ())), operations, log) return count kwargs = {} if spec.get("snapshot") else { "start": _iso(w0), "end": _iso(w1 - timedelta(seconds=1))} if spec.get("dated_snapshot"): day_end = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) kwargs = {"start": _iso(day_end - timedelta(days=1)), "end": _iso(day_end - timedelta(seconds=1))} options = dict(spec.get("options") or {}) if spec.get("period"): options["reportPeriod"] = spec["period"] if spec.get("date_options"): options.update(zip(spec["date_options"], (kwargs.pop("start"), kwargs.pop("end")))) if options: kwargs["options"] = options text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log) _DocumentProbe.downloaded(text) parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]} if spec.get("json_root") and meta.get("cancelled") and _blank(text): # ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already # returns an empty document for it, S66-T42), but the JSON parser read "" as a document # with no row root and REFUSED it. So the two B2B opportunity reports never earned a # watermark, the first load never finished, and every restart queued it again. An empty # period is a landed period; a real payload without its root is still refused below. columns, rows = [], [] elif spec.get("json_root"): columns, rows = parse_json_report(text, spec["json_root"], **parser_args) if spec.get("json_extra_roots"): rows = _stamp_rows(rows, {"_json_root": spec["json_root"]}) parts = [rows] for root in spec["json_extra_roots"]: extra_columns, extra = parse_json_report(text, root, **parser_args) parts.append(_stamp_rows(extra, {"_json_root": root})) columns = merge_columns(columns, extra_columns) rows = (_DocumentRows.chain(*parts) if any(isinstance(part, _DocumentRows) for part in parts) else [row for part in parts for row in part]) else: columns, rows = parse_flat_file(text, **parser_args) rows = normalise_app_dates(table, rows) if table in LEGACY_SNAPSHOT_TABLES: return _replace_legacy_snapshot(con, table, columns, rows, meta, document_row_count(text)) if spec.get("snapshot"): stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime") if not stamp: raise ValueError("Amazon snapshot returned no data date or report creation date") period, column = _utc(stamp).date().isoformat(), "_snapshot_date" elif table == "storage_fees": period, column = _utc(w0).strftime("%Y-%m"), "month" else: period, column = _iso(w0), "_period_start" def stamped(row): row[column] = period if w1 is not None: row["_period_end"] = _iso(w1) if table == "storage_fees": row["fulfillment_center"] = (row.get("fulfillment_center") or row.get("fulfillment_center_id") or row.get("fulfilment_center") or "all") return row if table == "recommended_removals": markets = meta.get("marketplaceIds") # Aged stock approves one marketplace. A report scoped to none or several stays unstamped and refuses there. if (isinstance(markets, list) and len(markets) == 1 and isinstance(markets[0], str) and markets[0].strip()): rows = _stamp_rows(rows, {"_marketplace_id": markets[0].strip()}) rows = rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows] columns = sorted(set(columns) | (rows.keys() if isinstance(rows, _DocumentRows) else {k for row in rows for k in row})) roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else () # HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count. empty = bool(roots) and ((meta.get("cancelled") and _blank(text)) or (table in ("b2b_recommended", "b2b_not_yet_on_amazon") and isinstance(json.loads(_document_text(text) or "{}"), dict) and spec["json_root"] not in json.loads(_document_text(text) or "{}"))) marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period, "column": column, "count": 0 if empty else document_row_count(text, roots)} if spec.get("snapshot") and not roots and meta.get("reportDocumentId"): marker["document"] = meta["reportDocumentId"] return _replace_period(con, table, rows, columns, column, period, marker=marker, snapshot=bool(spec.get("snapshot"))) def _stamp_rows(rows, values): """Set `values` on every row, on each pass when the rows are lazy.""" if isinstance(rows, _DocumentRows): return rows.with_values(values) def stamped(row): row.update(values) return row return [stamped(row) for row in rows] def _order_metric_totals(records, *, flattened=False): totals = {"orders": 0, "units": 0, "sales": {}} for row in records: orders = row.get("order_count" if flattened else "orderCount") units = row.get("unit_count" if flattened else "unitCount") money = ({"amount": row.get("total_sales_amount"), "currencyCode": row.get("total_sales_currency_code")} if flattened else row.get("totalSales") or {}) if orders is None or units is None or money.get("amount") is None or not money.get("currencyCode"): raise ValueError("Amazon order metrics omitted its order, unit or sales totals") totals["orders"] += int(orders) totals["units"] += int(units) currency = money["currencyCode"] totals["sales"][currency] = totals["sales"].get(currency, 0.0) + float(money["amount"]) return totals ORDER_LINE_PROOF = "reconcile_order_items_" def _order_report_months(start, end): first = _utc(start).replace(day=1, hour=0, minute=0, second=0, microsecond=0) while first < _utc(end): following = (first.replace(day=28) + timedelta(days=4)).replace(day=1) yield first, following first = following def _order_previous_months(con, orders): if not {"amazon_order_id", "purchase_date"} <= _columns_of(con, "amzn_order_items"): return set() result = con.execute("SELECT DISTINCT strftime(TRY_CAST(purchase_date AS TIMESTAMPTZ), '%Y-%m-01') " "FROM amzn_order_items WHERE amazon_order_id IN (SELECT unnest(?))", [list(orders)]) return {month for (month,) in result.fetchall()} def _order_line_summary(summary, row): status = str(row.get("order_status") or "").strip().lower() if status in ("cancelled", "canceled"): return summary["count"] += 1 origin = f"{row['_report_document_id']}:{row['_report_row_number']}".encode() summary["origin_xor"] = str(int(summary["origin_xor"]) ^ int.from_bytes( hashlib.md5(origin, usedforsecurity=False).digest(), "little")) currency = row.get("currency") if not status or not currency or (summary["currency"] and summary["currency"] != currency): summary["invalid"] = True summary["currency"] = currency try: price = Decimal(str(row.get("item_price"))) if not price.is_finite() or price < 0: raise ValueError("invalid order line price") summary["sales"] = str(Decimal(summary["sales"]) + price) except (InvalidOperation, ValueError): summary["invalid"] = True def _order_report_observations(con, cols, rows, meta, report_type, start, end): """Measure raw report windows before order grouping, typing or insertion.""" spec = _client().REPORT_TYPES["order_items"] required = {"amazon_order_id", "purchase_date", "asin", "order_status", "item_price", "currency"} proof = {"reportType": report_type, "reportDocumentId": meta.get("reportDocumentId"), "start": _iso(start), "end": _iso(end), "months": {}, "touched": set()} if report_type == spec["report"]: for first, following in _order_report_months(start, end): proof["months"][first.date().isoformat()] = { "start": _iso(max(first, start)), "end": _iso(min(following, end)), "asins": {}, "invalid": bool(cols and not required <= set(cols))} orders = set() for row in rows: orders.add(str(row.get("amazon_order_id") or "")) if len(orders) >= 1000: proof["touched"].update(_order_previous_months(con, orders)) orders.clear() try: purchased = _utc(row["purchase_date"]) if not row["purchase_date"]: raise ValueError("missing purchase date") except (KeyError, TypeError, ValueError, AttributeError): proof["touched"].add(None) continue month = purchased.strftime("%Y-%m-01") proof["touched"].add(month) if report_type != spec["report"] or not start <= purchased < end: continue window = proof["months"][month] if not row.get("asin") or not row.get("amazon_order_id"): window["invalid"] = True summary = window["asins"].setdefault(str(row.get("asin") or ""), {"count": 0, "sales": "0", "currency": None, "invalid": False, "origin_xor": "0"}) _order_line_summary(summary, row) proof["touched"].update(_order_previous_months(con, orders)) return proof def _order_report_totals(windows): totals = {} for window in windows: for asin, raw in window["asins"].items(): total = totals.setdefault(asin, {"count": 0, "sales": "0", "currency": None, "invalid": False, "origin_xor": "0"}) total["count"] += raw["count"] total["sales"] = str(Decimal(total["sales"]) + Decimal(raw["sales"])) total["origin_xor"] = str(int(total["origin_xor"]) ^ int(raw["origin_xor"])) total["invalid"] |= raw["invalid"] or bool( raw["count"] and total["currency"] and total["currency"] != raw["currency"]) if raw["count"]: total["currency"] = raw["currency"] return totals def _record_order_report(con, proof): """Publish raw line totals with disjoint purchase-date windows or an explicit invalidation.""" prefix = entity_key(ORDER_LINE_PROOF) touched = proof["touched"] | proof["months"].keys() unknown = None in touched if unknown: touched.update(row[0][len(prefix):].replace("_", "-") + "-01" for row in con.execute( "SELECT entity FROM _sync_state WHERE starts_with(entity, ?)", [prefix]).fetchall()) for month in sorted(touched - {None}): key = prefix + month[:7].replace("-", "_") saved = _read_state(con, key) or {} try: previous = json.loads(saved.get("detail") or "{}") windows = previous.get("windows", []) if not isinstance(windows, list): raise ValueError("invalid order source windows") window = proof["months"].get(month) windows = [old for old in windows if window and (old["end"] <= window["start"] or old["start"] >= window["end"])] except (ValueError, TypeError, KeyError, AttributeError): windows, window = [], proof["months"].get(month) if window and not unknown: windows.append({**window, "reportDocumentId": proof["reportDocumentId"]}) first = _utc(month + "T00:00:00Z") following = (first.replace(day=28) + timedelta(days=4)).replace(day=1) marker = {"version": 1, "source": _client().REPORT_TYPES["order_items"]["validates_against"], "reportType": _client().REPORT_TYPES["order_items"]["report"], "cancelled_excluded": True, "start": month, "end": following.date().isoformat(), "windows": windows, "asins": _order_report_totals(windows), "reportDocumentIds": list(dict.fromkeys(w["reportDocumentId"] for w in windows))} if not windows: marker["reason"] = "Order updates require purchase-month report proof to be refreshed" marker["invalidated_by"] = {key: proof[key] for key in ("reportType", "reportDocumentId", "start", "end")} _record_state(con, key, "done", detail=json.dumps(marker)) def _reset_order_report(con): if _columns_of(con, "amzn_order_items"): con.execute("DELETE FROM amzn_order_items") con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [entity_key(ORDER_LINE_PROOF)]) def _write_order_report_window(con, cols, types, rows, proof, *, reset=False): before = _reset_order_report if reset else None after = lambda target: _record_order_report(target, proof) if not cols: with _report_transaction(con): if before: before(con) after(con) return 0, None return replace_window(con, "amzn_order_items", cols, types, rows, replace_keys_for("order_items"), before=before, after=after) def _order_status_observations(con, rows, meta): """Retain status bounds for the current Sales window in the existing account sync state.""" saved = _read_state(con, entity_key("reconcile_order_metrics")) if not saved or not saved.get("detail"): return None metric = json.loads(saved["detail"]) scope = {"start": metric["start"], "end": metric["end"], "metric_read_at": saved["updated_at"]} previous = _read_state(con, entity_key("reconcile_orders")) proof = json.loads(previous["detail"]) if previous and previous.get("detail") else {} if any(proof.get(key) != value for key, value in scope.items()): proof = {**scope, "orders": {}} columns = _columns_of(con, "amzn_orders") if {"amazon_order_id", "purchase_date", "order_status"} <= columns: channel = " AND sales_channel='Amazon.com'" if "sales_channel" in columns else "" # Completion bounds the old source observation. The watermark itself is a request cutoff. mark = _read_state(con, entity_key(WATERMARK_PREFIX + "orders")) or {} before = mark.get("updated_at") if mark.get("state") == "done" else None updated = "TRY_CAST(last_updated_date AS TIMESTAMPTZ)" if "last_updated_date" in columns else "NULL" for oid, status, last_update in con.execute( "SELECT amazon_order_id, order_status, " + updated + " FROM amzn_orders " "WHERE CAST(purchase_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) " "AND CAST(purchase_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)" + channel, [scope["start"], scope["end"]]).fetchall(): proof["orders"][str(oid)] = { "cancelled_by": before if (str(status).lower() in ("cancelled", "canceled") and before and last_update and _utc(last_update) <= _utc(before)) else None, "active_after": None} received = _iso(datetime.now(timezone.utc)) created = meta.get("createdTime") lower = _iso(_utc(created)) if created and _utc(created) <= _utc(received) else None start, end = _utc(scope["start"]), _utc(scope["end"]) for row in rows: if (not row.get("amazon_order_id") or not row.get("purchase_date") or not start <= _utc(row["purchase_date"]) < end or row.get("sales_channel", "Amazon.com") != "Amazon.com"): continue entry = proof["orders"].setdefault(str(row["amazon_order_id"]), {"cancelled_by": None, "active_after": None}) if str(row.get("order_status") or "").lower() in ("cancelled", "canceled"): prior = entry.get("cancelled_by") entry["cancelled_by"] = min((prior, received), key=_utc) if prior else received else: entry["cancelled_by"] = None prior = entry.get("active_after") if lower: entry["active_after"] = max((prior, lower), key=_utc) if prior else lower proof["last_report"] = {"created_at": lower, "received_at": received, "document_id": meta.get("reportDocumentId")} return proof def _validate_orders(con): """Independent Sales API order count for one completed UTC interval.""" saved = _read_state(con, entity_key("reconcile_order_metrics")) if not saved or not saved.get("detail"): return {"state": "unproven", "reason": "Sync Amazon order metrics to reconcile orders independently"} try: marker = json.loads(saved["detail"]) if not marker.get("count"): return {"state": "unproven", "reason": "Amazon order metrics returned no intervals"} expected = marker["metrics"]["orders"] # getOrderMetrics counts the US marketplace only; the order report also carries CA, MX and MCF orders. columns = {row[0] for row in con.execute( "SELECT column_name FROM information_schema.columns WHERE table_name='amzn_orders'").fetchall()} channel = " AND sales_channel = 'Amazon.com'" if "sales_channel" in columns else "" window = ("FROM amzn_orders WHERE CAST(purchase_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) " "AND CAST(purchase_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)" + channel) cancelled = "lower(COALESCE(order_status, '')) IN ('cancelled', 'canceled')" counted, in_flux = con.execute( "SELECT COUNT(DISTINCT amazon_order_id), " "COUNT(DISTINCT amazon_order_id) FILTER (WHERE lower(COALESCE(order_status, '')) <> 'shipped') " + window + " AND NOT " + cancelled, [marker["start"], marker["end"]]).fetchone() # A watermark is a request cutoff. Only retained source observations bound status transitions. read_at, stored_at = saved.get("updated_at"), read_watermark(con, "orders") read_at = _utc(read_at) if read_at else None proof_state = _read_state(con, entity_key("reconcile_orders")) try: proof = json.loads(proof_state["detail"]) if proof_state and proof_state.get("detail") else {} if not isinstance(proof, dict) or not isinstance(proof.get("orders", {}), dict): raise ValueError("invalid status proof") for entry in proof.get("orders", {}).values(): for key in ("cancelled_by", "active_after"): if entry.get(key): _utc(entry[key]) except (ValueError, TypeError, AttributeError): return {"state": "refused", "expected": expected, "actual": counted, "reason": (f"Amazon Sales counted {expected} orders; stored orders contain {counted} " f"for {marker['start']} to {marker['end']}. Cancellation status evidence is invalid.")} if any(proof.get(key) != value for key, value in { "start": marker["start"], "end": marker["end"], "metric_read_at": saved.get("updated_at")}.items()): proof = {} observed = proof.get("orders", {}) identities = {str(row[0]) for row in con.execute( "SELECT DISTINCT amazon_order_id " + window, [marker["start"], marker["end"]]).fetchall()} missing = len(observed.keys() - identities) revived, uncertain = 0, 0 if read_at is not None: updated = "TRY_CAST(last_updated_date AS TIMESTAMPTZ)" if "last_updated_date" in columns else "NULL" for oid, last_update in con.execute( "SELECT amazon_order_id, MAX(" + updated + ") " + window + " AND " + cancelled + " GROUP BY amazon_order_id", [marker["start"], marker["end"]]).fetchall(): entry = observed.get(str(oid), {}) cancelled_by, active_after = entry.get("cancelled_by"), entry.get("active_after") if cancelled_by and _utc(cancelled_by) <= read_at: continue # An earlier final update bounds cancellation above. A later update proves no transition time. if last_update is not None and _utc(last_update) <= read_at: continue if active_after and _utc(active_after) >= read_at: revived += 1 else: uncertain += 1 actual = counted + revived span = f"for {marker['start']} to {marker['end']}" if missing: return {"state": "refused", "expected": expected, "actual": None if uncertain else actual, "actual_lower": actual, "actual_upper": actual + uncertain, "revived": revived, "uncertain_cancellations": uncertain, "missing_observed_orders": missing, "reason": (f"Amazon Sales counted {expected} orders; stored orders contain {counted} active orders {span}. " f"{missing} previously observed order identities are missing.")} if uncertain: return {"state": "unproven" if observed and actual <= expected <= actual + uncertain else "refused", "expected": expected, "actual": None, "actual_lower": actual, "actual_upper": actual + uncertain, "revived": revived, "uncertain_cancellations": uncertain, "reason": (f"Amazon Sales counted {expected} orders; stored orders contain {counted} active orders {span}. " f"{revived} post-metric cancellations are proved; {uncertain} cancellation times are unknown. " "LastUpdatedDate is not a cancellation transition. No exact adjusted count is proved.")} if expected == actual: return {"state": "ok", "expected": expected, "actual": actual, "revived": revived, "reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"} if (read_at is not None and stored_at is not None and stored_at < read_at and counted - in_flux <= expected < counted): # Stored orders predate the metric read, and only open orders can drop out of it since. return {"state": "unproven", "expected": expected, "actual": actual, "reason": (f"Amazon Sales counted {expected} orders after the stored orders were read; " f"stored orders contain {actual} {span}. The next orders sync reconciles them.")} # Two Amazon reports disagree: a check on the source's own data warns and publishes. return {"state": "refused", "expected": expected, "actual": actual, "revived": revived, "publish": True, "reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"} except Exception: return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"} def _validate_margin(con, table): """Compare stored detail to durable vendor evidence, per currency and latest period.""" state = _read_state(con, entity_key("reconcile_" + table)) if not state or state["state"] != "done" or not state.get("detail"): return {"state": "unproven", "reason": "Amazon reconciliation marker is absent"} try: marker = json.loads(state["detail"]) column = marker.get("column") or ("start_date" if table == "economics" else "posted_date") if column == "_snapshot_date" and marker.get("document"): return _validate_report_snapshot(con, table, marker) # Identifiers come only from this fixed table-kind selection, never stored JSON text. if column not in ("start_date", "posted_date", "_snapshot_date", "month", "_period_start"): raise ValueError("unrecognized reconciliation period column") exact = "count" in marker and not marker.get("range") predicate = f'"{column}" = ?' if exact else f'"{column}" >= ? AND "{column}" < ?' if column == "posted_date": predicate = ('CAST(posted_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) AND ' 'CAST(posted_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)') values = [marker["start"]] if exact else [marker["start"], marker["end"]] result = con.execute(f"SELECT * FROM {table_name(table)} WHERE {predicate}", values) names = [c[0] for c in result.description] rows = [dict(zip(names, row)) for row in result.fetchall()] if table == "review_trends": counts = dict.fromkeys(marker["source_counts"], 0) metrics = {} for row in rows: group = json.dumps([row.get("asin"), row.get("sentiment")], separators=(",", ":")) counts[group] = counts.get(group, 0) + 1 identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"), row.get("topic"), row.get("month")], separators=(",", ":")) metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"), *(row.get(name) for name, _path in _TREND_METRICS)] total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] trends_match = (marker["count"] == len(rows) == len(metrics) == total and counts == marker["source_counts"] and metrics == marker["source_metrics"]) return {"state": "ok" if trends_match else "refused", "expected": marker["count"], "actual": len(rows), "reason": (marker["empty_reason"] if trends_match and not rows else f"Amazon returned {marker['count']} review trend points. Stored {len(rows)} points with per-ASIN, sentiment, topic and month reconciliation.")} if table == "review_topics": counts = dict.fromkeys(marker["source_counts"], 0) metrics = {} for row in rows: group = json.dumps([row.get("asin"), row.get("sort_by"), row.get("sentiment")], separators=(",", ":")) counts[group] = counts.get(group, 0) + 1 identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sort_by"), row.get("sentiment"), row.get("topic")], separators=(",", ":")) metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS] total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] matches = (marker["count"] == len(rows) == len(metrics) == total and counts == marker["source_counts"] and metrics == marker["source_metrics"]) return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows), "reason": (marker["empty_reason"] if matches and not rows else f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")} if table == "browse_node_review_topics": counts = dict.fromkeys(marker["source_counts"], 0) metrics = {} for row in rows: group = json.dumps([row.get("browse_node_id"), row.get("sort_by"), row.get("sentiment")], separators=(",", ":")) counts[group] = counts.get(group, 0) + 1 identity = json.dumps([row.get("browse_node_id"), row.get("marketplace_id"), row.get("sort_by"), row.get("sentiment"), row.get("topic")], separators=(",", ":")) metrics[identity] = [row.get(name) for name, _path in BROWSE_NODE_TOPIC_METRICS] total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] matches = (marker["count"] == len(rows) == len(metrics) == total and counts == marker["source_counts"] and metrics == marker["source_metrics"]) return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows), "reason": (marker["empty_reason"] if matches and not rows else f"Amazon returned {marker['count']} category review topics. Stored {len(rows)} topics with per-node, sort, sentiment and metric reconciliation.")} if table == "fee_estimates_api": expected, actual = marker["count"], len(rows) identities = {row.get("estimate_id") for row in rows} distinct = len(identities - {None, ""}) return {"state": "ok" if expected == actual == distinct else "refused", "expected": expected, "actual": actual, "distinct": distinct, "reason": f"Amazon returned {expected} estimates; stored latest batch has {actual} rows and {distinct} distinct estimates"} if table == "replenishment": expected = {"count": marker["count"], "seller": marker["seller_count"], "offer": marker["offer_count"]} actual = {"count": len(rows), "seller": sum(row.get("metric_scope") == "seller" for row in rows), "offer": sum(row.get("metric_scope") == "offer" for row in rows)} distinct = len({row.get("metric_id") for row in rows} - {None, ""}) total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] matches = expected == actual and distinct == len(rows) == total return {"state": "ok" if matches else "refused", "expected": expected, "actual": actual, "reason": (_client().REPLENISHMENT_EMPTY if matches and not marker["offer_count"] else f"Amazon raw replenishment metrics {expected}; stored metrics {actual}")} if table == "inbound_shipments": expected = marker["shipments"] actual = {identity: {"count": 0, "quantity": 0} for identity in expected} sku_quantities = {identity: {} for identity in expected} for row in rows: identity = json.dumps([row.get("inbound_plan_id"), row.get("shipment_id")], separators=(",", ":")) sku, quantity = row.get("sku"), row.get("quantity") if not isinstance(sku, str) or not sku or type(quantity) is not int or quantity < 1: raise ValueError("stored inbound item has an invalid seller SKU or quantity") group = actual.setdefault(identity, {"count": 0, "quantity": 0}) group["count"] += 1 group["quantity"] += quantity skus = sku_quantities.setdefault(identity, {}) skus[sku] = skus.get(sku, 0) + quantity distinct = len({row.get("shipment_item_id") for row in rows} - {None, ""}) total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] matches = marker["count"] == len(rows) == distinct == total and actual == expected if "sku_quantities" in marker: source_skus = marker["sku_quantities"] if not isinstance(source_skus, dict): raise ValueError("invalid inbound seller-SKU evidence") for skus in source_skus.values(): if not isinstance(skus, dict) or any(not sku or type(units) is not int or units < 1 for sku, units in skus.items()): raise ValueError("invalid inbound seller-SKU quantities") matches = matches and sku_quantities == source_skus elif matches and rows: return {"state": "unproven", "expected": marker["count"], "actual": len(rows), "reason": "Amazon legacy inbound marker lacks seller-SKU quantities; sync inbound shipments to reconcile"} return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows), "reason": f"Amazon returned {marker['count']} shipment items; stored {len(rows)} with per-shipment seller-SKU quantity reconciliation"} if not rows: return {"state": "unproven", "reason": "latest reconciliation period is empty"} if "count" in marker: expected, actual = marker["count"], len(rows) matches = expected == actual if table == "order_metrics": expected = {"count": expected, **marker["metrics"]} actual = {"count": actual, **_order_metric_totals(rows, flattened=True)} matches = (expected["count"] == actual["count"] and expected["orders"] == actual["orders"] and expected["units"] == actual["units"] and set(expected["sales"]) == set(actual["sales"]) and all(abs(amount - actual["sales"][currency]) <= 0.01 for currency, amount in expected["sales"].items())) if table == "outbound_fulfillment": totals = {key: sum(int(row.get(amazon_schema._snake(key)) or 0) for row in rows) for key in _OUTBOUND_QUANTITIES} distinct = len({(row.get("seller_fulfillment_order_id"), row.get("seller_fulfillment_order_item_id")) for row in rows}) matches = matches and distinct == actual and totals == marker["quantities"] expected, actual = {"count": expected, "quantities": marker["quantities"]}, {"count": actual, "quantities": totals} elif table == "economics": expected, actual = marker["totals"], {} for row in rows: group = actual.setdefault(row["currency_code"], {"sales": 0.0, "fees": 0.0, "net": 0.0}) pivoted_fees = sum(float(row.get(c.removesuffix("_type_name") + "_amount") or 0) for c in names if c.startswith("fee_") and c.endswith("_type_name") and row.get(c)) if abs(pivoted_fees - float(row["fees_total_amount"])) > 0.010000001: raise ValueError(f"economics fee columns {pivoted_fees} vs row total {row['fees_total_amount']}") for key, col in (("sales", "sales_ordered_product_sales_amount"), ("fees", "fees_total_amount"), ("net", "net_proceeds_total_amount")): group[key] += float(row[col]) matches = _totals_match(expected, actual) else: expected, actual, groups = marker["totals"], {}, {} for row in rows: groups.setdefault(row["transaction_id"], []).append(row) missing_breakdowns = False for transaction_id, items in groups.items(): heads = [r for r in items if r["item_position"] == 0] if len(heads) != 1 or sorted(r["item_position"] for r in items) != list(range(len(items))): raise ValueError(f"transaction {transaction_id} has missing or duplicate items") head = heads[0] if len(items) != head["transaction_item_count"]: raise ValueError(f"transaction {transaction_id} has missing items") if any(r["total_amount"] is not None for r in items if r["item_position"] != 0): raise ValueError(f"transaction {transaction_id} repeats its total on an item") group = actual.setdefault(head["currency_code"], {"amount": 0.0, "count": 0}) group["amount"] += float(head["total_amount"]) group["count"] += 1 parts = [r for r in head if r.startswith("transaction_breakdown_") and r.endswith("_amount") and head[r] is not None] missing_breakdowns |= not bool(parts) if parts: amount = sum(float(head[c]) for c in parts) if abs(amount - float(head["total_amount"])) > 0.010000001: return {"state": "refused", "reason": f"transaction {transaction_id}: " f"breakdowns {amount} vs Amazon total {head['total_amount']}"} if any(head[c + "_currency_code"] != head["currency_code"] for c in parts): raise ValueError("transaction breakdown currency differs from its total") for col in parts: name = col.removeprefix("transaction_breakdown_").removesuffix("_amount") children = [c for c in head if c.startswith(f"transaction_sub_breakdown_{name}__") and c.endswith("_amount") and head[c] is not None] if children and (abs(sum(float(head[c]) for c in children) - float(head[col])) > 0.010000001 or any(head[c + "_currency_code"] != head["currency_code"] for c in children)): raise ValueError(f"transaction {transaction_id}: second-level breakdowns disagree with {col}") matches = _totals_match(expected, actual) if matches and missing_breakdowns: return {"state": "unproven", "reason": "Amazon omitted transaction breakdowns", "expected": expected, "actual": actual} return {"state": "ok" if matches else "refused", "expected": expected, "actual": actual, "reason": f"Amazon summary {expected}; stored detail {actual}"} except Exception as exc: return {"state": "refused", "reason": f"Amazon reconciliation failed: {exc}"} def _totals_match(expected, actual): return bool(expected) and set(expected) == set(actual) and all( set(expected[currency]) == set(actual[currency]) and all( abs(float(value) - float(actual[currency][key])) <= (0 if key == "count" else 0.010000001) for key, value in totals.items()) for currency, totals in expected.items()) def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, max_wait=900, *, scheduled=False, first_load=False, report_budget=None, refresh_loaded=False, slot_pass=False): """Publish a whole Amazon source after its requested work completes.""" rt = rt if rt is not None else _runtime_for(tenant_key) entries = account_entries(tenant_key, rt) admitted = False for entry in entries: try: _sync_credential(tenant_key, rt, connection_id=entry["connection_id"]) admitted = True except Exception: continue if not admitted: raise RuntimeError("Amazon sync requires a stored production amazon_sp credential; SANDBOX credentials cannot sync") if (tables is None and os.environ.get("AIOS_AMAZON_BRAND_ANALYTICS") != "1" and os.environ.get("AIOS_PROCESS_ROLE") != "worker"): # Large weekly reports belong to the dedicated sync worker. tables = [name for name in TABLES if name != "brand_analytics"] tables = None if tables is None else list(tables) options = dict(tables=tables, since=since, log=log, rt=rt, max_wait=max_wait, scheduled=scheduled, first_load=first_load, report_budget=report_budget, refresh_loaded=refresh_loaded, slot_pass=slot_pass) if getattr(datastore._SYNC_TLS, "candidate", None) is not None: return _sync_accounts_candidate(tenant_key, **options) report = {} expected = [name for name in (TABLES if tables is None else tables) if name in TABLES] if scheduled and not first_load and expected and "settlements" not in expected: expected.append("settlements") _PUBLISHED.flag = False # S82-T14: each new createReport draws from the store's per-seller ledger, which the first-load job shares. with _report_ledger(rt), _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected): report.update(_sync_accounts_candidate(tenant_key, **options)) _refuse_unreconciled(tenant_key, rt, report, log) if getattr(_PUBLISHED, "flag", False): _fill_catalog_pictures(tenant_key, rt, report, log) # S82-T11: App callbacks never run inside the pass. The runner drains this queue after it returns. _queue_after_sync(tenant_key, report, log, rt) # A weekly history repair publishes each week on its own, and each publication queues its callbacks. for table, row in list((report.get("tables") or {}).items()): week = row.get("nextWeek") if isinstance(row, dict) else None if not week: continue following = sync(tenant_key, tables=[table], since=_utc(week), log=log, rt=rt, max_wait=max_wait) report["tables"][table] = (following.get("tables") or {}).get(table, row) for key in ("problems", "audit_problems"): report.setdefault(key, []).extend(following.get(key) or []) return report #: Set when this thread's sync pass installs a published snapshot. _PUBLISHED = threading.local() def _note_published(): _PUBLISHED.flag = True def _synced_tables(report): """The tables this pass landed: state `done` with a row count.""" return [table for table, row in (report.get("tables") or {}).items() if isinstance(row, dict) and row.get("state", "done") == "done" and "rows" in row] def _catalog_picture_items(tenant_key, rt): """`(asin, main_image_url, _report_document_id)` rows of the latest published catalog snapshot. None when the mirror or the `main_image_url` column is absent: an older snapshot must not empty the index. """ import duckdb path = _tenant_mirror_path(tenant_key, rt) if not path.exists(): return None try: shared = Path(datastore.DB_PATH).resolve() == path.resolve() except Exception: # noqa: BLE001 shared = False # The process may hold the writer on this file: a second read-only connect would refuse. cur = datastore.ro_cursor() if shared else duckdb.connect(str(path), read_only=True) try: table = table_name("catalog_items") columns = {r[0] for r in cur.execute( "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()} if not {"asin", "main_image_url", "_snapshot_date"} <= columns: return None document = '"_report_document_id"' if "_report_document_id" in columns else "NULL" full_image = 'CAST(main_image_full_url AS VARCHAR)' if "main_image_full_url" in columns else "NULL" return cur.execute( f'SELECT CAST(asin AS VARCHAR), CAST(main_image_url AS VARCHAR), {document}, {full_image} FROM "{table}" ' f'WHERE "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM "{table}")').fetchall() finally: cur.close() def _fill_catalog_pictures(tenant_key, rt, report, log): """Store each catalog ASIN's MAIN picture after a published catalog pass. Never raises.""" try: if "catalog_items" not in _synced_tables(report): return from harness import amazon_pictures items = _catalog_picture_items(tenant_key, rt) if items is None: report["pictures"] = {"state": "skipped", "reason": "the catalog snapshot has no picture links"} else: report["pictures"] = amazon_pictures.fill(rt, items, log=log) log(amazon_pictures.summary_line(report["pictures"])) except Exception as exc: # noqa: BLE001 report["pictures"] = {"state": "failed", "reason": type(exc).__name__} try: log(f"Amazon catalog pictures failed ({type(exc).__name__}); the sync result is unchanged.") except Exception: # noqa: BLE001 pass def _after_sync(tenant_key, report, log): """Contracts C2 and C5: each callback is imported lazily; any failure is one log line and changes nothing.""" synced = tuple(_synced_tables(report)) if not synced: return for target in AFTER_SYNC: _run_after_sync_target(tenant_key, target, synced, log) def _run_after_sync_target(tenant_key, target, synced, log): """One App callback: a lazy import, and any failure is one log line. True when it completed.""" try: import importlib module_name, _, name = target.partition(":") getattr(importlib.import_module(module_name), name)(tenant_key, synced) log(f"Amazon after-sync callback {target} completed") return True except Exception as exc: # noqa: BLE001 log(f"Amazon after-sync callback {target} failed ({type(exc).__name__}); the sync result is unchanged.") return False #: Set when a publication in this process queues callbacks, so the runner drains without waiting. _AFTER_SYNC_WAKE = threading.Event() #: (tenant, target) -> the thread running that App's callback. An App never runs twice at once. _AFTER_SYNC_RUNNING = {} _AFTER_SYNC_LOCK = threading.Lock() #: Held by the one runner loop in this process. _AFTER_SYNC_RUNNER = threading.Lock() #: Claim expiry recovers crashed runners. Budget expiry never releases a live callback's ownership. _AFTER_SYNC_CLAIM_SECONDS = 300 _AFTER_SYNC_RENEW_SECONDS = 60 def _queue_after_sync(tenant_key, report, log, rt): """S82-T11: add this publication's synced tables to every App's queue entry. Runs nothing, never raises.""" synced = _synced_tables(report) if not synced: return stamp = datetime.now(timezone.utc).isoformat() def add(doc): doc["tenant"] = tenant_key apps = doc.setdefault("apps", {}) for target in AFTER_SYNC: entry = apps.setdefault(target, {"tables": [], "queuedAt": stamp}) entry["tables"] = list(entry.get("tables") or []) + [ table for table in synced if table not in (entry.get("tables") or [])] try: rt.update(AFTER_SYNC_QUEUE_KEY, add, flush="sync") _AFTER_SYNC_WAKE.set() except Exception as exc: # noqa: BLE001 try: log(f"Amazon after-sync queue could not be saved ({type(exc).__name__}); the sync result is unchanged.") except Exception: # noqa: BLE001 pass def _claim_after_sync(rt, target): """Claim pending or expired work durably, returning the committed claim or None.""" token = uuid.uuid4().hex def take(doc): entry = (doc.get("apps") or {}).get(target) if not isinstance(entry, dict): return previous = entry.get("claim") or {} now = time.time() if previous and previous.get("expiresAt", 0) > now: return tables = list(dict.fromkeys(list(previous.get("tables") or []) + list(entry.get("tables") or []))) if not tables: return entry["claim"] = {"id": token, "tables": tables, "expiresAt": now + _AFTER_SYNC_CLAIM_SECONDS} entry["tables"] = [] doc = rt.update(AFTER_SYNC_QUEUE_KEY, take, flush="sync") claim = (((doc or {}).get("apps") or {}).get(target) or {}).get("claim") or {} # Read the committed token because the store can replay the mutation after a write conflict. return dict(claim) if claim.get("id") == token else None def _renew_after_sync(rt, target, token): """Extend ownership only if this token still owns the durable claim.""" def renew(doc): claim = ((doc.get("apps") or {}).get(target) or {}).get("claim") or {} if claim.get("id") == token: claim["expiresAt"] = time.time() + _AFTER_SYNC_CLAIM_SECONDS doc = rt.update(AFTER_SYNC_QUEUE_KEY, renew, flush="sync") claim = (((doc or {}).get("apps") or {}).get(target) or {}).get("claim") or {} return claim.get("id") == token def _ack_after_sync(rt, target, token): """Acknowledge this claim without discarding publications queued during its callback.""" def ack(doc): apps = doc.get("apps") or {} entry = apps.get(target) or {} if (entry.get("claim") or {}).get("id") != token: return entry.pop("claim", None) if not entry.get("tables"): apps.pop(target, None) rt.update(AFTER_SYNC_QUEUE_KEY, ack, flush="sync") def _run_after_sync_claim(tenant_key, target, claim, rt, log): """Renew a live callback's claim and acknowledge success or terminal callback failure.""" warned = False def maintenance(fn): nonlocal warned try: return fn(rt, target, claim["id"]) except Exception as exc: # noqa: BLE001 if not warned: warned = True try: log(f"Amazon after-sync queue maintenance refused for {target} ({type(exc).__name__}). " "Unacknowledged tables remain recoverable.") except Exception: # noqa: BLE001 pass return None if maintenance(_renew_after_sync) is not True: return callback = threading.Thread(target=_run_after_sync_target, args=(tenant_key, target, tuple(claim["tables"]), log), daemon=True, name=f"amazon-after-sync callback {target}") callback.start() owned = True while callback.is_alive(): callback.join(_AFTER_SYNC_RENEW_SECONDS) if callback.is_alive() and owned: if maintenance(_renew_after_sync) is False: owned = False # Callback failures are terminal, so they cannot create an unbounded retry loop. if owned: maintenance(_ack_after_sync) def drain_after_sync(tenant_key, rt=None, log=print, *, budget_minutes=None): """S82-T11: run each queued App callback on its own thread under its own minute budget. The runner calls this, never a sync pass, so a callback reads published rows only. Returns `{target: state}`. """ if getattr(datastore._SYNC_TLS, "candidate", None) is not None: raise RuntimeError("Amazon after-sync callbacks never run inside a sync pass") key = _tenant_key(tenant_key) rt = rt if rt is not None else _runtime_for(key) if rt is None: return {} pending = (rt.get(AFTER_SYNC_QUEUE_KEY, fresh=True) or {}).get("apps") or {} states = {} for target in AFTER_SYNC: if target not in pending: continue with _AFTER_SYNC_LOCK: running = _AFTER_SYNC_RUNNING.get((key, target)) if running is not None and running.is_alive(): # Its entry stays queued, so the tables of this publication reach the App on its next run. states[target] = "running" continue claim = _claim_after_sync(rt, target) if claim is None: states[target] = "running" continue budget = budget_minutes if budget_minutes is not None else AFTER_SYNC_BUDGETS.get( target, AFTER_SYNC_BUDGET_MINUTES) job = threading.Thread(target=_run_after_sync_claim, args=(key, target, claim, rt, log), daemon=True, name=f"amazon-after-sync {target}") with _AFTER_SYNC_LOCK: _AFTER_SYNC_RUNNING[(key, target)] = job job.start() job.join(max(float(budget), 0.0) * 60) if job.is_alive(): states[target] = "over budget" log(f"Amazon after-sync callback {target} is past its {budget:g} minute budget. " "It keeps running, and the next App starts now.") else: states[target] = "done" return states def run_after_sync_loop(*, every=900, log=print, stop=None, tenants=None): """S82-T11: the one after-sync runner in this process. It drains each tenant's queue, then waits. It holds no writer lease. A second call while a runner lives returns at once. """ if not _AFTER_SYNC_RUNNER.acquire(blocking=False): return try: stop = stop if stop is not None else threading.Event() while not stop.is_set(): _AFTER_SYNC_WAKE.clear() try: if tenants is not None: keys = list(tenants()) else: from harness import runtime as _rt keys = sorted(_rt.known_tenants()) except Exception as exc: # noqa: BLE001 keys = [] log(f"Amazon after-sync runner could not list tenants ({type(exc).__name__}).") for key in keys: if stop.is_set(): break try: drain_after_sync(key, log=log) except Exception as exc: # noqa: BLE001 log(f"Amazon after-sync runner skipped {key} ({type(exc).__name__}). The queue keeps its tables.") _AFTER_SYNC_WAKE.wait(every) finally: _AFTER_SYNC_RUNNER.release() def _refuse_unreconciled(tenant_key, rt, report, log): """Standing rule 8: a table synced in this pass whose reconciliation refuses keeps its previous rows.""" if "accounts" in report: return # Each isolated account was reconciled before it joined the combined candidate. synced = _synced_tables(report) if not synced or report.get("state") in {"partial", "failed", "done"}: return try: checked = validate(tenant_key, rt=rt, log=lambda *_: None) except Exception as exc: # noqa: BLE001 # D-11771: an unavailable check holds each table it would have checked, never the whole source. checked = {table: {"state": "refused", "reason": f"the check could not run ({type(exc).__name__}); " "the table keeps its previous rows until it runs"} for table in synced} con, _path = _connect_tenant_mirror(tenant_key, rt) for table in synced: verdict, warned = checked.get(table) or {}, entity_key("warning_" + table) if verdict.get("state") == "refused" and verdict.get("publish") is True: # A check on the source's own data warns and publishes. It never holds the sync. expected, actual = verdict.get("expected"), verdict.get("actual") warning = (f"Amazon Sales counted {expected:,} orders and the orders report holds {actual:,}. " "The table is published." if type(expected) is int and type(actual) is int else f"{str(verdict.get('reason') or 'Amazon reports disagree').rstrip('. ')}. The table is published.") report["tables"][table] = {**report["tables"][table], "warning": warning} _record_state(con, warned, "warning", 0, warning) log(f"{table}: published with a warning: {warning}") continue if verdict.get("state") != "refused": # A clean reconcile clears the last warning. A held table keeps it with its kept rows. con.execute("DELETE FROM _sync_state WHERE entity=?", [warned]) continue reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given") report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason} report["problems"].append(f"{table}: {reason}") _record_state(con, entity_key(table), "refused", 0, reason) log(f"{table}: {reason}") def _sync_credential(tenant_key, rt, connection_id=None): """Refuse unusable credentials before storage or vendor work at either sync entry.""" fields = credential(tenant_key, rt, connection_id=connection_id) if not fields: raise RuntimeError( f"tenant `{tenant_key}` has no stored `amazon_sp` credential. Add one under " "Settings, Connectors. There is deliberately no environment fallback: " "AMAZON_SP_* is an operator proving credential, not a tenant's") cx = _client() # ⛔⛔ A SANDBOX CREDENTIAL IS REFUSED HERE, AND THIS IS THE FENCE THAT MAKES THE SANDBOX APP # SAFE TO OWN. Amazon's sandbox returns STATIC MOCK responses. Standing rule 8 forbids an # unverifiable aggregate reaching a screen and R3 requires reconciliation to Amazon's OWN # summary -- a mock figure satisfies neither. Worse, once mock rows are written into the mirror # they spawn into `ut_amzn_*` and become indistinguishable from real ones in a grid; there is no # marker a person would see. So the sandbox is allowed for the LIVENESS PROBE and for # `connectors_amazon.py --check`, which prove the client end to end, and it stops there. # ⚠ REFUSED BY NAME, WITH THE REASON, rather than silently syncing nothing -- an empty table is # exactly what a caller would misread as "Amazon has no data". if cx.is_sandbox(fields): raise RuntimeError( "this is a SANDBOX credential and sync refuses it. The sandbox returns static mock " "responses, and mock rows in ut_amzn_* are indistinguishable from real ones once they " "reach a grid (standing rule 8, and R3's reconciliation). Use it to prove the client " "(`connectors_amazon.py --check`) and sync only a PRODUCTION credential.") return fields def _summarize_account_table(row, total_accounts): """Sum all reported resync progress while retaining each account's window and safe failure details.""" accounts = row["accounts"] progress = [r["resync"] for r in accounts.values() if isinstance(r.get("resync"), dict)] if progress: row["resync"] = {key: sum(p.get(key, 0) for p in progress) for key in ("landed", "left")} if total_accounts > 1: complete = sum(_mirrors._table_sync_complete("amazon", r) for r in accounts.values()) summary = [f"{len(accounts)} of {total_accounts} stores reported", f"{complete} complete, {total_accounts - complete} incomplete"] if progress: summary.append(f"resync: {row['resync']['landed']} windows landed, {row['resync']['left']} left") row["window"] = "; ".join(summary) def _completed_account_tables(con, result, tables): """Retain this account's completed first-load tables when a later pass only reports its remaining tasks.""" if result.get("state") != "done": return for table in TABLES if tables is None else tables: if table not in TABLES or table in result["tables"]: continue state = _read_state(con, entity_key(table)) if not state or state["state"] != "done": continue columns = _columns_of(con, table_name(table)) if not columns: continue result["tables"][table] = {"state": "done", "rows": con.execute( f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0], "columns": len(columns), "provenance": amazon_schema.PROVENANCE[table], "reason": "This connection completed this table in an earlier first-load pass"} def _sync_accounts_candidate(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, max_wait=900, *, scheduled=False, first_load=False, report_budget=None, refresh_loaded=False, slot_pass=False): """Attempt every admitted account before publishing one combined source.""" rt = rt if rt is not None else _runtime_for(tenant_key) entries = account_entries(tenant_key, rt) restored = restore_mirror(tenant_key, rt, log=log) con, _path = _connect_tenant_mirror(tenant_key, rt) changed = _adopt_legacy_account(con, entries[0]) report = {"tenant": tenant_key, "tables": {}, "accounts": {}, "started": _iso(_utc()), "problems": list(restored["problems"]), "audit_problems": []} try: for entry in entries: cid = entry["connection_id"] try: with _account_workspace(con, tenant_key, rt, entry) as scope, \ _client().rate_limit_scope(tenant_key, cid): _sync_credential(tenant_key, rt, connection_id=cid) result = _sync_candidate(tenant_key, tables=tables, since=since, log=log, rt=rt, max_wait=max_wait, scheduled=scheduled, first_load=first_load, report_budget=report_budget, refresh_loaded=refresh_loaded, slot_pass=slot_pass) if first_load: _completed_account_tables(scope["root"], result, tables) _refuse_unreconciled(tenant_key, rt, result, log) changed.update(_merge_account(con, scope)) report["accounts"][cid] = {"store": entry["label"], **result} except Exception as exc: # Credential and vendor exception text can contain source secrets. report["accounts"][cid] = {"store": entry["label"], "state": "failed", "tables": {}, "problems": [f"Amazon account sync failed ({type(exc).__name__})"]} for cid, result in report["accounts"].items(): for key in ("problems", "audit_problems"): report[key].extend(result.get(key, ())) for table, row in result.get("tables", {}).items(): aggregate = report["tables"].setdefault(table, {**row, "rows": 0, "accounts": {}}) aggregate["accounts"][cid] = row aggregate["rows"] += row.get("rows") or 0 if not _mirrors._table_sync_complete("amazon", row): aggregate.update({k: v for k, v in row.items() if k != "rows"}) aggregate["state"] = row.get("state") or "failed" expected = list(TABLES if tables is None else tables) for table in expected: row = report["tables"].setdefault(table, {"rows": 0, "state": "failed", "accounts": {}}) if any(cid not in row["accounts"] for cid in (e["connection_id"] for e in entries)): row.update(state="failed", reason="An Amazon account did not complete this table") for row in report["tables"].values(): _summarize_account_table(row, len(entries)) if first_load: states = [r.get("state") for r in report["accounts"].values()] report.update(state="failed" if "failed" in states else "partial" if "partial" in states else "done", remaining=sum(r.get("remaining", 0) for r in report["accounts"].values()), landedTasks=sum(r.get("landedTasks", 0) for r in report["accounts"].values()), etaSeconds=sum(r.get("etaSeconds", 0) for r in report["accounts"].values())) waits = [r["nextDueSeconds"] for r in report["accounts"].values() if r.get("nextDueSeconds")] if waits: report["nextDueSeconds"] = min(waits) _refresh_account_status(con, entries, rt) _restore_first_load_wait(tenant_key, con) report["available"] = all(result.get("available", False) for result in report["accounts"].values()) if first_load: progress = first_load_status(tenant_key, rt) or {} report["firstLoad"] = {**progress, "state": report["state"]} report["remaining"] = progress.get("remaining", report["remaining"]) refresh_sku_keywords(con) persist_mirror(tenant_key, rt, con, changed, report=report, log=log) report["finished"] = _iso(_utc()) return report finally: con.close() def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, max_wait=900, *, scheduled=False, first_load=False, report_budget=None, refresh_loaded=False, slot_pass=False): """Pull into private progress; refresh loaded tables before bounded history and waits.""" rt = rt if rt is not None else _runtime_for(tenant_key) fields = _sync_credential(tenant_key, rt) restored = restore_mirror(tenant_key, rt, log=log) cx = _client() want = [t for t in (TABLES if tables is None else tables) if t in TABLES] if scheduled and not first_load and want and "settlements" not in want: want.append("settlements") end = datetime.now(timezone.utc) report = {"tenant": tenant_key, "tables": {}, "started": _iso(end), "problems": list(restored["problems"]), "audit_problems": []} enablement = read_database_enablement(rt) report["available"] = enablement["available"] for table in want: if not enablement["databases"][amazon_schema.DATABASE_PREFIX + table]: reason = ("database disabled; re-enable it in Settings, Connectors to resume; existing data preserved" if enablement["available"] else "enablement document malformed or unreadable; capability unavailable; " "repair amazon_connector_enablement with known database keys and boolean values " "and restore tenant-store access; existing data preserved") report["tables"][table] = {"state": "disabled", "reason": reason} log(f"{table}: SKIPPED: {reason}") if not enablement["available"]: report["problems"].append(f"{table}: {reason}") want = [table for table in want if table not in report["tables"]] if not want: report["finished"] = _iso(datetime.now(timezone.utc)) return report con, _path = _connect_tenant_mirror(tenant_key, rt) try: obligation = retention_check(tenant_key, rt=rt, raise_overdue=True) except Exception: con.close() raise if obligation["earliestTriggerAt"]: con.close() raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data") load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) # Table completion evidence also admits empty histories behind a stale aggregate marker. completed, repairs, invalid_proofs, reopened = set(), set(), set(), set() needs_first_load(tenant_key, rt=rt, completed_tables=completed, repair_tables=repairs, invalid_proofs=invalid_proofs, reopened_tables=reopened) prior_refusals = {row["table"]: row for row in _refused_reports(con)} exhausted = {table for table in want if prior_refusals.get(table, {}).get("state") == "failed"} # D-17409: only a first-load job pulls history now. With none serving this tenant, a table the first # load owes refreshes from its last refresh, else the frozen first-load end, and keeps its history owed. catch_up = {} def first_load_served(): if first_load_job_holds(rt): return True try: # A first-load pass rewrites this record, so a recent stamp means a producer is running. if (end - _utc(load["updated_at"])).total_seconds() <= FIRST_LOAD_IDLE_SECONDS: return True except (KeyError, TypeError, ValueError, AttributeError, OverflowError): pass chain = _doc(rt, FIRST_LOAD_GENERATIONS_KEY) try: idle = (end - _utc(chain["updatedAt"])).total_seconds() return int(chain.get("remaining") or 0) > 0 and idle <= FIRST_LOAD_IDLE_SECONDS except (KeyError, TypeError, ValueError, AttributeError, OverflowError): return False if (scheduled and not first_load and not refresh_loaded and load and load["state"] != "done" and _store_documents(rt) and not first_load_served()): try: cohort = min(_utc(load["detail"]), end) if load.get("detail") else None except (TypeError, ValueError, AttributeError, OverflowError): cohort = None cohort = cohort or end - timedelta(days=FIRST_LOAD_RECENT_DAYS) for table in want: if (read_watermark(con, table) or table in completed or table in FEEDBACK_TABLES or table == "settlements"): continue landed = (_read_state(con, entity_key(CATCH_UP_PREFIX + table)) or {}).get("detail") try: catch_up[table] = min(_utc(landed), end) if landed else cohort except (TypeError, ValueError, AttributeError, OverflowError): catch_up[table] = cohort if catch_up: log(f"no first-load job serves {tenant_key}: {', '.join(catch_up)} refresh from their last " "landing; their history stays owed to the first-load job") def current_table(table): # A cadence wait can reuse completed rows, never a failed/bounded checkpoint # or a legacy month whose completion proof still needs repair. return (table not in repairs and table not in invalid_proofs and (_read_state(con, entity_key(table)) or {}).get("state") == "done" and bool(read_watermark(con, table) or table in completed or (table in catch_up and _read_state(con, entity_key(CATCH_UP_PREFIX + table))))) if first_load: # D-11622: each loaded table's own floor decides its refresh, whatever `refresh_loaded` says. loaded = [table for table in want if table not in exhausted and prior_refusals.get(table, {}).get("state") != "not_offered" and (table in completed or (read_watermark(con, table) and table not in repairs))] due_loaded = [] for table in loaded: history = table == "brand_analytics" and table in reopened hold = _loaded_refresh_hold(con, table, end, slot_pass=slot_pass, history=history) if hold is None: if not history: due_loaded.append(table) continue next_due, reason = hold report["tables"][table] = {"state": "deferred", "reason": reason, "nextDue": _iso(next_due), "current": current_table(table)} log(f"{table}: DEFERRED: {reason}") if history: want.remove(table) if due_loaded: # Refreshes use their own floors; the history partition retains its bounded budget. con.close() refreshed = _sync_candidate(tenant_key, tables=due_loaded, log=log, rt=rt, max_wait=max_wait, scheduled=scheduled, refresh_loaded=True, slot_pass=slot_pass) report["tables"].update(refreshed.get("tables") or {}) for key in ("problems", "audit_problems"): report[key].extend(refreshed.get(key) or []) con, _path = _connect_tenant_mirror(tenant_key, rt) # A reopened table refreshed above like any loaded table; its history continues in this pass. want = [table for table in want if table not in loaded or table in reopened] due = _restore_first_load_wait(tenant_key, con) if due and due > end and not _client().prefetch_active(): progress = first_load_status(tenant_key, rt) report.update(progress or {"state": "partial", "remaining": 1, "etaSeconds": 0}) report.update(state="partial", landedTasks=0, nextDueSeconds=(due - end).total_seconds(), firstLoad=dict(progress or {}), finished=_iso(end)) con.close() return report # The history partition cannot call the vendor before its durable due time. if scheduled and not first_load and (refresh_loaded or (load and load["state"] != "done")): # The pinned inline producer still calls sync(scheduled=True). Do not let it start an # unbounded history pull between the resident writer's bounded first-load passes. # HOTFIX 2026-09-29 (D-3496): only a table WITHOUT a watermark belongs to the first load. # A watermark is written when a table's history is complete, so its next window is # bounded; deferring it too froze every loaded table behind one refused report. for table in want: # An entity snapshot owes no history, so the first load never holds it. if (table in catch_up or (read_watermark(con, table) and (not refresh_loaded or table not in repairs)) or table in completed or table in FEEDBACK_TABLES or (table == "settlements" and not refresh_loaded)): continue tick = _read_state(con, entity_key("sync_tick_" + table)) reason = ("This table has no recorded sync yet. Its first load is still running." if not tick else "This table's first load is still running. Its previous rows were kept.") report["tables"][table] = _deferred_table(reason, prior_refusals.get(table)) # The planner waits for the next phase on this hold. The card's tick stays as it was. _record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred") if table == SQP_TABLE and tick and tick.get("state") == "throttled": # The first load owns SQP now; a passed SQP deadline must not make it due every minute. _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail")) want = [table for table in want if (read_watermark(con, table) and (not refresh_loaded or table not in repairs)) or table in completed or table in FEEDBACK_TABLES or (table == "settlements" and not refresh_loaded)] # A catch-up table stays in this pass; the planner below decides when it is due. want += [table for table in catch_up if table not in want] if scheduled and not first_load: ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES} # A table standing rule 8 held is asked again at this pass, with its pair, inside Amazon's own floor. pairs = {**SHARED_REPORT_SIBLINGS, **{line: head for head, line in SHARED_REPORT_SIBLINGS.items()}} for table in list(want): mark = _read_state(con, entity_key(table)) or {} if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused") and not _interval_reason(table, ticks.get(table), end)): # The re-asked tick keeps its stamp, so the phase rule decides as the planner does. ticks.update({name: ticks[name] and {**ticks[name], "state": "refused"} for name in (table, pairs.get(table)) if name in want}) for table in want: tick = ticks.get(table) # A queued resync is a repair: only Amazon's own floor and the stagger hold it. if (tick and tick["state"] == "done" and queued_resync(con, table) is not None and read_watermark(con, table) and not _interval_reason(table, tick, end)): ticks[table] = {**tick, "state": "resync"} plan = sync_plan(ticks, want, now=end, slot_pass=slot_pass) want = plan["tables"] for table, reason in plan["deferred"].items(): report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table)) log(f"{table}: DEFERRED: {reason}") elif not first_load: # The vendor's report floor also applies to an operator's Sync now click. for table in list(want): # A Customer Feedback refusal stored nothing, so its weekly floor counts from a pull that landed. if (table in FEEDBACK_TABLES and (_read_state(con, entity_key(table)) or {}).get("state") in ("refused", "failed")): continue reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)), datetime.now(timezone.utc)) if reason: report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table)) want.remove(table) # Heads go first even when the caller listed lines first. Scheduled plans still select one. pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items() if head in want and sibling in want] + want)) handled, noted = set(), set() # D-3031: a mirror that already holds two pulls of a snapshot heals here too, idempotently. changed_tables = set() try: changed_tables.update(_collapse_snapshots(con, log)) except Exception as exc: # noqa: BLE001 report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next " "resync of each snapshot table replaces its earlier copies.") # Backstop: a candidate reused across restarts never passed through the restore heal. try: changed_tables.update(_unquote_mirror_values(con, log)) except Exception as exc: # noqa: BLE001 report["problems"].append(f"Amazon quoted value heal failed ({type(exc).__name__}); the next " "resync of each quoted table writes clean values.") try: changed_tables.update(_collapse_identity(con, log)) except Exception as exc: # noqa: BLE001 report["problems"].append(f"Amazon identity collapse failed ({type(exc).__name__}); the next " "resync of each doubled table replaces its earlier document's rows.") if prepare_brand_analytics(con): changed_tables.add(table_name("brand_analytics")) def write_window(table, cols, types, rows, after=None, order_proof=None, reset=False): if table == "order_items": count, note = _write_order_report_window(con, cols, types, rows, order_proof, reset=reset) elif cx.REPORT_TYPES[table].get("weekly"): count, note = write_week(con, table, cols, types, rows, tenant_key, report, log) else: count, note = replace_window(con, table_name(table), cols, types, rows, replace_keys_for(table), after=after) if note and table not in noted: noted.add(table) report["problems"].append(note) log(note) return count work, plans, paired, contexts = [], {}, {}, {} documents, settlement_remaining, pass_rows = 0, 0, 0 failed, landed = set(), set() if first_load: failed.update(exhausted) pending_reports = set() sqp_asins, sqp_limit = [], None if SQP_TABLE in pending: saved = _read_state(con, SQP_LIMIT_KEY) old_limit = json.loads(saved["detail"]) if saved and saved.get("detail") else {} # Freeze a first-load cohort between passes. A later incremental pass refreshes it # from our own catalogue, never from a separate vendor lookup. sqp_asins = (old_limit.get("asins") if first_load and load and load["state"] != "done" else None) sqp_asins = sqp_asins or _sqp_catalogue(con) spec = cx.REPORT_TYPES[SQP_TABLE] batches = sqp_batches(sqp_asins, spec["asin_option_max_chars"]) sqp_limit = {"asins": sqp_asins, "asinCount": len(sqp_asins), "batchCount": len(batches), "maxChars": spec["asin_option_max_chars"], "maxPerBatch": (spec["asin_option_max_chars"] + 1) // 11, "period": spec["period"], "periods": [], "refusedAsins": old_limit.get("refusedAsins", {}), "errors": old_limit.get("errors", []), "partialMonthReason": "The current partial month is deferred until it closes; Search Query Performance requires a complete calendar month."} weekly_cause_logged, weekly_completed = False, False weekly_pending = None def spend_rows(rows): """A first-load pass past its row bound spends its document budget; the next pass continues.""" nonlocal documents, pass_rows pass_rows += rows if first_load and isinstance(report_budget, int) and pass_rows >= FIRST_LOAD_ROWS_PER_PASS: documents = max(documents, report_budget) def sqp_problem(message): report["problems"].append(message) log(message) sqp_documents = 0 def sqp_request(*args, **kwargs): nonlocal documents, sqp_documents if isinstance(report_budget, int) and documents >= max(0, report_budget): raise SQPBudgetDeferred("Search Query Performance report budget reached; remaining batches continue next pass.") if scheduled and not first_load and sqp_documents >= SQP_PASS_REPORTS: raise SQPPassDeferred(f"Search Query Performance requested {SQP_PASS_REPORTS} reports this pass; " f"remaining batches continue in {SQP_RESUME_SECONDS // 60} minutes.") documents += 1 sqp_documents += 1 _sync_event("amazon.access", tenant_key, SQP_TABLE, {"state": "access", "operation": "report_document"}, rt, report, log) return _report_document(con, SQP_TABLE, *args, **kwargs, max_wait=max_wait, log=log) def pause_sqp(exc): """Stop SQP for this pass only, and leave its tick due after SQP_RESUME_SECONDS.""" failed.add(SQP_TABLE) report["tables"][SQP_TABLE] = {"state": "deferred", "reason": str(exc)} due = datetime.now(timezone.utc) + timedelta(seconds=SQP_RESUME_SECONDS) _record_state(con, entity_key("sync_tick_" + SQP_TABLE), "throttled", detail=_iso(due)) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or SQP_RESUME_SECONDS, SQP_RESUME_SECONDS) def sqp_progress(): sqp_limit["progress"] = _sqp_progress(con, sqp_months, sqp_weeks, sqp_asins, batches, end) try: measure_end = _utc(load["detail"]) if first_load and load and load.get("detail") else end except (ValueError, TypeError, AttributeError, OverflowError): measure_end = end # H2: one getOrderMetrics walk per seller sets every table's history floor (or records why not). _ensure_seller_first_month(con, fields, measure_end, log=log) if first_load: # Freeze the cohort end across passes. Recomputing it would move every window boundary. end = _utc(load["detail"]) if load else _utc(_iso(end)) _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end)) # One floor per table, shared with needs_first_load and the job planner, so the load converges. starts = history_starts(con, pending, end) plans = {table: sync_window(table, None, now=end, floor=starts.get(table)) for table in pending} undocumented = repairs.intersection(pending) floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending} wanted = [table for table in pending if (table not in completed or table in exhausted) and (table == SQP_TABLE or not read_watermark(con, table) or table in undocumented or table in reopened or read_watermark(con, table) < plans[table]["end"])] planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins, starts=starts) for entry in _sqp_owed_tasks(con, _sqp_after_cohort(planned, end, sqp_asins)): task = entry if len(entry) == 4 else (*entry, None) table, w0, w1, batch = task if table == SQP_TABLE: work.append(task) elif _after_cohort(table, w0, end): # A month after the frozen end has its own marker: the backfill floor only moves backwards. if not _cohort_month_done(con, table, w0): work.append(task) else: owed_window = _owed_window(table, w0, w1, floors[table]) if owed_window is not None: work.append((table, *owed_window, batch)) # A failed enrichment after the final report retries completion without fetching again. for table in wanted: if not any(task[0] == table for task in work): work.append((table, None, None, None)) # Reuse a document for head and lines only when BOTH durable cursors agree. for head, sibling in SHARED_REPORT_SIBLINGS.items(): head_tasks = [(w0, w1) for table, w0, w1, _batch in work if table == head] line_tasks = [(w0, w1) for table, w0, w1, _batch in work if table == sibling] if head_tasks and head_tasks == line_tasks: paired[head] = sibling work = [task for task in work if task[0] != sibling] else: for table in pending: mark = read_watermark(con, table) if not mark and since is None and table in catch_up: # An incremental window keeps the table's rows; its history stays the first load's. mark = catch_up[table] if table == "settlements" and since is None: # Empty discovery earns a request cutoff, never a stored source watermark. tick = _read_state(con, entity_key("sync_tick_" + table)) if tick and tick.get("state") == "done": mark = _utc(tick["updated_at"]) elif not mark: day = _read_state(con, entity_key("lookback_day_" + table)) if day and day.get("state") == "done": mark = _utc(day["updated_at"]) if not mark and since is None and table != "settlements" and table in completed: # A local fallback cannot reopen old daily history or invent a stored watermark. recent = end - timedelta(days=FIRST_LOAD_RECENT_DAYS) try: mark = _utc(load["detail"]) if load and load.get("detail") else recent except (ValueError, TypeError, AttributeError, OverflowError): mark = recent if not recent <= mark <= end: mark = recent plan = plans[table] = sync_window(table, mark, now=end, since=since, floor=history_floor(con, table, end), intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass)) if table == SQP_TABLE: # Monthly history precedes every completed week's ASIN batches. start = plan["start"] if since is not None else history_floor(con, table, end) months = list(reversed(_month_windows(start, plan["end"]))) tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches] weekly_tasks = _sqp_week_tasks( _utc(since) if since is not None else history_floor(con, table, end, period="WEEK"), end, batches) owed = [*tasks, *weekly_tasks] if since is None: owed = _sqp_owed_tasks(con, owed) work.extend(owed) if not any(_sqp_period(w0, w1) == "MONTH" for _table, w0, w1, _batch in owed): work.append((table, None, None, None)) else: work.append((table, None, None, None)) resumes = {} for table in pending: saved = _read_state(con, entity_key("resume_" + table)) continuation = json.loads(saved["detail"]) if saved and saved.get("detail") else {} if (continuation and continuation.get("first_load") == first_load and continuation.get("since") == (_iso(_utc(since)) if since is not None else None)): resumes[table] = continuation plans[table] = {**continuation["plan"], "start": _utc(continuation["plan"]["start"]), "end": _utc(continuation["plan"]["end"])} if sqp_limit is not None: def continuation_first(task): _table, w0, w1, batch = task if w0 is None or batch is None: return True marker = _sqp_month(con, w0, w1) retry = marker["retries"].get(batch) requested = retry["asins"] if retry else batch.split() key = _pending_report_key(SQP_TABLE, cx.REPORT_TYPES[SQP_TABLE]["report"], start=_iso(w0), end=_iso(w1 - timedelta(seconds=1)), options={"reportPeriod": marker.get("period", "MONTH"), "asin": " ".join(requested)}) saved = _read_state(con, key) return not (saved and saved.get("detail") and datetime.now(timezone.utc).timestamp() - json.loads(saved["detail"])["createdAt"] < 86400) # Saved report IDs take a bounded slot before new work can spend their budget. sqp_work = iter(sorted((task for task in work if task[0] == SQP_TABLE), key=continuation_first)) work = [next(sqp_work) if task[0] == SQP_TABLE else task for task in work] sqp_limit["weeklyStart"] = (_week_start(end) - timedelta(days=7)).date().isoformat() if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]: sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "") plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since, floor=history_floor(con, SQP_TABLE, end)) # A first load also owes the months that closed after its frozen cohort end. sqp_months = sorted({*_month_windows(plan["start"], plan["end"]), *(_month_windows(plan["end"], datetime.now(timezone.utc)) if first_load else ()), *((w0, w1) for name, w0, w1, _batch in work if name == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "MONTH")}) sqp_weeks = (_week_windows(history_floor(con, SQP_TABLE, end, period="WEEK"), end) if "WEEK" in cx.REPORT_TYPES[SQP_TABLE].get("periods", ()) else []) sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in sqp_months] sqp_progress() sqp_limit["deferredReason"] = ( "Search Query Performance is waiting for ASINs in the listings or pricing mirror. Sync the catalogue first." if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "") _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) history_cut = {} # Each table and period owns its observed floor. for index, (table, task_start, task_end, task_batch) in enumerate(work): if (first_load or table == SQP_TABLE) and table in failed: continue history_key = (table, _history_task_period((table, task_start, task_end, task_batch))) if history_cut.get(history_key) is not None and task_end is not None and task_end <= history_cut[history_key]: continue if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget) and (task_end is not None or table == "settlements" or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)): break tick = _read_state(con, entity_key("sync_tick_" + table)) if first_load and (tick or {}).get("state") == "throttled": reason = _interval_reason(table, tick, datetime.now(timezone.utc)) if reason: report["tables"][table] = {"state": "deferred", "reason": reason} wait = _interval_wait(table, tick, datetime.now(timezone.utc))[0] report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) continue if table == SQP_TABLE and task_start is not None and _sqp_period(task_start, task_end) == "WEEK": weekly_limit = {**sqp_limit, "refusedAsins": {}, "errors": []} causes = [] weekly_deferred = False changed_tables.add(table_name(table)) try: count = _sync_sqp_batch(con, fields, task_start, task_end, task_batch, weekly_limit, sqp_request, causes.append) weekly_completed = weekly_completed or not _sqp_month( con, task_start, task_end)["batches"][task_batch].get("refused", False) spend_rows(count) landed.add((table, task_start, task_end, task_batch)) except SQPPassDeferred as exc: pause_sqp(exc) continue except SQPBudgetDeferred as exc: report["tables"][table] = {"state": "deferred", "reason": str(exc)} weekly_deferred = True except cx.AmazonReportPending as exc: weekly_pending = _sync_problem(exc, table, "report_document", tenant_key, rt, report, log) report["tables"][table] = weekly_pending pending_reports.add(table) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900) _record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"]) except cx.AmazonError as exc: if isinstance(exc, cx.AmazonThrottled): problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log) report["tables"][table] = problem failed.add(table) pending_reports.add(table) wait = _save_throttled_resume(con, table, exc, plans[table], [], first_load=first_load, since=since) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) _record_state(con, entity_key(table), "throttled", detail=problem["reason"]) continue limit_reason = _close_history_floor(con, table, exc, task_start, task_end) if limit_reason: history_cut[history_key] = task_end landed.update(_history_cut_tasks(work, table, "WEEK", task_end)) sqp_limit["weeklyCause"] = limit_reason sqp_limit["errors"] = [] sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) log(limit_reason) continue causes.append(str(exc)) if not _AMAZON_FLOOR_TEXT.search(str(getattr(exc, "vendor_reason", "") or exc)): marker = _sqp_month(con, task_start, task_end) marker["batches"][task_batch] = {"refused": True, "reason": str(exc), "asins": []} marker.setdefault("refusedAsins", {}).update({a: str(exc) for a in task_batch.split()}) _record_state(con, _sqp_month_key(task_start, "WEEK"), "refused", detail=json.dumps(marker)) landed.add((table, task_start, task_end, task_batch)) else: failed.add(table) report["tables"][table] = {"state": "deferred", "reason": str(exc)} except ValueError as exc: # Unreadable source content remains owed without creating an ASIN refusal or history floor. causes.append(str(exc)) if causes and not weekly_cause_logged: sqp_limit["weeklyCause"] = "Amazon weekly Search Query Performance: " + "; ".join(dict.fromkeys(causes)) sqp_problem(sqp_limit["weeklyCause"]) weekly_cause_logged = True sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) if weekly_deferred: break continue if first_load and (table in GROUP_A_TABLES or (table in prior_refusals and cx.REPORT_TYPES[table].get("min_interval_s"))): reason = _interval_reason(table, tick, datetime.now(timezone.utc)) if reason: report["tables"][table] = {"state": "deferred", "reason": reason} # HOTFIX 2026-09-29: when the next floor opens, so the desk can wait for it # instead of re-queuing a pass that can only defer again. wait = _interval_wait(table, tick, datetime.now(timezone.utc))[0] report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) continue if table in handled: continue plan = plans[table] rtype = plan["report"] key, mirror = entity_key(table), table_name(table) if not rtype: report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"} log(f" {table:<16} SKIPPED: no confirmed bulk report flavor") continue prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync") root = cx.REPORT_TYPES[table].get("json_root") # History completion does not prove that legacy rows carry a report period. period_repair_mark = period_repair_next = waited_start = None if not first_load and cx.REPORT_TYPES[table].get("weekly"): names = set(_columns_of(con, mirror)) dated = ({"_period_start", "_period_end"} <= names and con.execute( f'SELECT 1 FROM {mirror} WHERE "_period_start" IS NOT NULL ' 'AND "_period_end" IS NOT NULL LIMIT 1').fetchone()) if not dated: period_repair_mark = read_watermark(con, table) repair_start, repair_end = _weekly_period_repair(con, table, end) # Two finished weeks, oldest first: this pass lands the older, `sync` publishes the newer next. period_repair_next = repair_start plan = {**plan, "start": repair_start - timedelta(days=7), "end": repair_end, "cause": "repair missing period fields from the two completed earlier weeks"} log(f"{table}: {plan['cause']}") sibling = SHARED_REPORT_SIBLINGS.get(table) sibling_plan = None if first_load: sibling = paired.get(table) sibling_plan = plans.get(sibling) elif sibling in want and sibling not in handled: sibling_plan = plans[sibling] differs = [part for part in ("start", "end", "report") if plan[part] != sibling_plan[part]] if differs: log(f"{table} and {sibling}: cannot share this pass; differing {', '.join(differs)}") sibling = None else: handled.add(sibling) log(f"{sibling}: {sibling_plan['cause']}") else: sibling = None operation = "report_document" # Include attempts: a later vendor/validation failure can follow a successful row write. changed_tables.add(mirror) if sibling: changed_tables.add(table_name(sibling)) proof_key = entity_key("reconcile_" + table) previous_proof = _read_state(con, proof_key) if table in invalid_proofs else None resync, resync_count = [], 0 completed_windows = list(resumes.get(table, {}).get("completed", [])) try: if previous_proof: # The recorder accepts an empty object; only a parsed document can replace it with proof. _record_state(con, proof_key, "unproven", detail="{}") # Attempt first, so a refusal or process interruption cannot starve the other tables. if not first_load and table != "settlements": _record_state(con, entity_key("sync_tick_" + table), "attempted") if sibling and not first_load: _record_state(con, entity_key("sync_tick_" + sibling), "attempted") sibling_written, sibling_cleared = 0, bool(completed_windows) written, cols, cleared = 0, [], bool(completed_windows) landed_window = bool(completed_windows) pricing_asins = [] operations = {} if first_load: cleared = floors[table] is not None sibling_cleared = bool(sibling and floors[sibling] is not None) pricing_asins = contexts.setdefault(table, []) if table == "settlements": operation = "getReports" _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": operation}, rt, report, log) discovered = cx.settlement_reports( fields, since=_iso(plan["start"]) if plan["incremental"] else None, log=lambda message: log(f"settlements: {message}")) downloaded, skipped = 0, 0 for issued_index, issued in enumerate(discovered): document_id = str(issued.get("reportDocumentId") or "") if not document_id: raise cx.AmazonError("Settlement discovery returned no document ID.") if not settlement_is_new(con, document_id): skipped += 1 continue if first_load and isinstance(report_budget, int) and documents >= max(0, report_budget): settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or "")) for item in discovered[issued_index:]) break probe = _DocumentProbe(table, rtype) operation = "getReportDocument" documents += 1 _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": operation}, rt, report, log) text = cx.settlement_document(fields, document_id) _DocumentProbe.downloaded(text) cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype) count = 0 if cols and rows: types = _ensure(con, mirror, cols, rows, preserve_rows=True) # Never clear settlement history: durable markers prevent its replay. # A newly issued document still replaces its own settlement's lines. count = write_window(table, cols, types, rows) written += count probe.finish(count) # Record completion after the write, including header-only and blank files. _record_state(con, entity_key("settlement_doc_" + document_id), "done", count, issued.get("dataEndTime")) downloaded += 1 operations["getReports"] = { "state": "done", "reports": len(discovered), "downloaded": downloaded, "skipped": skipped} if not discovered: con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)]) report["tables"][table] = {"rows": 0, "operations": operations, "state": "empty", "reason": "No issued settlement reports found."} landed.add((table, task_start, task_end, task_batch)) if first_load: _record_state(con, key, "done", 0, "No issued settlement reports found.") # A successful empty scheduled discovery earns cadence, but no source cursor or document marker. if scheduled and not first_load: _record_state(con, entity_key("sync_tick_" + table), "done", 0) if not first_load and since is None: _record_lookback_day(con, table, {**plan, "incremental": True}, end) continue if first_load and settlement_remaining: break # getReports filters createdTime. Never substitute the time we happened to pull. issued_times = [_utc(item.get("createdTime") or item["dataEndTime"]) for item in discovered if item.get("createdTime") or item.get("dataEndTime")] prior = read_watermark(con, table) settlement_mark = max(issued_times + ([prior] if prior else []), default=None) windows = () if table == SQP_TABLE: if sqp_limit["deferredReason"]: if weekly_completed: total = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] report["tables"][table] = {"state": "done", "rows": total, "provenance": prov, "report": rtype} _record_state(con, key, "done", total) _record_state(con, entity_key("sync_tick_" + table), "done", total) continue report["tables"][table] = {"state": "deferred", "reason": sqp_limit["deferredReason"], "current": bool(sqp_asins) and current_table(table)} _record_state(con, entity_key("sync_tick_" + table), "done") continue if task_batch is not None: written += _sync_sqp_batch(con, fields, task_start, task_end, task_batch, sqp_limit, sqp_request, sqp_problem) sqp_limit["errors"] = [] sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES: operation = cx.REPORT_TYPES[table].get("api") or "report_document" spec = cx.REPORT_TYPES[table] after = first_load and _after_cohort(table, task_start, end) if first_load: # A month after the cohort end lies past the frozen plan end, so it keeps its own bounds. margin_windows = [(task_start, task_end if after else min(task_end, plan["end"]))] if task_end else ( [(None, None)] if spec.get("snapshot") else []) elif spec.get("snapshot"): margin_windows = [(None, None)] elif spec.get("monthly"): margin_windows = _month_windows(plan["start"], plan["end"]) else: margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS)) resync = [] if first_load or spec.get("snapshot") else queued_resyncs(con, table) if resync and read_watermark(con, table): # Queued windows take this tick, several within the quota left; the watermark stays. budget = report_budget if isinstance(report_budget, int) else RESYNC_REPORTS_PER_PASS margin_windows = resync[:max(1, budget - documents)] plan["end"] = read_watermark(con, table) plan["cause"] = (f"resync of {len(margin_windows)} stored windows " f"from {_iso(margin_windows[0][0])}") log(f"{table}: {plan['cause']}") elif (table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1 and not spec.get("restatement_keys")): # One due window per table tick respects the vendor floor. The watermark # advances only to that window's end, so the following tick loses no history. remaining_windows = len(margin_windows) - 1 margin_windows = [margin_windows[0]] plan["end"] = margin_windows[0][1] _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({ "reason": (f"{remaining_windows} history windows remain. Amazon allows this report at most " f"once every {_readable_duration(spec['min_interval_s'])}. Keep the connection enabled; " "sync resumes at its saved date. Use Sync now without a custom start date.")})) elif table in GROUP_A_TABLES and table != "catalog_items": saved_limit = _read_state(con, entity_key("limit_" + table)) or {} reason = json.loads(saved_limit.get("detail") or "{}").get("reason", "") if not reason.startswith("Amazon keeps this report back to "): _record_state(con, entity_key("limit_" + table), "done", detail="{}") for w0, w1 in margin_windows: window = [_iso(w0) if w0 is not None else None, _iso(w1) if w1 is not None else None] if window in completed_windows: continue # One detail window is one budget task, including its reconciliation query. probe = _DocumentProbe(table, spec.get("report") or operation) documents += 1 _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": operation}, rt, report, log) count = _sync_margin_window(con, table, fields, w0, w1, max_wait=max_wait, log=log, operations=operations, rt=rt) if (w0, w1) in resync: # Per window, so a failure later in this tick never downloads a landed window again. resync_landed(con, table, (w0, w1)) resync_count += 1 probe.finish(count) spend_rows(count) written += count landed_window = True completed_windows.append(window) if table in LEGACY_SNAPSHOT_TABLES: # The dated replacement already landed; AWD must not clear that # document or historical FBA/AWD rows on an explicit first load. cleared = True if first_load and table in GROUP_A_TABLES: _record_state(con, entity_key("sync_tick_" + table), "done") if after: _record_state(con, _cohort_month_key(table, w0), "done", count) elif first_load and w0 is not None: record_backfill_floor(con, table, w0) floors[table] = w0 stop = _note_history_window(con, table, w0, count) if stop: history_cut[history_key] = w0 landed.update(task for task in work if task[0] == table and task[2] is not None and task[2] <= w0) log(stop) elif table != "settlements": period_spec = cx.REPORT_TYPES[table] windows = (([(task_start, task_end)] if task_end is not None else []) if first_load else _period_windows(period_spec, plan["start"], plan["end"]) if _periodic(period_spec) else _windows(plan["start"], plan["end"])) if period_spec.get("weekly"): _record_brand_analytics_limit(con) if not first_load and len(windows) > 1: # One week per pass, oldest first. The watermark stops at its end, so no week is asked twice. log(f"{table}: {len(windows) - 1} more completed weeks follow on later passes") windows = windows[:1] plan["end"] = windows[0][1] weekly = bool(table != "settlements" and cx.REPORT_TYPES[table].get("weekly")) for w0, w1 in (() if table == "settlements" else windows): window = [_iso(w0), _iso(w1)] if window in completed_windows: continue # Each vendor attempt records access before the call, without credentials or source rows. documents += 1 probe = _DocumentProbe(table, rtype) try: with _report_resume( con, table, rtype, start=_iso(w0), end=_iso(w1 - timedelta(seconds=1)) if _periodic(cx.REPORT_TYPES[table]) else _iso(w1), max_wait=max_wait, log=lambda message: log(f"{table}: {message}"), **({"options": {**REPORT_OPTIONS.get(table, {}), "reportPeriod": cx.REPORT_TYPES[table]["period"]}} if cx.REPORT_TYPES[table].get("period") else {"options": dict(REPORT_OPTIONS[table])} if table in REPORT_OPTIONS else {})) as resume_kwargs: _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": "report_document"}, rt, report, log) text, meta = cx.report_document(fields, rtype, **resume_kwargs) except cx.AmazonReportError as exc: if not (weekly and _week_not_ready(exc)): raise # An unfinished week is a wait, never a refusal: ask the week before it, and this one next pass. probe._halt.set() waited_start = w0 if waited_start is None else min(waited_start, w0) _record_week_wait(con, table, w0, w1, exc) prior = (w0 - timedelta(days=7), w0) held = _newest_week(con, mirror) log(f"{table}: Amazon has not finished the week starting {w0.date().isoformat()}; " "sync asks for it again next pass") if prior not in windows and (held is None or held < prior[0].date()): windows.append(prior) continue _DocumentProbe.downloaded(text) landed_window = True if root: parsed_cols, rows = parse_json_report( text, root, document_id=meta.get("reportDocumentId"), report_type=rtype) pinned = [c for c, _t in amazon_schema.columns_for(table)] # ⛔⛔ THE UNION, NOT THE PIN, AND AMAZON'S OWN ARTIFACT IS WHY. # # `sellerSalesAndTrafficReport.json` CONTRADICTS ITSELF: its `properties` declare # `mobileAppPageViews` while its own `examples` carry `mobilePageViews`, four # columns' worth. So the pinned list -- which is built from `properties`, and is # the right authority for a COLUMN LIST -- can be missing a field Amazon # actually returns. # # ⭐ `pinned or parsed_cols` would then have DROPPED that field silently, which # is the one thing R2's *"do not drop any column"* forbids and which nothing # would have gone red for. The union keeps every field the payload really # carried AND every field the schema promises, so a disagreement costs a NULL # column rather than lost data. [[a-published-fixture-corpus-can-answer-wrongly]] # # ⚠ Pinned order first, so the grid's column order still follows the schema and # only the surprises land at the end where they are visible. extra = [c for c in parsed_cols if c not in set(pinned)] if extra: log(f" {table}: {len(extra)} field(s) returned that the pinned schema " f"does not declare, KEPT: {', '.join(extra[:6])}") cols = merge_columns(pinned, parsed_cols) if pinned else parsed_cols declared = dict(amazon_schema.columns_for(table)) else: # ⚠ NO DECLARED TYPES HERE ON PURPOSE: a flat-file report publishes no schema, # so sniffing the data is the ONLY evidence there is. That is the whole # provenance split, arriving one layer down. cols, rows = parse_flat_file( text, document_id=meta.get("reportDocumentId"), report_type=rtype) declared = {} if cols and (weekly or "_period_start" in replace_keys_for(table)): cols, rows = stamp_week(cols, rows, w0, w1) order_proof = (_order_report_observations(con, cols, rows, meta, rtype, w0, w1) if "order_items" in (table, sibling) else None) if not cols: if first_load: # Even an empty recent window clears old rows exactly once. A weekly # table never clears: R18's retention keeps watched history. for empty_table in (table, sibling): if empty_table and empty_table != "order_items" and floors[empty_table] is None and not weekly: if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?", [table_name(empty_table)]).fetchone(): con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1") cleared = table != "order_items" elif order_proof is None: completed_windows.append(window) continue if sibling: sibling_mirror = table_name(sibling) sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared, preserve_rows=True) if cols else {} sibling_reset = not sibling_cleared and not sibling_plan["incremental"] if cols and sibling_reset and sibling != "order_items": con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1") if cols or order_proof is not None: sibling_written += write_window(sibling, cols, sibling_types, rows, order_proof=order_proof, reset=sibling_reset) sibling_cleared = True order_observations = None if table == "orders": rows = _dedupe_orders(rows) order_observations = _order_status_observations(con, rows, meta) if table == "inventory": rows = _tag_fba_inventory(rows) if "inventory_source" not in cols: cols.append("inventory_source") if table == "pricing": pricing_asins.extend(_pricing_asins(rows)) if not cleared and _pricing_regrain(con, mirror, rows, log): cleared = True types = _ensure(con, mirror, cols, rows, declared=declared, preserve_rows=True) if cols else {} # Explicit windows and first loads replace the mirror; incremental passes keep history. # A weekly table replaces by its period identity instead (D-2675). order_reset = not cleared and not plan["incremental"] if order_reset and not weekly and table != "order_items": con.execute(f"DELETE FROM {mirror} WHERE 1=1") cleared = True if cols or table == "order_items": after = (lambda target: _record_state(target, entity_key("reconcile_orders"), "done", detail=json.dumps(order_observations))) if order_observations else None written += write_window(table, cols, types, rows, after=after, order_proof=order_proof, reset=order_reset) if table == "order_items": cleared = True if _periodic(cx.REPORT_TYPES[table]): _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1), "count": len(rows)}) if weekly: _clear_week_wait(con, table, w0) probe.finish(len(rows) if cols else 0) spend_rows(len(rows) if cols else 0) completed_windows.append(window) if first_load: for reached in (table, sibling): if reached: record_backfill_floor(con, reached, w0) floors[reached] = w0 stop = _note_history_window(con, table, w0, len(rows) if cols else 0) if stop: history_cut[history_key] = w0 landed.update(task for task in work if task[0] == table and task[2] is not None and task[2] <= w0) log(stop) if (first_load and table != "settlements" and table != SQP_TABLE and not cx.REPORT_TYPES[table].get("snapshot") and floors[table] is None): report["tables"][table] = {"state": "deferred", "reason": "No completed data window has landed; keep the connection enabled for the next completed period."} continue if first_load or table == SQP_TABLE: landed.add((table, task_start, task_end, task_batch)) # SQP weekly tasks never close the table, so the last month task does. if any(task[0] == table and not (table == SQP_TABLE and task[1] is not None and _sqp_period(task[1], task[2]) == "WEEK") for task in work[index + 1:]): continue if table == "pricing": if first_load: # Completion can be in another process/pass than the recent ASIN report. # One ASIN per row, as `_pricing_asins` picks it: a parent beside its children # would answer for no row and `validate()` would refuse the pass. available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()} picked = [c for c in ("child_asin", "parent_asin") if c in available] select = ", ".join(f'"{c}"' for c in picked) mirror_rows = [dict(zip(picked, row)) for row in con.execute( f"SELECT DISTINCT {select} FROM {mirror}").fetchall()] if picked else [] pricing_asins = list(dict.fromkeys(pricing_asins + _pricing_asins(mirror_rows))) state_key = ENTITY_STATE_KEYS[table] if not pricing_asins: _record_state(con, state_key, "unproven", 0, "Sales and Traffic returned no ASIN to test") operations["getCompetitiveSummary"] = { "state": "unproven", "responses": 0} log(" pricing: getCompetitiveSummary UNPROVEN, no ASIN was returned") else: try: _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": "getCompetitiveSummary"}, rt, report, log) responses = cx.competitive_summaries(fields, pricing_asins) response_rows, updated = _apply_competitive(con, mirror, responses) _record_state(con, state_key, "done", response_rows) operations["getCompetitiveSummary"] = { "state": "done", "responses": response_rows, "mirror_rows_enriched": updated} log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, " f"{updated:,} mirror rows enriched") except Exception as e: # noqa: BLE001 if isinstance(e, cx.AmazonThrottled): raise problem = _sync_problem(e, table, "getCompetitiveSummary", tenant_key, rt, report, log) _record_state(con, state_key, problem["state"], 0, problem["reason"]) operations["getCompetitiveSummary"] = problem if table == "inventory": state_key = ENTITY_STATE_KEYS[table] try: _sync_event("amazon.access", tenant_key, table, {"state": "access", "operation": "listInventory"}, rt, report, log) inventory = cx.awd_inventory(fields) awd_rows = _awd_rows(inventory) written += _replace_awd_snapshot(con, awd_rows) _record_state(con, state_key, "done", len(awd_rows)) operations["listInventory"] = { "state": "done", "rows": len(awd_rows)} log(f" inventory: AWD listInventory {len(awd_rows):,} rows") except Exception as e: # noqa: BLE001 problem = _sync_problem(e, table, "listInventory", tenant_key, rt, report, log) _record_state(con, state_key, problem["state"], 0, problem["reason"]) operations["listInventory"] = problem raise try: column_count = len(con.execute(f"PRAGMA table_info('{mirror}')").fetchall()) total_rows = int(con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]) except Exception: # noqa: BLE001 column_count, total_rows = len(cols), written report["tables"][table] = { "rows": total_rows, "columns": column_count, "provenance": prov, "report": rtype, "window": plan["cause"], "incremental": plan["incremental"], "operations": operations} if waited_start is not None: report["tables"][table]["waiting"] = json.loads( (_read_state(con, entity_key("waiting_" + table)) or {}).get("detail") or "{}") elif period_repair_next is not None and landed_window: report["tables"][table]["nextWeek"] = _iso(period_repair_next) nothing_new = _landed_nothing(table, landed_window, landed) _record_state(con, key, "done", total_rows, NOTHING_NEW_DETAIL if nothing_new else None) prior_tick = _read_state(con, entity_key("sync_tick_" + table)) if nothing_new and first_load else None if not (prior_tick and prior_tick.get("state") == "done"): # A first-load pass that requested nothing keeps the last landing's tick, so it never reads as fresh. _record_state(con, entity_key("sync_tick_" + table), "done", total_rows, NOTHING_NEW_DETAIL if nothing_new else None) con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)]) if weekly and landed_window: # A landed week closes any earlier refusal of this table. con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("refused_" + table)]) if not any(v["state"] in ("refused", "throttled") for v in operations.values()): mark = settlement_mark if table == "settlements" else plan["end"] if mark is not None and period_repair_mark is not None: # Repairing an earlier week cannot rewind completed history. mark = max(mark, period_repair_mark) if mark is not None and waited_start is not None: # The week Amazon has not finished stays owed, so the next pass asks for it again. mark = min(mark, waited_start) has_window = (cx.REPORT_TYPES[table].get("snapshot") or table in ("settlements", SQP_TABLE) or landed_window or (first_load and floors[table] is not None)) if first_load and mark is not None and has_window: # Months landed after the cohort end advance the watermark, so a refresh never asks for them again. for w0, w1 in _month_windows(end, datetime.now(timezone.utc)): if not (_after_cohort(table, w0, end) and _cohort_month_done(con, table, w0)): break mark = max(mark, w1) if (first_load and mark is not None and waited_start is None and read_watermark(con, table) is not None): # A reopened history pass never rewinds the recent data the scheduled refresh landed. mark = max(mark, read_watermark(con, table)) if mark is not None and has_window and table in catch_up and not read_watermark(con, table): # A catch-up landed recent data only, so the history watermark stays unwritten. _record_state(con, entity_key(CATCH_UP_PREFIX + table), "done", total_rows, _iso(_utc(mark))) elif mark is not None and has_window: record_watermark(con, table, mark, total_rows) if not first_load and not resync: completed_plan = ({**plan, "incremental": True} if table == "settlements" and since is None else plan) _record_lookback_day(con, table, completed_plan, end) _sync_event("amazon.sync", tenant_key, table, {"state": "done", "rows": total_rows, "operations_ok": all(v["state"] == "done" for v in operations.values())}, rt, report, log) log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}") if sibling: sibling_mirror = table_name(sibling) try: sibling_columns = len(con.execute(f"PRAGMA table_info('{sibling_mirror}')").fetchall()) sibling_total = int(con.execute(f"SELECT COUNT(*) FROM {sibling_mirror}").fetchone()[0]) except Exception: sibling_columns, sibling_total = len(cols), sibling_written sibling_prov = amazon_schema.PROVENANCE.get(sibling, "measured_at_sync") report["tables"][sibling] = { "rows": sibling_total, "columns": sibling_columns, "provenance": sibling_prov, "report": rtype, "window": sibling_plan["cause"], "incremental": sibling_plan["incremental"], "operations": {}} _record_state(con, entity_key(sibling), "done", sibling_total) _record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total) con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + sibling)]) record_watermark(con, sibling, sibling_plan["end"], sibling_total) if not first_load: _record_lookback_day(con, sibling, sibling_plan, end) _sync_event("amazon.sync", tenant_key, sibling, {"state": "done", "rows": sibling_total, "operations_ok": True}, rt, report, log) log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}") except SQPPassDeferred as exc: # The SQP share is spent; the other tables in this pass still run. pause_sqp(exc) continue except SQPBudgetDeferred as exc: report["tables"][table] = {"state": "deferred", "reason": str(exc)} break except Exception as e: # noqa: BLE001 limit_reason = (_close_history_floor(con, table, e, task_start, task_end) if first_load or table == SQP_TABLE else None) if limit_reason and table == SQP_TABLE: history_cut[history_key] = task_end landed.update(_history_cut_tasks(work, table, "MONTH", task_end)) if sqp_limit is not None: sqp_limit["errors"] = [] sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) report["tables"][table] = {"state": "deferred", "current": True, "reason": limit_reason} log(limit_reason) continue if limit_reason: landed.update(task for task in work if task[0] == table) handled.add(table) total = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] _record_state(con, key, "done", total) record_watermark(con, table, max(plan["end"], read_watermark(con, table) or plan["end"]), total) report["tables"][table] = {"rows": total, "state": "done", "reason": limit_reason} log(limit_reason) continue # A failed pass retains its previous watermark so its window will be retried. # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN # ticket answering itself, and it must not cost the six tables that did work. problem = _sync_problem(e, table, getattr(e, "operation", None) or operation, tenant_key, rt, report, log) failed.add(table) report["tables"][table] = {"rows": 0, **problem} if problem["state"] == "throttled": landed.discard((table, task_start, task_end, task_batch)) pending_reports.add(table) if _columns_of(con, mirror): report["tables"][table]["rows"] = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] wait = _save_throttled_resume(con, table, e, plan, completed_windows, first_load=first_load, since=since) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) if problem["state"] == "pending": pending_reports.add(table) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900) # S66-T42 (lead, observed live 2026-09-25): a refusal lived only in this pass's report, # so Connectors > Amazon said "not loaded yet, sync it" beside a table Amazon had just # refused by name (a 400 "not allowed", a FATAL). The table's own marker now carries the # refusal, which `connectors_amazon.sync_state` shows verbatim, and the tick closes so the # table rotates behind its peers instead of reading as syncing forever. try: for refused in (table, sibling) if sibling else (table,): _record_state(con, entity_key(refused), problem["state"], 0, problem["reason"]) if problem["state"] == "refused": attempts = prior_refusals.get(refused, {}).get("attempts", 0) + 1 _record_state(con, entity_key("refused_" + refused), "refused", detail=json.dumps({ "reason": problem["reason"][:500], "amazonStatus": problem["amazonStatus"], "diagnostic": problem.get("diagnostic", "")[:500], "error": problem.get("error", ""), "attempts": attempts})) if problem["state"] == "throttled": if refused != table: _save_throttled_resume(con, refused, e, sibling_plan, completed_windows, first_load=first_load, since=since) else: _record_state(con, entity_key("sync_tick_" + refused), "attempted" if problem["state"] == "pending" else "done") except Exception: # noqa: BLE001 pass _sync_event("amazon.sync", tenant_key, table, {"state": problem["state"], "rows": 0}, rt, report, log) if sibling: problem = _sync_problem(e, sibling, "report_document", tenant_key, rt, report, log) report["tables"][sibling] = {"rows": 0, **problem} _sync_event("amazon.sync", tenant_key, sibling, {"state": problem["state"], "rows": 0}, rt, report, log) finally: if resync: resync_left = len(queued_resyncs(con, table)) report["tables"].setdefault(table, {})["resync"] = { "landed": resync_count, "left": resync_left} report["tables"][table].setdefault("window", plan["cause"]) log(f"Amazon resync {table}: {resync_count} windows landed, {resync_left} left") if previous_proof: proof = _read_state(con, proof_key) or {} if proof.get("state") == "unproven" and proof.get("detail") == "{}": # Pending, refused or unlanded work retains its original evidence and diagnostics. con.execute("UPDATE _sync_state SET phase=?, cursor_wd=?, rows=?, updated_at=? WHERE entity=?", [previous_proof["state"], previous_proof["detail"], previous_proof["rows"], previous_proof["updated_at"], proof_key]) else: invalid_proofs.discard(table) if SQP_TABLE in pending: remaining_sqp = sum(task[0] == SQP_TABLE and task not in landed for task in work) if remaining_sqp and SQP_TABLE not in failed and weekly_pending: report["tables"][SQP_TABLE] = weekly_pending elif remaining_sqp and SQP_TABLE not in failed and not sqp_limit["deferredReason"]: reason = f"Search Query Performance has {remaining_sqp} month and batch tasks remaining; continuing next pass." sqp_limit["deferredReason"] = reason report["tables"][SQP_TABLE] = {"state": "deferred", "reason": reason} # Landed batches publish with their month markers; the backlog stays owed in the progress. landed_batches = sum(task[0] == SQP_TABLE and task[1] is not None for task in landed) row = report["tables"].get(SQP_TABLE) if landed_batches and isinstance(row, dict) and row.get("state") in {"deferred", "throttled", "pending"}: report["tables"][SQP_TABLE] = {**row, "state": "deferred", "current": True, "landedBatches": landed_batches} sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) for table in pending if first_load else (): # Landed periods publish with their markers while the remaining history stays owed. months = sum(task[0] == table and task[2] is not None for task in landed) row = report["tables"].get(table) if (table == SQP_TABLE or not _periodic(cx.REPORT_TYPES[table]) or not months or (row is not None and not (isinstance(row, dict) and row.get("state") in {"deferred", "throttled", "pending"}))): continue grain = "week" if cx.REPORT_TYPES[table].get("weekly") else "month" row = row or {"reason": f"{months} {grain}{'' if months == 1 else 's'} landed. The remaining {grain}s continue next pass."} count_key = "landedWeeks" if grain == "week" else "landedMonths" row_count = (con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] if _columns_of(con, table_name(table)) else 0) report["tables"][table] = {**row, "state": "deferred", "current": True, count_key: months, "rows": row_count} if first_load: refusals ={row["table"]: row for row in _refused_reports(con)} not_offered = {table for table, row in refusals.items() if row["state"] == "not_offered"} retrying = {table for table, row in refusals.items() if row["state"] == "retrying"} landed.update(task for task in work if task[0] in not_offered) remaining = sum(task not in landed for task in work) if settlement_remaining: remaining += settlement_remaining - 1 # Replace the discovery slot with known documents. # Keep healthy tables moving after an isolated refusal. Once only failed work remains, # stop automatic continuation instead of endlessly retrying the same vendor refusal. continuable = any(task not in landed and (task[0] not in failed or task[0] in pending_reports or task[0] in retrying) for task in work) unexpected = [problem for problem in report["problems"] if not any(problem.startswith(table + ":") for table in not_offered)] state = ("partial" if remaining and continuable else "failed" if remaining or unexpected or report["audit_problems"] else "done") # Requests for one report type must also respect that report's own refresh floor. # Different tables can progress between those waits; use the longest table chain. counts = {} for task in work: if task not in landed and cx.REPORT_TYPES[task[0]].get("min_interval_s"): counts[task[0]] = counts.get(task[0], 0) + 1 floor_eta = 0 for table, count in counts.items(): floor = cx.REPORT_TYPES[table]["min_interval_s"] tick = _read_state(con, entity_key("sync_tick_" + table)) wait = (max(0, floor - (datetime.now(timezone.utc) - _utc(tick["updated_at"])).total_seconds()) if tick and tick["state"] == "done" else 0) floor_eta = max(floor_eta, int(wait + max(0, count - 1) * floor + 0.999)) # HOTFIX 2026-09-29: whether this pass FETCHED anything. A continuation that only met # refresh floors must not re-queue itself every 30 seconds; the daily pass resumes it. report["landedTasks"] = len(landed) markers = {row[0]: {"state": row[1], "detail": row[3]} for row in _sync_state_rows(con)} enabled_tables = [table for table in TABLES if enablement["databases"].get(amazon_schema.DATABASE_PREFIX + table)] progress = {"state": state, **first_load_payload(markers, remaining, max(first_load_eta(remaining, burst=0), floor_eta), state=state, tables=enabled_tables), "backfillDays": DEFAULT_BACKFILL_DAYS, "historyFloors": history_status(con, enabled_tables, end)} due = (datetime.now(timezone.utc) + timedelta(seconds=report["nextDueSeconds"])) if ( state == "partial" and not landed and report.get("nextDueSeconds")) else None _record_state(con, entity_key(FIRST_LOAD_DUE_KEY), "pending" if due else "done", detail=_iso(due) if due else None) _restore_first_load_wait(tenant_key, con) progress["nextDue"] = _iso(due) if due else None report.update(progress) report["firstLoad"] = dict(progress) _record_state(con, entity_key(FIRST_LOAD_END_KEY), state, detail=_iso(end)) _record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), state, detail=json.dumps({key: progress[key] for key in ("remaining", "etaSeconds")})) persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log) report["finished"] = _iso(datetime.now(timezone.utc)) con.close() return report def _dedupe_orders(rows): """The flat-file order report is one row per ITEM. `orders` is its deduplicated head. ⚠ FIRST ROW WINS PER ORDER ID, and the order-level columns are identical across an order's lines by construction, so this drops line detail rather than losing an order. `order_items` keeps every row, which is why both tables read the same document instead of costing two. """ if isinstance(rows, _DocumentRows): return rows.pipe(_first_row_per_order) return list(_first_row_per_order(rows)) def _first_row_per_order(rows): seen = set() for r in rows: oid = str(r.get("amazon_order_id") or r.get("order_id") or "") if oid and oid in seen: continue if oid: seen.add(oid) yield r def validate(tenant_key="royal-imports", rt=None, log=print): """Reconcile each connection against its own source summaries before combining verdicts.""" if _account_scope() is not None: return _validate_account(tenant_key, rt=rt, log=log) runtime = rt if rt is not None else _runtime_for(tenant_key) con, _path = _connect_tenant_mirror(tenant_key, runtime) try: if not _read_state(con, "amzn_account_binding"): return _validate_account(tenant_key, rt=runtime, log=log) entries = account_entries(tenant_key, runtime) results = {} for entry in entries: with _account_workspace(con, tenant_key, runtime, entry): results[entry["connection_id"]] = _validate_account(tenant_key, rt=runtime, log=log) out = {} for table in DATABASE_TABLES: accounts = {cid: rows[table] for cid, rows in results.items()} if len(accounts) == 1: out[table] = {**next(iter(accounts.values())), "accounts": accounts} continue state = ("refused" if any(row["state"] == "refused" for row in accounts.values()) else "ok" if all(row["state"] == "ok" for row in accounts.values()) else "unproven") out[table] = {"state": state, "rows": sum(row.get("rows", 0) for row in accounts.values()), "accounts": accounts} return out finally: con.close() def _validate_account(tenant_key="royal-imports", rt=None, log=print): """⭐⭐ R3 — RECONCILE EACH TABLE TO AMAZON'S OWN SUMMARY, NOT TO ODOO. -> a report dict. Standing rule 8 says a number that does not tie to an independent aggregate does not ship. Settlements, FBA inventory and Brand Analytics have NO Odoo counterpart at all, and the owner ruled (wave 52, R3) that for an API-only source the reconciliation is SOURCE-INTERNAL: summary versus detail, the report's own period totals versus our rows. ⛔ **THE REASON IS THE RULE, and it is what keeps rule 8's teeth:** the bug class rule 8 exists to catch is OUR transformation, not Amazon's arithmetic. A summary-versus-detail check catches exactly that and nothing less. Rule 8 is NARROWED here, never loosened to "any independent aggregate" — that broader version was offered and refused, because a loosened rule is far harder to tighten again than a narrow one is to widen. ⚠ A TABLE THAT HAS NEVER SYNCED IS `unproven`, NOT `ok`. An empty table trivially reconciles with an empty summary, and a check that cannot fail is broken [[a-check-that-cannot-reach-green-is-broken]]. """ rt = rt if rt is not None else _runtime_for(tenant_key) con, _path = _connect_tenant_mirror(tenant_key, rt) cx = _client() out = {} for table in DATABASE_TABLES: mirror, spec = table_name(table), (cx.REPORT_TYPES.get(table) or {}) if table == "keyword_ranks": out[table] = _validate_keyword_ranks(con) continue if table == "sku_keywords": out[table] = {"reconciles": "raw Search Query Performance purchases for every month", **_validate_sku_keywords(con)} continue if table in HISTORY_TABLES: out[table] = {"reconciles": "the source table's newest snapshot", **_validate_history(con, table.removesuffix("_history"))} continue try: n = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] except Exception: # noqa: BLE001 out[table] = {"state": "unproven", "reason": "table has never synced", "reconciles": spec.get("validates_against")} continue if table in ("fee_estimates_api", "replenishment", "inbound_shipments", *FEEDBACK_TABLES): out[table] = {"rows": n, "reconciles": spec.get("validates_against"), **_validate_margin(con, table)} continue if not n: out[table] = {"state": "unproven", "reason": "table is empty", "reconciles": spec.get("validates_against")} continue if table == SQP_TABLE: out[table] = {"rows": n, "reconciles": spec.get("validates_against"), **_validate_sqp(con)} continue if table == "listings": out[table] = {"rows": n, "reconciles": spec.get("validates_against"), **_validate_snapshot(con, table)} continue if table in PERIOD_TABLES: out[table] = {"rows": n, "reconciles": spec.get("validates_against"), **_validate_margin(con, table)} continue if table == "orders": out[table] = {"rows": n, "reconciles": "Amazon Sales getOrderMetrics order count", **_validate_orders(con)} continue if table in ENTITY_STATE_KEYS: operation = _read_state(con, ENTITY_STATE_KEYS[table]) if not operation or operation.get("state") != "done": out[table] = { "state": "unproven", "reason": (operation or {}).get("detail") or "required role operation has not completed", "reconciles": spec.get("validates_against")} continue expected = int(operation.get("rows") or 0) if table == "inventory": measured = int(con.execute( f"SELECT COUNT(*) FROM {mirror} WHERE inventory_source='AWD'").fetchone()[0]) else: measured = int(con.execute( f"SELECT COUNT(DISTINCT competitive_body_asin) FROM {mirror} " "WHERE competitive_body_asin IS NOT NULL").fetchone()[0]) if measured != expected: out[table] = { "state": "refused", "rows": n, "reason": f"role operation returned {expected} but mirror carries {measured}", "reconciles": spec.get("validates_against")} continue if table == "inventory": out[table] = {"rows": n, "reconciles": spec.get("validates_against"), **_validate_snapshot(con, table)} continue out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")} for t, r in out.items(): log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}") con.close() return out def status(tenant_key="royal-imports", rt=None): """What is in the mirror now, restoring once from the tenant store when needed.""" rt = rt if rt is not None else _runtime_for(tenant_key) restored = restore_mirror(tenant_key, rt) try: con, _path = _connect_tenant_mirror(tenant_key, rt) except Exception as e: # noqa: BLE001 return {"error": f"{type(e).__name__}: {e}"} out = {"error": "; ".join(restored["problems"])} if restored["problems"] else {} for table in TABLES: mirror = table_name(table) try: n = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] cols = len(con.execute(f"PRAGMA table_info('{mirror}')").fetchall()) except Exception: # noqa: BLE001 n, cols = None, 0 out[table] = {"rows": n, "columns": cols, "provenance": amazon_schema.PROVENANCE.get(table)} if table in ENTITY_STATE_KEYS: out[table]["operation"] = _read_state(con, ENTITY_STATE_KEYS[table]) con.close() return out # --------------------------------------------------------------------------- selftest class _FixtureMirrorRuntime: """Dictionary blobs for this module's offline sync and purge fixtures only.""" def __new__(cls, *args, **kwargs): import tempfile rt = super().__new__(cls) if not hasattr(cls, "datastore_path"): rt._mirror_directory = tempfile.TemporaryDirectory(prefix="amazon-fixture-") rt.datastore_path = Path(rt._mirror_directory.name) / "mirror.duckdb" return rt def upload_bytes(self, path, data, message=None): if not hasattr(self, "blobs"): self.blobs = {} self.blobs[path] = data def download_bytes(self, path): return getattr(self, "blobs", {}).get(path) def delete_path(self, path): getattr(self, "blobs", {}).pop(path, None) def _fixture_account_cursor(con, rt): """Lend owned cursors from the active account, publication candidate or fixture publication in that order.""" scope = _account_scope() if scope is not None: return scope["root"].cursor(), rt.datastore_path candidate = datastore._candidate_connection(rt.key, rt.datastore_path) return (candidate if candidate is not None else con.cursor()), rt.datastore_path class _FixtureSyncRuntime(_FixtureMirrorRuntime, _ProcessDocuments): """Keep one synthetic tenant's documents, blobs and mirror path isolated.""" def __init__(self, key, tables, connection_id="snapshot-account"): _ProcessDocuments.__init__(self) self.key = key self.docs = { "keychain": {"entries": {"fixture-entry": { "connection_id": connection_id, "type": "amazon_sp", "slot": 1, "label": "Fixture store"}}}, ENABLEMENT_KEY: {"databases": { amazon_schema.DATABASE_PREFIX + table: table in tables for table in TABLES}}} @contextmanager def _fixture_sync_store(con, rt): """Bind offline key lookup and cursors to this fixture's tenant without replacing publication.""" from unittest.mock import patch from core import tenant_keys documents = _ProcessDocuments() documents.key = rt.key documents._read_strict = documents.get def address(tenant): if tenant != documents.key: raise AssertionError("Fixture key lookup crossed its tenant") return documents, tenant_keys.RECORD_KEY with patch.dict(os.environ, {"AIOS_DATA_KEY_BACKEND": "", "AIOS_DATA_KEY_LOCAL": ""}), \ patch.object(tenant_keys, "_CACHE", {}), \ patch.object(tenant_keys, "_address", side_effect=address), \ patch.object(sys.modules[__name__], "_connect_tenant_mirror", side_effect=lambda *_: _fixture_account_cursor(con, rt)): yield @contextmanager def _fixture_account_state(con, rt, *, write=False): """Read or amend this fixture account's current markers through its real workspace.""" if not _read_state(con, "amzn_account_binding"): yield con return entries = account_entries(rt.key, rt) if len(entries) != 1: raise AssertionError("Fixture requires exactly one Amazon account") with _account_workspace(con, rt.key, rt, entries[0]) as scope: yield scope["root"] if write: _merge_account(con, scope) _refresh_account_status(con, entries, rt) def _fixture_damaged_parquet(data): """Damage a fixture's checksum while preserving the legacy blob format signature.""" if not data.startswith(b"PAR1") or not data.endswith(b"PAR1") or len(data) < 9: raise AssertionError("Fixture requires a Parquet blob") return data[:4] + bytes([data[4] ^ 1]) + data[5:] def inventory_status_selftest(log=print, check=None): """Exercise request retention and empty-ledger evidence using memory tables and stubbed Amazon responses.""" from unittest.mock import patch import duckdb cx = _client() failures = [] def ck(name, ok): if check: check("[inventory status] " + name, bool(ok)) else: log(f" {'ok' if ok else 'FAIL'} {name}") if not ok: failures.append(name) report_type = cx.REPORT_TYPES["inventory"]["report"] window = {"start": "2026-11-09T00:00:00Z", "end": "2026-11-09T23:59:59Z"} header = "date\tmsku\tfnsku\tdisposition\tlocation\tending-warehouse-balance\n" meta = {"reportId": "fixture-inventory-pending", "processingStatus": "DONE", "reportDocumentId": "fixture-ledger-document", "dataEndTime": window["end"]} con = duckdb.connect(":memory:") try: con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") con.execute("CREATE TABLE amzn_orders AS SELECT i AS fixture_row FROM range(2) t(i)") con.execute("CREATE TABLE amzn_listings AS SELECT i AS fixture_row FROM range(3) t(i)") with patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("offline fixture")), \ patch.object(cx, "access_token", return_value="fixture-access"), \ patch.object(cx, "_create_report", return_value="fixture-inventory-terminal"): unknown = inventory_table_status(con) ck("missing metadata and an absent mirror remain unknown", unknown["code"] == "request_unknown" and unknown["rows"] is None and unknown["empty"] is None) projected = first_load_table_status([], set(), {}, {}, table_status={"inventory": unknown})["inventory"] ck("job projection preserves an absent mirror's reason and unknown count", projected["rows"] is None and projected["code"] == "request_unknown") pending = cx.AmazonReportPending("fixture-inventory-pending", "IN_PROGRESS") with patch.object(cx, "report_document", side_effect=pending): try: _report_document(con, "inventory", {}, report_type, **window) except cx.AmazonReportPending: pass status = inventory_table_status(con) ck("pending report keeps its actual ID and state", status["code"] == "first_load_pending" and status["reports"][0]["reportId"] == pending.report_id and status["reports"][0]["processingStatus"] == "IN_PROGRESS") with patch.object(cx, "report_document", return_value=(header, meta)) as document: text, response = _report_document(con, "inventory", {}, report_type, **window) ck("completion resumes the saved ID and clears only the pending marker", document.call_args.kwargs["report_id"] == pending.report_id and not _read_state(con, _pending_report_key("inventory", report_type, **window)) and json.loads(_read_state(con, entity_key(INVENTORY_REQUEST_KEY))["detail"])["reportId"] == meta["reportId"]) def land_empty(response): columns, records = parse_flat_file(header) _replace_legacy_snapshot(con, "inventory", columns, records, response, document_row_count(header)) _record_state(con, "amzn_inventory", "done", 0) _record_state(con, "amzn_inventory_awd", "done", 0) land_empty(response) status = inventory_table_status(con) ck("empty source document names cause and request evidence", status["code"] == "no_source_rows" and status["rows"] == 0 and "retry Amazon sync" in status["reason"] and status["reports"][0]["reportId"] == meta["reportId"] and status["reports"][0]["processingStatus"] == "DONE") ck("an empty source does not certify a metric", _validate_snapshot(con, "inventory")["state"] == "unproven") _record_state(con, "amzn_inventory_awd", "pending", 0) ck("unfinished AWD cannot prove empty inventory", inventory_table_status(con)["code"] != "no_source_rows") _record_state(con, "amzn_inventory_awd", "done", 0) with patch.object(cx, "_request", return_value={"processingStatus": "CANCELLED", "dataEndTime": window["end"]}): text, cancelled = _report_document(con, "inventory", {}, report_type, **window) land_empty(cancelled) status = inventory_table_status(con) ck("confirmed no-data cancellation needs no document ID", text == "" and status["code"] == "no_source_rows" and status["reports"][0]["processingStatus"] == "CANCELLED" and status["reports"][0]["reportId"] == "fixture-inventory-terminal") saved = _read_state(con, "amzn_reconcile_inventory") proof = json.loads(saved["detail"]) proof.pop("cancelled") _record_state(con, "amzn_reconcile_inventory", "done", detail=json.dumps(proof)) ck("a cancellation label alone proves no empty source", inventory_table_status(con)["code"] != "no_source_rows") _record_state(con, "amzn_reconcile_inventory", "done", detail=saved["detail"]) for terminal in ("FATAL", "CANCELLED"): with patch.object(cx, "_request", return_value={"processingStatus": terminal, "errorMessage": "Fixture source rejection"}): try: _report_document(con, "inventory", {}, report_type, **window) except cx.AmazonReportError as exc: ck(terminal + " error carries the report ID", exc.report_id == "fixture-inventory-terminal") else: ck(terminal + " error carries the report ID", False) status = inventory_table_status(con) ck(terminal + " refusal retains request state instead of claiming empty", status["code"] == "refused_by_amazon" and status["reports"][0]["processingStatus"] == terminal and status["reports"][0]["reportId"] == "fixture-inventory-terminal" and status["reason"].count(".") == 1 and ", so " in status["reason"]) with patch.object(cx, "_request", side_effect=cx.AmazonError("Fixture status read failed")): try: _report_document(con, "inventory", {}, report_type, **window) except cx.AmazonError: pass status = inventory_table_status(con) ck("a failed status read retains the created ID and unknown vendor state", status["reports"][0]["reportId"] == "fixture-inventory-terminal" and status["reports"][0]["processingStatus"] is None) with patch.object(cx, "report_document", side_effect=cx.AmazonThrottled("Fixture throttle")): try: _report_document(con, "inventory", {}, report_type, **window) except cx.AmazonThrottled: pass status = inventory_table_status(con) ck("creation throttle retains an unknown ID and an actionable cause", status["code"] == "first_load_pending" and status["reports"][0]["reportId"] is None and status["reports"][0]["state"] == "throttled" and "limited" in status["reason"]) with patch.object(cx, "report_document", return_value=(header, meta)): _report_document(con, "inventory", {}, report_type, **window) long_meta = {**meta, "reportId": "fixture-" + "r" * 160, "reportDocumentId": "fixture-" + "d" * 160} with patch.object(cx, "report_document", return_value=(header, long_meta)): _report_document(con, "inventory", {}, report_type, **window) ck("request metadata survives beyond the short-marker limit", json.loads(_read_state(con, entity_key(INVENTORY_REQUEST_KEY))["detail"])["reportDocumentId"] == long_meta["reportDocumentId"]) with patch.object(cx, "report_document", return_value=(header, meta)): _report_document(con, "inventory", {}, report_type, **window) scoped = _account_prefix("fixture-seller") con.execute("INSERT INTO _sync_state SELECT ? || entity, phase, last_id, cursor_wd, rows, updated_at " "FROM _sync_state", [scoped]) _record_state(con, "amzn_pending_report_stale", "pending", detail=json.dumps({ "table": "inventory", "reportId": "fixture-stale-unscoped", "processingStatus": "IN_PROGRESS"})) status = inventory_table_status(con) ck("scoped proof ignores stale unscoped pending requests", status["code"] == "no_source_rows" and all(row["reportId"] != "fixture-stale-unscoped" for row in status["reports"])) _record_state(con, scoped + "amzn_pending_report_previous_day", "pending", detail=json.dumps({ "table": "inventory", "reportId": "fixture-previous-day", "processingStatus": "IN_PROGRESS"})) con.execute("UPDATE _sync_state SET updated_at='2026-11-08T00:00:00Z' WHERE entity=?", [scoped + "amzn_pending_report_previous_day"]) ck("a completed newer request supersedes an old pending window", inventory_table_status(con)["code"] == "no_source_rows") _record_state(con, _account_prefix("fixture-other") + "amzn_inventory", "pending") ck("another incomplete seller prevents a combined empty claim", inventory_table_status(con)["code"] != "no_source_rows") con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [_account_prefix("fixture-other")]) _record_state(con, "amzn_pending_report_other", "pending", detail=json.dumps({"table": "orders"})) con.execute("INSERT INTO amzn_inventory (_snapshot_date, inventory_source) VALUES ('2026-11-09', 'FBA')") status = inventory_table_status(con) ck("nonempty inventory still returns request metadata", status["rows"] == 1 and not status["empty"] and status["reports"][0]["reportId"] == meta["reportId"]) kept = _restored_state_rows(_sync_state_rows(con), ["amzn_inventory"], ["amzn_inventory"]) ck("a refused restore clears the request with its inventory proof", not any(row[0].endswith(entity_key(INVENTORY_REQUEST_KEY)) for row in kept)) ck("inventory diagnostics preserve orders and listings", con.execute("SELECT COUNT(*) FROM amzn_orders").fetchone()[0] == 2 and con.execute("SELECT COUNT(*) FROM amzn_listings").fetchone()[0] == 3) finally: con.close() return len(failures) def snapshot_selftest(log=print, check=None): """S66-T47: real sync with in-memory storage and fixture-only vendor transports.""" from unittest.mock import patch import duckdb cx = _client() failures = [] def ck(name, probe): try: ok = bool(probe()) except Exception as exc: ok = False log(f" {name}: {type(exc).__name__}: {exc}") if check: check("[snapshots] " + name, ok) else: log(f" {'ok' if ok else 'FAIL'} {name}") if not ok: failures.append(name) class Clock(datetime): instant = datetime(2026, 9, 25, 12, tzinfo=timezone.utc) @classmethod def now(cls, tz=None): return cls.instant.astimezone(tz) start = Clock.instant - timedelta(days=65) ck("both legacy snapshots plan one task and reduce the ETA", lambda: first_load_plan(["listings", "inventory"], start, Clock.instant) == [("listings", None, None), ("inventory", None, None)] and first_load_eta(2, burst=0) < first_load_eta(6, burst=0)) listing = "seller-sku\tquantity\nA\t2\nB\t3\n" ledger_header = "date\tmsku\tfnsku\tasin\tdisposition\tlocation\tending-warehouse-balance\n" ledger = ledger_header + ("09/23/2026\tA\tFA\tAA\tSELLABLE\tUS\t2\n" "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t3\n" "09/24/2026\tA\tFA\tAA\tDAMAGED\tUS\t1\n" "09/24/2026\tA\tFA\tAA\tSELLABLE\tCA\t4\n") for table in ("listings", "inventory"): con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") rt = _FixtureSyncRuntime("qa-snapshot-" + table, [table]) calls = [] awd_payload = [] awd_rows = _awd_rows payload = [listing if table == "listings" else ledger] meta = {"createdTime": "2026-09-25T11:00:00Z"} mirror = table_name(table) def document(_fields, rtype, **kwargs): calls.append(kwargs) return payload[0], {**meta, "reportDocumentId": f"snapshot-{len(calls)}"} def count(): return con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] def valid(): return validate(rt.key, rt=rt, log=lambda *_a: None)[table] with _fixture_sync_store(con, rt), \ patch.object(sys.modules[__name__], "credential", return_value={"fixture": True}), \ patch.object(sys.modules[__name__], "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(sys.modules[__name__], "_sync_event"), \ patch.object(sys.modules[__name__], "DEFAULT_BACKFILL_DAYS", 65), \ patch.object(sys.modules[__name__], "datetime", Clock), \ patch.object(cx, "report_document", side_effect=document), \ patch.object(cx, "awd_inventory", side_effect=lambda *_: list(awd_payload)), \ patch.object(sys.modules[__name__], "_awd_rows", side_effect=lambda records, operation_id=None: awd_rows(records, "operation:fixture-awd")), \ patch.object(cx, "access_token", side_effect=AssertionError("offline fixture")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("offline fixture")): ck(table + " never synced is unproven", lambda: valid()["state"] == "unproven") deferred = sync(rt.key, tables=[table], rt=rt, first_load=True, report_budget=0, log=lambda *_a: None) ck(table + " a zero budget defers its one snapshot task", lambda: not calls and deferred["remaining"] == 1 and deferred["etaSeconds"] == first_load_eta(1, burst=0)) result = sync(rt.key, tables=[table], rt=rt, first_load=True, report_budget=3, log=lambda *_a: None) ck(table + " three-window first load requests one snapshot with its declared dates", lambda: len(calls) == 1 and (("start" in calls[0] and "end" in calls[0]) if table == "inventory" else ("start" not in calls[0] and "end" not in calls[0])) and result["state"] == "done" and result["remaining"] == 0) ck(table + " all document rows reconcile", lambda: count() == (2 if table == "listings" else 4) and valid()["state"] == "ok") ck(table + " real sync publishes attributed rows and a durable snapshot", lambda: bool(datastore._publication(con, "amazon").get("snapshot_id")) and bool(rt.blobs.get(MIRROR_PREFIX + _MANIFEST_KEY)) and con.execute(f"SELECT COUNT(*) FROM {mirror} WHERE _connection_id=?", ["snapshot-account"]).fetchone()[0] == count()) # Prove the check is independent of the parser, without mutating any real store. before = con.execute(f"SELECT * FROM {mirror}").fetchall() parse = parse_flat_file def dropping_parser(*args, **kwargs): columns, records = parse(*args, **kwargs) return columns, records[:-1] with patch.object(sys.modules[__name__], "parse_flat_file", side_effect=dropping_parser): dropped = sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) ck(table + " parser row loss refuses before replacing the stored snapshot", lambda: dropped["tables"][table].get("state") == "refused" and con.execute(f"SELECT * FROM {mirror}").fetchall() == before) calls.pop() # The deliberate refused request is not part of the replay count below. if table == "listings": ck("listings row count equals distinct date and SKU", lambda: con.execute("SELECT COUNT(*), COUNT(DISTINCT (_snapshot_date, seller_sku)) " "FROM amzn_listings").fetchone() == (2, 2)) meta["createdTime"] = "2026-09-26T11:00:00Z" sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) # S67-T32 (D-3031, reversed): a second day REPLACES the first, one load's rows. ck("a second listing day replaces the first, leaving one copy", lambda: con.execute("SELECT _snapshot_date, COUNT(*) FROM amzn_listings " "GROUP BY 1 ORDER BY 1").fetchall() == [("2026-09-26", 2)] and len(calls) == 2 and all("start" not in c and "end" not in c for c in calls)) payload[0] = "seller-sku\tquantity\nA\t9\n" sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None) ck("same listing date replaces removed SKUs", lambda: con.execute("SELECT _snapshot_date, seller_sku, quantity FROM amzn_listings " "ORDER BY 1, 2").fetchall() == [("2026-09-26", "A", 9)] and valid()["state"] == "ok") else: ck("ledger keeps one row per data date, SKU, disposition and location", lambda: con.execute("SELECT COUNT(*), COUNT(DISTINCT (date, msku, disposition, location)), " "COUNT(DISTINCT _snapshot_date) FROM amzn_inventory").fetchone() == (4, 4, 2) and calls[0].get("options", {}).get("aggregateByTimePeriod") == "DAILY") payload[0] = ledger_header + "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t9\n" sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None) # S67-T32 (D-3031, reversed): the next document replaces every earlier FBA date. ck("ledger second document replaces every earlier data date", lambda: con.execute("SELECT _snapshot_date, ending_warehouse_balance FROM amzn_inventory " "WHERE inventory_source='FBA' ORDER BY 1").fetchall() == [("2026-09-24", 9)]) # Keep the count constant but turn a second SKU into a duplicate: a count-only # implementation would falsely reconcile this source document. payload[0] = (listing if table == "listings" else ledger_header + "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t2\n" "09/24/2026\tB\tFB\tAB\tSELLABLE\tUS\t3\n") sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) date = "2026-09-26" if table == "listings" else "2026-09-24" sku = "seller_sku" if table == "listings" else "msku" try: con.execute(f"UPDATE {mirror} SET {sku}='A' WHERE _snapshot_date=?", [date]) ck(table + " validation refuses duplicate identity at unchanged row count", lambda: valid()["state"] == "refused") con.execute(f"DELETE FROM {mirror} WHERE _snapshot_date=? AND _report_row_number=2", [date]) ck(table + " validation refuses document row loss", lambda: valid()["state"] == "refused") except duckdb.Error: ck(table + " validation refuses duplicate identity at unchanged row count", lambda: False) ck(table + " validation refuses document row loss", lambda: False) if table == "listings": payload[0] = "seller-sku\tquantity\n" sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) ck("empty listing restatement replaces the copy and stays unproven", lambda: count() == 0 and valid()["state"] == "unproven") meta["dataEndTime"] = "2026-09-27T23:00:00Z" payload[0] = "seller-sku\tquantity\n" + "".join(f"S{i}\t1\n" for i in range(1205)) sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) ck("the report data date wins over creation time and no row cap is applied", lambda: con.execute("SELECT COUNT(*) FROM amzn_listings WHERE _snapshot_date='2026-09-27'").fetchone()[0] == 1205 and valid()["state"] == "ok") else: payload[0] = ledger_header + "09/25/2026\t\tF-fallback\tAA\tSELLABLE\tUS\t5\n" sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) ck("the ledger uses FNSKU when MSKU is absent", lambda: valid()["state"] == "ok" and con.execute( "SELECT COUNT(*) FROM amzn_inventory WHERE _snapshot_date='2026-09-25' " "AND fnsku='F-fallback'").fetchone()[0] == 1) awd_payload.append({"sku": "AWD-kept"}) sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) before_awd = con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall() ck("ledger source proof excludes AWD", lambda: valid()["state"] == "ok") payload[0] = ledger_header + "09/25/2026\t\tF-fallback\tAA\tSELLABLE\tUS\t9\n" replaced = sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) ck("FBA replacement preserves AWD rows exactly", lambda: replaced["tables"][table].get("state", "done") == "done" and con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall() == before_awd and con.execute("SELECT ending_warehouse_balance FROM amzn_inventory " "WHERE inventory_source='FBA'").fetchall() == [(9,)]) con.close() ck("inventory request retention and empty-source controls", lambda: inventory_status_selftest(log, check) == 0) ck("S67-T32 saved mirror: one copy, content keys, kill-safe checkpoint, full purge", lambda: mirror_selftest(log, check) == 0) return len(failures) def _fixture_fee_records(estimates): """Answer fee estimates as Amazon does: echo the IdType sent and refuse one Product Fees v0 lacks.""" cx = _client() result = [] for estimate in estimates: sent = cx.fee_estimate_request(estimate, cx.US_MARKETPLACE_ID) request = sent["FeesEstimateRequest"] if sent["IdType"] not in ("SellerSKU", "ASIN"): # Amazon's live answer on 2026-11-02 to IdType SKU. raise cx.AmazonReportError("getMyFeesEstimates refused an estimate. Amazon reason", f"{sent['IdType']} is not a supported IdType", "InvalidParameterValue") result.append({"Status": "Success", "FeesEstimateIdentifier": { **{k: v for k, v in request.items() if k != "Identifier"}, "IdType": sent["IdType"], "IdValue": sent["IdValue"], "SellerInputIdentifier": request["Identifier"]}, "FeesEstimate": {"TotalFeesEstimate": {"Amount": 2.5, "CurrencyCode": "USD"}, "FeeDetailList": []}}) return result def snapshot_sync_selftest(log=print, check=None): """S70-T34: every current-state source lands once while history spans several windows. Exercise the candidate pull with an in-memory mirror. The report fixture returns the same document rows for every requested window, so repeated pulls cannot hide behind disjoint fixture SKUs. Saved-mirror restore and checkpoint integrity are separate tests. """ sync = _sync_candidate from contextlib import ExitStack from collections import Counter from copy import deepcopy from types import SimpleNamespace from unittest.mock import patch import duckdb cx = _client() module = sys.modules[__name__] failures = [] def ck(name, probe): try: ok = bool(probe()) except Exception as exc: ok = False log(f" {name}: {type(exc).__name__}: {exc}") if check: check("[snapshot sync] " + name, ok) else: log(f" {'ok' if ok else 'FAIL'} {name}") if not ok: failures.append(name) class Clock(datetime): instant = datetime(2026, 9, 25, 12, tzinfo=timezone.utc) @classmethod def now(cls, tz=None): return cls.instant.astimezone(tz) # Enumerate the vendor declarations independently of the store's snapshot helper. snapshots = tuple(table for table, spec in cx.REPORT_TYPES.items() if spec.get("snapshot")) selected = ("orders", "order_items", "transactions", *snapshots) ck("snapshot census includes every declared current-state source", lambda: bool(snapshots) and set(_snapshot_tables()) == set(snapshots)) rt = SimpleNamespace(key="qa-snapshot-sync", get=lambda key, **_kwargs: ( {"databases": {amazon_schema.DATABASE_PREFIX + t: t in selected for t in TABLES}} if key == ENABLEMENT_KEY else None)) start = Clock.instant - timedelta(days=65) calls, generation, feedback_asins = [], [0], [] report_tables = {spec["report"]: table for table, spec in cx.REPORT_TYPES.items() if table in snapshots and spec.get("report")} order_reports = {cx.REPORT_TYPES["orders"][key] for key in ("report", "report_incremental")} api_tables = {cx.REPORT_TYPES[table]["api"]: table for table in snapshots if cx.REPORT_TYPES[table].get("api")} orders = ("amazon-order-id\tsku\tquantity\tpurchase-date\torder-status\n" f"O1\tS1\t1\t{_iso(start + timedelta(days=1))}\tShipped\n" f"O1\tS2\t2\t{_iso(start + timedelta(days=1))}\tShipped\n" f"O2\tS3\t3\t{_iso(start + timedelta(days=61))}\tShipped\n") transactions = [{"transactionId": f"T{i}", "postedDate": _iso(start + timedelta(days=day)), "totalAmount": {"currencyAmount": 10 * i, "currencyCode": "USD"}, "breakdowns": [{"breakdownType": "Sales", "breakdownAmount": { "currencyAmount": 10 * i, "currencyCode": "USD"}}], "items": [{"contexts": [{"contextType": "ProductContext", "sku": f"S{j}"}]} for j in (1, 2)]} for i, day in enumerate((1, 30, 35), 1)] def size(table): return (986 if table == "restock_recommendations" else 2) - generation[0] def expected_rows(table): return size(table) + len(cx.REPORT_TYPES[table].get("json_extra_roots", ())) def document(_fields, report_type, **kwargs): table = "orders" if report_type in order_reports else report_tables[report_type] calls.append((table, kwargs)) if table == "orders": body = orders else: spec = cx.REPORT_TYPES[table] records = [{"sku": f"S{i}", "asin": f"B{i + 1:09d}", "quantity": generation[0] + 2} for i in range(size(table))] if spec.get("json_root"): data = {spec["json_root"]: records} data.update({root: [{"status": "NORMAL"}] for root in spec.get("json_extra_roots", ())}) body = json.dumps(data) elif table == "inventory": day = (Clock.instant - timedelta(days=1)).strftime("%m/%d/%Y") body = ("date\tmsku\tfnsku\tasin\tdisposition\tlocation\tending-warehouse-balance\n" + "".join(f"{day}\tS{i}\tF{i}\tB{i + 1:09d}\tSELLABLE\tUS\t2\n" for i in range(size(table)))) else: header = "seller-sku\tasin1\tquantity\n" if table == "listings" else "sku\tasin\tquantity\n" body = header + "".join(f"{r['sku']}\t{r['asin']}\t{r['quantity']}\n" for r in records) return body, {"reportDocumentId": f"snapshot-sync-{len(calls)}", "createdTime": _iso(Clock.instant)} def records(_fields, operation, **kwargs): table = api_tables[operation] calls.append((table, kwargs)) return [{"asin": f"B{i + 1:09d}", "sku": f"S{i}"} for i in range(size(table))] def finances(_fields, posted_after, posted_before): calls.append(("transactions", {"start": posted_after, "end": posted_before})) # Return the same full-history payload on every call, including boundary-overlap rows. # The loader must partition and replace it without multiplying amounts or item rows. return deepcopy(transactions) def fee_inputs(_con, _fields): return [{"sku": f"S{i}", "price": {"Amount": 12.5, "CurrencyCode": "USD"}, "is_amazon_fulfilled": True} for i in range(size("fee_estimates_api"))] def fee_records(_fields, estimates): calls.append(("fee_estimates_api", {})) return _fixture_fee_records(estimates) def outbound_records(_fields): calls.append(("outbound_fulfillment", {})) return [{"fulfillmentOrder": {"sellerFulfillmentOrderId": "OUT-1"}, "fulfillmentOrderItems": [{"sellerSku": f"S{i}", "sellerFulfillmentOrderItemId": f"ITEM-{i}", "quantity": generation[0] + 2} for i in range(size("outbound_fulfillment"))]}] def inbound_records(_fields): calls.append(("inbound_shipments", {})) return [{"inboundPlanId": "plan-1", "planStatus": "ACTIVE", "shipmentId": "shipment-1", "status": "WORKING", "items": [{"msku": f"S{i}", "quantity": generation[0] + 2} for i in range(size("inbound_shipments"))]}] def feedback_records(_fields, asins, *, operation="getItemReviewTopics"): if operation == "getItemReviewTrends": calls.append(("review_trends", {})) return [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "reviewTrends": { "positiveTopics": [{"topic": f"Fixture trend {generation[0]}", "trendMetrics": [{ "dateRange": {"startDate": "2026-10-01", "endDate": "2026-10-31"}, "asinMetrics": {"occurrencePercentage": 12.5}}]}], "negativeTopics": None}} for asin in asins] calls.append(("review_topics", {})) feedback_asins.append(tuple(asins)) if any(not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in asins): raise AssertionError("Snapshot fixture supplied an invalid child ASIN") return [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": { "positiveTopics": [{"topic": f"Fixture topic {generation[0]}", "asinMetrics": { "numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5, "starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins] def browse_nodes(_fields, asins): return {asin: {"browse_node_id": "1000", "browse_node_name": "Fixture node"} for asin in asins} def review_topic_records(fields, asins): empty = [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {"positiveTopics": None}} for asin in asins] return {"MENTIONS": feedback_records(fields, asins), "STAR_RATING_IMPACT": empty, "browse_nodes": browse_nodes(fields, asins)} def category_records(_fields, nodes): calls.append(("browse_node_review_topics", {})) topics = [{"topic": f"Fixture category {generation[0]} {i}", "browseNodeMetrics": { "occurrencePercentage": {"allProducts": 12.5}}} for i in range(size("browse_node_review_topics"))] return {"MENTIONS": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {"positiveTopics": topics}} for node in nodes], "STAR_RATING_IMPACT": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {}} for node in nodes]} def snapshot_counts(requests): return Counter(table for table, _kwargs in requests if table in snapshots) def history_counts(): return (con.execute("SELECT COUNT(*), COUNT(DISTINCT amazon_order_id) FROM amzn_orders").fetchone(), con.execute("SELECT COUNT(*), SUM(quantity) FROM amzn_order_items").fetchone(), con.execute("SELECT COUNT(*), COUNT(DISTINCT transaction_id), SUM(total_amount) " "FROM amzn_transactions").fetchone()) con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") def replenishment_records(_fields): calls.append(("replenishment", {})) offers = [{"asin": f"B{i + 1:09d}", "sku": f"S{i}", "fulfillmentChannelType": "AMAZON", "timeInterval": {"startDate": "2026-08-01T00:00:00Z", "endDate": "2026-08-31T23:59:59Z"}} for i in range(size("replenishment"))] return {"sellingPartnerMetrics": [], "offerMetrics": offers, "offerCount": len(offers), "marketplaceId": cx.US_MARKETPLACE_ID, "start": "2026-08-01T00:00:00Z", "end": "2026-08-31T23:59:59Z"} try: with ExitStack() as stack: for fixture in ( patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_a: (con.cursor(), None)), patch.object(module, "credential", return_value={"fixture": True}), patch.object(module, "restore_mirror", return_value={"problems": []}), patch.object(module, "persist_mirror"), patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), patch.object(module, "_sync_event"), patch.object(module, "DEFAULT_BACKFILL_DAYS", 65), patch.object(module, "datetime", Clock), patch.object(cx, "report_document", side_effect=document), patch.object(cx, "group_a_records", side_effect=records), patch.object(cx, "transactions", side_effect=finances), patch.object(module, "_fee_estimate_inputs", side_effect=fee_inputs), patch.object(cx, "fee_estimates", side_effect=fee_records), patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records), patch.object(cx, "inbound_shipment_items", side_effect=inbound_records), patch.object(cx, "item_review_feedback", side_effect=feedback_records), patch.object(cx, "item_review_topics", side_effect=review_topic_records), patch.object(cx, "item_browse_nodes", side_effect=browse_nodes), patch.object(cx, "browse_node_review_topics", side_effect=category_records), patch.object(cx, "replenishment_metrics", side_effect=replenishment_records), patch.object(cx, "awd_inventory", return_value=[]), patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")), patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("offline snapshot sync fixture")), ): stack.enter_context(fixture) first = sync(rt.key, tables=selected, rt=rt, first_load=True, report_budget=len(snapshots) + 2, log=lambda *_a: None) ck("first load requests every snapshot once alongside multiwindow history", lambda: not first["problems"] and first["state"] == "partial" and first["remaining"] == 4 and snapshot_counts(calls) == Counter(dict.fromkeys(snapshots, 1))) offset = len(calls) resumed = sync(rt.key, tables=selected, rt=rt, first_load=True, report_budget=10, log=lambda *_a: None) ck("history continuation never requests an already landed snapshot", lambda: not resumed["problems"] and resumed["state"] == "done" and resumed["remaining"] == 0 and not snapshot_counts(calls[offset:]) and Counter(table for table, _kwargs in calls if table in ("orders", "transactions")) == {"orders": 3, "transactions": 3}) ck("repeated history documents retain full counts without multiplying totals", lambda: history_counts() == ((2, 2), (3, 6), (6, 3, 60))) for table in snapshots: ck(table + " first load stores one document's rows", lambda: con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] == expected_rows(table)) ck("review topics first load reads every current listing ASIN once", lambda: feedback_asins == [("B000000001", "B000000002")] and con.execute("SELECT asin, topic FROM amzn_review_topics ORDER BY asin").fetchall() == [("B000000001", "Fixture topic 0"), ("B000000002", "Fixture topic 0")]) # Advance beyond every real refresh floor; do not weaken vendor cadence in the fixture. Clock.instant += timedelta(seconds=max(cx.REPORT_TYPES[t].get("min_interval_s", 0) for t in snapshots) + 86400) generation[0] = 1 offset = len(calls) resynced = sync(rt.key, tables=selected, since=start, rt=rt, log=lambda *_a: None) resync_calls = calls[offset:] ck("explicit multiwindow resync requests every snapshot exactly once", lambda: not resynced["problems"] and snapshot_counts(resync_calls) == Counter(dict.fromkeys(snapshots, 1)) and Counter(table for table, _kwargs in resync_calls if table in ("orders", "transactions")) == {"orders": 3, "transactions": 3}) ck("snapshot transports use no backfill window except the declared ledger day", lambda: all((kwargs.get("start") == _iso(Clock.instant.replace(hour=0) - timedelta(days=1)) and kwargs.get("end") == _iso(Clock.instant.replace(hour=0) - timedelta(seconds=1))) if cx.REPORT_TYPES[table].get("dated_snapshot") else kwargs.get("start") is None and kwargs.get("end") is None for table, kwargs in resync_calls if table in snapshots)) for table in snapshots: date = (Clock.instant - timedelta(days=1) if table == "inventory" else Clock.instant).date().isoformat() ck(table + " a new snapshot replaces older dates and removed rows", lambda: con.execute(f"SELECT COUNT(*), COUNT(DISTINCT _snapshot_date), MAX(_snapshot_date) " f"FROM {table_name(table)}").fetchone() == (expected_rows(table), 1, date)) ck("review topics resync replaces removed ASINs and changed topics", lambda: feedback_asins == [("B000000001", "B000000002"), ("B000000001",)] and con.execute("SELECT asin, topic, asin_metrics_number_of_mentions " "FROM amzn_review_topics ORDER BY asin").fetchall() == [("B000000001", "Fixture topic 1", 3)]) ck("resync history totals still match the full fixture history", lambda: history_counts() == ((2, 2), (3, 6), (6, 3, 60))) # A watermark replay preserves the mirror, so this catches plain append behavior # that an explicit since (which clears history first) would otherwise conceal. # Retain older rows outside the replay, so clearing all history cannot pass either. older = _iso(start - timedelta(days=1)) for table in ("orders", "order_items"): con.execute(f"INSERT INTO {table_name(table)} SELECT * REPLACE " "('kept-order' AS amazon_order_id, 'kept-sku' AS sku, 4 AS quantity, " "? AS purchase_date, 'kept-history' AS _report_document_id) " f"FROM {table_name(table)} WHERE amazon_order_id='O2'", [older]) con.execute("INSERT INTO amzn_transactions SELECT * REPLACE " "('kept-transaction' AS transaction_id, ? AS posted_date, 7 AS total_amount, " "1 AS transaction_item_count, 7 AS transaction_breakdown_sales_amount, " "'kept-history' AS _report_document_id) " "FROM amzn_transactions WHERE transaction_id='T1' AND item_position=0", [older]) for table in ("orders", "order_items", "transactions"): record_watermark(con, table, start) replay = sync(rt.key, tables=("orders", "order_items", "transactions"), rt=rt, log=lambda *_a: None) ck("incremental multiwindow replay removes duplication without clearing history", lambda: not replay["problems"] and all(replay["tables"][t]["incremental"] for t in ("orders", "order_items", "transactions")) and history_counts() == ((3, 3), (4, 10), (7, 4, 67))) finally: con.close() return len(failures) def mirror_selftest(log=print, check=None): """Exercise offline snapshot restore, partial refusal, checkpoint retention and purge fixtures. DuckDB memory tables and dictionary blobs cover legacy and chunked layouts without vendor calls. """ from unittest.mock import patch import duckdb failures = [] def ck(name, probe): try: ok = bool(probe()) except Exception as exc: # noqa: BLE001 ok = False log(f" {name}: {type(exc).__name__}: {exc}") if check: check(name if name.startswith("[restore]") else "[mirror] " + name, ok) else: log(f" {'ok' if ok else 'FAIL'} {name}") if not ok: failures.append(name) class Store(_FixtureMirrorRuntime): key = "qa-mirror" def __init__(self): self.blobs, self.uploads, self.hook = {}, [], None def get(self, *_a, **_k): return None def upload_bytes(self, path, data, message=None): if self.hook: self.hook("upload", path) self.blobs[path] = bytes(data) self.uploads.append(path) def download_bytes(self, path): if self.hook: self.hook("download", path) return self.blobs.get(path) def delete_path(self, path): self.blobs.pop(path, None) return True rt = Store() manifest_path, ledger_path = MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") def forget(): _MIRROR_CACHE.pop(_mirror_cache_key(rt.key, rt), None) def wipe(): """A new process whose local disk was lost.""" for name, _kind in _amazon_objects(con): con.execute(f"DROP TABLE {_quoted_identifier(name)}") con.execute("DELETE FROM _sync_state WHERE starts_with(entity, 'amzn_')") forget() def dump(): out = {name: sorted(con.execute(f"SELECT * FROM {_quoted_identifier(name)}").fetchall(), key=repr) for name, _kind in _amazon_objects(con)} out["_sync_state"] = sorted(r for r in _sync_state_rows(con) if r[0].startswith(PREFIX)) return out def ledger_doc(): try: return json.loads(rt.blobs[ledger_path]) except (KeyError, ValueError): return {"version": 1, "tenant": rt.key, "keys": {}} def mirror_blobs(): return {k for k in rt.blobs if k.startswith(MIRROR_PREFIX)} def persist(tables=()): report = {"problems": []} persist_mirror(rt.key, rt, con, set(tables), report=report, log=lambda *_a: None) return report def restore(): return restore_mirror(rt.key, rt, log=lambda *_a: None) def legacy_save(): """Today's staging layout, byte for byte: v1, in-place keys, no ledger.""" manifest = {"version": 1, "tenant": rt.key, "tables": {}} import tempfile as _tmp with _tmp.TemporaryDirectory(prefix="amazon-legacy-") as tmp: for name, _kind in _amazon_objects(con): path = Path(tmp) / (name + ".parquet") con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? " "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)]) data = path.read_bytes() rt.blobs[MIRROR_PREFIX + name + ".parquet"] = data manifest["tables"][name] = {**_blob_metadata(data), "written_at": "2026-09-27T00:00:00Z", "rows": con.execute(f"SELECT count(*) FROM {name}").fetchone()[0]} rows = [list(r) for r in _sync_state_rows(con) if r[0].startswith(PREFIX)] state = _mirror_json(rows) rt.blobs[MIRROR_PREFIX + "sync_state.json"] = state manifest.update(sync_state={**_blob_metadata(state), "rows": len(rows)}, written_at="2026-09-27T00:00:00Z") rt.blobs[manifest_path] = _mirror_json(manifest) def seed(): con.execute("CREATE TABLE amzn_restock_recommendations AS SELECT * FROM (VALUES " "('2026-09-25', 'A', 1), ('2026-09-25', 'B', 2), ('2026-09-25', 'C', 3), " "('2026-09-26', 'A', 4), ('2026-09-26', 'B', 5), ('2026-09-26', 'C', 6)) " "AS t(_snapshot_date, sku, qty)") con.execute("CREATE TABLE amzn_inventory AS SELECT * FROM (VALUES " "('2026-09-22', 'A', 'FBA', 1), ('2026-09-23', 'A', 'FBA', 2), " "('2026-09-24', 'A', 'FBA', 3), (NULL, 'AWD-1', 'AWD', 9)) " "AS t(_snapshot_date, msku, inventory_source, qty)") con.execute("CREATE TABLE amzn_orders AS SELECT * FROM (VALUES " "('2026-09-01', 'O1', 10.5), ('2026-09-02', 'O2', 11.5)) AS t(_period_start, id, amount)") con.execute("CREATE TABLE amzn_listings AS SELECT * FROM (VALUES ('2026-09-26', 'S1', 1)) " "AS t(_snapshot_date, seller_sku, quantity)") _record_state(con, entity_key("reconcile_inventory"), "done", detail=json.dumps( {"start": "2026-09-23", "end": "2026-09-24", "column": "_snapshot_date", "count": 2, "dates": {"2026-09-23": 1, "2026-09-24": 1}})) record_watermark(con, "orders", datetime(2026, 9, 26, tzinfo=timezone.utc), 2) _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z") try: with _fixture_sync_store(con, rt): # D-3031 at the write: an API snapshot pulled on a new day replaces the old day. for day, qty in (("2026-09-25", 1), ("2026-09-26", 2)): rows = [_origin({"sku": s, "qty": qty, "_snapshot_date": day}, "doc-" + day, i, "fixture") for i, s in enumerate(("A", "B"), 1)] _replace_period(con, "fba_inventory", rows, ["sku", "qty", "_snapshot_date"], "_snapshot_date", day, snapshot=True) ck("two snapshot days leave one copy of one load's rows", lambda: con.execute( "SELECT _snapshot_date, count(*) FROM amzn_fba_inventory GROUP BY 1").fetchall() == [("2026-09-26", 2)]) ck("a period table keeps its history", lambda: _replace_period( con, "storage_fees", [_origin({"month": "2026-08", "fee": 1}, "d", 1, "f")], ["month", "fee"], "month", "2026-08") == 1 and _replace_period( con, "storage_fees", [_origin({"month": "2026-09", "fee": 1}, "d", 1, "f")], ["month", "fee"], "month", "2026-09") == 1 and con.execute( "SELECT count(DISTINCT month) FROM amzn_storage_fees").fetchone()[0] == 2) con.execute("DROP TABLE amzn_fba_inventory") con.execute("DROP TABLE amzn_storage_fees") # The legacy manifest staging boots from, holding a doubled snapshot. seed() original = dump() legacy_save() legacy_blobs = dict(rt.blobs) wipe() result = restore() ck("a legacy v1 manifest with in-place keys restores every table and marker", lambda: result["state"] == "restored" and not result["problems"] and dump()["amzn_orders"] == original["amzn_orders"] and dump()["amzn_listings"] == original["amzn_listings"] and dump()["_sync_state"] == original["_sync_state"]) ck("a doubled mirror collapses on restore to the latest pull and is marked dirty", lambda: con.execute("SELECT _snapshot_date, count(*) FROM amzn_restock_recommendations " "GROUP BY 1").fetchall() == [("2026-09-26", 3)] and con.execute("SELECT _snapshot_date FROM amzn_inventory WHERE inventory_source='FBA' " "ORDER BY 1").fetchall() == [("2026-09-23",), ("2026-09-24",)] and con.execute("SELECT count(*) FROM amzn_inventory WHERE inventory_source='AWD'").fetchone()[0] == 1 and {"amzn_restock_recommendations", "amzn_inventory"} <= result["dirty"] and "amzn_orders" not in result["dirty"]) ck("the collapse is idempotent", lambda: _collapse_snapshots(con) == {}) # First checkpoint after the legacy boot: every blob moves to a content key. legacy_keys = set(_manifest_keys(json.loads(legacy_blobs[manifest_path]))) rt.uploads.clear() report = persist() saved = json.loads(rt.blobs[manifest_path]) named = set(_manifest_keys(saved)) ck("the first checkpoint migrates to content keys and retires the in-place blobs", lambda: not report["problems"] and saved["version"] == MANIFEST_VERSION and all(_TABLE_BLOB_RE.fullmatch(k).group(2) for k in named if k.endswith(".parquet")) and not any(MIRROR_PREFIX + k in rt.blobs for k in legacy_keys) and not any(MIRROR_PREFIX + k in rt.uploads for k in legacy_keys) and set(ledger_doc()["keys"]) == named and rt.uploads.index(ledger_path) < min(rt.uploads.index(MIRROR_PREFIX + k) for k in named) and rt.uploads[-2 if rt.uploads[-1] == ledger_path else -1] == manifest_path) healed = dump() wipe() ck("NC a normal checkpoint restores exactly, one copy each", lambda: restore()["state"] == "restored" and dump() == healed and con.execute("SELECT count(*) FROM amzn_restock_recommendations").fetchone()[0] == 3) rt.uploads.clear() before = dict(rt.blobs) persist(set(original)) ck("an unchanged checkpoint uploads nothing", lambda: rt.uploads == [] and rt.blobs == before) # A changed table never overwrites a key the published manifest names. con.execute("UPDATE amzn_orders SET amount = amount + 1") rt.uploads.clear() persist({"amzn_orders"}) ck("content keys never overwrite a named key", lambda: not any(path[len(MIRROR_PREFIX):] in named for path in rt.uploads) and _manifest_keys(json.loads(rt.blobs[manifest_path])) and not (MIRROR_PREFIX + saved["tables"]["amzn_orders"]["key"]) in rt.blobs) # A checkpoint that dies after its first table blob lands. published = rt.blobs[manifest_path] good = dump() con.execute("UPDATE amzn_orders SET amount = amount + 1") con.execute("UPDATE amzn_listings SET quantity = quantity + 1") parquet = [] def die_on_second(kind, path): if kind == "upload" and path.endswith(".parquet"): parquet.append(path) if len(parquet) == 2: raise OSError("fixture: the process died mid-checkpoint") rt.hook = die_on_second report = persist({"amzn_orders", "amzn_listings"}) rt.hook = None orphan = parquet[0][len(MIRROR_PREFIX):] if parquet else "absent" ck("a checkpoint that dies after one table upload leaves the manifest untouched", lambda: any("persistence failed" in p for p in report["problems"]) and rt.blobs[manifest_path] == published and MIRROR_PREFIX + orphan in rt.blobs) ck("the key ledger lists the unpublished orphan", lambda: orphan in ledger_doc()["keys"] and orphan not in _manifest_keys(json.loads(published))) wipe() ck("the next process restores the previous snapshot whole", lambda: restore()["state"] == "restored" and dump() == good) # An aged orphan is retired by the next successful publish; a fresh one is not. ledger = ledger_doc() ledger["keys"][orphan] = "2026-01-01T00:00:00Z" rt.blobs[ledger_path] = _mirror_json(ledger) con.execute("UPDATE amzn_orders SET amount = amount + 5") persist({"amzn_orders"}) ck("an aged ledger orphan is retired after the next publish", lambda: MIRROR_PREFIX + orphan not in rt.blobs and orphan not in ledger_doc()["keys"] and set(ledger_doc()["keys"]) >= set( _manifest_keys(json.loads(rt.blobs[manifest_path])))) # A refused restore (store unreachable at boot) must not publish its rebuild. valid_blobs = dict(rt.blobs) valid_rows = dump() wipe() def unreachable(kind, path): if kind == "download" and path == manifest_path: raise OSError("fixture: store unreachable at boot") rt.hook = unreachable refused = restore() rt.hook = None con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-02' AS _period_start, 'O2' AS id, 1.0 AS amount") _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z") rt.uploads.clear() report = persist({"amzn_orders"}) ck("a refused restore does not publish a partial rebuild over the valid manifest", lambda: refused["state"] == "refused" and rt.blobs == valid_blobs and rt.uploads == [] and any("checkpoint held" in p for p in report["problems"])) _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z") report = persist({"amzn_orders"}) ck("the refused process cannot publish after its own first load completes", lambda: rt.blobs == valid_blobs and rt.uploads == [] and any("checkpoint held" in p for p in report["problems"])) # A corrupt state refuses publication until a new process restores valid saved bytes. wipe() corrupt = json.loads(rt.blobs[manifest_path]) corrupt["sync_state"]["rows"] += 1 rt.blobs[manifest_path] = _mirror_json(corrupt) corrupt_blobs = dict(rt.blobs) rt.uploads.clear() refused = restore() con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-03' AS _period_start, 'O3' AS id, 2.0 AS amount") _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z") report = persist({"amzn_orders"}) ck("a corrupt stored state refuses whole and cannot publish its rebuild", lambda: refused["state"] == "refused" and rt.blobs == corrupt_blobs and rt.uploads == [] and any("checkpoint held" in p for p in report["problems"])) rt.blobs = dict(valid_blobs) wipe() recovered = restore() ck("a new process restores the valid snapshot before later checkpoints", lambda: recovered["state"] == "restored" and not recovered["problems"] and dump() == valid_rows) # Purge reaches every version: legacy keys, named keys and a killed checkpoint's orphan. for path, data in legacy_blobs.items(): if path != manifest_path: rt.blobs[path] = data con.execute("UPDATE amzn_orders SET amount = 7") parquet.clear() rt.hook = lambda kind, path: (_ for _ in ()).throw(OSError("fixture death")) if ( kind == "upload" and path == manifest_path) else None persist({"amzn_orders"}) rt.hook = None rt.blobs["unrelated/keep.bin"] = b"keep" orphans = set(ledger_doc()["keys"]) - set( _manifest_keys(json.loads(rt.blobs[manifest_path]))) _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)]) ck("purge deletes every stored version including the orphan", lambda: orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"}) # ⭐⭐ D-2639 (owner 2026-09-29 "make sure the chunking works"): a table over the cap is # stored as month parts, none over the cap, and a checkpoint uploads only changed parts. module = sys.modules[__name__] con.execute("CREATE OR REPLACE TABLE amzn_transactions AS SELECT " "'2026-0' || (1 + i % 3) || '-15' AS posted_date, i AS id, md5(i::VARCHAR) AS a, " "md5((i * 7)::VARCHAR) AS b FROM range(9000) t(i)") cap = 60000 with patch.object(module, "MIRROR_PART_MAX_BYTES", 10 ** 9): persist({name for name, _kind in _amazon_objects(con)}) whole = json.loads(rt.blobs[manifest_path])["tables"]["amzn_transactions"] ck("NC: under a huge cap the same table is one blob, and that blob is over the real cap", lambda: "parts" not in whole and whole["bytes"] > cap) def is_part(path): return bool(re.search(r"\.p[0-9a-z-]+\.[0-9a-f]{16}\.parquet$", path)) def txn_keys(): entry = json.loads(rt.blobs[manifest_path])["tables"]["amzn_transactions"] return [part["key"] for part in entry.get("parts", [])] with patch.object(module, "MIRROR_PART_MAX_BYTES", cap): start = len(rt.uploads) chunk_report = persist({"amzn_transactions"}) if chunk_report["problems"]: log(" chunked persist problems: " + " | ".join(chunk_report["problems"])) saved = json.loads(rt.blobs[manifest_path]) entry = saved["tables"]["amzn_transactions"] written = [p for p in rt.uploads[start:] if is_part(p)] ck("a table over the cap is saved as parts, none over the cap, rows intact (v3)", lambda: saved["version"] == 3 and len(entry["parts"]) >= 3 and written and all(len(rt.blobs[p]) <= cap for p in written) and entry["rows"] == 9000 == sum(part["rows"] for part in entry["parts"]) and {p["part"][:7] for p in entry["parts"]} == {"2026-01", "2026-02", "2026-03"}) before = txn_keys() start = len(rt.uploads) persist({"amzn_transactions"}) ck("an unchanged chunked table uploads no part again", lambda: not [p for p in rt.uploads[start:] if is_part(p)] and txn_keys() == before) con.execute("INSERT INTO amzn_transactions SELECT '2026-03-20', 9000 + i, " "md5(('x' || i)::VARCHAR), 'new' FROM range(50) t(i)") start = len(rt.uploads) persist({"amzn_transactions"}) moved = [p for p in rt.uploads[start:] if is_part(p)] ck("new rows in one month upload only that month's parts", lambda: moved and all(".p2026-03" in p for p in moved) and set(txn_keys()) - set(before) == {p[len(MIRROR_PREFIX):] for p in moved} and [k for k in before if ".p2026-01" in k] == [k for k in txn_keys() if ".p2026-01" in k]) expected = dump() wipe() restored = restore() ck("a chunked snapshot restores every row exactly", lambda: restored["state"] == "restored" and dump() == expected) saved_blobs = dict(rt.blobs) corrupt_part = MIRROR_PREFIX + txn_keys()[1] rt.blobs[corrupt_part] = _fixture_damaged_parquet(rt.blobs[corrupt_part]) wipe() partial = restore() ck("[restore] one corrupt part refuses its whole table and restores other tables", lambda: partial["state"] == "partial" and partial["refused_tables"] == ["amzn_transactions"] and "amzn_transactions" not in dump() and dump()["amzn_orders"] == expected["amzn_orders"]) rt.blobs = saved_blobs wipe() restore() _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)]) ck("purge deletes every part of a chunked table", lambda: not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"}) wipe() for name, value in (("amzn_inventory", 1), ("amzn_inventory_ledger_detail", 2), ("amzn_orders", 3)): con.execute(f"CREATE TABLE {name} AS SELECT {value} AS id") table = name[len(PREFIX):] _record_state(con, name, "done", 1) record_watermark(con, table, datetime(2026, 9, 26, tzinfo=timezone.utc), 1) record_backfill_floor(con, table, datetime(2026, 9, 1, tzinfo=timezone.utc)) for family in ("sync_tick_", "reconcile_", "limit_", "refused_"): _record_state(con, entity_key(family + table), "done", 1, "{}") _record_state(con, "amzn_inventory_awd", "done", 1) _record_state(con, "amzn_reconcile_inventory_2026_09", "done", 1, "{}") from harness.amazon_client import AmazonReportPending refused_pending = set() for table in ("inventory", "orders"): before_pending = {row[0] for row in _sync_state_rows(con)} try: with _report_resume(con, table, _client().REPORT_TYPES[table]["report"], start="2026-09-01", end="2026-09-26"): raise AmazonReportPending("fixture-" + table, "IN_PROGRESS") except AmazonReportPending: pass if table == "inventory": refused_pending = {row[0] for row in _sync_state_rows(con)} - before_pending for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY): _record_state(con, entity_key(key), "done", detail="2026-09-26T00:00:00Z") persist({name for name, _kind in _amazon_objects(con)}) snapshot = dict(rt.blobs) expected = dump() saved = json.loads(snapshot[manifest_path]) bad_entry = saved["tables"]["amzn_inventory"] bad_blob = MIRROR_PREFIX + bad_entry["key"] refused_markers = {"amzn_inventory", "amzn_sync_mark_inventory", "amzn_sync_floor_inventory", "amzn_sync_tick_inventory", "amzn_reconcile_inventory", "amzn_limit_inventory", "amzn_refused_inventory", "amzn_inventory_awd", "amzn_reconcile_inventory_2026_09", "amzn_sync_first_load_end", "amzn_sync_first_load_status", "amzn_sync_first_load_next_due"} refused_markers.update(refused_pending) good_markers = [row for row in expected["_sync_state"] if row[0] not in refused_markers] damaged_inventory = _fixture_damaged_parquet(rt.blobs[bad_blob]) rt.blobs[bad_blob] = damaged_inventory ledger = ledger_doc() ledger["keys"][bad_entry["key"]] = "2026-01-01T00:00:00Z" rt.blobs[ledger_path] = _mirror_json(ledger) wipe() strict_refused = False with tempfile.TemporaryDirectory(prefix="amazon-strict-recheck-") as tmp: try: _validated_snapshot(rt.key, rt, con, rt.blobs[manifest_path], tmp) except ValueError: strict_refused = True ck("[restore] strict revalidation still refuses table content faults", lambda: strict_refused) partial = restore() ck("[restore] one bad table restores the rest", lambda: partial["state"] == "partial" and partial["refused_tables"] == ["amzn_inventory"] and partial["manifest"]["tables"]["amzn_inventory"] == bad_entry and set(dump()) == {"_sync_state", "amzn_orders", "amzn_inventory_ledger_detail"} and all(dump()[name] == expected[name] for name in ("amzn_orders", "amzn_inventory_ledger_detail"))) ck("[restore] a refused table has no watermark floor reconciliation or progress markers", lambda: not refused_markers.intersection(row[0] for row in dump()["_sync_state"]) and read_watermark(con, "inventory") is None and read_backfill_floor(con, "inventory") is None and _read_state(con, entity_key(FIRST_LOAD_END_KEY)) is None) ck("[restore] good table markers survive exactly including overlapping names", lambda: dump()["_sync_state"] == good_markers) if "amzn_orders" in dump(): con.execute("UPDATE amzn_orders SET id = 4") rt.uploads.clear() report = persist({"amzn_orders"}) checkpoint = json.loads(rt.blobs[manifest_path]) ck("[restore] checkpoint retains the refused entry and stored blob", lambda: not report["problems"] and checkpoint["tables"].get("amzn_inventory") == bad_entry and rt.blobs.get(bad_blob) == damaged_inventory and bad_blob not in rt.uploads and bad_entry["key"] in ledger_doc()["keys"] and json.loads(rt.blobs[MIRROR_PREFIX + checkpoint["sync_state"]["key"]]) == [list(row) for row in good_markers]) wipe() partial = restore() ck("[restore] a checkpoint after partial restore remains partial on restart", lambda: partial["state"] == "partial" and partial["refused_tables"] == ["amzn_inventory"] and dump()["amzn_orders"] == [(4,)] and dump()["_sync_state"] == good_markers) for fault in ("JSON", "root", "tenant", "table map", "unsafe name", "table entry", "state metadata", "state checksum", "state access"): wipe() rt.blobs = dict(snapshot) manifest = json.loads(snapshot[manifest_path]) if fault == "JSON": rt.blobs[manifest_path] = b"{" elif fault == "root": manifest = [] elif fault == "tenant": manifest["tenant"] = "other-tenant" elif fault == "table map": manifest["tables"] = [] elif fault == "unsafe name": manifest["tables"]["amzn_bad/name"] = bad_entry elif fault == "table entry": manifest["tables"]["amzn_inventory"] = [] elif fault == "state metadata": manifest["sync_state"] = [] elif fault == "state checksum": rt.blobs[MIRROR_PREFIX + manifest["sync_state"]["key"]] = b"[]" if fault != "JSON": rt.blobs[manifest_path] = _mirror_json(manifest) downloaded = [] def observe(kind, path): if kind == "download": downloaded.append(path) if fault == "state access" and path == MIRROR_PREFIX + manifest["sync_state"]["key"]: raise OSError("fixture: state is inaccessible") rt.hook = observe try: whole = restore() finally: rt.hook = None ck("[restore] " + fault + " refuses before table content validation", lambda: whole["state"] == "refused" and whole["refused_tables"] == [] and dump() == {"_sync_state": []} and not any(path.endswith(".parquet") for path in downloaded)) # S79-T19, S79-T23: a PUBLISHED snapshot and a hand-off heal the rows they serve. split_window = ("2026-03-20T18:01:25Z", "2026-04-19T18:01:25Z") def seed_quoted(): origin = "'GET_FIXTURE', 'Amazon Services API'" con.execute("CREATE TABLE amzn_pricing AS SELECT * FROM (VALUES " f"('P1', 'C1', 'S1', 5, 'doc-old', 1, {origin}), " f"('P1', 'C2', 'S2', 7, 'doc-old', 2, {origin}), " f"('P2', 'C1', 'S1', 9, 'doc-new', 1, {origin})) AS t(parent_asin, child_asin, sku, " "units, _report_document_id, _report_row_number, _report_type, _data_origin)") con.execute("CREATE TABLE amzn_inventory_ledger_detail AS SELECT * FROM (VALUES " "('2026-02-18T18:01:25Z', '2026-03-20T18:01:25Z', '\"SKU-1\"', '\"Plain\"', " "'\"Receipts\"', '\"12\"', 'doc-a', 1), " f"('{split_window[0]}', '{split_window[1]}', '\"SKU-2\"', '\"A vase', " "'with a tab\"', '\"Shipments\"', 'doc-b', 1)) AS t(_period_start, _period_end, msku, " "title, event_type, quantity, _report_document_id, _report_row_number)") def served(): return (con.execute("SELECT child_asin, parent_asin FROM amzn_pricing ORDER BY 1").fetchall(), con.execute("SELECT msku, quantity FROM amzn_inventory_ledger_detail " "ORDER BY _period_start").fetchall()) healed_rows = ([("C1", "P2"), ("C2", "P1")], [("SKU-1", "12"), ("SKU-2", "Shipments")]) rt.blobs = {} wipe() restore() seed_quoted() record_watermark(con, "inventory_ledger_detail", datetime(2026, 10, 23, tzinfo=timezone.utc), 2) report = {"problems": []} _MIRROR_STORE.persist_mirror(rt.key, rt, con, {"amzn_pricing", "amzn_inventory_ledger_detail"}, report=report, log=lambda *_a: None, _publication={ "snapshot_id": uuid.uuid4().hex, "started_at": "2026-10-24T00:00:00Z", "state": "complete"}) wipe() published = restore() ck("[restore] a published snapshot restore heals quoted and doubled rows, not marked dirty", lambda: not report["problems"] and published["state"] == "restored" and bool(published["manifest"].get("snapshot_id")) and not published["problems"] and not published["dirty"] and served() == healed_rows) ck("[restore] a split ledger row queues its own report window for download again", lambda: queued_resync(con, "inventory_ledger_detail") == (_utc(split_window[0]), _utc(split_window[1]))) settled = dump() forget() again = restore() ck("[restore] the heal on a local restart of the same snapshot changes nothing", lambda: again["state"] == "local" and dump() == settled) resync_landed(con, "inventory_ledger_detail", queued_resync(con, "inventory_ledger_detail")) ck("[restore] a landed resync window leaves the queue", lambda: queued_resync(con, "inventory_ledger_detail") is None and _read_state(con, entity_key(RESYNC_PREFIX + "inventory_ledger_detail")) is None) import tempfile as _tmp with _tmp.TemporaryDirectory(prefix="amazon-handoff-") as tmp: wipe() seed_quoted() paths = {} for name in ("amzn_pricing", "amzn_inventory_ledger_detail"): paths[name] = [Path(tmp) / (name + ".parquet")] con.execute(f"COPY {name} TO ? (FORMAT PARQUET)", [str(paths[name][0])]) wipe() _mirrors.install_handoff("amazon", rt.key, rt, {"snapshot_id": uuid.uuid4().hex, "tables": {}}, [], paths) ck("[restore] a hand-off install heals the rows the web process serves", lambda: served() == healed_rows) # A re-parented child replaces its own window's row; another window keeps its own. con.execute("DROP TABLE amzn_pricing") def land_pricing(doc, parent, start): w0 = datetime(2026, 10, start, tzinfo=timezone.utc) rows = [_origin({"parent_asin": parent, "child_asin": "C1", "sku": "S1", "units": 3}, doc, 1, "GET_SALES_AND_TRAFFIC_REPORT")] cols, rows = stamp_week(["parent_asin", "child_asin", "sku", "units", *_ORIGIN_TYPES], rows, w0, w0 + timedelta(days=7)) types = _ensure(con, "amzn_pricing", cols, rows) return replace_window(con, "amzn_pricing", cols, types, rows, replace_keys_for("pricing")) landed = [land_pricing("doc-1", "P1", 1), land_pricing("doc-2", "P2", 1)] ck("[restore] a re-parented pricing child replaces its older document's row", lambda: landed == [(1, None), (1, None)] and con.execute( "SELECT parent_asin, _report_document_id FROM amzn_pricing").fetchall() == [("P2", "doc-2")]) land_pricing("doc-3", "P2", 8) ck("[restore] a later pricing window keeps its own row beside the earlier window", lambda: con.execute("SELECT count(*), count(DISTINCT _period_start) FROM amzn_pricing").fetchone() == (2, 2)) finally: forget() con.close() return len(failures) def margin_selftest(log=print, check=None): """S66-T40 offline fixtures and red-first controls, shared with api_amazon. Every new invariant first sees a planted defect, then the restored implementation. Only in-memory DuckDB and mocked vendor transports are used. """ from copy import deepcopy from unittest.mock import patch import duckdb cx = _client() failures = [] def ck(name, ok): if check: check(name, bool(ok)) else: log(f" {'ok' if ok else 'FAIL'} {name}") if not ok: failures.append(name) def rejects(call): try: call() except (cx.AmazonError, ValueError): return True return False with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")): excluded = "GET_SELLER_FEEDBACK_DATA" approved = cx.OPERATIONS["createReport"] with patch.dict(approved, {"report_types": (*approved["report_types"], excluded)}): probe = lambda: rejects(lambda: cx.require_operation("POST", cx._REPORTS_CREATE_PATH, report_type=excluded)) with patch.object(cx, "EXCLUDED_REPORT_TYPES", frozenset()): red = not probe() ck("margin excluded report refusal survives a future allowlist addition", red and probe()) probe = lambda: not set(cx._APPROVED_REPORT_TYPES).intersection(cx.EXCLUDED_REPORT_TYPES) with patch.object(cx, "_APPROVED_REPORT_TYPES", (*cx._APPROVED_REPORT_TYPES, excluded)): red = not probe() ck("margin every approved report is outside the compliance exclusion set", red and probe()) injected = '2026-09-01") { otherDataset { secret } }' probe = lambda: rejects(lambda: cx.economics_query(injected, "2026-09-02", "MKT")) with patch.object(cx, "economics_query", lambda *a, **k: a[0]): red = not probe() ck("margin economics dates cannot inject caller GraphQL", red and probe()) query = cx.economics_query("2026-09-01", "2026-09-02", "MKT") ck("margin query fixes the economics dataset, DAY x MSKU and every money family", all(word in query for word in ("analytics_economics_2024_03_15", "date: DAY", "productId: MSKU", "fees", "ads", "netProceeds", "cost", "refundedProductSales")) and "date: RANGE" in cx.economics_query("2026-09-01", "2026-09-02", "MKT", summary=True)) clock = [0.0] waits = [] report_tokens = cx._report_pacer["tokens"] def advance(seconds): waits.append(seconds) clock[0] += seconds with patch.dict(cx._query_pacer, {"tokens": 15.0, "updated": None}): for _ in range(16): cx._pace_query_create(sleep=advance, now=lambda: clock[0]) ck("margin createQuery has an independent burst-15 one-per-minute bucket", waits == [60.0] and cx._report_pacer["tokens"] == report_tokens) pages = [{"payload": {"transactions": [{"transactionId": "p1"}], "nextToken": "next-1"}}, {"payload": {"transactions": [], "nextToken": "next-2"}}, {"payload": {"transactions": [{"transactionId": "p2"}]}}] with patch.object(cx, "access_token", return_value="fixture"), \ patch.object(cx, "_request", side_effect=pages) as request, \ patch.object(cx, "_pace_entity") as pace: returned = cx.transactions({}, "2026-09-01T00:00:00Z", "2026-09-03T00:00:00Z") ck("margin Finances follows every cursor through empty pages at 0.5 requests per second", [r["transactionId"] for r in returned] == ["p1", "p2"] and [c.kwargs.get("nextToken") for c in request.call_args_list] == [None, "next-1", "next-2"] and all(c.args[:2] == ("transactions", 2.0) for c in pace.call_args_list)) with patch.object(cx, "access_token", return_value="fixture"), \ patch.object(cx, "_create_query", return_value="query-fixture"), \ patch.object(cx, "_request", side_effect=[{"processingStatus": "IN_PROGRESS"}, {"processingStatus": "DONE", "dataDocumentId": "data-fixture"}, {"documentUrl": "https://fixture.invalid/document"}]), \ patch.object(cx, "_download_document", return_value='{"startDate":"2026-09-01"}'): document, meta = cx.economics_document({}, "2026-09-01", "2026-09-02", sleep=lambda _: None) ck("margin Data Kiosk polls DONE and downloads its JSONL document", "startDate" in document and meta["reportDocumentId"] == "data-fixture") with patch.object(cx, "access_token", return_value="fixture"), \ patch.object(cx, "_create_query", return_value="query-fixture"), \ patch.object(cx, "_request", side_effect=[{"processingStatus": "FATAL", "errorDocumentId": "error-fixture"}, {"documentUrl": "https://fixture.invalid/error"}]), \ patch.object(cx, "_download_document", return_value='{"message":"Economics query refused"}'): try: cx.economics_document({}, "2026-09-01", "2026-09-02") message = "" except cx.AmazonError as exc: message = str(exc) ck("margin Data Kiosk surfaces the vendor error document", "Economics query refused" in message) end, start = _utc("2026-09-03T00:00:00Z"), _utc("2026-09-01T00:00:00Z") for table in ("fba_fee_estimates", "referral_fees"): spec = cx.REPORT_TYPES[table] probe = lambda: first_load_plan([table], end-timedelta(days=730), end) == [(table, None, None)] with patch.dict(spec, {"snapshot": False}): red = not probe() ck(f"margin {table} plans exactly one snapshot task", red and probe()) ck(f"margin {table} transport refuses any snapshot data window", rejects(lambda: cx._create_report("fixture", spec["report"], ["MKT"], start=_iso(start)))) states = {table: {"state": "done", "updated_at": _iso(end-timedelta(hours=1))}} probe = lambda: table in sync_plan(states, [table], now=end)["deferred"] with patch.dict(spec, {"min_interval_s": 0}), \ patch.object(sys.modules[__name__], "SCHEDULED_FLOOR_SECONDS", SCHEDULED_FLOOR_GRACE_SECONDS): red = not probe() result = sync_plan(states, [table], now=end) ck(f"margin {table} refresh floor defers with a reason", red and probe() and "next due" in result["deferred"][table]) ck("margin storage windows follow complete calendar months", _month_windows(_utc("2026-07-15T00:00:00Z"), end) == [(_utc("2026-07-01T00:00:00Z"), _utc("2026-08-01T00:00:00Z")), (_utc("2026-08-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z"))]) ck("margin economics incremental sync restates three days", sync_window("economics", start, now=end)["start"] == start-timedelta(days=3)) for table in MARGIN_TABLES: role = cx.REPORT_TYPES[table]["role"] def probe_role(): report = {"problems": []} got = _sync_problem(cx.AmazonError("vendor answered 403"), table, cx.REPORT_TYPES[table].get("api", "report_document"), "fixture", None, report, lambda *_: None) return role in got["reason"] with patch.dict(cx.REPORT_TYPES[table], {"role": "wrong role"}): red = not probe_role() ck(f"margin {table} 403 names its required role", red and probe_role()) money = lambda value: {"amount": value, "currencyCode": "USD"} economy = {"startDate": "2026-09-01", "endDate": "2026-09-01", "msku": "S1", "marketplaceId": "MKT", "childAsin": "A1", "parentAsin": "P1", "fnsku": "F1", "sales": {"orderedProductSales": money(100), "unitsOrdered": 2, "refundedProductSales": money(0), "unitsRefunded": 0}, "fees": [{"feeTypeName": "FBA fulfillment fees", "charges": [ {"aggregatedDetail": {"totalAmount": money(3), "amount": money(3)}}]}], "ads": [{"adTypeName": "Sponsored Products charge", "charge": {"totalAmount": money(1)}}], "netProceeds": {"total": money(95), "perUnit": money(47.5)}, "cost": {"costOfGoodsSold": money(0.5)}} raw = [economy, {**deepcopy(economy), "startDate": "2026-09-02", "endDate": "2026-09-02"}] rows = economics_rows(raw, "economics-fixture") probe = lambda values: (isinstance(values[0].get("fee_fba_fulfillment_fees_amount"), float) and values[0]["fee_fba_fulfillment_fees_amount"] == 3 and values[0]["ads_sponsored_products_charge_amount"] == 1) broken = deepcopy(rows) broken[0]["fee_fba_fulfillment_fees_amount"] = "3" ck("margin economics fees and ads pivot into summable numeric columns", not probe(broken) and probe(rows)) currency = lambda n: {"currencyAmount": n, "currencyCode": "USD"} transaction = {"transactionId": "T1", "postedDate": "2026-09-01T12:00:00Z", "transactionType": "Shipment", "transactionStatus": "RELEASED", "totalAmount": currency(10), "relatedIdentifiers": [{"relatedIdentifierName": "ORDER_ID", "relatedIdentifierValue": "O1"}], "breakdowns": [{"breakdownType": "Sales", "breakdownAmount": currency(12), "breakdowns": [{"breakdownType": "Principal", "breakdownAmount": currency(12)}]}, {"breakdownType": "Fees", "breakdownAmount": currency(-2)}], "items": [{"totalAmount": currency(n), "contexts": [ {"contextType": "ProductContext", "sku": "S"+str(i), "asin": "A"+str(i)}]} for i, n in enumerate((6, 4), 1)]} transactions = [transaction, {**deepcopy(transaction), "transactionId": "T2", "items": []}] items = transaction_rows(transactions, "operation:fixture") probe = lambda values: (len(values) == 3 and sum(r.get("total_amount") or 0 for r in values) == 20 and values[1]["sku"] == "S2" and values[0]["amazon_order_id"] == "O1" and values[0]["transaction_sub_breakdown_sales__principal_amount"] == 12) broken = deepcopy(items) broken[1]["total_amount"] = 10 ck("margin transactions explode items without multiplying transaction totals", not probe(broken) and probe(items)) con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") try: for table in MARGIN_TABLES: ck(f"margin {table} missing marker is unproven", _validate_margin(con, table)["state"] == "unproven") coarse = deepcopy(economy) coarse["sales"]["orderedProductSales"] = money(200) coarse["fees"][0]["charges"][0]["aggregatedDetail"]["totalAmount"] = money(6) coarse["netProceeds"]["total"] = money(190) def fake_economics(*args, summary=False, **kwargs): values = [coarse] if summary else raw return "\n".join(json.dumps(r) for r in values), {"reportDocumentId": "fixture-economics"} with patch.object(cx, "economics_document", fake_economics): _sync_margin_window(con, "economics", {}, start, end, max_wait=1, log=log) _sync_margin_window(con, "economics", {}, start, end, max_wait=1, log=log) with patch.object(cx, "transactions", return_value=transactions): _sync_margin_window(con, "transactions", {}, start, end, max_wait=1, log=log) _sync_margin_window(con, "transactions", {}, start, end, max_wait=1, log=log) ck("margin window replacement is idempotent", con.execute("SELECT COUNT(*) FROM amzn_economics").fetchone()[0] == 2 and con.execute("SELECT COUNT(*) FROM amzn_transactions").fetchone()[0] == 3) snapshots = [] def fake_report(*args, **kwargs): snapshots.append(kwargs) return ("sku\tasin\tfnsku\testimated-fee-total\nS1\tA1\tF1\t2.50\n", {"reportDocumentId": "fixture-report", "dataEndTime": "2026-09-02T00:00:00Z"}) with patch.object(cx, "report_document", fake_report): for table in ("fba_fee_estimates", "referral_fees"): for _ in range(2): _sync_margin_window(con, table, {}, None, None, max_wait=1, log=log) ck("margin snapshot requests never pass start or end", all("start" not in k and "end" not in k for k in snapshots)) _sync_margin_window(con, "storage_fees", {}, _utc("2026-08-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z"), max_wait=1, log=log) for table in MARGIN_TABLES: mirror = table_name(table) # Deliberately damage a stored period before accepting its positive proof. column = ("sales_ordered_product_sales_amount" if table == "economics" else "total_amount" if table == "transactions" else None) if column: con.execute(f'UPDATE {mirror} SET "{column}"="{column}"+5 WHERE "{column}" IS NOT NULL') red = _validate_margin(con, table)["state"] == "refused" con.execute(f'UPDATE {mirror} SET "{column}"="{column}"-5 WHERE "{column}" IS NOT NULL') else: con.execute(f"INSERT INTO {mirror} SELECT * FROM {mirror}") red = _validate_margin(con, table)["state"] == "refused" con.execute(f"DELETE FROM {mirror} WHERE rowid IN (SELECT MAX(rowid) FROM {mirror})") ck(f"margin {table} validation refuses a mismatch then accepts matching vendor evidence", red and _validate_margin(con, table)["state"] == "ok") with patch.object(sys.modules[__name__], "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)): results = validate("fixture", rt=object(), log=lambda *_: None) ck("margin public validate dispatches every new reconciliation", all(results[t]["state"] == "ok" for t in MARGIN_TABLES)) con.execute("UPDATE amzn_transactions SET transaction_breakdown_fees_amount=-3 WHERE item_position=0") ck("margin each transaction breakdown must reconcile to its vendor total", _validate_margin(con, "transactions")["state"] == "refused") con.execute("UPDATE amzn_transactions SET transaction_breakdown_fees_amount=-2 WHERE item_position=0") with patch.object(cx, "report_document", return_value=( "sku\tasin\nS1\tA1\n", {"reportDocumentId": "next-snapshot", "dataEndTime": "2026-09-03T00:00:00Z"})): _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log) _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log) # S67-T32 (D-3031, reversed): a new day REPLACES the earlier one; one copy, one load's rows. ck("margin a second snapshot day replaces the first and same-date replay stays idempotent", con.execute("SELECT COUNT(*), COUNT(DISTINCT _snapshot_date), MAX(_snapshot_date) " "FROM amzn_referral_fees").fetchone() == (1, 1, "2026-09-03") and _validate_margin(con, "referral_fees")["state"] == "ok") kept = con.execute("SELECT * FROM amzn_referral_fees").fetchall() with patch.object(cx, "report_document", side_effect=RuntimeError("fixture vendor failure")): try: _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log) except RuntimeError: pass ck("margin a failed snapshot pull keeps the stored copy", con.execute("SELECT * FROM amzn_referral_fees").fetchall() == kept and len(kept) == 1) _replace_period(con, "economics", [], [c[1] for c in con.execute("PRAGMA table_info('amzn_economics')").fetchall()], "start_date", "2026-09-01", "2026-09-03") ck("margin empty economics restatement deletes stale detail and stays unproven", _validate_margin(con, "economics")["state"] == "unproven") finally: con.close() # Exercise the actual outer loop across bounded passes, not only its per-window helper. class Clock(datetime): @classmethod def now(cls, tz=None): return end.astimezone(tz) class Runtime(_FixtureMirrorRuntime): key = "margin-fixture" def get(self, name, *_args, **_kwargs): if name == "keychain": return {"entries": {"margin-entry": { "connection_id": "margin-account", "type": "amazon_sp", "slot": 1, "label": "Margin store"}}} return None rt = Runtime() def dynamic_economics(_fields, date_start, date_end, **_kwargs): row = {**deepcopy(economy), "startDate": date_start, "endDate": date_start} return json.dumps(row), {"reportDocumentId": "economics-" + date_start} def dynamic_transactions(_fields, after, before): when = _utc(after) + timedelta(seconds=2) return [{**deepcopy(transaction), "transactionId": "T-" + _iso(when), "postedDate": _iso(when)}] con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") try: module = sys.modules[__name__] with _fixture_sync_store(con, rt), \ patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 40), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), \ patch.object(cx, "economics_document", side_effect=dynamic_economics) as kiosk, \ patch.object(cx, "transactions", side_effect=dynamic_transactions) as finances, \ patch.object(cx, "report_document", side_effect=fake_report) as reports: first = sync(Runtime.key, tables=MARGIN_TABLES, rt=rt, first_load=True, report_budget=2, log=lambda *_: None) ck("margin API windows consume the first-load pass budget", first["state"] == "partial" and kiosk.call_count == 2 and finances.call_count == 1 and reports.call_count == 0) for _ in range(3): last = sync(Runtime.key, tables=MARGIN_TABLES, rt=rt, first_load=True, report_budget=2, log=lambda *_: None) checked = validate(Runtime.key, rt=rt, log=lambda *_: None) with _fixture_account_state(con, rt) as account: watermarks = {t: read_watermark(account, t) for t in MARGIN_TABLES} ck("margin bounded passes resume to completion across all five table kinds", last["state"] == "done" and last["remaining"] == 0 and all(watermarks.values()) and all(checked[t]["state"] == "ok" for t in MARGIN_TABLES)) ck("margin real sync publishes a durable snapshot without store problems", bool(datastore._publication(con, "amazon").get("snapshot_id")) and bool(rt.blobs.get(MIRROR_PREFIX + _MANIFEST_KEY)) and not last["problems"]) ck("margin final account pass retains tables completed by earlier passes", all(_mirrors._table_sync_complete("amazon", last["tables"][t]) and "margin-account" in last["tables"][t]["accounts"] for t in MARGIN_TABLES)) calls = reports.call_count deferred = sync(Runtime.key, tables=["referral_fees", "fba_fee_estimates", "storage_fees"], rt=rt, log=lambda *_: None) ck("margin manual sync also honors vendor report floors", reports.call_count == calls and all(r["state"] == "deferred" for r in deferred["tables"].values())) finally: con.close() return 1 if failures else 0 def sqp_selftest(log=print, check=None): """S66-T41 fixtures, with broken inputs exercised before restored positives. No network.""" # This fixture inspects private windows; section_publication_boundary proves public sync. sync = _sync_candidate from unittest.mock import patch import duckdb cx, module = _client(), sys.modules[__name__] failures = [] def ck(name, ok): if check: check(name, bool(ok)) else: log(f" {'ok' if ok else 'FAIL'} {name}") if not ok: failures.append(name) asins = [f"B{i:09d}" for i in range(37)] valid_batches = lambda batches: (all(len(b) <= 200 for b in batches) and [a for b in batches for a in b.split()] == asins) ck("SQP planted 201-character batch fails before lossless stable batching passes", not valid_batches(["X" * 201]) and valid_batches(sqp_batches(asins[::-1] + asins))) try: sqp_batches(["X" * 201]) oversized = False except ValueError: oversized = True ck("SQP a single oversized identifier is refused by name, never dropped", oversized) ck("SQP empty and exact-boundary batches are supported", sqp_batches([]) == [] and sqp_batches(["A" * 100, "B" * 99]) == ["A" * 100 + " " + "B" * 99]) start, end = _utc("2026-07-17T00:00:00Z"), _utc("2026-09-25T00:00:00Z") def calendar_probe(): plan = first_load_plan([SQP_TABLE], start, end, sqp_asins=asins) plan = [task for task in plan if _sqp_period(task[1], task[2]) == "MONTH"] return (len(plan) == 6 and [task[1].strftime("%Y-%m") for task in plan] == ["2026-08"] * 3 + ["2026-07"] * 3 and all(task[1].day == task[2].day == 1 and len(task[3]) <= 200 for task in plan)) with patch.object(module, "_month_windows", return_value=[(start, end)]): red = not calendar_probe() ck("SQP first-load plan counts every completed calendar month and batch, newest first", red and calendar_probe()) plan = sync_window(SQP_TABLE, _utc("2026-09-01T00:00:00Z"), now=end) ck("SQP incremental window defers the partial month", plan["start"] == plan["end"] and not _month_windows(plan["start"], plan["end"])) con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") con.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR)") con.execute("INSERT INTO amzn_pricing VALUES ('B999999999')") ck("SQP catalogue falls back to pricing without a listings mirror", _sqp_catalogue(con) == ["B999999999"]) con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins[:19][::-1] + asins[:2]]) ck("SQP own listings take precedence and are sorted and deduplicated", _sqp_catalogue(con) == asins[:19]) foreign = asins[1] reason = f"ASIN {foreign} is not part of your brand" calls, problems = [], [] class Clock(datetime): @classmethod def now(cls, tz=None): return end.astimezone(tz) class Runtime(_FixtureMirrorRuntime): key = "sqp-fixture" def get(self, *_args, **_kwargs): return None def document(_fields, _report, **kwargs): calls.append(kwargs) requested = kwargs["options"]["asin"].split() if foreign in requested: raise cx.AmazonReportError("FATAL", reason, "FATAL") raw = [{"asin": a, "startDate": kwargs["start"][:10], "endDate": kwargs["end"][:10], "searchQueryData": {"searchQuery": "000123"}, "purchaseData": {"asinPurchaseCount": 2}, "impressionData": {"asinImpressionCount": 7}, "clickData": {"asinClickCount": 4}, "cartAddData": {"asinCartAddCount": 3}} for a in requested] return json.dumps({"dataByAsin": raw}), {"reportDocumentId": "sqp-fixture-" + str(len(calls))} try: with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")), \ patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 70), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \ patch.dict(cx.REPORT_TYPES, {SQP_TABLE: {k: v for k, v in cx.REPORT_TYPES[SQP_TABLE].items() if k != "history"}}), \ patch.dict(cx.REPORT_TYPES[SQP_TABLE], {"periods": ("MONTH",)}): # The fixture's months sit on the 70-day fallback, so the measured SQP `from` rule is patched out. def run(**kwargs): result = sync(Runtime.key, tables=[SQP_TABLE], rt=Runtime(), log=lambda *_: None, **kwargs) problems.extend(result["problems"]) return result ck("SQP no reconciliation marker is unproven", _validate_sqp(con)["state"] == "unproven") first = run(first_load=True, report_budget=1) ck("SQP a refused request consumes the pass budget and retains its retry task", len(calls) == 1 and first["remaining"] == 4 and first["etaSeconds"] == first_load_eta(4, burst=0) and read_watermark(con, SQP_TABLE) is None) second = run(first_load=True, report_budget=1) ck("SQP next pass retries once without the named foreign ASIN", len(calls) == 2 and foreign not in calls[1]["options"]["asin"].split() and len(calls[1]["options"]["asin"].split()) == 17 and second["remaining"] == 3 and any(sqp_refusal_message(foreign, reason) == p for p in problems)) for _ in range(4): last = run(first_load=True, report_budget=1) ck("SQP bounded passes finish all months without losing other ASINs", len(calls) == 6 and last["remaining"] == 0 and last["state"] == "done" and read_watermark(con, SQP_TABLE) == _utc("2026-09-01T00:00:00Z") and con.execute("SELECT COUNT(*), COUNT(DISTINCT asin) FROM amzn_search_query_performance").fetchone() == (36, 18)) ck("SQP JSON rows keep the query as text, all nested families and report period", con.execute("SELECT DISTINCT search_query_data_search_query, _report_period, " "impression_data_asin_impression_count, click_data_asin_click_count, " "cart_add_data_asin_cart_add_count FROM amzn_search_query_performance").fetchall() == [("000123", "MONTH", 7, 4, 3)]) limit = json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"]) ck("SQP durable limits retain the entire catalogue and named refusal beyond 300 characters", len(_read_state(con, SQP_LIMIT_KEY)["detail"]) > 300 and limit["asins"] == asins[:19] and limit["refusedAsins"] == {foreign: reason} and limit["batchCount"] == 2) con.execute("UPDATE amzn_search_query_performance SET purchase_data_asin_purchase_count=9 WHERE start_date='2026-08-01'") bad = _validate_sqp(con) con.execute("UPDATE amzn_search_query_performance SET purchase_data_asin_purchase_count=2 WHERE start_date='2026-08-01'") good = validate(Runtime.key, rt=Runtime(), log=lambda *_: None)[SQP_TABLE] ck("SQP validate refuses damaged counts before reconciling the newest month", bad["state"] == "refused" and bad["expected"] == {"count": 18, "purchases": 36} and bad["actual"] == {"count": 18, "purchases": 162} and good["state"] == "ok") before = len(calls) partial = run(report_budget=1) ck("SQP a current partial month makes no vendor request and states its cause", len(calls) == before and partial["tables"][SQP_TABLE]["state"] == "deferred" and "partial month" in partial["tables"][SQP_TABLE]["reason"]) # Explicit replay must replace the whole batch, including queries removed upstream. con.execute("UPDATE amzn_search_query_performance SET search_query_data_search_query='stale'") run(since="2026-08-01T00:00:00Z", report_budget=1) ck("SQP explicit replay replaces one ASIN batch without deleting another batch or month", con.execute("SELECT COUNT(*) FROM amzn_search_query_performance").fetchone()[0] == 36 and con.execute("SELECT COUNT(*) FROM amzn_search_query_performance WHERE search_query_data_search_query='000123'").fetchone()[0] == 17) ck("SQP newest proof survives older backfill", json.loads(_read_state( con, entity_key("reconcile_" + SQP_TABLE))["detail"])["start"] == "2026-08-01") w0, w1 = _utc("2026-08-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z") def empty_document(*_a, **_k): return '{"dataByAsin":[]}', {"reportDocumentId": "empty-sqp"} _sync_sqp_batch(con, {}, w0, w1, sqp_batches(asins[:18])[0], limit, empty_document, problems.append) ck("SQP empty replay removes stale batch rows and cannot validate an empty month", con.execute("SELECT COUNT(*) FROM amzn_search_query_performance WHERE start_date='2026-08-01'").fetchone()[0] == 1) _sync_sqp_batch(con, {}, w0, w1, asins[18], limit, empty_document, problems.append) ck("SQP no rows is unproven even with a summary marker", _validate_sqp(con)["state"] == "unproven") # Exercise the real FATAL-document path. The red mutation loses the offending ASIN. def fatal_probe(payload): with patch.object(cx, "access_token", return_value="fixture"), \ patch.object(cx, "_create_report", return_value="fatal-fixture"), \ patch.object(cx, "_request", side_effect=[{"processingStatus": "FATAL", "reportDocumentId": "error-fixture"}, {"url": "https://fixture.invalid/error"}]), \ patch.object(cx, "_download_document", return_value=payload): try: cx.report_document({}, cx.REPORT_TYPES[SQP_TABLE]["report"]) except cx.AmazonReportError as exc: return cx.sqp_refused_asins(exc, asins) return [] ck("SQP FATAL error document preserves Amazon's named foreign ASIN", fatal_probe("Report failed without naming an ASIN") != [foreign] and fatal_probe(reason) == [foreign]) from urllib.error import HTTPError with patch.object(cx.urllib.request, "urlopen", side_effect=HTTPError( "https://fixture.invalid", 400, "Bad Request", {}, io.BytesIO(json.dumps( {"errors": [{"message": reason}]}).encode()))), patch.object(cx, "_pace_report_create", return_value=0): try: cx._create_report("fixture", cx.REPORT_TYPES[SQP_TABLE]["report"], ["MKT"]) named = [] except cx.AmazonReportError as exc: named = cx.sqp_refused_asins(exc, asins) ck("SQP HTTP 400 preserves the same named refusal", named == [foreign]) # The second error entry and its details can carry the ASIN, past the old 200-char cut. long_reason = "Search Query Performance brand refusal: " + "more vendor context " * 20 with patch.object(cx.urllib.request, "urlopen", side_effect=HTTPError( "https://fixture.invalid", 400, "Bad Request", {}, io.BytesIO(json.dumps( {"errors": [{"message": long_reason}, {"message": "Invalid ASINs", "details": foreign}]}).encode()))), \ patch.object(cx, "_pace_report_create", return_value=0) as pace: try: cx._create_report("fixture", cx.REPORT_TYPES[SQP_TABLE]["report"], ["MKT"]) named = [] except cx.AmazonReportError as exc: named = cx.sqp_refused_asins(exc, asins) ck("SQP 400 preserves every error and details after 200 characters and uses the shared pacer", named == [foreign] and pace.call_count == 1) ck("SQP unrelated failures cannot silently exclude a named valid ASIN", cx.sqp_refused_asins(cx.AmazonReportError("FATAL", f"Your brand owns {foreign}; service unavailable", "FATAL"), asins) == []) # Red first: a dropped error document cannot cause the successful one-retry path. def retry_probe(named): month_start, month_end = _utc("2026-05-01T00:00:00Z"), _utc("2026-06-01T00:00:00Z") con.execute("DELETE FROM _sync_state WHERE entity=?", [_sqp_month_key(month_start)]) attempts, named_problems = [], [] def request(fields, rtype, **kwargs): attempts.append(kwargs["options"]["asin"]) if len(attempts) == 1: raise cx.AmazonReportError("FATAL", reason if named else "Amazon report failed", "FATAL") return document(fields, rtype, **kwargs) try: _sync_sqp_batch(con, {}, month_start, month_end, " ".join(asins[:2]), limit, request, named_problems.append) except cx.AmazonError: return False return attempts == [" ".join(asins[:2]), asins[0]] and named_problems == [sqp_refusal_message(foreign, reason)] ck("SQP named FATAL triggers exactly one immediate retry without the foreign ASIN", not retry_probe(False) and retry_probe(True)) month_start, month_end = _utc("2026-04-01T00:00:00Z"), _utc("2026-05-01T00:00:00Z") attempts = [] def twice_refused(*_args, **kwargs): attempts.append(kwargs["options"]["asin"]) raise cx.AmazonReportError("FATAL", reason, "FATAL") for _ in range(2): try: _sync_sqp_batch(con, {}, month_start, month_end, " ".join(asins[:2]), limit, twice_refused, problems.append) except cx.AmazonError: pass ck("SQP a failed retry is durable and never creates a third request", attempts == [" ".join(asins[:2]), asins[0]]) requests = [] def unnamed(*_a, **_k): requests.append(True) raise cx.AmazonReportError("FATAL", "Amazon cannot generate this report", "FATAL") try: _sync_sqp_batch(con, {}, _utc("2026-06-01T00:00:00Z"), _utc("2026-07-01T00:00:00Z"), asins[0], limit, unnamed, problems.append) except cx.AmazonReportError: pass ck("SQP unnamed errors stay verbatim and are never retried", len(requests) == 1 and json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])["errors"] == ["Amazon cannot generate this report"]) finally: con.close() _sqp_resume_legs(ck, Runtime, asins[:19]) return 1 if failures else 0 def _sqp_resume_legs(ck, runtime, asins): """H1: a throttled or capped SQP backlog resumes at its own deadline. No network.""" from unittest.mock import patch import duckdb cx, module = _client(), sys.modules[__name__] t0 = _utc("2026-11-14T00:00:30Z") clock, holder, calls = [t0], {}, [] class Clock(datetime): @classmethod def now(cls, tz=None): return clock[0].astimezone(tz) def fresh(watermark=None): con = holder["con"] = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins]) if watermark: record_watermark(con, SQP_TABLE, _utc(watermark)) # A fixture watermark's earlier months need the same per-ASIN evidence as a real completed load. for w0, w1 in _month_windows(history_floor(con, SQP_TABLE, _utc(watermark)), _utc(watermark)): marker = _sqp_month(con, w0, w1) marker["asins"] = {a: {"count": 0, "purchases": 0} for a in asins} _sqp_save_month(con, w0, marker) calls.clear() return con throttle_at = [None] def document(_fields, _report, **kwargs): calls.append((kwargs["start"][:7], kwargs["options"]["asin"])) if throttle_at[0] is not None and len(calls) - 1 == throttle_at[0]: raise cx.AmazonThrottled("Amazon is limiting requests", retry_after=5) raw = [{"asin": a, "startDate": kwargs["start"][:10], "endDate": kwargs["end"][:10], "searchQueryData": {"searchQuery": "q"}, "purchaseData": {"asinPurchaseCount": 1}} for a in kwargs["options"]["asin"].split()] return json.dumps({"dataByAsin": raw}), {"reportDocumentId": f"h1-{len(calls)}"} def run(**kwargs): return _sync_candidate(runtime.key, tables=[SQP_TABLE], rt=runtime(), log=lambda *_: None, **kwargs) tick = lambda con: _read_state(con, entity_key("sync_tick_" + SQP_TABLE)) month = lambda w0: (w0, (w0.replace(day=28) + timedelta(days=4)).replace(day=1)) sep, octo = month(_utc("2026-09-01T00:00:00Z")), month(_utc("2026-10-01T00:00:00Z")) batches = sqp_batches(asins, cx.REPORT_TYPES[SQP_TABLE]["asin_option_max_chars"]) con = None try: with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")), \ patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 70), \ patch.object(module, "_SCHEDULE_OFFSET_READER", lambda: 0), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (holder["con"].cursor(), None)), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \ patch.object(cx, "settlement_reports", return_value=[]) as settlements, \ patch.dict(cx.REPORT_TYPES[SQP_TABLE], {"periods": ("MONTH",)}): # A pass throttled at batch k keeps batches < k and resumes at k after the deadline. con = fresh("2026-09-01T00:00:00Z") throttle_at[0] = 2 first = run(scheduled=True) saved = tick(con) kept = [_sqp_done(con, *octo, b) for b in batches] + [_sqp_done(con, *sep, batches[0])] holds = _MIRROR_STORE._publication_holds(first, ()) row = first["tables"][SQP_TABLE] throttle_at[0] = None floor = _cadence_seconds(SQP_TABLE, saved) waits = [_interval_wait(SQP_TABLE, saved, t0 + timedelta(seconds=s), floor_s=floor)[0] for s in (60, 901)] daily = _interval_wait("storage_fees", saved, t0 + timedelta(seconds=901), floor_s=_cadence_seconds("storage_fees", saved))[0] ck("SQP throttled tick follows its own deadline while another table keeps its daily phase", saved["state"] == "throttled" and _utc(saved["detail"]) == t0 + timedelta(seconds=SQP_RESUME_SECONDS) and 0 < waits[0] <= SQP_RESUME_SECONDS and waits[1] == 0 and daily > SQP_RESUME_SECONDS) before = list(calls) clock[0] = t0 + timedelta(seconds=901) second = run(scheduled=True) resumed = calls[len(before):] ck("SQP throttled scheduled pass resumes at the throttled batch on the next pass, never re-requesting done batches", bool(row.get("reason")) and kept == [True, True, False] and [m for m, _a in before] == ["2026-10", "2026-10", "2026-09"] and resumed == [("2026-09", batches[0]), ("2026-09", batches[1])] and tick(con)["state"] == "done" and read_watermark(con, SQP_TABLE) == _utc("2026-11-01T00:00:00Z") and second["tables"][SQP_TABLE].get("state", "done") == "done") con.close() # A pass that landed SQP batches publishes them; a pass that landed none holds SQP. clock[0] = t0 con = fresh("2026-09-01T00:00:00Z") throttle_at[0] = 0 nothing = run(scheduled=True) throttle_at[0] = None empty_holds = _MIRROR_STORE._publication_holds(nothing, ()) ck("SQP landed batches publish with their month markers; a pass that landed none stays held", row.get("state") == "deferred" and row.get("current") is True and row.get("landedBatches") == 2 and _mirrors._table_sync_complete("amazon", row) and (holds is None or SQP_TABLE not in holds["held"]) and nothing["tables"][SQP_TABLE]["state"] == "throttled" and empty_holds is not None and SQP_TABLE in empty_holds["held"]) con.close() # A scheduled pass during an unfinished first load leaves SQP on the daily phase, not due every minute. t1 = t0 + timedelta(minutes=30) clock[0] = t1 con = fresh() _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail="2026-09-26T00:00:00Z") _record_state(con, entity_key("sync_tick_" + SQP_TABLE), "throttled", detail=_iso(t1 - timedelta(seconds=10))) stale = tick(con) run(scheduled=True) later = t1 + timedelta(seconds=60) ck("SQP throttled tick deferred by an unfinished first load is not due a minute later", sync_plan({SQP_TABLE: stale}, [SQP_TABLE], now=later)["tables"] == [SQP_TABLE] and tick(con)["state"] == "deferred" and tick(con)["detail"] == stale["detail"] and sync_plan({SQP_TABLE: tick(con)}, [SQP_TABLE], now=later)["tables"] == [] and _interval_wait(SQP_TABLE, tick(con), later, floor_s=_cadence_seconds(SQP_TABLE, tick(con)))[0] > 0 and not calls) con.close() clock[0] = t0 # A scheduled pass spends at most SQP_PASS_REPORTS SQP documents; other due tables still run. clock[0] = t0 with patch.object(module, "SQP_PASS_REPORTS", 10 ** 6): con = fresh("2026-01-01T00:00:00Z") run(scheduled=True) unbounded = len(calls) con.close() con = fresh("2026-01-01T00:00:00Z") settlements.reset_mock() capped = run(scheduled=True) saved = tick(con) ck("SQP scheduled pass stops at SQP_PASS_REPORTS documents and other due tables still run", unbounded == 20 and len(calls) == SQP_PASS_REPORTS and capped["tables"][SQP_TABLE]["state"] == "deferred" and settlements.call_count == 1 and capped["tables"]["settlements"]["state"] == "empty" and saved["state"] == "throttled" and _utc(saved["detail"]) == t0 + timedelta(seconds=SQP_RESUME_SECONDS)) limit = json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"]) progress = limit.get("progress") or {} sentence = sqp_limit_sentence(limit) bare = sqp_limit_sentence({k: v for k, v in limit.items() if k != "progress"}) ck("SQP progress sentence names months and batches done of total, without a dash", "of 20 batches" not in bare and progress.get("batchesDone") == 15 and progress.get("batchesTotal") == 20 and progress.get("monthsTotal") == 10 and progress.get("etaSeconds") == 300 and progress["months"][2] == {"month": "2026-03", "total": 2, "done": 1} and "7 of 10 months complete, 15 of 20 batches." in sentence and "About 5 minutes left" in sentence and len(sentence) - len(bare) < 300 and not any(d in sentence for d in ("\u2013", "\u2014")) and sqp_progress_sentence({}) == "" and sentence.endswith(" " + sqp_progress_sentence(limit)) and len(sqp_progress_sentence(limit)) < 140) con.close() # A first load with a cohort frozen at 2026-09-26 plans 2026-10 and 2026-09 first. chunk = {} def frozen(spliced=True): con = fresh() _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail="2026-09-26T00:00:00Z") if spliced: chunk.update(run(first_load=True, report_budget=4)) else: with patch.object(module, "_sqp_after_cohort", side_effect=lambda tasks, *_: tasks): run(first_load=True, report_budget=4) con.close() return [m for m, _a in calls] end = _utc("2026-09-26T00:00:00Z") planned = _sqp_after_cohort(first_load_plan([SQP_TABLE], end - timedelta(days=70), end, sqp_asins=asins), end, asins) red = frozen(spliced=False) ck("SQP first load with a frozen cohort plans the months closed since, newest first", red[:1] == ["2026-08"] and frozen() == ["2026-10", "2026-10", "2026-09", "2026-09"] and [t[1].strftime("%Y-%m") for t in planned[:4]] == ["2026-10", "2026-10", "2026-09", "2026-09"]) # A first-load chunk whose only progress is SQP batches publishes them; other tables stay held. waiting = {table_name("orders")} unmarked = {**chunk, "tables": {**chunk["tables"], SQP_TABLE: { k: v for k, v in chunk["tables"][SQP_TABLE].items() if k != "current"}}} chunk_holds = _MIRROR_STORE._publication_holds(chunk, waiting) ck("SQP first-load chunk of only SQP batches publishes them, never nothing", chunk.get("state") == "partial" and _MIRROR_STORE._publication_holds(unmarked, waiting) is None and chunk_holds is not None and SQP_TABLE not in chunk_holds["held"] and "orders" in chunk_holds["held"]) con = None finally: if con is not None: con.close() def group_a_selftest(log=print, check=None): """S66-T42 red-first fixture controls; memory mirrors and no credentials or network.""" from unittest.mock import patch import duckdb cx, module = _client(), sys.modules[__name__] failures = [] def ck(name, ok): if check: check("[group a] " + name, bool(ok)) else: log(f" {'ok' if ok else 'FAIL'} group a {name}") if not ok: failures.append(name) def coverage(tables, declared): return (set(tables).isdisjoint(declared) and set(tables) | set(declared) == set(cx.GROUP_A)) ck("catalogue has no omitted dataset or overlapping declaration", not coverage(TABLES[1:], cx.DECLARED_DATASETS) and coverage(TABLES, cx.DECLARED_DATASETS)) excluded = lambda reports: not set(reports).intersection(cx.EXCLUDED_REPORT_TYPES) reports = [rtype for spec in cx.GROUP_A.values() for rtype in spec.get("report_types", ())] ck("no group a report is excluded", not excluded([*reports, "GET_ORDERS_DATA"]) and excluded(reports)) operation_ok = lambda ops: (not any(any(word in str(spec).lower() for word in ("wallet", "financial_holds", "advertising-api", "amazon-ads")) for spec in ops.values())) ck("no wallet, deferred transaction or Ads operation is admitted", not operation_ok({**cx.OPERATIONS, "planted": {"path": "sellerWallet/accounts"}}) and operation_ok(cx.OPERATIONS)) begin, end = _utc("2026-07-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z") for table in GROUP_A_TABLES: spec = cx.REPORT_TYPES[table] if spec.get("report"): with patch.dict(cx.OPERATIONS["createReport"], {"report_types": ()}): try: cx.require_operation("POST", cx._REPORTS_CREATE_PATH, report_type=spec["report"]) red = False except cx.AmazonOperationRefused: red = True admitted = cx.require_operation("POST", cx._REPORTS_CREATE_PATH, report_type=spec["report"]) ck(table + " report is admitted by the guarded operation", red and bool(admitted)) if spec.get("snapshot"): def snapshot_plan(): return first_load_plan([table], begin, end) == [(table, None, None)] with patch.dict(spec, {"snapshot": False}): red = not snapshot_plan() ck(table + " snapshot plans one task", red and snapshot_plan()) floor = {"state": "done", "updated_at": _iso(end)} ck(table + " refresh floor defers then becomes due", bool(_interval_reason(table, floor, end)) and not _interval_reason(table, floor, end + timedelta(seconds=spec["min_interval_s"]))) plan = first_load_plan(["long_term_storage_fees", "market_basket", "mfn_returns"], begin, end) ck("monthly tasks are full months and MFN windows are at most sixty days", all((w0.day == w1.day == 1) if t != "mfn_returns" else (w1 - w0).total_seconds() <= 60 * 86400 for t, w0, w1 in plan) and sum(t == "long_term_storage_fees" for t, *_ in plan) == 2) con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") calls = [] def document(_fields, report, **kwargs): spec = next(s for s in cx.GROUP_A_TABLES.values() if s.get("report") == report) calls.append((report, kwargs)) if spec.get("json_root"): row = {"asin": "B000000001", "sku": "SKU-1", "marketplaceId": "MKT", "promotionId": "promo-1", "couponId": "coupon-1", "buyer-email": "planted@example.invalid", "children": [{"buyerName": "planted", "amount": 1}]} body = {spec["json_root"]: [row, dict(row, sku="SKU-2")]} body.update({root: [{"marketplaceId": "MKT", "status": "NORMAL"}] for root in spec.get("json_extra_roots", ())}) text = json.dumps(body) else: text = ("sku\tasin\tbuyer-email\tship-address-1\tdestination-address-1\tamount\n" "SKU-1\tB000000001\tplanted@example.invalid\tPrivate\tPrivate\t1.25\n" "SKU-2\tB000000002\tplanted@example.invalid\tPrivate\tPrivate\t2.50\n") return text, {"reportDocumentId": "fixture-" + report, "createdTime": _iso(end)} def records(_fields, operation, **kwargs): if operation == "getOrderMetrics": return [{"interval": kwargs["start"] + "--" + kwargs["end"], "orderCount": 2, "unitCount": 3, "totalSales": {"amount": 12.5, "currencyCode": "USD"}}] if operation == "listAllFulfillmentOrders": return [{"fulfillmentOrder": {"sellerFulfillmentOrderId": "OUT-1", "fulfillmentOrderStatus": "Complete", "destinationAddress": {"name": "planted", "addressLine1": "Private"}}, "fulfillmentOrderItems": [{"sellerSku": "SKU-1", "sellerFulfillmentOrderItemId": "ITEM-1", "quantity": 2}]}] return [{"asin": "B000000001", "sku": "SKU-1", "buyerEmail": "planted@example.invalid", "attributes": {"recipientName": "planted", "value": "retained"}}] fee_input = {"sku": "SKU-1", "price": {"Amount": 12.5, "CurrencyCode": "USD"}, "is_amazon_fulfilled": True} replenishment = {"sellingPartnerMetrics": [], "offerMetrics": [{"asin": "B000000001", "sku": "SKU-1", "fulfillmentChannelType": "AMAZON", "timeInterval": {"startDate": _iso(begin), "endDate": _iso(end)}}], "offerCount": 1, "marketplaceId": cx.US_MARKETPLACE_ID, "start": _iso(begin), "end": _iso(end)} inbound = [{"inboundPlanId": "plan-1", "planStatus": "ACTIVE", "shipmentId": "shipment-1", "status": "WORKING", "items": [{"msku": "SKU-1", "quantity": 2, "buyerEmail": "planted@example.invalid"}]}] rt = _FixtureSyncRuntime("group-a-document-fixture", GROUP_A_TABLES) con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") con.execute("INSERT INTO amzn_listings VALUES ('B000000001')") feedback = [{"asin": "B000000001", "marketplaceId": cx.US_MARKETPLACE_ID, "reviewTrends": { "positiveTopics": [{"topic": "Fixture durability", "trendMetrics": [{ "dateRange": {"startDate": "2026-10-01", "endDate": "2026-10-31"}, "asinMetrics": {"occurrencePercentage": 12.5}}]}], "negativeTopics": None}, "topics": { "positiveTopics": [{"topic": "Fixture durability", "asinMetrics": {"numberOfMentions": 4, "occurrencePercentage": 12.5, "starRatingImpact": 0.2}}], "negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2, "occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}] node = {"browse_node_id": "3734491", "browse_node_name": "Fixture node"} review_topics = {**{sort: feedback for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {"B000000001": node}} category = {sort: [{"browseNodeId": "3734491", "displayName": "Fixture node", "marketplaceId": cx.US_MARKETPLACE_ID, "dateRange": {"startDate": "2026-05-06T00:00:00.000Z", "endDate": "2026-11-06T00:00:00.000Z"}, "topics": {"positiveTopics": [{"topic": "Fixture glow", "browseNodeMetrics": { "occurrencePercentage": {"allProducts": 18.15, "topTwentyFivePercentProducts": 22.56}, "starRatingImpact": {"allProducts": 1.58, "topTwentyFivePercentProducts": 1.15}}}], "negativeTopics": [{"topic": "Fixture size", "browseNodeMetrics": { "occurrencePercentage": {"allProducts": 14.31}}}]}}] for sort in cx.FEEDBACK_SORTS} try: with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")), \ patch.object(cx, "report_document", side_effect=document), \ patch.object(cx, "group_a_records", side_effect=records), \ patch.object(cx, "outbound_fulfillment_orders", side_effect=lambda fields: records(fields, "listAllFulfillmentOrders")), \ patch.object(cx, "inbound_shipment_items", return_value=inbound), \ patch.object(cx, "replenishment_metrics", return_value=replenishment), \ patch.object(cx, "item_review_feedback", return_value=feedback), \ patch.object(cx, "item_review_topics", return_value=review_topics), \ patch.object(cx, "item_browse_nodes", return_value={"B000000001": node}), \ patch.object(cx, "browse_node_review_topics", return_value=category), \ patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \ patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)): for table in GROUP_A_TABLES: spec = cx.REPORT_TYPES[table] window = (None, None) if spec.get("snapshot") else (begin, end) def ingest(): return _sync_margin_window(con, table, {}, *window, max_wait=1, log=lambda *_: None, rt=rt) ck(table + " without evidence is unproven", _validate_margin(con, table)["state"] == "unproven") count = ingest() # A real broken mirror must fail before the matching document count passes. con.execute(f"DELETE FROM {table_name(table)} WHERE _report_row_number=1") red = _validate_margin(con, table)["state"] != "ok" ingest() actual = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] ck(table + " latest-period row proof rejects dropped rows and replay is idempotent", red and actual == count and _validate_margin(con, table)["state"] == "ok") fields = [r[1] for r in con.execute(f"PRAGMA table_info('{table_name(table)}')").fetchall()] payload = str(con.execute(f"SELECT * FROM {table_name(table)}").fetchall()) ck(table + " buyer identity and removal addresses never reach the mirror", not any(buyer_column(c) for c in fields) and "planted" not in payload and "Private" not in payload) if spec.get("report"): request = calls[-1][1] ck(table + " request respects snapshot and report options", (not spec.get("snapshot") or ("start" not in request and "end" not in request)) and (not spec.get("period") or request.get("options", {}).get("reportPeriod") == "MONTH") and (not spec.get("date_options") or all(k in request.get("options", {}) for k in spec["date_options"]))) # Explicit planted bypass, then the production parser drops buyer-email. sample = "sku\tbuyer-email\nSKU-1\tplanted@example.invalid\n" with patch.object(module, "buyer_column", return_value=False): red = "buyer_email" in parse_flat_file(sample)[0] ck("buyer-column control fails when the deny rule is bypassed", red and "buyer_email" not in parse_flat_file(sample)[0]) original_parser = parse_flat_file def drop_source_row(*args, **kwargs): columns, rows = original_parser(*args, **kwargs) return columns, rows[1:] with patch.object(module, "parse_flat_file", side_effect=drop_source_row): _sync_margin_window(con, "reimbursements", {}, begin, end, max_wait=1, log=lambda *_: None) red = _validate_margin(con, "reimbursements")["state"] == "refused" _sync_margin_window(con, "reimbursements", {}, begin, end, max_wait=1, log=lambda *_: None) ck("document proof counts raw source records independently of a dropping parser", red and _validate_margin(con, "reimbursements")["state"] == "ok") for body in ("{bad", "{}", '{"dataByAsin":null}'): try: parse_json_report(body, "dataByAsin") refused = False except ValueError: refused = True ck("malformed or absent JSON rows cannot masquerade as empty data " + body, refused) con.execute("UPDATE amzn_order_metrics SET order_count=99") red = _validate_margin(con, "order_metrics")["state"] == "refused" con.execute("UPDATE amzn_order_metrics SET order_count=2") ck("order metrics reconcile counts and totals, not just row count", red and _validate_margin(con, "order_metrics")["state"] == "ok") con.execute("CREATE TABLE amzn_orders (amazon_order_id VARCHAR, purchase_date VARCHAR, " "order_status VARCHAR, last_updated_date VARCHAR)") con.execute("INSERT INTO amzn_orders VALUES ('O1','2026-07-02T00:00:00Z','Shipped',NULL)") red = _validate_orders(con)["state"] == "refused" con.execute("INSERT INTO amzn_orders VALUES ('O2','2026-07-03T00:00:00Z','Shipped',NULL), " "('O3','2026-07-04T00:00:00Z','Cancelled','2026-07-04T12:00:00Z')") ck("orders independently reconcile Sales API counts and exclude cancelled orders", red and _validate_orders(con)["state"] == "ok") con.execute("ALTER TABLE amzn_orders ADD COLUMN sales_channel VARCHAR") con.execute("UPDATE amzn_orders SET sales_channel='Amazon.com'") con.execute("INSERT INTO amzn_orders VALUES ('O4','2026-07-03T00:00:00Z','Shipped',NULL,'Amazon.ca'), " "('O5','2026-07-03T00:00:00Z','Shipped',NULL,'Non-Amazon')") ck("orders reconcile only the US marketplace that Amazon Sales counts", _validate_orders(con)["state"] == "ok") con.execute("UPDATE amzn_orders SET last_updated_date=NULL WHERE amazon_order_id='O3'") ck("NC: a cancellation without a source time bound cannot prove an exact order count", _validate_orders(con)["state"] == "refused") con.execute("UPDATE amzn_orders SET last_updated_date='2026-07-04T12:00:00Z' WHERE amazon_order_id='O3'") spec = cx.REPORT_TYPES["reimbursements"] with patch.object(cx, "report_document", return_value=( "sku\tamount\n", {"reportDocumentId": "empty", "createdTime": _iso(end)})): _sync_margin_window(con, "reimbursements", {}, begin, end, max_wait=1, log=lambda *_: None) ck("an empty restatement deletes stale period rows and stays unproven", con.execute("SELECT COUNT(*) FROM amzn_reimbursements").fetchone()[0] == 0 and _validate_margin(con, "reimbursements")["state"] == "unproven") # HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" for a JSON report too. with patch.object(cx, "report_document", return_value=( "", {"reportId": "cancelled", "reportDocumentId": None, "cancelled": True, "createdTime": _iso(end)})): _sync_margin_window(con, "b2b_recommended", {}, None, None, max_wait=1, log=lambda *_: None) ck("a CANCELLED JSON report lands as an empty snapshot instead of a refusal", con.execute("SELECT COUNT(*) FROM amzn_b2b_recommended").fetchone()[0] == 0) try: parse_json_report("", "recommendations") still_strict = False except ValueError: still_strict = True ck("NC: an empty JSON body that Amazon did NOT cancel is still refused", still_strict) finally: con.close() # Exercise the outer dispatcher and restart markers, not only the document helper. con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") clock = [end] class Clock(datetime): @classmethod def now(cls, tz=None): return clock[0].astimezone(tz) selected = ("inventory_planning", "long_term_storage_fees", "order_metrics", "listing_items", "account_health") rt = _FixtureSyncRuntime("group-a-sync-fixture", selected, "group-a-account") def watermark(table): with _fixture_account_state(con, rt) as account: return read_watermark(account, table) def queued(): with _fixture_account_state(con, rt) as account: return queued_resyncs(account, "long_term_storage_fees") def owed(): with _fixture_account_state(con, rt): return needs_first_load(rt.key, rt) try: with _fixture_sync_store(con, rt), \ patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 62), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \ patch.object(cx, "group_a_records", side_effect=records), \ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")): record_watermark(con, "orders", end) _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail=_iso(end)) ck("new enabled tables trigger bounded first load on an existing account", owed()) def run(): return sync(rt.key, tables=selected, first_load=True, report_budget=20, rt=rt, log=lambda *_: None) first = run() before = len(calls) paused = run() ck("first-load refresh floors preserve unfinished tasks without repeating requests", first["remaining"] > 0 and paused["remaining"] == first["remaining"] and len(calls) == before and paused["etaSeconds"] > first_load_eta(paused["remaining"], burst=0)) for _ in range(5): clock[0] += timedelta(days=2) last = run() if not last["remaining"]: break checked = validate(rt.key, rt=rt, log=lambda *_: None) if (last["state"] != "done" or last["remaining"] or any(checked[table]["state"] != "ok" for table in selected) or owed()): log("group a sync fixture detail: " + json.dumps({ "result": last, "validation": {t: checked[t] for t in selected}, "watermarks": {t: str(watermark(t)) for t in selected}})) ck("outer sync resumes every window and validates all new dispatch kinds", last["state"] == "done" and last["remaining"] == 0 and all(watermark(table) for table in selected) and all(checked[table]["state"] == "ok" for table in selected) and not owed()) ck("outer sync publishes durable account-attributed rows and independent account verdicts", bool(datastore._publication(con, "amazon").get("snapshot_id")) and bool(rt.blobs.get(MIRROR_PREFIX + _MANIFEST_KEY)) and not last["problems"] and all(checked[t]["accounts"]["group-a-account"]["state"] == "ok" for t in selected) and all(con.execute(f"SELECT COUNT(*) FROM {table_name(t)} WHERE _connection_id=?", ["group-a-account"]).fetchone()[0] > 0 for t in selected)) # HOTFIX 2026-09-29 (D-3496): a load left unfinished ONLY by a report Amazon refused is # no longer owed, so a restart stops queueing it and the scheduled producer stops # deferring every loaded table to it. The refused table is still retried by a pass. refused_table = "account_health" with _fixture_account_state(con, rt, write=True) as account: account.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(WATERMARK_PREFIX + refused_table)]) _record_state(account, entity_key(refused_table), "refused", 0, "fixture refusal") _record_state(account, entity_key("refused_" + refused_table), "refused", detail=json.dumps({ "reason": "fixture refusal", "amazonStatus": "FATAL", "attempts": 3})) _record_state(account, entity_key(FIRST_LOAD_END_KEY), "failed", detail=_iso(end)) ck("a load left unfinished only by a refused report is no longer owed", not owed()) with _fixture_account_state(con, rt, write=True) as account: account.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(refused_table)]) ck("NC: the same table never refused (no watermark) still owes a first load", owed()) # S79-T23: a queued resync window downloads again alone; the watermark stays. fees = "amzn_long_term_storage_fees" stored = con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} " "GROUP BY 1, 2 ORDER BY 1").fetchall() mark = watermark("long_term_storage_fees") with _fixture_account_state(con, rt, write=True) as account: _record_state(account, entity_key(RESYNC_PREFIX + "long_term_storage_fees"), "pending", 1, json.dumps({"windows": [list(stored[0][:2])]})) clock[0] += timedelta(days=2) before = len(calls) sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None) asked = [kwargs for _report, kwargs in calls[before:]] ck("a queued resync window downloads again alone and keeps the watermark", len(asked) == 1 and str(asked[0].get("start", ""))[:10] == stored[0][0][:10] and watermark("long_term_storage_fees") == mark and not queued() and con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} " "GROUP BY 1, 2 ORDER BY 1").fetchall() == stored) # S79-T23: a scheduled pass inside the daily floor drains every queued window in one turn. queue = [list(window[:2]) for window in stored] with _fixture_account_state(con, rt, write=True) as account: _record_state(account, entity_key(RESYNC_PREFIX + "long_term_storage_fees"), "pending", 1, json.dumps({"windows": queue})) _record_state(account, entity_key("sync_tick_long_term_storage_fees"), "done") clock[0] += timedelta(seconds=60) before = len(calls) held = sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None, scheduled=True)["tables"]["long_term_storage_fees"] ck("NC: a queued resync still waits out Amazon's own report floor", held.get("state") == "deferred" and len(calls) == before and len(queued()) == len(queue)) clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"]) lines, before = [], len(calls) drained = sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lines.append, scheduled=True)["tables"]["long_term_storage_fees"] ck("a scheduled resync turn drains several windows inside the daily floor, one log line", len(queue) > 1 and len(calls) - before == len(queue) and drained.get("resync") == {"landed": len(queue), "left": 0} and f"Amazon resync long_term_storage_fees: {len(queue)} windows landed, 0 left" in lines and not queued() and watermark("long_term_storage_fees") == mark) clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"] + 60) idle = sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None, scheduled=True)["tables"]["long_term_storage_fees"] ck("NC: with the queue empty the table cadence defers it again", idle.get("state") == "deferred" and "next due " in idle.get("reason", "")) finally: con.close() # A scheduled pass during an unfinished first load still runs a Customer Feedback snapshot. def feedback_pass(exempt): con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") rt = _FixtureSyncRuntime("group-a-feedback-fixture", ("review_topics",), "group-a-account") asked = [] try: with _fixture_sync_store(con, rt), \ patch.object(module, "datetime", Clock), \ patch.object(module, "FEEDBACK_TABLES", FEEDBACK_TABLES if exempt else ()), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), \ patch.object(cx, "item_review_topics", side_effect=lambda _f, asins: asked.append(asins) or { **{sort: [] for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {}}), \ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")): with _fixture_account_state(con, rt, write=True) as account: record_watermark(account, "orders", end) _record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end)) result = sync(rt.key, tables=("review_topics",), rt=rt, log=lambda *_: None, scheduled=True) with _fixture_account_state(con, rt) as account: tick = _read_state(account, entity_key("sync_tick_review_topics")) return asked, (result.get("tables") or {}).get("review_topics") or {}, tick finally: con.close() asked, row, tick = feedback_pass(True) red_asked, red_row, red_tick = feedback_pass(False) ck("a scheduled pass during an unfinished first load runs review topics and records its tick", asked == [[]] and (tick or {}).get("state") == "done" and row.get("state") != "deferred" and not red_asked and not red_tick and red_row.get("state") == "deferred" and "first load" in red_row.get("reason", "")) # A Sync now click retries a refused Customer Feedback table inside its weekly floor. def click_after(landed): con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") rt = _FixtureSyncRuntime("group-a-feedback-click", ("review_topics",), "group-a-account") asked = [] try: with _fixture_sync_store(con, rt), \ patch.object(module, "datetime", Clock), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), \ patch.object(cx, "item_review_topics", side_effect=lambda _f, asins: asked.append(asins) or { **{sort: [] for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {}}), \ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")): with _fixture_account_state(con, rt, write=True) as account: record_watermark(account, "orders", end) _record_state(account, entity_key(FIRST_LOAD_END_KEY), "done", detail=_iso(end)) _record_state(account, entity_key("sync_tick_review_topics"), "done") _record_state(account, entity_key("review_topics"), landed, 0, "fixture landed state") result = sync(rt.key, tables=("review_topics",), rt=rt, log=lambda *_: None) return asked, (result.get("tables") or {}).get("review_topics") or {} finally: con.close() retried, retried_row = click_after("refused") held, held_row = click_after("done") ck("Sync now retries a refused review table inside its weekly floor and defers a landed one", retried == [[]] and retried_row.get("state") != "deferred" and not held and held_row.get("state") == "deferred") # Client pagination: an empty middle page must not end the traversal. for operation, interval in cx.ENTITY_INTERVALS.items(): if operation == "getMyFeesEstimates": continue # The per-price wire fixture covers this POST reader separately. requests, pace = [], [] def request(path, token, **params): requests.append((path, params)) i = len(requests) following = "page-2" if i % 3 == 1 else "page-3" if i % 3 == 2 else None rows = [] if i % 3 == 2 else [{"asin": "B000000001", "sku": "SKU-" + str(i)}] if operation == "getOrderMetrics": return {"payload": rows} if operation == "getInventorySummaries": return {"payload": {"inventorySummaries": rows}, "pagination": {"nextToken": following}} return {"items": rows, "pagination": {"nextToken": following}} args = {"start": _iso(begin), "end": _iso(end), "asins": [f"B{i:09d}" for i in range(41)]} with patch.object(cx, "access_token", return_value="fixture"), \ patch.object(cx, "_request", side_effect=request), \ patch.object(cx, "_pace_entity", side_effect=lambda name, seconds, **_: pace.append((name, seconds))): got = cx.group_a_records({"seller_id": "SELLER"}, operation, **args) want = 9 if operation == "searchCatalogItems" else 1 if operation == "getOrderMetrics" else 3 ck(operation + " follows every page and uses its default request pace", len(requests) == want and len(pace) == want and all(p == (operation, interval) for p in pace) and len(got) == (6 if operation == "searchCatalogItems" else 1 if want == 1 else 2)) if operation == "searchCatalogItems": sent = [a for _, params in requests[::3] for a in params["identifiers"].split(",")] ck("catalog search covers all forty-one own ASINs in batches of twenty", sent == args["asins"] and all(len(p["identifiers"].split(",")) <= 20 for _, p in requests)) ck("catalog search asks for relationships and images and nothing beyond the approved includes", all(p["includedData"].split(",") == ["summaries", "salesRanks", "dimensions", "attributes", "relationships", "images"] for _, p in requests)) if operation == "getInventorySummaries": ck("FBA inventory requests detailed inventory on every page", all(p["details"] == "true" for _, p in requests)) def partition_probe(partition): requests = [] def request(*_a, **kwargs): requests.append(kwargs) if len(requests) == 1: return {"numberOfResults": 1001, "items": []} if partition else {"numberOfResults": 0, "items": []} return {"numberOfResults": 1, "items": [{"sku": "SKU-" + str(len(requests))}]} with patch.object(cx, "access_token", return_value="fixture"), \ patch.object(cx, "_request", side_effect=request), patch.object(cx, "_pace_entity"): rows = cx.group_a_records({"seller_id": "SELLER"}, "searchListingsItems") return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"] ck("Listings search splits date ranges at the real thousand-SKU ceiling", not partition_probe(False) and partition_probe(True)) # D-17409: a table the first load owes refreshes in the scheduled pass while no first-load job runs. class CatchUpClock(datetime): instant = datetime(2026, 11, 17, 12, 0, tzinfo=timezone.utc) @classmethod def now(cls, tz=None): return cls.instant.astimezone(tz) def catch_up_pass(served): # served: None = a first load stalled since its frozen end, "lease" = a job holds the Amazon # lease, "fresh" = a first-load pass wrote its record an hour ago, "worker" = the scheduled # pass's own resident writer holds the Amazon lease, "job" = a generation-style holder does. con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") table, cohort = "fba_customer_returns", _utc("2026-09-29T00:00:00Z") mirror = table_name(table) con.execute(f"CREATE TABLE {mirror} (return_date VARCHAR, order_id VARCHAR, sku VARCHAR, asin VARCHAR, " "fnsku VARCHAR)") con.execute(f"INSERT INTO {mirror} VALUES ('2026-08-01T10:00:00+00:00', '111-0000000-0000009', 'OLD-1', " "'B00000000A', 'X00000000A')") rt = _FixtureSyncRuntime("group-a-catch-up", (table,), "group-a-account") asked = [] def returns(_fields, report, **kwargs): asked.append(kwargs) return ("return-date\torder-id\tsku\tasin\tfnsku\tquantity\tdetailed-disposition\n" "2026-10-01T10:00:00+00:00\t111-0000000-0000001\tSKU-1\tB000000001\tX000000001\t1\tSELLABLE\n", {"reportDocumentId": "fixture-catch-up", "createdTime": "2026-11-17T11:00:00Z"}) try: with _fixture_sync_store(con, rt), \ patch.object(module, "datetime", CatchUpClock), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), \ patch.object(module, "_ensure_seller_first_month"), \ patch.object(cx, "report_document", side_effect=returns), \ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")): now = datetime(2026, 11, 17, 12, 0, tzinfo=timezone.utc) CatchUpClock.instant = now - timedelta(hours=1) if served == "fresh" else cohort with _fixture_account_state(con, rt, write=True) as account: record_watermark(account, "orders", cohort) _record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(cohort)) CatchUpClock.instant = now holder = {"lease": FIRST_LOAD_HOLDER_PREFIX + "group-a", "worker": "local@host:amazon:abc", "job": FIRST_LOAD_HOLDER_PREFIX + "1234abcd"}.get(served) if holder: acquire_amazon_lease(rt, holder) first = (sync(rt.key, tables=(table,), rt=rt, log=lambda *_: None, scheduled=True) .get("tables") or {}).get(table) or {} first_asked = list(asked) second = sync(rt.key, tables=(table,), rt=rt, log=lambda *_: None, scheduled=True) with _fixture_account_state(con, rt) as account: marks = (read_watermark(account, table), _read_state(account, entity_key(CATCH_UP_PREFIX + table)), _read_state(account, entity_key(PHASE_HOLD_PREFIX + table))) kept = con.execute(f"SELECT count(*) FROM {mirror} WHERE sku = 'OLD-1'").fetchone()[0] holds = module._MIRROR_STORE._publication_holds(second, ()) or {} return first, first_asked, (second.get("tables") or {}).get(table) or {}, marks, kept, holds finally: con.close() first, first_asked, second, marks, kept, holds = catch_up_pass(None) controls = [catch_up_pass(mode) for mode in ("lease", "fresh")] ck("with no first-load job a scheduled pass refreshes a table the first load owes and leaves it current", first.get("state") not in ("deferred", "failed", "throttled") and first.get("rows") == 2 and "since watermark 2026-09-29T00:00:00Z" in first.get("window", "") and [str(a.get("start", ""))[:10] for a in first_asked] == ["2026-09-18", "2026-10-18"] and marks[0] is None and (marks[1] or {}).get("state") == "done" and marks[2] is None and kept == 1 and second.get("current") is True and "fba_customer_returns" not in (holds.get("held") or {}) and all(nc[0].get("state") == "deferred" and "first load" in nc[0].get("reason", "") and not nc[1] and nc[3][0] is None and nc[3][1] is None for nc in controls)) ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0) worker = catch_up_pass("worker") ck("a scheduled pass with no producer refreshes an owed table while its own writer holds the Amazon lease", worker[0].get("state") not in ("deferred", "failed", "throttled") and worker[0].get("rows") == 2 and len(worker[1]) == 2 and (worker[3][1] or {}).get("state") == "done") job = catch_up_pass("job") ck("NC: a first-load generation holding the Amazon lease still defers an owed table", job[0].get("state") == "deferred" and "first load" in job[0].get("reason", "") and not job[1] and job[3][0] is None and job[3][1] is None) return 1 if failures else 0 def catalog_parent_selftest(log=print, check=None): """S67-T29 (O23) offline fixtures: the catalog Parent ASIN map covers every own ASIN. `python -c "import sys; sys.path[:0]=['platform','platform/harness']; import amazon_store as s; sys.exit(s.catalog_parent_selftest())"` from the repo root; no credentials, no network. """ from unittest.mock import patch import duckdb cx = _client() failures = [] def ck(name, ok): if check: check("[catalog parent] " + name, bool(ok)) else: log(f" {'ok' if ok else 'FAIL'} catalog parent {name}") if not ok: failures.append(name) mkt = cx.US_MARKETPLACE_ID child, parent, standalone, lonely, broken = ("B0CHILD001", "B0PARENT01", "B0STAND001", "B0LONELY01", "B0BROKEN01") items = [ {"asin": child, "relationships": [ {"marketplaceId": "A1F83G8C2ARO7P", "relationships": [ {"type": "VARIATION", "parentAsins": ["B0WRONGMKT"]}]}, {"marketplaceId": mkt, "relationships": [ {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]}, {"type": "VARIATION", "parentAsins": [parent], "variationTheme": {"attributes": ["color"], "theme": "COLOR"}}]}]}, {"asin": parent, "relationships": [{"marketplaceId": mkt, "relationships": [ {"type": "VARIATION", "childAsins": [child, "B0SIBLING1"]}]}]}, {"asin": standalone, "relationships": [{"marketplaceId": mkt, "relationships": []}]}, {"asin": lonely}, {"asin": broken, "relationships": "not-a-list"}, {"asin": "B0BROKEN02", "relationships": [None, {"marketplaceId": mkt, "relationships": {"x": 1}}, {"marketplaceId": mkt, "relationships": [ {"type": "VARIATION", "parentAsins": "B0NOTALIST"}, {"type": "VARIATION", "parentAsins": [None, "bad"]}]}]}, ] eco_only = "B0ECONLY01" items.append({"asin": eco_only, "relationships": [{"marketplaceId": mkt, "relationships": []}]}) picture_link = "https://m.media-amazon.com/images/I/child-main.jpg" items[0]["images"] = [{"marketplaceId": mkt, "images": [ {"variant": "MAIN", "link": picture_link, "width": 500, "height": 500}, {"variant": "PT01", "link": "https://m.media-amazon.com/images/I/child-pt.jpg", "width": 500, "height": 500}]}] pricing = {standalone: "B0PRICEPAR", broken: "not an asin"} economics = {eco_only: "B0ECOPARNT", standalone: "B0ECOLOSER", child: "B0ECOWRONG", broken: "junk"} got = catalog_parent_map(items, pricing, mkt, economics) ck("variation child maps to its parent from relationships", got.get(child) == (parent, "relationships")) ck("variation parent maps to itself", got.get(parent) == (parent, "relationships")) ck("standalone falls back to pricing parent", got.get(standalone) == ("B0PRICEPAR", "pricing")) ck("no relationships and no pricing maps to itself", got.get(lonely) == (lonely, "self")) ck("malformed relationships degrade to itself, never an error", got.get(broken) == (broken, "self") and got.get("B0BROKEN02") == ("B0BROKEN02", "self")) ck("economics child to parent is the next fallback after pricing", got.get(eco_only) == ("B0ECOPARNT", "economics")) ck("every item gets a parent", set(got) == {i["asin"] for i in items}) # Red-first: economics never beats relationships or pricing, and is absent without its map. ck("economics never outranks relationships or pricing, and needs its map", got.get(child) == (parent, "relationships") and got.get(standalone) == ("B0PRICEPAR", "pricing") and catalog_parent_map(items, pricing, mkt).get(eco_only) == (eco_only, "self")) # Red-first: another marketplace or a PACKAGE_HIERARCHY parent must never win. solo_other = catalog_parent_map([dict(items[0], relationships=items[0]["relationships"][:1])], {}, mkt) pack_only = catalog_parent_map([{"asin": child, "relationships": [{"marketplaceId": mkt, "relationships": [ {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]}]}]}], {}, mkt) ck("another marketplace and a package parent are ignored", solo_other.get(child) == (child, "self") and pack_only.get(child) == (child, "self")) ck("empty and junk inputs never raise", catalog_parent_map(None) == {} and catalog_parent_map([None, "x", {"asin": ""}]) == {}) con = duckdb.connect(":memory:") try: con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") # Resolver before any table exists: no own-ASIN source means None, never an error. ck("resolver with no listings or catalog is None and an empty map", asin_parents_sql(con) is None and asin_parents(con) == {}) own = [child, parent, standalone, lonely, broken, eco_only] # Listed ASINs Amazon's catalog returns no item for (the live 5 of 1164). eco_listed, nothing = "B0ECOLIST1", "B0NOTHING1" listed = own + [eco_listed, nothing, child] con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") con.executemany("INSERT INTO amzn_listings VALUES (?)", [[a] for a in listed]) only_listings = asin_parents(con) ck("resolver with listings only maps every listed ASIN to itself", set(only_listings) == set(listed) and all(v == (k, "self") for k, v in only_listings.items())) con.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR, start_date VARCHAR)") con.executemany("INSERT INTO amzn_economics VALUES (?, ?, ?)", [ [eco_only, "B0ECOOLDPR", "2026-08-01"], [eco_only, "B0ECOPARNT", "2026-09-01"], [eco_listed, "B0ECOLDPAR", "2026-07-01"], [eco_listed, "B0ECOLSTPR", "2026-09-10"], [eco_listed, None, "2026-09-20"]]) con.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR, _period_start VARCHAR)") con.executemany("INSERT INTO amzn_pricing VALUES (?, ?, ?)", [ [standalone, "B0OLDPARNT", "2026-08-01T00:00:00Z"], [standalone, "B0PRICEPAR", "2026-09-01T00:00:00Z"]]) asked = [] def records(_fields, operation, **kwargs): asked.append(list(kwargs.get("asins") or ())) return [i for i in items if i["asin"] in kwargs.get("asins", ())] # An older snapshot day with a different parent: the readback must use the LATEST day. with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")), \ patch.object(cx, "group_a_records", side_effect=records): count = _sync_margin_window(con, "catalog_items", {}, None, None, max_wait=1, log=lambda *_: None, rt=_FixtureSyncRuntime("catalog-parent-fixture", ("catalog_items",))) con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) " "VALUES (?, 'B0STALEPAR', 'relationships', '2000-01-01')", [child]) latest = con.execute("SELECT MAX(_snapshot_date) FROM amzn_catalog_items").fetchone()[0] mapped = dict((a, (p, s)) for a, p, s in con.execute( "SELECT asin, parent_asin, parent_asin_source FROM amzn_catalog_items WHERE _snapshot_date=?", [latest]).fetchall()) columns = {r[1] for r in con.execute("PRAGMA table_info('amzn_catalog_items')").fetchall()} state = _read_state(con, entity_key("reconcile_catalog_items")) or {} marker = json.loads(state.get("detail") or "{}") ck("store readback shows the parent of a known child ASIN on the latest snapshot", mapped.get(child) == (parent, "relationships")) ck("catalog asks for every listed ASIN and maps every item it returns", asked == [sorted(set(listed))] and set(mapped) == set(own) and all(p for p, _ in mapped.values())) ck("catalog sync stamps the economics fallback", mapped.get(eco_only) == ("B0ECOPARNT", "economics")) pictures = dict(con.execute("SELECT asin, main_image_url FROM amzn_catalog_items WHERE _snapshot_date=?", [latest]).fetchall()) ck("catalog sync names the MAIN picture link per ASIN and keeps the raw images column", pictures.get(child) == picture_link and pictures.get(parent) is None and "images" in columns) # Resolver: two catalog snapshot days, an old-day-only ASIN, a latest-day catalog-only ASIN. con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) " "VALUES ('B0OLDONLY1', 'B0OLDONLY1', 'self', '2000-01-01')") con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) " "VALUES ('B0CATONLY1', 'B0CATPARNT', 'relationships', ?)", [latest]) resolved = asin_parents(con) rows = con.execute(asin_parents_sql(con)).fetchall() ck("resolver covers every own ASIN exactly once, latest catalog day only", set(resolved) == set(listed) | {"B0CATONLY1"} and len(rows) == len(resolved) and "B0OLDONLY1" not in resolved and all(p for p, _ in resolved.values())) ck("resolver readback shows the latest parent of a known child", resolved.get(child) == (parent, "relationships") and resolved.get("B0CATONLY1") == ("B0CATPARNT", "relationships")) ck("resolver falls back to the newest economics parent, then to itself", resolved.get(eco_listed) == ("B0ECOLSTPR", "economics") and resolved.get(nothing) == (nothing, "self")) # A catalog mirror synced before S67-T29 has no parent_asin column: economics and self carry it. old = duckdb.connect(":memory:") try: old.execute("CREATE TABLE amzn_catalog_items (asin VARCHAR, _snapshot_date VARCHAR)") old.execute(f"INSERT INTO amzn_catalog_items VALUES ('{child}', '2026-09-01'), ('{eco_listed}', '2026-09-01')") old.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)") old.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR')") ck("resolver degrades on a pre-T29 catalog with no listings table", asin_parents(old) == {child: (child, "self"), eco_listed: ("B0ECOLSTPR", "economics")}) finally: old.close() # O23 order in the READ-TIME resolver: catalog, then pricing, then economics, then self. pri = duckdb.connect(":memory:") try: pri.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") pri.execute(f"INSERT INTO amzn_listings VALUES ('{eco_listed}'), ('{nothing}')") pri.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)") pri.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR'), " f"('{nothing}', 'B0ECONOTHG')") pri.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)") pri.execute(f"INSERT INTO amzn_pricing VALUES ('{eco_listed}', 'B0PRICEPA2')") got = asin_parents(pri) ck("resolver: a pricing parent wins over economics, economics still answers the rest", got.get(eco_listed) == ("B0PRICEPA2", "pricing") and got.get(nothing) == ("B0ECONOTHG", "economics")) pri.execute("CREATE OR REPLACE TABLE amzn_pricing (parent_asin VARCHAR)") ck("NC: a pricing mirror with no child_asin (the live mirror today) adds no pricing step", asin_parents(pri).get(eco_listed) == ("B0ECOLSTPR", "economics")) finally: pri.close() # SKU grain (S67-T29 finding): the request asks for it, and the first SKU-grain pull # replaces a parent-grain mirror instead of sitting beside it. grain = duckdb.connect(":memory:") try: quiet = lambda *_a, **_k: None # noqa: E731 sku_rows = [{"parent_asin": "B0PARENT01", "child_asin": "B0CHILD001", "sku": "RI-1"}] ck("regrain: an absent mirror is left alone", _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False) grain.execute("CREATE TABLE amzn_pricing (parent_asin VARCHAR, report_date VARCHAR)") grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-01'), ('B0PARENT02', '2026-09-02')") ck("regrain: parent-grain rows WITHOUT child_asin never clear the mirror", _pricing_regrain(grain, "amzn_pricing", [{"parent_asin": "B0PARENT03"}], quiet) is False and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 2) ck("regrain: the first SKU-grain pull clears the parent-grain mirror (one grain per table)", _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is True and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 0) grain.execute("ALTER TABLE amzn_pricing ADD COLUMN child_asin VARCHAR") grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-03', 'B0CHILD001')") ck("regrain: an already SKU-grain mirror keeps its history", _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 1) finally: grain.close() with open(__file__, encoding="utf-8") as _src: passes_options = _src.read().count('REPORT_OPTIONS[table]') >= 2 ck("the Sales and Traffic request asks for SKU grain, the same report and role", REPORT_OPTIONS.get("pricing") == {"asinGranularity": "SKU"} and passes_options) ck("pricing fallback reads the newest pricing period", mapped.get(standalone) == ("B0PRICEPAR", "pricing")) ck("raw relationships JSON is kept beside the derived columns", {"relationships", "parent_asin", "parent_asin_source"} <= columns) ck("row count stays Amazon's record count", count == len(own) and marker.get("count") == len(own)) finally: con.close() return 1 if failures else 0 def _without_history_rules(tables): """Specs for a fixture that predates the measured `history` rules: same tables, rules removed.""" specs = _client().REPORT_TYPES return {table: {k: v for k, v in specs[table].items() if k != "history"} for table in tables} def monthly_after_cohort_selftest(log=print): """H5: monthly tables plan, land and publish the months closed after a frozen cohort end. No network.""" from unittest.mock import patch import duckdb cx, module = _client(), sys.modules[__name__] fails, checks = [], [0] def ck(name, cond, detail=""): checks[0] += 1 if cond: log(f" ok {name}") return log(f" FAIL {name} {str(detail).encode('ascii', 'backslashreplace').decode('ascii')}") fails.append(name) tables = ["search_catalog_performance", "market_basket", "repeat_purchase"] end, t0 = _utc("2026-09-26T00:00:00Z"), _utc("2026-11-14T00:00:30Z") clock, holder, calls, bounds = [t0], {}, [], [] by_report = {cx.REPORT_TYPES[t]["report"]: t for t in tables} class Clock(datetime): @classmethod def now(cls, tz=None): return clock[0].astimezone(tz) class Runtime(_FixtureMirrorRuntime): key = "h5-fixture" def get(self, *_args, **_kwargs): return None def fresh(floor="2026-08-01T00:00:00Z"): con = holder["con"] = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end)) for table in tables: record_backfill_floor(con, table, _utc(floor)) calls.clear() return con def document(_fields, report_type, **kwargs): calls.append((by_report[report_type], kwargs["start"][:7])) bounds.append((kwargs["start"], kwargs["end"])) rows = [{"asin": f"B0H5{len(calls):06d}", "startDate": kwargs["start"][:10], "endDate": kwargs["end"][:10]}] return json.dumps({"dataByAsin": rows}), {"reportDocumentId": f"h5-{len(calls)}"} def run(at): clock[0] = at return _sync_candidate(Runtime.key, tables=list(tables), rt=Runtime(), log=lambda *_: None, scheduled=True, first_load=True, report_budget=10) def asked(start=0): return sorted({month for _t, month in calls[start:]}) def months(con, table): return sorted(str(r[0])[:7] for r in con.execute( f'SELECT DISTINCT "_period_start" FROM {table_name(table)}').fetchall()) day = timedelta(days=1, minutes=1) try: with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")), \ patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 70), \ patch.object(module, "_SCHEDULE_OFFSET_READER", lambda: 0), \ patch.object(module, "credential", return_value={"fixture": True}), \ patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (holder["con"].cursor(), None)), \ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \ patch.object(cx, "settlement_reports", return_value=[]), \ patch.dict(cx.REPORT_TYPES, _without_history_rules(tables)): # The plan for each table holds 2026-10 then 2026-09 before its own 2026-08 task. bare = first_load_plan(tables, end - timedelta(days=70), end) spliced = _sqp_after_cohort(bare, end, ()) order = {t: [w0.strftime("%Y-%m") for name, w0, *_ in spliced if name == t] for t in tables} ck("H5 a frozen cohort plans 2026-10 and 2026-09 first for each Brand Analytics month table", all(order[t][:3] == ["2026-10", "2026-09", "2026-08"] for t in tables) and not any(w0.strftime("%Y-%m") in ("2026-09", "2026-10") for _n, w0, *_ in bare), order) # Scheduled first-load passes land 2026-10, then 2026-09 after Amazon's daily floor. con = fresh() with patch.object(module, "_sqp_after_cohort", side_effect=lambda tasks, *_: list(tasks)): run(t0) red = asked() con.close() con = fresh() first = run(t0) one, mark = asked(), len(calls) run(t0 + day) two = asked(mark) ck("H5 scheduled first-load passes land 2026-09 and 2026-10 rows for search catalog performance", red == ["2026-07"] and one == ["2026-10"] and two == ["2026-09"] and months(con, "search_catalog_performance") == ["2026-09", "2026-10"] and bool(bounds) and all(a[:7] == b[:7] and a < b for a, b in bounds) and all(_cohort_month_done(con, t, _utc(m + "-01T00:00:00Z")) for t in tables for m in ("2026-09", "2026-10")) and all(read_backfill_floor(con, t) == _utc("2026-08-01T00:00:00Z") for t in tables), (red, one, two, months(con, "search_catalog_performance"))) # A landed month publishes; the same chunk without the landed mark publishes nothing. waiting = {table_name("orders")} row = first["tables"]["search_catalog_performance"] unmarked = {**first, "tables": {t: {k: v for k, v in r.items() if k not in ("current", "landedMonths")} for t, r in first["tables"].items()}} holds = _MIRROR_STORE._publication_holds(first, waiting) ck("H5 a first-load chunk that landed months publishes them, never holds them at the old rows", first.get("state") == "partial" and row.get("current") is True and row.get("landedMonths") == 1 and holds is not None and not set(tables) & set(holds["held"]) and _MIRROR_STORE._publication_holds(unmarked, waiting) is None, (row, holds)) # The next pass takes the owed history month only, then the table closes past 2026-10. before = len(calls) third = run(t0 + 2 * day) later = asked(before) run(t0 + 3 * day) ck("H5 a later pass never asks Amazon again for a landed month after the cohort", later == ["2026-07"] and asked(before + 3) == [] and all(read_watermark(con, t) == _utc("2026-11-01T00:00:00Z") for t in tables) and third["tables"]["search_catalog_performance"].get("state", "done") == "done", (later, asked(before + 3), [read_watermark(con, t) for t in tables])) con.close() con = fresh() run(t0) run(t0 + day) with patch.object(module, "_cohort_month_done", return_value=False): before = len(calls) run(t0 + 2 * day) ck("H5 red control: without the month marker the next pass asks for 2026-10 again", asked(before) == ["2026-10"], asked(before)) con.close() # A pass whose table landed no window keeps the last landing's tick and says nothing new landed. def zero(nothing=None): clock[0] = t0 con = fresh("2026-07-01T00:00:00Z") for t in tables: for m in ("2026-09", "2026-10"): _record_state(con, _cohort_month_key(t, _utc(m + "-01T00:00:00Z")), "done", 1) _record_state(con, entity_key("sync_tick_" + t), "done", 1) stamp = _read_state(con, entity_key("sync_tick_" + tables[0]))["updated_at"] if nothing is None: run(t0 + 3 * day) else: with patch.object(module, "_landed_nothing", return_value=nothing): run(t0 + 3 * day) tick = _read_state(con, entity_key("sync_tick_" + tables[0])) state = _read_state(con, entity_key(tables[0])) con.close() return stamp, tick, state, list(calls) stamp, tick, state, seen = zero() _s, red_tick, red_state, _c = zero(False) ck("H5 a pass that landed no window never stamps a plain done", seen == [] and state["state"] == "done" and state["detail"] == NOTHING_NEW_DETAIL and tick["state"] == "done" and tick["updated_at"] == stamp and red_state["detail"] is None and red_tick["updated_at"] != stamp, (seen, state, tick, red_state, red_tick)) except Exception as exc: # noqa: BLE001 ck("H5 monthly after-cohort fixture ran", False, f"{type(exc).__name__}: {exc}") log(f" monthly after cohort: {checks[0] - len(fails)}/{checks[0]}") return 1 if fails else 0 def selftest(log=print): """Prove the parsers, the typing, the naming rule and validate's refusal. No network, no store. ⛔ EVERY CHECK HERE WAS DRIVEN RED BEFORE BEING TRUSTED. Green on a first run proves a check exists, not that it discriminates [[revert-one-file-to-see-if-a-gate-discriminates]]. """ fails = [] checks = 0 # Offline fixtures never measure a seller floor; the H2 legs in verify_amazon turn it on. os.environ.setdefault("AIOS_AMAZON_SELLER_FLOOR_PROBE", "0") def ck(name, cond, detail=""): """⛔ THE DETAIL IS ASCII-ESCAPED, AND THAT IS NOT COSMETIC. A perturbation run printed a measured header still carrying its BOM into `detail`, and the Windows console encoder raised on it: the selftest DIED at check 1 of 15 instead of reporting 14 more. A gate that crashes reports a SMALLER total, and a shrinking total reads as a passing run to anything counting reds [[a-shrinking-gate-total-is-a-red-result]]. This is a connector whose whole job is ingesting other people's text, so hostile bytes in a failure message are the normal case, not the exotic one. """ nonlocal checks checks += 1 if cond: log(f" ok {name}") return try: shown = str(detail).encode("ascii", "backslashreplace").decode("ascii") except Exception: # noqa: BLE001 shown = "" log(f" FAIL {name} {shown}") fails.append(name) log("amazon_store selftest") # NC-AZ1 — the header IS the column list, and hostile headers survive it. tsv = ("amazon-order-id\tpurchase-date\tItem Price\t2nd Ref\tsku\n" "111-2223334-4445556\t2026-08-01T00:00:00Z\t27.30\tx\tRI-001\n" "111-2223334-4445557\t2026-08-02T00:00:00Z\t14.00\ty\tRI-002\n") origin = {"document_id": "selftest-document", "report_type": "selftest-report"} cols, rows = parse_flat_file(tsv, **origin) ck("NC-AZ1 flat-file columns are measured from the header", cols == ["amazon_order_id", "purchase_date", "item_price", "c_2nd_ref", "sku"] + list(_ORIGIN_TYPES), cols) # ⚠ ASSERTED AGAINST THE SANITIZER, NOT AGAINST A DEAD `.replace`. The first version of this # check asked "is there a BOM left in the output", which is true no matter what the code does # once every non-alphanumeric becomes `_`. It survived its own perturbation. It now asserts # that a BOM'd header and a clean one produce the SAME column, which goes red the moment the # sanitizer stops normalising. ck("NC-AZ2 a BOM'd header yields the same column as a clean one", _snake("amazon-order-id") == "amazon_order_id" == _snake("amazon-order-id"), _snake("amazon-order-id")) ck("NC-AZ3 rows parse to the measured width", len(rows) == 2 and rows[0]["amazon_order_id"] == "111-2223334-4445556", rows[:1]) # NC-AZ4 — QUOTE_NONE, and the FIXTURE had to be fixed before the check could fail. # ⛔ A first version used `12" Vase`, where the quote sits mid-field. `csv` only treats `"` # specially at the START of a field, so QUOTE_MINIMAL parsed it identically and the check # survived swapping the dialect. The hazard is a field that OPENS with a quote: minimal # quoting then treats the rest of the document as one quoted string and silently returns a # single merged row. That is the real Amazon case, a title like `"Special" Vase`. q = 'sku\ttitle\nRI-9\t"12 inch Vase\nRI-10\tPlain\n' qc, qr = parse_flat_file(q, **origin) ck("NC-AZ4 a field opening with a quote does not swallow the rows after it", len(qr) == 2 and qr[0]["title"] == '"12 inch Vase' and qr[1]["sku"] == "RI-10", qr) # NC-AZ5 — a duplicate header cell must not collide. dc, _ = parse_flat_file("sku\tsku\tqty\nA\tB\t1\n", **origin) ck("NC-AZ5 duplicate headers are disambiguated", dc == ["sku", "sku_2", "qty"] + list(_ORIGIN_TYPES), dc) # NC-AZ6..7 — typing sniffs the DATA, and ids stay text however numeric they look. numeric = [{"qty": "3", "price": "27.30", "settlement_id": "9007199254740993"}] ck("NC-AZ6 a numeric text column is typed from its values", _coltype("qty", numeric) == "BIGINT" and _coltype("price", numeric) == "DOUBLE", (_coltype("qty", numeric), _coltype("price", numeric))) ck("NC-AZ7 an id stays VARCHAR past 2^53", _coltype("settlement_id", numeric) == "VARCHAR" and _cell("9007199254740993", "VARCHAR") == "9007199254740993") # NC-AZ8 — text wins ties, so one 'N/A' cannot break an insert after five tables are written. mixed = [{"q": "3"}, {"q": "3"}, {"q": "N/A"}] ck("NC-AZ8 a mixed column falls back to VARCHAR", _coltype("q", mixed) == "VARCHAR") # NC-AZ9 — the JSON path flattens onto the PINNED names, not the payload's. doc = json.dumps({"salesAndTrafficByAsin": [ {"parentAsin": "B01", "salesByAsin": {"unitsOrdered": 3, "orderedProductSales": {"amount": 27.3}}, "trafficByAsin": {"buyBoxPercentage": 91.2}}]}) jc, jr = parse_json_report(doc, "salesAndTrafficByAsin", **origin) ck("NC-AZ9 nested JSON flattens to the pinned column spelling", jr and jr[0].get("sales_by_asin_ordered_product_sales_amount") == 27.3 and jr[0].get("traffic_by_asin_buy_box_percentage") == 91.2, jr[:1]) # NC-AZ10 — ⛔ THE COLLISION GUARD, ASSERTED AGAINST THE LITERAL AND NOT AGAINST `PREFIX`. # A first version wrote `entity_key(t).startswith(PREFIX)`, which is vacuously TRUE when # `PREFIX` is the empty string: setting `PREFIX = ""` — the exact regression this guards — # left it green. A check whose ruler is the constant under test cannot fail # [[a-source-check-matches-its-own-assertion]]. The literal is spelled out here on purpose. bad = [t for t in TABLES if not entity_key(t).startswith("amzn_") or not table_name(t).startswith("amzn_")] ck("NC-AZ10 every entity key and mirror table is amzn_-prefixed", not bad, bad) # NC-AZ11 — and the concrete collision: `orders` must never be the bare key Odoo could own. ck("NC-AZ11 the orders key is not the bare name", entity_key("orders") == "amzn_orders" and entity_key("orders") != "orders") # NC-AZ12 — a backfill is sliced to Amazon's ceiling, and the ceiling is not silently exceeded. w = _windows(datetime(2026, 1, 1, tzinfo=timezone.utc), datetime(2026, 4, 1, tzinfo=timezone.utc)) ck("NC-AZ12 a backfill slices to Amazon's window ceiling", len(w) == 3 and all((b - a).days <= MAX_WINDOW_DAYS for a, b in w), len(w)) # NC-AZ13 — orders dedupes to the order head; order_items keeps every line. lines = [{"amazon_order_id": "1", "sku": "A"}, {"amazon_order_id": "1", "sku": "B"}, {"amazon_order_id": "2", "sku": "C"}] ck("NC-AZ13 orders dedupes by order id and items do not", len(_dedupe_orders(lines)) == 2 and len(lines) == 3) # Include derived databases and declarations that have no sync transport. declared = set(DATABASE_TABLES) | set(amazon_schema.DECLARED_DATASETS) provenance = set(amazon_schema.PROVENANCE) ck("NC-AZ14 complete declared databases agree with the schema provenance map", declared == provenance, declared ^ provenance) ck("NC-AZ14 missing derived databases and undeclared additions fail provenance parity", all(candidate != provenance for candidate in ( declared - {"sku_keywords"}, declared - {"keyword_ranks"}, declared | {"undeclared_fixture"}))) # NC-AZ15 — an empty parse is empty, never a phantom row. ck("NC-AZ15 an empty document yields no columns and no rows", parse_flat_file("", **origin) == ([], []) and parse_json_report('{"x":[]}', "x", **origin) == ([], [])) # NC-AZ16..18 - both role-specific payloads keep their complete source shape and source label. price_input = [ {"child_asin": "B01", "parent_asin": "P01"}, {"child_asin": "B01", "parent_asin": "P01"}, {"parent_asin": "P02"}, ] ck("NC-AZ16 pricing ASINs are complete and de-duplicated", _pricing_asins(price_input) == ["B01", "P02"], _pricing_asins(price_input)) competitive = [{ "status": {"statusCode": 200, "reasonPhrase": "Success"}, "body": {"asin": "B01", "marketplaceId": "ATVPDKIKX0DER", "featuredBuyingOptions": [{"listingPrice": {"amount": 12.5}}], "referencePrices": [{"name": "CompetitivePrice", "price": {"amount": 11.0}}]}, }] cr = _competitive_rows(competitive) ck("NC-AZ17 competitive summary keeps status and every returned data family", bool(cr) and cr[0].get("competitive_status_status_code") == 200 and "competitive_body_featured_buying_options" in cr[0] and "competitive_body_reference_prices" in cr[0], cr) awd = _awd_rows([{"sku": "RI-1", "totalOnhandQuantity": 9, "inventoryDetails": {"replenishmentQuantity": 2}}]) ck("NC-AZ18 AWD inventory remains source-labelled and nested fields survive", bool(awd) and awd[0].get("inventory_source") == "AWD" and awd[0].get("inventory_details_replenishment_quantity") == 2, awd) import duckdb mem = duckdb.connect(":memory:") mem.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)") mem.execute("INSERT INTO amzn_pricing VALUES ('B01','P01'),('B01','P01')") response_count, updated = _apply_competitive(mem, "amzn_pricing", competitive) enriched = mem.execute( "SELECT COUNT(*) FROM amzn_pricing WHERE competitive_body_asin='B01'").fetchone()[0] ck("NC-AZ19 competitive summary enriches every matching report row without appending", response_count == 1 and updated == 2 and enriched == 2, (response_count, updated, enriched)) held = duckdb.connect(":memory:") held.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR, " "competitive_body_asin VARCHAR, competitive_status_status_code BIGINT)") held.execute("INSERT INTO amzn_pricing VALUES ('B01','P01',NULL,NULL), ('B02','P02','B02',200)") response_count, _ = _apply_competitive(held, "amzn_pricing", competitive) left = held.execute("SELECT competitive_body_asin, competitive_status_status_code FROM amzn_pricing " "WHERE child_asin='B02'").fetchone() carried = held.execute("SELECT COUNT(DISTINCT competitive_body_asin) FROM amzn_pricing " "WHERE competitive_body_asin IS NOT NULL").fetchone()[0] held.close() ck("NC-AZ32 an ASIN absent from this pass keeps no earlier competitive price", left == (None, None) and carried == response_count == 1, (left, carried, response_count)) sku = duckdb.connect(":memory:") sku.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)") sku.execute("INSERT INTO amzn_pricing VALUES ('B01','P01'), ('B03','P01'), (NULL,'P04')") family = [{"body": {"asin": a, "price": p}} for a, p in (("B01", 1), ("B03", 3), ("P01", 9), ("P04", 4))] response_count, _ = _apply_competitive(sku, "amzn_pricing", family) priced = sku.execute("SELECT COALESCE(child_asin, parent_asin), competitive_body_asin, competitive_body_price " "FROM amzn_pricing ORDER BY 1").fetchall() sku.close() ck("NC-AZ33 a parent ASIN's summary never overwrites its SKU rows", priced == [("B01", "B01", 1), ("B03", "B03", 3), ("P04", "P04", 4)] and response_count == 4, priced) mem.execute("""CREATE TABLE _sync_state ( entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") _record_state(mem, "amzn_inventory_awd", "done", 7) marker = mem.execute( "SELECT phase, last_id, cursor_wd, rows, updated_at FROM _sync_state").fetchone() ck("NC-AZ20 sync state writes rows and timestamp into their declared columns", marker[0] == "done" and marker[1] is None and marker[2] is None and marker[3] == 7 and bool(marker[4]), marker) # NC-AZ21..24 - the destructive path is hermetic, prefix-complete, and collateral-free. mem.execute("CREATE TABLE amzn_orders (id BIGINT)") mem.execute("INSERT INTO amzn_orders VALUES (1), (2)") mem.execute("CREATE TABLE amzn_future_table (id BIGINT)") mem.execute("INSERT INTO amzn_future_table VALUES (3)") mem.execute("CREATE TABLE keep_unrelated (id BIGINT)") mem.execute("INSERT INTO keep_unrelated VALUES (99)") _record_state(mem, "amzn_orders", "done", 2) _record_state(mem, "odoo_orders", "live", 11) purged = _purge_mirror(mem, "fixture", "fixture") ck("NC-AZ21 purge removes every amzn_ table, including a future one", purged["tables"] == ["amzn_future_table", "amzn_orders", "amzn_pricing"] and purged["rows"] == 5, purged) ck("NC-AZ22 purge removes every Amazon sync marker", purged["syncMarkers"] == 2 and not [r for r in _sync_state_rows(mem) if str(r[0]).startswith("amzn_")], _sync_state_rows(mem)) kept = mem.execute("SELECT id FROM keep_unrelated").fetchall() kept_state = mem.execute( "SELECT phase, rows FROM _sync_state WHERE entity='odoo_orders'").fetchone() ck("NC-AZ23 purge preserves unrelated tables and sync markers byte for byte", kept == [(99,)] and kept_state == ("live", 11), (kept, kept_state)) second = _purge_mirror(mem, "fixture", "fixture") ck("NC-AZ24 purge readback is residue-free and a retry is idempotent", second == {"tables": [], "rows": 0, "syncMarkers": 0, "residueTables": [], "residueSyncMarkers": []}, second) # NC-AZ25..26 - neither a typo nor a still-live credential can reach the DROP transaction. refused_confirmation = False try: _require_purge_confirmation("royal-imports", "royal_imports") except RuntimeError: refused_confirmation = True ck("NC-AZ25 purge refuses when the exact tenant confirmation mismatches", refused_confirmation) class _FakeRuntime(_FixtureMirrorRuntime): def __init__(self, entries): self.entries = entries def get(self, name): from core import keychain as _kc return {"entries": self.entries} if name == _kc.KEY else None absent_ok, present_refused = False, False try: absent_ok = _assert_amazon_credential_revoked(_FakeRuntime({})) _assert_amazon_credential_revoked(_FakeRuntime({ "k_probe": {"type": "amazon_sp", "label": "Amazon", "enc": "not-decrypted-by-this-check"}, })) except RuntimeError: present_refused = True ck("NC-AZ26 purge refuses while any Amazon credential metadata remains", absent_ok and present_refused, (absent_ok, present_refused)) mem.close() # NC-AZ27 - drive the public operator wrapper against an isolated file, including its readback. import tempfile prior_path = Path(datastore.DB_PATH) public_ok, public_detail = False, None try: with tempfile.TemporaryDirectory() as tmp: purge_path = Path(tmp) / "qa-purge.duckdb" seed = duckdb.connect(str(purge_path)) seed.execute("""CREATE TABLE _sync_state ( entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") seed.execute("CREATE TABLE amzn_orders (id BIGINT)") seed.execute("INSERT INTO amzn_orders VALUES (1)") _record_state(seed, "amzn_orders", "done", 1) seed.close() class _PurgeRuntime(_FakeRuntime): key = "qa-purge" datastore_path = purge_path def __init__(self, entries): super().__init__(entries) self.docs = {} def get(self, name, fresh=False): from copy import deepcopy return deepcopy(self.docs[name]) if name in self.docs else super().get(name) def update(self, name, fn, flush="sync"): from copy import deepcopy assert flush == "sync" doc = deepcopy(self.docs.get(name) or {}) fn(doc) self.docs[name] = doc def assert_datastore_matches(self): if Path(datastore.DB_PATH).resolve() != purge_path.resolve(): raise RuntimeError("wrong test mirror") # A C2 shim is confined to this offline fixture; product paths never substitute it. from types import ModuleType from unittest.mock import patch audit_fixture = ModuleType("core.audit") events = [] audit_fixture.record = lambda *args: events.append(args) or {"recorded": True} with patch.dict(sys.modules, {"core.audit": audit_fixture}): public_detail = purge( "qa-purge", "qa-purge", rt=_PurgeRuntime({}), log=lambda *_a: None) datastore.use_path(prior_path) readback = duckdb.connect(str(purge_path), read_only=True) tables_after = sorted(str(row[0]) for row in readback.execute("SHOW TABLES").fetchall()) state_after = readback.execute("SELECT entity FROM _sync_state").fetchall() readback.close() public_ok = (public_detail["tenant"] == "qa-purge" and public_detail["database"] == "qa-purge.duckdb" and public_detail["rows"] == 1 and tables_after == ["_sync_state"] and state_after == [] and len(events) == 1 and events[0][0] == "data.purge") finally: datastore.use_path(prior_path) ck("NC-AZ27 the public purge wrapper deletes and reads back an isolated tenant file", public_ok, public_detail) # W-8734: "AttributeError: 'Runtime' object has no attribute 'assert_datastore_matches'" at NC-AZ28, # run as a script: the patches below land on this module, sync resolves harness.amazon_store's store. ck("NC-AZ36 the selftest runs in the module whose store the Amazon mirror resolves", _mirrors.for_connector("amazon") is _MIRROR_STORE, __name__) import importlib.util spec = importlib.util.spec_from_file_location(f"_amazon_store_copy_{os.getpid()}", __file__) second = importlib.util.module_from_spec(spec) spec.loader.exec_module(second) ck("NC-AZ37 a second copy of this file holds a store the Amazon mirror never resolves", _mirrors.for_connector("amazon") is not second._MIRROR_STORE) ck("NC-AZ28 margin fixtures and red-first controls", margin_selftest(log) == 0) ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0) ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0) ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0) same, detail = _spooled_parse_selftest() ck("NC-AZ34 a spooled document parses and counts exactly as its text does", same, detail) bounded, detail = _bounded_document_selftest() ck("NC-AZ35 a large synthetic report parses and writes in bounded memory", bounded, detail) ck("NC-AZ38 brand analytics weekly wait and two week history fixtures", weekly_wait_selftest(log) == 0) ck("NC-AZ39 catalog picture fill fixtures", catalog_picture_selftest(log) == 0) ck("NC-AZ40 monthly tables after a frozen cohort fixtures", monthly_after_cohort_selftest(log) == 0) ck("NC-AZ41 staging report hold during live's daily pass fixtures", report_hold_selftest(log) == 0) log(f"\n{'GREEN' if not fails else 'RED'}: {checks - len(fails)}/{checks}") return 0 if not fails else 1 def report_hold_selftest(log=print, check=None): """D-17410 offline fixtures: staging's report draws wait for live's daily pass, and every leg goes red without the hold.""" from contextlib import redirect_stdout from unittest.mock import patch module = sys.modules[__name__] cx = _client() failures = [] def ck(name, ok): if check: check("[report hold] " + name, bool(ok)) else: log(f" {'ok' if ok else 'FAIL'} report hold {name}") if not ok: failures.append(name) at = lambda hour, minute=0: datetime(2026, 11, 18, hour, minute, tzinfo=timezone.utc).timestamp() # noqa: E731 fields = {"seller_id": "SELFTEST-SELLER"} def target(value): env = {k: v for k, v in os.environ.items() if k != "AIOS_FIRST_LOAD_TARGET"} if value is not None: env["AIOS_FIRST_LOAD_TARGET"] = value return patch.dict(os.environ, env, clear=True) def legs(): out = {} with patch.object(module, "_LIVE_HOLD_NOTICED", set()): docs = _ProcessDocuments() printed = io.StringIO() with target("staging"), redirect_stdout(printed): held = take_report_token(docs, "s", now=at(1, 30)) again = take_report_token(docs, "s", now=at(1, 45)) lines = [line for line in printed.getvalue().splitlines() if "staging holds" in line] out["held"] = (held == (False, 5400.0) and again[0] is False and REPORT_LEDGER_KEY not in docs.docs and len(lines) == 1) with target("staging"), redirect_stdout(io.StringIO()): granted = take_report_token(docs, "s", now=at(3)) row = ((docs.docs.get(REPORT_LEDGER_KEY) or {}).get("sellers") or {}).get("s") or {} out["after"] = granted == (True, 0.0) and row.get("hours") == {"2026-11-18T03": 1} and row.get("spent") == 1 live, unset = _ProcessDocuments(), _ProcessDocuments() with redirect_stdout(io.StringIO()): with target("staging"): staged = take_report_token(_ProcessDocuments(), "s", now=at(1, 30)) with target("live"): live_draw = take_report_token(live, "s", now=at(1, 30)) with target(None): unset_draw = take_report_token(unset, "s", now=at(1, 30)) out["scoped"] = staged[0] is False and live_draw == (True, 0.0) and unset_draw == (True, 0.0) raised = None with target("staging"), redirect_stdout(io.StringIO()), patch.object(time, "time", return_value=at(1, 30)): try: _ledger_draw(_ProcessDocuments())(fields) except cx.AmazonThrottled as exc: raised = exc out["raise"] = (raised is not None and "Staging holds" in str(raised) and "03:00 UTC" in str(raised) and raised.retry_after == 5400.0 and raised.operation == "createReport") return out real = legs() with patch.object(module, "live_daily_hold_seconds", return_value=0.0): removed = legs() ck("staging at 01:30Z is refused for 5400 s, spends no token and prints one hold line", real["held"] and not removed["held"]) ck("staging at 03:00Z is granted and its hour bucket counts 1", real["after"] and not removed["after"]) ck("live or an unset target at 01:30Z is granted while staging waits", real["scoped"] and not removed["scoped"]) ck("the daily pass draw raises AmazonThrottled naming the hold, with the hold as retry_after", real["raise"] and not removed["raise"]) return 1 if failures else 0 def catalog_picture_selftest(log=print, check=None): """Offline fixtures: `amazon_pictures` itself, and the post-publish fill reading the latest catalog snapshot.""" from unittest.mock import patch import duckdb from harness import amazon_pictures failures = [] def ck(name, ok): if check: check("[catalog pictures] " + name, bool(ok)) else: log(f" {'ok' if ok else 'FAIL'} catalog pictures {name}") if not ok: failures.append(name) ck("amazon_pictures selftest", amazon_pictures.selftest(log) == 0) picture, calls = amazon_pictures.fixture_jpeg(400), [] def fetch(url): calls.append(url) return picture link = "https://m.media-amazon.com/images/I/{}.jpg".format rt = _FixtureMirrorRuntime() rt.key = "royal-imports" with closing(duckdb.connect(str(rt.datastore_path))) as con: con.execute("CREATE TABLE amzn_catalog_items (asin VARCHAR, main_image_url VARCHAR, " "_report_document_id VARCHAR, _snapshot_date VARCHAR)") con.executemany("INSERT INTO amzn_catalog_items VALUES (?, ?, ?, ?)", [ ["B0PICNEW01", link("new"), "operation:latest", "2026-10-31"], ["B0PICNONE1", None, "operation:latest", "2026-10-31"], ["B0PICOLD01", link("old"), "operation:older", "2026-10-30"]]) synced = {"tables": {"catalog_items": {"state": "done", "rows": 2}}} report = json.loads(json.dumps(synced)) with patch.dict(os.environ, {"STORE_BACKEND": "pg"}), \ patch.object(amazon_pictures, "_download", side_effect=fetch), \ patch.object(amazon_pictures.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")): _fill_catalog_pictures("royal-imports", rt, report, lambda *_: None) index = amazon_pictures.read_index(rt) ck("a published catalog pass stores the latest snapshot's pictures only", (report.get("pictures") or {}).get("fetched") == 1 and calls == [link("new")] and set(index["items"]) == {"B0PICNEW01"} and index["items"]["B0PICNEW01"]["_report_document_id"] == "operation:latest") other = {"tables": {"listings": {"state": "done", "rows": 1}}} _fill_catalog_pictures("royal-imports", rt, other, lambda *_: None) ck("a pass that did not sync the catalog fills nothing", "pictures" not in other and len(calls) == 1) wrong = _FixtureMirrorRuntime() wrong.key = "another-tenant" failed = json.loads(json.dumps(synced)) _fill_catalog_pictures("royal-imports", wrong, failed, lambda *_: None) ck("a fill that cannot read the mirror is reported failed and never raises", (failed.get("pictures") or {}).get("state") == "failed") with closing(duckdb.connect(str(rt.datastore_path))) as con: con.execute("ALTER TABLE amzn_catalog_items DROP COLUMN main_image_url") old = json.loads(json.dumps(synced)) with patch.dict(os.environ, {"STORE_BACKEND": "pg"}), patch.object(amazon_pictures, "_download", side_effect=fetch): _fill_catalog_pictures("royal-imports", rt, old, lambda *_: None) ck("a snapshot without picture links skips and keeps the index", (old.get("pictures") or {}).get("state") == "skipped" and set(amazon_pictures.read_index(rt)["items"]) == {"B0PICNEW01"}) return len(failures) def weekly_wait_selftest(log=print, check=None): """Brand analytics: an unfinished week is a wait with a fallback, and an undated mirror loads two weeks.""" from contextlib import contextmanager from types import SimpleNamespace from unittest.mock import patch import duckdb cx, module, failures = _client(), sys.modules[__name__], [] def ck(name, probe): try: ok = bool(probe()) except Exception as exc: # noqa: BLE001 ok = False log(f" {name}: {type(exc).__name__}: {exc}") if check: check("[weekly wait] " + name, ok) else: log(f" {'ok' if ok else 'FAIL'} {name}") if not ok: failures.append(name) class Clock(datetime): instant = datetime(2026, 10, 25, 8, 27, tzinfo=timezone.utc) @classmethod def now(cls, tz=None): return cls.instant.astimezone(tz) pending = ("Reporting data for end date [2026-10-24] and time period [weekly] is not available yet. " "This date may not have finished processing, or may fall outside the available data range.") def fixture(*, undated=False, refusal=None): con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") state = SimpleNamespace(con=con, requests=[], unready={"2026-10-18"}, refusal=refusal, callbacks=[]) if undated: con.execute("CREATE TABLE amzn_brand_analytics AS SELECT 'legacy' AS search_term, " "'Home' AS department_name, 'B0LEGACY01' AS clicked_asin, 1::BIGINT AS click_share_rank, " "NULL::VARCHAR AS _period_start, NULL::VARCHAR AS _period_end") record_watermark(con, "brand_analytics", datetime(2026, 10, 1, tzinfo=timezone.utc), 1) record_backfill_floor(con, "brand_analytics", datetime(2026, 8, 1, tzinfo=timezone.utc)) _record_state(con, entity_key("brand_analytics"), "refused", 1, "fixture prior not available yet") _record_state(con, entity_key("refused_brand_analytics"), "refused", detail=json.dumps({"reason": pending, "amazonStatus": "FATAL", "attempts": 1})) _record_reconciliation(con, "brand_analytics", { "start": "2026-09-01T00:00:00Z", "end": "2026-10-01T00:00:00Z", "count": 3521339}) _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-10-01T00:00:00Z") return state def document(state): def answer(_fields, report_type, **kwargs): week = kwargs["start"][:10] state.requests.append(week) if state.refusal: raise cx.AmazonReportError(f"Amazon ended the {report_type} report as FATAL (reportId f)", state.refusal, "FATAL") if week in state.unready: raise cx.AmazonReportError(f"Amazon ended the {report_type} report as FATAL (reportId f)", json.dumps({"errorDetails": pending}), "FATAL") body = json.dumps({"dataByDepartmentAndSearchTerm": [ {"departmentName": "Home", "searchTerm": term, "clickedAsin": "B0WEEK0001", "clickShareRank": 1, "searchFrequencyRank": 3} for term in ("legacy", "term " + week)]}) return body, {"reportDocumentId": "week-" + week} return answer def patches(state): @contextmanager def published(*_args, **_kwargs): yield _note_published() def after(_tenant, report, _log, _rt): if "brand_analytics" in _synced_tables(report): state.callbacks.append(_newest_week(state.con, "amzn_brand_analytics").isoformat()) def accounts(tenant_key, **options): return _sync_candidate(tenant_key, **options) # This fixture's history is the latest week only, so weekly history (H2) plans nothing older. one_week = {**cx.REPORT_TYPES["brand_analytics"], "history": {"from": "2026-10-18", "evidence": "fixture"}} return [patch.dict(cx.REPORT_TYPES, {"brand_analytics": one_week}), patch.object(module, "datetime", Clock), patch.object(module, "_sync_credential", return_value={}), patch.object(module, "account_entries", return_value=[{"connection_id": "c", "label": "c"}]), patch.object(module, "restore_mirror", return_value={"dirty": set(), "problems": []}), patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (state.con.cursor(), None)), patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), patch.object(module, "read_database_enablement", return_value={"available": True, "databases": { amazon_schema.DATABASE_PREFIX + t: t == "brand_analytics" for t in TABLES}}), patch.object(module, "_sync_event"), patch.object(module, "persist_mirror"), patch.object(module, "watched_terms", return_value=({"legacy"}, None)), patch.object(module, "_sync_accounts_candidate", side_effect=accounts), patch.object(module, "_refuse_unreconciled"), patch.object(module, "_queue_after_sync", side_effect=after), patch.object(_mirrors, "sync_mirror", published), patch.object(cx, "report_document", side_effect=document(state))] def run(state, call, **kwargs): from contextlib import ExitStack with ExitStack() as stack: for item in patches(state): stack.enter_context(item) return call("fixture", tables=["brand_analytics"], rt=SimpleNamespace(key="fixture"), max_wait=1, log=lambda *_: None, **kwargs) def marker(state, name): return _read_state(state.con, entity_key(name)) or {} def waited(state): detail = json.loads(marker(state, "waiting_brand_analytics").get("detail") or "{}") return (marker(state, "waiting_brand_analytics").get("state") == "waiting" and detail == {"weekStart": "2026-10-18", "weekEnd": "2026-10-24", "error": pending} and marker(state, "brand_analytics").get("state") == "done" and not marker(state, "refused_brand_analytics") and not _refused_reports(state.con)) # Fix 1: a fresh first load asks the newest closed week, waits on it, and lands the week before. fresh = fixture() first = run(fresh, _sync_candidate, first_load=True, report_budget=6) ck("BA-W1 an unfinished week records a wait and the preceding week lands in the same pass", lambda: fresh.requests == ["2026-10-18", "2026-10-11"] and waited(fresh) and not first["problems"] and first["tables"]["brand_analytics"]["waiting"]["weekStart"] == "2026-10-18" and _newest_week(fresh.con, "amzn_brand_analytics").isoformat() == "2026-10-11" and read_watermark(fresh.con, "brand_analytics") == datetime(2026, 10, 18, tzinfo=timezone.utc)) Clock.instant += timedelta(days=1) fresh.requests.clear() run(fresh, _sync_candidate) ck("BA-W2 the next pass asks the waited week again and skips a fallback the mirror holds", lambda: fresh.requests == ["2026-10-18"] and waited(fresh)) fresh.unready.clear() fresh.requests.clear() _record_state(fresh.con, entity_key("brand_analytics"), "refused", 0, "fixture stale refusal") _record_state(fresh.con, entity_key("refused_brand_analytics"), "refused", detail="{}") landed = run(fresh, _sync_candidate) ck("BA-W3 the waited week lands, its wait is deleted and a stale refusal clears", lambda: fresh.requests == ["2026-10-18"] and "waiting" not in landed["tables"]["brand_analytics"] and not marker(fresh, "waiting_brand_analytics") and marker(fresh, "brand_analytics").get("state") == "done" and not marker(fresh, "refused_brand_analytics") and read_watermark(fresh.con, "brand_analytics") == datetime(2026, 10, 25, tzinfo=timezone.utc)) Clock.instant -= timedelta(days=1) # Negative controls: another FATAL stays a refusal, and the old classifier stores the wait as one. refused = fixture(refusal="Brand Registry enrollment is required for this report.") run(refused, _sync_candidate, first_load=True, report_budget=6) ck("BA-W4 NC a FATAL that is not a wait still stores the refusal", lambda: refused.requests == ["2026-10-18"] and marker(refused, "brand_analytics").get("state") == "refused" and marker(refused, "refused_brand_analytics") and not marker(refused, "waiting_brand_analytics")) blind = fixture() with patch.object(module, "_week_not_ready", return_value=False): run(blind, _sync_candidate, first_load=True, report_budget=6) ck("BA-W5 NC without the wait classifier the same answer reads refused", lambda: blind.requests == ["2026-10-18"] and not waited(blind) and marker(blind, "brand_analytics").get("state") == "refused") # Fix 2: an undated mirror loads the two newest finished weeks, oldest first, one publication each. staging = fixture(undated=True) result = run(staging, sync) periods = staging.con.execute("SELECT _period_start, count(*) FROM amzn_brand_analytics " "WHERE _period_start IS NOT NULL GROUP BY 1 ORDER BY 1").fetchall() ck("BA-W6 an undated mirror loads two weeks oldest first and each week queues the callbacks", lambda: staging.requests == ["2026-10-04", "2026-10-11"] and staging.callbacks == ["2026-10-04", "2026-10-11"] and periods == [("2026-10-04", 1), ("2026-10-11", 2)] and not result["problems"] and marker(staging, "brand_analytics").get("state") == "done" and not marker(staging, "refused_brand_analytics") and staging.con.execute("SELECT count(*) FROM amzn_brand_analytics WHERE _period_start IS NULL").fetchone()[0] == 1 and read_watermark(staging.con, "brand_analytics") == datetime(2026, 10, 18, tzinfo=timezone.utc)) staging.requests.clear() run(staging, sync) ck("BA-W7 the following pass asks the newest closed week and waits on it without a refusal", lambda: staging.requests == ["2026-10-18"] and waited(staging) and len(staging.callbacks) == 3 and read_watermark(staging.con, "brand_analytics") == datetime(2026, 10, 18, tzinfo=timezone.utc)) single, publish = fixture(undated=True), sync with patch.object(module, "sync", side_effect=lambda *_a, **_k: {"tables": {}}): run(single, publish) ck("BA-W8 NC one publication per pass alone queues the callbacks for one week", lambda: single.requests == ["2026-10-04"] and single.callbacks == ["2026-10-04"]) for state in (fresh, refused, blind, staging, single): state.con.close() return len(failures) def _spool_fixture(text, *, gz=False, encoding="utf-8"): """A `ReportDocument` made by the client's own spool path from local bytes, no network.""" import gzip raw = text.encode(encoding) if isinstance(text, str) else text handle, path = tempfile.mkstemp(prefix="amazon-selftest-") with os.fdopen(handle, "wb") as out: out.write(gzip.compress(raw) if gz else raw) return _client()._spooled_document(path, "GZIP" if gz else "") def _spooled_parse_selftest(): """D-7081: the streaming readers answer every fixture exactly as the whole-text parsers do.""" origin = {"document_id": "selftest-document", "report_type": "selftest-report"} def outcome(parse): try: cols, rows = parse() return "ok", list(cols), [dict(row) for row in rows] except Exception as exc: # noqa: BLE001 return "error", type(exc).__name__, str(exc) def count(text, roots): try: return document_row_count(text, roots) except Exception as exc: # noqa: BLE001 return type(exc).__name__ flat = ["", " \n\t", "a\tb\r\n1\t2\r\n\r\n3\t4", "\ufeffsku\tsku\tbuyer-email\nA\tB\tx\n\"q\t1\n", "x\ty\r1\r2\t3\r\r", "sku\tname\nA\tsmart \u201cquote\u201d\n"] blobs = [(t, {}) for t in flat] + [(t, {"gz": True}) for t in flat] + [(flat[-1], {"encoding": "cp1252"})] misses = [] for index, (text, how) in enumerate(blobs): expected = text if how.get("encoding") != "cp1252" else text.encode("cp1252").decode("cp1252") got = outcome(lambda: parse_flat_file(_spool_fixture(text, **how), **origin)) if got != outcome(lambda: parse_flat_file(expected, **origin)): misses.append(f"flat {index}") if expected.strip() and count(_spool_fixture(text, **how), ()) != count(expected, ()): misses.append(f"flat count {index}") documents = ['', '[]', '{"rows": []}', '{"rows": {"a": 1}}', '{"rows": [1]}', '{"rows": [{"a": 1},]}', '{"rows": [{"a": 1}]} x', '{"rows": [{"a": 1}], "rows": [{"b": 2}]}', '{"other": 1}', ' {"spec": {"q": [1, [2]]}, "rows": [{"a": 1, "b": null, "n": {"c": "\u201c"}}, ' '{"a": 2.5, "l": [1, 2]}], "tail": [[1], []]} ', '{"rows": [{"_report_document_id": "vendor"}]}', '{"rows": [{"buyerEmail": "e", "a": 123456789012345678901}]}', '{"x": NaN, "rows": [{"v": Infinity, "w": -0.0, "s": "\\ud83d"}]}'] for index, text in enumerate(documents): if outcome(lambda: parse_json_report(_spool_fixture(text), "rows", **origin)) != outcome( lambda: parse_json_report(text, "rows", **origin)): misses.append(f"json {index}") if count(_spool_fixture(text), ("rows",)) != count(text, ("rows",)): misses.append(f"json count {index}") return not misses, misses def _bounded_document_selftest(rows=24_000): """D-7081: a report larger than the bound parses, sniffs and writes with a bounded Python peak. The whole-text path holds the decoded text and every row at once, so it fails this bound; the spooled path holds one read piece and one insert batch. """ import gzip import tracemalloc bound = 10 << 20 handle, path = tempfile.mkstemp(prefix="amazon-selftest-") os.close(handle) with gzip.open(path, "wt", encoding="utf-8") as out: out.write('{"reportSpecification": {"reportType": "selftest"}, "dataByDepartmentAndSearchTerm": [') for i in range(rows): out.write(("," if i else "") + json.dumps({ "departmentName": "Amazon.com", "searchTerm": f"term {i} \u2122 caf\u00e9", "searchFrequencyRank": i + 1, "clickedAsin": f"B0{i:08d}", "clickedItemName": f"Product title number {i} with several words", "clickShareRank": i % 3 + 1, "clickShare": round(i / 7 % 1, 4), "conversionShare": round(i / 11 % 1, 4)})) out.write("]}") document = _client()._spooled_document(path, "GZIP") text_bytes = 2 * len(document) con = datastore.duckdb.connect(":memory:") table = "amzn_brand_analytics" # A first bound statement imports DuckDB's lazy modules; that one-time cost is not a row's. con.execute("SELECT ?", [1]).fetchall() tracemalloc.start() try: cols, parsed = parse_json_report(document, "dataByDepartmentAndSearchTerm", document_id="selftest-document", report_type="selftest") cols, parsed = stamp_week(cols, parsed, _utc("2026-10-11T00:00:00Z"), _utc("2026-10-18T00:00:00Z")) cols = merge_columns([c for c, _t in amazon_schema.columns_for("brand_analytics")], cols) types = _ensure(con, table, cols, parsed, declared=dict(amazon_schema.columns_for("brand_analytics"))) first, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics")) peak = tracemalloc.get_traced_memory()[1] finally: tracemalloc.stop() again, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics")) stored = con.execute(f"SELECT COUNT(*), COUNT(DISTINCT _report_row_number) FROM {table}").fetchone() con.close() document.close() detail = {"rows": len(parsed), "first": first, "again": again, "stored": stored, "text_bytes": text_bytes, "peak": peak, "bound": bound} return (len(parsed) == first == again == rows and stored == (rows, rows) and text_bytes > bound and peak < bound), detail def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--sync", action="store_true") ap.add_argument("--status", action="store_true") ap.add_argument("--validate", action="store_true") ap.add_argument("--purge", action="store_true", help="permanently delete every Amazon mirror table after credential revocation") ap.add_argument("--deletion-trigger", choices=DELETION_TRIGGERS) ap.add_argument("--trigger-at", help="original event timestamp with UTC offset") ap.add_argument("--retention-check", action="store_true") ap.add_argument("--restore-from", help="existing backup file to restore as a checked copy") ap.add_argument("--restore-to", help="new copy beside the tenant mirror, never the live file") ap.add_argument("--selftest", action="store_true") ap.add_argument("--tenant", default=None) ap.add_argument("--confirm-tenant", default=None, help="for --purge or --restore-from, repeat the exact tenant slug") ap.add_argument("--table", action="append", help="limit to one table (repeatable)") a = ap.parse_args(argv) if sum(bool(v) for v in (a.sync, a.status, a.validate, a.purge, a.selftest, a.deletion_trigger, a.retention_check, a.restore_from)) > 1: ap.error("choose exactly one action") if a.trigger_at and not a.deletion_trigger: ap.error("--trigger-at requires --deletion-trigger") if bool(a.restore_from) != bool(a.restore_to): ap.error("--restore-from and --restore-to are required together") if a.deletion_trigger or a.retention_check or a.restore_from: if not a.tenant or a.table: ap.error("lifecycle actions require an explicit --tenant and no --table") if a.deletion_trigger: rep = record_deletion_trigger(a.tenant, a.deletion_trigger, occurred_at=a.trigger_at) elif a.restore_from: rep = restore(a.tenant, a.confirm_tenant, a.restore_from, a.restore_to) else: rep = retention_check(a.tenant) print(json.dumps(rep, indent=2)) return 1 if rep.get("state") == "OVERDUE" else 0 tenant = a.tenant or "royal-imports" if a.purge: if not a.tenant: ap.error("--purge requires an explicit --tenant") if a.table or a.sync or a.status or a.validate or a.selftest: ap.error("--purge is all-or-nothing and cannot be combined with another action") rep = purge(a.tenant, a.confirm_tenant) print(json.dumps(rep, indent=2)) return 0 if a.selftest: return selftest() if a.status: print(json.dumps(status(tenant), indent=2)) return 0 if a.validate: validate(tenant) return 0 if a.sync: t0 = time.time() rep = sync(tenant, tables=a.table) print(json.dumps(rep, indent=2)) print(f"\n{time.time() - t0:.1f}s") return 0 ap.print_help() return 2 if __name__ == "__main__": # Run harness.amazon_store, the copy mirror_store resolves, never this __main__ copy. from harness import amazon_store as _canonical # noqa: E402 sys.exit(_canonical.main())